netbox-agent/netbox_agent/inventory.py

import json
import logging
import re
import sys
import traceback

import pynetbox

from netbox_agent.config import config
from netbox_agent.config import netbox_instance as nb
from netbox_agent.lshw import LSHW
from netbox_agent.misc import get_vendor, is_tool
from netbox_agent.raid.hp import HPRaid
from netbox_agent.raid.omreport import OmreportRaid
from netbox_agent.raid.storcli import StorcliRaid

INVENTORY_TAG = {
    "cpu": {"name": "hw:cpu", "slug": "hw-cpu"},
    "gpu": {"name": "hw:gpu", "slug": "hw-gpu"},
    "disk": {"name": "hw:disk", "slug": "hw-disk"},
    "interface": {"name": "hw:interface", "slug": "hw-interface"},
    "memory": {"name": "hw:memory", "slug": "hw-memory"},
    "motherboard": {"name": "hw:motherboard", "slug": "hw-motherboard"},
    "raid_card": {"name": "hw:raid_card", "slug": "hw-raid-card"},
}


class Inventory:
    """
    Better Inventory items coming, see:
    - https://github.com/netbox-community/netbox/issues/3087
    - https://github.com/netbox-community/netbox/issues/3333

    This class implements for:
    * memory
    * cpu
    * raid cards
    * disks
    * gpus

    methods that:
    * get local item
    * get netbox item
    * create netbox item
    * update netbox item

    Known issues:
    - no scan of non-raid devices
    - no scan of NVMe devices
    """

    def __init__(self, server, update_expansion=False):
        self.create_netbox_tags()
        self.server = server
        self.update_expansion = update_expansion
        netbox_server = self.server.get_netbox_server(update_expansion)

        self.device_id = netbox_server.id if netbox_server else None
        self.raid = None
        self.disks = []

        self.lshw = LSHW()

    def create_netbox_tags(self):
        ret = []
        for key, tag in INVENTORY_TAG.items():
            nb_tag = nb.extras.tags.get(name=tag["name"])
            if not nb_tag:
                nb_tag = nb.extras.tags.create(
                    name=tag["name"],
                    slug=tag["slug"],
                    comments=tag["name"],
                )
            ret.append(nb_tag)
        return ret

    def find_or_create_manufacturer(self, name):
        if name is None:
            return None

        manufacturer = nb.dcim.manufacturers.get(
            name=name,
        )
        if not manufacturer:
            logging.info("Creating missing manufacturer {name}".format(name=name))
            manufacturer = nb.dcim.manufacturers.create(
                name=name,
                slug=re.sub("[^A-Za-z0-9]+", "-", name).lower(),
            )

            logging.info("Creating missing manufacturer {name}".format(name=name))

        return manufacturer

    def get_netbox_inventory(self, device_id, tag):
        try:
            items = nb.dcim.inventory_items.filter(device_id=device_id, tag=tag)
        except pynetbox.core.query.RequestError:
            logging.info("Tag {tag} is missing, returning empty array.".format(tag=tag))
            items = []

        return list(items)

    def create_netbox_inventory_item(
        self, device_id, tags, vendor, name, serial, description
    ):
        manufacturer = self.find_or_create_manufacturer(vendor)

        _ = nb.dcim.inventory_items.create(
            device=device_id,
            manufacturer=manufacturer.id,
            discovered=True,
            tags=tags,
            name="{}".format(name),
            serial="{}".format(serial),
            description=description,
        )

        logging.info(
            "Creating inventory item {} {}/{} {} ".format(
                vendor, name, serial, description
            )
        )

    def get_hw_motherboards(self):
        motherboards = []

        m = {}
        m["serial"] = self.lshw.motherboard_serial
        m["vendor"] = self.lshw.vendor
        m["name"] = "{} {}".format(self.lshw.vendor, self.lshw.motherboard)
        m["description"] = "{} Motherboard".format(self.lshw.motherboard)

        motherboards.append(m)

        return motherboards

    def do_netbox_motherboard(self):

        motherboards = self.get_hw_motherboards()
        nb_motherboards = self.get_netbox_inventory(
            device_id=self.device_id, tag=INVENTORY_TAG["motherboard"]["slug"]
        )

        for nb_motherboard in nb_motherboards:
            if nb_motherboard.serial not in [x["serial"] for x in motherboards]:
                logging.info(
                    "Deleting unknown motherboard {motherboard}/{serial}".format(
                        motherboard=self.lshw.motherboard,
                        serial=nb_motherboard.serial,
                    )
                )
                nb_motherboard.delete()

        # create interfaces that are not in netbox
        for motherboard in motherboards:
            if motherboard.get("serial") not in [x.serial for x in nb_motherboards]:
                self.create_netbox_inventory_item(
                    device_id=self.device_id,
                    tags=[{"name": INVENTORY_TAG["motherboard"]["name"]}],
                    vendor="{}".format(motherboard.get("vendor", "N/A")),
                    serial="{}".format(motherboard.get("serial", "No SN")),
                    name="{}".format(motherboard.get("name")),
                    description="{}".format(motherboard.get("description")),
                )

    def create_netbox_interface(self, iface):
        manufacturer = self.find_or_create_manufacturer(iface["vendor"])
        _ = nb.dcim.inventory_items.create(
            device=self.device_id,
            manufacturer=manufacturer.id,
            discovered=True,
            tags=[{"name": INVENTORY_TAG["interface"]["name"]}],
            name="{}".format(iface["product"]),
            serial="{}".format(iface["serial"]),
            description="{} {}".format(iface["description"], iface["name"]),
        )

    def do_netbox_interfaces(self):
        nb_interfaces = self.get_netbox_inventory(
            device_id=self.device_id, tag=INVENTORY_TAG["interface"]["slug"]
        )
        interfaces = self.lshw.interfaces

        # delete interfaces that are in netbox but not locally
        # use the serial_number has the comparison element
        for nb_interface in nb_interfaces:
            if nb_interface.serial not in [x["serial"] for x in interfaces]:
                logging.info(
                    "Deleting unknown interface {serial}".format(
                        serial=nb_interface.serial,
                    )
                )
                nb_interface.delete()

        # create interfaces that are not in netbox
        for iface in interfaces:
            if iface.get("serial") not in [x.serial for x in nb_interfaces]:
                self.create_netbox_interface(iface)

    def create_netbox_cpus(self):
        for cpu in self.lshw.get_hw_linux("cpu"):
            manufacturer = self.find_or_create_manufacturer(cpu["vendor"])
            _ = nb.dcim.inventory_items.create(
                device=self.device_id,
                manufacturer=manufacturer.id,
                discovered=True,
                tags=[{"name": INVENTORY_TAG["cpu"]["name"]}],
                name=cpu["product"],
                description="CPU {}".format(cpu["location"]),
                # asset_tag=cpu['location']
            )

            logging.info("Creating CPU model {}".format(cpu["product"]))

    def do_netbox_cpus(self):
        cpus = self.lshw.get_hw_linux("cpu")
        nb_cpus = self.get_netbox_inventory(
            device_id=self.device_id,
            tag=INVENTORY_TAG["cpu"]["slug"],
        )

        if not len(nb_cpus) or len(nb_cpus) and len(cpus) != len(nb_cpus):
            for x in nb_cpus:
                x.delete()

            self.create_netbox_cpus()

    def get_raid_cards(self, filter_cards=False):
        raid_class = None
        if self.server.manufacturer in ("Dell", "Huawei"):
            if is_tool("omreport"):
                raid_class = OmreportRaid
            if is_tool("storcli"):
                raid_class = StorcliRaid
        elif self.server.manufacturer in ("HP", "HPE"):
            if is_tool("ssacli"):
                raid_class = HPRaid

        if not raid_class:
            return []

        self.raid = raid_class()

        if (
            filter_cards
            and config.expansion_as_device
            and self.server.own_expansion_slot()
        ):
            return [
                c
                for c in self.raid.get_controllers()
                if c.is_external() is self.update_expansion
            ]
        else:
            return self.raid.get_controllers()

    def create_netbox_raid_card(self, raid_card):
        manufacturer = self.find_or_create_manufacturer(raid_card.get_manufacturer())

        name = raid_card.get_product_name()
        serial = raid_card.get_serial_number()
        nb_raid_card = nb.dcim.inventory_items.create(
            device=self.device_id,
            discovered=True,
            manufacturer=manufacturer.id if manufacturer else None,
            tags=[{"name": INVENTORY_TAG["raid_card"]["name"]}],
            name="{}".format(name),
            serial="{}".format(serial),
            description="RAID Card",
        )
        logging.info(
            "Creating RAID Card {name} (SN: {serial})".format(
                name=name,
                serial=serial,
            )
        )
        return nb_raid_card

    def do_netbox_raid_cards(self):
        """
        Update raid cards in netbobx
        Since we only push:
        * Name
        * Manufacturer
        * Serial

        We only need to handle destroy and new cards
        """

        nb_raid_cards = self.get_netbox_inventory(
            device_id=self.device_id, tag=[INVENTORY_TAG["raid_card"]["slug"]]
        )
        raid_cards = self.get_raid_cards(filter_cards=True)

        # delete cards that are in netbox but not locally
        # use the serial_number has the comparison element
        for nb_raid_card in nb_raid_cards:
            if nb_raid_card.serial not in [x.get_serial_number() for x in raid_cards]:
                logging.info(
                    "Deleting unknown locally RAID Card {serial}".format(
                        serial=nb_raid_card.serial,
                    )
                )
                nb_raid_card.delete()

        # create card that are not in netbox
        for raid_card in raid_cards:
            if raid_card.get_serial_number() not in [x.serial for x in nb_raid_cards]:
                self.create_netbox_raid_card(raid_card)

    def is_virtual_disk(self, disk, raid_devices):
        disk_type = disk.get("type")
        logicalname = disk.get("logicalname")
        description = disk.get("description")
        size = disk.get("size")
        product = disk.get("product")
        if (
            logicalname in raid_devices
            or disk_type is None
            or product is None
            or description is None
        ):
            return True
        non_raid_disks = [
            "MR9361-8i",
        ]

        if (
            logicalname in raid_devices
            or product in non_raid_disks
            or "virtual" in product.lower()
            or "logical" in product.lower()
            or "volume" in description.lower()
            or "dvd-ram" in description.lower()
            or description == "SCSI Enclosure"
            or (size is None and logicalname is None)
        ):
            return True
        return False

    def get_hw_disks(self):
        disks = []

        for raid_card in self.get_raid_cards(filter_cards=True):
            disks.extend(raid_card.get_physical_disks())

        raid_devices = [
            d.get("custom_fields", {}).get("vd_device")
            for d in disks
            if d.get("custom_fields", {}).get("vd_device")
        ]

        for disk in self.lshw.get_hw_linux("storage"):
            if self.is_virtual_disk(disk, raid_devices):
                continue
            size = round(int(disk.get("size", 0)) / 1073741824, 1)
            d = {
                "name": "",
                "Size": "{} GB".format(size),
                "logicalname": disk.get("logicalname"),
                "description": disk.get("description"),
                "SN": disk.get("serial"),
                "Model": disk.get("product"),
                "Type": disk.get("type"),
            }
            if disk.get("vendor"):
                d["Vendor"] = disk["vendor"]
            else:
                d["Vendor"] = get_vendor(disk["product"])
            disks.append(d)

        # remove duplicate serials
        seen = set()
        uniq = [x for x in disks if x["SN"] not in seen and not seen.add(x["SN"])]
        return uniq

    def create_netbox_disk(self, disk):
        manufacturer = None
        if "Vendor" in disk:
            manufacturer = self.find_or_create_manufacturer(disk["Vendor"])

        name = "{} ({})".format(disk["Model"], disk["Size"])
        description = disk["Type"]
        sn = disk.get("SN", "unknown")

        parms = {
            "device": self.device_id,
            "discovered": True,
            "tags": [{"name": INVENTORY_TAG["disk"]["name"]}],
            "name": name,
            "serial": sn,
            "part_id": disk["Model"],
            "description": description,
            "manufacturer": getattr(manufacturer, "id", None),
        }
        if config.process_virtual_drives:
            parms["custom_fields"] = disk.get("custom_fields", {})

        _ = nb.dcim.inventory_items.create(**parms)

        logging.info(
            "Creating Disk {model} {serial}".format(
                model=disk["Model"],
                serial=sn,
            )
        )

    def dump_disks_map(self, disks):
        disk_map = [d["custom_fields"] for d in disks if "custom_fields" in d]
        if config.dump_disks_map == "-":
            f = sys.stdout
        else:
            f = open(config.dump_disks_map, "w")
        f.write(json.dumps(disk_map, separators=(",", ":"), indent=4, sort_keys=True))
        if config.dump_disks_map != "-":
            f.close()

    def do_netbox_disks(self):
        nb_disks = self.get_netbox_inventory(
            device_id=self.device_id, tag=INVENTORY_TAG["disk"]["slug"]
        )
        disks = self.get_hw_disks()
        if config.dump_disks_map:
            try:
                self.dump_disks_map(disks)
            except Exception as e:
                logging.error("Failed to dump disks map: {}".format(e))
                logging.debug(traceback.format_exc())
        disk_serials = [d["SN"] for d in disks if "SN" in d]

        # delete disks that are in netbox but not locally
        # use the serial_number has the comparison element
        for nb_disk in nb_disks:
            if nb_disk.serial not in disk_serials or config.force_disk_refresh:
                logging.info(
                    "Deleting unknown locally Disk {serial}".format(
                        serial=nb_disk.serial,
                    )
                )
                nb_disk.delete()

        if config.force_disk_refresh:
            nb_disks = self.get_netbox_inventory(
                device_id=self.device_id, tag=INVENTORY_TAG["disk"]["slug"]
            )

        # create disks that are not in netbox
        for disk in disks:
            if disk.get("SN") not in [d.serial for d in nb_disks]:
                self.create_netbox_disk(disk)

    def create_netbox_memory(self, memory):
        manufacturer = self.find_or_create_manufacturer(memory["vendor"])
        name = "Slot {} ({}GB)".format(memory["slot"], memory["size"])
        nb_memory = nb.dcim.inventory_items.create(
            device=self.device_id,
            discovered=True,
            manufacturer=manufacturer.id,
            tags=[{"name": INVENTORY_TAG["memory"]["name"]}],
            name=name,
            part_id=memory["product"],
            serial=memory["serial"],
            description=memory["description"],
        )

        logging.info(
            "Creating Memory {location} {type} {size}GB".format(
                location=memory["slot"],
                type=memory["product"],
                size=memory["size"],
            )
        )

        return nb_memory

    def do_netbox_memories(self):
        memories = self.lshw.memories
        nb_memories = self.get_netbox_inventory(
            device_id=self.device_id, tag=INVENTORY_TAG["memory"]["slug"]
        )

        for nb_memory in nb_memories:
            if nb_memory.serial not in [x["serial"] for x in memories]:
                logging.info(
                    "Deleting unknown locally Memory {serial}".format(
                        serial=nb_memory.serial,
                    )
                )
                nb_memory.delete()

        for memory in memories:
            if memory.get("serial") not in [x.serial for x in nb_memories]:
                self.create_netbox_memory(memory)

    def create_netbox_gpus(self, gpus):
        for gpu in gpus:
            if "product" in gpu and len(gpu["product"]) > 50:
                gpu["product"] = gpu["product"][:48] + ".."

            manufacturer = self.find_or_create_manufacturer(gpu["vendor"])
            _ = nb.dcim.inventory_items.create(
                device=self.device_id,
                manufacturer=manufacturer.id,
                discovered=True,
                tags=[{"name": INVENTORY_TAG["gpu"]["name"]}],
                name=gpu["product"],
                description=gpu["description"],
            )

            logging.info("Creating GPU model {}".format(gpu["product"]))

    def is_external_gpu(self, gpu):
        is_3d_gpu = gpu["description"].startswith("3D")
        return (
            self.server.is_blade()
            and self.server.own_gpu_expansion_slot()
            and is_3d_gpu
        )

    def do_netbox_gpus(self):
        gpus = []
        gpu_models = {}
        for gpu in self.lshw.get_hw_linux("gpu"):
            # Filters GPU if an expansion bay is detected:
            # The internal (VGA) GPU only goes into the blade inventory,
            # the external (3D) GPU goes into the expansion blade.
            if (
                config.expansion_as_device
                and self.update_expansion ^ self.is_external_gpu(gpu)
            ):
                continue
            gpus.append(gpu)
            gpu_models.setdefault(gpu["product"], 0)
            gpu_models[gpu["product"]] += 1

        nb_gpus = self.get_netbox_inventory(
            device_id=self.device_id,
            tag=INVENTORY_TAG["gpu"]["slug"],
        )
        nb_gpu_models = {}
        for gpu in nb_gpus:
            nb_gpu_models.setdefault(str(gpu), 0)
            nb_gpu_models[str(gpu)] += 1
        up_to_date = set(gpu_models) == set(nb_gpu_models)
        if not gpus or not up_to_date:
            for x in nb_gpus:
                x.delete()
        if gpus and not up_to_date:
            self.create_netbox_gpus(gpus)

    def create_or_update(self):
        if config.inventory is None or config.update_inventory is None:
            return False
        if self.update_expansion is False:
            self.do_netbox_cpus()
            self.do_netbox_memories()
            self.do_netbox_interfaces()
            self.do_netbox_motherboard()
        self.do_netbox_gpus()
        self.do_netbox_disks()
        self.do_netbox_raid_cards()
        return True