Local inventory (#16)

- Create and update CPU
- Create and update Memory
- Create and update RAID Cards (HP and Storcli)
- Create and update Physical disks

README.md

@@ -65,6 +65,8 @@ rack_location:
 #
 # driver: "file:/tmp/datacenter"
 # regex: "(.*)"
+
+inventory: true
 ```
 
 # Hardware
@@ -124,5 +126,5 @@ Feel free to send me a dmidecode output for Supermicro's blade!
 
 # TODO
 
-- [ ] CPU, RAID Card(s), RAM, Disks in `Device`'s `Inventory`
+- [ ] handle non-raid and NVMe drive in Inventory
 - [ ] `CustomFields` support with firmware versions for Device (BIOS), RAID Cards and disks

netbox_agent.yaml.example

@@ -23,4 +23,6 @@ rack_location:
 # regex: 'SysName:[ ]+[A-Za-z]+-[A-Za-z]+-([A-Za-z0-9]+)'
 #
 # driver: "file:/tmp/datacenter"
-# regex: "(.*)"
+# regex: "(.*)"
+
+inventory: true

netbox_agent/cli.py

@@ -18,7 +18,7 @@ MANUFACTURERS = {
 
 def run(args):
     manufacturer = dmidecode.get_by_type('Chassis')[0].get('Manufacturer')
-    server = MANUFACTURERS[manufacturer](dmidecode)
+    server = MANUFACTURERS[manufacturer](dmi=dmidecode)
     if args.debug:
         server.print_debug()
     if args.register:

netbox_agent/config.py

@@ -38,3 +38,5 @@ if config.get('network'):
     NETWORK_IGNORE_INTERFACES = config['network'].get('ignore_interfaces')
     NETWORK_IGNORE_IPS = config['network'].get('ignore_ips')
     NETWORK_LLDP = config['network'].get('lldp') is True
+
+INVENTORY_ENABLED = config.get('inventory') is True

netbox_agent/inventory.py

@@ -0,0 +1,328 @@
+import logging
+import subprocess
+import re
+
+from netbox_agent.config import netbox_instance as nb, INVENTORY_ENABLED
+from netbox_agent.misc import is_tool
+from netbox_agent.raid.hp import HPRaid
+from netbox_agent.raid.storcli import StorcliRaid
+
+INVENTORY_TAG = {
+    'cpu': {'name': 'hw:cpu', 'slug': 'hw-cpu'},
+    'memory': {'name': 'hw:memory', 'slug': 'hw-memory'},
+    'disk': {'name': 'hw:disk', 'slug': 'hw-disk'},
+    'raid_card': {'name': 'hw:raid_card', 'slug': 'hw-raid-card'},
+    }
+
+for key, tag in INVENTORY_TAG.items():
+    nb_tag = nb.extras.tags.get(
+        name=tag['name']
+        )
+    if not nb_tag:
+        nb_tag = nb.extras.tags.create(
+            name=tag['name'],
+            slug=tag['slug'],
+            comments=tag['name'],
+            )
+
+
+class Inventory():
+    """
+    Better Inventory items coming, see:
+    - https://github.com/netbox-community/netbox/issues/3087
+    - https://github.com/netbox-community/netbox/issues/3333
+
+    This class implements for:
+    * memory
+    * cpu
+    * raid cards
+    * disks
+
+    methods that:
+    * get local item
+    * get netbox item
+    * create netbox item
+    * update netbox item
+
+    Known issues:
+    - no scan of non-raid devices
+    - no scan of NVMe devices
+    """
+
+    def __init__(self, server):
+        self.server = server
+        self.device_id = self.server.get_netbox_server().id
+        self.raid = None
+        self.disks = []
+
+    def get_cpus(self):
+        model = None
+        nb = None
+
+        output = subprocess.getoutput('lscpu')
+        model_re = re.search(r'Model name: (.*)', output)
+        if len(model_re.groups()) > 0:
+            model = model_re.groups()[0].strip()
+        socket_re = re.search(r'Socket\(s\): (.*)', output)
+        if len(socket_re.groups()) > 0:
+            nb = int(socket_re.groups()[0].strip())
+        return nb, model
+
+    def create_netbox_cpus(self):
+        nb_cpus, model = self.get_cpus()
+        for i in range(nb_cpus):
+            _ = nb.dcim.inventory_items.create(
+                device=self.device_id,
+                tags=[INVENTORY_TAG['cpu']['name']],
+                name=model,
+                discovered=True,
+                description='CPU',
+            )
+            logging.info('Creating CPU model {model}'.format(model=model))
+
+    def update_netbox_cpus(self):
+        cpus_number, model = self.get_cpus()
+        nb_cpus = nb.dcim.inventory_items.filter(
+            device_id=self.device_id,
+            tag=INVENTORY_TAG['cpu']['slug'],
+        )
+
+        if not len(nb_cpus) or \
+           len(nb_cpus) and cpus_number != len(nb_cpus):
+            for x in nb_cpus:
+                x.delete()
+            self.create_netbox_cpus()
+
+    def get_raid_cards(self):
+        if self.server.manufacturer == 'Dell':
+            if is_tool('storcli'):
+                self.raid = StorcliRaid()
+        elif self.server.manufacturer == 'HP':
+            if is_tool('ssacli'):
+                self.raid = HPRaid()
+
+        if not self.raid:
+            return
+
+        controllers = self.raid.get_controllers()
+        if len(self.raid.get_controllers()):
+            return controllers
+
+    def get_netbox_raid_cards(self):
+        raid_cards = nb.dcim.inventory_items.filter(
+            device_id=self.device_id,
+            tag=INVENTORY_TAG['raid_card']['slug'],
+            )
+        return raid_cards
+
+    def find_or_create_manufacturer(self, name):
+        if name is None:
+            return None
+        manufacturer = nb.dcim.manufacturers.get(
+            name=name,
+        )
+        if not manufacturer:
+            manufacturer = nb.dcim.manufacturers.create(
+                name=name,
+                slug=name.lower(),
+            )
+            logging.info('Creating missing manufacturer {name}'.format(name=name))
+        return manufacturer
+
+    def create_netbox_raid_card(self, raid_card):
+        manufacturer = self.find_or_create_manufacturer(
+            raid_card.get_manufacturer()
+        )
+        name = raid_card.get_product_name()
+        serial = raid_card.get_serial_number()
+        nb_raid_card = nb.dcim.inventory_items.create(
+            device=self.device_id,
+            discovered=True,
+            manufacturer=manufacturer.id if manufacturer else None,
+            tags=[INVENTORY_TAG['raid_card']['name']],
+            name='{}'.format(name),
+            serial='{}'.format(serial),
+            description='RAID Card',
+        )
+        logging.info('Creating RAID Card {name} (SN: {serial})'.format(
+            name=name,
+            serial=serial,
+        ))
+        return nb_raid_card
+
+    def create_netbox_raid_cards(self):
+        for raid_card in self.get_raid_cards():
+            self.create_netbox_raid_card(raid_card)
+
+    def update_netbox_raid_cards(self):
+        """
+        Update raid cards in netbobx
+        Since we only push:
+        * Name
+        * Manufacturer
+        * Serial
+
+        We only need to handle destroy and new cards
+        """
+
+        nb_raid_cards = self.get_netbox_raid_cards()
+        raid_cards = self.get_raid_cards()
+
+        # delete cards that are in netbox but not locally
+        # use the serial_number has the comparison element
+        for nb_raid_card in nb_raid_cards:
+            if nb_raid_card.serial not in [x.get_serial_number() for x in raid_cards]:
+                logging.info('Deleting unknown locally RAID Card {serial}'.format(
+                    serial=nb_raid_card.serial,
+                ))
+                nb_raid_card.delete()
+
+        # create card that are not in netbox
+        for raid_card in raid_cards:
+            if raid_card.get_serial_number() not in [x.serial for x in nb_raid_cards]:
+                self.create_netbox_raid_card(raid_card)
+
+    def get_disks(self):
+        ret = []
+        for raid_card in self.get_raid_cards():
+            ret += raid_card.get_physical_disks()
+        return ret
+
+    def get_netbox_disks(self):
+        disks = nb.dcim.inventory_items.filter(
+            device_id=self.device_id,
+            tag=INVENTORY_TAG['disk']['slug'],
+            )
+        return disks
+
+    def create_netbox_disks(self):
+        for disk in self.get_disks():
+            _ = nb.dcim.inventory_items.create(
+                device=self.device_id,
+                discovered=True,
+                tags=[INVENTORY_TAG['disk']['name']],
+                name='{} ({})'.format(disk['Model'], disk['Size']),
+                serial=disk['SN'],
+            )
+            logging.info('Creating Disk {model} {serial}'.format(
+                model=disk['Model'],
+                serial=disk['SN'],
+            ))
+
+    def update_netbox_disks(self):
+        nb_disks = self.get_netbox_disks()
+        disks = self.get_disks()
+
+        # delete disks that are in netbox but not locally
+        # use the serial_number has the comparison element
+        for nb_disk in nb_disks:
+            if nb_disk.serial not in [x['SN'] for x in disks]:
+                logging.info('Deleting unknown locally Disk {serial}'.format(
+                    serial=nb_disk.serial,
+                ))
+                nb_disk.delete()
+
+        # create disks that are not in netbox
+        for disk in disks:
+            if disk['SN'] not in [x.serial for x in nb_disks]:
+                nb_disk = nb.dcim.inventory_items.create(
+                    device=self.device_id,
+                    discovered=True,
+                    tags=[INVENTORY_TAG['disk']['name']],
+                    name='{} ({})'.format(disk['Model'], disk['Size']),
+                    serial=disk['SN'],
+                    description=disk.get('Type', ''),
+                )
+                logging.info('Creating Disk {model} {serial}'.format(
+                    model=disk['Model'],
+                    serial=disk['SN'],
+                ))
+
+    def get_memory(self):
+        memories = []
+        for _, value in self.server.dmi.parse().items():
+            if value['DMIName'] == 'Memory Device' and \
+               value['Size'] != 'No Module Installed':
+                memories.append({
+                    'Manufacturer': value['Manufacturer'].strip(),
+                    'Size': value['Size'].strip(),
+                    'PN': value['Part Number'].strip(),
+                    'SN': value['Serial Number'].strip(),
+                    'Locator': value['Locator'].strip(),
+                    'Type': value['Type'].strip(),
+                    })
+        return memories
+
+    def get_memory_total_size(self):
+        total_size = 0
+        for memory in self.get_memory():
+            total_size += int(memory['Size'].split()[0])
+        return total_size
+
+    def get_netbox_memory(self):
+        memories = nb.dcim.inventory_items.filter(
+            device_id=self.device_id,
+            tag=INVENTORY_TAG['memory']['slug'],
+            )
+        return memories
+
+    def create_netbox_memory(self, memory):
+        manufacturer = nb.dcim.manufacturers.get(
+            name=memory['Manufacturer']
+        )
+        if not manufacturer:
+            manufacturer = nb.dcim.manufacturers.create(
+                name=memory['Manufacturer'],
+                slug=memory['Manufacturer'].lower(),
+            )
+        nb_memory = nb.dcim.inventory_items.create(
+            device=self.device_id,
+            discovered=True,
+            manufacturer=manufacturer.id,
+            tags=[INVENTORY_TAG['memory']['name']],
+            name='{} ({} {})'.format(memory['Locator'], memory['Size'], memory['Type']),
+            part_id=memory['PN'],
+            serial=memory['SN'],
+            description='RAM',
+        )
+        logging.info('Creating Memory {type} {size}'.format(
+            type=memory['Type'],
+            size=memory['Size'],
+        ))
+        return nb_memory
+
+    def create_netbox_memories(self):
+        for memory in self.get_memory():
+            self.create_netbox_memory(memory)
+
+    def update_netbox_memory(self):
+        memories = self.get_memory()
+        nb_memories = self.get_netbox_memory()
+
+        for nb_memory in nb_memories:
+            if nb_memory.serial not in [x['SN'] for x in memories]:
+                logging.info('Deleting unknown locally Memory {serial}'.format(
+                    serial=nb_memory.serial,
+                ))
+                nb_memory.delete()
+        for memory in memories:
+            if memory['SN'] not in [x.serial for x in nb_memories]:
+                self.create_netbox_memory(memory)
+
+    def create(self):
+        if not INVENTORY_ENABLED:
+            return False
+        self.create_netbox_cpus()
+        self.create_netbox_memory()
+        self.create_netbox_raid_cards()
+        self.create_netbox_disks()
+        return True
+
+    def update(self):
+        if not INVENTORY_ENABLED:
+            return False
+        self.update_netbox_cpus()
+        self.update_netbox_memory()
+        self.update_netbox_raid_cards()
+        self.update_netbox_disks()
+        return True

netbox_agent/misc.py

@@ -0,0 +1,6 @@
+from shutil import which
+
+
+def is_tool(name):
+    '''Check whether `name` is on PATH and marked as executable.'''
+    return which(name) is not None

netbox_agent/raid/base.py

@@ -0,0 +1,21 @@
+class RaidController():
+
+    def get_product_name(self):
+        raise NotImplementedError
+
+    def get_serial_number(self):
+        raise NotImplementedError
+
+    def get_manufacturer(self):
+        raise NotImplementedError
+
+    def get_firmware_version(self):
+        raise NotImplementedError
+
+    def get_physical_disks(self):
+        raise NotImplementedError
+
+
+class Raid():
+    def get_controllers(self):
+        raise NotImplementedError

netbox_agent/raid/hp.py

@@ -0,0 +1,154 @@
+import re
+import subprocess
+
+from netbox_agent.raid.base import Raid, RaidController
+
+REGEXP_CONTROLLER_HP = re.compile(r'Smart Array ([a-zA-Z0-9- ]+) in Slot ([0-9]+)')
+
+
+def _get_indentation(string):
+    """Return the number of spaces before the current line."""
+    return len(string) - len(string.lstrip(' '))
+
+
+def _get_key_value(string):
+    """Return the (key, value) as a tuple from a string."""
+    # Normally all properties look like this:
+    #   Unique Identifier: 600508B1001CE4ACF473EE9C826230FF
+    #   Disk Name: /dev/sda
+    #   Mount Points: None
+    key = ''
+    value = ''
+    try:
+        key, value = string.split(':')
+    except ValueError:
+        # This handles the case when the property of a logical drive
+        # returned is as follows. Here we cannot split by ':' because
+        # the disk id has colon in it. So if this is about disk,
+        # then strip it accordingly.
+        #   Mirror Group 0: physicaldrive 6I:1:5
+        string = string.lstrip(' ')
+        if string.startswith('physicaldrive'):
+            fields = string.split(' ')
+            key = fields[0]
+            value = fields[1]
+        else:
+            # TODO(rameshg87): Check if this ever occurs.
+            return None, None
+
+    return key.lstrip(' ').rstrip(' '), value.lstrip(' ').rstrip(' ')
+
+
+def _get_dict(lines, start_index, indentation):
+    """Recursive function for parsing hpssacli/ssacli output."""
+
+    info = {}
+    current_item = None
+
+    i = start_index
+    while i < len(lines):
+        current_line = lines[i]
+        if current_line.startswith('Note:'):
+            i = i + 1
+            continue
+
+        current_line_indentation = _get_indentation(current_line)
+        if current_line_indentation == indentation:
+            current_item = current_line.lstrip(' ')
+
+            info[current_item] = {}
+            i = i + 1
+            continue
+
+        if i >= len(lines) - 1:
+            key, value = _get_key_value(current_line)
+            # If this is some unparsable information, then
+            # just skip it.
+            if key:
+                info[current_item][key] = value
+            return info, i
+
+        next_line = lines[i + 1]
+        next_line_indentation = _get_indentation(next_line)
+
+        if current_line_indentation == next_line_indentation:
+            key, value = _get_key_value(current_line)
+            if key:
+                info[current_item][key] = value
+            i = i + 1
+        elif next_line_indentation > current_line_indentation:
+            ret_dict, j = _get_dict(lines, i, current_line_indentation)
+            info[current_item].update(ret_dict)
+            i = j + 1
+        elif next_line_indentation < current_line_indentation:
+            key, value = _get_key_value(current_line)
+            if key:
+                info[current_item][key] = value
+            return info, i
+
+    return info, i
+
+
+class HPRaidController(RaidController):
+    def __init__(self, controller_name, data):
+        self.controller_name = controller_name
+        self.data = data
+
+    def get_product_name(self):
+        return self.controller_name
+
+    def get_manufacturer(self):
+        return 'HP'
+
+    def get_serial_number(self):
+        return self.data['Serial Number']
+
+    def get_firmware_version(self):
+        return self.data['Firmware Version']
+
+    def get_physical_disks(self):
+        ret = []
+        output = subprocess.getoutput(
+            'ssacli ctrl slot={slot} pd all show detail'.format(slot=self.data['Slot'])
+        )
+        lines = output.split('\n')
+        lines = list(filter(None, lines))
+        j = -1
+        while j < len(lines):
+            info_dict, j = _get_dict(lines, j + 1, 0)
+
+        key = next(iter(info_dict))
+        for array, physical_disk in info_dict[key].items():
+            for _, pd_attr in physical_disk.items():
+                ret.append({
+                    'Model': pd_attr.get('Model', '').strip(),
+                    'SN': pd_attr.get('Serial Number', '').strip(),
+                    'Size': pd_attr.get('Size', '').strip(),
+                    'Type': 'SSD' if pd_attr.get('Interface Type') == 'Solid State SATA'
+                    else 'HDD',
+                })
+        return ret
+
+
+class HPRaid(Raid):
+    def __init__(self):
+        self.output = subprocess.getoutput('ssacli ctrl all show detail')
+        self.controllers = []
+        self.convert_to_dict()
+
+    def convert_to_dict(self):
+        lines = self.output.split('\n')
+        lines = list(filter(None, lines))
+        j = -1
+        while j < len(lines):
+            info_dict, j = _get_dict(lines, j + 1, 0)
+            if len(info_dict.keys()):
+                _product_name = list(info_dict.keys())[0]
+                product_name = REGEXP_CONTROLLER_HP.search(_product_name)
+                if product_name:
+                    self.controllers.append(
+                        HPRaidController(product_name.group(1), info_dict[_product_name])
+                    )
+
+    def get_controllers(self):
+        return self.controllers

netbox_agent/raid/storcli.py

@@ -0,0 +1,69 @@
+import subprocess
+import json
+
+from netbox_agent.raid.base import Raid, RaidController
+
+
+class StorcliController(RaidController):
+    def __init__(self, controller_index, data):
+        self.data = data
+        self.controller_index = controller_index
+
+    def get_product_name(self):
+        return self.data['Product Name']
+
+    def get_manufacturer(self):
+        return None
+
+    def get_serial_number(self):
+        return self.data['Serial Number']
+
+    def get_firmware_version(self):
+        return self.data['FW Package Build']
+
+    def get_physical_disks(self):
+        ret = []
+        output = subprocess.getoutput(
+            'storcli /c{}/eall/sall show all J'.format(self.controller_index)
+        )
+        drive_infos = json.loads(output)['Controllers'][self.controller_index]['Response Data']
+
+        for physical_drive in self.data['PD LIST']:
+            enclosure = physical_drive.get('EID:Slt').split(':')[0]
+            slot = physical_drive.get('EID:Slt').split(':')[1]
+            size = physical_drive.get('Size').strip()
+            media_type = physical_drive.get('Med').strip()
+            drive_identifier = 'Drive /c{}/e{}/s{}'.format(
+                str(self.controller_index), str(enclosure), str(slot)
+            )
+            drive_attr = drive_infos['{} - Detailed Information'.format(drive_identifier)][
+                '{} Device attributes'.format(drive_identifier)]
+            ret.append({
+                'Model': drive_attr.get('Model Number', '').strip(),
+                'SN': drive_attr.get('SN', '').strip(),
+                'Size': size,
+                'Type': media_type,
+                })
+        return ret
+
+
+class StorcliRaid(Raid):
+    def __init__(self):
+        self.output = subprocess.getoutput('storcli /call show J')
+        self.data = json.loads(self.output)
+        self.controllers = []
+
+        if len([
+                x for x in self.data['Controllers']
+                if x['Command Status']['Status'] == 'Success'
+        ]) > 0:
+            for controller in self.data['Controllers']:
+                self.controllers.append(
+                    StorcliController(
+                        controller['Command Status']['Controller'],
+                        controller['Response Data']
+                    )
+                )
+
+    def get_controllers(self):
+        return self.controllers

netbox_agent/server.py

@@ -5,6 +5,7 @@ import socket
 from netbox_agent.config import netbox_instance as nb
 import netbox_agent.dmidecode as dmidecode
 from netbox_agent.location import Datacenter, Rack
+from netbox_agent.inventory import Inventory
 from netbox_agent.network import Network
 
 
@@ -204,6 +205,8 @@ class ServerBase():
 
         self.network = Network(server=self)
         self.network.create_netbox_network_cards()
+        self.inventory = Inventory(server=self)
+        self.inventory.create()
         logging.debug('Server created!')
 
     def _netbox_update_chassis_for_blade(self, server, datacenter):
@@ -271,12 +274,15 @@ class ServerBase():
         # check network cards
         self.network = Network(server=self)
         self.network.update_netbox_network_cards()
+        # update inventory
+        self.inventory = Inventory(server=self)
+        self.inventory.update()
         if update:
             server.save()
         logging.debug('Finished updating Server!')
 
     def print_debug(self):
-        # FIXME: do something more generic by looping on every get_* methods
+        self.network = Network(server=self)
         print('Datacenter:', self.get_datacenter())
         print('Netbox Datacenter:', self.get_netbox_datacenter())
         print('Rack:', self.get_rack())

netbox_agent/vendors/dell.py

@@ -2,6 +2,10 @@ from netbox_agent.server import ServerBase
 
 
 class DellHost(ServerBase):
+    def __init__(self, *args, **kwargs):
+        super(DellHost, self).__init__(*args, **kwargs)
+        self.manufacturer = 'Dell'
+
     def is_blade(self):
         return self.get_product_name().startswith('PowerEdge M')
 

netbox_agent/vendors/hp.py

@@ -6,6 +6,7 @@ class HPHost(ServerBase):
         super(HPHost, self).__init__(*args, **kwargs)
         if self.is_blade():
             self.hp_rack_locator = self._find_rack_locator()
+        self.manufacturer = 'HP'
 
     def is_blade(self):
         return self.get_product_name().startswith('ProLiant BL')