diff --git a/README.md b/README.md index 7c224db..60861f7 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,8 @@ rack_location: # # driver: "file:/tmp/datacenter" # regex: "(.*)" + +inventory: true ``` # Hardware @@ -124,5 +126,5 @@ Feel free to send me a dmidecode output for Supermicro's blade! # TODO -- [ ] CPU, RAID Card(s), RAM, Disks in `Device`'s `Inventory` +- [ ] handle non-raid and NVMe drive in Inventory - [ ] `CustomFields` support with firmware versions for Device (BIOS), RAID Cards and disks diff --git a/netbox_agent.yaml.example b/netbox_agent.yaml.example index f751dab..3a0481a 100644 --- a/netbox_agent.yaml.example +++ b/netbox_agent.yaml.example @@ -23,4 +23,6 @@ rack_location: # regex: 'SysName:[ ]+[A-Za-z]+-[A-Za-z]+-([A-Za-z0-9]+)' # # driver: "file:/tmp/datacenter" -# regex: "(.*)" \ No newline at end of file +# regex: "(.*)" + +inventory: true diff --git a/netbox_agent/cli.py b/netbox_agent/cli.py index 8ef6f56..644983b 100644 --- a/netbox_agent/cli.py +++ b/netbox_agent/cli.py @@ -18,7 +18,7 @@ MANUFACTURERS = { def run(args): manufacturer = dmidecode.get_by_type('Chassis')[0].get('Manufacturer') - server = MANUFACTURERS[manufacturer](dmidecode) + server = MANUFACTURERS[manufacturer](dmi=dmidecode) if args.debug: server.print_debug() if args.register: diff --git a/netbox_agent/config.py b/netbox_agent/config.py index 9b34c41..194c78f 100644 --- a/netbox_agent/config.py +++ b/netbox_agent/config.py @@ -38,3 +38,5 @@ if config.get('network'): NETWORK_IGNORE_INTERFACES = config['network'].get('ignore_interfaces') NETWORK_IGNORE_IPS = config['network'].get('ignore_ips') NETWORK_LLDP = config['network'].get('lldp') is True + +INVENTORY_ENABLED = config.get('inventory') is True diff --git a/netbox_agent/inventory.py b/netbox_agent/inventory.py new file mode 100644 index 0000000..06aee92 --- /dev/null +++ b/netbox_agent/inventory.py @@ -0,0 +1,328 @@ +import logging +import subprocess +import re + +from netbox_agent.config import netbox_instance as nb, INVENTORY_ENABLED +from netbox_agent.misc import is_tool +from netbox_agent.raid.hp import HPRaid +from netbox_agent.raid.storcli import StorcliRaid + +INVENTORY_TAG = { + 'cpu': {'name': 'hw:cpu', 'slug': 'hw-cpu'}, + 'memory': {'name': 'hw:memory', 'slug': 'hw-memory'}, + 'disk': {'name': 'hw:disk', 'slug': 'hw-disk'}, + 'raid_card': {'name': 'hw:raid_card', 'slug': 'hw-raid-card'}, + } + +for key, tag in INVENTORY_TAG.items(): + nb_tag = nb.extras.tags.get( + name=tag['name'] + ) + if not nb_tag: + nb_tag = nb.extras.tags.create( + name=tag['name'], + slug=tag['slug'], + comments=tag['name'], + ) + + +class Inventory(): + """ + Better Inventory items coming, see: + - https://github.com/netbox-community/netbox/issues/3087 + - https://github.com/netbox-community/netbox/issues/3333 + + This class implements for: + * memory + * cpu + * raid cards + * disks + + methods that: + * get local item + * get netbox item + * create netbox item + * update netbox item + + Known issues: + - no scan of non-raid devices + - no scan of NVMe devices + """ + + def __init__(self, server): + self.server = server + self.device_id = self.server.get_netbox_server().id + self.raid = None + self.disks = [] + + def get_cpus(self): + model = None + nb = None + + output = subprocess.getoutput('lscpu') + model_re = re.search(r'Model name: (.*)', output) + if len(model_re.groups()) > 0: + model = model_re.groups()[0].strip() + socket_re = re.search(r'Socket\(s\): (.*)', output) + if len(socket_re.groups()) > 0: + nb = int(socket_re.groups()[0].strip()) + return nb, model + + def create_netbox_cpus(self): + nb_cpus, model = self.get_cpus() + for i in range(nb_cpus): + _ = nb.dcim.inventory_items.create( + device=self.device_id, + tags=[INVENTORY_TAG['cpu']['name']], + name=model, + discovered=True, + description='CPU', + ) + logging.info('Creating CPU model {model}'.format(model=model)) + + def update_netbox_cpus(self): + cpus_number, model = self.get_cpus() + nb_cpus = nb.dcim.inventory_items.filter( + device_id=self.device_id, + tag=INVENTORY_TAG['cpu']['slug'], + ) + + if not len(nb_cpus) or \ + len(nb_cpus) and cpus_number != len(nb_cpus): + for x in nb_cpus: + x.delete() + self.create_netbox_cpus() + + def get_raid_cards(self): + if self.server.manufacturer == 'Dell': + if is_tool('storcli'): + self.raid = StorcliRaid() + elif self.server.manufacturer == 'HP': + if is_tool('ssacli'): + self.raid = HPRaid() + + if not self.raid: + return + + controllers = self.raid.get_controllers() + if len(self.raid.get_controllers()): + return controllers + + def get_netbox_raid_cards(self): + raid_cards = nb.dcim.inventory_items.filter( + device_id=self.device_id, + tag=INVENTORY_TAG['raid_card']['slug'], + ) + return raid_cards + + def find_or_create_manufacturer(self, name): + if name is None: + return None + manufacturer = nb.dcim.manufacturers.get( + name=name, + ) + if not manufacturer: + manufacturer = nb.dcim.manufacturers.create( + name=name, + slug=name.lower(), + ) + logging.info('Creating missing manufacturer {name}'.format(name=name)) + return manufacturer + + def create_netbox_raid_card(self, raid_card): + manufacturer = self.find_or_create_manufacturer( + raid_card.get_manufacturer() + ) + name = raid_card.get_product_name() + serial = raid_card.get_serial_number() + nb_raid_card = nb.dcim.inventory_items.create( + device=self.device_id, + discovered=True, + manufacturer=manufacturer.id if manufacturer else None, + tags=[INVENTORY_TAG['raid_card']['name']], + name='{}'.format(name), + serial='{}'.format(serial), + description='RAID Card', + ) + logging.info('Creating RAID Card {name} (SN: {serial})'.format( + name=name, + serial=serial, + )) + return nb_raid_card + + def create_netbox_raid_cards(self): + for raid_card in self.get_raid_cards(): + self.create_netbox_raid_card(raid_card) + + def update_netbox_raid_cards(self): + """ + Update raid cards in netbobx + Since we only push: + * Name + * Manufacturer + * Serial + + We only need to handle destroy and new cards + """ + + nb_raid_cards = self.get_netbox_raid_cards() + raid_cards = self.get_raid_cards() + + # delete cards that are in netbox but not locally + # use the serial_number has the comparison element + for nb_raid_card in nb_raid_cards: + if nb_raid_card.serial not in [x.get_serial_number() for x in raid_cards]: + logging.info('Deleting unknown locally RAID Card {serial}'.format( + serial=nb_raid_card.serial, + )) + nb_raid_card.delete() + + # create card that are not in netbox + for raid_card in raid_cards: + if raid_card.get_serial_number() not in [x.serial for x in nb_raid_cards]: + self.create_netbox_raid_card(raid_card) + + def get_disks(self): + ret = [] + for raid_card in self.get_raid_cards(): + ret += raid_card.get_physical_disks() + return ret + + def get_netbox_disks(self): + disks = nb.dcim.inventory_items.filter( + device_id=self.device_id, + tag=INVENTORY_TAG['disk']['slug'], + ) + return disks + + def create_netbox_disks(self): + for disk in self.get_disks(): + _ = nb.dcim.inventory_items.create( + device=self.device_id, + discovered=True, + tags=[INVENTORY_TAG['disk']['name']], + name='{} ({})'.format(disk['Model'], disk['Size']), + serial=disk['SN'], + ) + logging.info('Creating Disk {model} {serial}'.format( + model=disk['Model'], + serial=disk['SN'], + )) + + def update_netbox_disks(self): + nb_disks = self.get_netbox_disks() + disks = self.get_disks() + + # delete disks that are in netbox but not locally + # use the serial_number has the comparison element + for nb_disk in nb_disks: + if nb_disk.serial not in [x['SN'] for x in disks]: + logging.info('Deleting unknown locally Disk {serial}'.format( + serial=nb_disk.serial, + )) + nb_disk.delete() + + # create disks that are not in netbox + for disk in disks: + if disk['SN'] not in [x.serial for x in nb_disks]: + nb_disk = nb.dcim.inventory_items.create( + device=self.device_id, + discovered=True, + tags=[INVENTORY_TAG['disk']['name']], + name='{} ({})'.format(disk['Model'], disk['Size']), + serial=disk['SN'], + description=disk.get('Type', ''), + ) + logging.info('Creating Disk {model} {serial}'.format( + model=disk['Model'], + serial=disk['SN'], + )) + + def get_memory(self): + memories = [] + for _, value in self.server.dmi.parse().items(): + if value['DMIName'] == 'Memory Device' and \ + value['Size'] != 'No Module Installed': + memories.append({ + 'Manufacturer': value['Manufacturer'].strip(), + 'Size': value['Size'].strip(), + 'PN': value['Part Number'].strip(), + 'SN': value['Serial Number'].strip(), + 'Locator': value['Locator'].strip(), + 'Type': value['Type'].strip(), + }) + return memories + + def get_memory_total_size(self): + total_size = 0 + for memory in self.get_memory(): + total_size += int(memory['Size'].split()[0]) + return total_size + + def get_netbox_memory(self): + memories = nb.dcim.inventory_items.filter( + device_id=self.device_id, + tag=INVENTORY_TAG['memory']['slug'], + ) + return memories + + def create_netbox_memory(self, memory): + manufacturer = nb.dcim.manufacturers.get( + name=memory['Manufacturer'] + ) + if not manufacturer: + manufacturer = nb.dcim.manufacturers.create( + name=memory['Manufacturer'], + slug=memory['Manufacturer'].lower(), + ) + nb_memory = nb.dcim.inventory_items.create( + device=self.device_id, + discovered=True, + manufacturer=manufacturer.id, + tags=[INVENTORY_TAG['memory']['name']], + name='{} ({} {})'.format(memory['Locator'], memory['Size'], memory['Type']), + part_id=memory['PN'], + serial=memory['SN'], + description='RAM', + ) + logging.info('Creating Memory {type} {size}'.format( + type=memory['Type'], + size=memory['Size'], + )) + return nb_memory + + def create_netbox_memories(self): + for memory in self.get_memory(): + self.create_netbox_memory(memory) + + def update_netbox_memory(self): + memories = self.get_memory() + nb_memories = self.get_netbox_memory() + + for nb_memory in nb_memories: + if nb_memory.serial not in [x['SN'] for x in memories]: + logging.info('Deleting unknown locally Memory {serial}'.format( + serial=nb_memory.serial, + )) + nb_memory.delete() + for memory in memories: + if memory['SN'] not in [x.serial for x in nb_memories]: + self.create_netbox_memory(memory) + + def create(self): + if not INVENTORY_ENABLED: + return False + self.create_netbox_cpus() + self.create_netbox_memory() + self.create_netbox_raid_cards() + self.create_netbox_disks() + return True + + def update(self): + if not INVENTORY_ENABLED: + return False + self.update_netbox_cpus() + self.update_netbox_memory() + self.update_netbox_raid_cards() + self.update_netbox_disks() + return True diff --git a/netbox_agent/misc.py b/netbox_agent/misc.py new file mode 100644 index 0000000..b28e4b3 --- /dev/null +++ b/netbox_agent/misc.py @@ -0,0 +1,6 @@ +from shutil import which + + +def is_tool(name): + '''Check whether `name` is on PATH and marked as executable.''' + return which(name) is not None diff --git a/netbox_agent/raid/__init__.py b/netbox_agent/raid/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/netbox_agent/raid/base.py b/netbox_agent/raid/base.py new file mode 100644 index 0000000..a3d3081 --- /dev/null +++ b/netbox_agent/raid/base.py @@ -0,0 +1,21 @@ +class RaidController(): + + def get_product_name(self): + raise NotImplementedError + + def get_serial_number(self): + raise NotImplementedError + + def get_manufacturer(self): + raise NotImplementedError + + def get_firmware_version(self): + raise NotImplementedError + + def get_physical_disks(self): + raise NotImplementedError + + +class Raid(): + def get_controllers(self): + raise NotImplementedError diff --git a/netbox_agent/raid/hp.py b/netbox_agent/raid/hp.py new file mode 100644 index 0000000..b2a4858 --- /dev/null +++ b/netbox_agent/raid/hp.py @@ -0,0 +1,154 @@ +import re +import subprocess + +from netbox_agent.raid.base import Raid, RaidController + +REGEXP_CONTROLLER_HP = re.compile(r'Smart Array ([a-zA-Z0-9- ]+) in Slot ([0-9]+)') + + +def _get_indentation(string): + """Return the number of spaces before the current line.""" + return len(string) - len(string.lstrip(' ')) + + +def _get_key_value(string): + """Return the (key, value) as a tuple from a string.""" + # Normally all properties look like this: + # Unique Identifier: 600508B1001CE4ACF473EE9C826230FF + # Disk Name: /dev/sda + # Mount Points: None + key = '' + value = '' + try: + key, value = string.split(':') + except ValueError: + # This handles the case when the property of a logical drive + # returned is as follows. Here we cannot split by ':' because + # the disk id has colon in it. So if this is about disk, + # then strip it accordingly. + # Mirror Group 0: physicaldrive 6I:1:5 + string = string.lstrip(' ') + if string.startswith('physicaldrive'): + fields = string.split(' ') + key = fields[0] + value = fields[1] + else: + # TODO(rameshg87): Check if this ever occurs. + return None, None + + return key.lstrip(' ').rstrip(' '), value.lstrip(' ').rstrip(' ') + + +def _get_dict(lines, start_index, indentation): + """Recursive function for parsing hpssacli/ssacli output.""" + + info = {} + current_item = None + + i = start_index + while i < len(lines): + current_line = lines[i] + if current_line.startswith('Note:'): + i = i + 1 + continue + + current_line_indentation = _get_indentation(current_line) + if current_line_indentation == indentation: + current_item = current_line.lstrip(' ') + + info[current_item] = {} + i = i + 1 + continue + + if i >= len(lines) - 1: + key, value = _get_key_value(current_line) + # If this is some unparsable information, then + # just skip it. + if key: + info[current_item][key] = value + return info, i + + next_line = lines[i + 1] + next_line_indentation = _get_indentation(next_line) + + if current_line_indentation == next_line_indentation: + key, value = _get_key_value(current_line) + if key: + info[current_item][key] = value + i = i + 1 + elif next_line_indentation > current_line_indentation: + ret_dict, j = _get_dict(lines, i, current_line_indentation) + info[current_item].update(ret_dict) + i = j + 1 + elif next_line_indentation < current_line_indentation: + key, value = _get_key_value(current_line) + if key: + info[current_item][key] = value + return info, i + + return info, i + + +class HPRaidController(RaidController): + def __init__(self, controller_name, data): + self.controller_name = controller_name + self.data = data + + def get_product_name(self): + return self.controller_name + + def get_manufacturer(self): + return 'HP' + + def get_serial_number(self): + return self.data['Serial Number'] + + def get_firmware_version(self): + return self.data['Firmware Version'] + + def get_physical_disks(self): + ret = [] + output = subprocess.getoutput( + 'ssacli ctrl slot={slot} pd all show detail'.format(slot=self.data['Slot']) + ) + lines = output.split('\n') + lines = list(filter(None, lines)) + j = -1 + while j < len(lines): + info_dict, j = _get_dict(lines, j + 1, 0) + + key = next(iter(info_dict)) + for array, physical_disk in info_dict[key].items(): + for _, pd_attr in physical_disk.items(): + ret.append({ + 'Model': pd_attr.get('Model', '').strip(), + 'SN': pd_attr.get('Serial Number', '').strip(), + 'Size': pd_attr.get('Size', '').strip(), + 'Type': 'SSD' if pd_attr.get('Interface Type') == 'Solid State SATA' + else 'HDD', + }) + return ret + + +class HPRaid(Raid): + def __init__(self): + self.output = subprocess.getoutput('ssacli ctrl all show detail') + self.controllers = [] + self.convert_to_dict() + + def convert_to_dict(self): + lines = self.output.split('\n') + lines = list(filter(None, lines)) + j = -1 + while j < len(lines): + info_dict, j = _get_dict(lines, j + 1, 0) + if len(info_dict.keys()): + _product_name = list(info_dict.keys())[0] + product_name = REGEXP_CONTROLLER_HP.search(_product_name) + if product_name: + self.controllers.append( + HPRaidController(product_name.group(1), info_dict[_product_name]) + ) + + def get_controllers(self): + return self.controllers diff --git a/netbox_agent/raid/storcli.py b/netbox_agent/raid/storcli.py new file mode 100644 index 0000000..48989b8 --- /dev/null +++ b/netbox_agent/raid/storcli.py @@ -0,0 +1,69 @@ +import subprocess +import json + +from netbox_agent.raid.base import Raid, RaidController + + +class StorcliController(RaidController): + def __init__(self, controller_index, data): + self.data = data + self.controller_index = controller_index + + def get_product_name(self): + return self.data['Product Name'] + + def get_manufacturer(self): + return None + + def get_serial_number(self): + return self.data['Serial Number'] + + def get_firmware_version(self): + return self.data['FW Package Build'] + + def get_physical_disks(self): + ret = [] + output = subprocess.getoutput( + 'storcli /c{}/eall/sall show all J'.format(self.controller_index) + ) + drive_infos = json.loads(output)['Controllers'][self.controller_index]['Response Data'] + + for physical_drive in self.data['PD LIST']: + enclosure = physical_drive.get('EID:Slt').split(':')[0] + slot = physical_drive.get('EID:Slt').split(':')[1] + size = physical_drive.get('Size').strip() + media_type = physical_drive.get('Med').strip() + drive_identifier = 'Drive /c{}/e{}/s{}'.format( + str(self.controller_index), str(enclosure), str(slot) + ) + drive_attr = drive_infos['{} - Detailed Information'.format(drive_identifier)][ + '{} Device attributes'.format(drive_identifier)] + ret.append({ + 'Model': drive_attr.get('Model Number', '').strip(), + 'SN': drive_attr.get('SN', '').strip(), + 'Size': size, + 'Type': media_type, + }) + return ret + + +class StorcliRaid(Raid): + def __init__(self): + self.output = subprocess.getoutput('storcli /call show J') + self.data = json.loads(self.output) + self.controllers = [] + + if len([ + x for x in self.data['Controllers'] + if x['Command Status']['Status'] == 'Success' + ]) > 0: + for controller in self.data['Controllers']: + self.controllers.append( + StorcliController( + controller['Command Status']['Controller'], + controller['Response Data'] + ) + ) + + def get_controllers(self): + return self.controllers diff --git a/netbox_agent/server.py b/netbox_agent/server.py index f9bff13..f57fa9e 100644 --- a/netbox_agent/server.py +++ b/netbox_agent/server.py @@ -5,6 +5,7 @@ import socket from netbox_agent.config import netbox_instance as nb import netbox_agent.dmidecode as dmidecode from netbox_agent.location import Datacenter, Rack +from netbox_agent.inventory import Inventory from netbox_agent.network import Network @@ -204,6 +205,8 @@ class ServerBase(): self.network = Network(server=self) self.network.create_netbox_network_cards() + self.inventory = Inventory(server=self) + self.inventory.create() logging.debug('Server created!') def _netbox_update_chassis_for_blade(self, server, datacenter): @@ -271,12 +274,15 @@ class ServerBase(): # check network cards self.network = Network(server=self) self.network.update_netbox_network_cards() + # update inventory + self.inventory = Inventory(server=self) + self.inventory.update() if update: server.save() logging.debug('Finished updating Server!') def print_debug(self): - # FIXME: do something more generic by looping on every get_* methods + self.network = Network(server=self) print('Datacenter:', self.get_datacenter()) print('Netbox Datacenter:', self.get_netbox_datacenter()) print('Rack:', self.get_rack()) diff --git a/netbox_agent/vendors/dell.py b/netbox_agent/vendors/dell.py index 1e03e9f..8c0493b 100644 --- a/netbox_agent/vendors/dell.py +++ b/netbox_agent/vendors/dell.py @@ -2,6 +2,10 @@ from netbox_agent.server import ServerBase class DellHost(ServerBase): + def __init__(self, *args, **kwargs): + super(DellHost, self).__init__(*args, **kwargs) + self.manufacturer = 'Dell' + def is_blade(self): return self.get_product_name().startswith('PowerEdge M') diff --git a/netbox_agent/vendors/hp.py b/netbox_agent/vendors/hp.py index cb2f516..e9a0c24 100644 --- a/netbox_agent/vendors/hp.py +++ b/netbox_agent/vendors/hp.py @@ -6,6 +6,7 @@ class HPHost(ServerBase): super(HPHost, self).__init__(*args, **kwargs) if self.is_blade(): self.hp_rack_locator = self._find_rack_locator() + self.manufacturer = 'HP' def is_blade(self): return self.get_product_name().startswith('ProLiant BL')