diff --git a/netbox_agent/config.py b/netbox_agent/config.py index cf88f69..68d7d82 100644 --- a/netbox_agent/config.py +++ b/netbox_agent/config.py @@ -28,6 +28,10 @@ def get_config(): p.add_argument('--update-inventory', action='store_true', help='Update inventory') p.add_argument('--update-location', action='store_true', help='Update location') p.add_argument('--update-psu', action='store_true', help='Update PSU') + p.add_argument('--purge-old-devices', action='store_true', + help='Purge existing (old ?) devices having same name but different serial') + p.add_argument('--expansion-as-device', action='store_true', + help='Manage blade expansions as external devices') p.add_argument('--log_level', default='debug') p.add_argument('--netbox.url', help='Netbox URL') diff --git a/netbox_agent/inventory.py b/netbox_agent/inventory.py index dfef77c..6525bea 100644 --- a/netbox_agent/inventory.py +++ b/netbox_agent/inventory.py @@ -46,10 +46,11 @@ class Inventory(): - no scan of NVMe devices """ - def __init__(self, server): + def __init__(self, server, update_expansion=False): self.create_netbox_tags() self.server = server - netbox_server = self.server.get_netbox_server() + self.update_expansion = update_expansion + netbox_server = self.server.get_netbox_server(update_expansion) self.device_id = netbox_server.id if netbox_server else None self.raid = None @@ -223,7 +224,7 @@ class Inventory(): self.create_netbox_cpus() - def get_raid_cards(self): + def get_raid_cards(self, filter_cards=False): raid_class = None if self.server.manufacturer == 'Dell': if is_tool('omreport'): @@ -238,9 +239,15 @@ class Inventory(): return [] self.raid = raid_class() - controllers = self.raid.get_controllers() - if len(self.raid.get_controllers()): - return controllers + + if filter_cards and config.expansion_as_device \ + and self.server.own_expansion_slot(): + return [ + c for c in self.raid.get_controllers() + if c.is_external() is self.update_expansion + ] + else: + return self.raid.get_controllers() def create_netbox_raid_card(self, raid_card): manufacturer = self.find_or_create_manufacturer( @@ -279,7 +286,7 @@ class Inventory(): device_id=self.device_id, tag=[INVENTORY_TAG['raid_card']['slug']] ) - raid_cards = self.get_raid_cards() + raid_cards = self.get_raid_cards(filter_cards=True) # delete cards that are in netbox but not locally # use the serial_number has the comparison element @@ -339,7 +346,7 @@ class Inventory(): d['Vendor'] = get_vendor(disk['product']) disks.append(d) - for raid_card in self.get_raid_cards(): + for raid_card in self.get_raid_cards(filter_cards=True): disks += raid_card.get_physical_disks() # remove duplicate serials @@ -466,21 +473,24 @@ class Inventory(): tag=INVENTORY_TAG['gpu']['slug'], ) - if not len(nb_gpus) or \ + if config.expansion_as_device and len(nb_gpus): + for x in nb_gpus: + x.delete() + elif not len(nb_gpus) or \ len(nb_gpus) and len(gpus) != len(nb_gpus): for x in nb_gpus: x.delete() - self.create_netbox_gpus() def create_or_update(self): if config.inventory is None or config.update_inventory is None: return False - self.do_netbox_cpus() - self.do_netbox_memories() - self.do_netbox_raid_cards() - self.do_netbox_disks() - self.do_netbox_interfaces() - self.do_netbox_motherboard() + if self.update_expansion is False: + self.do_netbox_cpus() + self.do_netbox_memories() + self.do_netbox_interfaces() + self.do_netbox_motherboard() self.do_netbox_gpus() + self.do_netbox_disks() + self.do_netbox_raid_cards() return True diff --git a/netbox_agent/lshw.py b/netbox_agent/lshw.py index 0a5a870..8070633 100644 --- a/netbox_agent/lshw.py +++ b/netbox_agent/lshw.py @@ -104,7 +104,10 @@ class LSHW(): d['product'] = device["ModelNumber"] d['serial'] = device["SerialNumber"] d["version"] = device["Firmware"] - d['size'] = device["UsedSize"] + if "UsedSize" in device: + d['size'] = device["UsedSize"] + if "UsedBytes" in device: + d['size'] = device["UsedBytes"] d['description'] = "NVME Disk" self.disks.append(d) diff --git a/netbox_agent/raid/base.py b/netbox_agent/raid/base.py index a3d3081..97b8274 100644 --- a/netbox_agent/raid/base.py +++ b/netbox_agent/raid/base.py @@ -15,6 +15,9 @@ class RaidController(): def get_physical_disks(self): raise NotImplementedError + def is_external(self): + return False + class Raid(): def get_controllers(self): diff --git a/netbox_agent/raid/hp.py b/netbox_agent/raid/hp.py index 144e73b..5d5d140 100644 --- a/netbox_agent/raid/hp.py +++ b/netbox_agent/raid/hp.py @@ -1,106 +1,65 @@ import re import subprocess +from netbox_agent.config import config from netbox_agent.misc import get_vendor from netbox_agent.raid.base import Raid, RaidController REGEXP_CONTROLLER_HP = re.compile(r'Smart Array ([a-zA-Z0-9- ]+) in Slot ([0-9]+)') -def _get_indentation(string): - """Return the number of spaces before the current line.""" - return len(string) - len(string.lstrip(' ')) +def _parse_ctrl_output(lines): + controllers = {} + current_ctrl = None - -def _get_key_value(string): - """Return the (key, value) as a tuple from a string.""" - # Normally all properties look like this: - # Unique Identifier: 600508B1001CE4ACF473EE9C826230FF - # Disk Name: /dev/sda - # Mount Points: None - key = '' - value = '' - try: - key, value = string.split(':') - except ValueError: - # This handles the case when the property of a logical drive - # returned is as follows. Here we cannot split by ':' because - # the disk id has colon in it. So if this is about disk, - # then strip it accordingly. - # Mirror Group 0: physicaldrive 6I:1:5 - string = string.lstrip(' ') - if string.startswith('physicaldrive'): - fields = string.split(' ') - key = fields[0] - value = fields[1] - else: - # TODO(rameshg87): Check if this ever occurs. - return None, None - - return key.lstrip(' ').rstrip(' '), value.lstrip(' ').rstrip(' ') - - -def _get_dict(lines, start_index, indentation): - """Recursive function for parsing hpssacli/ssacli output.""" - - info = {} - current_item = None - - i = start_index - while i < len(lines): - current_line = lines[i] - if current_line.startswith('Note:'): - i = i + 1 + for line in lines: + if not line or line.startswith('Note:'): continue - - current_line_indentation = _get_indentation(current_line) - # This check ignore some useless information that make - # crash the parsing - product_name = REGEXP_CONTROLLER_HP.search(current_line) - if current_line_indentation == 0 and not product_name: - i = i + 1 + ctrl = REGEXP_CONTROLLER_HP.search(line) + if ctrl is not None: + current_ctrl = ctrl.group(1) + controllers[current_ctrl] = {"Slot": ctrl.group(2)} + if "Embedded" not in line: + controllers[current_ctrl]["External"] = True continue + attr, val = line.split(": ", 1) + attr = attr.strip() + val = val.strip() + controllers[current_ctrl][attr] = val + return controllers - if current_line_indentation == indentation: - current_item = current_line.lstrip(' ') - info[current_item] = {} - i = i + 1 +def _parse_pd_output(lines): + drives = {} + current_array = None + current_drv = None + + for line in lines: + line = line.strip() + if not line or line.startswith('Note:'): continue - - if i >= len(lines) - 1: - key, value = _get_key_value(current_line) - # If this is some unparsable information, then - # just skip it. - if key: - info[current_item][key] = value - return info, i - - next_line = lines[i + 1] - next_line_indentation = _get_indentation(next_line) - - if current_line_indentation == next_line_indentation: - key, value = _get_key_value(current_line) - if key: - info[current_item][key] = value - i = i + 1 - elif next_line_indentation > current_line_indentation: - ret_dict, j = _get_dict(lines, i, current_line_indentation) - info[current_item].update(ret_dict) - i = j + 1 - elif next_line_indentation < current_line_indentation: - key, value = _get_key_value(current_line) - if key: - info[current_item][key] = value - return info, i - - return info, i + # Parses the Array the drives are in + if line.startswith("Array"): + current_array = line.split(None, 1)[1] + # Detects new physical drive + if line.startswith("physicaldrive"): + current_drv = line.split(None, 1)[1] + drives[current_drv] = {} + if current_array is not None: + drives[current_drv]["Array"] = current_array + continue + if ": " not in line: + continue + attr, val = line.split(": ", 1) + drives.setdefault(current_drv, {})[attr] = val + return drives class HPRaidController(RaidController): def __init__(self, controller_name, data): self.controller_name = controller_name self.data = data + self.drives = self._get_physical_disks() def get_product_name(self): return self.controller_name @@ -114,40 +73,42 @@ class HPRaidController(RaidController): def get_firmware_version(self): return self.data['Firmware Version'] - def get_physical_disks(self): - ret = [] + def is_external(self): + return self.data.get('External', False) + + def _get_physical_disks(self): output = subprocess.getoutput( 'ssacli ctrl slot={slot} pd all show detail'.format(slot=self.data['Slot']) ) lines = output.split('\n') lines = list(filter(None, lines)) - j = -1 - while j < len(lines): - info_dict, j = _get_dict(lines, j + 1, 0) + drives = _parse_pd_output(lines) + ret = [] - key = next(iter(info_dict)) - for array, physical_disk in info_dict[key].items(): - for _, pd_attr in physical_disk.items(): - model = pd_attr.get('Model', '').strip() - vendor = None - if model.startswith('HP'): - vendor = 'HP' - elif len(model.split()) > 1: - vendor = get_vendor(model.split()[1]) - else: - vendor = get_vendor(model) + for name, attrs in drives.items(): + model = attrs.get('Model', '').strip() + vendor = None + if model.startswith('HP'): + vendor = 'HP' + elif len(model.split()) > 1: + vendor = get_vendor(model.split()[1]) + else: + vendor = get_vendor(model) - ret.append({ - 'Model': model, - 'Vendor': vendor, - 'SN': pd_attr.get('Serial Number', '').strip(), - 'Size': pd_attr.get('Size', '').strip(), - 'Type': 'SSD' if pd_attr.get('Interface Type') == 'Solid State SATA' - else 'HDD', - '_src': self.__class__.__name__, - }) + ret.append({ + 'Model': model, + 'Vendor': vendor, + 'SN': attrs.get('Serial Number', '').strip(), + 'Size': attrs.get('Size', '').strip(), + 'Type': 'SSD' if attrs.get('Interface Type') == 'Solid State SATA' + else 'HDD', + '_src': self.__class__.__name__, + }) return ret + def get_physical_disks(self): + return self.drives + class HPRaid(Raid): def __init__(self): @@ -158,16 +119,11 @@ class HPRaid(Raid): def convert_to_dict(self): lines = self.output.split('\n') lines = list(filter(None, lines)) - j = -1 - while j < len(lines): - info_dict, j = _get_dict(lines, j + 1, 0) - if len(info_dict.keys()): - _product_name = list(info_dict.keys())[0] - product_name = REGEXP_CONTROLLER_HP.search(_product_name) - if product_name: - self.controllers.append( - HPRaidController(product_name.group(1), info_dict[_product_name]) - ) + controllers = _parse_ctrl_output(lines) + for controller, attrs in controllers.items(): + self.controllers.append( + HPRaidController(controller, attrs) + ) def get_controllers(self): return self.controllers diff --git a/netbox_agent/server.py b/netbox_agent/server.py index 06a2742..a20023b 100644 --- a/netbox_agent/server.py +++ b/netbox_agent/server.py @@ -25,6 +25,7 @@ class ServerBase(): self.bios = dmidecode.get_by_type(self.dmi, 'BIOS') self.chassis = dmidecode.get_by_type(self.dmi, 'Chassis') self.system = dmidecode.get_by_type(self.dmi, 'System') + self.inventory = Inventory(server=self) self.network = None @@ -67,7 +68,6 @@ class ServerBase(): def update_netbox_location(self, server): dc = self.get_datacenter() - rack = self.get_rack() nb_rack = self.get_netbox_rack() nb_dc = self.get_netbox_datacenter() @@ -80,7 +80,11 @@ class ServerBase(): update = True server.site = nb_dc.id - if rack and server.rack and server.rack.id != nb_rack.id: + if ( + server.rack + and nb_rack + and server.rack.id != nb_rack.id + ): logging.info('Rack location has changed from {} to {}, updating'.format( server.rack, nb_rack, @@ -92,6 +96,19 @@ class ServerBase(): server.position = None return update, server + def update_netbox_expansion_location(self, server, expansion): + update = False + if expansion.tenant != server.tenant: + expansion.tenant = server.tenant + update = True + if expansion.site != server.site: + expansion.site = server.site + update = True + if expansion.rack != server.rack: + expansion.rack = server.rack + update = True + return update + def get_rack(self): rack = Rack() return rack.get() @@ -226,6 +243,13 @@ class ServerBase(): ) return new_blade + def _netbox_deduplicate_server(self): + serial = self.get_service_tag() + hostname = self.get_hostname() + server = nb.dcim.devices.get(name=hostname) + if server and server.serial != serial: + server.delete() + def _netbox_create_server(self, datacenter, tenant, rack): device_role = get_device_role(config.device.server_role) device_type = get_device_type(self.get_product_name()) @@ -247,8 +271,11 @@ class ServerBase(): ) return new_server - def get_netbox_server(self): - return nb.dcim.devices.get(serial=self.get_service_tag()) + def get_netbox_server(self, expansion=False): + if expansion is False: + return nb.dcim.devices.get(serial=self.get_service_tag()) + else: + return nb.dcim.devices.get(serial=self.get_expansion_service_tag()) def _netbox_set_or_update_blade_slot(self, server, chassis, datacenter): # before everything check if right chassis @@ -283,9 +310,9 @@ class ServerBase(): slot=slot )) - def _netbox_set_or_update_blade_expansion_slot(self, server, chassis, datacenter): + def _netbox_set_or_update_blade_expansion_slot(self, expansion, chassis, datacenter): # before everything check if right chassis - actual_device_bay = server.parent_device.device_bay if server.parent_device else None + actual_device_bay = expansion.parent_device.device_bay if expansion.parent_device else None actual_chassis = actual_device_bay.device if actual_device_bay else None slot = self.get_blade_expansion_slot() if actual_chassis and \ @@ -293,30 +320,28 @@ class ServerBase(): actual_device_bay.name == slot: return - server.name += " expansion" - real_device_bays = nb.dcim.device_bays.filter( device_id=chassis.id, name=slot, ) - if len(real_device_bays) > 0: - logging.info( - 'Setting device expansion ({serial}) new slot on {slot} ' - '(Chassis {chassis_serial})..'.format( - serial=server.serial, slot=slot, chassis_serial=chassis.serial - )) - # reset actual device bay if set - if actual_device_bay: - actual_device_bay.installed_device = None - actual_device_bay.save() - # setup new device bay - real_device_bay = real_device_bays[0] - real_device_bay.installed_device = server - real_device_bay.save() - else: + if len(real_device_bays) == 0: logging.error('Could not find slot {slot} expansion for chassis'.format( slot=slot )) + return + logging.info( + 'Setting device expansion ({serial}) new slot on {slot} ' + '(Chassis {chassis_serial})..'.format( + serial=expansion.serial, slot=slot, chassis_serial=chassis.serial + )) + # reset actual device bay if set + if actual_device_bay: + actual_device_bay.installed_device = None + actual_device_bay.save() + # setup new device bay + real_device_bay = real_device_bays[0] + real_device_bay.installed_device = expansion + real_device_bay.save() def netbox_create_or_update(self, config): """ @@ -334,6 +359,9 @@ class ServerBase(): rack = self.get_netbox_rack() tenant = self.get_netbox_tenant() + if config.purge_old_devices: + self._netbox_deduplicate_server() + if self.is_blade(): chassis = nb.dcim.devices.get( serial=self.get_chassis_service_tag() @@ -358,9 +386,10 @@ class ServerBase(): if config.register or config.update_all or config.update_network: self.network = ServerNetwork(server=self) self.network.create_or_update_netbox_network_cards() + update_inventory = config.inventory and (config.register or + config.update_all or config.update_inventory) # update inventory if feature is enabled - if config.inventory and (config.register or config.update_all or config.update_inventory): - self.inventory = Inventory(server=self) + if update_inventory: self.inventory.create_or_update() # update psu if config.register or config.update_all or config.update_psu: @@ -368,14 +397,21 @@ class ServerBase(): self.power.create_or_update_power_supply() self.power.report_power_consumption() - if self.own_expansion_slot(): + expansion = nb.dcim.devices.get(serial=self.get_expansion_service_tag()) + if self.own_expansion_slot() and config.expansion_as_device: logging.debug('Update Server expansion...') - expansion = nb.dcim.devices.get(serial=self.get_expansion_service_tag()) if not expansion: expansion = self._netbox_create_blade_expansion(chassis, datacenter, tenant, rack) # set slot for blade expansion self._netbox_set_or_update_blade_expansion_slot(expansion, chassis, datacenter) + if update_inventory: + # Updates expansion inventory + inventory = Inventory(server=self, update_expansion=True) + inventory.create_or_update() + elif self.own_expansion_slot() and expansion: + expansion.delete() + expansion = None update = 0 # for every other specs @@ -394,6 +430,17 @@ class ServerBase(): if update: server.save() + + if expansion: + update = 0 + expansion_name = server.name + ' expansion' + if expansion.name != expansion_name: + expansion.name = expansion_name + update += 1 + if self.update_netbox_expansion_location(server, expansion): + update += 1 + if update: + expansion.save() logging.debug('Finished updating Server!') def print_debug(self): diff --git a/netbox_agent/vendors/hp.py b/netbox_agent/vendors/hp.py index ef479cf..5545972 100644 --- a/netbox_agent/vendors/hp.py +++ b/netbox_agent/vendors/hp.py @@ -67,35 +67,49 @@ class HPHost(ServerBase): return self.hp_rack_locator["Enclosure Serial"].strip() return self.get_service_tag() + def get_blade_expansion_slot(self): + """ + Expansion slot are always the compute bay number + 1 + """ + if self.is_blade() and self.own_gpu_expansion_slot() or \ + self.own_disk_expansion_slot() or True: + return 'Bay {}'.format( + str(int(self.hp_rack_locator['Server Bay'].strip()) + 1) + ) + return None + def get_expansion_product(self): """ Get the extension slot that is on a pair slot number next to the compute slot that is on an odd slot number I only know on model of slot GPU extension card that. """ - if self.own_expansion_slot(): + if self.own_gpu_expansion_slot(): return "ProLiant BL460c Graphics Expansion Blade" - return None - - def is_expansion_slot(self, server): - """ - Return True if its an extension slot, based on the name - """ - return server.name.endswith(" expansion") - - def get_blade_expansion_slot(self): - """ - Expansion slot are always the compute bay number + 1 - """ - if self.is_blade() and self.own_expansion_slot(): - return 'Bay {}'.format( - str(int(self.hp_rack_locator['Server Bay'].strip()) + 1) - ) + elif self.own_disk_expansion_slot(): + return "ProLiant BL460c Disk Expansion Blade" return None def own_expansion_slot(self): + """ + Say if the device can host an extension card based + on the product name + """ + return self.own_gpu_expansion_slot() or self.own_disk_expansion_slot() + + def own_gpu_expansion_slot(self): """ Say if the device can host an extension card based on the product name """ return self.get_product_name().endswith('Graphics Exp') + + def own_disk_expansion_slot(self): + """ + Say if the device can host an extension card based + on the product name + """ + for raid_card in self.inventory.get_raid_cards(): + if self.is_blade() and raid_card.is_external(): + return True + return False