From 305d4d41ec13db32da113973074207c568456e70 Mon Sep 17 00:00:00 2001 From: Christophe Simon Date: Mon, 21 Feb 2022 18:12:16 +0100 Subject: [PATCH] Various changes - Added an option to specify an SSL CA certificates file to talk to the Netbox API - Enhanced GPU expansion bays inventory: internal GPU (VGA) goes into the blade, external GPU (3D) goes into the expansion bay - Unified the way expansion bays are managed (GPU and drive exansion bays) - Started to refactor `network` module to make it more readable - Dependencies in `setup.py` now reads its requirements from `requirements.txt` to avoid double maintenance --- netbox_agent/cli.py | 7 +- netbox_agent/config.py | 13 +++- netbox_agent/inventory.py | 41 ++++++++--- netbox_agent/network.py | 113 ++++++++++++++--------------- netbox_agent/server.py | 38 +++++++--- netbox_agent/vendors/dell.py | 7 -- netbox_agent/vendors/generic.py | 26 ------- netbox_agent/vendors/hp.py | 17 +++-- netbox_agent/vendors/supermicro.py | 19 ----- setup.py | 26 ++++--- 10 files changed, 155 insertions(+), 152 deletions(-) diff --git a/netbox_agent/cli.py b/netbox_agent/cli.py index a379ae2..e112469 100644 --- a/netbox_agent/cli.py +++ b/netbox_agent/cli.py @@ -1,5 +1,4 @@ from packaging import version - import netbox_agent.dmidecode as dmidecode from netbox_agent.config import config from netbox_agent.config import netbox_instance as nb @@ -39,11 +38,11 @@ def run(config): print('netbox-agent is not compatible with Netbox prior to verison 2.9') return False + if config.register or config.update_all or config.update_network or \ + config.update_location or config.update_inventory or config.update_psu: + server.netbox_create_or_update(config) if config.debug: server.print_debug() - if config.register or config.update_all or config.update_network or config.update_location or \ - config.update_inventory or config.update_psu: - server.netbox_create_or_update(config) return True diff --git a/netbox_agent/config.py b/netbox_agent/config.py index 68d7d82..6913f65 100644 --- a/netbox_agent/config.py +++ b/netbox_agent/config.py @@ -34,6 +34,7 @@ def get_config(): help='Manage blade expansions as external devices') p.add_argument('--log_level', default='debug') + p.add_argument('--netbox.ssl_ca_certs_file', help='SSL CA certificates file') p.add_argument('--netbox.url', help='Netbox URL') p.add_argument('--netbox.token', help='Netbox API Token') p.add_argument('--netbox.ssl_verify', default=True, action='store_true', @@ -80,8 +81,10 @@ def get_config(): return options +config = get_config() + + def get_netbox_instance(): - config = get_config() if config.netbox.url is None or config.netbox.token is None: logging.error('Netbox URL and token are mandatory') sys.exit(1) @@ -90,7 +93,12 @@ def get_netbox_instance(): url=get_config().netbox.url, token=get_config().netbox.token, ) - if get_config().netbox.ssl_verify is False: + ca_certs_file = config.netbox.ssl_ca_certs_file + if ca_certs_file is not None: + session = requests.Session() + session.verify = ca_certs_file + nb.http_session = session + elif config.netbox.ssl_verify is False: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) session = requests.Session() session.verify = False @@ -99,5 +107,4 @@ def get_netbox_instance(): return nb -config = get_config() netbox_instance = get_netbox_instance() diff --git a/netbox_agent/inventory.py b/netbox_agent/inventory.py index 6525bea..6f7fab6 100644 --- a/netbox_agent/inventory.py +++ b/netbox_agent/inventory.py @@ -450,10 +450,11 @@ class Inventory(): if memory.get('serial') not in [x.serial for x in nb_memories]: self.create_netbox_memory(memory) - def create_netbox_gpus(self): - for gpu in self.lshw.get_hw_linux('gpu'): + def create_netbox_gpus(self, gpus): + for gpu in gpus: if 'product' in gpu and len(gpu['product']) > 50: gpu['product'] = (gpu['product'][:48] + '..') + manufacturer = self.find_or_create_manufacturer(gpu["vendor"]) _ = nb.dcim.inventory_items.create( device=self.device_id, @@ -461,26 +462,44 @@ class Inventory(): discovered=True, tags=[{'name': INVENTORY_TAG['gpu']['name']}], name=gpu['product'], - description='GPU {}'.format(gpu['product']), + description=gpu['description'], ) logging.info('Creating GPU model {}'.format(gpu['product'])) + def is_external_gpu(self, gpu): + is_3d_gpu = gpu['description'].startswith('3D') + return self.server.is_blade() and \ + self.server.own_gpu_expansion_slot() and is_3d_gpu + def do_netbox_gpus(self): - gpus = self.lshw.get_hw_linux('gpu') + gpus = [] + gpu_models = {} + for gpu in self.lshw.get_hw_linux('gpu'): + # Filters GPU if an expansion bay is detected: + # The internal (VGA) GPU only goes into the blade inventory, + # the external (3D) GPU goes into the expansion blade. + if config.expansion_as_device and \ + self.update_expansion ^ self.is_external_gpu(gpu): + continue + gpus.append(gpu) + gpu_models.setdefault(gpu["product"], 0) + gpu_models[gpu["product"]] += 1 + nb_gpus = self.get_netbox_inventory( device_id=self.device_id, tag=INVENTORY_TAG['gpu']['slug'], ) - - if config.expansion_as_device and len(nb_gpus): + nb_gpu_models = {} + for gpu in nb_gpus: + nb_gpu_models.setdefault(str(gpu), 0) + nb_gpu_models[str(gpu)] += 1 + up_to_date = set(gpu_models) == set(nb_gpu_models) + if not gpus or not up_to_date: for x in nb_gpus: x.delete() - elif not len(nb_gpus) or \ - len(nb_gpus) and len(gpus) != len(nb_gpus): - for x in nb_gpus: - x.delete() - self.create_netbox_gpus() + if gpus and not up_to_date: + self.create_netbox_gpus(gpus) def create_or_update(self): if config.inventory is None or config.update_inventory is None: diff --git a/netbox_agent/network.py b/netbox_agent/network.py index 3a390eb..cb5437e 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -148,19 +148,19 @@ class Network(object): if nic['mac'] is None: interface = self.nb_net.interfaces.get( name=nic['name'], - **self.custom_arg_id, + **self.custom_arg_id ) else: interface = self.nb_net.interfaces.get( mac_address=nic['mac'], name=nic['name'], - **self.custom_arg_id, + **self.custom_arg_id ) return interface def get_netbox_network_cards(self): return self.nb_net.interfaces.filter( - **self.custom_arg_id, + **self.custom_arg_id ) def get_netbox_type_for_nic(self, nic): @@ -267,12 +267,12 @@ class Network(object): nb_vlan = None - params = { + params = dict(self.custom_arg) + params.update({ 'name': nic['name'], 'type': type, 'mgmt_only': mgmt, - **self.custom_arg, - } + }) if not nic.get('virtual', False): params['mac_address'] = nic['mac'] @@ -338,59 +338,58 @@ class Network(object): netbox_ip = nb.ipam.ip_addresses.create( **query_params ) - else: - netbox_ip = list(netbox_ips)[0] - # If IP exists in anycast - if netbox_ip.role and netbox_ip.role.label == 'Anycast': - logging.debug('IP {} is Anycast..'.format(ip)) - unassigned_anycast_ip = [x for x in netbox_ips if x.interface is None] - assigned_anycast_ip = [x for x in netbox_ips if - x.interface and x.interface.id == interface.id] - # use the first available anycast ip - if len(unassigned_anycast_ip): - logging.info('Assigning existing Anycast IP {} to interface'.format(ip)) - netbox_ip = unassigned_anycast_ip[0] - netbox_ip.interface = interface - netbox_ip.save() - # or if everything is assigned to other servers - elif not len(assigned_anycast_ip): - logging.info('Creating Anycast IP {} and assigning it to interface'.format(ip)) - query_params = { - "address": ip, - "status": "active", - "role": self.ipam_choices['ip-address:role']['Anycast'], - "tenant": self.tenant.id if self.tenant else None, - "assigned_object_type": self.assigned_object_type, - "assigned_object_id": interface.id - } - netbox_ip = nb.ipam.ip_addresses.create(**query_params) - return netbox_ip - else: - if hasattr(netbox_ip, 'interface') and netbox_ip.interface is None or \ - hasattr(netbox_ip, 'assigned_object') and netbox_ip.assigned_object is None: - logging.info('Assigning existing IP {ip} to {interface}'.format( - ip=ip, interface=interface)) - elif hasattr(netbox_ip, 'interface') and \ - netbox_ip.interface.id != interface.id or \ - hasattr(netbox_ip, 'assigned_object') and \ - netbox_ip.assigned_object_id != interface.id: + return netbox_ip - old_interface = netbox_ip.assigned_object - logging.info( - 'Detected interface change for ip {ip}: old interface is ' - '{old_interface} (id: {old_id}), new interface is {new_interface} ' - ' (id: {new_id})' - .format( - old_interface=old_interface, new_interface=interface, - old_id=netbox_ip.id, new_id=interface.id, ip=netbox_ip.address - )) - else: - return netbox_ip - - netbox_ip.assigned_object_type = self.assigned_object_type - netbox_ip.assigned_object_id = interface.id + netbox_ip = next(netbox_ips) + # If IP exists in anycast + if netbox_ip.role and netbox_ip.role.label == 'Anycast': + logging.debug('IP {} is Anycast..'.format(ip)) + unassigned_anycast_ip = [x for x in netbox_ips if x.interface is None] + assigned_anycast_ip = [x for x in netbox_ips if + x.interface and x.interface.id == interface.id] + # use the first available anycast ip + if len(unassigned_anycast_ip): + logging.info('Assigning existing Anycast IP {} to interface'.format(ip)) + netbox_ip = unassigned_anycast_ip[0] + netbox_ip.interface = interface netbox_ip.save() - return netbox_ip + # or if everything is assigned to other servers + elif not len(assigned_anycast_ip): + logging.info('Creating Anycast IP {} and assigning it to interface'.format(ip)) + query_params = { + "address": ip, + "status": "active", + "role": self.ipam_choices['ip-address:role']['Anycast'], + "tenant": self.tenant.id if self.tenant else None, + "assigned_object_type": self.assigned_object_type, + "assigned_object_id": interface.id + } + netbox_ip = nb.ipam.ip_addresses.create(**query_params) + return netbox_ip + else: + ip_interface = getattr(netbox_ip, 'interface', None) + assigned_object = getattr(netbox_ip, 'assigned_object', None) + if not ip_interface or not assigned_object: + logging.info('Assigning existing IP {ip} to {interface}'.format( + ip=ip, interface=interface)) + elif (ip_interface and ip_interface.id != interface.id) or \ + (assigned_object and assigned_object_id != interface.id): + + old_interface = getattr(netbox_ip, "assigned_object", "n/a") + logging.info( + 'Detected interface change for ip {ip}: old interface is ' + '{old_interface} (id: {old_id}), new interface is {new_interface} ' + ' (id: {new_id})' + .format( + old_interface=old_interface, new_interface=interface, + old_id=netbox_ip.id, new_id=interface.id, ip=netbox_ip.address + )) + else: + return netbox_ip + + netbox_ip.assigned_object_type = self.assigned_object_type + netbox_ip.assigned_object_id = interface.id + netbox_ip.save() def create_or_update_netbox_network_cards(self): if config.update_all is None or config.update_network is None: diff --git a/netbox_agent/server.py b/netbox_agent/server.py index a20023b..24758ae 100644 --- a/netbox_agent/server.py +++ b/netbox_agent/server.py @@ -25,7 +25,6 @@ class ServerBase(): self.bios = dmidecode.get_by_type(self.dmi, 'BIOS') self.chassis = dmidecode.get_by_type(self.dmi, 'Chassis') self.system = dmidecode.get_by_type(self.dmi, 'System') - self.inventory = Inventory(server=self) self.network = None @@ -279,8 +278,10 @@ class ServerBase(): def _netbox_set_or_update_blade_slot(self, server, chassis, datacenter): # before everything check if right chassis - actual_device_bay = server.parent_device.device_bay if server.parent_device else None - actual_chassis = actual_device_bay.device if actual_device_bay else None + actual_device_bay = server.parent_device.device_bay \ + if server.parent_device else None + actual_chassis = actual_device_bay.device \ + if actual_device_bay else None slot = self.get_blade_slot() if actual_chassis and \ actual_chassis.serial == chassis.serial and \ @@ -291,7 +292,11 @@ class ServerBase(): device_id=chassis.id, name=slot, ) - if len(real_device_bays) > 0: + real_device_bays = nb.dcim.device_bays.filter( + device_id=chassis.id, + name=slot, + ) + if real_device_bays: logging.info( 'Setting device ({serial}) new slot on {slot} ' '(Chassis {chassis_serial})..'.format( @@ -299,10 +304,14 @@ class ServerBase(): )) # reset actual device bay if set if actual_device_bay: + # Forces the evaluation of the installed_device attribute to + # workaround a bug probably due to lazy loading optimization + # that prevents the value change detection + actual_device_bay.installed_device actual_device_bay.installed_device = None actual_device_bay.save() # setup new device bay - real_device_bay = real_device_bays[0] + real_device_bay = next(real_device_bays) real_device_bay.installed_device = server real_device_bay.save() else: @@ -324,7 +333,7 @@ class ServerBase(): device_id=chassis.id, name=slot, ) - if len(real_device_bays) == 0: + if not real_device_bays: logging.error('Could not find slot {slot} expansion for chassis'.format( slot=slot )) @@ -336,10 +345,14 @@ class ServerBase(): )) # reset actual device bay if set if actual_device_bay: + # Forces the evaluation of the installed_device attribute to + # workaround a bug probably due to lazy loading optimization + # that prevents the value change detection + actual_device_bay.installed_device actual_device_bay.installed_device = None actual_device_bay.save() # setup new device bay - real_device_bay = real_device_bays[0] + real_device_bay = next(real_device_bays) real_device_bay.installed_device = expansion real_device_bay.save() @@ -389,6 +402,7 @@ class ServerBase(): update_inventory = config.inventory and (config.register or config.update_all or config.update_inventory) # update inventory if feature is enabled + self.inventory = Inventory(server=self) if update_inventory: self.inventory.create_or_update() # update psu @@ -417,12 +431,12 @@ class ServerBase(): # for every other specs # check hostname if server.name != self.get_hostname(): - update += 1 server.name = self.get_hostname() + update += 1 if sorted(set([x.name for x in server.tags])) != sorted(set(self.tags)): - update += 1 server.tags = [x.id for x in self.nb_tags] + update += 1 if config.update_all or config.update_location: ret, server = self.update_netbox_location(server) @@ -458,3 +472,9 @@ class ServerBase(): print('NIC:',) pprint(self.network.get_network_cards()) pass + + def own_expansion_slot(self): + """ + Indicates if the device hosts an expansion card + """ + return False diff --git a/netbox_agent/vendors/dell.py b/netbox_agent/vendors/dell.py index b9a3212..dddb67a 100644 --- a/netbox_agent/vendors/dell.py +++ b/netbox_agent/vendors/dell.py @@ -86,10 +86,3 @@ class DellHost(ServerBase): Expansion slot are always the compute bay number + 1 """ raise NotImplementedError - - def own_expansion_slot(self): - """ - Say if the device can host an extension card based - on the product name - """ - pass diff --git a/netbox_agent/vendors/generic.py b/netbox_agent/vendors/generic.py index eddd8fa..6080112 100644 --- a/netbox_agent/vendors/generic.py +++ b/netbox_agent/vendors/generic.py @@ -21,29 +21,3 @@ class GenericHost(ServerBase): def get_chassis_service_tag(self): return self.get_service_tag() - - def get_expansion_product(self): - """ - Get the extension slot that is on a pair slot number - next to the compute slot that is on an odd slot number - """ - raise NotImplementedError - - def is_expansion_slot(self, server): - """ - Return True if its an extension slot - """ - raise NotImplementedError - - def get_blade_expansion_slot(self): - """ - Expansion slot are always the compute bay number + 1 - """ - raise NotImplementedError - - def own_expansion_slot(self): - """ - Say if the device can host an extension card based - on the product name - """ - pass diff --git a/netbox_agent/vendors/hp.py b/netbox_agent/vendors/hp.py index 5545972..78b3092 100644 --- a/netbox_agent/vendors/hp.py +++ b/netbox_agent/vendors/hp.py @@ -1,5 +1,6 @@ import netbox_agent.dmidecode as dmidecode from netbox_agent.server import ServerBase +from netbox_agent.inventory import Inventory class HPHost(ServerBase): @@ -92,24 +93,28 @@ class HPHost(ServerBase): def own_expansion_slot(self): """ - Say if the device can host an extension card based - on the product name + Indicates if the device hosts an expension card """ return self.own_gpu_expansion_slot() or self.own_disk_expansion_slot() def own_gpu_expansion_slot(self): """ - Say if the device can host an extension card based + Indicates if the device hosts a GPU expansion card based on the product name """ return self.get_product_name().endswith('Graphics Exp') def own_disk_expansion_slot(self): """ - Say if the device can host an extension card based - on the product name + Indicates if the device hosts a drive expansion card based + on raid card attributes. """ - for raid_card in self.inventory.get_raid_cards(): + # Uses already parsed inventory if available + # parses it otherwise + inventory = getattr(self, "inventory", None) + if inventory is None: + inventory = Inventory(self) + for raid_card in inventory.get_raid_cards(): if self.is_blade() and raid_card.is_external(): return True return False diff --git a/netbox_agent/vendors/supermicro.py b/netbox_agent/vendors/supermicro.py index 32e8fdf..d27b343 100644 --- a/netbox_agent/vendors/supermicro.py +++ b/netbox_agent/vendors/supermicro.py @@ -77,22 +77,3 @@ class SupermicroHost(ServerBase): I only know on model of slot GPU extension card that. """ raise NotImplementedError - - def is_expansion_slot(self, server): - """ - Return True if its an extension slot, based on the name - """ - raise NotImplementedError - - def get_blade_expansion_slot(self): - """ - Expansion slot are always the compute bay number + 1 - """ - raise NotImplementedError - - def own_expansion_slot(self): - """ - Say if the device can host an extension card based - on the product name - """ - pass diff --git a/setup.py b/setup.py index 1ecdeb2..ad2ff7f 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,22 @@ from setuptools import find_packages, setup +import os + +def get_requirements(): + reqs_path = os.path.join( + os.path.dirname(__file__), + 'requirements.txt' + ) + with open(reqs_path, 'r') as f: + reqs = [ + r.strip() for r in f + if r.strip() + ] + return reqs + setup( name='netbox_agent', - version='0.6.2', + version='0.6.3', description='NetBox agent for server', long_description=open('README.md', encoding="utf-8").read(), long_description_content_type='text/markdown', @@ -13,15 +27,7 @@ setup( include_package_data=True, packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), use_scm_version=True, - install_requires=[ - 'pynetbox==6.1.2', - 'netaddr==0.8.0', - 'netifaces==0.10.9', - 'pyyaml==5.4.1', - 'jsonargparse==2.32.2', - 'python-slugify==5.0.2', - 'packaging==20.9', - ], + install_requires=get_requirements(), zip_safe=False, keywords=['netbox'], classifiers=[