Various changes

- Added an option to specify an SSL CA certificates file to talk to the
  Netbox API
- Enhanced GPU expansion bays inventory: internal GPU (VGA) goes into
  the blade, external GPU (3D) goes into the expansion bay
- Unified the way expansion bays are managed (GPU and drive exansion
  bays)
- Started to refactor `network` module to make it more readable
- Dependencies in `setup.py` now reads its requirements from
  `requirements.txt` to avoid double maintenance

netbox_agent/cli.py

@@ -1,5 +1,4 @@
 from packaging import version
-
 import netbox_agent.dmidecode as dmidecode
 from netbox_agent.config import config
 from netbox_agent.config import netbox_instance as nb
@@ -39,11 +38,11 @@ def run(config):
         print('netbox-agent is not compatible with Netbox prior to verison 2.9')
         return False
 
+    if config.register or config.update_all or config.update_network or \
+       config.update_location or config.update_inventory or config.update_psu:
+        server.netbox_create_or_update(config)
     if config.debug:
         server.print_debug()
-    if config.register or config.update_all or config.update_network or config.update_location or \
-       config.update_inventory or config.update_psu:
-        server.netbox_create_or_update(config)
     return True
 
 

netbox_agent/config.py

@@ -34,6 +34,7 @@ def get_config():
                    help='Manage blade expansions as external devices')
 
     p.add_argument('--log_level', default='debug')
+    p.add_argument('--netbox.ssl_ca_certs_file', help='SSL CA certificates file')
     p.add_argument('--netbox.url', help='Netbox URL')
     p.add_argument('--netbox.token', help='Netbox API Token')
     p.add_argument('--netbox.ssl_verify', default=True, action='store_true',
@@ -80,8 +81,10 @@ def get_config():
     return options
 
 
+config = get_config()
+
+
 def get_netbox_instance():
-    config = get_config()
     if config.netbox.url is None or config.netbox.token is None:
         logging.error('Netbox URL and token are mandatory')
         sys.exit(1)
@@ -90,7 +93,12 @@ def get_netbox_instance():
         url=get_config().netbox.url,
         token=get_config().netbox.token,
     )
-    if get_config().netbox.ssl_verify is False:
+    ca_certs_file = config.netbox.ssl_ca_certs_file
+    if ca_certs_file is not None:
+        session = requests.Session()
+        session.verify = ca_certs_file
+        nb.http_session = session
+    elif config.netbox.ssl_verify is False:
         urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
         session = requests.Session()
         session.verify = False
@@ -99,5 +107,4 @@ def get_netbox_instance():
     return nb
 
 
-config = get_config()
 netbox_instance = get_netbox_instance()

netbox_agent/inventory.py

@@ -450,10 +450,11 @@ class Inventory():
             if memory.get('serial') not in [x.serial for x in nb_memories]:
                 self.create_netbox_memory(memory)
 
-    def create_netbox_gpus(self):
+    def create_netbox_gpus(self, gpus):
-        for gpu in self.lshw.get_hw_linux('gpu'):
+        for gpu in gpus:
             if 'product' in gpu and len(gpu['product']) > 50:
                 gpu['product'] = (gpu['product'][:48] + '..')
+
             manufacturer = self.find_or_create_manufacturer(gpu["vendor"])
             _ = nb.dcim.inventory_items.create(
                 device=self.device_id,
@@ -461,26 +462,44 @@ class Inventory():
                 discovered=True,
                 tags=[{'name': INVENTORY_TAG['gpu']['name']}],
                 name=gpu['product'],
-                description='GPU {}'.format(gpu['product']),
+                description=gpu['description'],
             )
 
             logging.info('Creating GPU model {}'.format(gpu['product']))
 
+    def is_external_gpu(self, gpu):
+        is_3d_gpu = gpu['description'].startswith('3D')
+        return self.server.is_blade() and \
+            self.server.own_gpu_expansion_slot() and is_3d_gpu
+
     def do_netbox_gpus(self):
-        gpus = self.lshw.get_hw_linux('gpu')
+        gpus = []
+        gpu_models = {}
+        for gpu in  self.lshw.get_hw_linux('gpu'):
+            # Filters GPU if an expansion bay is detected:
+            # The internal (VGA) GPU only goes into the blade inventory,
+            # the external (3D) GPU goes into the expansion blade.
+            if config.expansion_as_device and \
+                    self.update_expansion ^ self.is_external_gpu(gpu):
+                continue
+            gpus.append(gpu)
+            gpu_models.setdefault(gpu["product"], 0)
+            gpu_models[gpu["product"]] += 1
+
         nb_gpus = self.get_netbox_inventory(
             device_id=self.device_id,
             tag=INVENTORY_TAG['gpu']['slug'],
         )
-
+        nb_gpu_models = {}
-        if config.expansion_as_device and len(nb_gpus):
+        for gpu in nb_gpus:
+            nb_gpu_models.setdefault(str(gpu), 0)
+            nb_gpu_models[str(gpu)] += 1
+        up_to_date = set(gpu_models) == set(nb_gpu_models)
+        if not gpus or not up_to_date:
             for x in nb_gpus:
                 x.delete()
-        elif not len(nb_gpus) or \
+        if gpus and not up_to_date:
-           len(nb_gpus) and len(gpus) != len(nb_gpus):
+            self.create_netbox_gpus(gpus)
-            for x in nb_gpus:
-                x.delete()
-            self.create_netbox_gpus()
 
     def create_or_update(self):
         if config.inventory is None or config.update_inventory is None:

netbox_agent/network.py

@@ -148,19 +148,19 @@ class Network(object):
         if nic['mac'] is None:
             interface = self.nb_net.interfaces.get(
                 name=nic['name'],
-                **self.custom_arg_id,
+                **self.custom_arg_id
             )
         else:
             interface = self.nb_net.interfaces.get(
                 mac_address=nic['mac'],
                 name=nic['name'],
-                **self.custom_arg_id,
+                **self.custom_arg_id
             )
         return interface
 
     def get_netbox_network_cards(self):
         return self.nb_net.interfaces.filter(
-            **self.custom_arg_id,
+            **self.custom_arg_id
         )
 
     def get_netbox_type_for_nic(self, nic):
@@ -267,12 +267,12 @@ class Network(object):
 
         nb_vlan = None
 
-        params = {
+        params = dict(self.custom_arg)
+        params.update({
             'name': nic['name'],
             'type': type,
             'mgmt_only': mgmt,
-            **self.custom_arg,
+        })
-        }
 
         if not nic.get('virtual', False):
             params['mac_address'] = nic['mac']
@@ -338,59 +338,58 @@ class Network(object):
             netbox_ip = nb.ipam.ip_addresses.create(
                 **query_params
             )
-        else:
+            return netbox_ip
-            netbox_ip = list(netbox_ips)[0]
-            # If IP exists in anycast
-            if netbox_ip.role and netbox_ip.role.label == 'Anycast':
-                logging.debug('IP {} is Anycast..'.format(ip))
-                unassigned_anycast_ip = [x for x in netbox_ips if x.interface is None]
-                assigned_anycast_ip = [x for x in netbox_ips if
-                                       x.interface and x.interface.id == interface.id]
-                # use the first available anycast ip
-                if len(unassigned_anycast_ip):
-                    logging.info('Assigning existing Anycast IP {} to interface'.format(ip))
-                    netbox_ip = unassigned_anycast_ip[0]
-                    netbox_ip.interface = interface
-                    netbox_ip.save()
-                # or if everything is assigned to other servers
-                elif not len(assigned_anycast_ip):
-                    logging.info('Creating Anycast IP {} and assigning it to interface'.format(ip))
-                    query_params = {
-                        "address": ip,
-                        "status": "active",
-                        "role": self.ipam_choices['ip-address:role']['Anycast'],
-                        "tenant": self.tenant.id if self.tenant else None,
-                        "assigned_object_type": self.assigned_object_type,
-                        "assigned_object_id": interface.id
-                    }
-                    netbox_ip = nb.ipam.ip_addresses.create(**query_params)
-                return netbox_ip
-            else:
-                if hasattr(netbox_ip, 'interface') and netbox_ip.interface is None or \
-                   hasattr(netbox_ip, 'assigned_object') and netbox_ip.assigned_object is None:
-                    logging.info('Assigning existing IP {ip} to {interface}'.format(
-                        ip=ip, interface=interface))
-                elif hasattr(netbox_ip, 'interface') and \
-                    netbox_ip.interface.id != interface.id or \
-                    hasattr(netbox_ip, 'assigned_object') and \
-                    netbox_ip.assigned_object_id != interface.id:
 
-                    old_interface = netbox_ip.assigned_object
+        netbox_ip = next(netbox_ips)
-                    logging.info(
+        # If IP exists in anycast
-                        'Detected interface change for ip {ip}: old interface is '
+        if netbox_ip.role and netbox_ip.role.label == 'Anycast':
-                        '{old_interface} (id: {old_id}), new interface is {new_interface} '
+            logging.debug('IP {} is Anycast..'.format(ip))
-                        ' (id: {new_id})'
+            unassigned_anycast_ip = [x for x in netbox_ips if x.interface is None]
-                        .format(
+            assigned_anycast_ip = [x for x in netbox_ips if
-                            old_interface=old_interface, new_interface=interface,
+                                   x.interface and x.interface.id == interface.id]
-                            old_id=netbox_ip.id, new_id=interface.id, ip=netbox_ip.address
+            # use the first available anycast ip
-                        ))
+            if len(unassigned_anycast_ip):
-                else:
+                logging.info('Assigning existing Anycast IP {} to interface'.format(ip))
-                    return netbox_ip
+                netbox_ip = unassigned_anycast_ip[0]
-
+                netbox_ip.interface = interface
-                netbox_ip.assigned_object_type = self.assigned_object_type
-                netbox_ip.assigned_object_id = interface.id
                 netbox_ip.save()
-        return netbox_ip
+            # or if everything is assigned to other servers
+            elif not len(assigned_anycast_ip):
+                logging.info('Creating Anycast IP {} and assigning it to interface'.format(ip))
+                query_params = {
+                    "address": ip,
+                    "status": "active",
+                    "role": self.ipam_choices['ip-address:role']['Anycast'],
+                    "tenant": self.tenant.id if self.tenant else None,
+                    "assigned_object_type": self.assigned_object_type,
+                    "assigned_object_id": interface.id
+                }
+                netbox_ip = nb.ipam.ip_addresses.create(**query_params)
+            return netbox_ip
+        else:
+            ip_interface = getattr(netbox_ip, 'interface', None)
+            assigned_object = getattr(netbox_ip, 'assigned_object', None)
+            if not ip_interface or not assigned_object:
+                logging.info('Assigning existing IP {ip} to {interface}'.format(
+                    ip=ip, interface=interface))
+            elif (ip_interface and ip_interface.id != interface.id) or \
+                 (assigned_object and assigned_object_id != interface.id):
+
+                old_interface = getattr(netbox_ip, "assigned_object", "n/a")
+                logging.info(
+                    'Detected interface change for ip {ip}: old interface is '
+                    '{old_interface} (id: {old_id}), new interface is {new_interface} '
+                    ' (id: {new_id})'
+                    .format(
+                        old_interface=old_interface, new_interface=interface,
+                        old_id=netbox_ip.id, new_id=interface.id, ip=netbox_ip.address
+                    ))
+            else:
+                return netbox_ip
+
+            netbox_ip.assigned_object_type = self.assigned_object_type
+            netbox_ip.assigned_object_id = interface.id
+            netbox_ip.save()
 
     def create_or_update_netbox_network_cards(self):
         if config.update_all is None or config.update_network is None:

netbox_agent/server.py

@@ -25,7 +25,6 @@ class ServerBase():
         self.bios = dmidecode.get_by_type(self.dmi, 'BIOS')
         self.chassis = dmidecode.get_by_type(self.dmi, 'Chassis')
         self.system = dmidecode.get_by_type(self.dmi, 'System')
-        self.inventory = Inventory(server=self)
 
         self.network = None
 
@@ -279,8 +278,10 @@ class ServerBase():
 
     def _netbox_set_or_update_blade_slot(self, server, chassis, datacenter):
         # before everything check if right chassis
-        actual_device_bay = server.parent_device.device_bay if server.parent_device else None
+        actual_device_bay = server.parent_device.device_bay \
-        actual_chassis = actual_device_bay.device if actual_device_bay else None
+                if server.parent_device else None
+        actual_chassis = actual_device_bay.device \
+                if actual_device_bay else None
         slot = self.get_blade_slot()
         if actual_chassis and \
            actual_chassis.serial == chassis.serial and \
@@ -291,7 +292,11 @@ class ServerBase():
             device_id=chassis.id,
             name=slot,
         )
-        if len(real_device_bays) > 0:
+        real_device_bays = nb.dcim.device_bays.filter(
+            device_id=chassis.id,
+            name=slot,
+        )
+        if real_device_bays:
             logging.info(
                 'Setting device ({serial}) new slot on {slot} '
                 '(Chassis {chassis_serial})..'.format(
@@ -299,10 +304,14 @@ class ServerBase():
                 ))
             # reset actual device bay if set
             if actual_device_bay:
+                # Forces the evaluation of the installed_device attribute to
+                # workaround a bug probably due to lazy loading optimization
+                # that prevents the value change detection
+                actual_device_bay.installed_device
                 actual_device_bay.installed_device = None
                 actual_device_bay.save()
             # setup new device bay
-            real_device_bay = real_device_bays[0]
+            real_device_bay = next(real_device_bays)
             real_device_bay.installed_device = server
             real_device_bay.save()
         else:
@@ -324,7 +333,7 @@ class ServerBase():
             device_id=chassis.id,
             name=slot,
         )
-        if len(real_device_bays) == 0:
+        if not real_device_bays:
             logging.error('Could not find slot {slot} expansion for chassis'.format(
                 slot=slot
             ))
@@ -336,10 +345,14 @@ class ServerBase():
             ))
         # reset actual device bay if set
         if actual_device_bay:
+            # Forces the evaluation of the installed_device attribute to
+            # workaround a bug probably due to lazy loading optimization
+            # that prevents the value change detection
+            actual_device_bay.installed_device
             actual_device_bay.installed_device = None
             actual_device_bay.save()
         # setup new device bay
-        real_device_bay = real_device_bays[0]
+        real_device_bay = next(real_device_bays)
         real_device_bay.installed_device = expansion
         real_device_bay.save()
 
@@ -389,6 +402,7 @@ class ServerBase():
         update_inventory = config.inventory and (config.register or
                 config.update_all or config.update_inventory)
         # update inventory if feature is enabled
+        self.inventory = Inventory(server=self)
         if update_inventory:
             self.inventory.create_or_update()
         # update psu
@@ -417,12 +431,12 @@ class ServerBase():
         # for every other specs
         # check hostname
         if server.name != self.get_hostname():
-            update += 1
             server.name = self.get_hostname()
+            update += 1
 
         if sorted(set([x.name for x in server.tags])) != sorted(set(self.tags)):
-            update += 1
             server.tags = [x.id for x in self.nb_tags]
+            update += 1
 
         if config.update_all or config.update_location:
             ret, server = self.update_netbox_location(server)
@@ -458,3 +472,9 @@ class ServerBase():
         print('NIC:',)
         pprint(self.network.get_network_cards())
         pass
+
+    def own_expansion_slot(self):
+        """
+        Indicates if the device hosts an expansion card
+        """
+        return False

netbox_agent/vendors/dell.py

@@ -86,10 +86,3 @@ class DellHost(ServerBase):
         Expansion slot are always the compute bay number + 1
         """
         raise NotImplementedError
-
-    def own_expansion_slot(self):
-        """
-        Say if the device can host an extension card based
-        on the product name
-        """
-        pass

netbox_agent/vendors/generic.py

@@ -21,29 +21,3 @@ class GenericHost(ServerBase):
 
     def get_chassis_service_tag(self):
         return self.get_service_tag()
-
-    def get_expansion_product(self):
-        """
-        Get the extension slot that is on a pair slot number
-        next to the compute slot that is on an odd slot number
-        """
-        raise NotImplementedError
-
-    def is_expansion_slot(self, server):
-        """
-        Return True if its an extension slot
-        """
-        raise NotImplementedError
-
-    def get_blade_expansion_slot(self):
-        """
-        Expansion slot are always the compute bay number + 1
-        """
-        raise NotImplementedError
-
-    def own_expansion_slot(self):
-        """
-        Say if the device can host an extension card based
-        on the product name
-        """
-        pass

netbox_agent/vendors/hp.py

@@ -1,5 +1,6 @@
 import netbox_agent.dmidecode as dmidecode
 from netbox_agent.server import ServerBase
+from netbox_agent.inventory import Inventory
 
 
 class HPHost(ServerBase):
@@ -92,24 +93,28 @@ class HPHost(ServerBase):
 
     def own_expansion_slot(self):
         """
-        Say if the device can host an extension card based
+        Indicates if the device hosts an expension card
-        on the product name
         """
         return self.own_gpu_expansion_slot() or self.own_disk_expansion_slot()
 
     def own_gpu_expansion_slot(self):
         """
-        Say if the device can host an extension card based
+        Indicates if the device hosts a GPU expansion card based
         on the product name
         """
         return self.get_product_name().endswith('Graphics Exp')
 
     def own_disk_expansion_slot(self):
         """
-        Say if the device can host an extension card based
+        Indicates if the device hosts a drive expansion card based
-        on the product name
+        on raid card attributes.
         """
-        for raid_card in self.inventory.get_raid_cards():
+        # Uses already parsed inventory if available
+        # parses it otherwise
+        inventory = getattr(self, "inventory", None)
+        if inventory is None:
+            inventory = Inventory(self)
+        for raid_card in inventory.get_raid_cards():
             if self.is_blade() and raid_card.is_external():
                 return True
         return False

netbox_agent/vendors/supermicro.py

@@ -77,22 +77,3 @@ class SupermicroHost(ServerBase):
         I only know on model of slot GPU extension card that.
         """
         raise NotImplementedError
-
-    def is_expansion_slot(self, server):
-        """
-        Return True if its an extension slot, based on the name
-        """
-        raise NotImplementedError
-
-    def get_blade_expansion_slot(self):
-        """
-        Expansion slot are always the compute bay number + 1
-        """
-        raise NotImplementedError
-
-    def own_expansion_slot(self):
-        """
-        Say if the device can host an extension card based
-        on the product name
-        """
-        pass

setup.py

@@ -1,8 +1,22 @@
 from setuptools import find_packages, setup
+import os
+
+def get_requirements():
+    reqs_path = os.path.join(
+        os.path.dirname(__file__),
+        'requirements.txt'
+    )
+    with open(reqs_path, 'r') as f:
+        reqs = [
+            r.strip() for r in f
+            if r.strip()
+        ]
+    return reqs
+
 
 setup(
     name='netbox_agent',
-    version='0.6.2',
+    version='0.6.3',
     description='NetBox agent for server',
     long_description=open('README.md', encoding="utf-8").read(),
     long_description_content_type='text/markdown',
@@ -13,15 +27,7 @@ setup(
     include_package_data=True,
     packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]),
     use_scm_version=True,
-    install_requires=[
+    install_requires=get_requirements(),
-        'pynetbox==6.1.2',
-        'netaddr==0.8.0',
-        'netifaces==0.10.9',
-        'pyyaml==5.4.1',
-        'jsonargparse==2.32.2',
-        'python-slugify==5.0.2',
-        'packaging==20.9',
-    ],
     zip_safe=False,
     keywords=['netbox'],
     classifiers=[