Merge branch 'master' into fix/chriss/purge_remaining_devices

netbox_agent/config.py

@@ -30,6 +30,8 @@ def get_config():
     p.add_argument('--update-psu', action='store_true', help='Update PSU')
     p.add_argument('--purge-old-devices', action='store_true',
                    help='Purge existing (old ?) devices having same name but different serial')
+    p.add_argument('--expansion-as-device', action='store_true',
+                   help='Manage blade expansions as external devices')
 
     p.add_argument('--log_level', default='debug')
     p.add_argument('--netbox.url', help='Netbox URL')

netbox_agent/inventory.py

@@ -46,10 +46,11 @@ class Inventory():
     - no scan of NVMe devices
     """
 
-    def __init__(self, server):
+    def __init__(self, server, update_expansion=False):
         self.create_netbox_tags()
         self.server = server
-        netbox_server = self.server.get_netbox_server()
+        self.update_expansion = update_expansion
+        netbox_server = self.server.get_netbox_server(update_expansion)
 
         self.device_id = netbox_server.id if netbox_server else None
         self.raid = None
@@ -220,7 +221,7 @@ class Inventory():
 
             self.create_netbox_cpus()
 
-    def get_raid_cards(self):
+    def get_raid_cards(self, filter_cards=False):
         raid_class = None
         if self.server.manufacturer == 'Dell':
             if is_tool('omreport'):
@@ -235,9 +236,15 @@ class Inventory():
             return []
 
         self.raid = raid_class()
-        controllers = self.raid.get_controllers()
+
-        if len(self.raid.get_controllers()):
+        if filter_cards and config.expansion_as_device \
-            return controllers
+                and self.server.own_expansion_slot():
+            return [
+                c for c in self.raid.get_controllers()
+                if c.is_external() is self.update_expansion
+            ]
+        else:
+            return self.raid.get_controllers()
 
     def create_netbox_raid_card(self, raid_card):
         manufacturer = self.find_or_create_manufacturer(
@@ -276,7 +283,7 @@ class Inventory():
             device_id=self.device_id,
             tag=[INVENTORY_TAG['raid_card']['slug']]
         )
-        raid_cards = self.get_raid_cards()
+        raid_cards = self.get_raid_cards(filter_cards=True)
 
         # delete cards that are in netbox but not locally
         # use the serial_number has the comparison element
@@ -336,7 +343,7 @@ class Inventory():
                 d['Vendor'] = get_vendor(disk['product'])
             disks.append(d)
 
-        for raid_card in self.get_raid_cards():
+        for raid_card in self.get_raid_cards(filter_cards=True):
             disks += raid_card.get_physical_disks()
 
         # remove duplicate serials
@@ -463,21 +470,24 @@ class Inventory():
             tag=INVENTORY_TAG['gpu']['slug'],
         )
 
-        if not len(nb_gpus) or \
+        if config.expansion_as_device and len(nb_gpus):
+            for x in nb_gpus:
+                x.delete()
+        elif not len(nb_gpus) or \
            len(nb_gpus) and len(gpus) != len(nb_gpus):
             for x in nb_gpus:
                 x.delete()
-
             self.create_netbox_gpus()
 
     def create_or_update(self):
         if config.inventory is None or config.update_inventory is None:
             return False
-        self.do_netbox_cpus()
+        if self.update_expansion is False:
-        self.do_netbox_memories()
+            self.do_netbox_cpus()
-        self.do_netbox_raid_cards()
+            self.do_netbox_memories()
-        self.do_netbox_disks()
+            self.do_netbox_interfaces()
-        self.do_netbox_interfaces()
+            self.do_netbox_motherboard()
-        self.do_netbox_motherboard()
         self.do_netbox_gpus()
+        self.do_netbox_disks()
+        self.do_netbox_raid_cards()
         return True

netbox_agent/raid/base.py

@@ -15,6 +15,9 @@ class RaidController():
     def get_physical_disks(self):
         raise NotImplementedError
 
+    def is_external(self):
+        return False
+
 
 class Raid():
     def get_controllers(self):

netbox_agent/raid/hp.py

@@ -1,106 +1,65 @@
 import re
 import subprocess
 
+from netbox_agent.config import config
 from netbox_agent.misc import get_vendor
 from netbox_agent.raid.base import Raid, RaidController
 
 REGEXP_CONTROLLER_HP = re.compile(r'Smart Array ([a-zA-Z0-9- ]+) in Slot ([0-9]+)')
 
 
-def _get_indentation(string):
+def _parse_ctrl_output(lines):
-    """Return the number of spaces before the current line."""
+    controllers = {}
-    return len(string) - len(string.lstrip(' '))
+    current_ctrl = None
 
-
+    for line in lines:
-def _get_key_value(string):
+        if not line or line.startswith('Note:'):
-    """Return the (key, value) as a tuple from a string."""
-    # Normally all properties look like this:
-    #   Unique Identifier: 600508B1001CE4ACF473EE9C826230FF
-    #   Disk Name: /dev/sda
-    #   Mount Points: None
-    key = ''
-    value = ''
-    try:
-        key, value = string.split(':')
-    except ValueError:
-        # This handles the case when the property of a logical drive
-        # returned is as follows. Here we cannot split by ':' because
-        # the disk id has colon in it. So if this is about disk,
-        # then strip it accordingly.
-        #   Mirror Group 0: physicaldrive 6I:1:5
-        string = string.lstrip(' ')
-        if string.startswith('physicaldrive'):
-            fields = string.split(' ')
-            key = fields[0]
-            value = fields[1]
-        else:
-            # TODO(rameshg87): Check if this ever occurs.
-            return None, None
-
-    return key.lstrip(' ').rstrip(' '), value.lstrip(' ').rstrip(' ')
-
-
-def _get_dict(lines, start_index, indentation):
-    """Recursive function for parsing hpssacli/ssacli output."""
-
-    info = {}
-    current_item = None
-
-    i = start_index
-    while i < len(lines):
-        current_line = lines[i]
-        if current_line.startswith('Note:'):
-            i = i + 1
             continue
-
+        ctrl = REGEXP_CONTROLLER_HP.search(line)
-        current_line_indentation = _get_indentation(current_line)
+        if ctrl is not None:
-        # This check ignore some useless information that make
+            current_ctrl = ctrl.group(1)
-        # crash the parsing
+            controllers[current_ctrl] = {"Slot": ctrl.group(2)}
-        product_name = REGEXP_CONTROLLER_HP.search(current_line)
+            if "Embedded" not in line:
-        if current_line_indentation == 0 and not product_name:
+                controllers[current_ctrl]["External"] = True
-            i = i + 1
             continue
+        attr, val = line.split(": ", 1)
+        attr = attr.strip()
+        val = val.strip()
+        controllers[current_ctrl][attr] = val
+    return controllers
 
-        if current_line_indentation == indentation:
-            current_item = current_line.lstrip(' ')
 
-            info[current_item] = {}
+def _parse_pd_output(lines):
-            i = i + 1
+    drives = {}
+    current_array = None
+    current_drv = None
+
+    for line in lines:
+        line = line.strip()
+        if not line or line.startswith('Note:'):
             continue
-
+        # Parses the Array the drives are in
-        if i >= len(lines) - 1:
+        if line.startswith("Array"):
-            key, value = _get_key_value(current_line)
+            current_array = line.split(None, 1)[1]
-            # If this is some unparsable information, then
+        # Detects new physical drive
-            # just skip it.
+        if line.startswith("physicaldrive"):
-            if key:
+            current_drv = line.split(None, 1)[1]
-                info[current_item][key] = value
+            drives[current_drv] = {}
-            return info, i
+            if current_array is not None:
-
+                drives[current_drv]["Array"] = current_array
-        next_line = lines[i + 1]
+            continue
-        next_line_indentation = _get_indentation(next_line)
+        if ": " not in line:
-
+            continue
-        if current_line_indentation == next_line_indentation:
+        attr, val = line.split(": ", 1)
-            key, value = _get_key_value(current_line)
+        drives.setdefault(current_drv, {})[attr] = val
-            if key:
+    return drives
-                info[current_item][key] = value
-            i = i + 1
-        elif next_line_indentation > current_line_indentation:
-            ret_dict, j = _get_dict(lines, i, current_line_indentation)
-            info[current_item].update(ret_dict)
-            i = j + 1
-        elif next_line_indentation < current_line_indentation:
-            key, value = _get_key_value(current_line)
-            if key:
-                info[current_item][key] = value
-            return info, i
-
-    return info, i
 
 
 class HPRaidController(RaidController):
     def __init__(self, controller_name, data):
         self.controller_name = controller_name
         self.data = data
+        self.drives = self._get_physical_disks()
 
     def get_product_name(self):
         return self.controller_name
@@ -114,40 +73,42 @@ class HPRaidController(RaidController):
     def get_firmware_version(self):
         return self.data['Firmware Version']
 
-    def get_physical_disks(self):
+    def is_external(self):
-        ret = []
+        return self.data.get('External', False)
+
+    def _get_physical_disks(self):
         output = subprocess.getoutput(
             'ssacli ctrl slot={slot} pd all show detail'.format(slot=self.data['Slot'])
         )
         lines = output.split('\n')
         lines = list(filter(None, lines))
-        j = -1
+        drives = _parse_pd_output(lines)
-        while j < len(lines):
+        ret = []
-            info_dict, j = _get_dict(lines, j + 1, 0)
 
-        key = next(iter(info_dict))
+        for name, attrs in drives.items():
-        for array, physical_disk in info_dict[key].items():
+            model = attrs.get('Model', '').strip()
-            for _, pd_attr in physical_disk.items():
+            vendor = None
-                model = pd_attr.get('Model', '').strip()
+            if model.startswith('HP'):
-                vendor = None
+                vendor = 'HP'
-                if model.startswith('HP'):
+            elif len(model.split()) > 1:
-                    vendor = 'HP'
+                vendor = get_vendor(model.split()[1])
-                elif len(model.split()) > 1:
+            else:
-                    vendor = get_vendor(model.split()[1])
+                vendor = get_vendor(model)
-                else:
-                    vendor = get_vendor(model)
 
-                ret.append({
+            ret.append({
-                    'Model': model,
+                'Model': model,
-                    'Vendor': vendor,
+                'Vendor': vendor,
-                    'SN': pd_attr.get('Serial Number', '').strip(),
+                'SN': attrs.get('Serial Number', '').strip(),
-                    'Size': pd_attr.get('Size', '').strip(),
+                'Size': attrs.get('Size', '').strip(),
-                    'Type': 'SSD' if pd_attr.get('Interface Type') == 'Solid State SATA'
+                'Type': 'SSD' if attrs.get('Interface Type') == 'Solid State SATA'
-                    else 'HDD',
+                else 'HDD',
-                    '_src': self.__class__.__name__,
+                '_src': self.__class__.__name__,
-                })
+            })
         return ret
 
+    def get_physical_disks(self):
+        return self.drives
+
 
 class HPRaid(Raid):
     def __init__(self):
@@ -158,16 +119,11 @@ class HPRaid(Raid):
     def convert_to_dict(self):
         lines = self.output.split('\n')
         lines = list(filter(None, lines))
-        j = -1
+        controllers = _parse_ctrl_output(lines)
-        while j < len(lines):
+        for controller, attrs in controllers.items():
-            info_dict, j = _get_dict(lines, j + 1, 0)
+            self.controllers.append(
-            if len(info_dict.keys()):
+                HPRaidController(controller, attrs)
-                _product_name = list(info_dict.keys())[0]
+            )
-                product_name = REGEXP_CONTROLLER_HP.search(_product_name)
-                if product_name:
-                    self.controllers.append(
-                        HPRaidController(product_name.group(1), info_dict[_product_name])
-                    )
 
     def get_controllers(self):
         return self.controllers

netbox_agent/server.py

@@ -25,6 +25,7 @@ class ServerBase():
         self.bios = dmidecode.get_by_type(self.dmi, 'BIOS')
         self.chassis = dmidecode.get_by_type(self.dmi, 'Chassis')
         self.system = dmidecode.get_by_type(self.dmi, 'System')
+        self.inventory = Inventory(server=self)
 
         self.network = None
 
@@ -94,6 +95,19 @@ class ServerBase():
                 server.position = None
         return update, server
 
+    def update_netbox_expansion_location(self, server, expansion):
+        update = False
+        if expansion.tenant != server.tenant:
+            expansion.tenant = server.tenant
+            update = True
+        if expansion.site != server.site:
+            expansion.site = server.site
+            update = True
+        if expansion.rack != server.rack:
+            expansion.rack = server.rack
+            update = True
+        return update
+
     def get_rack(self):
         rack = Rack()
         return rack.get()
@@ -256,8 +270,11 @@ class ServerBase():
         )
         return new_server
 
-    def get_netbox_server(self):
+    def get_netbox_server(self, expansion=False):
-        return nb.dcim.devices.get(serial=self.get_service_tag())
+        if expansion is False:
+            return nb.dcim.devices.get(serial=self.get_service_tag())
+        else:
+            return nb.dcim.devices.get(serial=self.get_expansion_service_tag())
 
     def _netbox_set_or_update_blade_slot(self, server, chassis, datacenter):
         # before everything check if right chassis
@@ -292,9 +309,9 @@ class ServerBase():
                 slot=slot
             ))
 
-    def _netbox_set_or_update_blade_expansion_slot(self, server, chassis, datacenter):
+    def _netbox_set_or_update_blade_expansion_slot(self, expansion, chassis, datacenter):
         # before everything check if right chassis
-        actual_device_bay = server.parent_device.device_bay if server.parent_device else None
+        actual_device_bay = expansion.parent_device.device_bay if expansion.parent_device else None
         actual_chassis = actual_device_bay.device if actual_device_bay else None
         slot = self.get_blade_expansion_slot()
         if actual_chassis and \
@@ -302,30 +319,28 @@ class ServerBase():
            actual_device_bay.name == slot:
             return
 
-        server.name += " expansion"
-
         real_device_bays = nb.dcim.device_bays.filter(
             device_id=chassis.id,
             name=slot,
         )
-        if len(real_device_bays) > 0:
+        if len(real_device_bays) == 0:
-            logging.info(
-                'Setting device expansion ({serial}) new slot on {slot} '
-                '(Chassis {chassis_serial})..'.format(
-                    serial=server.serial, slot=slot, chassis_serial=chassis.serial
-                ))
-            # reset actual device bay if set
-            if actual_device_bay:
-                actual_device_bay.installed_device = None
-                actual_device_bay.save()
-            # setup new device bay
-            real_device_bay = real_device_bays[0]
-            real_device_bay.installed_device = server
-            real_device_bay.save()
-        else:
             logging.error('Could not find slot {slot} expansion for chassis'.format(
                 slot=slot
             ))
+            return
+        logging.info(
+            'Setting device expansion ({serial}) new slot on {slot} '
+            '(Chassis {chassis_serial})..'.format(
+                serial=expansion.serial, slot=slot, chassis_serial=chassis.serial
+            ))
+        # reset actual device bay if set
+        if actual_device_bay:
+            actual_device_bay.installed_device = None
+            actual_device_bay.save()
+        # setup new device bay
+        real_device_bay = real_device_bays[0]
+        real_device_bay.installed_device = expansion
+        real_device_bay.save()
 
     def netbox_create_or_update(self, config):
         """
@@ -370,9 +385,10 @@ class ServerBase():
         if config.register or config.update_all or config.update_network:
             self.network = ServerNetwork(server=self)
             self.network.create_or_update_netbox_network_cards()
+        update_inventory = config.inventory and (config.register or
+                config.update_all or config.update_inventory)
         # update inventory if feature is enabled
-        if config.inventory and (config.register or config.update_all or config.update_inventory):
+        if update_inventory:
-            self.inventory = Inventory(server=self)
             self.inventory.create_or_update()
         # update psu
         if config.register or config.update_all or config.update_psu:
@@ -380,14 +396,21 @@ class ServerBase():
             self.power.create_or_update_power_supply()
             self.power.report_power_consumption()
 
-        if self.own_expansion_slot():
+        expansion = nb.dcim.devices.get(serial=self.get_expansion_service_tag())
+        if self.own_expansion_slot() and config.expansion_as_device:
             logging.debug('Update Server expansion...')
-            expansion = nb.dcim.devices.get(serial=self.get_expansion_service_tag())
             if not expansion:
                 expansion = self._netbox_create_blade_expansion(chassis, datacenter, tenant, rack)
 
             # set slot for blade expansion
             self._netbox_set_or_update_blade_expansion_slot(expansion, chassis, datacenter)
+            if update_inventory:
+                # Updates expansion inventory
+                inventory = Inventory(server=self, update_expansion=True)
+                inventory.create_or_update()
+        elif self.own_expansion_slot() and expansion:
+            expansion.delete()
+            expansion = None
 
         update = 0
         # for every other specs
@@ -396,6 +419,7 @@ class ServerBase():
             update += 1
             server.name = self.get_hostname()
 
+
         if sorted(set(server.tags)) != sorted(set(self.tags)):
             server.tags = self.tags
             update += 1
@@ -406,6 +430,17 @@ class ServerBase():
 
         if update:
             server.save()
+
+        if expansion:
+            update = 0
+            expansion_name = server.name + ' expansion'
+            if expansion.name != expansion_name:
+                expansion.name = expansion_name
+                update += 1
+            if self.update_netbox_expansion_location(server, expansion):
+                update += 1
+            if update:
+                expansion.save()
         logging.debug('Finished updating Server!')
 
     def print_debug(self):

netbox_agent/vendors/hp.py

@@ -67,35 +67,49 @@ class HPHost(ServerBase):
             return self.hp_rack_locator["Enclosure Serial"].strip()
         return self.get_service_tag()
 
+    def get_blade_expansion_slot(self):
+        """
+        Expansion slot are always the compute bay number + 1
+        """
+        if self.is_blade() and self.own_gpu_expansion_slot() or \
+                self.own_disk_expansion_slot() or True:
+            return 'Bay {}'.format(
+                str(int(self.hp_rack_locator['Server Bay'].strip()) + 1)
+            )
+        return None
+
     def get_expansion_product(self):
         """
         Get the extension slot that is on a pair slot number
         next to the compute slot that is on an odd slot number
         I only know on model of slot GPU extension card that.
         """
-        if self.own_expansion_slot():
+        if self.own_gpu_expansion_slot():
             return "ProLiant BL460c Graphics Expansion Blade"
-        return None
+        elif self.own_disk_expansion_slot():
-
+            return "ProLiant BL460c Disk Expansion Blade"
-    def is_expansion_slot(self, server):
-        """
-        Return True if its an extension slot, based on the name
-        """
-        return server.name.endswith(" expansion")
-
-    def get_blade_expansion_slot(self):
-        """
-        Expansion slot are always the compute bay number + 1
-        """
-        if self.is_blade() and self.own_expansion_slot():
-            return 'Bay {}'.format(
-                str(int(self.hp_rack_locator['Server Bay'].strip()) + 1)
-            )
         return None
 
     def own_expansion_slot(self):
+        """
+        Say if the device can host an extension card based
+        on the product name
+        """
+        return self.own_gpu_expansion_slot() or self.own_disk_expansion_slot()
+
+    def own_gpu_expansion_slot(self):
         """
         Say if the device can host an extension card based
         on the product name
         """
         return self.get_product_name().endswith('Graphics Exp')
+
+    def own_disk_expansion_slot(self):
+        """
+        Say if the device can host an extension card based
+        on the product name
+        """
+        for raid_card in self.inventory.get_raid_cards():
+            if self.is_blade() and raid_card.is_external():
+                return True
+        return False