Merge branch 'master' into device-roles

2020-07-01 17:56:05 +02:00 · 2020-07-01 17:56:05 +02:00 · c29ddaf0d5
commit c29ddaf0d5
parent 8a152cd15f f654187074
8 changed files with 160 additions and 57 deletions
--- a/README.md
+++ b/README.md
@ -32,24 +32,71 @@ The goal is to generate an existing infrastructure on Netbox and have the abilit
 - lldpd
 - lshw

-# Known limitations
+## Inventory requirement
+- hpassacli
+- storcli
+- omreport

-* The project is only compatible with Linux.
-Since it uses `ethtool` and parses `/sys/` directory, it's not compatible with *BSD distributions.
-* Netbox `>=2.6.0,<=2.6.2` has a caching problem ; if the cache lifetime is too high, the script can get stale data after modification.
-We advise to set `CACHE_TIME` to `0`.
+# Installation
+
+```
+# pip3 install netbox-agent
+```
+
+# Usage
+
+The agent can be run from a shell and get its configuration from either the configuration file or environment variables.
+
+Configuration values are overridden based on the following precedence: command line arguments (might include config file) > environment variables > default config file > defaults.
+
+```
+# netbox_agent -c /etc/netbox_agent.yml --register
+INFO:root:Creating chassis blade (serial: QTFCQ574502EF)
+INFO:root:Creating blade (serial: QTFCQ574502D2) myserver on chassis QTFCQ574502EF
+INFO:root:Setting device (QTFCQ574502D2) new slot on Slot 9 (Chassis QTFCQ574502EF)..
+INFO:root:Interface a8:1e:84:f2:9e:6a not found, creating..
+INFO:root:Creating NIC enp1s0f1 (a8:1e:84:f2:9e:6a) on myserver
+INFO:root:Interface 02:42:7a:89:cf:a4 not found, creating..
+INFO:root:Creating NIC br-07ea1e4a2f0e (02:42:7a:89:cf:a4) on myserver
+INFO:root:Create new IP 172.19.0.1/16 on br-07ea1e4a2f0e
+INFO:root:Interface a8:1e:84:f2:9e:69 not found, creating..
+INFO:root:Creating NIC enp1s0f0 (a8:1e:84:f2:9e:69) on myserver
+INFO:root:Create new IP 42.42.42.42/24 on enp1s0f0
+INFO:root:Create new IP fe80::aa1e:84ff:fef2:9e69/64 on enp1s0f0
+INFO:root:Interface a8:1e:84:cd:9d:d6 not found, creating..
+INFO:root:Creating NIC IPMI (a8:1e:84:cd:9d:d6) on myserver
+INFO:root:Create new IP 10.191.122.10/24 on IPMI
+```
+
+If you need, you can update only specific informations like:
+* Network
+* Inventory
+* Location
+* PSUs
+
+```
+# ip a add 42.42.42.43/24 dev enp1s0f1
+# netbox_agent -c /etc/netbox_agent.yaml --update-network
+INFO:root:Create new IP 42.42.42.43/24 on enp1s0f1
+# netbox_agent --update-inventory
+INFO:root:Creating Disk Samsung SSD 850 S2RBNX0K101698D
+```

 # Configuration

 ```
+# Netbox configuration
 netbox:
 url: 'http://netbox.internal.company.com'
 token: supersecrettoken

+# Network configuration
 network:
+  # Regex to ignore interfaces 
  ignore_interfaces: "(dummy.*|docker.*)"
+  # Regex to ignore IP addresses
  ignore_ips: (127\.0\.0\..*)
-  # enable auto-cabling
+  # enable auto-cabling by parsing LLDP answers
  lldp: true

 #
@ -69,12 +116,14 @@ network:
 # driver: "file:/tmp/tenant"
 # regex: "(.*)"

+## Enable virtual machine support 
 # virtual:
 #   # not mandatory, can be guessed
 #   enabled: True
 #   # see https://netbox.company.com/virtualization/clusters/
 #   cluster_name: my_vm_cluster

+# Enable datacenter location feature in Netbox
 datacenter_location:
 driver: "cmd:cat /etc/qualification | tr [a-z] [A-Z]"
 regex: "DATACENTER: (?P<datacenter>[A-Za-z0-9]+)"
@ -84,6 +133,7 @@ datacenter_location:
 # driver: "file:/tmp/datacenter"
 # regex: "(.*)"

+# Enable rack location feature in Netbox
 rack_location:
 # driver: 'cmd:lldpctl'
 # match SysName: sw-dist-a1.dc42
@ -92,6 +142,7 @@ rack_location:
 # driver: "file:/tmp/datacenter"
 # regex: "(.*)"

+# Enable local inventory reporting 
 inventory: true
 ```

@ -178,6 +229,27 @@ Feel free to send me a dmidecode output for Supermicro's blade!

 * Nothing ATM, feel free to send me a dmidecode or make a PR!

-# TODO
+# Known limitations

- [ ] `CustomFields` support with firmware versions for Device (BIOS), RAID Cards and disks
+* The project is only compatible with Linux.
+Since it uses `ethtool` and parses `/sys/` directory, it's not compatible with *BSD distributions.
+* Netbox `>=2.6.0,<=2.6.2` has a caching problem ; if the cache lifetime is too high, the script can get stale data after modification.
+We advise to set `CACHE_TIME` to `0`.
+
+# Developing
+
+If you want to run the agent while adding features or just for debugging purposes
+
+```
+# git clone https://github.com/Solvik/netbox-agent.git
+# cd netbox-agent
+# python3 -m netbox_agent.cli --register
+```
+
+On a personal note, I use the docker image from [netbox-community/netbox-docker](https://github.com/netbox-community/netbox-docker)
+```
+# git clone https://github.com/netbox-community/netbox-docker
+# cd netbox-docker
+# docker-compose pull
+# docker-compose up 
+```
--- a/netbox_agent/inventory.py
+++ b/netbox_agent/inventory.py
@ -139,7 +139,7 @@ class Inventory():

        for nb_motherboard in nb_motherboards:
            if nb_motherboard.serial not in [x['serial'] for x in motherboards]:
-                logging.info('Deleting unknown motherboard {vendor} {motherboard}/{serial}'.format(
+                logging.info('Deleting unknown motherboard {motherboard}/{serial}'.format(
                    motherboard=self.lshw.motherboard,
                    serial=nb_motherboard.serial,
                ))
--- a/netbox_agent/lshw.py
+++ b/netbox_agent/lshw.py
@ -86,21 +86,21 @@ class LSHW():

        elif "nvme" in obj["configuration"]["driver"]:
            nvme = json.loads(
-                            subprocess.check_output(["nvme", '-list', '-o', 'json'],
-                            encoding='utf8'))  # noqa: E128
+                subprocess.check_output(
+                    ["nvme", '-list', '-o', 'json'],
+                    encoding='utf8')
+            )

-            d = {}
-            d["vendor"] = obj["vendor"]
-            d["version"] = obj["version"]
-            d["product"] = obj["product"]
+            for device in nvme["Devices"]:
+                d = {}
+                d['logicalname'] = device["DevicePath"]
+                d['product'] = device["ModelNumber"]
+                d['serial'] = device["SerialNumber"]
+                d["version"] = device["Firmware"]
+                d['size'] = device["UsedSize"]
+                d['description'] = "NVME Disk"

-            d['description'] = "NVME Disk"
-            d['product'] = nvme["Devices"][0]["ModelNumber"]
-            d['size'] = nvme["Devices"][0]["PhysicalSize"]
-            d['serial'] = nvme["Devices"][0]["SerialNumber"]
-            d['logicalname'] = nvme["Devices"][0]["DevicePath"]
-
-            self.disks.append(d)
+                self.disks.append(d)

    def find_cpus(self, obj):
        if "product" in obj:
@ -127,7 +127,7 @@ class LSHW():
            d["id"] = dimm.get("id")
            d["serial"] = dimm.get("serial", 'N/A')
            d["vendor"] = dimm.get("vendor", 'N/A')
-            d["product"] = dimm.get("product")
+            d["product"] = dimm.get("product", 'N/A')
            d["size"] = dimm.get("size", 0) / 2 ** 20 / 1024

            self.memories.append(d)
--- a/netbox_agent/network.py
+++ b/netbox_agent/network.py
@ -24,20 +24,20 @@ class Network(object):
        self.nics = self.scan()
        self.ipmi = None
        self.dcim_choices = {}
-        dcim_c = nb.dcim.choices()
-
-        for choice in dcim_c:
-            self.dcim_choices[choice] = {}
-            for c in dcim_c[choice]:
-                self.dcim_choices[choice][c['label']] = c['value']
+        dcim_c = nb.dcim.interfaces.choices()
+        for _choice_type in dcim_c:
+            key = 'interface:{}'.format(_choice_type)
+            self.dcim_choices[key] = {}
+            for choice in dcim_c[_choice_type]:
+                self.dcim_choices[key][choice['display_name']] = choice['value']

        self.ipam_choices = {}
-        ipam_c = nb.ipam.choices()
-
-        for choice in ipam_c:
-            self.ipam_choices[choice] = {}
-            for c in ipam_c[choice]:
-                self.ipam_choices[choice][c['label']] = c['value']
+        ipam_c = nb.ipam.ip_addresses.choices()
+        for _choice_type in ipam_c:
+            key = 'ip-address:{}'.format(_choice_type)
+            self.ipam_choices[key] = {}
+            for choice in ipam_c[_choice_type]:
+                self.ipam_choices[key][choice['display_name']] = choice['value']

    def get_network_type():
        return NotImplementedError
@ -93,6 +93,12 @@ class Network(object):
                bonding_slaves = open(
                    '/sys/class/net/{}/bonding/slaves'.format(interface)
                ).read().split()
+
+            # Tun and TAP support
+            virtual = os.path.isfile(
+                '/sys/class/net/{}/tun_flags'.format(interface)
+            )
+
            nic = {
                'name': interface,
                'mac': mac if mac != '00:00:00:00:00:00' else None,
@ -103,6 +109,7 @@ class Network(object):
                    ) for x in ip_addr
                ] if ip_addr else None,  # FIXME: handle IPv6 addresses
                'ethtool': Ethtool(interface).parse(),
+                'virtual': virtual,
                'vlan': vlan,
                'bonding': bonding,
                'bonding_slaves': bonding_slaves,
@ -162,6 +169,10 @@ class Network(object):

        if nic.get('bonding'):
            return self.dcim_choices['interface:type']['Link Aggregation Group (LAG)']
+
+        if nic.get('virtual'):
+            return self.dcim_choices['interface:type']['Virtual']
+
        if nic.get('ethtool') is None:
            return self.dcim_choices['interface:type']['Other']

@ -240,13 +251,18 @@ class Network(object):
            name=nic['name'], mac=nic['mac'], device=self.device.name))

        nb_vlan = None
-        interface = self.nb_net.interfaces.create(
-            name=nic['name'],
-            mac_address=nic['mac'],
-            type=type,
-            mgmt_only=mgmt,
+
+        params = {
+            'name': nic['name'],
+            'type': type,
+            'mgmt_only': mgmt,
            **self.custom_arg,
-        )
+        }
+
+        if not nic.get('virtual', False):
+            params['mac_address'] = nic['mac']
+
+        interface = self.nb_net.interfaces.create(**params)

        if nic['vlan']:
            nb_vlan = self.get_or_create_vlan(nic['vlan'])
@ -442,8 +458,8 @@ class ServerNetwork(Network):
        self.server = server
        self.device = self.server.get_netbox_server()
        self.nb_net = nb.dcim
-        self.custom_arg = {'device': self.device.id}
-        self.custom_arg_id = {'device_id': self.device.id}
+        self.custom_arg = {'device': getattr(self.device, "id", None)}
+        self.custom_arg_id = {'device_id': getattr(self.device, "id", None)}

    def get_network_type(self):
        return 'server'
@ -562,15 +578,15 @@ class VirtualNetwork(Network):
        self.server = server
        self.device = self.server.get_netbox_vm()
        self.nb_net = nb.virtualization
-        self.custom_arg = {'virtual_machine': self.device.id}
-        self.custom_arg_id = {'virtual_machine_id': self.device.id}
+        self.custom_arg = {'virtual_machine': getattr(self.device, "id", None)}
+        self.custom_arg_id = {'virtual_machine_id': getattr(self.device, "id", None)}

-        dcim_c = nb.virtualization.choices()
-
-        for choice in dcim_c:
-            self.dcim_choices[choice] = {}
-            for c in dcim_c[choice]:
-                self.dcim_choices[choice][c['label']] = c['value']
+        dcim_c = nb.virtualization.interfaces.choices()
+        for _choice_type in dcim_c:
+            key = 'interface:{}'.format(_choice_type)
+            self.dcim_choices[key] = {}
+            for choice in dcim_c[_choice_type]:
+                self.dcim_choices[key][choice['display_name']] = choice['value']

    def get_network_type(self):
        return 'virtual'
--- a/netbox_agent/power.py
+++ b/netbox_agent/power.py
@ -29,8 +29,15 @@ class PowerSupply():
                psu.get('Manufacturer', 'No Manufacturer').strip(),
                psu.get('Name', 'No name').strip(),
            )
+
+            sn = psu.get('Serial Number', '').strip()
+            # Let's assume that if no serial and no power reported we skip it
+            if sn == '' and max_power is None:
+                continue
+            if sn == '':
+                sn = 'N/A'
            power_supply.append({
-                'name': psu.get('Serial Number', 'No S/N').strip(),
+                'name': sn,
                'description': desc,
                'allocated_draw': None,
                'maximum_draw': max_power,
--- a/netbox_agent/raid/hp.py
+++ b/netbox_agent/raid/hp.py
@ -54,6 +54,13 @@ def _get_dict(lines, start_index, indentation):
            continue

        current_line_indentation = _get_indentation(current_line)
+        # This check ignore some useless information that make
+        # crash the parsing
+        product_name = REGEXP_CONTROLLER_HP.search(current_line)
+        if current_line_indentation == 0 and not product_name:
+            i = i + 1
+            continue
+
        if current_line_indentation == indentation:
            current_item = current_line.lstrip(' ')

--- a/requirements.txt
+++ b/requirements.txt
@ -1,5 +1,5 @@
 pynetbox==4.3.1
-netaddr==0.7.19
+netaddr==0.7.20
 netifaces==0.10.9
 pyyaml==5.3.1
-jsonargparse==2.25.3
+jsonargparse==2.31.0
--- a/setup.py
+++ b/setup.py
@ -2,9 +2,10 @@ from setuptools import find_packages, setup

 setup(
    name='netbox_agent',
-    version='0.5.0',
+    version='0.6.1',
    description='NetBox agent for server',
    long_description=open('README.md', encoding="utf-8").read(),
+    long_description_content_type='text/markdown',
    url='https://github.com/solvik/netbox_agent',
    author='Solvik Blum',
    author_email='solvik@solvik.fr',
@ -14,10 +15,10 @@ setup(
    use_scm_version=True,
    install_requires=[
        'pynetbox==4.3.1',
-        'netaddr==0.7.19',
+        'netaddr==0.7.20',
        'netifaces==0.10.9',
        'pyyaml==5.3.1',
-        'jsonargparse==2.25.3',
+        'jsonargparse==2.31.0',
    ],
    zip_safe=False,
    keywords=['netbox'],