README.md

@@ -8,16 +8,26 @@ The goal is to generate an existing infrastructure on Netbox and have the abilit
 # Features
 
 * Create servers, chassis and blade through standard tools (`dmidecode`)
-* Create physical network interfaces with IPs
+* Create physical, bonding and vlan network interfaces with IPs
+* Create IPMI interface if found
+* Create or get existing VLAN and associate it to interfaces
 * Generic ability to guess datacenters and rack location through drivers (`cmd` and `file` and custom ones)
 * Update existing `Device` and `Interfaces`
 * Handle blade moving (new slot, new chassis)
 
+# Requirements
+
+- Netbox >= 2.6
+- Python >= 3.4
+- [python3-netaddr](https://github.com/drkjam/netaddr)
+- [python3-netifaces](https://github.com/al45tair/netifaces)
+
 # Known limitations
 
 * The project is only compatible with Linux.
 Since it uses `ethtool` and parses `/sys/` directory, it's not compatible with *BSD distributions.
 * Netbox `>=2.6.0,<=2.6.2` has a caching problem ; if the cache lifetime is too high, the script can get stale data after modification.
+We advise to set `CACHE_TIME` to `0`.
 
 # Configuration
 
@@ -26,12 +36,26 @@ netbox:
  url: 'http://netbox.internal.company.com'
  token: supersecrettoken
 
+network:
+  ignore_interfaces: "(dummy.*|docker.*)"
+  ignore_ips: (127\.0\.0\..*)
+
 datacenter_location:
- # driver_file: /opt/netbox_driver_dc.py
- driver: file:/etc/qualification
- regex: "datacenter: (?P<datacenter>[A-Za-z0-9]+)"
+ driver: "cmd:cat /etc/qualification | tr [a-z] [A-Z]"
+ regex: "DATACENTER: (?P<datacenter>[A-Za-z0-9]+)"
 # driver: 'cmd:lldpctl'
-# regex = 'SysName: .*\.(?P<datacenter>[A-Za-z0-9]+)'```
+# regex: 'SysName: .*\.([A-Za-z0-9]+)'
+#
+# driver: "file:/tmp/datacenter"
+# regex: "(.*)"
+
+rack_location:
+# driver: 'cmd:lldpctl'
+# match SysName: sw-dist-a1.dc42
+# regex: 'SysName:[ ]+[A-Za-z]+-[A-Za-z]+-([A-Za-z0-9]+)'
+#
+# driver: "file:/tmp/datacenter"
+# regex: "(.*)"
 ```
 
 # Hardware

netbox_agent.yaml.example

@@ -2,6 +2,10 @@ netbox:
  url: 'http://netbox.internal.company.com'
  token: supersecrettoken
 
+network:
+  ignore_interfaces: "(dummy.*|docker.*)"
+  ignore_ips: (127\.0\.0\..*)
+
 datacenter_location:
  driver: "cmd:cat /etc/qualification | tr [a-z] [A-Z]"
  regex: "DATACENTER: (?P<datacenter>[A-Za-z0-9]+)"

netbox_agent/config.py

@@ -30,3 +30,9 @@ if config.get('rack_location'):
     RACK_LOCATION_DRIVER_FILE = rack_location.get('driver_file')
     RACK_LOCATION = rack_location.get('driver')
     RACK_LOCATION_REGEX = rack_location.get('regex')
+
+NETWORK_IGNORE_INTERFACES = None
+NETWORK_IGNORE_IPS = None
+if config.get('network'):
+    NETWORK_IGNORE_INTERFACES = config['network']['ignore_interfaces']
+    NETWORK_IGNORE_IPS = config['network']['ignore_ips']

netbox_agent/ipmi.py

@@ -0,0 +1,49 @@
+import logging
+import subprocess
+
+
+class Ipmi():
+    """
+    Parse IPMI output
+    ie:
+
+    Set in Progress         : Set Complete
+    Auth Type Support       :
+    Auth Type Enable        : Callback :
+                            : User     :
+                            : Operator :
+                            : Admin    :
+                            : OEM      :
+    IP Address Source       : DHCP Address
+    IP Address              : 10.192.2.1
+    Subnet Mask             : 255.255.240.0
+    MAC Address             : 98:f2:b3:f0:ee:1e
+    SNMP Community String   :
+    BMC ARP Control         : ARP Responses Enabled, Gratuitous ARP Disabled
+    Default Gateway IP      : 10.192.2.254
+    802.1q VLAN ID          : Disabled
+    802.1q VLAN Priority    : 0
+    RMCP+ Cipher Suites     : 0,1,2,3
+    Cipher Suite Priv Max   : XuuaXXXXXXXXXXX
+                            :     X=Cipher Suite Unused
+                            :     c=CALLBACK
+                            :     u=USER
+                            :     o=OPERATOR
+                            :     a=ADMIN
+                            :     O=OEM
+    Bad Password Threshold  : Not Available
+    """
+    def __init__(self):
+        self.ret, self.output = subprocess.getstatusoutput('ipmitool lan print')
+        if self.ret != 0:
+            logging.error('Cannot get ipmi info: {}'.format(self.output))
+
+    def parse(self):
+        ret = {}
+        if self.ret != 0:
+            return ret
+        for line in self.output.split('\n'):
+            key = line.split(':')[0].strip()
+            value = ':'.join(line.split(':')[1:]).strip()
+            ret[key] = value
+        return ret

netbox_agent/location.py

@@ -22,11 +22,9 @@ class LocationBase():
         self.driver = driver
         self.driver_value = driver_value
         self.driver_file = driver_file
-        print(self.driver_file)
         self.regex = regex
 
         if self.driver_file:
-            print('if', self.driver_file)
             try:
                 # FIXME: Works with Python 3.3+, support older version?
                 loader = importlib.machinery.SourceFileLoader('driver_file', self.driver_file)

netbox_agent/network.py

@@ -3,11 +3,13 @@ import logging
 import os
 import re
 
-from netaddr import IPAddress
+from netaddr import IPAddress, IPNetwork
 import netifaces
 
 from netbox_agent.config import netbox_instance as nb
+from netbox_agent.config import NETWORK_IGNORE_INTERFACES, NETWORK_IGNORE_IPS
 from netbox_agent.ethtool import Ethtool
+from netbox_agent.ipmi import Ipmi
 
 IFACE_TYPE_100ME_FIXED = 800
 IFACE_TYPE_1GE_FIXED = 1000
@@ -33,10 +35,7 @@ IFACE_TYPE_200GE_CFP2 = 1650
 IFACE_TYPE_200GE_QSFP56 = 1700
 IFACE_TYPE_400GE_QSFP_DD = 1750
 IFACE_TYPE_OTHER = 32767
-
-# Regex to match base interface name
-# Doesn't match vlan interfaces and other loopback etc
-INTERFACE_REGEX = re.compile('^(eth[0-9]+|ens[0-9]+|enp[0-9]+s[0-9]f[0-9])$')
+IFACE_TYPE_LAG = 200
 
 
 class Network():
@@ -44,29 +43,99 @@ class Network():
         self.nics = []
 
         self.server = server
+        self.device = self.server.get_netbox_server()
         self.scan()
 
     def scan(self):
         for interface in os.listdir('/sys/class/net/'):
-            if re.match(INTERFACE_REGEX, interface):
+            # ignore if it's not a link (ie: bonding_masters etc)
+            if not os.path.islink('/sys/class/net/{}'.format(interface)):
+                continue
+
+            if NETWORK_IGNORE_INTERFACES and \
+               re.match(NETWORK_IGNORE_INTERFACES, interface):
+                logging.debug('Ignore interface {interface}'.format(interface=interface))
+                continue
+            else:
                 ip_addr = netifaces.ifaddresses(interface).get(netifaces.AF_INET)
+                if NETWORK_IGNORE_IPS and ip_addr:
+                    for i, ip in enumerate(ip_addr):
+                        if re.match(NETWORK_IGNORE_IPS, ip['addr']):
+                            ip_addr.pop(i)
+
+                mac = open('/sys/class/net/{}/address'.format(interface), 'r').read().strip()
+                vlan = None
+                if len(interface.split('.')) > 1:
+                    vlan = int(interface.split('.')[1])
+                bonding = False
+                bonding_slaves = []
+                if os.path.isdir('/sys/class/net/{}/bonding'.format(interface)):
+                    bonding = True
+                    bonding_slaves = open(
+                        '/sys/class/net/{}/bonding/slaves'.format(interface)
+                    ).read().split()
                 nic = {
                     'name': interface,
-                    'mac': open('/sys/class/net/{}/address'.format(interface), 'r').read().strip(),
+                    'mac': mac if mac != '00:00:00:00:00:00' else None,
                     'ip': [
                         '{}/{}'.format(
                             x['addr'],
                             IPAddress(x['netmask']).netmask_bits()
                         ) for x in ip_addr
                         ] if ip_addr else None,  # FIXME: handle IPv6 addresses
-                    'ethtool': Ethtool(interface).parse()
+                    'ethtool': Ethtool(interface).parse(),
+                    'vlan': vlan,
+                    'bonding': bonding,
+                    'bonding_slaves': bonding_slaves,
                 }
                 self.nics.append(nic)
 
+    def _set_bonding_interfaces(self):
+        bonding_nics = [x for x in self.nics if x['bonding']]
+        if not len(bonding_nics):
+            return False
+        logging.debug('Setting bonding interfaces..')
+        for nic in bonding_nics:
+            bond_int = self.get_netbox_network_card(nic)
+            logging.debug('Setting slave interface for {name}'.format(
+                name=bond_int.name
+            ))
+            for slave in nic['bonding_slaves']:
+                slave_nic = next(item for item in self.nics if item['name'] == slave)
+                slave_int = self.get_netbox_network_card(slave_nic)
+                logging.debug('Settting interface {name} as slave of {master}'.format(
+                    name=slave_int.name, master=bond_int.name
+                ))
+                slave_int.lag = bond_int
+                slave_int.save()
+        return True
+
     def get_network_cards(self):
         return self.nics
 
+    def get_netbox_network_card(self, nic):
+        if nic['mac'] is None:
+            interface = nb.dcim.interfaces.get(
+                device_id=self.device.id,
+                name=nic['name'],
+            )
+        else:
+            interface = nb.dcim.interfaces.get(
+                device_id=self.device.id,
+                mac_address=nic['mac'],
+                name=nic['name'],
+            )
+        return interface
+
+    def get_netbox_network_cards(self):
+        return nb.dcim.interfaces.filter(
+            device_id=self.device.id,
+            mgmt_only=False,
+        )
+
     def get_netbox_type_for_nic(self, nic):
+        if nic.get('bonding'):
+            return IFACE_TYPE_LAG
         if nic.get('ethtool') is None:
             return IFACE_TYPE_OTHER
         if nic['ethtool']['speed'] == '10000Mb/s':
@@ -79,58 +148,169 @@ class Network():
             return IFACE_TYPE_1GE_FIXED
         return IFACE_TYPE_OTHER
 
-    def create_netbox_nic(self, device, nic):
+    def get_ipmi(self):
+        ipmi = Ipmi().parse()
+        return ipmi
+
+    def get_netbox_ipmi(self):
+        ipmi = self.get_ipmi()
+        mac = ipmi['MAC Address']
+        return nb.dcim.interfaces.get(
+            mac=mac
+        )
+
+    def get_or_create_vlan(self, vlan_id):
+        # FIXME: we may need to specify the datacenter
+        # since users may have same vlan id in multiple dc
+        vlan = nb.ipam.vlans.get(
+            vid=vlan_id,
+        )
+        if vlan is None:
+            vlan = nb.ipam.vlans.create(
+                name='VLAN {}'.format(vlan_id),
+                vid=vlan_id,
+            )
+        return vlan
+
+    def reset_vlan_on_interface(self, vlan_id, interface):
+        update = False
+        if vlan_id is None and \
+           (interface.mode is not None or len(interface.tagged_vlans) > 0):
+            logging.info('Interface {interface} is not tagged, reseting mode'.format(
+                interface=interface))
+            update = True
+            interface.mode = None
+            interface.tagged_vlans = []
+        elif vlan_id and (
+                interface.mode is None or
+                len(interface.tagged_vlans) != 1 or
+                interface.tagged_vlans[0].vid != vlan_id):
+            logging.info('Resetting VLAN on interface {interface}'.format(
+                interface=interface))
+            update = True
+            nb_vlan = self.get_or_create_vlan(vlan_id)
+            interface.mode = 200
+            interface.tagged_vlans = [nb_vlan] if nb_vlan else []
+        return update, interface
+
+    def create_or_update_ipmi(self):
+        ipmi = self.get_ipmi()
+        mac = ipmi['MAC Address']
+        ip = ipmi['IP Address']
+        netmask = ipmi['Subnet Mask']
+        vlan = int(ipmi['802.1q VLAN ID']) if ipmi['802.1q VLAN ID'] != 'Disabled' else None
+        address = IPNetwork('{}/{}'.format(ip, netmask)).__str__()
+
+        interface = nb.dcim.interfaces.get(
+            device_id=self.device.id,
+            mgmt_only=True,
+            )
+        nic = {
+            'name': 'IPMI',
+            'mac': mac,
+            'vlan': vlan,
+            'ip': [address],
+        }
+        if interface is None:
+            interface = self.create_netbox_nic(nic, mgmt=True)
+            self.create_or_update_netbox_ip_on_interface(address, interface)
+        else:
+            # let the user chose the name of mgmt ?
+            # guess it with manufacturer (IDRAC, ILO, ...) ?
+            update = False
+            self.create_or_update_netbox_ip_on_interface(address, interface)
+            update, interface = self.reset_vlan_on_interface(nic['vlan'], interface)
+            if mac != interface.mac_address:
+                interface.mac_address = mac
+                update = True
+            if update:
+                interface.save()
+        return interface
+
+    def create_netbox_nic(self, nic, mgmt=False):
         # TODO: add Optic Vendor, PN and Serial
         type = self.get_netbox_type_for_nic(nic)
         logging.info('Creating NIC {name} ({mac}) on {device}'.format(
-            name=nic['name'], mac=nic['mac'], device=device.name))
+            name=nic['name'], mac=nic['mac'], device=self.device.name))
+
+        nb_vlan = None
+        if nic['vlan']:
+            nb_vlan = self.get_or_create_vlan(nic['vlan'])
         return nb.dcim.interfaces.create(
-            device=device.id,
+            device=self.device.id,
             name=nic['name'],
             mac_address=nic['mac'],
             type=type,
+            mode=200 if nic['vlan'] else None,
+            tagged_vlans=[nb_vlan.id] if nb_vlan is not None else [],
+            mgmt_only=mgmt,
         )
 
+    def create_or_update_netbox_ip_on_interface(self, ip, interface):
+        netbox_ip = nb.ipam.ip_addresses.get(
+            address=ip,
+        )
+        if netbox_ip:
+            if netbox_ip.interface is None:
+                logging.info('Assigning existing IP {ip} to {interface}'.format(
+                    ip=ip, interface=interface))
+            elif netbox_ip.interface.id != interface.id:
+                logging.info(
+                    'Detected interface change for ip {ip}: old interface is '
+                    '{old_interface} (id: {old_id}), new interface is {new_interface} '
+                    ' (id: {new_id})'
+                    .format(
+                        old_interface=netbox_ip.interface, new_interface=interface,
+                        old_id=netbox_ip.id, new_id=interface.id, ip=netbox_ip.address
+                    ))
+            else:
+                return netbox_ip
+            netbox_ip.interface = interface
+            netbox_ip.save()
+        else:
+            logging.info('Create new IP {ip} on {interface}'.format(
+                ip=ip, interface=interface))
+            netbox_ip = nb.ipam.ip_addresses.create(
+                address=ip,
+                interface=interface.id,
+                status=1,
+            )
+        return netbox_ip
+
     def create_netbox_network_cards(self):
         logging.debug('Creating NIC...')
-        device = self.server.get_netbox_server()
         for nic in self.nics:
-            interface = nb.dcim.interfaces.get(
-                mac_address=nic['mac'],
-                )
+            interface = self.get_netbox_network_card(nic)
             # if network doesn't exist we create it
             if not interface:
-                new_interface = self.create_netbox_nic(device, nic)
+                new_interface = self.create_netbox_nic(nic)
                 if nic['ip']:
                     # for each ip, we try to find it
                     # assign the device's interface to it
                     # or simply create it
                     for ip in nic['ip']:
-                        netbox_ip = nb.ipam.ip_addresses.get(
-                            address=ip,
-                        )
-                        if netbox_ip:
-                            logging.info('Assigning existing IP {ip} to {interface}'.format(
-                                ip=ip, interface=new_interface))
-                            netbox_ip.interface = new_interface
-                            netbox_ip.save()
-                        else:
-                            logging.info('Create new IP {ip} on {interface}'.format(
-                                ip=ip, interface=new_interface))
-                            netbox_ip = nb.ipam.ip_addresses.create(
-                                address=ip,
-                                interface=new_interface.id,
-                                status=1,
-                            )
+                        self.create_or_update_netbox_ip_on_interface(ip, new_interface)
+        self._set_bonding_interfaces()
+        self.create_or_update_ipmi()
         logging.debug('Finished creating NIC!')
 
     def update_netbox_network_cards(self):
         logging.debug('Updating NIC...')
-        device = self.server.get_netbox_server()
+
+        # delete unknown interface
+        nb_nics = self.get_netbox_network_cards()
+        local_nics = [x['name'] for x in self.nics]
+        for nic in nb_nics:
+            if nic.name not in local_nics:
+                logging.info('Deleting netbox interface {name} because not present locally'.format(
+                    name=nic.name
+                ))
+                nic.delete()
 
         # delete IP on netbox that are not known on this server
         netbox_ips = nb.ipam.ip_addresses.filter(
-            device=device
+            device_id=self.device.id,
+            interface_id=[x.id for x in nb_nics],
         )
         all_local_ips = list(chain.from_iterable([
             x['ip'] for x in self.nics if x['ip'] is not None
@@ -144,14 +324,12 @@ class Network():
 
         # update each nic
         for nic in self.nics:
-            interface = nb.dcim.interfaces.get(
-                mac_address=nic['mac'],
-                )
+            interface = self.get_netbox_network_card(nic)
             if not interface:
-                logging.info('Interface {} not found, creating..'.format(
+                logging.info('Interface {mac_address} not found, creating..'.format(
                     mac_address=nic['mac'])
                 )
-                interface = self.create_netbox_nic(device, nic)
+                interface = self.create_netbox_nic(nic)
 
             nic_update = False
             if nic['name'] != interface.name:
@@ -160,32 +338,34 @@ class Network():
                     interface=interface, name=nic['name']))
                 interface.name = nic['name']
 
+            nic_update, interface = self.reset_vlan_on_interface(nic['vlan'], interface)
+
+            type = self.get_netbox_type_for_nic(nic)
+            if not interface.type or \
+               type != interface.type.value:
+                logging.info('Interface type is wrong, resetting')
+                nic_update = True
+                interface.type = type
+
+            if interface.lag is not None:
+                local_lag_int = next(
+                    item for item in self.nics if item['name'] == interface.lag.name
+                )
+                if nic['name'] not in local_lag_int['bonding_slaves']:
+                    logging.info('Interface has no LAG, resetting')
+                    nic_update = True
+                    interface.lag = None
+
             if nic['ip']:
                 # sync local IPs
                 for ip in nic['ip']:
                     netbox_ip = nb.ipam.ip_addresses.get(
                         address=ip,
                     )
-                    if not netbox_ip:
-                        # create netbbox_ip on device
-                        netbox_ip = nb.ipam.ip_addresses.create(
-                            address=ip,
-                            interface=interface.id,
-                            status=1,
-                        )
-                        logging.info('Created new IP {ip} on {interface}'.format(
-                            ip=ip, interface=interface))
-                    else:
-                        if netbox_ip.interface.id != interface.id:
-                            logging.info(
-                                'Detected interface change: old interface is {old_interface} '
-                                '(id: {old_id}), new interface is {new_interface} (id: {new_id})'
-                                .format(
-                                    old_interface=netbox_ip.interface, new_interface=interface,
-                                    old_id=netbox_ip.id, new_id=interface.id
-                                ))
-                            netbox_ip.interface = interface
-                            netbox_ip.save()
+                    self.create_or_update_netbox_ip_on_interface(ip, interface)
             if nic_update:
                 interface.save()
+
+        self._set_bonding_interfaces()
+        self.create_or_update_ipmi()
         logging.debug('Finished updating NIC!')

netbox_agent/server.py

@@ -17,7 +17,7 @@ class ServerBase():
         self.system = self.dmi.get_by_type('System')
         self.bios = self.dmi.get_by_type('BIOS')
 
-        self.network = Network(server=self)
+        self.network = None
 
     def get_datacenter(self):
         dc = Datacenter()
@@ -198,6 +198,7 @@ class ServerBase():
             if not server:
                 self._netbox_create_server(datacenter)
 
+        self.network = Network(server=self)
         self.network.create_netbox_network_cards()
         logging.debug('Server created!')
 
@@ -264,6 +265,7 @@ class ServerBase():
             update = True
             server.hostname = self.get_hostname()
         # check network cards
+        self.network = Network(server=self)
         self.network.update_netbox_network_cards()
         if update:
             server.save()