From 79ebcdb323343b8b50aaa78f0c7b5560e32f42be Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Wed, 7 Aug 2019 17:42:36 +0200 Subject: [PATCH 01/14] rework network part by ignoring devices and ip from config file --- netbox_agent/config.py | 6 +++ netbox_agent/network.py | 91 ++++++++++++++++++++++++++++++----------- netbox_agent/server.py | 4 +- 3 files changed, 76 insertions(+), 25 deletions(-) diff --git a/netbox_agent/config.py b/netbox_agent/config.py index 6d24c48..941a83c 100644 --- a/netbox_agent/config.py +++ b/netbox_agent/config.py @@ -30,3 +30,9 @@ if config.get('rack_location'): RACK_LOCATION_DRIVER_FILE = rack_location.get('driver_file') RACK_LOCATION = rack_location.get('driver') RACK_LOCATION_REGEX = rack_location.get('regex') + +NETWORK_IGNORE_INTERFACES = None +NETWORK_IGNORE_IPS = None +if config.get('network'): + NETWORK_IGNORE_INTERFACES = config['network']['ignore_interfaces'] + NETWORK_IGNORE_IPS = config['network']['ignore_ips'] diff --git a/netbox_agent/network.py b/netbox_agent/network.py index 07ec6c0..8078d35 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -7,6 +7,7 @@ from netaddr import IPAddress import netifaces from netbox_agent.config import netbox_instance as nb +from netbox_agent.config import NETWORK_IGNORE_INTERFACES, NETWORK_IGNORE_IPS from netbox_agent.ethtool import Ethtool IFACE_TYPE_100ME_FIXED = 800 @@ -33,10 +34,7 @@ IFACE_TYPE_200GE_CFP2 = 1650 IFACE_TYPE_200GE_QSFP56 = 1700 IFACE_TYPE_400GE_QSFP_DD = 1750 IFACE_TYPE_OTHER = 32767 - -# Regex to match base interface name -# Doesn't match vlan interfaces and other loopback etc -INTERFACE_REGEX = re.compile('^(eth[0-9]+|ens[0-9]+|enp[0-9]+s[0-9]f[0-9])$') +IFACE_TYPE_LAG = 200 class Network(): @@ -44,29 +42,69 @@ class Network(): self.nics = [] self.server = server + self.device = self.server.get_netbox_server() self.scan() def scan(self): for interface in os.listdir('/sys/class/net/'): - if re.match(INTERFACE_REGEX, interface): + if NETWORK_IGNORE_INTERFACES and \ + re.match(NETWORK_IGNORE_INTERFACES, interface): + logging.debug('Ignore interface {interface}'.format(interface=interface)) + continue + else: ip_addr = netifaces.ifaddresses(interface).get(netifaces.AF_INET) + mac = open('/sys/class/net/{}/address'.format(interface), 'r').read().strip() + vlan = None + if len(interface.split('.')) > 1: + vlan = int(interface.split('.')[1]) + bonding = False + bonding_slaves = [] + if os.path.isdir('/sys/class/net/{}/bonding'.format(interface)): + bonding = True + bonding_slaves = open( + '/sys/class/net/{}/bonding/slaves'.format(interface) + ).read().split() nic = { 'name': interface, - 'mac': open('/sys/class/net/{}/address'.format(interface), 'r').read().strip(), + 'mac': mac if mac != '00:00:00:00:00:00' else None, 'ip': [ '{}/{}'.format( x['addr'], IPAddress(x['netmask']).netmask_bits() - ) for x in ip_addr + ) for x in ip_addr if not re.match(NETWORK_IGNORE_IPS, x['addr']) ] if ip_addr else None, # FIXME: handle IPv6 addresses - 'ethtool': Ethtool(interface).parse() + 'ethtool': Ethtool(interface).parse(), + 'vlan': vlan, + 'bonding': bonding, + 'bonding_slaves': bonding_slaves, } self.nics.append(nic) def get_network_cards(self): return self.nics + def get_netbox_network_card(self, nic): + if nic['mac'] is None: + interface = nb.dcim.interfaces.get( + device_id=self.device.id, + name=nic['name'], + ) + else: + interface = nb.dcim.interfaces.get( + device_id=self.device.id, + mac_address=nic['mac'], + name=nic['name'], + ) + return interface + + def get_netbox_network_cards(self): + return nb.dcim.interfaces.filter( + device_id=self.device.id, + ) + def get_netbox_type_for_nic(self, nic): + if nic['bonding']: + return IFACE_TYPE_LAG if nic.get('ethtool') is None: return IFACE_TYPE_OTHER if nic['ethtool']['speed'] == '10000Mb/s': @@ -79,28 +117,26 @@ class Network(): return IFACE_TYPE_1GE_FIXED return IFACE_TYPE_OTHER - def create_netbox_nic(self, device, nic): + def create_netbox_nic(self, nic): # TODO: add Optic Vendor, PN and Serial type = self.get_netbox_type_for_nic(nic) logging.info('Creating NIC {name} ({mac}) on {device}'.format( - name=nic['name'], mac=nic['mac'], device=device.name)) + name=nic['name'], mac=nic['mac'], device=self.device.name)) return nb.dcim.interfaces.create( - device=device.id, + device=self.device.id, name=nic['name'], mac_address=nic['mac'], type=type, + mode=200 if nic['vlan'] else None, ) def create_netbox_network_cards(self): logging.debug('Creating NIC...') - device = self.server.get_netbox_server() for nic in self.nics: - interface = nb.dcim.interfaces.get( - mac_address=nic['mac'], - ) + interface = self.get_netbox_network_card(nic) # if network doesn't exist we create it if not interface: - new_interface = self.create_netbox_nic(device, nic) + new_interface = self.create_netbox_nic(nic) if nic['ip']: # for each ip, we try to find it # assign the device's interface to it @@ -126,11 +162,20 @@ class Network(): def update_netbox_network_cards(self): logging.debug('Updating NIC...') - device = self.server.get_netbox_server() + + # delete unknown interface + nb_nics = self.get_netbox_network_cards() + local_nics = [x['name'] for x in self.nics] + for nic in nb_nics: + if nic.name not in local_nics: + logging.info('Deleting netbox interface {name} because not present locally'.format( + name=nic.name + )) + nic.delete() # delete IP on netbox that are not known on this server netbox_ips = nb.ipam.ip_addresses.filter( - device=device + device_id=self.device.id ) all_local_ips = list(chain.from_iterable([ x['ip'] for x in self.nics if x['ip'] is not None @@ -144,14 +189,12 @@ class Network(): # update each nic for nic in self.nics: - interface = nb.dcim.interfaces.get( - mac_address=nic['mac'], - ) + interface = self.get_netbox_network_card(nic) if not interface: - logging.info('Interface {} not found, creating..'.format( + logging.info('Interface {mac_address} not found, creating..'.format( mac_address=nic['mac']) ) - interface = self.create_netbox_nic(device, nic) + interface = self.create_netbox_nic(nic) nic_update = False if nic['name'] != interface.name: @@ -167,7 +210,7 @@ class Network(): address=ip, ) if not netbox_ip: - # create netbbox_ip on device + # create netbox_ip on device netbox_ip = nb.ipam.ip_addresses.create( address=ip, interface=interface.id, diff --git a/netbox_agent/server.py b/netbox_agent/server.py index ba27b29..cdb8f2f 100644 --- a/netbox_agent/server.py +++ b/netbox_agent/server.py @@ -17,7 +17,7 @@ class ServerBase(): self.system = self.dmi.get_by_type('System') self.bios = self.dmi.get_by_type('BIOS') - self.network = Network(server=self) + self.network = None def get_datacenter(self): dc = Datacenter() @@ -198,6 +198,7 @@ class ServerBase(): if not server: self._netbox_create_server(datacenter) + self.network = Network(server=self) self.network.create_netbox_network_cards() logging.debug('Server created!') @@ -264,6 +265,7 @@ class ServerBase(): update = True server.hostname = self.get_hostname() # check network cards + self.network = Network(server=self) self.network.update_netbox_network_cards() if update: server.save() -- 2.47.0 From 6538493fd5019b544d80eb13b9a2276af6424b92 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Wed, 7 Aug 2019 17:48:12 +0200 Subject: [PATCH 02/14] update config file --- README.md | 22 ++++++++++++++++++---- netbox_agent.yaml.example | 4 ++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c90182c..96a2169 100644 --- a/README.md +++ b/README.md @@ -26,12 +26,26 @@ netbox: url: 'http://netbox.internal.company.com' token: supersecrettoken +network: + ignore_interfaces: "(dummy.*|docker.*)" + ignore_ips: (127\.0\.0\..*) + datacenter_location: - # driver_file: /opt/netbox_driver_dc.py - driver: file:/etc/qualification - regex: "datacenter: (?P[A-Za-z0-9]+)" + driver: "cmd:cat /etc/qualification | tr [a-z] [A-Z]" + regex: "DATACENTER: (?P[A-Za-z0-9]+)" # driver: 'cmd:lldpctl' -# regex = 'SysName: .*\.(?P[A-Za-z0-9]+)'``` +# regex: 'SysName: .*\.([A-Za-z0-9]+)' +# +# driver: "file:/tmp/datacenter" +# regex: "(.*)" + +rack_location: +# driver: 'cmd:lldpctl' +# match SysName: sw-dist-a1.dc42 +# regex: 'SysName:[ ]+[A-Za-z]+-[A-Za-z]+-([A-Za-z0-9]+)' +# +# driver: "file:/tmp/datacenter" +# regex: "(.*)" ``` # Hardware diff --git a/netbox_agent.yaml.example b/netbox_agent.yaml.example index cca08ef..fe42b05 100644 --- a/netbox_agent.yaml.example +++ b/netbox_agent.yaml.example @@ -2,6 +2,10 @@ netbox: url: 'http://netbox.internal.company.com' token: supersecrettoken +network: + ignore_interfaces: "(dummy.*|docker.*)" + ignore_ips: (127\.0\.0\..*) + datacenter_location: driver: "cmd:cat /etc/qualification | tr [a-z] [A-Z]" regex: "DATACENTER: (?P[A-Za-z0-9]+)" -- 2.47.0 From f1a1e73aa3898e4a47f8d1752da18ac262aa0372 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Wed, 7 Aug 2019 18:16:52 +0200 Subject: [PATCH 03/14] associate slave device to bond device with Netbox LAG --- netbox_agent/network.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/netbox_agent/network.py b/netbox_agent/network.py index 8078d35..468efb8 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -47,6 +47,10 @@ class Network(): def scan(self): for interface in os.listdir('/sys/class/net/'): + # ignore if it's not a link (ie: bonding_masters etc) + if not os.path.islink('/sys/class/net/{}'.format(interface)): + continue + if NETWORK_IGNORE_INTERFACES and \ re.match(NETWORK_IGNORE_INTERFACES, interface): logging.debug('Ignore interface {interface}'.format(interface=interface)) @@ -80,6 +84,22 @@ class Network(): } self.nics.append(nic) + def _set_bonding_interfaces(self): + logging.debug('Setting bonding interfaces..') + for nic in [x for x in self.nics if x['bonding']]: + bond_int = self.get_netbox_network_card(nic) + logging.debug('Setting slave interface for {name}'.format( + name=bond_int.name + )) + for slave in nic['bonding_slaves']: + slave_nic = next(item for item in self.nics if item['name'] == slave) + slave_int = self.get_netbox_network_card(slave_nic) + logging.debug('Settting interface {name} as slave of {master}'.format( + name=slave_int.name, master=bond_int.name + )) + slave_int.lag = bond_int + slave_int.save() + def get_network_cards(self): return self.nics @@ -158,6 +178,7 @@ class Network(): interface=new_interface.id, status=1, ) + self._set_bonding_interfaces() logging.debug('Finished creating NIC!') def update_netbox_network_cards(self): @@ -231,4 +252,6 @@ class Network(): netbox_ip.save() if nic_update: interface.save() + + self._set_bonding_interfaces() logging.debug('Finished updating NIC!') -- 2.47.0 From 1988bd0f9c09acb19e98d5753aa8f9cfb39bddfd Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Wed, 7 Aug 2019 18:56:39 +0200 Subject: [PATCH 04/14] update README --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 96a2169..7a0f804 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,18 @@ The goal is to generate an existing infrastructure on Netbox and have the abilit # Features * Create servers, chassis and blade through standard tools (`dmidecode`) -* Create physical network interfaces with IPs +* Create physical, bonding and vlan network interfaces with IPs * Generic ability to guess datacenters and rack location through drivers (`cmd` and `file` and custom ones) * Update existing `Device` and `Interfaces` * Handle blade moving (new slot, new chassis) + # Known limitations * The project is only compatible with Linux. Since it uses `ethtool` and parses `/sys/` directory, it's not compatible with *BSD distributions. * Netbox `>=2.6.0,<=2.6.2` has a caching problem ; if the cache lifetime is too high, the script can get stale data after modification. +We advise to set `CACHE_TIME` to `0`. # Configuration -- 2.47.0 From ab660b4fcf3c14364aef4c25025d11ce84bead59 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Wed, 7 Aug 2019 19:09:09 +0200 Subject: [PATCH 05/14] make sure every interface attribute is ok --- netbox_agent/network.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/netbox_agent/network.py b/netbox_agent/network.py index 468efb8..4326991 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -93,7 +93,7 @@ class Network(): )) for slave in nic['bonding_slaves']: slave_nic = next(item for item in self.nics if item['name'] == slave) - slave_int = self.get_netbox_network_card(slave_nic) + slave_int = self.get_netbox_network_card(slave_nic) logging.debug('Settting interface {name} as slave of {master}'.format( name=slave_int.name, master=bond_int.name )) @@ -224,6 +224,23 @@ class Network(): interface=interface, name=nic['name'])) interface.name = nic['name'] + if nic['vlan'] is None and interface.mode is not None: + logging.info('Interface is not tagged, reseting mode') + nic_update = True + interface.mode = None + + type = self.get_netbox_type_for_nic(nic) + if type != interface.type.value: + logging.info('Interface type is wrong, resetting') + nic_update = True + interface.type = type + + if interface.lag is not None: + if nic['name'] not in self.nics[interface.lag.name]['bonding_slaves']: + logging.info('Interface has no LAG, resetting') + nic_update = True + interface.lag = None + if nic['ip']: # sync local IPs for ip in nic['ip']: -- 2.47.0 From a68bc10f38c89099febefb31806691307e689ff5 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Thu, 8 Aug 2019 15:09:43 +0200 Subject: [PATCH 06/14] some fixes after test --- netbox_agent/network.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/netbox_agent/network.py b/netbox_agent/network.py index 4326991..6935a3a 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -57,6 +57,11 @@ class Network(): continue else: ip_addr = netifaces.ifaddresses(interface).get(netifaces.AF_INET) + if NETWORK_IGNORE_IPS and ip_addr: + for i, ip in enumerate(ip_addr): + if re.match(NETWORK_IGNORE_IPS, ip['addr']): + ip_addr.pop(i) + mac = open('/sys/class/net/{}/address'.format(interface), 'r').read().strip() vlan = None if len(interface.split('.')) > 1: @@ -75,7 +80,7 @@ class Network(): '{}/{}'.format( x['addr'], IPAddress(x['netmask']).netmask_bits() - ) for x in ip_addr if not re.match(NETWORK_IGNORE_IPS, x['addr']) + ) for x in ip_addr ] if ip_addr else None, # FIXME: handle IPv6 addresses 'ethtool': Ethtool(interface).parse(), 'vlan': vlan, @@ -230,7 +235,8 @@ class Network(): interface.mode = None type = self.get_netbox_type_for_nic(nic) - if type != interface.type.value: + if not interface.type or \ + type != interface.type.value: logging.info('Interface type is wrong, resetting') nic_update = True interface.type = type @@ -257,13 +263,14 @@ class Network(): logging.info('Created new IP {ip} on {interface}'.format( ip=ip, interface=interface)) else: - if netbox_ip.interface.id != interface.id: + if not netbox_ip.interface or \ + netbox_ip.interface.id != interface.id: logging.info( - 'Detected interface change: old interface is {old_interface} ' + 'Detected interface change for ip {ip}: old interface is {old_interface} ' '(id: {old_id}), new interface is {new_interface} (id: {new_id})' .format( old_interface=netbox_ip.interface, new_interface=interface, - old_id=netbox_ip.id, new_id=interface.id + old_id=netbox_ip.id, new_id=interface.id, ip=netbox_ip.address )) netbox_ip.interface = interface netbox_ip.save() -- 2.47.0 From 9e9b1d2199453b44adab90245d006ca5e7c95255 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Thu, 8 Aug 2019 16:00:34 +0200 Subject: [PATCH 07/14] tox --- netbox_agent/network.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/netbox_agent/network.py b/netbox_agent/network.py index 6935a3a..6dee433 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -266,8 +266,9 @@ class Network(): if not netbox_ip.interface or \ netbox_ip.interface.id != interface.id: logging.info( - 'Detected interface change for ip {ip}: old interface is {old_interface} ' - '(id: {old_id}), new interface is {new_interface} (id: {new_id})' + 'Detected interface change for ip {ip}: old interface is ' + '{old_interface} (id: {old_id}), new interface is {new_interface} ' + ' (id: {new_id})' .format( old_interface=netbox_ip.interface, new_interface=interface, old_id=netbox_ip.id, new_id=interface.id, ip=netbox_ip.address -- 2.47.0 From 21c79e5b435e79db425951a4ed8ee9e9738c9c4f Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Thu, 8 Aug 2019 17:56:23 +0200 Subject: [PATCH 08/14] add ipmi feature --- netbox_agent/network.py | 60 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/netbox_agent/network.py b/netbox_agent/network.py index 6935a3a..5b8a141 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -3,12 +3,13 @@ import logging import os import re -from netaddr import IPAddress +from netaddr import IPAddress, IPNetwork import netifaces from netbox_agent.config import netbox_instance as nb from netbox_agent.config import NETWORK_IGNORE_INTERFACES, NETWORK_IGNORE_IPS from netbox_agent.ethtool import Ethtool +from netbox_agent.ipmi import Ipmi IFACE_TYPE_100ME_FIXED = 800 IFACE_TYPE_1GE_FIXED = 1000 @@ -142,6 +143,57 @@ class Network(): return IFACE_TYPE_1GE_FIXED return IFACE_TYPE_OTHER + def get_ipmi(self): + ipmi = Ipmi().parse() + return ipmi + + def get_netbox_ipmi(self): + ipmi = self.get_ipmi() + mac = ipmi['MAC Address'] + return nb.dcim.interfaces.get( + mac=mac + ) + + def create_or_update_ipmi(self): + ipmi = self.get_ipmi() + mac = ipmi['MAC Address'] + ip = ipmi['IP Address'] + netmask = ipmi['Subnet Mask'] + vlan = ipmi['802.1q VLAN ID'] if ipmi['802.1q VLAN ID'] != 'Disabled' else None + + address = IPNetwork('{}/{}'.format(ip, netmask)).__str__() + + device = nb.dcim.interfaces.get( + device_id=self.device.id, + mgmt_only=True, + ) + if device is None: + return nb.dcim.interfaces.create( + device=self.device.id, + name='IPMI', + mac_address=ipmi['MAC Address'], + address=address, + mode=200 if vlan else None, + type=IFACE_TYPE_1GE_FIXED, + mgmt_only=True, + ) + else: + # let the user chose the name of mgmt ? + # guess it with manufacturer (IDRAC, ILO, ...) ? + update = False + if device.address != address: + device.address = address + update = True + if vlan and device.mode != 200: + device.mode = 200 + update = True + if mac != device.mac_address: + device.mac_address = mac + update = True + if update: + device.save() + return device + def create_netbox_nic(self, nic): # TODO: add Optic Vendor, PN and Serial type = self.get_netbox_type_for_nic(nic) @@ -266,8 +318,9 @@ class Network(): if not netbox_ip.interface or \ netbox_ip.interface.id != interface.id: logging.info( - 'Detected interface change for ip {ip}: old interface is {old_interface} ' - '(id: {old_id}), new interface is {new_interface} (id: {new_id})' + 'Detected interface change for ip {ip}: old interface is ' + '{old_interface} (id: {old_id}), new interface is {new_interface} ' + ' (id: {new_id})' .format( old_interface=netbox_ip.interface, new_interface=interface, old_id=netbox_ip.id, new_id=interface.id, ip=netbox_ip.address @@ -278,4 +331,5 @@ class Network(): interface.save() self._set_bonding_interfaces() + self.create_or_update_ipmi() logging.debug('Finished updating NIC!') -- 2.47.0 From 98643ee4ac7a584a5fb5a882461a470cbc6a98eb Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Thu, 8 Aug 2019 17:56:55 +0200 Subject: [PATCH 09/14] fix bug in LAG --- netbox_agent/network.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/netbox_agent/network.py b/netbox_agent/network.py index 5b8a141..b557512 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -294,7 +294,8 @@ class Network(): interface.type = type if interface.lag is not None: - if nic['name'] not in self.nics[interface.lag.name]['bonding_slaves']: + local_lag_int = next(item for item in self.nics if item['name'] == interface.lag.name) + if nic['name'] not in local_lag_int['bonding_slaves']: logging.info('Interface has no LAG, resetting') nic_update = True interface.lag = None -- 2.47.0 From 93ab0159b6389cf55cda929de3a51019d206c483 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Thu, 8 Aug 2019 17:59:02 +0200 Subject: [PATCH 10/14] ipmi class --- netbox_agent/ipmi.py | 49 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 netbox_agent/ipmi.py diff --git a/netbox_agent/ipmi.py b/netbox_agent/ipmi.py new file mode 100644 index 0000000..f747712 --- /dev/null +++ b/netbox_agent/ipmi.py @@ -0,0 +1,49 @@ +import logging +import subprocess + +class Ipmi(): + """ + Parse IPMI output + ie: + + Set in Progress : Set Complete + Auth Type Support : + Auth Type Enable : Callback : + : User : + : Operator : + : Admin : + : OEM : + IP Address Source : DHCP Address + IP Address : 10.192.2.1 + Subnet Mask : 255.255.240.0 + MAC Address : 98:f2:b3:f0:ee:1e + SNMP Community String : + BMC ARP Control : ARP Responses Enabled, Gratuitous ARP Disabled + Default Gateway IP : 10.192.2.254 + 802.1q VLAN ID : Disabled + 802.1q VLAN Priority : 0 + RMCP+ Cipher Suites : 0,1,2,3 + Cipher Suite Priv Max : XuuaXXXXXXXXXXX + : X=Cipher Suite Unused + : c=CALLBACK + : u=USER + : o=OPERATOR + : a=ADMIN + : O=OEM + Bad Password Threshold : Not Available + """ + def __init__(self): + self.ret, self.output = subprocess.getstatusoutput('ipmitool lan print') + if not self.ret: + logging.error('Cannot get ipmi info: {}'.format(self.output)) + + + def parse(self): + ret = {} + if self.ret != 0: + return ret + for line in self.output.split('\n'): + key = line.split(':')[0] + value = ':'.join(line.split(':')[1:]) + ret[key] = value + return ret -- 2.47.0 From d2d7ca6587d5462589d40040a76413051b6d804c Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Thu, 8 Aug 2019 18:23:53 +0200 Subject: [PATCH 11/14] update README --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 7a0f804..c32b5c7 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,17 @@ The goal is to generate an existing infrastructure on Netbox and have the abilit * Create servers, chassis and blade through standard tools (`dmidecode`) * Create physical, bonding and vlan network interfaces with IPs +* Create IPMI interface if found * Generic ability to guess datacenters and rack location through drivers (`cmd` and `file` and custom ones) * Update existing `Device` and `Interfaces` * Handle blade moving (new slot, new chassis) +# Requirements + +- Netbox >= 2.6 +- Python >= 3.4 +- [python3-netaddr](https://github.com/drkjam/netaddr) +- [python3-netifaces](https://github.com/al45tair/netifaces) # Known limitations -- 2.47.0 From f7809eb3be6efdf3f11ab82fea47b1cc27067ad9 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Fri, 9 Aug 2019 12:00:48 +0200 Subject: [PATCH 12/14] network update --- netbox_agent/ipmi.py | 8 +- netbox_agent/network.py | 175 ++++++++++++++++++++++++---------------- 2 files changed, 109 insertions(+), 74 deletions(-) diff --git a/netbox_agent/ipmi.py b/netbox_agent/ipmi.py index f747712..85b1c6c 100644 --- a/netbox_agent/ipmi.py +++ b/netbox_agent/ipmi.py @@ -1,6 +1,7 @@ import logging import subprocess + class Ipmi(): """ Parse IPMI output @@ -34,16 +35,15 @@ class Ipmi(): """ def __init__(self): self.ret, self.output = subprocess.getstatusoutput('ipmitool lan print') - if not self.ret: + if self.ret != 0: logging.error('Cannot get ipmi info: {}'.format(self.output)) - def parse(self): ret = {} if self.ret != 0: return ret for line in self.output.split('\n'): - key = line.split(':')[0] - value = ':'.join(line.split(':')[1:]) + key = line.split(':')[0].strip() + value = ':'.join(line.split(':')[1:]).strip() ret[key] = value return ret diff --git a/netbox_agent/network.py b/netbox_agent/network.py index b557512..f1ad80b 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -91,8 +91,11 @@ class Network(): self.nics.append(nic) def _set_bonding_interfaces(self): + bonding_nics = [x for x in self.nics if x['bonding']] + if not len(bonding_nics): + return False logging.debug('Setting bonding interfaces..') - for nic in [x for x in self.nics if x['bonding']]: + for nic in bonding_nics: bond_int = self.get_netbox_network_card(nic) logging.debug('Setting slave interface for {name}'.format( name=bond_int.name @@ -105,6 +108,7 @@ class Network(): )) slave_int.lag = bond_int slave_int.save() + return True def get_network_cards(self): return self.nics @@ -126,10 +130,11 @@ class Network(): def get_netbox_network_cards(self): return nb.dcim.interfaces.filter( device_id=self.device.id, + mgmt_only=False, ) def get_netbox_type_for_nic(self, nic): - if nic['bonding']: + if nic.get('bonding'): return IFACE_TYPE_LAG if nic.get('ethtool') is None: return IFACE_TYPE_OTHER @@ -154,59 +159,124 @@ class Network(): mac=mac ) + def get_or_create_vlan(self, vlan_id): + # FIXME: we may need to specify the datacenter + # since users may have same vlan id in multiple dc + vlan = nb.ipam.vlans.get( + vid=vlan_id, + ) + if vlan is None: + vlan = nb.ipam.vlans.create( + name='VLAN {}'.format(vlan_id), + vid=vlan_id, + ) + return vlan + + def reset_vlan_on_interface(self, vlan_id, interface): + update = False + if vlan_id is None and \ + (interface.mode is not None or len(interface.tagged_vlans) > 0): + logging.info('Interface {interface} is not tagged, reseting mode'.format( + interface=interface)) + update = True + interface.mode = None + interface.tagged_vlans = [] + elif vlan_id and ( + interface.mode is None or + len(interface.tagged_vlans) != 1 or + interface.tagged_vlans[0].vid != vlan_id): + logging.info('Resetting VLAN on interface {interface}'.format( + interface=interface)) + update = True + nb_vlan = self.get_or_create_vlan(vlan_id) + interface.mode = 200 + interface.tagged_vlans = [nb_vlan] if nb_vlan else [] + return update, interface + def create_or_update_ipmi(self): ipmi = self.get_ipmi() mac = ipmi['MAC Address'] ip = ipmi['IP Address'] netmask = ipmi['Subnet Mask'] - vlan = ipmi['802.1q VLAN ID'] if ipmi['802.1q VLAN ID'] != 'Disabled' else None - + vlan = int(ipmi['802.1q VLAN ID']) if ipmi['802.1q VLAN ID'] != 'Disabled' else None address = IPNetwork('{}/{}'.format(ip, netmask)).__str__() - device = nb.dcim.interfaces.get( + interface = nb.dcim.interfaces.get( device_id=self.device.id, mgmt_only=True, ) - if device is None: - return nb.dcim.interfaces.create( - device=self.device.id, - name='IPMI', - mac_address=ipmi['MAC Address'], - address=address, - mode=200 if vlan else None, - type=IFACE_TYPE_1GE_FIXED, - mgmt_only=True, - ) + nic = { + 'name': 'IPMI', + 'mac': mac, + 'vlan': vlan, + 'ip': [address], + } + if interface is None: + interface = self.create_netbox_nic(nic, mgmt=True) + self.create_or_update_netbox_ip_on_interface(address, interface) else: # let the user chose the name of mgmt ? # guess it with manufacturer (IDRAC, ILO, ...) ? update = False - if device.address != address: - device.address = address - update = True - if vlan and device.mode != 200: - device.mode = 200 - update = True - if mac != device.mac_address: - device.mac_address = mac + self.create_or_update_netbox_ip_on_interface(address, interface) + update, interface = self.reset_vlan_on_interface(nic['vlan'], interface) + if mac != interface.mac_address: + interface.mac_address = mac update = True if update: - device.save() - return device + interface.save() + return interface - def create_netbox_nic(self, nic): + def create_netbox_nic(self, nic, mgmt=False): # TODO: add Optic Vendor, PN and Serial type = self.get_netbox_type_for_nic(nic) logging.info('Creating NIC {name} ({mac}) on {device}'.format( name=nic['name'], mac=nic['mac'], device=self.device.name)) + + nb_vlan = None + if nic['vlan']: + nb_vlan = self.get_or_create_vlan(nic['vlan']) return nb.dcim.interfaces.create( device=self.device.id, name=nic['name'], mac_address=nic['mac'], type=type, mode=200 if nic['vlan'] else None, + tagged_vlans=[nb_vlan.id] if nb_vlan is not None else [], + mgmt_only=mgmt, ) + def create_or_update_netbox_ip_on_interface(self, ip, interface): + netbox_ip = nb.ipam.ip_addresses.get( + address=ip, + ) + if netbox_ip: + if netbox_ip.interface is None: + logging.info('Assigning existing IP {ip} to {interface}'.format( + ip=ip, interface=interface)) + elif netbox_ip.interface.id != interface.id: + logging.info( + 'Detected interface change for ip {ip}: old interface is ' + '{old_interface} (id: {old_id}), new interface is {new_interface} ' + ' (id: {new_id})' + .format( + old_interface=netbox_ip.interface, new_interface=interface, + old_id=netbox_ip.id, new_id=interface.id, ip=netbox_ip.address + )) + else: + return netbox_ip + netbox_ip.interface = interface + netbox_ip.save() + else: + logging.info('Create new IP {ip} on {interface}'.format( + ip=ip, interface=interface)) + netbox_ip = nb.ipam.ip_addresses.create( + address=ip, + interface=interface.id, + status=1, + ) + return netbox_ip + def create_netbox_network_cards(self): logging.debug('Creating NIC...') for nic in self.nics: @@ -219,23 +289,9 @@ class Network(): # assign the device's interface to it # or simply create it for ip in nic['ip']: - netbox_ip = nb.ipam.ip_addresses.get( - address=ip, - ) - if netbox_ip: - logging.info('Assigning existing IP {ip} to {interface}'.format( - ip=ip, interface=new_interface)) - netbox_ip.interface = new_interface - netbox_ip.save() - else: - logging.info('Create new IP {ip} on {interface}'.format( - ip=ip, interface=new_interface)) - netbox_ip = nb.ipam.ip_addresses.create( - address=ip, - interface=new_interface.id, - status=1, - ) + self.create_or_update_netbox_ip_on_interface(ip, new_interface) self._set_bonding_interfaces() + self.create_or_update_ipmi() logging.debug('Finished creating NIC!') def update_netbox_network_cards(self): @@ -253,7 +309,8 @@ class Network(): # delete IP on netbox that are not known on this server netbox_ips = nb.ipam.ip_addresses.filter( - device_id=self.device.id + device_id=self.device.id, + interface_id=[x.id for x in nb_nics], ) all_local_ips = list(chain.from_iterable([ x['ip'] for x in self.nics if x['ip'] is not None @@ -281,10 +338,7 @@ class Network(): interface=interface, name=nic['name'])) interface.name = nic['name'] - if nic['vlan'] is None and interface.mode is not None: - logging.info('Interface is not tagged, reseting mode') - nic_update = True - interface.mode = None + nic_update, interface = self.reset_vlan_on_interface(nic['vlan'], interface) type = self.get_netbox_type_for_nic(nic) if not interface.type or \ @@ -294,7 +348,9 @@ class Network(): interface.type = type if interface.lag is not None: - local_lag_int = next(item for item in self.nics if item['name'] == interface.lag.name) + local_lag_int = next( + item for item in self.nics if item['name'] == interface.lag.name + ) if nic['name'] not in local_lag_int['bonding_slaves']: logging.info('Interface has no LAG, resetting') nic_update = True @@ -306,28 +362,7 @@ class Network(): netbox_ip = nb.ipam.ip_addresses.get( address=ip, ) - if not netbox_ip: - # create netbox_ip on device - netbox_ip = nb.ipam.ip_addresses.create( - address=ip, - interface=interface.id, - status=1, - ) - logging.info('Created new IP {ip} on {interface}'.format( - ip=ip, interface=interface)) - else: - if not netbox_ip.interface or \ - netbox_ip.interface.id != interface.id: - logging.info( - 'Detected interface change for ip {ip}: old interface is ' - '{old_interface} (id: {old_id}), new interface is {new_interface} ' - ' (id: {new_id})' - .format( - old_interface=netbox_ip.interface, new_interface=interface, - old_id=netbox_ip.id, new_id=interface.id, ip=netbox_ip.address - )) - netbox_ip.interface = interface - netbox_ip.save() + self.create_or_update_netbox_ip_on_interface(ip, interface) if nic_update: interface.save() -- 2.47.0 From 38d417b43654be470429bbfd61a99779e913c2ec Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Fri, 9 Aug 2019 12:01:06 +0200 Subject: [PATCH 13/14] delete print --- netbox_agent/location.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/netbox_agent/location.py b/netbox_agent/location.py index e3f73d4..f01ece5 100644 --- a/netbox_agent/location.py +++ b/netbox_agent/location.py @@ -22,11 +22,9 @@ class LocationBase(): self.driver = driver self.driver_value = driver_value self.driver_file = driver_file - print(self.driver_file) self.regex = regex if self.driver_file: - print('if', self.driver_file) try: # FIXME: Works with Python 3.3+, support older version? loader = importlib.machinery.SourceFileLoader('driver_file', self.driver_file) -- 2.47.0 From 0e456669c0bc02580be173f9bc1cc83af14192b2 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Fri, 9 Aug 2019 12:03:10 +0200 Subject: [PATCH 14/14] update README --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 7a0f804..84c5fc1 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,18 @@ The goal is to generate an existing infrastructure on Netbox and have the abilit * Create servers, chassis and blade through standard tools (`dmidecode`) * Create physical, bonding and vlan network interfaces with IPs +* Create IPMI interface if found +* Create or get existing VLAN and associate it to interfaces * Generic ability to guess datacenters and rack location through drivers (`cmd` and `file` and custom ones) * Update existing `Device` and `Interfaces` * Handle blade moving (new slot, new chassis) +# Requirements + +- Netbox >= 2.6 +- Python >= 3.4 +- [python3-netaddr](https://github.com/drkjam/netaddr) +- [python3-netifaces](https://github.com/al45tair/netifaces) # Known limitations -- 2.47.0