From 1b9bf467626cfbabb47297c13f7ba10b09d05d7e Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Fri, 23 Aug 2019 15:00:59 +0200 Subject: [PATCH 1/8] Add LLDP auto cabling --- netbox_agent/config.py | 2 + netbox_agent/lldp.py | 42 +++++++++++++++ netbox_agent/network.py | 114 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 netbox_agent/lldp.py diff --git a/netbox_agent/config.py b/netbox_agent/config.py index 2590582..9b34c41 100644 --- a/netbox_agent/config.py +++ b/netbox_agent/config.py @@ -33,6 +33,8 @@ if config.get('rack_location'): NETWORK_IGNORE_INTERFACES = None NETWORK_IGNORE_IPS = None +NETWORK_LLDP = None if config.get('network'): NETWORK_IGNORE_INTERFACES = config['network'].get('ignore_interfaces') NETWORK_IGNORE_IPS = config['network'].get('ignore_ips') + NETWORK_LLDP = config['network'].get('lldp') is True diff --git a/netbox_agent/lldp.py b/netbox_agent/lldp.py new file mode 100644 index 0000000..770a8f6 --- /dev/null +++ b/netbox_agent/lldp.py @@ -0,0 +1,42 @@ +import subprocess + + +class LLDP(): + def __init__(self): + self.output = subprocess.getoutput('lldpctl -f keyvalue') + self.data = self.parse() + + def parse(self): + output_dict = {} + for entry in self.output.splitlines(): + path, value = entry.strip().split("=", 1) + path = path.split(".") + path_components, final = path[:-1], path[-1] + + current_dict = output_dict + for path_component in path_components: + current_dict[path_component] = current_dict.get(path_component, {}) + current_dict = current_dict[path_component] + current_dict[final] = value + return output_dict + + def get_switch_ip(self, interface): + # lldp.eth0.chassis.mgmt-ip=100.66.7.222 + if self.data['lldp'].get(interface) is None: + return None + return self.data['lldp'][interface]['chassis']['mgmt-ip'] + + def get_switch_port(self, interface): + # lldp.eth0.port.descr=GigabitEthernet1/0/1 + if self.data['lldp'].get(interface) is None: + return None + return self.data['lldp'][interface]['port']['descr'] + + def get_switch_vlan(self, interface): + # lldp.eth0.vlan.vlan-id=296 + if self.data['lldp'].get(interface) is None: + return None + + if self.data['lldp'][interface].get('vlan'): + return int(self.data['lldp'][interface]['vlan'].replace('vlan-', '')) + return None diff --git a/netbox_agent/network.py b/netbox_agent/network.py index 875ce00..200f073 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -7,9 +7,10 @@ from netaddr import IPAddress, IPNetwork import netifaces from netbox_agent.config import netbox_instance as nb -from netbox_agent.config import NETWORK_IGNORE_INTERFACES, NETWORK_IGNORE_IPS +from netbox_agent.config import NETWORK_IGNORE_INTERFACES, NETWORK_IGNORE_IPS, NETWORK_LLDP from netbox_agent.ethtool import Ethtool from netbox_agent.ipmi import IPMI +from netbox_agent.lldp import LLDP IFACE_TYPE_100ME_FIXED = 800 IFACE_TYPE_1GE_FIXED = 1000 @@ -44,6 +45,7 @@ class Network(): self.server = server self.device = self.server.get_netbox_server() + self.lldp = LLDP() self.scan() def scan(self): @@ -297,6 +299,105 @@ class Network(): self.create_or_update_ipmi() logging.debug('Finished creating NIC!') + def connect_interface_to_switch(self, switch_ip, switch_interface, nb_server_interface): + logging.info('Interface {} is not connected to switch, trying to connect..'.format( + nb_server_interface.name + )) + nb_mgmt_ip = nb.ipam.ip_addresses.get( + address=switch_ip, + ) + if not nb_mgmt_ip: + logging.error('Switch IP {} cannot be found in Netbox'.format(switch_ip)) + return nb_server_interface + + try: + nb_switch = nb_mgmt_ip.interface.device + logging.info('Found a switch in Netbox based on LLDP infos: {} (id: {})'.format( + switch_ip, + nb_switch.id + )) + except KeyError: + logging.error( + 'Switch IP {} is found but not associated to a Netbox Switch Device'.format( + switch_ip + ) + ) + return nb_server_interface + + switch_interface = self.lldp.get_switch_port(nb_server_interface.name) + nb_switch_interface = nb.dcim.interfaces.get( + device=nb_switch, + name=switch_interface, + ) + if nb_switch_interface is None: + logging.error('Switch interface {} cannot be found'.format(switch_interface)) + return nb_server_interface + + logging.info('Found interface {} on switch {}'.format( + switch_interface, + switch_ip, + )) + cable = nb.dcim.cables.create( + termination_a_id=nb_server_interface.id, + termination_a_type="dcim.interface", + termination_b_id=nb_switch_interface.id, + termination_b_type="dcim.interface", + ) + nb_server_interface.cable = cable + logging.info( + 'Connected interface {interface} with {switch_interface} of {switch_ip}'.format( + interface=nb_server_interface.name, + switch_interface=switch_interface, + switch_ip=switch_ip, + ) + ) + return nb_server_interface + + def create_or_update_cable(self, switch_ip, switch_interface, nb_server_interface): + if nb_server_interface.cable is None: + update = True + nb_server_interface = self.connect_interface_to_switch( + switch_ip, switch_interface, nb_server_interface + ) + else: + update = False + nb_sw_int = nb_server_interface.cable.termination_b + nb_sw = nb_sw_int.device + nb_mgmt_int = nb.dcim.interfaces.get( + device_id=nb_sw.id, + mgmt_only=True + ) + nb_mgmt_ip = nb.ipam.ip_addresses.get( + interface_id=nb_mgmt_int.id + ) + if nb_mgmt_ip is None: + pass + + # Netbox IP is always IP/Netmask + nb_mgmt_ip = nb_mgmt_ip.address.split('/')[0] + if nb_mgmt_ip != switch_ip or \ + nb_sw_int.name != switch_interface: + logging.info('Netbox cable is not connected to correct ports, fixing..') + logging.info( + 'Deleting cable {cable_id} from {interface} to {switch_interface} of ' + '{switch_ip}'.format( + cable_id=nb_server_interface.cable.id, + interface=nb_server_interface.name, + switch_interface=nb_sw_int.name, + switch_ip=nb_mgmt_ip, + ) + ) + cable = nb.dcim.cables.get( + nb_server_interface.cable.id + ) + print(cable) + cable.delete() + update = True + nb_server_interface = self.connect_interface_to_switch( + switch_ip, switch_interface, nb_server_interface + ) + return update, nb_server_interface + def update_netbox_network_cards(self): logging.debug('Updating NIC...') @@ -359,12 +460,17 @@ class Network(): nic_update = True interface.lag = None + # cable the interface + if NETWORK_LLDP: + switch_ip = self.lldp.get_switch_ip(interface.name) + switch_interface = self.lldp.get_switch_port(interface.name) + nic_update, interface = self.create_or_update_cable( + switch_ip, switch_interface, interface + ) + if nic['ip']: # sync local IPs for ip in nic['ip']: - netbox_ip = nb.ipam.ip_addresses.get( - address=ip, - ) self.create_or_update_netbox_ip_on_interface(ip, interface) if nic_update: interface.save() -- 2.47.1 From 767fb2c5616bbb5bb9eb60c6106559e5e8f9eed9 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Fri, 23 Aug 2019 15:13:04 +0200 Subject: [PATCH 2/8] handle cabling while creating interface --- netbox_agent/network.py | 48 +++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/netbox_agent/network.py b/netbox_agent/network.py index 200f073..05c782f 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -241,7 +241,7 @@ class Network(): nb_vlan = None if nic['vlan']: nb_vlan = self.get_or_create_vlan(nic['vlan']) - return nb.dcim.interfaces.create( + interface = nb.dcim.interfaces.create( device=self.device.id, name=nic['name'], mac_address=nic['mac'], @@ -250,6 +250,17 @@ class Network(): tagged_vlans=[nb_vlan.id] if nb_vlan is not None else [], mgmt_only=mgmt, ) + # cable the interface + if NETWORK_LLDP: + switch_ip = self.lldp.get_switch_ip(interface.name) + switch_interface = self.lldp.get_switch_port(interface.name) + if switch_ip is not None and switch_interface is not None: + nic_update, interface = self.create_or_update_cable( + switch_ip, switch_interface, interface + ) + if nic_update: + interface.save() + return interface def create_or_update_netbox_ip_on_interface(self, ip, interface): netbox_ip = nb.ipam.ip_addresses.get( @@ -282,23 +293,6 @@ class Network(): ) return netbox_ip - def create_netbox_network_cards(self): - logging.debug('Creating NIC...') - for nic in self.nics: - interface = self.get_netbox_network_card(nic) - # if network doesn't exist we create it - if not interface: - new_interface = self.create_netbox_nic(nic) - if nic['ip']: - # for each ip, we try to find it - # assign the device's interface to it - # or simply create it - for ip in nic['ip']: - self.create_or_update_netbox_ip_on_interface(ip, new_interface) - self._set_bonding_interfaces() - self.create_or_update_ipmi() - logging.debug('Finished creating NIC!') - def connect_interface_to_switch(self, switch_ip, switch_interface, nb_server_interface): logging.info('Interface {} is not connected to switch, trying to connect..'.format( nb_server_interface.name @@ -390,7 +384,6 @@ class Network(): cable = nb.dcim.cables.get( nb_server_interface.cable.id ) - print(cable) cable.delete() update = True nb_server_interface = self.connect_interface_to_switch( @@ -398,6 +391,23 @@ class Network(): ) return update, nb_server_interface + def create_netbox_network_cards(self): + logging.debug('Creating NIC...') + for nic in self.nics: + interface = self.get_netbox_network_card(nic) + # if network doesn't exist we create it + if not interface: + new_interface = self.create_netbox_nic(nic) + if nic['ip']: + # for each ip, we try to find it + # assign the device's interface to it + # or simply create it + for ip in nic['ip']: + self.create_or_update_netbox_ip_on_interface(ip, new_interface) + self._set_bonding_interfaces() + self.create_or_update_ipmi() + logging.debug('Finished creating NIC!') + def update_netbox_network_cards(self): logging.debug('Updating NIC...') -- 2.47.1 From c9ddee73e8e4e8544153d327d66b6981995a82e5 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Fri, 23 Aug 2019 15:15:36 +0200 Subject: [PATCH 3/8] error handling in case switch mgmt interface doesnt have IP --- netbox_agent/network.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/netbox_agent/network.py b/netbox_agent/network.py index 05c782f..f8817b0 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -348,13 +348,13 @@ class Network(): return nb_server_interface def create_or_update_cable(self, switch_ip, switch_interface, nb_server_interface): + update = False if nb_server_interface.cable is None: update = True nb_server_interface = self.connect_interface_to_switch( switch_ip, switch_interface, nb_server_interface ) else: - update = False nb_sw_int = nb_server_interface.cable.termination_b nb_sw = nb_sw_int.device nb_mgmt_int = nb.dcim.interfaces.get( @@ -365,7 +365,12 @@ class Network(): interface_id=nb_mgmt_int.id ) if nb_mgmt_ip is None: - pass + logging.error( + 'Switch {switch_ip} does not have IP on its management interface'.format( + switch_ip=switch_ip, + ) + ) + return update, nb_server_interface # Netbox IP is always IP/Netmask nb_mgmt_ip = nb_mgmt_ip.address.split('/')[0] -- 2.47.1 From bac6da46e3f04da82413f2798b32c711e15fbc9c Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Fri, 23 Aug 2019 15:33:14 +0200 Subject: [PATCH 4/8] update README and config example file --- README.md | 2 ++ netbox_agent.yaml.example | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 0023a51..4838fcd 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,8 @@ netbox: network: ignore_interfaces: "(dummy.*|docker.*)" ignore_ips: (127\.0\.0\..*) + # enable auto-cabling + lldp: true datacenter_location: driver: "cmd:cat /etc/qualification | tr [a-z] [A-Z]" diff --git a/netbox_agent.yaml.example b/netbox_agent.yaml.example index fe42b05..f751dab 100644 --- a/netbox_agent.yaml.example +++ b/netbox_agent.yaml.example @@ -5,6 +5,8 @@ netbox: network: ignore_interfaces: "(dummy.*|docker.*)" ignore_ips: (127\.0\.0\..*) + # enable auto-cabling + lldp: true datacenter_location: driver: "cmd:cat /etc/qualification | tr [a-z] [A-Z]" -- 2.47.1 From 6b5715c7c6f7ce2b640b879a129086d07cdf1ba8 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Fri, 23 Aug 2019 16:56:35 +0200 Subject: [PATCH 5/8] use lldp vlan info to tag interface in access with the vlan --- netbox_agent/network.py | 63 ++++++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/netbox_agent/network.py b/netbox_agent/network.py index f8817b0..6a2ea05 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -175,25 +175,43 @@ class Network(): ) return vlan - def reset_vlan_on_interface(self, vlan_id, interface): + def reset_vlan_on_interface(self, nic, interface): update = False - if vlan_id is None and \ + vlan_id = nic['vlan'] + lldp_vlan = self.lldp.get_switch_vlan(nic['name']) + + # if local interface isn't a interface vlan or lldp doesn't report a vlan-id + if vlan_id is None and lldp_vlan is None and \ (interface.mode is not None or len(interface.tagged_vlans) > 0): logging.info('Interface {interface} is not tagged, reseting mode'.format( interface=interface)) update = True interface.mode = None interface.tagged_vlans = [] + interface.untagged_vlan = None + # if it's a vlan interface elif vlan_id and ( - interface.mode is None or + interface.mode.value != 200 or len(interface.tagged_vlans) != 1 or interface.tagged_vlans[0].vid != vlan_id): - logging.info('Resetting VLAN on interface {interface}'.format( + logging.info('Resetting tagged VLAN(s) on interface {interface}'.format( interface=interface)) update = True nb_vlan = self.get_or_create_vlan(vlan_id) interface.mode = 200 interface.tagged_vlans = [nb_vlan] if nb_vlan else [] + interface.untagged_vlan = None + # if lldp reports a vlan-id + elif lldp_vlan and ( + interface.mode.value != 100 or + interface.untagged_vlan is None or + interface.untagged_vlan.vid != lldp_vlan): + logging.info('Resetting access VLAN on interface {interface}'.format( + interface=interface)) + update = True + nb_vlan = self.get_or_create_vlan(lldp_vlan) + interface.mode = 100 + interface.untagged_vlan = nb_vlan.id return update, interface def create_or_update_ipmi(self): @@ -222,7 +240,7 @@ class Network(): # guess it with manufacturer (IDRAC, ILO, ...) ? update = False self.create_or_update_netbox_ip_on_interface(address, interface) - update, interface = self.reset_vlan_on_interface(nic['vlan'], interface) + update, interface = self.reset_vlan_on_interface(nic, interface) if mac.upper() != interface.mac_address: logging.info('IPMI mac changed from {old_mac} to {new_mac}'.format( old_mac=interface.mac_address, new_mac=mac.upper())) @@ -239,21 +257,31 @@ class Network(): name=nic['name'], mac=nic['mac'], device=self.device.name)) nb_vlan = None - if nic['vlan']: - nb_vlan = self.get_or_create_vlan(nic['vlan']) interface = nb.dcim.interfaces.create( device=self.device.id, name=nic['name'], mac_address=nic['mac'], type=type, - mode=200 if nic['vlan'] else None, - tagged_vlans=[nb_vlan.id] if nb_vlan is not None else [], mgmt_only=mgmt, ) + if nic['vlan']: + nb_vlan = self.get_or_create_vlan(nic['vlan']) + interface.mode = 200 + interface.tagged_vlans = [nb_vlan.id] + interface.save() + elif self.lldp.get_switch_vlan(nic['name']) is not None: + # if lldp reports a vlan on an interface, tag the interface in access and set the vlan + vlan_id = self.lldp.get_switch_vlan(nic['name']) + nb_vlan = self.get_or_create_vlan(vlan_id) + interface.mode = 100 + interface.untagged_vlan = nb_vlan.id + interface.save() + # cable the interface if NETWORK_LLDP: switch_ip = self.lldp.get_switch_ip(interface.name) switch_interface = self.lldp.get_switch_port(interface.name) + if switch_ip is not None and switch_interface is not None: nic_update, interface = self.create_or_update_cable( switch_ip, switch_interface, interface @@ -450,21 +478,22 @@ class Network(): ) interface = self.create_netbox_nic(nic) - nic_update = False + nic_update = 0 if nic['name'] != interface.name: - nic_update = True logging.info('Updating interface {interface} name to: {name}'.format( interface=interface, name=nic['name'])) interface.name = nic['name'] + nic_update += 1 - nic_update, interface = self.reset_vlan_on_interface(nic['vlan'], interface) + ret, interface = self.reset_vlan_on_interface(nic, interface) + nic_update += ret type = self.get_netbox_type_for_nic(nic) if not interface.type or \ type != interface.type.value: logging.info('Interface type is wrong, resetting') - nic_update = True interface.type = type + nic_update += 1 if interface.lag is not None: local_lag_int = next( @@ -472,22 +501,24 @@ class Network(): ) if nic['name'] not in local_lag_int['bonding_slaves']: logging.info('Interface has no LAG, resetting') - nic_update = True + nic_update += 1 interface.lag = None # cable the interface if NETWORK_LLDP: switch_ip = self.lldp.get_switch_ip(interface.name) switch_interface = self.lldp.get_switch_port(interface.name) - nic_update, interface = self.create_or_update_cable( + ret, interface = self.create_or_update_cable( switch_ip, switch_interface, interface ) + if ret: + nic_update += 1 if nic['ip']: # sync local IPs for ip in nic['ip']: self.create_or_update_netbox_ip_on_interface(ip, interface) - if nic_update: + if nic_update > 0: interface.save() self._set_bonding_interfaces() -- 2.47.1 From d5a2a3047fd762599300172772069d17c1b4eaa7 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Fri, 23 Aug 2019 17:51:49 +0200 Subject: [PATCH 6/8] handle more lldp formats --- netbox_agent/lldp.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/netbox_agent/lldp.py b/netbox_agent/lldp.py index 770a8f6..4a9d148 100644 --- a/netbox_agent/lldp.py +++ b/netbox_agent/lldp.py @@ -9,6 +9,8 @@ class LLDP(): def parse(self): output_dict = {} for entry in self.output.splitlines(): + if '=' not in entry: + continue path, value = entry.strip().split("=", 1) path = path.split(".") path_components, final = path[:-1], path[-1] @@ -37,6 +39,10 @@ class LLDP(): if self.data['lldp'].get(interface) is None: return None - if self.data['lldp'][interface].get('vlan'): - return int(self.data['lldp'][interface]['vlan'].replace('vlan-', '')) + lldp = self.data['lldp'][interface] + if lldp.get('vlan'): + if lldp['vlan'].get('vlan-id'): + return int(lldp['vlan'].get('vlan-id')) + elif type(lldp['vlan']) is str: + return int(lldp['vlan'].replace('vlan-', '')) return None -- 2.47.1 From a16d9bf82c0723d1854417ea8f7430f608622265 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Mon, 26 Aug 2019 10:43:12 +0200 Subject: [PATCH 7/8] minor fixes --- netbox_agent/network.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/netbox_agent/network.py b/netbox_agent/network.py index 6a2ea05..caa950b 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -178,7 +178,7 @@ class Network(): def reset_vlan_on_interface(self, nic, interface): update = False vlan_id = nic['vlan'] - lldp_vlan = self.lldp.get_switch_vlan(nic['name']) + lldp_vlan = self.lldp.get_switch_vlan(nic['name']) if NETWORK_LLDP else None # if local interface isn't a interface vlan or lldp doesn't report a vlan-id if vlan_id is None and lldp_vlan is None and \ @@ -264,12 +264,13 @@ class Network(): type=type, mgmt_only=mgmt, ) + if nic['vlan']: nb_vlan = self.get_or_create_vlan(nic['vlan']) interface.mode = 200 interface.tagged_vlans = [nb_vlan.id] interface.save() - elif self.lldp.get_switch_vlan(nic['name']) is not None: + elif NETWORK_LLDP and self.lldp.get_switch_vlan(nic['name']) is not None: # if lldp reports a vlan on an interface, tag the interface in access and set the vlan vlan_id = self.lldp.get_switch_vlan(nic['name']) nb_vlan = self.get_or_create_vlan(vlan_id) @@ -511,8 +512,7 @@ class Network(): ret, interface = self.create_or_update_cable( switch_ip, switch_interface, interface ) - if ret: - nic_update += 1 + nic_update += ret if nic['ip']: # sync local IPs -- 2.47.1 From 74e5a16c0462d21e697c4ac3aa61def75c785cb1 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Mon, 26 Aug 2019 10:57:50 +0200 Subject: [PATCH 8/8] minor fixes --- netbox_agent/network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox_agent/network.py b/netbox_agent/network.py index caa950b..c37a891 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -191,7 +191,7 @@ class Network(): interface.untagged_vlan = None # if it's a vlan interface elif vlan_id and ( - interface.mode.value != 200 or + interface.mode is None or interface.mode.value != 200 or len(interface.tagged_vlans) != 1 or interface.tagged_vlans[0].vid != vlan_id): logging.info('Resetting tagged VLAN(s) on interface {interface}'.format( @@ -203,7 +203,7 @@ class Network(): interface.untagged_vlan = None # if lldp reports a vlan-id elif lldp_vlan and ( - interface.mode.value != 100 or + interface.mode is None or interface.mode.value != 100 or interface.untagged_vlan is None or interface.untagged_vlan.vid != lldp_vlan): logging.info('Resetting access VLAN on interface {interface}'.format( -- 2.47.1