DGNum/netbox-agent
6
0
Fork 0
Code Issues 28 Pull requests 9 Projects Releases 14 Packages Wiki Activity Actions

Rework all network part (#21)

Browse source 
* rework network part by ignoring devices and ip from config file

* update config file

* associate slave device to bond device with Netbox LAG

* update README

* make sure every interface attribute is ok

* some fixes after  test

* tox

* add ipmi feature

* fix bug in LAG

* ipmi class

* update README

* network update

* delete print

* update README
This commit is contained in:
Solvik 2019-08-09 12:08:11 +02:00 committed by GitHub
parent 215e66d62f
commit 2988a8bd6a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 330 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
README.md
View file

@ -8,16 +8,26 @@ The goal is to generate an existing infrastructure on Netbox and have the abilit
# Features
* Create servers, chassis and blade through standard tools (`dmidecode`)
* Create physical network interfaces with IPs
* Create physical, bonding and vlan network interfaces with IPs
* Create IPMI interface if found
* Create or get existing VLAN and associate it to interfaces
* Generic ability to guess datacenters and rack location through drivers (`cmd` and `file` and custom ones)
* Update existing `Device` and `Interfaces`
* Handle blade moving (new slot, new chassis)
# Requirements
- Netbox >= 2.6
- Python >= 3.4
- [python3-netaddr](https://github.com/drkjam/netaddr)
- [python3-netifaces](https://github.com/al45tair/netifaces)
# Known limitations
* The project is only compatible with Linux.
Since it uses `ethtool` and parses `/sys/` directory, it's not compatible with *BSD distributions.
* Netbox `>=2.6.0,<=2.6.2` has a caching problem ; if the cache lifetime is too high, the script can get stale data after modification.
We advise to set `CACHE_TIME` to `0`.
# Configuration
@ -26,12 +36,26 @@ netbox:
url: 'http://netbox.internal.company.com'
token: supersecrettoken
network:
ignore_interfaces: "(dummy.*|docker.*)"
ignore_ips: (127\.0\.0\..*)
datacenter_location:
# driver_file: /opt/netbox_driver_dc.py
driver: file:/etc/qualification
regex: "datacenter: (?P<datacenter>[A-Za-z0-9]+)"
driver: "cmd:cat /etc/qualification | tr [a-z] [A-Z]"
regex: "DATACENTER: (?P<datacenter>[A-Za-z0-9]+)"
# driver: 'cmd:lldpctl'
# regex = 'SysName: .*\.(?P<datacenter>[A-Za-z0-9]+)'```
# regex: 'SysName: .*\.([A-Za-z0-9]+)'
#
# driver: "file:/tmp/datacenter"
# regex: "(.*)"
rack_location:
# driver: 'cmd:lldpctl'
# match SysName: sw-dist-a1.dc42
# regex: 'SysName:[ ]+[A-Za-z]+-[A-Za-z]+-([A-Za-z0-9]+)'
#
# driver: "file:/tmp/datacenter"
# regex: "(.*)"
```
# Hardware

4
netbox_agent.yaml.example
View file

@ -2,6 +2,10 @@ netbox:
url: 'http://netbox.internal.company.com'
token: supersecrettoken
network:
ignore_interfaces: "(dummy.*|docker.*)"
ignore_ips: (127\.0\.0\..*)
datacenter_location:
driver: "cmd:cat /etc/qualification | tr [a-z] [A-Z]"
regex: "DATACENTER: (?P<datacenter>[A-Za-z0-9]+)"

6
netbox_agent/config.py
View file

@ -30,3 +30,9 @@ if config.get('rack_location'):
RACK_LOCATION_DRIVER_FILE = rack_location.get('driver_file')
RACK_LOCATION = rack_location.get('driver')
RACK_LOCATION_REGEX = rack_location.get('regex')
NETWORK_IGNORE_INTERFACES = None
NETWORK_IGNORE_IPS = None
if config.get('network'):
NETWORK_IGNORE_INTERFACES = config['network']['ignore_interfaces']
NETWORK_IGNORE_IPS = config['network']['ignore_ips']

49
netbox_agent/ipmi.py Normal file
View file

@ -0,0 +1,49 @@
import logging
import subprocess
class Ipmi():
"""
Parse IPMI output
ie:
Set in Progress : Set Complete
Auth Type Support :
Auth Type Enable : Callback :
: User :
: Operator :
: Admin :
: OEM :
IP Address Source : DHCP Address
IP Address : 10.192.2.1
Subnet Mask : 255.255.240.0
MAC Address : 98:f2:b3:f0:ee:1e
SNMP Community String :
BMC ARP Control : ARP Responses Enabled, Gratuitous ARP Disabled
Default Gateway IP : 10.192.2.254
802.1q VLAN ID : Disabled
802.1q VLAN Priority : 0
RMCP+ Cipher Suites : 0,1,2,3
Cipher Suite Priv Max : XuuaXXXXXXXXXXX
: X=Cipher Suite Unused
: c=CALLBACK
: u=USER
: o=OPERATOR
: a=ADMIN
: O=OEM
Bad Password Threshold : Not Available
"""
def __init__(self):
self.ret, self.output = subprocess.getstatusoutput('ipmitool lan print')
if self.ret != 0:
logging.error('Cannot get ipmi info: {}'.format(self.output))
def parse(self):
ret = {}
if self.ret != 0:
return ret
for line in self.output.split('\n'):
key = line.split(':')[0].strip()
value = ':'.join(line.split(':')[1:]).strip()
ret[key] = value
return ret

2
netbox_agent/location.py
View file

@ -22,11 +22,9 @@ class LocationBase():
self.driver = driver
self.driver_value = driver_value
self.driver_file = driver_file
print(self.driver_file)
self.regex = regex
if self.driver_file:
print('if', self.driver_file)
try:
# FIXME: Works with Python 3.3+, support older version?
loader = importlib.machinery.SourceFileLoader('driver_file', self.driver_file)

298
netbox_agent/network.py
View file

@ -3,11 +3,13 @@ import logging
import os
import re
from netaddr import IPAddress
from netaddr import IPAddress, IPNetwork
import netifaces
from netbox_agent.config import netbox_instance as nb
from netbox_agent.config import NETWORK_IGNORE_INTERFACES, NETWORK_IGNORE_IPS
from netbox_agent.ethtool import Ethtool
from netbox_agent.ipmi import Ipmi
IFACE_TYPE_100ME_FIXED = 800
IFACE_TYPE_1GE_FIXED = 1000
@ -33,10 +35,7 @@ IFACE_TYPE_200GE_CFP2 = 1650
IFACE_TYPE_200GE_QSFP56 = 1700
IFACE_TYPE_400GE_QSFP_DD = 1750
IFACE_TYPE_OTHER = 32767
# Regex to match base interface name
# Doesn't match vlan interfaces and other loopback etc
INTERFACE_REGEX = re.compile('^(eth[0-9]+|ens[0-9]+|enp[0-9]+s[0-9]f[0-9])$')
IFACE_TYPE_LAG = 200
class Network():
@ -44,29 +43,99 @@ class Network():
self.nics = []
self.server = server
self.device = self.server.get_netbox_server()
self.scan()
def scan(self):
for interface in os.listdir('/sys/class/net/'):
if re.match(INTERFACE_REGEX, interface):
# ignore if it's not a link (ie: bonding_masters etc)
if not os.path.islink('/sys/class/net/{}'.format(interface)):
continue
if NETWORK_IGNORE_INTERFACES and \
re.match(NETWORK_IGNORE_INTERFACES, interface):
logging.debug('Ignore interface {interface}'.format(interface=interface))
continue
else:
ip_addr = netifaces.ifaddresses(interface).get(netifaces.AF_INET)
if NETWORK_IGNORE_IPS and ip_addr:
for i, ip in enumerate(ip_addr):
if re.match(NETWORK_IGNORE_IPS, ip['addr']):
ip_addr.pop(i)
mac = open('/sys/class/net/{}/address'.format(interface), 'r').read().strip()
vlan = None
if len(interface.split('.')) > 1:
vlan = int(interface.split('.')[1])
bonding = False
bonding_slaves = []
if os.path.isdir('/sys/class/net/{}/bonding'.format(interface)):
bonding = True
bonding_slaves = open(
'/sys/class/net/{}/bonding/slaves'.format(interface)
).read().split()
nic = {
'name': interface,
'mac': open('/sys/class/net/{}/address'.format(interface), 'r').read().strip(),
'mac': mac if mac != '00:00:00:00:00:00' else None,
'ip': [
'{}/{}'.format(
x['addr'],
IPAddress(x['netmask']).netmask_bits()
) for x in ip_addr
] if ip_addr else None, # FIXME: handle IPv6 addresses
'ethtool': Ethtool(interface).parse()
'ethtool': Ethtool(interface).parse(),
'vlan': vlan,
'bonding': bonding,
'bonding_slaves': bonding_slaves,
}
self.nics.append(nic)
def _set_bonding_interfaces(self):
bonding_nics = [x for x in self.nics if x['bonding']]
if not len(bonding_nics):
return False
logging.debug('Setting bonding interfaces..')
for nic in bonding_nics:
bond_int = self.get_netbox_network_card(nic)
logging.debug('Setting slave interface for {name}'.format(
name=bond_int.name
))
for slave in nic['bonding_slaves']:
slave_nic = next(item for item in self.nics if item['name'] == slave)
slave_int = self.get_netbox_network_card(slave_nic)
logging.debug('Settting interface {name} as slave of {master}'.format(
name=slave_int.name, master=bond_int.name
))
slave_int.lag = bond_int
slave_int.save()
return True
def get_network_cards(self):
return self.nics
def get_netbox_network_card(self, nic):
if nic['mac'] is None:
interface = nb.dcim.interfaces.get(
device_id=self.device.id,
name=nic['name'],
)
else:
interface = nb.dcim.interfaces.get(
device_id=self.device.id,
mac_address=nic['mac'],
name=nic['name'],
)
return interface
def get_netbox_network_cards(self):
return nb.dcim.interfaces.filter(
device_id=self.device.id,
mgmt_only=False,
)
def get_netbox_type_for_nic(self, nic):
if nic.get('bonding'):
return IFACE_TYPE_LAG
if nic.get('ethtool') is None:
return IFACE_TYPE_OTHER
if nic['ethtool']['speed'] == '10000Mb/s':
@ -79,58 +148,169 @@ class Network():
return IFACE_TYPE_1GE_FIXED
return IFACE_TYPE_OTHER
def create_netbox_nic(self, device, nic):
def get_ipmi(self):
ipmi = Ipmi().parse()
return ipmi
def get_netbox_ipmi(self):
ipmi = self.get_ipmi()
mac = ipmi['MAC Address']
return nb.dcim.interfaces.get(
mac=mac
)
def get_or_create_vlan(self, vlan_id):
# FIXME: we may need to specify the datacenter
# since users may have same vlan id in multiple dc
vlan = nb.ipam.vlans.get(
vid=vlan_id,
)
if vlan is None:
vlan = nb.ipam.vlans.create(
name='VLAN {}'.format(vlan_id),
vid=vlan_id,
)
return vlan
def reset_vlan_on_interface(self, vlan_id, interface):
update = False
if vlan_id is None and \
(interface.mode is not None or len(interface.tagged_vlans) > 0):
logging.info('Interface {interface} is not tagged, reseting mode'.format(
interface=interface))
update = True
interface.mode = None
interface.tagged_vlans = []
elif vlan_id and (
interface.mode is None or
len(interface.tagged_vlans) != 1 or
interface.tagged_vlans[0].vid != vlan_id):
logging.info('Resetting VLAN on interface {interface}'.format(
interface=interface))
update = True
nb_vlan = self.get_or_create_vlan(vlan_id)
interface.mode = 200
interface.tagged_vlans = [nb_vlan] if nb_vlan else []
return update, interface
def create_or_update_ipmi(self):
ipmi = self.get_ipmi()
mac = ipmi['MAC Address']
ip = ipmi['IP Address']
netmask = ipmi['Subnet Mask']
vlan = int(ipmi['802.1q VLAN ID']) if ipmi['802.1q VLAN ID'] != 'Disabled' else None
address = IPNetwork('{}/{}'.format(ip, netmask)).__str__()
interface = nb.dcim.interfaces.get(
device_id=self.device.id,
mgmt_only=True,
)
nic = {
'name': 'IPMI',
'mac': mac,
'vlan': vlan,
'ip': [address],
}
if interface is None:
interface = self.create_netbox_nic(nic, mgmt=True)
self.create_or_update_netbox_ip_on_interface(address, interface)
else:
# let the user chose the name of mgmt ?
# guess it with manufacturer (IDRAC, ILO, ...) ?
update = False
self.create_or_update_netbox_ip_on_interface(address, interface)
update, interface = self.reset_vlan_on_interface(nic['vlan'], interface)
if mac != interface.mac_address:
interface.mac_address = mac
update = True
if update:
interface.save()
return interface
def create_netbox_nic(self, nic, mgmt=False):
# TODO: add Optic Vendor, PN and Serial
type = self.get_netbox_type_for_nic(nic)
logging.info('Creating NIC {name} ({mac}) on {device}'.format(
name=nic['name'], mac=nic['mac'], device=device.name))
name=nic['name'], mac=nic['mac'], device=self.device.name))
nb_vlan = None
if nic['vlan']:
nb_vlan = self.get_or_create_vlan(nic['vlan'])
return nb.dcim.interfaces.create(
device=device.id,
device=self.device.id,
name=nic['name'],
mac_address=nic['mac'],
type=type,
mode=200 if nic['vlan'] else None,
tagged_vlans=[nb_vlan.id] if nb_vlan is not None else [],
mgmt_only=mgmt,
)
def create_or_update_netbox_ip_on_interface(self, ip, interface):
netbox_ip = nb.ipam.ip_addresses.get(
address=ip,
)
if netbox_ip:
if netbox_ip.interface is None:
logging.info('Assigning existing IP {ip} to {interface}'.format(
ip=ip, interface=interface))
elif netbox_ip.interface.id != interface.id:
logging.info(
'Detected interface change for ip {ip}: old interface is '
'{old_interface} (id: {old_id}), new interface is {new_interface} '
' (id: {new_id})'
.format(
old_interface=netbox_ip.interface, new_interface=interface,
old_id=netbox_ip.id, new_id=interface.id, ip=netbox_ip.address
))
else:
return netbox_ip
netbox_ip.interface = interface
netbox_ip.save()
else:
logging.info('Create new IP {ip} on {interface}'.format(
ip=ip, interface=interface))
netbox_ip = nb.ipam.ip_addresses.create(
address=ip,
interface=interface.id,
status=1,
)
return netbox_ip
def create_netbox_network_cards(self):
logging.debug('Creating NIC...')
device = self.server.get_netbox_server()
for nic in self.nics:
interface = nb.dcim.interfaces.get(
mac_address=nic['mac'],
)
interface = self.get_netbox_network_card(nic)
# if network doesn't exist we create it
if not interface:
new_interface = self.create_netbox_nic(device, nic)
new_interface = self.create_netbox_nic(nic)
if nic['ip']:
# for each ip, we try to find it
# assign the device's interface to it
# or simply create it
for ip in nic['ip']:
netbox_ip = nb.ipam.ip_addresses.get(
address=ip,
)
if netbox_ip:
logging.info('Assigning existing IP {ip} to {interface}'.format(
ip=ip, interface=new_interface))
netbox_ip.interface = new_interface
netbox_ip.save()
else:
logging.info('Create new IP {ip} on {interface}'.format(
ip=ip, interface=new_interface))
netbox_ip = nb.ipam.ip_addresses.create(
address=ip,
interface=new_interface.id,
status=1,
)
self.create_or_update_netbox_ip_on_interface(ip, new_interface)
self._set_bonding_interfaces()
self.create_or_update_ipmi()
logging.debug('Finished creating NIC!')
def update_netbox_network_cards(self):
logging.debug('Updating NIC...')
device = self.server.get_netbox_server()
# delete unknown interface
nb_nics = self.get_netbox_network_cards()
local_nics = [x['name'] for x in self.nics]
for nic in nb_nics:
if nic.name not in local_nics:
logging.info('Deleting netbox interface {name} because not present locally'.format(
name=nic.name
))
nic.delete()
# delete IP on netbox that are not known on this server
netbox_ips = nb.ipam.ip_addresses.filter(
device=device
device_id=self.device.id,
interface_id=[x.id for x in nb_nics],
)
all_local_ips = list(chain.from_iterable([
x['ip'] for x in self.nics if x['ip'] is not None
@ -144,14 +324,12 @@ class Network():
# update each nic
for nic in self.nics:
interface = nb.dcim.interfaces.get(
mac_address=nic['mac'],
)
interface = self.get_netbox_network_card(nic)
if not interface:
logging.info('Interface {} not found, creating..'.format(
logging.info('Interface {mac_address} not found, creating..'.format(
mac_address=nic['mac'])
)
interface = self.create_netbox_nic(device, nic)
interface = self.create_netbox_nic(nic)
nic_update = False
if nic['name'] != interface.name:
@ -160,32 +338,34 @@ class Network():
interface=interface, name=nic['name']))
interface.name = nic['name']
nic_update, interface = self.reset_vlan_on_interface(nic['vlan'], interface)
type = self.get_netbox_type_for_nic(nic)
if not interface.type or \
type != interface.type.value:
logging.info('Interface type is wrong, resetting')
nic_update = True
interface.type = type
if interface.lag is not None:
local_lag_int = next(
item for item in self.nics if item['name'] == interface.lag.name
)
if nic['name'] not in local_lag_int['bonding_slaves']:
logging.info('Interface has no LAG, resetting')
nic_update = True
interface.lag = None
if nic['ip']:
# sync local IPs
for ip in nic['ip']:
netbox_ip = nb.ipam.ip_addresses.get(
address=ip,
)
if not netbox_ip:
# create netbbox_ip on device
netbox_ip = nb.ipam.ip_addresses.create(
address=ip,
interface=interface.id,
status=1,
)
logging.info('Created new IP {ip} on {interface}'.format(
ip=ip, interface=interface))
else:
if netbox_ip.interface.id != interface.id:
logging.info(
'Detected interface change: old interface is {old_interface} '
'(id: {old_id}), new interface is {new_interface} (id: {new_id})'
.format(
old_interface=netbox_ip.interface, new_interface=interface,
old_id=netbox_ip.id, new_id=interface.id
))
netbox_ip.interface = interface
netbox_ip.save()
self.create_or_update_netbox_ip_on_interface(ip, interface)
if nic_update:
interface.save()
self._set_bonding_interfaces()
self.create_or_update_ipmi()
logging.debug('Finished updating NIC!')

4
netbox_agent/server.py
View file

@ -17,7 +17,7 @@ class ServerBase():
self.system = self.dmi.get_by_type('System')
self.bios = self.dmi.get_by_type('BIOS')
self.network = Network(server=self)
self.network = None
def get_datacenter(self):
dc = Datacenter()
@ -198,6 +198,7 @@ class ServerBase():
if not server:
self._netbox_create_server(datacenter)
self.network = Network(server=self)
self.network.create_netbox_network_cards()
logging.debug('Server created!')
@ -264,6 +265,7 @@ class ServerBase():
update = True
server.hostname = self.get_hostname()
# check network cards
self.network = Network(server=self)
self.network.update_netbox_network_cards()
if update:
server.save()