From d12ac49d50424226d7a2d7864bbcdfeb563a5dbc Mon Sep 17 00:00:00 2001 From: Solvik Date: Tue, 3 Sep 2019 13:16:37 +0200 Subject: [PATCH] new config system (#53) --- netbox_agent/cli.py | 30 ++++------- netbox_agent/config.py | 109 +++++++++++++++++++++----------------- netbox_agent/dmidecode.py | 7 +++ netbox_agent/inventory.py | 33 ++++++------ netbox_agent/location.py | 35 ++++++------ netbox_agent/logging.py | 4 +- netbox_agent/network.py | 22 ++++---- netbox_agent/server.py | 56 ++++++++++++++++---- requirements.txt | 1 + 9 files changed, 177 insertions(+), 120 deletions(-) diff --git a/netbox_agent/cli.py b/netbox_agent/cli.py index 644983b..6a3170e 100644 --- a/netbox_agent/cli.py +++ b/netbox_agent/cli.py @@ -1,11 +1,10 @@ -import argparse - +from netbox_agent.logging import logging # NOQA from netbox_agent.vendors.dell import DellHost import netbox_agent.dmidecode as dmidecode +from netbox_agent.config import config from netbox_agent.vendors.hp import HPHost from netbox_agent.vendors.qct import QCTHost from netbox_agent.vendors.supermicro import SupermicroHost -from netbox_agent.logging import logging # NOQA MANUFACTURERS = { 'Dell Inc.': DellHost, @@ -16,29 +15,22 @@ MANUFACTURERS = { } -def run(args): +def run(config): manufacturer = dmidecode.get_by_type('Chassis')[0].get('Manufacturer') server = MANUFACTURERS[manufacturer](dmi=dmidecode) - if args.debug: + + if config.debug: server.print_debug() - if args.register: - server.netbox_create() - if args.update: - server.netbox_update() + if config.register: + server.netbox_create(config) + if config.update_all or config.update_network or config.update_location or \ + config.update_inventory: + server.netbox_update(config) return True def main(): - parser = argparse.ArgumentParser(description='Netbox agent command line') - parser.add_argument('-r', '--register', action='store_true', - help='Register server in Netbox') - parser.add_argument('-u', '--update', action='store_true', - help='Update server in Netbox') - parser.add_argument('-d', '--debug', action='store_true', - help='Print debug informations') - - args = parser.parse_args() - return run(args) + return run(config) if __name__ == '__main__': diff --git a/netbox_agent/config.py b/netbox_agent/config.py index bf6c34e..8408283 100644 --- a/netbox_agent/config.py +++ b/netbox_agent/config.py @@ -1,52 +1,65 @@ +import logging import pynetbox -import yaml - -with open('/etc/netbox_agent.yaml', 'r') as ymlfile: - # FIXME: validate configuration file - config = yaml.load(ymlfile) - -netbox_instance = pynetbox.api( - url=config['netbox']['url'], - token=config['netbox']['token'] -) - -LOG_LEVEL = config.get('log_level', 'debug') - -DATACENTER_LOCATION_DRIVER_FILE = None -DATACENTER_LOCATION = None -DATACENTER_LOCATION_REGEX = None -RACK_LOCATION_DRIVER_FILE = None -RACK_LOCATION = None -RACK_LOCATION_REGEX = None -SLOT_LOCATION_DRIVER_FILE = None -SLOT_LOCATION = None -SLOT_LOCATION_REGEX = None - -if config.get('datacenter_location'): - dc_loc = config.get('datacenter_location') - DATACENTER_LOCATION_DRIVER_FILE = dc_loc.get('driver_file') - DATACENTER_LOCATION = dc_loc.get('driver') - DATACENTER_LOCATION_REGEX = dc_loc.get('regex') - -if config.get('rack_location'): - rack_location = config['rack_location'] - RACK_LOCATION_DRIVER_FILE = rack_location.get('driver_file') - RACK_LOCATION = rack_location.get('driver') - RACK_LOCATION_REGEX = rack_location.get('regex') - -if config.get('slot_location'): - slot_location = config['slot_location'] - SLOT_LOCATION_DRIVER_FILE = slot_location.get('driver_file') - SLOT_LOCATION = slot_location.get('driver') - SLOT_LOCATION_REGEX = slot_location.get('regex') +import jsonargparse +import sys -NETWORK_IGNORE_INTERFACES = None -NETWORK_IGNORE_IPS = None -NETWORK_LLDP = None -if config.get('network'): - NETWORK_IGNORE_INTERFACES = config['network'].get('ignore_interfaces') - NETWORK_IGNORE_IPS = config['network'].get('ignore_ips') - NETWORK_LLDP = config['network'].get('lldp') is True +def get_config(): + p = jsonargparse.ArgumentParser( + default_config_files=[ + '/etc/netbox_agent.yaml', + '~/.config/netbox_agent.yaml', + '~/.netbox_agent.yaml', + ], + prog='netbox_agent', + description="Netbox agent to run on your infrastructure's servers", + ) + p.add_argument('-c', '--config', action=jsonargparse.ActionConfigFile) -INVENTORY_ENABLED = config.get('inventory') is True + p.add_argument('-r', '--register', action='store_true', help='Register server to Netbox') + p.add_argument('-u', '--update-all', action='store_true', help='Update all infos in Netbox') + p.add_argument('-d', '--debug', action='store_true', help='Print debug infos') + p.add_argument('--update-network', action='store_true', help='Update network') + p.add_argument('--update-inventory', action='store_true', help='Update inventory') + p.add_argument('--update-location', action='store_true', help='Update location') + + p.add_argument('--log_level', default='debug') + p.add_argument('--netbox.url', help='Netbox URL') + p.add_argument('--netbox.token', help='Netbox API Token') + p.add_argument('--datacenter_location.driver', + help='Datacenter location driver, ie: cmd, file') + p.add_argument('--datacenter_location.driver_file', + help='Datacenter location custom driver file path') + p.add_argument('--datacenter_location.regex', + help='Datacenter location regex to extract Netbox DC slug') + p.add_argument('--rack_location.driver', help='Rack location driver, ie: cmd, file') + p.add_argument('--rack_location.driver_file', help='Rack location custom driver file path') + p.add_argument('--rack_location.regex', help='Rack location regex to extract Rack name') + p.add_argument('--slot_location.driver', help='Slot location driver, ie: cmd, file') + p.add_argument('--slot_location.driver_file', help='Slot location custom driver file path') + p.add_argument('--slot_location.regex', help='Slot location regex to extract slot name') + p.add_argument('--network.ignore_interfaces', default=r'(dummy.*|docker.*)', + help='Regex to ignore interfaces') + p.add_argument('--network.ignore_ips', default=r'^(127\.0\.0\..*|fe80.*|::1.*)', + help='Regex to ignore IPs') + p.add_argument('--network.lldp', help='Enable auto-cabling feature through LLDP infos') + p.add_argument('--inventory', action='store_true', + help='Enable HW inventory (CPU, Memory, RAID Cards, Disks) feature') + + options = p.parse_args() + return options + + +def get_netbox_instance(): + config = get_config() + if config.netbox.url is None or config.netbox.token is None: + logging.error('Netbox URL and token are mandatory') + sys.exit(1) + return pynetbox.api( + url=get_config().netbox.url, + token=get_config().netbox.token, + ) + + +config = get_config() +netbox_instance = get_netbox_instance() diff --git a/netbox_agent/dmidecode.py b/netbox_agent/dmidecode.py index ce90cb7..4780381 100644 --- a/netbox_agent/dmidecode.py +++ b/netbox_agent/dmidecode.py @@ -1,6 +1,9 @@ import re as _re import subprocess as _subprocess +import sys +from netbox_agent.misc import is_tool +import logging _handle_re = _re.compile('^Handle\\s+(.+),\\s+DMI\\s+type\\s+(\\d+),\\s+(\\d+)\\s+bytes$') _in_block_re = _re.compile('^\\t\\t(.+)$') @@ -131,6 +134,10 @@ def get_by_type(type_id): def _execute_cmd(): + if not is_tool('dmidecode'): + logging.error('Dmidecode does not seem to be present on your system. Add it your path or ' + 'check the compatibility of this project with your distro.') + sys.exit(1) return _subprocess.check_output(['dmidecode', ], stderr=_subprocess.PIPE) diff --git a/netbox_agent/inventory.py b/netbox_agent/inventory.py index 035a84b..58e0b73 100644 --- a/netbox_agent/inventory.py +++ b/netbox_agent/inventory.py @@ -2,7 +2,7 @@ import logging import subprocess import re -from netbox_agent.config import netbox_instance as nb, INVENTORY_ENABLED +from netbox_agent.config import netbox_instance as nb, config from netbox_agent.misc import is_tool from netbox_agent.raid.hp import HPRaid from netbox_agent.raid.storcli import StorcliRaid @@ -14,17 +14,6 @@ INVENTORY_TAG = { 'raid_card': {'name': 'hw:raid_card', 'slug': 'hw-raid-card'}, } -for key, tag in INVENTORY_TAG.items(): - nb_tag = nb.extras.tags.get( - name=tag['name'] - ) - if not nb_tag: - nb_tag = nb.extras.tags.create( - name=tag['name'], - slug=tag['slug'], - comments=tag['name'], - ) - class Inventory(): """ @@ -50,11 +39,25 @@ class Inventory(): """ def __init__(self, server): + self.create_netbox_tags() self.server = server - self.device_id = self.server.get_netbox_server().id + netbox_server = self.server.get_netbox_server() + self.device_id = netbox_server.id if netbox_server else None self.raid = None self.disks = [] + def create_netbox_tags(): + for key, tag in INVENTORY_TAG.items(): + nb_tag = nb.extras.tags.get( + name=tag['name'] + ) + if not nb_tag: + nb_tag = nb.extras.tags.create( + name=tag['name'], + slug=tag['slug'], + comments=tag['name'], + ) + def get_cpus(self): model = None nb = None @@ -310,7 +313,7 @@ class Inventory(): self.create_netbox_memory(memory) def create(self): - if not INVENTORY_ENABLED: + if config.inventory is None: return False self.create_netbox_cpus() self.create_netbox_memory() @@ -319,7 +322,7 @@ class Inventory(): return True def update(self): - if not INVENTORY_ENABLED: + if config.inventory is None or config.update_inventory is None: return False self.update_netbox_cpus() self.update_netbox_memory() diff --git a/netbox_agent/location.py b/netbox_agent/location.py index 0fb4fcf..ffdf0b8 100644 --- a/netbox_agent/location.py +++ b/netbox_agent/location.py @@ -1,9 +1,7 @@ import importlib import importlib.machinery -from netbox_agent.config import DATACENTER_LOCATION, DATACENTER_LOCATION_DRIVER_FILE, \ - DATACENTER_LOCATION_REGEX, RACK_LOCATION, RACK_LOCATION_DRIVER_FILE, RACK_LOCATION_REGEX, \ - SLOT_LOCATION, SLOT_LOCATION_DRIVER_FILE, SLOT_LOCATION_REGEX +from netbox_agent.config import config class LocationBase(): @@ -53,27 +51,32 @@ class LocationBase(): class Datacenter(LocationBase): def __init__(self): - driver = DATACENTER_LOCATION.split(':')[0] if DATACENTER_LOCATION else None - driver_value = ':'.join(DATACENTER_LOCATION.split(':')[1:]) if DATACENTER_LOCATION \ - else None - driver_file = DATACENTER_LOCATION_DRIVER_FILE - regex = DATACENTER_LOCATION_REGEX + driver = config.datacenter_location.driver.split(':')[0] if \ + config.datacenter_location.driver else None + driver_value = ':'.join(config.datacenter_location.driver.split(':')[1:]) if \ + config.datacenter_location.driver else None + driver_file = config.datacenter_location.driver_file + regex = config.datacenter_location.regex super().__init__(driver, driver_value, driver_file, regex) class Rack(LocationBase): def __init__(self): - driver = RACK_LOCATION.split(':')[0] if RACK_LOCATION else None - driver_value = ':'.join(RACK_LOCATION.split(':')[1:]) if RACK_LOCATION else None - driver_file = RACK_LOCATION_DRIVER_FILE - regex = RACK_LOCATION_REGEX + driver = config.rack_location.driver.split(':')[0] if \ + config.rack_location.driver else None + driver_value = ':'.join(config.rack_location.driver.split(':')[1:]) if \ + config.rack_location.driver else None + driver_file = config.rack_location.driver_file + regex = config.rack_location.regex super().__init__(driver, driver_value, driver_file, regex) class Slot(LocationBase): def __init__(self): - driver = SLOT_LOCATION.split(':')[0] if SLOT_LOCATION else None - driver_value = ':'.join(SLOT_LOCATION.split(':')[1:]) if SLOT_LOCATION else None - driver_file = SLOT_LOCATION_DRIVER_FILE - regex = SLOT_LOCATION_REGEX + driver = config.slot_location.driver.split(':')[0] if \ + config.slot_location.driver else None + driver_value = ':'.join(config.slot_location.driver.split(':')[1:]) if \ + config.slot_location.driver else None + driver_file = config.slot_location.driver_file + regex = config.slot_location.regex super().__init__(driver, driver_value, driver_file, regex) diff --git a/netbox_agent/logging.py b/netbox_agent/logging.py index 4a21dc3..db08c30 100644 --- a/netbox_agent/logging.py +++ b/netbox_agent/logging.py @@ -1,11 +1,11 @@ import logging -from netbox_agent.config import LOG_LEVEL +from netbox_agent.config import config logger = logging.getLogger() -if LOG_LEVEL == 'debug': +if config.log_level == 'debug': logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) diff --git a/netbox_agent/network.py b/netbox_agent/network.py index 45be10b..4bf3bab 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -6,8 +6,7 @@ import re from netaddr import IPAddress, IPNetwork import netifaces -from netbox_agent.config import netbox_instance as nb -from netbox_agent.config import NETWORK_IGNORE_INTERFACES, NETWORK_IGNORE_IPS, NETWORK_LLDP +from netbox_agent.config import netbox_instance as nb, config from netbox_agent.ethtool import Ethtool from netbox_agent.ipmi import IPMI from netbox_agent.lldp import LLDP @@ -56,8 +55,8 @@ class Network(): if not os.path.islink('/sys/class/net/{}'.format(interface)): continue - if NETWORK_IGNORE_INTERFACES and \ - re.match(NETWORK_IGNORE_INTERFACES, interface): + if config.network.ignore_interfaces and \ + re.match(config.network.ignore_interfaces, interface): logging.debug('Ignore interface {interface}'.format(interface=interface)) continue @@ -84,9 +83,9 @@ class Network(): addr["netmask"] = addr["netmask"].split('/')[0] ip_addr.append(addr) - if NETWORK_IGNORE_IPS and ip_addr: + if config.network.ignore_ips and ip_addr: for i, ip in enumerate(ip_addr): - if re.match(NETWORK_IGNORE_IPS, ip['addr']): + if re.match(config.network.ignore_ips, ip['addr']): ip_addr.pop(i) mac = open('/sys/class/net/{}/address'.format(interface), 'r').read().strip() @@ -202,7 +201,7 @@ class Network(): def reset_vlan_on_interface(self, nic, interface): update = False vlan_id = nic['vlan'] - lldp_vlan = self.lldp.get_switch_vlan(nic['name']) if NETWORK_LLDP else None + lldp_vlan = self.lldp.get_switch_vlan(nic['name']) if config.network.lldp else None # if local interface isn't a interface vlan or lldp doesn't report a vlan-id if vlan_id is None and lldp_vlan is None and \ @@ -294,7 +293,7 @@ class Network(): interface.mode = 200 interface.tagged_vlans = [nb_vlan.id] interface.save() - elif NETWORK_LLDP and self.lldp.get_switch_vlan(nic['name']) is not None: + elif config.network.lldp and self.lldp.get_switch_vlan(nic['name']) is not None: # if lldp reports a vlan on an interface, tag the interface in access and set the vlan vlan_id = self.lldp.get_switch_vlan(nic['name']) nb_vlan = self.get_or_create_vlan(vlan_id) @@ -303,7 +302,7 @@ class Network(): interface.save() # cable the interface - if NETWORK_LLDP: + if config.network.lldp: switch_ip = self.lldp.get_switch_ip(interface.name) switch_interface = self.lldp.get_switch_port(interface.name) @@ -503,6 +502,9 @@ class Network(): logging.debug('Finished creating NIC!') def update_netbox_network_cards(self): + if config.update_all is None or config.update_network is None: + print(config) + return None logging.debug('Updating NIC...') # delete unknown interface @@ -566,7 +568,7 @@ class Network(): interface.lag = None # cable the interface - if NETWORK_LLDP: + if config.network.lldp: switch_ip = self.lldp.get_switch_ip(interface.name) switch_interface = self.lldp.get_switch_port(interface.name) if switch_ip and switch_interface: diff --git a/netbox_agent/server.py b/netbox_agent/server.py index f57fa9e..6ad3cdd 100644 --- a/netbox_agent/server.py +++ b/netbox_agent/server.py @@ -30,6 +30,33 @@ class ServerBase(): ) return datacenter + def update_netbox_location(self, server): + dc = self.get_datacenter() + rack = self.get_rack() + nb_rack = self.get_netbox_rack() + nb_dc = self.get_netbox_datacenter() + + update = False + if dc and server.site.slug != nb_dc.slug: + logging.info('Datacenter location has changed from {} to {}, updating'.format( + server.site.slug, + nb_dc.slug, + )) + update = True + server.site = nb_dc.id + + if rack and server.rack != nb_rack: + logging.info('Rack location has changed from {} to {}, updating'.format( + server.rack, + nb_rack, + )) + update = True + server.rack = nb_rack + if nb_rack is None: + server.face = None + server.position = None + return update, server + def get_rack(self): rack = Rack() return rack.get() @@ -175,7 +202,7 @@ class ServerBase(): def get_netbox_server(self): return nb.dcim.devices.get(serial=self.get_service_tag()) - def netbox_create(self): + def netbox_create(self, config): logging.debug('Creating Server..') datacenter = self.get_netbox_datacenter() rack = self.get_netbox_rack() @@ -205,8 +232,9 @@ class ServerBase(): self.network = Network(server=self) self.network.create_netbox_network_cards() - self.inventory = Inventory(server=self) - self.inventory.create() + if config.inventory: + self.inventory = Inventory(server=self) + self.inventory.create() logging.debug('Server created!') def _netbox_update_chassis_for_blade(self, server, datacenter): @@ -235,7 +263,7 @@ class ServerBase(): # Set slot for blade self._netbox_set_blade_slot(chassis, server) - def netbox_update(self): + def netbox_update(self, config): """ Netbox method to update info about our server/blade @@ -246,11 +274,12 @@ class ServerBase(): * new network infos """ logging.debug('Updating Server...') + server = nb.dcim.devices.get(serial=self.get_service_tag()) if not server: raise Exception("The server (Serial: {}) isn't yet registered in Netbox, register" 'it before updating it'.format(self.get_service_tag())) - update = False + update = 0 if self.is_blade(): datacenter = self.get_netbox_datacenter() # if it's already linked to a chassis @@ -269,14 +298,21 @@ class ServerBase(): # for every other specs # check hostname if server.name != self.get_hostname(): - update = True + update += 1 server.hostname = self.get_hostname() + + if config.update_all or config.update_location: + ret, server = self.update_netbox_location(server) + update += ret + # check network cards - self.network = Network(server=self) - self.network.update_netbox_network_cards() + if config.update_all or config.update_network: + self.network = Network(server=self) + self.network.update_netbox_network_cards() # update inventory - self.inventory = Inventory(server=self) - self.inventory.update() + if config.update_all or config.update_inventory: + self.inventory = Inventory(server=self) + self.inventory.update() if update: server.save() logging.debug('Finished updating Server!') diff --git a/requirements.txt b/requirements.txt index 895d0a8..7107ec4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pynetbox==4.0.6 netaddr==0.7.19 netifaces==0.10.9 pyyaml==5.1.2 +jsonargparse==2.2.1