550 lines
19 KiB
Python
550 lines
19 KiB
Python
from netbox_agent.config import config
|
|
from netbox_agent.config import netbox_instance as nb
|
|
from netbox_agent.lshw import LSHW
|
|
from netbox_agent.misc import get_vendor, is_tool
|
|
from netbox_agent.raid.hp import HPRaid
|
|
from netbox_agent.raid.omreport import OmreportRaid
|
|
from netbox_agent.raid.storcli import StorcliRaid
|
|
import traceback
|
|
import pynetbox
|
|
import logging
|
|
import json
|
|
import re
|
|
|
|
|
|
INVENTORY_TAG = {
|
|
'cpu': {'name': 'hw:cpu', 'slug': 'hw-cpu'},
|
|
'gpu': {'name': 'hw:gpu', 'slug': 'hw-gpu'},
|
|
'disk': {'name': 'hw:disk', 'slug': 'hw-disk'},
|
|
'interface': {'name': 'hw:interface', 'slug': 'hw-interface'},
|
|
'memory': {'name': 'hw:memory', 'slug': 'hw-memory'},
|
|
'motherboard': {'name': 'hw:motherboard', 'slug': 'hw-motherboard'},
|
|
'raid_card': {'name': 'hw:raid_card', 'slug': 'hw-raid-card'},
|
|
}
|
|
|
|
|
|
class Inventory():
|
|
"""
|
|
Better Inventory items coming, see:
|
|
- https://github.com/netbox-community/netbox/issues/3087
|
|
- https://github.com/netbox-community/netbox/issues/3333
|
|
|
|
This class implements for:
|
|
* memory
|
|
* cpu
|
|
* raid cards
|
|
* disks
|
|
* gpus
|
|
|
|
methods that:
|
|
* get local item
|
|
* get netbox item
|
|
* create netbox item
|
|
* update netbox item
|
|
|
|
Known issues:
|
|
- no scan of non-raid devices
|
|
- no scan of NVMe devices
|
|
"""
|
|
|
|
def __init__(self, server, update_expansion=False):
|
|
self.create_netbox_tags()
|
|
self.server = server
|
|
self.update_expansion = update_expansion
|
|
netbox_server = self.server.get_netbox_server(update_expansion)
|
|
|
|
self.device_id = netbox_server.id if netbox_server else None
|
|
self.raid = None
|
|
self.disks = []
|
|
|
|
self.lshw = LSHW()
|
|
|
|
def create_netbox_tags(self):
|
|
ret = []
|
|
for key, tag in INVENTORY_TAG.items():
|
|
nb_tag = nb.extras.tags.get(
|
|
name=tag['name']
|
|
)
|
|
if not nb_tag:
|
|
nb_tag = nb.extras.tags.create(
|
|
name=tag['name'],
|
|
slug=tag['slug'],
|
|
comments=tag['name'],
|
|
)
|
|
ret.append(nb_tag)
|
|
return ret
|
|
|
|
def find_or_create_manufacturer(self, name):
|
|
if name is None:
|
|
return None
|
|
|
|
manufacturer = nb.dcim.manufacturers.get(
|
|
name=name,
|
|
)
|
|
if not manufacturer:
|
|
logging.info('Creating missing manufacturer {name}'.format(name=name))
|
|
manufacturer = nb.dcim.manufacturers.create(
|
|
name=name,
|
|
slug=re.sub('[^A-Za-z0-9]+', '-', name).lower(),
|
|
)
|
|
|
|
logging.info('Creating missing manufacturer {name}'.format(name=name))
|
|
|
|
return manufacturer
|
|
|
|
def get_netbox_inventory(self, device_id, tag):
|
|
try:
|
|
items = nb.dcim.inventory_items.filter(
|
|
device_id=device_id,
|
|
tag=tag
|
|
)
|
|
except pynetbox.core.query.RequestError:
|
|
logging.info('Tag {tag} is missing, returning empty array.'.format(tag=tag))
|
|
items = []
|
|
|
|
return list(items)
|
|
|
|
def create_netbox_inventory_item(self, device_id, tags, vendor, name, serial, description):
|
|
manufacturer = self.find_or_create_manufacturer(vendor)
|
|
|
|
_ = nb.dcim.inventory_items.create(
|
|
device=device_id,
|
|
manufacturer=manufacturer.id,
|
|
discovered=True,
|
|
tags=tags,
|
|
name='{}'.format(name),
|
|
serial='{}'.format(serial),
|
|
description=description
|
|
)
|
|
|
|
logging.info('Creating inventory item {} {}/{} {} '.format(
|
|
vendor,
|
|
name,
|
|
serial,
|
|
description)
|
|
)
|
|
|
|
def get_hw_motherboards(self):
|
|
motherboards = []
|
|
|
|
m = {}
|
|
m['serial'] = self.lshw.motherboard_serial
|
|
m['vendor'] = self.lshw.vendor
|
|
m['name'] = '{} {}'.format(self.lshw.vendor, self.lshw.motherboard)
|
|
m['description'] = '{} Motherboard'.format(self.lshw.motherboard)
|
|
|
|
motherboards.append(m)
|
|
|
|
return motherboards
|
|
|
|
def do_netbox_motherboard(self):
|
|
|
|
motherboards = self.get_hw_motherboards()
|
|
nb_motherboards = self.get_netbox_inventory(
|
|
device_id=self.device_id,
|
|
tag=INVENTORY_TAG['motherboard']['slug'])
|
|
|
|
for nb_motherboard in nb_motherboards:
|
|
if nb_motherboard.serial not in [x['serial'] for x in motherboards]:
|
|
logging.info('Deleting unknown motherboard {motherboard}/{serial}'.format(
|
|
motherboard=self.lshw.motherboard,
|
|
serial=nb_motherboard.serial,
|
|
))
|
|
nb_motherboard.delete()
|
|
|
|
# create interfaces that are not in netbox
|
|
for motherboard in motherboards:
|
|
if motherboard.get('serial') not in [x.serial for x in nb_motherboards]:
|
|
self.create_netbox_inventory_item(
|
|
device_id=self.device_id,
|
|
tags=[{'name': INVENTORY_TAG['motherboard']['name']}],
|
|
vendor='{}'.format(motherboard.get('vendor', 'N/A')),
|
|
serial='{}'.format(motherboard.get('serial', 'No SN')),
|
|
name='{}'.format(motherboard.get('name')),
|
|
description='{}'.format(motherboard.get('description'))
|
|
)
|
|
|
|
def create_netbox_interface(self, iface):
|
|
manufacturer = self.find_or_create_manufacturer(iface["vendor"])
|
|
_ = nb.dcim.inventory_items.create(
|
|
device=self.device_id,
|
|
manufacturer=manufacturer.id,
|
|
discovered=True,
|
|
tags=[{'name': INVENTORY_TAG['interface']['name']}],
|
|
name="{}".format(iface['product']),
|
|
serial='{}'.format(iface['serial']),
|
|
description='{} {}'.format(iface['description'], iface['name'])
|
|
)
|
|
|
|
def do_netbox_interfaces(self):
|
|
nb_interfaces = self.get_netbox_inventory(
|
|
device_id=self.device_id,
|
|
tag=INVENTORY_TAG['interface']['slug'])
|
|
interfaces = self.lshw.interfaces
|
|
|
|
# delete interfaces that are in netbox but not locally
|
|
# use the serial_number has the comparison element
|
|
for nb_interface in nb_interfaces:
|
|
if nb_interface.serial not in [x['serial'] for x in interfaces]:
|
|
logging.info('Deleting unknown interface {serial}'.format(
|
|
serial=nb_interface.serial,
|
|
))
|
|
nb_interface.delete()
|
|
|
|
# create interfaces that are not in netbox
|
|
for iface in interfaces:
|
|
if iface.get('serial') not in [x.serial for x in nb_interfaces]:
|
|
self.create_netbox_interface(iface)
|
|
|
|
def create_netbox_cpus(self):
|
|
for cpu in self.lshw.get_hw_linux('cpu'):
|
|
manufacturer = self.find_or_create_manufacturer(cpu["vendor"])
|
|
_ = nb.dcim.inventory_items.create(
|
|
device=self.device_id,
|
|
manufacturer=manufacturer.id,
|
|
discovered=True,
|
|
tags=[{'name': INVENTORY_TAG['cpu']['name']}],
|
|
name=cpu['product'],
|
|
description='CPU {}'.format(cpu['location']),
|
|
# asset_tag=cpu['location']
|
|
)
|
|
|
|
logging.info('Creating CPU model {}'.format(cpu['product']))
|
|
|
|
def do_netbox_cpus(self):
|
|
cpus = self.lshw.get_hw_linux('cpu')
|
|
nb_cpus = self.get_netbox_inventory(
|
|
device_id=self.device_id,
|
|
tag=INVENTORY_TAG['cpu']['slug'],
|
|
)
|
|
|
|
if not len(nb_cpus) or \
|
|
len(nb_cpus) and len(cpus) != len(nb_cpus):
|
|
for x in nb_cpus:
|
|
x.delete()
|
|
|
|
self.create_netbox_cpus()
|
|
|
|
def get_raid_cards(self, filter_cards=False):
|
|
raid_class = None
|
|
if self.server.manufacturer in ('Dell', 'Huawei'):
|
|
if is_tool('omreport'):
|
|
raid_class = OmreportRaid
|
|
if is_tool('storcli'):
|
|
raid_class = StorcliRaid
|
|
elif self.server.manufacturer in ('HP', 'HPE'):
|
|
if is_tool('ssacli'):
|
|
raid_class = HPRaid
|
|
|
|
if not raid_class:
|
|
return []
|
|
|
|
self.raid = raid_class()
|
|
|
|
if filter_cards and config.expansion_as_device \
|
|
and self.server.own_expansion_slot():
|
|
return [
|
|
c for c in self.raid.get_controllers()
|
|
if c.is_external() is self.update_expansion
|
|
]
|
|
else:
|
|
return self.raid.get_controllers()
|
|
|
|
def create_netbox_raid_card(self, raid_card):
|
|
manufacturer = self.find_or_create_manufacturer(
|
|
raid_card.get_manufacturer()
|
|
)
|
|
|
|
name = raid_card.get_product_name()
|
|
serial = raid_card.get_serial_number()
|
|
nb_raid_card = nb.dcim.inventory_items.create(
|
|
device=self.device_id,
|
|
discovered=True,
|
|
manufacturer=manufacturer.id if manufacturer else None,
|
|
tags=[{'name': INVENTORY_TAG['raid_card']['name']}],
|
|
name='{}'.format(name),
|
|
serial='{}'.format(serial),
|
|
description='RAID Card',
|
|
)
|
|
logging.info('Creating RAID Card {name} (SN: {serial})'.format(
|
|
name=name,
|
|
serial=serial,
|
|
))
|
|
return nb_raid_card
|
|
|
|
def do_netbox_raid_cards(self):
|
|
"""
|
|
Update raid cards in netbobx
|
|
Since we only push:
|
|
* Name
|
|
* Manufacturer
|
|
* Serial
|
|
|
|
We only need to handle destroy and new cards
|
|
"""
|
|
|
|
nb_raid_cards = self.get_netbox_inventory(
|
|
device_id=self.device_id,
|
|
tag=[INVENTORY_TAG['raid_card']['slug']]
|
|
)
|
|
raid_cards = self.get_raid_cards(filter_cards=True)
|
|
|
|
# delete cards that are in netbox but not locally
|
|
# use the serial_number has the comparison element
|
|
for nb_raid_card in nb_raid_cards:
|
|
if nb_raid_card.serial not in [x.get_serial_number() for x in raid_cards]:
|
|
logging.info('Deleting unknown locally RAID Card {serial}'.format(
|
|
serial=nb_raid_card.serial,
|
|
))
|
|
nb_raid_card.delete()
|
|
|
|
# create card that are not in netbox
|
|
for raid_card in raid_cards:
|
|
if raid_card.get_serial_number() not in [x.serial for x in nb_raid_cards]:
|
|
self.create_netbox_raid_card(raid_card)
|
|
|
|
def is_virtual_disk(self, disk, raid_devices):
|
|
disk_type = disk.get('type')
|
|
logicalname = disk.get('logicalname')
|
|
description = disk.get('description')
|
|
size = disk.get('size')
|
|
product = disk.get('product')
|
|
if logicalname in raid_devices or disk_type is None or product is None or description is None:
|
|
return True
|
|
non_raid_disks = [
|
|
'MR9361-8i',
|
|
]
|
|
|
|
if logicalname in raid_devices or \
|
|
product in non_raid_disks or \
|
|
'virtual' in product.lower() or \
|
|
'logical' in product.lower() or \
|
|
'volume' in description.lower() or \
|
|
'dvd-ram' in description.lower() or \
|
|
description == 'SCSI Enclosure' or \
|
|
(size is None and logicalname is None):
|
|
return True
|
|
return False
|
|
|
|
def get_hw_disks(self):
|
|
disks = []
|
|
|
|
for raid_card in self.get_raid_cards(filter_cards=True):
|
|
disks.extend(raid_card.get_physical_disks())
|
|
|
|
raid_devices = [
|
|
d.get('custom_fields', {}).get('vd_device')
|
|
for d in disks
|
|
if d.get('custom_fields', {}).get('vd_device')
|
|
]
|
|
|
|
for disk in self.lshw.get_hw_linux("storage"):
|
|
if self.is_virtual_disk(disk, raid_devices):
|
|
continue
|
|
size = int(getattr(disk, "size", 0)) / 1073741824
|
|
d = {
|
|
"name": "",
|
|
'Size': '{} GB'.format(size),
|
|
'logicalname': disk.get('logicalname'),
|
|
'description': disk.get('description'),
|
|
'SN': disk.get('serial'),
|
|
'Model': disk.get('product'),
|
|
'Type': disk.get('type'),
|
|
}
|
|
if disk.get('vendor'):
|
|
d['Vendor'] = disk['vendor']
|
|
else:
|
|
d['Vendor'] = get_vendor(disk['product'])
|
|
disks.append(d)
|
|
|
|
# remove duplicate serials
|
|
seen = set()
|
|
uniq = [x for x in disks if x['SN'] not in seen and not seen.add(x['SN'])]
|
|
return uniq
|
|
|
|
def create_netbox_disk(self, disk):
|
|
manufacturer = None
|
|
if "Vendor" in disk:
|
|
manufacturer = self.find_or_create_manufacturer(disk["Vendor"])
|
|
|
|
logicalname = disk.get('logicalname')
|
|
desc = disk.get('description')
|
|
name = '{} ({})'.format(disk['Model'], disk['Size'])
|
|
description = disk['Type']
|
|
sn = disk.get('SN', 'unknown')
|
|
|
|
parms = {
|
|
'device': self.device_id,
|
|
'discovered': True,
|
|
'tags': [{'name': INVENTORY_TAG['disk']['name']}],
|
|
'name': name,
|
|
'serial': sn,
|
|
'part_id': disk['Model'],
|
|
'description': description,
|
|
'manufacturer': getattr(manufacturer, "id", None),
|
|
}
|
|
if config.process_virtual_drives:
|
|
parms['custom_fields'] = disk.get("custom_fields", {})
|
|
|
|
_ = nb.dcim.inventory_items.create(**parms)
|
|
|
|
logging.info('Creating Disk {model} {serial}'.format(
|
|
model=disk['Model'],
|
|
serial=sn,
|
|
))
|
|
|
|
def dump_disks_map(self, disks):
|
|
disk_map = [d['custom_fields'] for d in disks if 'custom_fields' in d]
|
|
if config.dump_disks_map == "-":
|
|
f = sys.stdout
|
|
else:
|
|
f = open(config.dump_disks_map, "w")
|
|
f.write(
|
|
json.dumps(
|
|
disk_map,
|
|
separators=(',', ':'),
|
|
indent=4,
|
|
sort_keys=True
|
|
)
|
|
)
|
|
if config.dump_disks_map != "-":
|
|
f.close()
|
|
|
|
def do_netbox_disks(self):
|
|
nb_disks = self.get_netbox_inventory(
|
|
device_id=self.device_id,
|
|
tag=INVENTORY_TAG['disk']['slug']
|
|
)
|
|
disks = self.get_hw_disks()
|
|
if config.dump_disks_map:
|
|
try:
|
|
self.dump_disks_map(disks)
|
|
except Exception as e:
|
|
logging.error("Failed to dump disks map: {}".format(e))
|
|
logging.debug(traceback.format_exc())
|
|
disk_serials = [d['SN'] for d in disks if 'SN' in d]
|
|
|
|
# delete disks that are in netbox but not locally
|
|
# use the serial_number has the comparison element
|
|
for nb_disk in nb_disks:
|
|
if nb_disk.serial not in disk_serials or \
|
|
config.force_disk_refresh:
|
|
logging.info('Deleting unknown locally Disk {serial}'.format(
|
|
serial=nb_disk.serial,
|
|
))
|
|
nb_disk.delete()
|
|
|
|
if config.force_disk_refresh:
|
|
nb_disks = self.get_netbox_inventory(
|
|
device_id=self.device_id,
|
|
tag=INVENTORY_TAG['disk']['slug']
|
|
)
|
|
|
|
# create disks that are not in netbox
|
|
for disk in disks:
|
|
if disk.get('SN') not in [d.serial for d in nb_disks]:
|
|
self.create_netbox_disk(disk)
|
|
|
|
def create_netbox_memory(self, memory):
|
|
manufacturer = self.find_or_create_manufacturer(memory['vendor'])
|
|
name = 'Slot {} ({}GB)'.format(memory['slot'], memory['size'])
|
|
nb_memory = nb.dcim.inventory_items.create(
|
|
device=self.device_id,
|
|
discovered=True,
|
|
manufacturer=manufacturer.id,
|
|
tags=[{'name': INVENTORY_TAG['memory']['name']}],
|
|
name=name,
|
|
part_id=memory['product'],
|
|
serial=memory['serial'],
|
|
description=memory['description'],
|
|
)
|
|
|
|
logging.info('Creating Memory {location} {type} {size}GB'.format(
|
|
location=memory['slot'],
|
|
type=memory['product'],
|
|
size=memory['size'],
|
|
))
|
|
|
|
return nb_memory
|
|
|
|
def do_netbox_memories(self):
|
|
memories = self.lshw.memories
|
|
nb_memories = self.get_netbox_inventory(
|
|
device_id=self.device_id,
|
|
tag=INVENTORY_TAG['memory']['slug']
|
|
)
|
|
|
|
for nb_memory in nb_memories:
|
|
if nb_memory.serial not in [x['serial'] for x in memories]:
|
|
logging.info('Deleting unknown locally Memory {serial}'.format(
|
|
serial=nb_memory.serial,
|
|
))
|
|
nb_memory.delete()
|
|
|
|
for memory in memories:
|
|
if memory.get('serial') not in [x.serial for x in nb_memories]:
|
|
self.create_netbox_memory(memory)
|
|
|
|
def create_netbox_gpus(self, gpus):
|
|
for gpu in gpus:
|
|
if 'product' in gpu and len(gpu['product']) > 50:
|
|
gpu['product'] = (gpu['product'][:48] + '..')
|
|
|
|
manufacturer = self.find_or_create_manufacturer(gpu["vendor"])
|
|
_ = nb.dcim.inventory_items.create(
|
|
device=self.device_id,
|
|
manufacturer=manufacturer.id,
|
|
discovered=True,
|
|
tags=[{'name': INVENTORY_TAG['gpu']['name']}],
|
|
name=gpu['product'],
|
|
description=gpu['description'],
|
|
)
|
|
|
|
logging.info('Creating GPU model {}'.format(gpu['product']))
|
|
|
|
def is_external_gpu(self, gpu):
|
|
is_3d_gpu = gpu['description'].startswith('3D')
|
|
return self.server.is_blade() and \
|
|
self.server.own_gpu_expansion_slot() and is_3d_gpu
|
|
|
|
def do_netbox_gpus(self):
|
|
gpus = []
|
|
gpu_models = {}
|
|
for gpu in self.lshw.get_hw_linux('gpu'):
|
|
# Filters GPU if an expansion bay is detected:
|
|
# The internal (VGA) GPU only goes into the blade inventory,
|
|
# the external (3D) GPU goes into the expansion blade.
|
|
if config.expansion_as_device and \
|
|
self.update_expansion ^ self.is_external_gpu(gpu):
|
|
continue
|
|
gpus.append(gpu)
|
|
gpu_models.setdefault(gpu["product"], 0)
|
|
gpu_models[gpu["product"]] += 1
|
|
|
|
nb_gpus = self.get_netbox_inventory(
|
|
device_id=self.device_id,
|
|
tag=INVENTORY_TAG['gpu']['slug'],
|
|
)
|
|
nb_gpu_models = {}
|
|
for gpu in nb_gpus:
|
|
nb_gpu_models.setdefault(str(gpu), 0)
|
|
nb_gpu_models[str(gpu)] += 1
|
|
up_to_date = set(gpu_models) == set(nb_gpu_models)
|
|
if not gpus or not up_to_date:
|
|
for x in nb_gpus:
|
|
x.delete()
|
|
if gpus and not up_to_date:
|
|
self.create_netbox_gpus(gpus)
|
|
|
|
def create_or_update(self):
|
|
if config.inventory is None or config.update_inventory is None:
|
|
return False
|
|
if self.update_expansion is False:
|
|
self.do_netbox_cpus()
|
|
self.do_netbox_memories()
|
|
self.do_netbox_interfaces()
|
|
self.do_netbox_motherboard()
|
|
self.do_netbox_gpus()
|
|
self.do_netbox_disks()
|
|
self.do_netbox_raid_cards()
|
|
return True
|