From 3f095122560af45e92f39df97be87a30b1a7f86e Mon Sep 17 00:00:00 2001 From: Dalton Rardin <8892319+dalrrard@users.noreply.github.com> Date: Wed, 25 Aug 2021 09:52:50 -0500 Subject: [PATCH] Refactored import functions to use bulk POSTs and list GETs instead of individual ones --- nb-dt-import.py | 443 ++++++++++++++++++++++++++---------------------- 1 file changed, 245 insertions(+), 198 deletions(-) diff --git a/nb-dt-import.py b/nb-dt-import.py index 49c3997..9d4a346 100755 --- a/nb-dt-import.py +++ b/nb-dt-import.py @@ -81,252 +81,299 @@ def readYAMl(files): def createManufacturers(vendors, nb): + all_manufacturers = {str(item): item for item in nb.dcim.manufacturers.all()} + need_manufacturers = [] for vendor in vendors: try: - manGet = nb.dcim.manufacturers.get(name=vendor["name"]) - if manGet: - print(f'Manufacturer Exists: {manGet.name} - {manGet.id}') - else: - manSuccess = nb.dcim.manufacturers.create(vendor) - print(f'Manufacturer Created: {manSuccess.name} - ' - + f'{manSuccess.id}') - counter.update({'manufacturer': 1}) - except pynetbox.RequestError as e: - print(e.error) + manGet = all_manufacturers[vendor["name"]] + print(f'Manufacturer Exists: {manGet.name} - {manGet.id}') + except KeyError: + need_manufacturers.append(vendor) + + if not need_manufacturers: + return + + try: + manSuccess = nb.dcim.manufacturers.create(need_manufacturers) + for man in manSuccess: + print(f'Manufacturer Created: {man.name} - ' + + f'{man.id}') + counter.update({'manufacturer': 1}) + except pynetbox.RequestError as e: + print(e.error) def createInterfaces(interfaces, deviceType, nb): + all_interfaces = {str(item): item for item in nb.dcim.interface_templates.filter(devicetype_id=deviceType)} + need_interfaces = [] for interface in interfaces: - interface['device_type'] = deviceType try: - ifGet = nb.dcim.interface_templates.get(devicetype_id=deviceType, - name=interface["name"]) - if ifGet: - print(f'Interface Template Exists: {ifGet.name} - {ifGet.type}' - + f' - {ifGet.device_type.id} - {ifGet.id}') - else: - ifSuccess = nb.dcim.interface_templates.create(interface) - print(f'Interface Template Created: {ifSuccess.name} - ' - + f'{ifSuccess.type} - {ifSuccess.device_type.id} - ' - + f'{ifSuccess.id}') - counter.update({'updated': 1}) - except pynetbox.RequestError as e: - print(e.error) + ifGet = all_interfaces[interface["name"]] + print(f'Interface Template Exists: {ifGet.name} - {ifGet.type}' + + f' - {ifGet.device_type.id} - {ifGet.id}') + except KeyError: + interface['device_type'] = deviceType + need_interfaces.append(interface) + + if not need_interfaces: + return + + try: + ifSuccess = nb.dcim.interface_templates.create(need_interfaces) + for intf in ifSuccess: + print(f'Interface Template Created: {intf.name} - ' + + f'{intf.type} - {intf.device_type.id} - ' + + f'{intf.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) + def createConsolePorts(consoleports, deviceType, nb): + all_consoleports = {str(item): item for item in nb.dcim.console_port_templates.filter(devicetype_id=deviceType)} + need_consoleports = [] for consoleport in consoleports: - consoleport['device_type'] = deviceType try: - cpGet = nb.dcim.console_port_templates.get( - devicetype_id=deviceType, name=consoleport["name"]) - if cpGet: - print(f'Console Port Template Exists: {cpGet.name} - ' - + f'{cpGet.type} - {cpGet.device_type.id} - {cpGet.id}') - else: - cpSuccess = nb.dcim.console_port_templates.create(consoleport) - print(f'Console Port Created: {cpSuccess.name} - ' - + f'{cpSuccess.type} - {cpSuccess.device_type.id} - ' - + f'{cpSuccess.id}') - counter.update({'updated': 1}) - except pynetbox.RequestError as e: - print(e.error) + cpGet = all_consoleports[consoleport["name"]] + print(f'Console Port Template Exists: {cpGet.name} - ' + + f'{cpGet.type} - {cpGet.device_type.id} - {cpGet.id}') + except KeyError: + consoleport['device_type'] = deviceType + need_consoleports.append(consoleport) + + if not need_consoleports: + return + + try: + cpSuccess = nb.dcim.console_port_templates.create(need_consoleports) + for port in cpSuccess: + print(f'Console Port Created: {port.name} - ' + + f'{port.type} - {port.device_type.id} - ' + + f'{port.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) def createPowerPorts(powerports, deviceType, nb): + all_power_ports = {str(item): item for item in nb.dcim.power_port_templates.filter(devicetype_id=deviceType)} + need_power_ports = [] for powerport in powerports: - powerport['device_type'] = deviceType try: - ppGet = nb.dcim.power_port_templates.get(devicetype_id=deviceType, - name=powerport["name"]) - if ppGet: - print(f'Power Port Template Exists: {ppGet.name} - ' - + f'{ppGet.type} - {ppGet.device_type.id} - {ppGet.id}') - else: - ppSuccess = nb.dcim.power_port_templates.create(powerport) - print(f'Power Port Created: {ppSuccess.name} - ' - + f'{ppSuccess.type} - {ppSuccess.device_type.id} - ' - + f'{ppSuccess.id}') - counter.update({'updated': 1}) - except pynetbox.RequestError as e: - print(e.error) + ppGet = all_power_ports[powerport["name"]] + print(f'Power Port Template Exists: {ppGet.name} - ' + + f'{ppGet.type} - {ppGet.device_type.id} - {ppGet.id}') + except KeyError: + powerport['device_type'] = deviceType + need_power_ports.append(powerport) + + if not need_power_ports: + return + + try: + ppSuccess = nb.dcim.power_port_templates.create(need_power_ports) + for pp in ppSuccess: + print(f'Interface Template Created: {pp.name} - ' + + f'{pp.type} - {pp.device_type.id} - ' + + f'{pp.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) def createConsoleServerPorts(consoleserverports, deviceType, nb): + all_consoleserverports = {str(item): item for item in nb.dcim.console_server_port_templates.filter(devicetype_id=deviceType)} + need_consoleserverports = [] for csport in consoleserverports: - csport['device_type'] = deviceType try: - cspGet = nb.dcim.console_server_port_templates.get( - devicetype_id=deviceType, name=csport["name"]) - if cspGet: - print(f'Console Server Port Template Exists: {cspGet.name} - ' - + f'{cspGet.type} - {cspGet.device_type.id} - ' - + f'{cspGet.id}') - else: - cspSuccess = nb.dcim.console_server_port_templates.create( - csport) - print(f'Console Server Port Created: {cspSuccess.name} - ' - + f'{cspSuccess.type} - {cspSuccess.device_type.id} - ' - + f'{cspSuccess.id}') - counter.update({'updated': 1}) - except pynetbox.RequestError as e: - print(e.error) + cspGet = all_consoleserverports[csport["name"]] + print(f'Console Server Port Template Exists: {cspGet.name} - ' + + f'{cspGet.type} - {cspGet.device_type.id} - ' + + f'{cspGet.id}') + except KeyError: + csport['device_type'] = deviceType + need_consoleserverports.append(csport) + + if not need_consoleserverports: + return + + try: + cspSuccess = nb.dcim.console_server_port_templates.create( + need_consoleserverports) + for csp in cspSuccess: + print(f'Console Server Port Created: {csp.name} - ' + + f'{csp.type} - {csp.device_type.id} - ' + + f'{csp.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) def createFrontPorts(frontports, deviceType, nb): + all_frontports = {str(item): item for item in nb.dcim.front_port_templates.filter(devicetype_id=deviceType)} + need_frontports = [] for frontport in frontports: - frontport['device_type'] = deviceType try: - fpGet = nb.dcim.front_port_templates.get(devicetype_id=deviceType, - name=frontport["name"]) - if fpGet: - print(f'Front Port Template Exists: {fpGet.name} - ' - + f'{fpGet.type} - {fpGet.device_type.id} - {fpGet.id}') - else: - rpGet = nb.dcim.rear_port_templates.get( - devicetype_id=deviceType, name=frontport["rear_port"]) - if rpGet: - frontport['rear_port'] = rpGet.id - fpSuccess = nb.dcim.front_port_templates.create(frontport) - print(f'Front Port Created: {fpSuccess.name} - ' - + f'{fpSuccess.type} - {fpSuccess.device_type.id} - ' - + f'{fpSuccess.id}') - counter.update({'updated': 1}) - except pynetbox.RequestError as e: - print(e.error) + fpGet = all_frontports[frontport["name"]] + print(f'Front Port Template Exists: {fpGet.name} - ' + + f'{fpGet.type} - {fpGet.device_type.id} - {fpGet.id}') + except KeyError: + frontport['device_type'] = deviceType + need_frontports.append(frontport) + + if not need_frontports: + return + + all_rearports = {str(item): item for item in nb.dcim.rear_port_templates.filter(devicetype_id=deviceType)} + for port in need_frontports: + try: + rpGet = all_rearports[port["rear_port"]] + port['rear_port'] = rpGet.id + except KeyError: + print(f'Could not find Rear Port for Front Port: {port["name"]} - ' + + f'{port["type"]} - {deviceType}') + + try: + fpSuccess = nb.dcim.front_port_templates.create(need_frontports) + for fp in fpSuccess: + print(f'Front Port Created: {fp.name} - ' + + f'{fp.type} - {fp.device_type.id} - ' + + f'{fp.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) def createRearPorts(rearports, deviceType, nb): + all_rearports = {str(item): item for item in nb.dcim.rear_port_templates.filter(devicetype_id=deviceType)} + need_rearports = [] for rearport in rearports: - rearport['device_type'] = deviceType try: - rpGet = nb.dcim.rear_port_templates.get(devicetype_id=deviceType, - name=rearport["name"]) - if rpGet: - print(f'Rear Port Template Exists: {rpGet.name} - {rpGet.type}' - + f' - {rpGet.device_type.id} - {rpGet.id}') - else: - rpSuccess = nb.dcim.rear_port_templates.create(rearport) - print(f'Rear Port Created: {rpSuccess.name} - {rpSuccess.type}' - + f' - {rpSuccess.device_type.id} - {rpSuccess.id}') - counter.update({'updated': 1}) - except pynetbox.RequestError as e: - print(e.error) + rpGet = all_rearports[rearport["name"]] + print(f'Rear Port Template Exists: {rpGet.name} - {rpGet.type}' + + f' - {rpGet.device_type.id} - {rpGet.id}') + except KeyError: + rearport['device_type'] = deviceType + need_rearports.append(rearport) + + if not need_rearports: + return + + try: + rpSuccess = nb.dcim.rear_port_templates.create( + need_rearports) + for rp in rpSuccess: + print(f'Rear Port Created: {rp.name} - {rp.type}' + + f' - {rp.device_type.id} - {rp.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) def createDeviceBays(devicebays, deviceType, nb): + all_devicebays = {str(item): item for item in nb.dcim.device_bay_templates.filter(devicetype_id=deviceType)} + need_devicebays = [] for devicebay in devicebays: - devicebay['device_type'] = deviceType try: - dbGet = nb.dcim.device_bay_templates.get(devicetype_id=deviceType, - name=devicebay["name"]) - if dbGet: - print(f'Device Bay Template Exists: {dbGet.name} - ' - + f'{dbGet.device_type.id} - {dbGet.id}') - else: - dbSuccess = nb.dcim.device_bay_templates.create(devicebay) - print(f'Device Bay Created: {dbSuccess.name} - ' - + f'{dbSuccess.device_type.id} - {dbSuccess.id}') - counter.update({'updated': 1}) - except pynetbox.RequestError as e: - print(e.error) + dbGet = all_devicebays[devicebay["name"]] + print(f'Device Bay Template Exists: {dbGet.name} - ' + + f'{dbGet.device_type.id} - {dbGet.id}') + except KeyError: + devicebay['device_type'] = deviceType + need_devicebays.append(devicebay) + + if not need_devicebays: + return + + try: + dbSuccess = nb.dcim.device_bay_templates.create(need_devicebays) + for db in dbSuccess: + print(f'Device Bay Created: {db.name} - ' + + f'{db.device_type.id} - {db.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) def createPowerOutlets(poweroutlets, deviceType, nb): + all_poweroutlets = {str(item): item for item in nb.dcim.power_outlet_templates.filter(devicetype_id=deviceType)} + need_poweroutlets = [] for poweroutlet in poweroutlets: try: - poGet = nb.dcim.power_outlet_templates.get( - devicetype_id=deviceType, name=poweroutlet["name"]) - if poGet: - print(f'Power Outlet Template Exists: {poGet.name} - ' - + f'{poGet.type} - {poGet.device_type.id} - {poGet.id}') - else: - try: - ppGet = nb.dcim.power_port_templates.get( - devicetype_id=deviceType) - if ppGet: - poweroutlet["power_port"] = ppGet.id - poweroutlet["device_type"] = deviceType - poSuccess = nb.dcim.power_outlet_templates.create( - poweroutlet) - print(f'Power Outlet Created: {poSuccess.name} - ' - + f'{poSuccess.type} - ' - + f'{poSuccess.device_type.id} - {poSuccess.id}') - counter.update({'updated': 1}) - except: - poweroutlet["device_type"] = deviceType - poSuccess = nb.dcim.power_outlet_templates.create( - poweroutlet) - print(f'Power Outlet Created: {poSuccess.name} - ' - + f'{poSuccess.type} - {poSuccess.device_type.id} - ' - + f'{poSuccess.id}') - counter.update({'updated': 1}) - except pynetbox.RequestError as e: - print(e.error) + poGet = all_poweroutlets[poweroutlet["name"]] + print(f'Power Outlet Template Exists: {poGet.name} - ' + + f'{poGet.type} - {poGet.device_type.id} - {poGet.id}') + except KeyError: + poweroutlet["device_type"] = deviceType + need_poweroutlets.append(poweroutlet) + + if not need_poweroutlets: + return + + all_power_ports = {str(item): item for item in nb.dcim.power_port_templates.filter(devicetype_id=deviceType)} + for outlet in need_poweroutlets: + try: + ppGet = all_power_ports[outlet["power_port"]] + outlet['power_port'] = ppGet.id + except KeyError: + pass + + try: + poSuccess = nb.dcim.power_outlet_templates.create( + need_poweroutlets) + for po in poSuccess: + print(f'Power Outlet Created: {po.name} - ' + + f'{po.type} - {po.device_type.id} - ' + + f'{po.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) def createDeviceTypes(deviceTypes, nb): + all_device_types = {str(item): item for item in nb.dcim.device_types.all()} for deviceType in deviceTypes: try: - dtGet = nb.dcim.device_types.get(model=deviceType["model"]) - if dtGet: - print(f'Device Type Exists: {dtGet.manufacturer.name} - ' - + f'{dtGet.model} - {dtGet.id}') - if "interfaces" in deviceType: - createInterfaces(deviceType["interfaces"], dtGet.id, nb) - if "power-ports" in deviceType: - createPowerPorts(deviceType["power-ports"], dtGet.id, nb) - if "power-port" in deviceType: - createPowerPorts(deviceType["power-port"], dtGet.id, nb) - if "console-ports" in deviceType: - createConsolePorts(deviceType["console-ports"], - dtGet.id, nb) - if "power-outlets" in deviceType: - createPowerOutlets(deviceType["power-outlets"], - dtGet.id, nb) - if "console-server-ports" in deviceType: - createConsoleServerPorts( - deviceType["console-server-ports"], dtGet.id, nb) - if "rear-ports" in deviceType: - createRearPorts(deviceType["rear-ports"], dtGet.id, nb) - if "front-ports" in deviceType: - createFrontPorts(deviceType["front-ports"], dtGet.id, nb) - if "device-bays" in deviceType: - createDeviceBays(deviceType["device-bays"], dtGet.id, nb) - else: - dtSuccess = nb.dcim.device_types.create(deviceType) + dt = all_device_types[deviceType["model"]] + print(f'Device Type Exists: {dt.manufacturer.name} - ' + + f'{dt.model} - {dt.id}') + except KeyError: + try: + dt = nb.dcim.device_types.create(deviceType) counter.update({'added': 1}) - print(f'Device Type Created: {dtSuccess.manufacturer.name} - ' - + f'{dtSuccess.model} - {dtSuccess.id}') - if "interfaces" in deviceType: - createInterfaces(deviceType["interfaces"], - dtSuccess.id, nb) - if "power-ports" in deviceType: - createPowerPorts(deviceType["power-ports"], - dtSuccess.id, nb) - if "power-port" in deviceType: - createPowerPorts(deviceType["power-port"], - dtSuccess.id, nb) - if "console-ports" in deviceType: - createConsolePorts(deviceType["console-ports"], - dtSuccess.id, nb) - if "power-outlets" in deviceType: - createPowerOutlets(deviceType["power-outlets"], - dtSuccess.id, nb) - if "console-server-ports" in deviceType: - createConsoleServerPorts( - deviceType["console-server-ports"], dtSuccess.id, nb) - if "rear-ports" in deviceType: - createRearPorts(deviceType["rear-ports"], - dtSuccess.id, nb) - if "front-ports" in deviceType: - createFrontPorts(deviceType["front-ports"], - dtSuccess.id, nb) - if "device-bays" in deviceType: - createDeviceBays(deviceType["device-bays"], - dtSuccess.id, nb) - except pynetbox.RequestError as e: - print(e.error) + print(f'Device Type Created: {dt.manufacturer.name} - ' + + f'{dt.model} - {dt.id}') + except pynetbox.RequestError as e: + print(e.error) + + if "interfaces" in deviceType: + createInterfaces(deviceType["interfaces"], + dt.id, nb) + if "power-ports" in deviceType: + createPowerPorts(deviceType["power-ports"], + dt.id, nb) + if "power-port" in deviceType: + createPowerPorts(deviceType["power-port"], + dt.id, nb) + if "console-ports" in deviceType: + createConsolePorts(deviceType["console-ports"], + dt.id, nb) + if "power-outlets" in deviceType: + createPowerOutlets(deviceType["power-outlets"], + dt.id, nb) + if "console-server-ports" in deviceType: + createConsoleServerPorts( + deviceType["console-server-ports"], dt.id, nb) + if "rear-ports" in deviceType: + createRearPorts(deviceType["rear-ports"], + dt.id, nb) + if "front-ports" in deviceType: + createFrontPorts(deviceType["front-ports"], + dt.id, nb) + if "device-bays" in deviceType: + createDeviceBays(deviceType["device-bays"], + dt.id, nb) try: