Collection of CheckMK checks (see https://checkmk.com/). All checks and plugins are provided as is. Absolutely no warranty. Send any comments to thl-cmk[at]outlook[dot]com

Skip to content
Snippets Groups Projects
Commit 9976a268 authored by thl-cmk's avatar thl-cmk :flag_na:
Browse files

update project

parent 2161f487
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 418 additions and 305 deletions
README.md
+ 1
− 1
View file @ 9976a268
[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.0.8-20231020.mkp "create_topology_data-0.0.8-20231020.mkp"
[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.0.9-20231027.mkp "create_topology_data-0.0.9-20231027.mkp"
# PoC for Network Visualization data creation from inventory data
This script creates the topology data file needed for the [Checkmk Exchange Network visualization](https://exchange.checkmk.com/p/network-visualization) plugin by Andreas Boesl and [schnetz](https://exchange.checkmk.com/u/schnetz).\
......
bin/topology_data/create_topology_args.py 0 → 100755
+ 177
− 0
View file @ 9976a268
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# License: GNU General Public License v2
# Author: thl-cmk[at]outlook[dot]com
# URL : https://thl-cmk.hopto.org
# Date : 2023-10-12
# File : create_topology_utils.py
#
# options used
# -d --default
# -o --output-directory
# -s --seed-devices
# -u --user-data-file
# -v --version
# --check-user-data-only
# --data-source
# --debug
# --dont-compare
# --inventory-columns
# --keep-domain
# --keep
# --lldp
# --lowercase
# --min-age
# --path-in-inventory
# --time-format
# --uppercase
from argparse import (
Namespace as arg_Namespace,
ArgumentParser,
RawTextHelpFormatter,
)
from create_topology_utils import(
CREATE_TOPOLOGY_VERSION,
SCRIPT,
SAMPLE_SEEDS,
USER_DATA_FILE,
LABEL_CDP,
COLUMNS_CDP,
LABEL_LLDP,
PATH_LLDP,
COLUMNS_LLDP,
PATH_CDP
)
def parse_arguments() -> arg_Namespace:
parser = ArgumentParser(
prog='create_topology_data.py',
description='This script creates the topology data file needed for the Checkmk "network_visualization"\n'
'plugin by Andreas Boesl and schnetz. For more information see\n'
'the announcement from schnetz: https://forum.checkmk.com/t/network-visualization/41680\n'
'and the plugin on the Exchange: https://exchange.checkmk.com/p/network-visualization .\n'
'\n'
'The required inventory data can be created with my inventory plugins:\n'
'CDP: https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_cdp_cache\n'
'LLDP: https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_lldp_cache\n'
'\n'
f'\nVersion: {CREATE_TOPOLOGY_VERSION} | Written by: thl-cmk\n'
f'for more information see: https://thl-cmk.hopto.org',
formatter_class=RawTextHelpFormatter,
epilog='Usage:\n'
'for CDP (the default):\n'
f'{SCRIPT} -s {SAMPLE_SEEDS} -d\n'
'for LLDP:\n'
f'{SCRIPT} -s {SAMPLE_SEEDS} -d --lldp\n',
)
command_group = parser.add_mutually_exclusive_group()
parser.add_argument(
'-d', '--default', default=False, action='store_const', const=True,
help='Set the created topology data as default',
)
parser.add_argument(
'-m', '--merge',
nargs=2,
choices=['CDP', 'LLDP'],
help=f'Merge topologies. This runs the topology creation for CDP and LLDP.\n'
f'The topologies are then merged in the specified order.\n'
f'I.e. -m CDP LLDP merges the CDP topology into the LLDP topology, overwriting\n'
f'the LLDP data in case there are conflicts.\n'
f'NOTE: static connection data from the user file will always merged with the\n'
f' highest priority',
)
parser.add_argument(
'-o', '--output-directory', type=str,
help='Directory name where to save the topology data.\n'
'I.e.: my_topology. Default is the actual date/time\n'
'in "--time-format" format.\n'
'NOTE: the directory is a sub directory under "~/var/topology_data/"',
)
parser.add_argument(
'-s', '--seed-devices', type=str, nargs='+',
help=f'List of devices to start the topology discovery from.\n'
f'I.e. {SAMPLE_SEEDS}',
)
parser.add_argument(
'-u', '--user-data-file', type=str,
help='Set the name uf the user provided data file\n'
'Default is ~local/bin/topology_data/create_topology_data.toml\n',
)
parser.add_argument(
'-v', '--version', default=False, action='store_const', const=True,
help='Print version of this script and exit',
)
parser.add_argument(
'--check-user-data-only', default=False, action='store_const', const=True,
help=f'Only tries to read/parse the user data from {USER_DATA_FILE} and exits.',
)
parser.add_argument(
'--data-source', type=str,
help=f'The source from which the topology data originates.\n'
f'I.e. {LABEL_CDP} for CDP data from the inventory.\n'
'NOTE: right now this only an unused label.',
)
parser.add_argument(
'--debug', default=False, action='store_const', const=True,
help='Print debug information',
)
parser.add_argument(
'--dont-compare', default=False, action='store_const', const=True,
help='Do not compare the actual topology data with the default topology\n'
'data. By default, the actual topology is compared with the default\n'
'topology. If the data matches, the actual topology is not saved.\n'
'So, if you run this tool in a cron job, a new topology will be\n'
'created only if there was a change, unless you use "--dont-compare".'
)
parser.add_argument(
'--inventory-columns', type=str,
help=f'Columns used from the inventory data.\n'
f'I.e. "{COLUMNS_CDP}"\n'
'NOTE: the columns must be in the order: neighbour, local_port,\n'
'neighbour_port',
)
parser.add_argument(
'--keep-domain', default=False, action='store_const', const=True,
help='Do not remove the domain name from the neighbor name',
)
parser.add_argument(
'--keep', type=int,
help=f'Number of topologies to keep. The oldest topologies above keep\n'
f'max will be deleted. The minimum value for --keep is 1.\n'
f'NOTE: The default topologies will be always kept.\n'
)
parser.add_argument(
'--lldp', default=False, action='store_const', const=True,
help=f'Sets data source to {LABEL_LLDP}, inventory path \n'
f'to "{PATH_LLDP}" and columns\n'
f'to "{COLUMNS_LLDP}"',
)
command_group.add_argument(
'--lowercase', default=False, action='store_const', const=True,
help='Change neighbour names to all lower case',
)
parser.add_argument(
'--min-age', type=int,
help=f'The minimum number of days before a topology is deleted\n'
f'by "--keep".\n'
f'NOTE: Topologies that are not older than 1 days are always kept.',
)
parser.add_argument(
'--path-in-inventory', type=str,
help=f'Checkmk inventory path to the topology data.\n'
f'I.e. "{PATH_CDP}"',
)
parser.add_argument(
'--time-format', type=str,
help='Format string to render the time. (default: %%Y-%%m-%%dT%%H:%%M:%%S.%%m)',
)
command_group.add_argument(
'--uppercase', default=False, action='store_const', const=True,
help='Change neighbour names to all upper case',
)
return parser.parse_args()
bin/topology_data/create_topology_classes.py
+ 93
− 22
View file @ 9976a268
......@@ -10,17 +10,27 @@
from os import environ
from pathlib import Path
from time import strftime
from typing import Dict, List, Any, NamedTuple
from enum import Enum, unique
from create_topology_utils import (
cdp_path,
cdp_columns,
cdp_label,
lldp_path,
lldp_columns,
lldp_label,
user_data_file,
CREATE_TOPOLOGY_VERSION,
PATH_CDP,
COLUMNS_CDP,
LABEL_CDP,
PATH_LLDP,
COLUMNS_LLDP,
LABEL_LLDP,
PATH_INTERFACES,
USER_DATA_FILE,
LQ_INTERFACES,
SCRIPT,
get_inventory_data,
get_table_from_inventory,
get_interface_items_from_lq,
ExitCodes,
get_data_from_toml,
)
......@@ -32,8 +42,14 @@ class TopologyParams(NamedTuple):
@unique
class Topologies(Enum):
CDP = TopologyParams(path=cdp_path, columns=cdp_columns, label=cdp_label)
LLDP = TopologyParams(path=lldp_path, columns=lldp_columns, label=lldp_label)
CDP = TopologyParams(path=PATH_CDP, columns=COLUMNS_CDP, label=LABEL_CDP)
LLDP = TopologyParams(path=PATH_LLDP, columns=COLUMNS_LLDP, label=LABEL_LLDP)
@unique
class CacheSources(Enum):
inventory = 'inventory'
lq = 'lq'
class InventoryColumns(NamedTuple):
......@@ -72,11 +88,11 @@ class Settings:
self.__settings = {
'seed_devices': None,
'path_in_inventory': cdp_path,
'inventory_columns': cdp_columns,
'data_source': cdp_label,
'path_in_inventory': PATH_CDP,
'inventory_columns': COLUMNS_CDP,
'data_source': LABEL_CDP,
'time_format': '%Y-%m-%dT%H:%M:%S.%m',
'user_data_file': f'{self.__omd_root}/local/bin/topology_data/{user_data_file}',
'user_data_file': f'{self.__omd_root}/local/bin/topology_data/{USER_DATA_FILE}',
'merge': None,
'output_directory': None,
'default': False,
......@@ -96,16 +112,31 @@ class Settings:
self.__args = ({k.split(',')[-1].strip(' ').strip('_'): v for k, v in cli_args.items() if v})
# first set values
if self.__args.get('lldp'):
self.__settings['data_source'] = lldp_label
self.__settings['path_in_inventory'] = lldp_path
self.__settings['inventory_columns'] = lldp_columns
self.__settings['data_source'] = LABEL_LLDP
self.__settings['path_in_inventory'] = PATH_LLDP
self.__settings['inventory_columns'] = COLUMNS_LLDP
# Then update values with cli values
self.__settings.update(self.__args)
if not self.merge:
if not self.lldp:
self.merge = ['CDP']
else:
if self.version:
print(f'{Path(SCRIPT).name} version: {CREATE_TOPOLOGY_VERSION}')
exit(code=ExitCodes.OK.value)
if self.check_user_data_only:
user_data = get_data_from_toml(file=self.user_data_file)
print(f'Could read/parse the user data from {self.user_data_file}')
exit(code=ExitCodes.OK.value)
if self.merge:
_merge = list(set(self.merge.copy()))
if len(_merge) != len(self.merge):
print(f'-m/--merge options must be unique. Use "-m CDP LLDP" oder "-m LLDP CDP"')
exit(code=ExitCodes.BAD_OPTION_LIST.value)
else:
if self.lldp:
self.merge = ['LLDP']
else:
self.merge = ['CDP']
def set_topology_param(self, topology: TopologyParams):
self.__settings['path_in_inventory'] = topology.path
......@@ -199,8 +230,9 @@ class Settings:
@property
def path_in_inventory(self) -> List[str]:
path = ('Nodes,' + ',Nodes,'.join(self.__settings['path_in_inventory'].split(',')) + ',Table,Rows').split(',')
return path
# path = ('Nodes,' + ',Nodes,'.join(self.__settings['path_in_inventory'].split(',')) + ',Table,Rows').split(',')
# return path
return self.__settings['path_in_inventory']
@property
def path_to_if_table(self) -> List[str]:
......@@ -233,3 +265,42 @@ class Settings:
return f'{strftime(self.__settings["time_format"])}'
else:
return self.__settings['output_directory']
class HostCache:
def __init__(self):
self.__cache = {}
self.__inventory_pre_fetch_list: List[str] = [
PATH_CDP,
PATH_LLDP,
PATH_INTERFACES,
]
def __fill_cache(self, host: str):
# pre fill inventory data
inventory = get_inventory_data(host=host)
if inventory:
self.__cache[host][CacheSources.inventory.value] = {}
self.__cache[host][CacheSources.inventory.value].update({
entry: get_table_from_inventory(
inventory=inventory,
path=entry
) for entry in self.__inventory_pre_fetch_list
})
else:
self.__cache[host][CacheSources.inventory.value] = None
# prefill live status data
self.__cache[host][CacheSources.lq.value] = {}
self.__cache[host][CacheSources.lq.value][LQ_INTERFACES] = get_interface_items_from_lq(host)
def get_data(self, host: str, source: CacheSources, path: str):
if host not in self.__cache.keys():
self.__cache[host]: Dict[str, Any] = {}
self.__fill_cache(host=host)
try:
return self.__cache[host][source.value][path]
except (KeyError, TypeError) as e:
return None
def add_inventory_prefetch_path(self, path: str):
self.__inventory_pre_fetch_list = list(set(self.__inventory_pre_fetch_list + [path]))
bin/topology_data/create_topology_data.py
+ 74
− 125
View file @ 9976a268
......@@ -23,6 +23,8 @@
# added option -u, --user-data-file to provide a user specific data file
# 2023-10-20: changed option -m/--make-default to -d/--default (-m is needed for -m/--merge
# added -m/--merge option
# 2023-10-27: improved matching of port name from CDP/LLDP data to interface (item) name of service
# improved internal data handling (HostCache)
#
# PoC for creating topology_data.json from inventory data
......@@ -37,78 +39,39 @@
# NOTE: the topology_data configuration (layout etc.) is saved under ~/var/check_mk/topology
#
# The inventory data could be created with my CDP/LLDP inventory plugins:
#
# CDP: https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_cdp_cache\
# CDP path-in-inventory: 'networking,cdp_cache'
# CDP inventory-columns: 'device_id,local_port,device_port'
#
# LLDP: https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_lldp_cache\
# LLDP path-in-inventory: 'networking,lldp_cache'
# LLDP inventory-columns: 'system_name,local_port_num,port_id'
#
# USAGE CDP (is the default):
# ~/local/lib/topology_data/create_topology_data.py -s CORE01 -m
#
# USAGE LLDP:
# ~/local/lib/topology_data/create_topology_data.py -s CORE01 -m --lldp
# USAGE:
# ~/local/lib/topology_data/create_topology_data.py --help
#
from ast import literal_eval
from pathlib import Path
from typing import Dict, List, Any
from time import strftime, time_ns
from create_topology_utils import (
parse_arguments,
remove_old_data,
get_data_from_toml,
merge_topologies,
save_topology,
get_data_form_live_status,
ExitCodes,
LQ_INTERFACES,
PATH_INTERFACES,
)
from create_topology_args import parse_arguments
from create_topology_classes import (
InventoryColumns,
Settings,
StaticConnection,
Topologies,
TopologyParams,
HostCache,
CacheSources,
)
CREATE_TOPOLOGY_VERSION = '0.0.8-202310120'
ITEMS = {}
HOST_MAP: Dict[str, str] = {}
DROP_HOSTS: List[str] = []
STATIC_CONNECTIONS: Dict = {}
def get_interface_items_from_lq(host: str, debug: bool = False) -> Dict:
query = (
'GET services\n'
'Columns: host_name description check_command\n'
'Filter: description ~ Interface\n'
f'Filter: host_name = {host}\n'
'OutputFormat: python3\n'
)
data = get_data_form_live_status(query=query)
items = {}
for host, description, check_command in data:
if host:
if host not in items.keys():
items[host] = []
items[host].append(description[10:]) # remove 'Interface ' from description
interfaces = 0
for host in items.keys():
items[host] = list(set(items[host]))
items[host].sort()
interfaces += len(items[host])
# print(f'Interfaces found: {interfaces}')
return items
def get_list_of_devices(data) -> List[str]:
devices = []
for connection in data.values():
......@@ -116,66 +79,64 @@ def get_list_of_devices(data) -> List[str]:
return list(set(devices))
def get_service_by_interface(host: str, interface: str, debug: bool = False) -> str | None:
def get_service_by_interface(host: str, interface: str, debug: bool = False) -> str:
# try to find the item for an interface
def _match_entry_with_item(_entry: Dict[str, str]):
values = [_entry.get('name'), _entry.get('description'), entry.get('alias')]
for value in values:
if value in items:
return value
index = str(_entry.get('index'))
# for index try with padding
for i in range(1, 6):
index_padded = f'{index:0>{i}}'
if index_padded in items:
return index_padded
# still not found try values + index
for value in values:
if f'{value} {index_padded}' in items:
return f'{value} {index_padded}'
return interface
# empty host/neighbour should never happen here
if not host:
return interface
items = ITEMS.get(host, [])
# get list of interface items
items = HOST_CACHE.get_data(host=host, source=CacheSources.lq, path=LQ_INTERFACES)
# the easy case
if interface in items:
return interface
else:
if_padding = interface
if '/' in interface:
if_padding = '/'.join(interface.split('/')[:-1]) + f'/{interface.split("/")[-1]:0>2}'
for item in items:
# interface = Gi0/1
# item = Gi0/1 - Access port
# this might not catch the correct interface (fa0 and fa0/0)
item = item.split(' ')[0]
if item == interface:
return item
elif item == if_padding:
return item
# try to find the interface in the host interface inventory list
inventory = HOST_CACHE.get_data(host=host, source=CacheSources.inventory, path=PATH_INTERFACES)
if inventory:
for entry in inventory:
if interface in [
entry.get('name'),
entry.get('description'),
entry.get('alias'),
str(entry.get('index')),
entry.get('phys_address'),
]:
return _match_entry_with_item(entry)
if debug:
print(f'Device: {host}: interface ({interface}) not found in services')
print(f'Device: {host}: service for interface {interface} not found')
return interface
def get_inventory_data_lq(host: str, path: List[str], debug: bool = False, ) -> List[Dict[str, str]]:
query = f'GET hosts\nColumns: mk_inventory\nOutputFormat: python3\nFilter: host_name = {host}\n'
data = get_data_form_live_status(query=query)
if data:
try:
data = literal_eval(data[0][0].decode('utf-8'))
except SyntaxError as e:
if debug:
print(f'data: |{data}|')
print(f'type: {type(data)}')
print(f'exception: {e}')
return []
for m in path:
try:
data = data[m]
except KeyError:
return []
return data
return []
def create_device_from_inv(
host: str,
inv_data: List[Dict[str, str]],
inv_columns: InventoryColumns,
data_source: str,
debug: bool = False,
) -> Dict[str, Any] | None:
data = {'connections': {}, "interfaces": []}
if not host in ITEMS.keys():
ITEMS.update(get_interface_items_from_lq(host=host))
for topo_neighbour in inv_data:
neighbour = topo_neighbour.get(inv_columns.neighbour)
if not neighbour:
......@@ -194,18 +155,18 @@ def create_device_from_inv(
if neighbour in HOST_MAP.keys():
neighbour = HOST_MAP[neighbour]
# has to be done before checking interfaces
if not neighbour in ITEMS.keys():
ITEMS.update(get_interface_items_from_lq(host=neighbour))
# getting/checking interfaces
local_port = topo_neighbour.get(inv_columns.local_port)
local_port = get_service_by_interface(host, local_port)
_local_port = local_port
local_port = get_service_by_interface(host, local_port, debug=debug)
if debug and local_port != _local_port:
print(f'host: {host}, local_port: {local_port}, _local_port: {_local_port}')
neighbour_port = topo_neighbour.get(inv_columns.neighbour_port)
neighbour_port = get_service_by_interface(neighbour, neighbour_port)
_neighbour_port = neighbour_port
neighbour_port = get_service_by_interface(neighbour, neighbour_port, debug=debug)
if debug and neighbour_port != _neighbour_port:
print(f'neighbour: {neighbour}, neighbour_port {neighbour_port}, _neighbour_port {_neighbour_port}')
if neighbour and local_port and neighbour_port:
data['connections'].update({local_port: [neighbour, neighbour_port, data_source]})
......@@ -272,18 +233,20 @@ def create_topology(
if device in devices_done:
continue
topo_data = get_inventory_data_lq(host=device, path=path_in_inventory, debug=debug)
topology_data.update(
create_device_from_inv(
host=device,
inv_data=topo_data,
inv_columns=inv_columns,
data_source=data_source,
))
devices_list = get_list_of_devices(topology_data[device]['connections'])
for entry in devices_list:
if entry not in devices_done:
devices_to_go.append(entry)
# topo_data = get_inventory_data_lq(host=device, path=path_in_inventory, debug=debug)
topo_data = HOST_CACHE.get_data(host=device, source=CacheSources.inventory, path=path_in_inventory)
if topo_data:
topology_data.update(
create_device_from_inv(
host=device,
inv_data=topo_data,
inv_columns=inv_columns,
data_source=data_source,
))
devices_list = get_list_of_devices(topology_data[device]['connections'])
for entry in devices_list:
if entry not in devices_done:
devices_to_go.append(entry)
devices_to_go = list(set(devices_to_go))
devices_done.append(device)
......@@ -297,26 +260,12 @@ def create_topology(
if __name__ == '__main__':
SETTINGS = Settings(vars(parse_arguments(CREATE_TOPOLOGY_VERSION)))
if SETTINGS.version:
print(f'{Path(__file__).name} version: {CREATE_TOPOLOGY_VERSION}')
exit(code=ExitCodes.ok)
if SETTINGS.check_user_data_only:
user_data = get_data_from_toml(file=SETTINGS.user_data_file)
print(f'Could read/parse the user data from {SETTINGS.user_data_file}')
exit(code=ExitCodes.OK)
if SETTINGS.merge:
_merge = list(set(SETTINGS.merge.copy()))
if len(_merge) != len(SETTINGS.merge):
print(f'-m/--merge options must be unique. Use "-m cdp lldp" oder "-m lldp cdp"')
exit(code=ExitCodes.BAD_OPTION_LIST)
start_time = time_ns()
SETTINGS = Settings(vars(parse_arguments()))
HOST_CACHE = HostCache()
print()
print(f'Start time: {strftime(SETTINGS.time_format)}')
start_time = time_ns()
user_data = get_data_from_toml(file=SETTINGS.user_data_file)
HOST_MAP = user_data.get('HOST_MAP', {})
......@@ -353,7 +302,7 @@ if __name__ == '__main__':
remove_old_data(
keep=SETTINGS.keep,
min_age=SETTINGS.min_age,
path=Path(f'{SETTINGS.omd_root}/{SETTINGS.topology_save_path}'),
path=f'{SETTINGS.omd_root}/{SETTINGS.topology_save_path}',
protected=user_data.get('PROTECTED_TOPOLOGIES', []),
)
......
bin/topology_data/create_topology_data.toml 100755 → 100644
+ 12
− 12
View file @ 9976a268
......@@ -14,33 +14,33 @@
# list of (additional to -s/--seed-devices) seed devices
SEED_DEVICES = [
"CORE01",
"LOCATION01",
"LOCATION02",
# "CORE01",
# "LOCATION01",
# "LOCATION02",
]
# drop neighbours with invalid names
DROP_HOSTS = [
"not advertised",
"a nother invalid name",
# "not advertised",
# "a nother invalid name",
]
# topologies will not be deleted by "--keep"
PROTECTED_TOPOLOGIES = [
"2023-10-17T14:08:05.10",
"your_important_topology"
# "2023-10-17T14:08:05.10",
# "your_important_topology"
]
# user defined static connections
# connections will be added from host to neighbour and in reverese
# hosts/neighbours in this section will be added to SEED_DEVICES
STATIC_CONNECTIONS = [
["cmk_host1", "local-port1", "neighbour-port1", "neighbour1", "label"],
["cmk_host1", "local-port2", "neighbour-port2", "neighbour2", "label"],
# ["cmk_host1", "local-port1", "neighbour-port1", "neighbour1", "label"],
# ["cmk_host1", "local-port2", "neighbour-port2", "neighbour2", "label"],
]
# map inventory neighbour name to Checkmk host name
[HOST_MAP]
inventory_neighbour1 = "cmk_host1"
inventory_neighbour2 = "cmk_host2"
inventory_neighbour3 = "cmk_host3"
# inventory_neighbour1 = "cmk_host1"
# inventory_neighbour2 = "cmk_host2"
# inventory_neighbour3 = "cmk_host3"
bin/topology_data/create_topology_utils.py
+ 58
− 143
View file @ 9976a268
......@@ -37,12 +37,7 @@ from tomllib import loads as toml_loads
from tomllib import TOMLDecodeError
from re import match as re_match
from pathlib import Path
from typing import List, Dict
from argparse import (
Namespace as arg_Namespace,
ArgumentParser,
RawTextHelpFormatter,
)
from typing import List, Dict, Any
from enum import Enum, unique
......@@ -53,15 +48,19 @@ class ExitCodes(Enum):
BAD_TOML_FORMAT = 3
script = '~/local/bin/network-topology/create_topology_data.py'
sample_seeds = 'Core01 Core02'
cdp_path = 'networking,cdp_cache'
cdp_columns = 'device_id,local_port,device_port'
cdp_label = 'inv_CDP'
lldp_path = 'networking,lldp_cache'
lldp_columns = 'system_name,local_port_num,port_id'
lldp_label = 'inv_LLDP'
user_data_file = 'create_topology_data.toml'
# constants
CREATE_TOPOLOGY_VERSION = '0.0.9-202310127'
SCRIPT = '~/local/bin/network-topology/create_topology_data.py'
SAMPLE_SEEDS = 'Core01 Core02'
PATH_CDP = 'networking,cdp_cache'
PATH_LLDP = 'networking,lldp_cache'
PATH_INTERFACES = 'networking,interfaces'
LABEL_CDP = 'inv_CDP'
LABEL_LLDP = 'inv_LLDP'
COLUMNS_LLDP = 'system_name,local_port_num,port_id'
COLUMNS_CDP = 'device_id,local_port,device_port'
USER_DATA_FILE = 'create_topology_data.toml'
LQ_INTERFACES = 'interface_items'
def get_data_form_live_status(query: str):
......@@ -91,7 +90,7 @@ def get_data_from_toml(file: str, debug: bool = False) -> Dict:
except TOMLDecodeError as e:
print(
f'ERROR: data file {toml_file} is not in valid TOML format! ({e}), (see https://toml.io/en/)')
exit(code=ExitCodes.BAD_TOML_FORMAT)
exit(code=ExitCodes.BAD_TOML_FORMAT.value)
else:
print(f'WARNING: User data {file} not found.')
if debug:
......@@ -113,7 +112,8 @@ def rm_tree(root: Path):
root.rmdir()
def remove_old_data(keep: int, min_age: int, path: Path, protected: List[str], debug: bool = False):
def remove_old_data(keep: int, min_age: int, path: str, protected: List[str], debug: bool = False):
path = Path(path)
default_topo = path.joinpath('default')
directories = [str(directory) for directory in list(path.iterdir())]
# keep default top
......@@ -317,131 +317,46 @@ def is_equal_with_default(data: Dict, file: str) -> bool:
return compare_dicts(data, default_data)
def parse_arguments(create_topology_version: str) -> arg_Namespace:
parser = ArgumentParser(
prog='create_topology_data.py',
description='This script creates the topology data file needed for the Checkmk "network_visualization"\n'
'plugin by Andreas Boesl and schnetz. For more information see\n'
'the announcement from schnetz: https://forum.checkmk.com/t/network-visualization/41680\n'
'and the plugin on the Exchange: https://exchange.checkmk.com/p/network-visualization .\n'
'\n'
'The required inventory data can be created with my inventory plugins:\n'
'CDP: https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_cdp_cache\n'
'LLDP: https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_lldp_cache\n'
'\n'
f'\nVersion: {create_topology_version} | Written by: thl-cmk\n'
f'for more information see: https://thl-cmk.hopto.org',
formatter_class=RawTextHelpFormatter,
epilog='Usage:\n'
'for CDP (the default):\n'
f'{script} -s {sample_seeds} -d\n'
'for LLDP:\n'
f'{script} -s {sample_seeds} -d --lldp\n',
)
command_group = parser.add_mutually_exclusive_group()
def get_inventory_data(host: str, debug: bool = False, ) -> Dict[str, str] | None:
query = f'GET hosts\nColumns: mk_inventory\nOutputFormat: python3\nFilter: host_name = {host}\n'
data = get_data_form_live_status(query=query)
parser.add_argument(
'-d', '--default', default=False, action='store_const', const=True,
help='Set the created topology data as default',
)
parser.add_argument(
'-m', '--merge',
nargs=2,
choices=['CDP', 'LLDP'],
help=f'Merge topologies. This runs the topology creation for CDP and LLDP.\n'
f'The topologies are then merged in the specified order.\n'
f'I.e. -m CDP LLDP merges the CDP topology into the LLDP topology, overwriting\n'
f'the LLDP data in case there are conflicts.\n'
f'NOTE: static connection data from the user file will always merged with the\n'
f' highest priority',
)
parser.add_argument(
'-o', '--output-directory', type=str,
help='Directory name where to save the topology data.\n'
'I.e.: my_topology. Default is the actual date/time\n'
'in "--time-format" format.\n'
'NOTE: the directory is a sub directory under "~/var/topology_data/"',
)
parser.add_argument(
'-s', '--seed-devices', type=str, nargs='+',
help=f'List of devices to start the topology discovery from.\n'
f'I.e. {sample_seeds}',
)
parser.add_argument(
'-u', '--user-data-file', type=str,
help='Set the name uf the user provided data file\n'
'Default is ~local/bin/topology_data/create_topology_data.toml\n',
)
parser.add_argument(
'-v', '--version', default=False, action='store_const', const=True,
help='Print version of this script and exit',
)
parser.add_argument(
'--check-user-data-only', default=False, action='store_const', const=True,
help=f'Only tries to read/parse the user data from {user_data_file} and exits.',
)
parser.add_argument(
'--data-source', type=str,
help=f'The source from which the topology data originates.\n'
f'I.e. {cdp_label} for CDP data from the inventory.\n'
'NOTE: right now this only an unused label.',
)
parser.add_argument(
'--debug', default=False, action='store_const', const=True,
help='Print debug information',
)
parser.add_argument(
'--dont-compare', default=False, action='store_const', const=True,
help='Do not compare the actual topology data with the default topology\n'
'data. By default, the actual topology is compared with the default\n'
'topology. If the data matches, the actual topology is not saved.\n'
'So, if you run this tool in a cron job, a new topology will be\n'
'created only if there was a change, unless you use "--dont-compare".'
)
parser.add_argument(
'--inventory-columns', type=str,
help=f'Columns used from the inventory data.\n'
f'I.e. "{cdp_columns}"\n'
'NOTE: the columns must be in the order: neighbour, local_port,\n'
'neighbour_port',
)
parser.add_argument(
'--keep-domain', default=False, action='store_const', const=True,
help='Do not remove the domain name from the neighbor name',
)
parser.add_argument(
'--keep', type=int,
help=f'Number of topologies to keep. The oldest topologies above keep\n'
f'max will be deleted. The minimum value for --keep is 1.\n'
f'NOTE: The default topologies will be always kept.\n'
)
parser.add_argument(
'--lldp', default=False, action='store_const', const=True,
help=f'Sets data source to {lldp_label}, inventory path \n'
f'to "{lldp_path}" and columns\n'
f'to "{lldp_columns}"',
)
command_group.add_argument(
'--lowercase', default=False, action='store_const', const=True,
help='Change neighbour names to all lower case',
)
parser.add_argument(
'--min-age', type=int,
help=f'The minimum number of days before a topology is deleted\n'
f'by "--keep".\n'
f'NOTE: Topologies that are not older than 1 days are always kept.',
)
parser.add_argument(
'--path-in-inventory', type=str,
help=f'Checkmk inventory path to the topology data.\n'
f'I.e. "{cdp_path}"',
)
parser.add_argument(
'--time-format', type=str,
help='Format string to render the time. (default: %%Y-%%m-%%dT%%H:%%M:%%S.%%m)',
)
command_group.add_argument(
'--uppercase', default=False, action='store_const', const=True,
help='Change neighbour names to all upper case',
if data:
try:
data = literal_eval(data[0][0].decode('utf-8'))
except SyntaxError as e:
if debug:
print(f'data: |{data}|')
print(f'type: {type(data)}')
print(f'exception: {e}')
return
return data
def get_table_from_inventory(inventory: Dict[str, Any], path: List[str]) -> List | None:
path = ('Nodes,' + ',Nodes,'.join(path.split(',')) + ',Table,Rows').split(',')
table = inventory.copy()
for m in path:
try:
table = table[m]
except KeyError:
return
return table
def get_interface_items_from_lq(host: str, debug: bool = False) -> List:
query = (
'GET services\n'
'Columns: host_name description check_command\n'
'Filter: description ~ Interface\n'
f'Filter: host_name = {host}\n'
'OutputFormat: python3\n'
)
return parser.parse_args()
data = get_data_form_live_status(query=query)
items = []
for host, description, check_command in data:
items.append(description[10:]) # remove 'Interface ' from description
if debug:
print(f'Interfaces items found: {len(items)} an host {host}')
return items
mkp/create_topology_data-0.0.9-20231027.mkp 0 → 100644
+ 0
− 0
View file @ 9976a268
File added
packages/create_topology_data
+ 3
− 2
View file @ 9976a268
......@@ -53,10 +53,11 @@
'files': {'bin': ['topology_data/create_topology_classes.py',
'topology_data/create_topology_data.py',
'topology_data/create_topology_utils.py',
'topology_data/create_topology_data.toml']},
'topology_data/create_topology_data.toml',
'topology_data/create_topology_args.py']},
'name': 'create_topology_data',
'title': 'Network Visualization data creation',
'version': '0.0.8-20231020',
'version': '0.0.9-20231027',
'version.min_required': '2.2.0p1',
'version.packaged': '2.2.0p11',
'version.usable_until': '2.3.0p1'}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Collection of CheckMK checks (see https://checkmk.com/). All checks and plugins are provided as is. Absolutely no warranty. Send any comments to thl-cmk[at]outlook[dot]com