diff --git a/README.md b/README.md
index de840b40a508b0e6af58eef8e7f708cfca4f3d51..fbb988490a02d0ae44897765877e68996b9e03af 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.0.8-20231020.mkp "create_topology_data-0.0.8-20231020.mkp"
+[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.0.9-20231027.mkp "create_topology_data-0.0.9-20231027.mkp"
# PoC for Network Visualization data creation from inventory data
This script creates the topology data file needed for the [Checkmk Exchange Network visualization](https://exchange.checkmk.com/p/network-visualization) plugin by Andreas Boesl and [schnetz](https://exchange.checkmk.com/u/schnetz).\
diff --git a/bin/topology_data/create_topology_args.py b/bin/topology_data/create_topology_args.py
new file mode 100755
index 0000000000000000000000000000000000000000..0460d885298004eaa7337523a150553434330345
--- /dev/null
+++ b/bin/topology_data/create_topology_args.py
@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# License: GNU General Public License v2
+# Author: thl-cmk[at]outlook[dot]com
+# URL : https://thl-cmk.hopto.org
+# Date : 2023-10-12
+# File : create_topology_utils.py
+
+#
+# options used
+# -d --default
+# -o --output-directory
+# -s --seed-devices
+# -u --user-data-file
+# -v --version
+# --check-user-data-only
+# --data-source
+# --debug
+# --dont-compare
+# --inventory-columns
+# --keep-domain
+# --keep
+# --lldp
+# --lowercase
+# --min-age
+# --path-in-inventory
+# --time-format
+# --uppercase
+
+from argparse import (
+ Namespace as arg_Namespace,
+ ArgumentParser,
+ RawTextHelpFormatter,
+)
+from create_topology_utils import(
+ CREATE_TOPOLOGY_VERSION,
+ SCRIPT,
+ SAMPLE_SEEDS,
+ USER_DATA_FILE,
+ LABEL_CDP,
+ COLUMNS_CDP,
+ LABEL_LLDP,
+ PATH_LLDP,
+ COLUMNS_LLDP,
+ PATH_CDP
+)
+
+
+def parse_arguments() -> arg_Namespace:
+ parser = ArgumentParser(
+ prog='create_topology_data.py',
+ description='This script creates the topology data file needed for the Checkmk "network_visualization"\n'
+ 'plugin by Andreas Boesl and schnetz. For more information see\n'
+ 'the announcement from schnetz: https://forum.checkmk.com/t/network-visualization/41680\n'
+ 'and the plugin on the Exchange: https://exchange.checkmk.com/p/network-visualization .\n'
+ '\n'
+ 'The required inventory data can be created with my inventory plugins:\n'
+ 'CDP: https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_cdp_cache\n'
+ 'LLDP: https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_lldp_cache\n'
+ '\n'
+ f'\nVersion: {CREATE_TOPOLOGY_VERSION} | Written by: thl-cmk\n'
+ f'for more information see: https://thl-cmk.hopto.org',
+ formatter_class=RawTextHelpFormatter,
+ epilog='Usage:\n'
+ 'for CDP (the default):\n'
+ f'{SCRIPT} -s {SAMPLE_SEEDS} -d\n'
+ 'for LLDP:\n'
+ f'{SCRIPT} -s {SAMPLE_SEEDS} -d --lldp\n',
+ )
+ command_group = parser.add_mutually_exclusive_group()
+
+ parser.add_argument(
+ '-d', '--default', default=False, action='store_const', const=True,
+ help='Set the created topology data as default',
+ )
+ parser.add_argument(
+ '-m', '--merge',
+ nargs=2,
+ choices=['CDP', 'LLDP'],
+ help=f'Merge topologies. This runs the topology creation for CDP and LLDP.\n'
+ f'The topologies are then merged in the specified order.\n'
+ f'I.e. -m CDP LLDP merges the CDP topology into the LLDP topology, overwriting\n'
+ f'the LLDP data in case there are conflicts.\n'
+ f'NOTE: static connection data from the user file will always merged with the\n'
+ f' highest priority',
+ )
+ parser.add_argument(
+ '-o', '--output-directory', type=str,
+ help='Directory name where to save the topology data.\n'
+ 'I.e.: my_topology. Default is the actual date/time\n'
+ 'in "--time-format" format.\n'
+ 'NOTE: the directory is a sub directory under "~/var/topology_data/"',
+ )
+ parser.add_argument(
+ '-s', '--seed-devices', type=str, nargs='+',
+ help=f'List of devices to start the topology discovery from.\n'
+ f'I.e. {SAMPLE_SEEDS}',
+ )
+ parser.add_argument(
+ '-u', '--user-data-file', type=str,
+ help='Set the name uf the user provided data file\n'
+ 'Default is ~local/bin/topology_data/create_topology_data.toml\n',
+ )
+ parser.add_argument(
+ '-v', '--version', default=False, action='store_const', const=True,
+ help='Print version of this script and exit',
+ )
+ parser.add_argument(
+ '--check-user-data-only', default=False, action='store_const', const=True,
+ help=f'Only tries to read/parse the user data from {USER_DATA_FILE} and exits.',
+ )
+ parser.add_argument(
+ '--data-source', type=str,
+ help=f'The source from which the topology data originates.\n'
+ f'I.e. {LABEL_CDP} for CDP data from the inventory.\n'
+ 'NOTE: right now this only an unused label.',
+ )
+ parser.add_argument(
+ '--debug', default=False, action='store_const', const=True,
+ help='Print debug information',
+ )
+ parser.add_argument(
+ '--dont-compare', default=False, action='store_const', const=True,
+ help='Do not compare the actual topology data with the default topology\n'
+ 'data. By default, the actual topology is compared with the default\n'
+ 'topology. If the data matches, the actual topology is not saved.\n'
+ 'So, if you run this tool in a cron job, a new topology will be\n'
+ 'created only if there was a change, unless you use "--dont-compare".'
+ )
+ parser.add_argument(
+ '--inventory-columns', type=str,
+ help=f'Columns used from the inventory data.\n'
+ f'I.e. "{COLUMNS_CDP}"\n'
+ 'NOTE: the columns must be in the order: neighbour, local_port,\n'
+ 'neighbour_port',
+ )
+ parser.add_argument(
+ '--keep-domain', default=False, action='store_const', const=True,
+ help='Do not remove the domain name from the neighbor name',
+ )
+ parser.add_argument(
+ '--keep', type=int,
+ help=f'Number of topologies to keep. The oldest topologies above keep\n'
+ f'max will be deleted. The minimum value for --keep is 1.\n'
+ f'NOTE: The default topologies will be always kept.\n'
+ )
+ parser.add_argument(
+ '--lldp', default=False, action='store_const', const=True,
+ help=f'Sets data source to {LABEL_LLDP}, inventory path \n'
+ f'to "{PATH_LLDP}" and columns\n'
+ f'to "{COLUMNS_LLDP}"',
+ )
+ command_group.add_argument(
+ '--lowercase', default=False, action='store_const', const=True,
+ help='Change neighbour names to all lower case',
+ )
+ parser.add_argument(
+ '--min-age', type=int,
+ help=f'The minimum number of days before a topology is deleted\n'
+ f'by "--keep".\n'
+ f'NOTE: Topologies that are not older than 1 days are always kept.',
+ )
+ parser.add_argument(
+ '--path-in-inventory', type=str,
+ help=f'Checkmk inventory path to the topology data.\n'
+ f'I.e. "{PATH_CDP}"',
+ )
+ parser.add_argument(
+ '--time-format', type=str,
+ help='Format string to render the time. (default: %%Y-%%m-%%dT%%H:%%M:%%S.%%m)',
+ )
+ command_group.add_argument(
+ '--uppercase', default=False, action='store_const', const=True,
+ help='Change neighbour names to all upper case',
+ )
+ return parser.parse_args()
diff --git a/bin/topology_data/create_topology_classes.py b/bin/topology_data/create_topology_classes.py
index 145a84d72ddcd81092978bbba6319e81e80fdc29..8bf30df90c38504e43297dbdc998233fbb0aa084 100755
--- a/bin/topology_data/create_topology_classes.py
+++ b/bin/topology_data/create_topology_classes.py
@@ -10,17 +10,27 @@
from os import environ
+from pathlib import Path
from time import strftime
from typing import Dict, List, Any, NamedTuple
from enum import Enum, unique
from create_topology_utils import (
- cdp_path,
- cdp_columns,
- cdp_label,
- lldp_path,
- lldp_columns,
- lldp_label,
- user_data_file,
+ CREATE_TOPOLOGY_VERSION,
+ PATH_CDP,
+ COLUMNS_CDP,
+ LABEL_CDP,
+ PATH_LLDP,
+ COLUMNS_LLDP,
+ LABEL_LLDP,
+ PATH_INTERFACES,
+ USER_DATA_FILE,
+ LQ_INTERFACES,
+ SCRIPT,
+ get_inventory_data,
+ get_table_from_inventory,
+ get_interface_items_from_lq,
+ ExitCodes,
+ get_data_from_toml,
)
@@ -32,8 +42,14 @@ class TopologyParams(NamedTuple):
@unique
class Topologies(Enum):
- CDP = TopologyParams(path=cdp_path, columns=cdp_columns, label=cdp_label)
- LLDP = TopologyParams(path=lldp_path, columns=lldp_columns, label=lldp_label)
+ CDP = TopologyParams(path=PATH_CDP, columns=COLUMNS_CDP, label=LABEL_CDP)
+ LLDP = TopologyParams(path=PATH_LLDP, columns=COLUMNS_LLDP, label=LABEL_LLDP)
+
+
+@unique
+class CacheSources(Enum):
+ inventory = 'inventory'
+ lq = 'lq'
class InventoryColumns(NamedTuple):
@@ -72,11 +88,11 @@ class Settings:
self.__settings = {
'seed_devices': None,
- 'path_in_inventory': cdp_path,
- 'inventory_columns': cdp_columns,
- 'data_source': cdp_label,
+ 'path_in_inventory': PATH_CDP,
+ 'inventory_columns': COLUMNS_CDP,
+ 'data_source': LABEL_CDP,
'time_format': '%Y-%m-%dT%H:%M:%S.%m',
- 'user_data_file': f'{self.__omd_root}/local/bin/topology_data/{user_data_file}',
+ 'user_data_file': f'{self.__omd_root}/local/bin/topology_data/{USER_DATA_FILE}',
'merge': None,
'output_directory': None,
'default': False,
@@ -96,16 +112,31 @@ class Settings:
self.__args = ({k.split(',')[-1].strip(' ').strip('_'): v for k, v in cli_args.items() if v})
# first set values
if self.__args.get('lldp'):
- self.__settings['data_source'] = lldp_label
- self.__settings['path_in_inventory'] = lldp_path
- self.__settings['inventory_columns'] = lldp_columns
+ self.__settings['data_source'] = LABEL_LLDP
+ self.__settings['path_in_inventory'] = PATH_LLDP
+ self.__settings['inventory_columns'] = COLUMNS_LLDP
# Then update values with cli values
self.__settings.update(self.__args)
- if not self.merge:
- if not self.lldp:
- self.merge = ['CDP']
- else:
+
+ if self.version:
+ print(f'{Path(SCRIPT).name} version: {CREATE_TOPOLOGY_VERSION}')
+ exit(code=ExitCodes.OK.value)
+
+ if self.check_user_data_only:
+ user_data = get_data_from_toml(file=self.user_data_file)
+ print(f'Could read/parse the user data from {self.user_data_file}')
+ exit(code=ExitCodes.OK.value)
+
+ if self.merge:
+ _merge = list(set(self.merge.copy()))
+ if len(_merge) != len(self.merge):
+ print(f'-m/--merge options must be unique. Use "-m CDP LLDP" oder "-m LLDP CDP"')
+ exit(code=ExitCodes.BAD_OPTION_LIST.value)
+ else:
+ if self.lldp:
self.merge = ['LLDP']
+ else:
+ self.merge = ['CDP']
def set_topology_param(self, topology: TopologyParams):
self.__settings['path_in_inventory'] = topology.path
@@ -199,8 +230,9 @@ class Settings:
@property
def path_in_inventory(self) -> List[str]:
- path = ('Nodes,' + ',Nodes,'.join(self.__settings['path_in_inventory'].split(',')) + ',Table,Rows').split(',')
- return path
+ # path = ('Nodes,' + ',Nodes,'.join(self.__settings['path_in_inventory'].split(',')) + ',Table,Rows').split(',')
+ # return path
+ return self.__settings['path_in_inventory']
@property
def path_to_if_table(self) -> List[str]:
@@ -233,3 +265,42 @@ class Settings:
return f'{strftime(self.__settings["time_format"])}'
else:
return self.__settings['output_directory']
+
+
+class HostCache:
+ def __init__(self):
+ self.__cache = {}
+ self.__inventory_pre_fetch_list: List[str] = [
+ PATH_CDP,
+ PATH_LLDP,
+ PATH_INTERFACES,
+ ]
+
+ def __fill_cache(self, host: str):
+ # pre fill inventory data
+ inventory = get_inventory_data(host=host)
+ if inventory:
+ self.__cache[host][CacheSources.inventory.value] = {}
+ self.__cache[host][CacheSources.inventory.value].update({
+ entry: get_table_from_inventory(
+ inventory=inventory,
+ path=entry
+ ) for entry in self.__inventory_pre_fetch_list
+ })
+ else:
+ self.__cache[host][CacheSources.inventory.value] = None
+ # prefill live status data
+ self.__cache[host][CacheSources.lq.value] = {}
+ self.__cache[host][CacheSources.lq.value][LQ_INTERFACES] = get_interface_items_from_lq(host)
+
+ def get_data(self, host: str, source: CacheSources, path: str):
+ if host not in self.__cache.keys():
+ self.__cache[host]: Dict[str, Any] = {}
+ self.__fill_cache(host=host)
+ try:
+ return self.__cache[host][source.value][path]
+ except (KeyError, TypeError) as e:
+ return None
+
+ def add_inventory_prefetch_path(self, path: str):
+ self.__inventory_pre_fetch_list = list(set(self.__inventory_pre_fetch_list + [path]))
diff --git a/bin/topology_data/create_topology_data.py b/bin/topology_data/create_topology_data.py
index c47adbc4e7737b3f80eb8c6ef0ded4c34bbdea36..cded0835c6a54f4c7178529a4792660090a8400d 100755
--- a/bin/topology_data/create_topology_data.py
+++ b/bin/topology_data/create_topology_data.py
@@ -23,6 +23,8 @@
# added option -u, --user-data-file to provide a user specific data file
# 2023-10-20: changed option -m/--make-default to -d/--default (-m is needed for -m/--merge
# added -m/--merge option
+# 2023-10-27: improved matching of port name from CDP/LLDP data to interface (item) name of service
+# improved internal data handling (HostCache)
#
# PoC for creating topology_data.json from inventory data
@@ -37,78 +39,39 @@
# NOTE: the topology_data configuration (layout etc.) is saved under ~/var/check_mk/topology
#
# The inventory data could be created with my CDP/LLDP inventory plugins:
-#
# CDP: https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_cdp_cache\
-# CDP path-in-inventory: 'networking,cdp_cache'
-# CDP inventory-columns: 'device_id,local_port,device_port'
-#
# LLDP: https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_lldp_cache\
-# LLDP path-in-inventory: 'networking,lldp_cache'
-# LLDP inventory-columns: 'system_name,local_port_num,port_id'
#
-# USAGE CDP (is the default):
-# ~/local/lib/topology_data/create_topology_data.py -s CORE01 -m
-#
-# USAGE LLDP:
-# ~/local/lib/topology_data/create_topology_data.py -s CORE01 -m --lldp
+# USAGE:
+# ~/local/lib/topology_data/create_topology_data.py --help
#
-from ast import literal_eval
-from pathlib import Path
from typing import Dict, List, Any
from time import strftime, time_ns
from create_topology_utils import (
- parse_arguments,
remove_old_data,
get_data_from_toml,
merge_topologies,
save_topology,
- get_data_form_live_status,
- ExitCodes,
+ LQ_INTERFACES,
+ PATH_INTERFACES,
)
+from create_topology_args import parse_arguments
from create_topology_classes import (
InventoryColumns,
Settings,
StaticConnection,
Topologies,
- TopologyParams,
+ HostCache,
+ CacheSources,
)
-
-CREATE_TOPOLOGY_VERSION = '0.0.8-202310120'
-
-ITEMS = {}
HOST_MAP: Dict[str, str] = {}
DROP_HOSTS: List[str] = []
STATIC_CONNECTIONS: Dict = {}
-def get_interface_items_from_lq(host: str, debug: bool = False) -> Dict:
- query = (
- 'GET services\n'
- 'Columns: host_name description check_command\n'
- 'Filter: description ~ Interface\n'
- f'Filter: host_name = {host}\n'
- 'OutputFormat: python3\n'
- )
- data = get_data_form_live_status(query=query)
- items = {}
- for host, description, check_command in data:
- if host:
- if host not in items.keys():
- items[host] = []
- items[host].append(description[10:]) # remove 'Interface ' from description
-
- interfaces = 0
- for host in items.keys():
- items[host] = list(set(items[host]))
- items[host].sort()
- interfaces += len(items[host])
- # print(f'Interfaces found: {interfaces}')
- return items
-
-
def get_list_of_devices(data) -> List[str]:
devices = []
for connection in data.values():
@@ -116,66 +79,64 @@ def get_list_of_devices(data) -> List[str]:
return list(set(devices))
-def get_service_by_interface(host: str, interface: str, debug: bool = False) -> str | None:
+def get_service_by_interface(host: str, interface: str, debug: bool = False) -> str:
+ # try to find the item for an interface
+ def _match_entry_with_item(_entry: Dict[str, str]):
+ values = [_entry.get('name'), _entry.get('description'), entry.get('alias')]
+ for value in values:
+ if value in items:
+ return value
+
+ index = str(_entry.get('index'))
+ # for index try with padding
+ for i in range(1, 6):
+ index_padded = f'{index:0>{i}}'
+ if index_padded in items:
+ return index_padded
+ # still not found try values + index
+ for value in values:
+ if f'{value} {index_padded}' in items:
+ return f'{value} {index_padded}'
+
+ return interface
+
# empty host/neighbour should never happen here
if not host:
return interface
- items = ITEMS.get(host, [])
+ # get list of interface items
+ items = HOST_CACHE.get_data(host=host, source=CacheSources.lq, path=LQ_INTERFACES)
+
+ # the easy case
if interface in items:
return interface
else:
- if_padding = interface
- if '/' in interface:
- if_padding = '/'.join(interface.split('/')[:-1]) + f'/{interface.split("/")[-1]:0>2}'
-
- for item in items:
- # interface = Gi0/1
- # item = Gi0/1 - Access port
- # this might not catch the correct interface (fa0 and fa0/0)
- item = item.split(' ')[0]
- if item == interface:
- return item
- elif item == if_padding:
- return item
-
+ # try to find the interface in the host interface inventory list
+ inventory = HOST_CACHE.get_data(host=host, source=CacheSources.inventory, path=PATH_INTERFACES)
+ if inventory:
+ for entry in inventory:
+ if interface in [
+ entry.get('name'),
+ entry.get('description'),
+ entry.get('alias'),
+ str(entry.get('index')),
+ entry.get('phys_address'),
+ ]:
+ return _match_entry_with_item(entry)
if debug:
- print(f'Device: {host}: interface ({interface}) not found in services')
+ print(f'Device: {host}: service for interface {interface} not found')
return interface
-def get_inventory_data_lq(host: str, path: List[str], debug: bool = False, ) -> List[Dict[str, str]]:
- query = f'GET hosts\nColumns: mk_inventory\nOutputFormat: python3\nFilter: host_name = {host}\n'
- data = get_data_form_live_status(query=query)
-
- if data:
- try:
- data = literal_eval(data[0][0].decode('utf-8'))
- except SyntaxError as e:
- if debug:
- print(f'data: |{data}|')
- print(f'type: {type(data)}')
- print(f'exception: {e}')
- return []
- for m in path:
- try:
- data = data[m]
- except KeyError:
- return []
- return data
- return []
-
-
def create_device_from_inv(
host: str,
inv_data: List[Dict[str, str]],
inv_columns: InventoryColumns,
data_source: str,
+ debug: bool = False,
) -> Dict[str, Any] | None:
data = {'connections': {}, "interfaces": []}
- if not host in ITEMS.keys():
- ITEMS.update(get_interface_items_from_lq(host=host))
for topo_neighbour in inv_data:
neighbour = topo_neighbour.get(inv_columns.neighbour)
if not neighbour:
@@ -194,18 +155,18 @@ def create_device_from_inv(
if neighbour in HOST_MAP.keys():
neighbour = HOST_MAP[neighbour]
- # has to be done before checking interfaces
- if not neighbour in ITEMS.keys():
- ITEMS.update(get_interface_items_from_lq(host=neighbour))
-
# getting/checking interfaces
local_port = topo_neighbour.get(inv_columns.local_port)
-
- local_port = get_service_by_interface(host, local_port)
+ _local_port = local_port
+ local_port = get_service_by_interface(host, local_port, debug=debug)
+ if debug and local_port != _local_port:
+ print(f'host: {host}, local_port: {local_port}, _local_port: {_local_port}')
neighbour_port = topo_neighbour.get(inv_columns.neighbour_port)
-
- neighbour_port = get_service_by_interface(neighbour, neighbour_port)
+ _neighbour_port = neighbour_port
+ neighbour_port = get_service_by_interface(neighbour, neighbour_port, debug=debug)
+ if debug and neighbour_port != _neighbour_port:
+ print(f'neighbour: {neighbour}, neighbour_port {neighbour_port}, _neighbour_port {_neighbour_port}')
if neighbour and local_port and neighbour_port:
data['connections'].update({local_port: [neighbour, neighbour_port, data_source]})
@@ -272,18 +233,20 @@ def create_topology(
if device in devices_done:
continue
- topo_data = get_inventory_data_lq(host=device, path=path_in_inventory, debug=debug)
- topology_data.update(
- create_device_from_inv(
- host=device,
- inv_data=topo_data,
- inv_columns=inv_columns,
- data_source=data_source,
- ))
- devices_list = get_list_of_devices(topology_data[device]['connections'])
- for entry in devices_list:
- if entry not in devices_done:
- devices_to_go.append(entry)
+ # topo_data = get_inventory_data_lq(host=device, path=path_in_inventory, debug=debug)
+ topo_data = HOST_CACHE.get_data(host=device, source=CacheSources.inventory, path=path_in_inventory)
+ if topo_data:
+ topology_data.update(
+ create_device_from_inv(
+ host=device,
+ inv_data=topo_data,
+ inv_columns=inv_columns,
+ data_source=data_source,
+ ))
+ devices_list = get_list_of_devices(topology_data[device]['connections'])
+ for entry in devices_list:
+ if entry not in devices_done:
+ devices_to_go.append(entry)
devices_to_go = list(set(devices_to_go))
devices_done.append(device)
@@ -297,26 +260,12 @@ def create_topology(
if __name__ == '__main__':
- SETTINGS = Settings(vars(parse_arguments(CREATE_TOPOLOGY_VERSION)))
-
- if SETTINGS.version:
- print(f'{Path(__file__).name} version: {CREATE_TOPOLOGY_VERSION}')
- exit(code=ExitCodes.ok)
-
- if SETTINGS.check_user_data_only:
- user_data = get_data_from_toml(file=SETTINGS.user_data_file)
- print(f'Could read/parse the user data from {SETTINGS.user_data_file}')
- exit(code=ExitCodes.OK)
-
- if SETTINGS.merge:
- _merge = list(set(SETTINGS.merge.copy()))
- if len(_merge) != len(SETTINGS.merge):
- print(f'-m/--merge options must be unique. Use "-m cdp lldp" oder "-m lldp cdp"')
- exit(code=ExitCodes.BAD_OPTION_LIST)
+ start_time = time_ns()
+ SETTINGS = Settings(vars(parse_arguments()))
+ HOST_CACHE = HostCache()
print()
print(f'Start time: {strftime(SETTINGS.time_format)}')
- start_time = time_ns()
user_data = get_data_from_toml(file=SETTINGS.user_data_file)
HOST_MAP = user_data.get('HOST_MAP', {})
@@ -353,7 +302,7 @@ if __name__ == '__main__':
remove_old_data(
keep=SETTINGS.keep,
min_age=SETTINGS.min_age,
- path=Path(f'{SETTINGS.omd_root}/{SETTINGS.topology_save_path}'),
+ path=f'{SETTINGS.omd_root}/{SETTINGS.topology_save_path}',
protected=user_data.get('PROTECTED_TOPOLOGIES', []),
)
diff --git a/bin/topology_data/create_topology_data.toml b/bin/topology_data/create_topology_data.toml
old mode 100755
new mode 100644
index 5e31d3db6e75ea61380aef1877a373424564fa5c..9e1c7c889b60b49a3aa610bde79ee3d2866a0405
--- a/bin/topology_data/create_topology_data.toml
+++ b/bin/topology_data/create_topology_data.toml
@@ -14,33 +14,33 @@
# list of (additional to -s/--seed-devices) seed devices
SEED_DEVICES = [
- "CORE01",
- "LOCATION01",
- "LOCATION02",
+ # "CORE01",
+ # "LOCATION01",
+ # "LOCATION02",
]
# drop neighbours with invalid names
DROP_HOSTS = [
- "not advertised",
- "a nother invalid name",
+ # "not advertised",
+ # "a nother invalid name",
]
# topologies will not be deleted by "--keep"
PROTECTED_TOPOLOGIES = [
- "2023-10-17T14:08:05.10",
- "your_important_topology"
+ # "2023-10-17T14:08:05.10",
+ # "your_important_topology"
]
# user defined static connections
# connections will be added from host to neighbour and in reverese
# hosts/neighbours in this section will be added to SEED_DEVICES
STATIC_CONNECTIONS = [
- ["cmk_host1", "local-port1", "neighbour-port1", "neighbour1", "label"],
- ["cmk_host1", "local-port2", "neighbour-port2", "neighbour2", "label"],
+ # ["cmk_host1", "local-port1", "neighbour-port1", "neighbour1", "label"],
+ # ["cmk_host1", "local-port2", "neighbour-port2", "neighbour2", "label"],
]
# map inventory neighbour name to Checkmk host name
[HOST_MAP]
-inventory_neighbour1 = "cmk_host1"
-inventory_neighbour2 = "cmk_host2"
-inventory_neighbour3 = "cmk_host3"
+# inventory_neighbour1 = "cmk_host1"
+# inventory_neighbour2 = "cmk_host2"
+# inventory_neighbour3 = "cmk_host3"
diff --git a/bin/topology_data/create_topology_utils.py b/bin/topology_data/create_topology_utils.py
index bf6776fea969bb46e7e688d8f998ccdd950b21d0..ab4bda85f339f9cde288e512731e72ede04e6444 100755
--- a/bin/topology_data/create_topology_utils.py
+++ b/bin/topology_data/create_topology_utils.py
@@ -37,12 +37,7 @@ from tomllib import loads as toml_loads
from tomllib import TOMLDecodeError
from re import match as re_match
from pathlib import Path
-from typing import List, Dict
-from argparse import (
- Namespace as arg_Namespace,
- ArgumentParser,
- RawTextHelpFormatter,
-)
+from typing import List, Dict, Any
from enum import Enum, unique
@@ -53,15 +48,19 @@ class ExitCodes(Enum):
BAD_TOML_FORMAT = 3
-script = '~/local/bin/network-topology/create_topology_data.py'
-sample_seeds = 'Core01 Core02'
-cdp_path = 'networking,cdp_cache'
-cdp_columns = 'device_id,local_port,device_port'
-cdp_label = 'inv_CDP'
-lldp_path = 'networking,lldp_cache'
-lldp_columns = 'system_name,local_port_num,port_id'
-lldp_label = 'inv_LLDP'
-user_data_file = 'create_topology_data.toml'
+# constants
+CREATE_TOPOLOGY_VERSION = '0.0.9-202310127'
+SCRIPT = '~/local/bin/network-topology/create_topology_data.py'
+SAMPLE_SEEDS = 'Core01 Core02'
+PATH_CDP = 'networking,cdp_cache'
+PATH_LLDP = 'networking,lldp_cache'
+PATH_INTERFACES = 'networking,interfaces'
+LABEL_CDP = 'inv_CDP'
+LABEL_LLDP = 'inv_LLDP'
+COLUMNS_LLDP = 'system_name,local_port_num,port_id'
+COLUMNS_CDP = 'device_id,local_port,device_port'
+USER_DATA_FILE = 'create_topology_data.toml'
+LQ_INTERFACES = 'interface_items'
def get_data_form_live_status(query: str):
@@ -91,7 +90,7 @@ def get_data_from_toml(file: str, debug: bool = False) -> Dict:
except TOMLDecodeError as e:
print(
f'ERROR: data file {toml_file} is not in valid TOML format! ({e}), (see https://toml.io/en/)')
- exit(code=ExitCodes.BAD_TOML_FORMAT)
+ exit(code=ExitCodes.BAD_TOML_FORMAT.value)
else:
print(f'WARNING: User data {file} not found.')
if debug:
@@ -113,7 +112,8 @@ def rm_tree(root: Path):
root.rmdir()
-def remove_old_data(keep: int, min_age: int, path: Path, protected: List[str], debug: bool = False):
+def remove_old_data(keep: int, min_age: int, path: str, protected: List[str], debug: bool = False):
+ path = Path(path)
default_topo = path.joinpath('default')
directories = [str(directory) for directory in list(path.iterdir())]
# keep default top
@@ -317,131 +317,46 @@ def is_equal_with_default(data: Dict, file: str) -> bool:
return compare_dicts(data, default_data)
-def parse_arguments(create_topology_version: str) -> arg_Namespace:
- parser = ArgumentParser(
- prog='create_topology_data.py',
- description='This script creates the topology data file needed for the Checkmk "network_visualization"\n'
- 'plugin by Andreas Boesl and schnetz. For more information see\n'
- 'the announcement from schnetz: https://forum.checkmk.com/t/network-visualization/41680\n'
- 'and the plugin on the Exchange: https://exchange.checkmk.com/p/network-visualization .\n'
- '\n'
- 'The required inventory data can be created with my inventory plugins:\n'
- 'CDP: https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_cdp_cache\n'
- 'LLDP: https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_lldp_cache\n'
- '\n'
- f'\nVersion: {create_topology_version} | Written by: thl-cmk\n'
- f'for more information see: https://thl-cmk.hopto.org',
- formatter_class=RawTextHelpFormatter,
- epilog='Usage:\n'
- 'for CDP (the default):\n'
- f'{script} -s {sample_seeds} -d\n'
- 'for LLDP:\n'
- f'{script} -s {sample_seeds} -d --lldp\n',
- )
- command_group = parser.add_mutually_exclusive_group()
+def get_inventory_data(host: str, debug: bool = False, ) -> Dict[str, str] | None:
+ query = f'GET hosts\nColumns: mk_inventory\nOutputFormat: python3\nFilter: host_name = {host}\n'
+ data = get_data_form_live_status(query=query)
- parser.add_argument(
- '-d', '--default', default=False, action='store_const', const=True,
- help='Set the created topology data as default',
- )
- parser.add_argument(
- '-m', '--merge',
- nargs=2,
- choices=['CDP', 'LLDP'],
- help=f'Merge topologies. This runs the topology creation for CDP and LLDP.\n'
- f'The topologies are then merged in the specified order.\n'
- f'I.e. -m CDP LLDP merges the CDP topology into the LLDP topology, overwriting\n'
- f'the LLDP data in case there are conflicts.\n'
- f'NOTE: static connection data from the user file will always merged with the\n'
- f' highest priority',
- )
- parser.add_argument(
- '-o', '--output-directory', type=str,
- help='Directory name where to save the topology data.\n'
- 'I.e.: my_topology. Default is the actual date/time\n'
- 'in "--time-format" format.\n'
- 'NOTE: the directory is a sub directory under "~/var/topology_data/"',
- )
- parser.add_argument(
- '-s', '--seed-devices', type=str, nargs='+',
- help=f'List of devices to start the topology discovery from.\n'
- f'I.e. {sample_seeds}',
- )
- parser.add_argument(
- '-u', '--user-data-file', type=str,
- help='Set the name uf the user provided data file\n'
- 'Default is ~local/bin/topology_data/create_topology_data.toml\n',
- )
- parser.add_argument(
- '-v', '--version', default=False, action='store_const', const=True,
- help='Print version of this script and exit',
- )
- parser.add_argument(
- '--check-user-data-only', default=False, action='store_const', const=True,
- help=f'Only tries to read/parse the user data from {user_data_file} and exits.',
- )
- parser.add_argument(
- '--data-source', type=str,
- help=f'The source from which the topology data originates.\n'
- f'I.e. {cdp_label} for CDP data from the inventory.\n'
- 'NOTE: right now this only an unused label.',
- )
- parser.add_argument(
- '--debug', default=False, action='store_const', const=True,
- help='Print debug information',
- )
- parser.add_argument(
- '--dont-compare', default=False, action='store_const', const=True,
- help='Do not compare the actual topology data with the default topology\n'
- 'data. By default, the actual topology is compared with the default\n'
- 'topology. If the data matches, the actual topology is not saved.\n'
- 'So, if you run this tool in a cron job, a new topology will be\n'
- 'created only if there was a change, unless you use "--dont-compare".'
- )
- parser.add_argument(
- '--inventory-columns', type=str,
- help=f'Columns used from the inventory data.\n'
- f'I.e. "{cdp_columns}"\n'
- 'NOTE: the columns must be in the order: neighbour, local_port,\n'
- 'neighbour_port',
- )
- parser.add_argument(
- '--keep-domain', default=False, action='store_const', const=True,
- help='Do not remove the domain name from the neighbor name',
- )
- parser.add_argument(
- '--keep', type=int,
- help=f'Number of topologies to keep. The oldest topologies above keep\n'
- f'max will be deleted. The minimum value for --keep is 1.\n'
- f'NOTE: The default topologies will be always kept.\n'
- )
- parser.add_argument(
- '--lldp', default=False, action='store_const', const=True,
- help=f'Sets data source to {lldp_label}, inventory path \n'
- f'to "{lldp_path}" and columns\n'
- f'to "{lldp_columns}"',
- )
- command_group.add_argument(
- '--lowercase', default=False, action='store_const', const=True,
- help='Change neighbour names to all lower case',
- )
- parser.add_argument(
- '--min-age', type=int,
- help=f'The minimum number of days before a topology is deleted\n'
- f'by "--keep".\n'
- f'NOTE: Topologies that are not older than 1 days are always kept.',
- )
- parser.add_argument(
- '--path-in-inventory', type=str,
- help=f'Checkmk inventory path to the topology data.\n'
- f'I.e. "{cdp_path}"',
- )
- parser.add_argument(
- '--time-format', type=str,
- help='Format string to render the time. (default: %%Y-%%m-%%dT%%H:%%M:%%S.%%m)',
- )
- command_group.add_argument(
- '--uppercase', default=False, action='store_const', const=True,
- help='Change neighbour names to all upper case',
+ if data:
+ try:
+ data = literal_eval(data[0][0].decode('utf-8'))
+ except SyntaxError as e:
+ if debug:
+ print(f'data: |{data}|')
+ print(f'type: {type(data)}')
+ print(f'exception: {e}')
+ return
+ return data
+
+
+def get_table_from_inventory(inventory: Dict[str, Any], path: List[str]) -> List | None:
+ path = ('Nodes,' + ',Nodes,'.join(path.split(',')) + ',Table,Rows').split(',')
+ table = inventory.copy()
+ for m in path:
+ try:
+ table = table[m]
+ except KeyError:
+ return
+ return table
+
+
+def get_interface_items_from_lq(host: str, debug: bool = False) -> List:
+ query = (
+ 'GET services\n'
+ 'Columns: host_name description check_command\n'
+ 'Filter: description ~ Interface\n'
+ f'Filter: host_name = {host}\n'
+ 'OutputFormat: python3\n'
)
- return parser.parse_args()
+ data = get_data_form_live_status(query=query)
+ items = []
+ for host, description, check_command in data:
+ items.append(description[10:]) # remove 'Interface ' from description
+
+ if debug:
+ print(f'Interfaces items found: {len(items)} an host {host}')
+ return items
diff --git a/mkp/create_topology_data-0.0.9-20231027.mkp b/mkp/create_topology_data-0.0.9-20231027.mkp
new file mode 100644
index 0000000000000000000000000000000000000000..f47d7e12ba7db7499e18557f65dab4193738595a
Binary files /dev/null and b/mkp/create_topology_data-0.0.9-20231027.mkp differ
diff --git a/packages/create_topology_data b/packages/create_topology_data
index 85ce824cda9156d9599bf186b03c70a55e9aab33..e181976999ddb68f7e71290f19e4b3984d665410 100644
--- a/packages/create_topology_data
+++ b/packages/create_topology_data
@@ -53,10 +53,11 @@
'files': {'bin': ['topology_data/create_topology_classes.py',
'topology_data/create_topology_data.py',
'topology_data/create_topology_utils.py',
- 'topology_data/create_topology_data.toml']},
+ 'topology_data/create_topology_data.toml',
+ 'topology_data/create_topology_args.py']},
'name': 'create_topology_data',
'title': 'Network Visualization data creation',
- 'version': '0.0.8-20231020',
+ 'version': '0.0.9-20231027',
'version.min_required': '2.2.0p1',
'version.packaged': '2.2.0p11',
'version.usable_until': '2.3.0p1'}