diff --git a/README.md b/README.md
index 1730be2ba59876a208274f70acd30a086e42e63b..1a04927de215d18e5aef2fbfe87f41b0c383c372 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.0.9-20231027.mkp "create_topology_data-0.0.9-20231027.mkp"
+[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.1.0-20231028.mkp "create_topology_data-0.1.0-20231028.mkp"
# PoC for Network Visualization data creation from inventory data
This script creates the topology data file needed for the [Checkmk Exchange Network visualization](https://exchange.checkmk.com/p/network-visualization) plugin by Andreas Boesl and [schnetz](https://exchange.checkmk.com/u/schnetz).\
diff --git a/bin/topology_data/create_topology_args.py b/bin/topology_data/create_topology_args.py
index 0460d885298004eaa7337523a150553434330345..8ffb32839783a6244b755c813ce8fde8d98dd41a 100755
--- a/bin/topology_data/create_topology_args.py
+++ b/bin/topology_data/create_topology_args.py
@@ -10,21 +10,18 @@
#
# options used
# -d --default
+# -l --layer
# -o --output-directory
# -s --seed-devices
# -u --user-data-file
# -v --version
# --check-user-data-only
-# --data-source
# --debug
# --dont-compare
-# --inventory-columns
# --keep-domain
# --keep
-# --lldp
# --lowercase
# --min-age
-# --path-in-inventory
# --time-format
# --uppercase
@@ -33,17 +30,11 @@ from argparse import (
ArgumentParser,
RawTextHelpFormatter,
)
-from create_topology_utils import(
+from create_topology_utils import (
CREATE_TOPOLOGY_VERSION,
SCRIPT,
SAMPLE_SEEDS,
USER_DATA_FILE,
- LABEL_CDP,
- COLUMNS_CDP,
- LABEL_LLDP,
- PATH_LLDP,
- COLUMNS_LLDP,
- PATH_CDP
)
@@ -65,8 +56,6 @@ def parse_arguments() -> arg_Namespace:
epilog='Usage:\n'
'for CDP (the default):\n'
f'{SCRIPT} -s {SAMPLE_SEEDS} -d\n'
- 'for LLDP:\n'
- f'{SCRIPT} -s {SAMPLE_SEEDS} -d --lldp\n',
)
command_group = parser.add_mutually_exclusive_group()
@@ -74,17 +63,7 @@ def parse_arguments() -> arg_Namespace:
'-d', '--default', default=False, action='store_const', const=True,
help='Set the created topology data as default',
)
- parser.add_argument(
- '-m', '--merge',
- nargs=2,
- choices=['CDP', 'LLDP'],
- help=f'Merge topologies. This runs the topology creation for CDP and LLDP.\n'
- f'The topologies are then merged in the specified order.\n'
- f'I.e. -m CDP LLDP merges the CDP topology into the LLDP topology, overwriting\n'
- f'the LLDP data in case there are conflicts.\n'
- f'NOTE: static connection data from the user file will always merged with the\n'
- f' highest priority',
- )
+
parser.add_argument(
'-o', '--output-directory', type=str,
help='Directory name where to save the topology data.\n'
@@ -97,6 +76,14 @@ def parse_arguments() -> arg_Namespace:
help=f'List of devices to start the topology discovery from.\n'
f'I.e. {SAMPLE_SEEDS}',
)
+ parser.add_argument(
+ '-l', '--layers',
+ nargs='+',
+ choices=['CDP', 'CUSTOM', 'LLDP', 'STATIC'],
+ default=['CDP', 'STATIC'],
+ help=f'Layers with least significant layer first. Listed layers\n'
+ f'will be merged automatically. Default is "-l CDP STATIC"',
+ )
parser.add_argument(
'-u', '--user-data-file', type=str,
help='Set the name uf the user provided data file\n'
@@ -110,12 +97,6 @@ def parse_arguments() -> arg_Namespace:
'--check-user-data-only', default=False, action='store_const', const=True,
help=f'Only tries to read/parse the user data from {USER_DATA_FILE} and exits.',
)
- parser.add_argument(
- '--data-source', type=str,
- help=f'The source from which the topology data originates.\n'
- f'I.e. {LABEL_CDP} for CDP data from the inventory.\n'
- 'NOTE: right now this only an unused label.',
- )
parser.add_argument(
'--debug', default=False, action='store_const', const=True,
help='Print debug information',
@@ -128,13 +109,6 @@ def parse_arguments() -> arg_Namespace:
'So, if you run this tool in a cron job, a new topology will be\n'
'created only if there was a change, unless you use "--dont-compare".'
)
- parser.add_argument(
- '--inventory-columns', type=str,
- help=f'Columns used from the inventory data.\n'
- f'I.e. "{COLUMNS_CDP}"\n'
- 'NOTE: the columns must be in the order: neighbour, local_port,\n'
- 'neighbour_port',
- )
parser.add_argument(
'--keep-domain', default=False, action='store_const', const=True,
help='Do not remove the domain name from the neighbor name',
@@ -145,12 +119,6 @@ def parse_arguments() -> arg_Namespace:
f'max will be deleted. The minimum value for --keep is 1.\n'
f'NOTE: The default topologies will be always kept.\n'
)
- parser.add_argument(
- '--lldp', default=False, action='store_const', const=True,
- help=f'Sets data source to {LABEL_LLDP}, inventory path \n'
- f'to "{PATH_LLDP}" and columns\n'
- f'to "{COLUMNS_LLDP}"',
- )
command_group.add_argument(
'--lowercase', default=False, action='store_const', const=True,
help='Change neighbour names to all lower case',
@@ -161,11 +129,6 @@ def parse_arguments() -> arg_Namespace:
f'by "--keep".\n'
f'NOTE: Topologies that are not older than 1 days are always kept.',
)
- parser.add_argument(
- '--path-in-inventory', type=str,
- help=f'Checkmk inventory path to the topology data.\n'
- f'I.e. "{PATH_CDP}"',
- )
parser.add_argument(
'--time-format', type=str,
help='Format string to render the time. (default: %%Y-%%m-%%dT%%H:%%M:%%S.%%m)',
diff --git a/bin/topology_data/create_topology_classes.py b/bin/topology_data/create_topology_classes.py
index 8bf30df90c38504e43297dbdc998233fbb0aa084..f3d1794ffc4079f443ecdc08f985792ab4bc86d8 100755
--- a/bin/topology_data/create_topology_classes.py
+++ b/bin/topology_data/create_topology_classes.py
@@ -20,8 +20,6 @@ from create_topology_utils import (
COLUMNS_CDP,
LABEL_CDP,
PATH_LLDP,
- COLUMNS_LLDP,
- LABEL_LLDP,
PATH_INTERFACES,
USER_DATA_FILE,
LQ_INTERFACES,
@@ -40,12 +38,6 @@ class TopologyParams(NamedTuple):
label: str
-@unique
-class Topologies(Enum):
- CDP = TopologyParams(path=PATH_CDP, columns=COLUMNS_CDP, label=LABEL_CDP)
- LLDP = TopologyParams(path=PATH_LLDP, columns=COLUMNS_LLDP, label=LABEL_LLDP)
-
-
@unique
class CacheSources(Enum):
inventory = 'inventory'
@@ -87,20 +79,19 @@ class Settings:
# self.__autochecks_path = 'var/check_mk/autochecks'
self.__settings = {
+ 'layers': [],
'seed_devices': None,
'path_in_inventory': PATH_CDP,
'inventory_columns': COLUMNS_CDP,
'data_source': LABEL_CDP,
'time_format': '%Y-%m-%dT%H:%M:%S.%m',
'user_data_file': f'{self.__omd_root}/local/bin/topology_data/{USER_DATA_FILE}',
- 'merge': None,
'output_directory': None,
'default': False,
'keep_domain': False,
'lowercase': False,
'uppercase': False,
'debug': False,
- 'lldp': False,
'version': False,
'dont_compare': False,
'check_user_data_only': False,
@@ -110,12 +101,6 @@ class Settings:
# args in the form {'s, __seed_devices': 'CORE01', 'p, __path_in_inventory': None, ... }}
# we will remove 's, __'
self.__args = ({k.split(',')[-1].strip(' ').strip('_'): v for k, v in cli_args.items() if v})
- # first set values
- if self.__args.get('lldp'):
- self.__settings['data_source'] = LABEL_LLDP
- self.__settings['path_in_inventory'] = PATH_LLDP
- self.__settings['inventory_columns'] = COLUMNS_LLDP
- # Then update values with cli values
self.__settings.update(self.__args)
if self.version:
@@ -123,37 +108,28 @@ class Settings:
exit(code=ExitCodes.OK.value)
if self.check_user_data_only:
- user_data = get_data_from_toml(file=self.user_data_file)
+ _user_data = get_data_from_toml(file=self.user_data_file)
print(f'Could read/parse the user data from {self.user_data_file}')
exit(code=ExitCodes.OK.value)
- if self.merge:
- _merge = list(set(self.merge.copy()))
- if len(_merge) != len(self.merge):
- print(f'-m/--merge options must be unique. Use "-m CDP LLDP" oder "-m LLDP CDP"')
+ if self.layers:
+ layers = list(set(self.layers))
+ if len(layers) != len(self.layers):
+ print(f'-l/--layers options must be unique. Don~\'t use any layer more than once.')
exit(code=ExitCodes.BAD_OPTION_LIST.value)
- else:
- if self.lldp:
- self.merge = ['LLDP']
- else:
- self.merge = ['CDP']
- def set_topology_param(self, topology: TopologyParams):
- self.__settings['path_in_inventory'] = topology.path
- self.__settings['inventory_columns'] = topology.columns
- self.__settings['data_source'] = topology.label
+ def set_topology_param(self, topology: Dict[str, str]):
+ self.__settings['path_in_inventory'] = topology['path']
+ self.__settings['inventory_columns'] = topology['columns']
+ self.__settings['data_source'] = topology['label']
@property
def version(self) -> bool:
return self.__settings['version']
@property
- def merge(self) -> List[str] | None:
- return self.__settings['merge']
-
- @merge.setter
- def merge(self, topologies: List[str]):
- self.__settings['merge'] = topologies
+ def layers(self) -> List[str]:
+ return self.__settings['layers']
@property
def keep(self) -> int | None:
@@ -171,10 +147,6 @@ class Settings:
def check_user_data_only(self) -> bool:
return self.__settings['check_user_data_only']
- @property
- def lldp(self) -> bool:
- return self.__settings['lldp']
-
@property
def debug(self) -> bool:
return self.__settings['debug']
@@ -207,15 +179,6 @@ class Settings:
def omd_root(self) -> str:
return self.__omd_root
- # replaced by live status query
- # @property
- # def inventory_path(self) -> str:
- # return self.__inventory_path
-
- # @property
- # def autochecks_path(self) -> str:
- # return self.__autochecks_path
-
@property
def user_data_file(self) -> str:
return self.__settings['user_data_file']
@@ -230,14 +193,12 @@ class Settings:
@property
def path_in_inventory(self) -> List[str]:
- # path = ('Nodes,' + ',Nodes,'.join(self.__settings['path_in_inventory'].split(',')) + ',Table,Rows').split(',')
- # return path
return self.__settings['path_in_inventory']
- @property
- def path_to_if_table(self) -> List[str]:
- path = ('Nodes,' + ',Nodes,'.join(self.__path_to_if_table.split(',')) + ',Table,Rows').split(',')
- return path
+ # @property
+ # def path_to_if_table(self) -> List[str]:
+ # path = ('Nodes,' + ',Nodes,'.join(self.__path_to_if_table.split(',')) + ',Table,Rows').split(',')
+ # return path
@property
def inventory_columns(self) -> InventoryColumns:
@@ -299,7 +260,7 @@ class HostCache:
self.__fill_cache(host=host)
try:
return self.__cache[host][source.value][path]
- except (KeyError, TypeError) as e:
+ except (KeyError, TypeError) as _e:
return None
def add_inventory_prefetch_path(self, path: str):
diff --git a/bin/topology_data/create_topology_data.py b/bin/topology_data/create_topology_data.py
index cded0835c6a54f4c7178529a4792660090a8400d..70ce9bb1e1a9f23eb3ef580859df43b58f307618 100755
--- a/bin/topology_data/create_topology_data.py
+++ b/bin/topology_data/create_topology_data.py
@@ -25,16 +25,22 @@
# added -m/--merge option
# 2023-10-27: improved matching of port name from CDP/LLDP data to interface (item) name of service
# improved internal data handling (HostCache)
+# 2023-10-28: reworked handling of layers (see option -l/--layers)
+# added option -l/-layers {CDP,CUSTOM,LLDP,STATIC} [{CDP,CUSTOM,LLDP,STATIC} ...]
+# added section CUSTOM_LAYERS to user data file
+# removed option -m/--merge, included in option -l/--layers
+# removed option --lldp, included in option -l/--layers
+# removed option --data-source, now handled in CUSTOM_LAYERS
+# removed option --inventory-columns, now handled in CUSTOM_LAYERS
+# removed option --path-in-inventory, now handled in CUSTOM_LAYERS
#
# PoC for creating topology_data.json from inventory data
#
# This script creates the topology data file needed for the Checkmk "network_visualization" plugin by
-# Andreas Boesl and schnetz. See
+# Andreas Boesl and schnetz. For more information see
# https://forum.checkmk.com/t/network-visualization/41680
-# and
# https://exchange.checkmk.com/p/network-visualization
-# for more information.
#
# NOTE: the topology_data configuration (layout etc.) is saved under ~/var/check_mk/topology
#
@@ -56,13 +62,13 @@ from create_topology_utils import (
save_topology,
LQ_INTERFACES,
PATH_INTERFACES,
+ LAYERS,
)
from create_topology_args import parse_arguments
from create_topology_classes import (
InventoryColumns,
Settings,
StaticConnection,
- Topologies,
HostCache,
CacheSources,
)
@@ -270,24 +276,34 @@ if __name__ == '__main__':
user_data = get_data_from_toml(file=SETTINGS.user_data_file)
HOST_MAP = user_data.get('HOST_MAP', {})
DROP_HOSTS = user_data.get('DROP_HOSTS', [])
-
+ CUSTOM_LAYERS = user_data.get('CUSTOM_LAYERS', [])
+ jobs = []
final_topology = {}
- while SETTINGS.merge:
- SETTINGS.set_topology_param(Topologies[SETTINGS.merge[-1]].value)
- topology = create_topology(
- seed_devices=list(set(SETTINGS.seed_devices + user_data.get('SEED_DEVICES', []))),
- path_in_inventory=SETTINGS.path_in_inventory,
- inv_columns=SETTINGS.inventory_columns,
- data_source=SETTINGS.data_source,
- debug=SETTINGS.debug,
- )
+
+ for layer in SETTINGS.layers:
+ if layer == 'STATIC':
+ jobs.append('STATIC')
+ elif layer in LAYERS.keys():
+ jobs.append(LAYERS[layer])
+ elif layer == 'CUSTOM':
+ for entry in CUSTOM_LAYERS:
+ jobs.append(entry)
+
+ for job in jobs:
+ topology = {}
+ if job == 'STATIC':
+ topology = create_static_connections(connections=user_data.get('STATIC_CONNECTIONS', []))
+ else:
+ SETTINGS.set_topology_param(job)
+ topology = create_topology(
+ seed_devices=list(set(SETTINGS.seed_devices + user_data.get('SEED_DEVICES', []))),
+ path_in_inventory=SETTINGS.path_in_inventory,
+ inv_columns=SETTINGS.inventory_columns,
+ data_source=SETTINGS.data_source,
+ debug=SETTINGS.debug,
+ )
if topology:
final_topology = merge_topologies(topo_pri=topology, topo_sec=final_topology)
- SETTINGS.merge.remove(SETTINGS.merge[-1])
-
- STATIC_CONNECTIONS = create_static_connections(connections=user_data.get('STATIC_CONNECTIONS', []))
-
- final_topology = merge_topologies(topo_pri=STATIC_CONNECTIONS, topo_sec=final_topology)
save_topology(
data=final_topology,
diff --git a/bin/topology_data/create_topology_data.toml b/bin/topology_data/create_topology_data.toml
old mode 100644
new mode 100755
index 9e1c7c889b60b49a3aa610bde79ee3d2866a0405..bb5a2a07e09f2ce84423f3cf1d5acfec8e0dbf83
--- a/bin/topology_data/create_topology_data.toml
+++ b/bin/topology_data/create_topology_data.toml
@@ -31,6 +31,12 @@ PROTECTED_TOPOLOGIES = [
# "your_important_topology"
]
+# optional custom layers use option -l/--layers CUSTOM to include this layers
+CUSTOM_LAYERS = [
+ # { path = "networking,lldp_cache", columns = "system_name,local_port_num,port_id", label = "custom_LLDP" },
+ # { path = "networking,cdp_cache", columns = "device_id,local_port,device_port", label = "custom_CDP" },
+]
+
# user defined static connections
# connections will be added from host to neighbour and in reverese
# hosts/neighbours in this section will be added to SEED_DEVICES
diff --git a/bin/topology_data/create_topology_utils.py b/bin/topology_data/create_topology_utils.py
index ab4bda85f339f9cde288e512731e72ede04e6444..cd0d0237d926a0ec31a1ce0645ffea0edb58d2db 100755
--- a/bin/topology_data/create_topology_utils.py
+++ b/bin/topology_data/create_topology_utils.py
@@ -7,27 +7,6 @@
# Date : 2023-10-12
# File : create_topology_utils.py
-#
-# options used
-# -d --default
-# -o --output-directory
-# -s --seed-devices
-# -u --user-data-file
-# -v --version
-# --check-user-data-only
-# --data-source
-# --debug
-# --dont-compare
-# --inventory-columns
-# --keep-domain
-# --keep
-# --lldp
-# --lowercase
-# --min-age
-# --path-in-inventory
-# --time-format
-# --uppercase
-
from os import environ
from json import dumps
import socket
@@ -49,7 +28,7 @@ class ExitCodes(Enum):
# constants
-CREATE_TOPOLOGY_VERSION = '0.0.9-202310127'
+CREATE_TOPOLOGY_VERSION = '0.1.0-202310128'
SCRIPT = '~/local/bin/network-topology/create_topology_data.py'
SAMPLE_SEEDS = 'Core01 Core02'
PATH_CDP = 'networking,cdp_cache'
@@ -61,6 +40,10 @@ COLUMNS_LLDP = 'system_name,local_port_num,port_id'
COLUMNS_CDP = 'device_id,local_port,device_port'
USER_DATA_FILE = 'create_topology_data.toml'
LQ_INTERFACES = 'interface_items'
+LAYERS = {
+ 'CDP': {'path': PATH_CDP, 'columns': COLUMNS_CDP, 'label': LABEL_CDP},
+ 'LLDP': {'path': PATH_LLDP, 'columns': COLUMNS_LLDP, 'label': LABEL_LLDP},
+}
def get_data_form_live_status(query: str):
diff --git a/mkp/create_topology_data-0.1.0-20231028.mkp b/mkp/create_topology_data-0.1.0-20231028.mkp
new file mode 100644
index 0000000000000000000000000000000000000000..364cbc7b4d86abb72761f1b79cb1d2cbb89379c8
Binary files /dev/null and b/mkp/create_topology_data-0.1.0-20231028.mkp differ
diff --git a/packages/create_topology_data b/packages/create_topology_data
index e181976999ddb68f7e71290f19e4b3984d665410..1fc13fcdcf1cd90d7cd017347ec791d6cceb7ffd 100644
--- a/packages/create_topology_data
+++ b/packages/create_topology_data
@@ -57,7 +57,7 @@
'topology_data/create_topology_args.py']},
'name': 'create_topology_data',
'title': 'Network Visualization data creation',
- 'version': '0.0.9-20231027',
+ 'version': '0.1.0-20231028',
'version.min_required': '2.2.0p1',
'version.packaged': '2.2.0p11',
'version.usable_until': '2.3.0p1'}