diff --git a/README.md b/README.md
index bdcad01a6c9711bfcb674a7058b58ee065787442..f903ebd292a57ff2e956426f7382de04576d6349 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.1.0-20231028.mkp "create_topology_data-0.1.0-20231028.mkp"
+[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.2.0-20231116.mkp "create_topology_data-0.2.0-20231116.mkp"
Network Visualization data creation tool from inventory data
This script creates the topology data file needed for the [Checkmk Exchange Network visualization](https://exchange.checkmk.com/p/network-visualization) plugin by Andreas Boesl and [schnetz](https://exchange.checkmk.com/u/schnetz).\
diff --git a/bin/topology_data/create_topology_args.py b/bin/topology_data/create_topology_args.py
index fd8a8013ef2d9752b9d685b2b8a27759a1dc0323..af60d8655ae25d01e0efe796228f57cf387a92e0 100755
--- a/bin/topology_data/create_topology_args.py
+++ b/bin/topology_data/create_topology_args.py
@@ -9,6 +9,7 @@
#
# options used
+# -b --backend
# -d --default
# -l --layer
# -o --output-directory
@@ -59,6 +60,16 @@ def parse_arguments() -> arg_Namespace:
)
command_group = parser.add_mutually_exclusive_group()
+ parser.add_argument(
+ '-b', '--backend',
+ # nargs='+',
+ choices=['LIVESTATUS', 'FILESYSTEM', 'MULTISITE'],
+ default='LIVESTATUS',
+ help=f'Backend used to retrieve the topology data\n'
+ f'LIVESTATUS fetches the data only from the local site. In distributed environments use FILESYSTEM.\n'
+ f'For performance reasons LIVESTATUS is the default.',
+ )
+
parser.add_argument(
'-d', '--default', default=False, action='store_const', const=True,
help='Set the created topology data as default',
diff --git a/bin/topology_data/create_topology_classes.py b/bin/topology_data/create_topology_classes.py
index 14631a5d3aca46dfc875c0ca4a6a493eed99582f..2610eadb4383708fbb98a530117368f6a56a267b 100755
--- a/bin/topology_data/create_topology_classes.py
+++ b/bin/topology_data/create_topology_classes.py
@@ -14,26 +14,27 @@ from pathlib import Path
from time import strftime
from typing import Dict, List, Any, NamedTuple
from enum import Enum, unique
+from abc import abstractmethod
+from ast import literal_eval
+
from create_topology_utils import (
CREATE_TOPOLOGY_VERSION,
- PATH_CDP,
- PATH_LLDP,
PATH_INTERFACES,
USER_DATA_FILE,
- LQ_INTERFACES,
+ CACHE_INTERFACES_ITEM,
SCRIPT,
- get_inventory_data,
+ get_data_form_live_status,
get_table_from_inventory,
- get_interface_items_from_lq,
ExitCodes,
get_data_from_toml,
+ OMD_ROOT,
)
@unique
-class CacheSources(Enum):
+class CacheItems(Enum):
inventory = 'inventory'
- lq = 'lq'
+ interfaces = 'interfaces'
class InventoryColumns(NamedTuple):
@@ -59,9 +60,6 @@ class Settings:
self.__topology_save_path = 'var/topology_data'
self.__topology_file_name = 'network_data.json'
self.__path_to_if_table = 'networking,interfaces'
- # self.__inventory_path = 'var/check_mk/inventory'
- # self.__autochecks_path = 'var/check_mk/autochecks'
-
self.__settings = {
'layers': [],
'seed_devices': None,
@@ -78,6 +76,7 @@ class Settings:
'check_user_data_only': False,
'keep': None,
'min_age': None,
+ 'backend': 'LIVESTATUS',
}
# args in the form {'s, __seed_devices': 'CORE01', 'p, __path_in_inventory': None, ... }}
# we will remove 's, __'
@@ -99,6 +98,10 @@ class Settings:
print(f'-l/--layers options must be unique. Don~\'t use any layer more than once.')
exit(code=ExitCodes.BAD_OPTION_LIST.value)
+ @property
+ def backend(self) -> str:
+ return self.__settings['backend']
+
@property
def version(self) -> bool:
return self.__settings['version']
@@ -186,36 +189,151 @@ class HostCache:
def __init__(self):
self.__cache = {}
self.__inventory_pre_fetch_list: List[str] = [
- # PATH_CDP,
- # PATH_LLDP,
PATH_INTERFACES,
]
+ @abstractmethod
+ def get_inventory_data(self, host: str, debug: bool = False) -> Dict[str, str] | None:
+ """
+ Args:
+ host: the host name to return the inventory data for
+ debug: enable debug output
+
+ Returns:
+ the inventory data as dictionary
+ """
+
+ @abstractmethod
+ def get_interface_items(self, host: str, debug: bool = False) -> List:
+ """
+
+ Args:
+ host: the host name to return the interface items
+ debug: enable debug output
+
+ Returns:
+ list of the interface items
+ """
+
def __fill_cache(self, host: str):
# pre fill inventory data
- inventory = get_inventory_data(host=host)
+ inventory = self.get_inventory_data(host=host)
if inventory:
- self.__cache[host][CacheSources.inventory.value] = {}
- self.__cache[host][CacheSources.inventory.value].update({
+ self.__cache[host][CacheItems.inventory.value] = {}
+ self.__cache[host][CacheItems.inventory.value].update({
entry: get_table_from_inventory(
inventory=inventory,
path=entry
) for entry in self.__inventory_pre_fetch_list
})
else:
- self.__cache[host][CacheSources.inventory.value] = None
- # prefill live status data
- self.__cache[host][CacheSources.lq.value] = {}
- self.__cache[host][CacheSources.lq.value][LQ_INTERFACES] = get_interface_items_from_lq(host)
+ self.__cache[host][CacheItems.inventory.value] = None
+ self.__cache[host][CacheItems.interfaces.value] = {}
+ self.__cache[host][CacheItems.interfaces.value][CACHE_INTERFACES_ITEM] = self.get_interface_items(host)
- def get_data(self, host: str, source: CacheSources, path: str):
+ def get_data(self, host: str, item: CacheItems, path: str):
if host not in self.__cache.keys():
self.__cache[host]: Dict[str, Any] = {}
self.__fill_cache(host=host)
try:
- return self.__cache[host][source.value][path]
+ return self.__cache[host][item.value][path]
except (KeyError, TypeError) as _e:
return None
def add_inventory_prefetch_path(self, path: str):
self.__inventory_pre_fetch_list = list(set(self.__inventory_pre_fetch_list + [path]))
+
+
+class HostCacheLiveStatus(HostCache):
+ def get_inventory_data(self, host: str, debug: bool = False) -> Dict[str, str] | None:
+ query = f'GET hosts\nColumns: mk_inventory\nOutputFormat: python3\nFilter: host_name = {host}\n'
+ data = get_data_form_live_status(query=query)
+ if data:
+ try:
+ data = literal_eval(data[0][0].decode('utf-8'))
+ except SyntaxError as e:
+ if debug:
+ print(f'data: |{data}|')
+ print(f'type: {type(data)}')
+ print(f'exception: {e}')
+ return
+ return data
+
+ def get_interface_items(self, host: str, debug: bool = False) -> List:
+ """
+ Sample data from lq query, we keep only the item (description without "Interface" ).
+ [
+ ['C9540-7-1', 'Interface Vlan-999', 'check_mk-if64'],
+ ['C9540-7-1', 'Interface Vlan-998', 'check_mk-if64'],
+ ['C9540-7-1', 'Interface Vlan-997', 'check_mk-if64'],
+ ['C9540-7-1', 'Interface Vlan-996', 'check_mk-if64'],
+ ['C9540-7-1', 'Interface Vlan-8', 'check_mk-if64'],
+ ['C9540-7-1', 'Interface Te2/0/2', 'check_mk-if64'],
+ ['C9540-7-1', 'Interface Te2/0/30', 'check_mk-if64']
+ ]
+ Args:
+ host:
+ debug:
+
+ Returns:
+
+ """
+ query = (
+ 'GET services\n'
+ 'Columns: host_name description check_command\n'
+ 'Filter: description ~ Interface\n'
+ f'Filter: host_name = {host}\n'
+ 'OutputFormat: python3\n'
+ )
+ data = get_data_form_live_status(query=query)
+ items = []
+ for host, description, check_command in data:
+ items.append(description[10:]) # remove 'Interface ' from description
+
+ if debug:
+ print(f'Interfaces items found: {len(items)} an host {host}')
+ return items
+
+
+class HostCacheFileSystem(HostCache):
+ def get_inventory_data(self, host: str, debug: bool = False) -> Dict[str, str] | None:
+ __inventory_path = 'var/check_mk/inventory'
+ inventory_file = Path(f'{OMD_ROOT}/{__inventory_path}/{host}')
+ if inventory_file.exists():
+ data = literal_eval(inventory_file.read_text())
+ return data
+ else:
+ return None
+
+ def get_interface_items(self, host: str, debug: bool = False) -> List:
+ """
+ Sample autochecks data we keep only the item
+ [
+ {'check_plugin_name': 'if64', 'item': 'Fa0', 'parameters': {'discovered_oper_status': ['2'], ...}},
+ {'check_plugin_name': 'if64', 'item': 'Fa1/0/1', 'parameters': {'discovered_oper_status': ['1'], ...}},
+ {'check_plugin_name': 'if64', 'item': 'Fa1/0/10', 'parameters': {'discovered_oper_status': ['2'], ...}},
+ {'check_plugin_name': 'if64', 'item': 'Fa1/0/11', 'parameters': {'discovered_oper_status': ['2'], ...}},
+ {'check_plugin_name': 'if64', 'item': 'Fa1/0/12', 'parameters': {'discovered_oper_status': ['1'], ...}}
+ ]
+
+ Args:
+ host:
+ debug:
+
+ Returns:
+
+ """
+ __autochecks_path = 'var/check_mk/autochecks'
+ autochecks_file = Path(f'{OMD_ROOT}/{__autochecks_path}/{host}.mk')
+ __data = []
+ if autochecks_file.exists():
+ data: List[Dict[str, str]] = literal_eval(autochecks_file.read_text())
+ for service in data:
+ if service['check_plugin_name'] in ['if64']:
+ __data.append(service['item'])
+ else:
+ if debug:
+ print(f'Device: {host}: not found in auto checks path!')
+ return []
+
+ return __data
diff --git a/bin/topology_data/create_topology_data.py b/bin/topology_data/create_topology_data.py
index 86d43c523096a4ed0824958d67e3e3bf88b5b089..f0b3578d804aff1190495a703767ee845d536a52 100755
--- a/bin/topology_data/create_topology_data.py
+++ b/bin/topology_data/create_topology_data.py
@@ -34,6 +34,7 @@
# removed option --inventory-columns, now handled in CUSTOM_LAYERS
# removed option --path-in-inventory, now handled in CUSTOM_LAYERS
# removed lower limits for --keep and --min-age
+# 2023-1ß-29: fixed missing path from custom layer in host prefetch
#
# PoC for creating topology_data.json from inventory data
@@ -61,7 +62,7 @@ from create_topology_utils import (
get_data_from_toml,
merge_topologies,
save_topology,
- LQ_INTERFACES,
+ CACHE_INTERFACES_ITEM,
PATH_INTERFACES,
LAYERS,
)
@@ -70,8 +71,9 @@ from create_topology_classes import (
InventoryColumns,
Settings,
StaticConnection,
- HostCache,
- CacheSources,
+ HostCacheLiveStatus,
+ HostCacheFileSystem,
+ CacheItems,
)
HOST_MAP: Dict[str, str] = {}
@@ -89,7 +91,7 @@ def get_list_of_devices(data) -> List[str]:
def get_service_by_interface(host: str, interface: str, debug: bool = False) -> str:
# try to find the item for an interface
def _match_entry_with_item(_entry: Dict[str, str]):
- values = [_entry.get('name'), _entry.get('description'), entry.get('alias')]
+ values = [_entry.get('name'), _entry.get('description'), _entry.get('alias')]
for value in values:
if value in items:
return value
@@ -112,24 +114,24 @@ def get_service_by_interface(host: str, interface: str, debug: bool = False) ->
return interface
# get list of interface items
- items = HOST_CACHE.get_data(host=host, source=CacheSources.lq, path=LQ_INTERFACES)
+ items = HOST_CACHE.get_data(host=host, item=CacheItems.interfaces, path=CACHE_INTERFACES_ITEM)
# the easy case
if interface in items:
return interface
else:
# try to find the interface in the host interface inventory list
- inventory = HOST_CACHE.get_data(host=host, source=CacheSources.inventory, path=PATH_INTERFACES)
+ inventory = HOST_CACHE.get_data(host=host, item=CacheItems.inventory, path=PATH_INTERFACES)
if inventory:
- for entry in inventory:
+ for _entry in inventory:
if interface in [
- entry.get('name'),
- entry.get('description'),
- entry.get('alias'),
- str(entry.get('index')),
- entry.get('phys_address'),
+ _entry.get('name'),
+ _entry.get('description'),
+ _entry.get('alias'),
+ str(_entry.get('index')),
+ _entry.get('phys_address'),
]:
- return _match_entry_with_item(entry)
+ return _match_entry_with_item(_entry)
if debug:
print(f'Device: {host}: service for interface {interface} not found')
@@ -242,7 +244,7 @@ def create_topology(
continue
# topo_data = get_inventory_data_lq(host=device, path=path_in_inventory, debug=debug)
- topo_data = HOST_CACHE.get_data(host=device, source=CacheSources.inventory, path=path_in_inventory)
+ topo_data = HOST_CACHE.get_data(host=device, item=CacheItems.inventory, path=path_in_inventory)
if topo_data:
topology_data.update(
create_device_from_inv(
@@ -252,9 +254,9 @@ def create_topology(
label=label,
))
devices_list = get_list_of_devices(topology_data[device]['connections'])
- for entry in devices_list:
- if entry not in devices_done:
- devices_to_go.append(entry)
+ for _entry in devices_list:
+ if _entry not in devices_done:
+ devices_to_go.append(_entry)
devices_to_go = list(set(devices_to_go))
devices_done.append(device)
@@ -270,7 +272,14 @@ def create_topology(
if __name__ == '__main__':
start_time = time_ns()
SETTINGS = Settings(vars(parse_arguments()))
- HOST_CACHE = HostCache()
+ match SETTINGS.backend:
+ case 'LIVESTATUS':
+ HOST_CACHE = HostCacheLiveStatus()
+ case 'FILESYSTEM':
+ HOST_CACHE = HostCacheFileSystem()
+ case _:
+ print(f'Backend {SETTINGS.backend} not (yet) implemented')
+ exit()
print()
print(f'Start time: {strftime(SETTINGS.time_format)}')
@@ -287,7 +296,7 @@ if __name__ == '__main__':
jobs.append('STATIC')
elif layer in LAYERS.keys():
jobs.append(LAYERS[layer])
- HOST_CACHE.add_inventory_prefetch_path(LAYERS[layer]['path'])
+ HOST_CACHE.add_inventory_prefetch_path(path=LAYERS[layer]['path'])
elif layer == 'CUSTOM':
for entry in CUSTOM_LAYERS:
jobs.append(entry)
diff --git a/bin/topology_data/create_topology_utils.py b/bin/topology_data/create_topology_utils.py
index cd0d0237d926a0ec31a1ce0645ffea0edb58d2db..a59bbdef80fc13a17c0be834d5401baf56cd4df6 100755
--- a/bin/topology_data/create_topology_utils.py
+++ b/bin/topology_data/create_topology_utils.py
@@ -39,15 +39,16 @@ LABEL_LLDP = 'inv_LLDP'
COLUMNS_LLDP = 'system_name,local_port_num,port_id'
COLUMNS_CDP = 'device_id,local_port,device_port'
USER_DATA_FILE = 'create_topology_data.toml'
-LQ_INTERFACES = 'interface_items'
+CACHE_INTERFACES_ITEM = 'interface_items'
LAYERS = {
'CDP': {'path': PATH_CDP, 'columns': COLUMNS_CDP, 'label': LABEL_CDP},
'LLDP': {'path': PATH_LLDP, 'columns': COLUMNS_LLDP, 'label': LABEL_LLDP},
}
+OMD_ROOT = environ["OMD_ROOT"]
def get_data_form_live_status(query: str):
- address = f'{environ.get("OMD_ROOT")}/tmp/run/live'
+ address = f'{OMD_ROOT}/tmp/run/live'
family = socket.AF_INET if type(address) is tuple else socket.AF_UNIX
sock = socket.socket(family, socket.SOCK_STREAM)
sock.connect(address)
@@ -84,7 +85,7 @@ def get_data_from_toml(file: str, debug: bool = False) -> Dict:
def rm_tree(root: Path):
# safety
- if not str(root).startswith(f'{environ["OMD_ROOT"]}/var/topology_data'):
+ if not str(root).startswith(f'{OMD_ROOT}/var/topology_data'):
print(f"WARNING: bad path to remove, {str(root)}, don\'t delete it.")
return
for p in root.iterdir():
@@ -300,22 +301,6 @@ def is_equal_with_default(data: Dict, file: str) -> bool:
return compare_dicts(data, default_data)
-def get_inventory_data(host: str, debug: bool = False, ) -> Dict[str, str] | None:
- query = f'GET hosts\nColumns: mk_inventory\nOutputFormat: python3\nFilter: host_name = {host}\n'
- data = get_data_form_live_status(query=query)
-
- if data:
- try:
- data = literal_eval(data[0][0].decode('utf-8'))
- except SyntaxError as e:
- if debug:
- print(f'data: |{data}|')
- print(f'type: {type(data)}')
- print(f'exception: {e}')
- return
- return data
-
-
def get_table_from_inventory(inventory: Dict[str, Any], path: List[str]) -> List | None:
path = ('Nodes,' + ',Nodes,'.join(path.split(',')) + ',Table,Rows').split(',')
table = inventory.copy()
@@ -325,21 +310,3 @@ def get_table_from_inventory(inventory: Dict[str, Any], path: List[str]) -> List
except KeyError:
return
return table
-
-
-def get_interface_items_from_lq(host: str, debug: bool = False) -> List:
- query = (
- 'GET services\n'
- 'Columns: host_name description check_command\n'
- 'Filter: description ~ Interface\n'
- f'Filter: host_name = {host}\n'
- 'OutputFormat: python3\n'
- )
- data = get_data_form_live_status(query=query)
- items = []
- for host, description, check_command in data:
- items.append(description[10:]) # remove 'Interface ' from description
-
- if debug:
- print(f'Interfaces items found: {len(items)} an host {host}')
- return items
diff --git a/mkp/create_topology_data-0.2.0-20231116.mkp b/mkp/create_topology_data-0.2.0-20231116.mkp
new file mode 100644
index 0000000000000000000000000000000000000000..3dfb59c892d676c4d87023841ca96f214fbbc05a
Binary files /dev/null and b/mkp/create_topology_data-0.2.0-20231116.mkp differ
diff --git a/packages/create_topology_data b/packages/create_topology_data
index 1fc13fcdcf1cd90d7cd017347ec791d6cceb7ffd..49be6c70566befd421ec6179e058f50f5b6a76b8 100644
--- a/packages/create_topology_data
+++ b/packages/create_topology_data
@@ -1,6 +1,5 @@
{'author': 'Th.L. (thl-cmk[at]outlook[dot]com)',
- 'description': 'PoC for Network Visualization data creation from inventory '
- 'data\n'
+ 'description': 'Network Visualization data creation tool from inventory data\n'
'\n'
'This tool creates the topology data file needed for the '
'Checkmk "Network Visualization" plugin \n'
@@ -34,6 +33,9 @@
'LLDP: '
'https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_lldp_cache\n'
'\n'
+ 'For the latest version and documentation see:\n'
+ 'https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/create_topology_data\n'
+ '\n'
'#########################################################################\n'
'Important:\n'
'\n'
@@ -57,7 +59,7 @@
'topology_data/create_topology_args.py']},
'name': 'create_topology_data',
'title': 'Network Visualization data creation',
- 'version': '0.1.0-20231028',
+ 'version': '0.2.0-20231116',
'version.min_required': '2.2.0p1',
'version.packaged': '2.2.0p11',
'version.usable_until': '2.3.0p1'}