diff --git a/README.md b/README.md
index 9bfcfca494d35623478a0d944d1f9914ff5887f3..e42d629affb5cd86002d24359dad5384ee4d6ec8 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.3.0-20231128.mkp "create_topology_data-0.3.0-20231128.mkp"
+[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.4.0-20231130.mkp "create_topology_data-0.4.0-20231130.mkp"
Network Visualization data creation tool from inventory data
This script creates the topology data file needed for the [Checkmk Exchange Network visualization](https://exchange.checkmk.com/p/network-visualization) plugin by Andreas Boesl and [schnetz](https://exchange.checkmk.com/u/schnetz).\
diff --git a/bin/topology_data/create_topology_args.py b/bin/topology_data/create_topology_args.py
index 8fde0d9c41100a72f92eedf307c6f965da96f8ac..f0b696b3590be66f2bc1a5f9f61d282460fc9105 100755
--- a/bin/topology_data/create_topology_args.py
+++ b/bin/topology_data/create_topology_args.py
@@ -63,12 +63,13 @@ def parse_arguments() -> arg_Namespace:
parser.add_argument(
'-b', '--backend',
# nargs='+',
- choices=['LIVESTATUS', 'FILESYSTEM', 'MULTISITE'],
+ choices=['LIVESTATUS', 'FILESYSTEM', 'MULTISITE', 'RESTAPI'],
default='LIVESTATUS',
help=f'Backend used to retrieve the topology data\n'
f' LIVESTATUS is the default for performance reasons -> only local site\n'
f' FILESYSTEM fetches the data directly form the inventory files -> use in distributed environments\n'
- f' MULTISITE like LIVESTATUS but for distribute environments -> may become the default',
+ f' MULTISITE like LIVESTATUS but for distribute environments -> may become the default\n'
+ f' RESTAPI uses the CMK REST API.',
)
parser.add_argument(
diff --git a/bin/topology_data/create_topology_classes.py b/bin/topology_data/create_topology_classes.py
index 6b3a176282105c78f4c8cc13aed21cc72e4303cf..edbb2bc1979be20e88e9ca52b58ebfa3c0dcf552 100755
--- a/bin/topology_data/create_topology_classes.py
+++ b/bin/topology_data/create_topology_classes.py
@@ -11,11 +11,12 @@
from os import environ
from pathlib import Path
-from time import strftime
+from time import strftime, time_ns
from typing import Dict, List, Any, NamedTuple
from enum import Enum, unique
from abc import abstractmethod
from ast import literal_eval
+from requests import session
import livestatus
@@ -193,6 +194,10 @@ class HostCache:
self.__inventory_pre_fetch_list: List[str] = [
PATH_INTERFACES,
]
+ self._count = 0
+ self._debug = debug
+ if debug:
+ print('init HOST_CACHE')
@abstractmethod
def get_inventory_data(self, host: str, debug: bool = False) -> Dict[str, str] | None:
@@ -221,7 +226,15 @@ class HostCache:
def __fill_cache(self, host: str):
# pre fill inventory data
+ if self._debug:
+ self._count += 1
+ _pre_query = time_ns()
+
inventory = self.get_inventory_data(host=host)
+
+ if self._debug:
+ print(f'{(time_ns() - _pre_query) / 1e9}|{self._count:0>4}|inventory|{host}')
+
if inventory:
self.__cache[host][CacheItems.inventory.value] = {}
self.__cache[host][CacheItems.inventory.value].update({
@@ -233,8 +246,16 @@ class HostCache:
else:
self.__cache[host][CacheItems.inventory.value] = None
self.__cache[host][CacheItems.interfaces.value] = {}
+
+ if self._debug:
+ self._count += 1
+ _pre_query = time_ns()
+
self.__cache[host][CacheItems.interfaces.value][CACHE_INTERFACES_ITEM] = self.get_interface_items(host)
+ if self._debug:
+ print(f'{(time_ns() - _pre_query) / 1e9}|{self._count:0>4}|items|{host}')
+
def get_data(self, host: str, item: CacheItems, path: str):
if host not in self.__cache.keys():
self.__cache[host]: Dict[str, Any] = {}
@@ -262,6 +283,8 @@ class HostCacheLiveStatus(HostCache):
print(f'exception: {e}')
return
return data
+ if debug:
+ print(f'Device: {host}: no inventory data found!')
def get_interface_items(self, host: str, debug: bool = False) -> List:
"""
@@ -307,24 +330,27 @@ class HostCacheFileSystem(HostCache):
data = literal_eval(inventory_file.read_text())
return data
else:
+ if debug:
+ print(f'Device: {host}: not found in inventory data path!')
return None
def get_interface_items(self, host: str, debug: bool = False) -> List:
"""
- Sample autochecks data we keep only the item
+ Sample autochecks data, we keep only the item
[
- {'check_plugin_name': 'if64', 'item': 'Fa0', 'parameters': {'discovered_oper_status': ['2'], ...}},
- {'check_plugin_name': 'if64', 'item': 'Fa1/0/1', 'parameters': {'discovered_oper_status': ['1'], ...}},
- {'check_plugin_name': 'if64', 'item': 'Fa1/0/10', 'parameters': {'discovered_oper_status': ['2'], ...}},
- {'check_plugin_name': 'if64', 'item': 'Fa1/0/11', 'parameters': {'discovered_oper_status': ['2'], ...}},
- {'check_plugin_name': 'if64', 'item': 'Fa1/0/12', 'parameters': {'discovered_oper_status': ['1'], ...}}
+ {'check_plugin_name': 'if64', 'item': 'Fa0', 'parameters': {'discovered_oper_status': ['2'], ...}},\n
+ {'check_plugin_name': 'if64', 'item': 'Fa1/0/1', 'parameters': {'discovered_oper_status': ['1'], ...}},\n
+ {'check_plugin_name': 'if64', 'item': 'Fa1/0/10', 'parameters': {'discovered_oper_status': ['2'], ...}},\n
+ {'check_plugin_name': 'if64', 'item': 'Fa1/0/11', 'parameters': {'discovered_oper_status': ['2'], ...}},\n
+ {'check_plugin_name': 'if64', 'item': 'Fa1/0/12', 'parameters': {'discovered_oper_status': ['1'], ...}}\n
]
Args:
- host:
- debug:
+ host: name of the host object in cmk to fetch the data for
+ debug: output debug information
Returns:
+ List of interface service items
"""
__autochecks_path = 'var/check_mk/autochecks'
@@ -348,12 +374,14 @@ class HostCacheMultiSite(HostCache):
super().__init__(debug=debug)
self.__sites = {}
self.get_sites(debug=debug)
+ if debug:
+ print('Create livestatus connection(s)')
self.__c = livestatus.MultiSiteConnection(self.__sites)
# self.__c.set_prepend_site(False) # is default
# self.__c.parallelize = True # is default
self.__dead_sites = [site['site']['alias'] for site in self.__c.dead_sites().values()]
if self.__dead_sites:
- self.__dead_sites = ', '.join( self.__dead_sites)
+ self.__dead_sites = ', '.join(self.__dead_sites)
print(f'WARNING: use of dead site(s) {self.__dead_sites} is disabled')
self.__c.set_only_sites(self.__c.alive_sites())
@@ -416,7 +444,74 @@ class HostCacheMultiSite(HostCache):
items = []
for host, description, check_command in data:
items.append(description[10:]) # remove 'Interface ' from description
-
if debug:
- print(f'Interfaces items found: {len(items)} an host {host}')
+ if items:
+ print(f'Interfaces items found: {len(items)} an host {host}')
+ else:
+ print(f'No Interfaces items found for host {host}')
return items
+
+
+class HostCacheRestApi(HostCache):
+ def __init__(self, debug: bool = False):
+ super().__init__(debug=debug)
+ try:
+ self.__secret = Path(f'{OMD_ROOT}/var/check_mk/web/automation/automation.secret').read_text().strip('\n)')
+ except FileNotFoundError as e:
+ print(f'automation.secret not found, {e}')
+ exit()
+ self.__hostname = 'localhost'
+ self.__site = OMD_ROOT.split('/')[-1]
+ self.__api_url = f"http://{self.__hostname}/{self.__site}/check_mk/api/1.0"
+ self.__user = 'automation'
+ if debug:
+ print('Create REST API session')
+ self.__session = session()
+ self.__session.headers['Authorization'] = f"Bearer {self.__user} {self.__secret}"
+ self.__session.headers['Accept'] = 'application/json'
+
+ def get_inventory_data(self, host: str, debug: bool = False) -> Dict[str, str] | None:
+ # self._count += 1
+
+ __query = '{"op": "=", "left": "name", "right": "' + host + '"}'
+ resp = self.__session.get(
+ f"{self.__api_url}/domain-types/host/collections/all",
+ params={
+ "query": __query,
+ "columns": ['mk_inventory'],
+ },
+ )
+ if resp.status_code == 200:
+ # print(f'{resp.elapsed}|{self._count:0>4}|inventory|{host}')
+ try:
+ return resp.json()['value'][0]['extensions']['mk_inventory']
+ except IndexError:
+ return None
+ else:
+ if debug:
+ print(f'Device: {host}: no inventory data found!')
+ return None
+
+ def get_interface_items(self, host: str, debug: bool = False) -> List:
+ # self._count += 1
+
+ __query = '{"op": "~", "left": "description", "right": "Interface "}'
+ resp = self.__session.get(
+ f"{self.__api_url}/objects/host/NX01/collections/services",
+ params={
+ "query": __query,
+ "columns": ['host_name', 'description', 'check_command'],
+ },
+ )
+
+ if resp.status_code == 200:
+ # print(f'{resp.elapsed}|{self._count:0>4}|items|{host}')
+
+ items = [service['extensions']['description'][10:] for service in resp.json()['value']]
+ if debug:
+ print(f'Interfaces items found: {len(items)} an host {host}')
+ return items
+ else:
+ if debug:
+ print(f'No Interfaces items found for host {host}')
+ return []
diff --git a/bin/topology_data/create_topology_data.py b/bin/topology_data/create_topology_data.py
index 96198b1f7985d43f24cc3a8b24fa5cccacd7ccba..445249deac7e5ee2acc8975657d6ad830c442e73 100755
--- a/bin/topology_data/create_topology_data.py
+++ b/bin/topology_data/create_topology_data.py
@@ -39,6 +39,7 @@
# LIVESTATUS is the default for performance reasons -> for now only local site
# FILESYSTEM fetches the data directly form the inventory files -> use in distributed environments
# 2023-11-28: implemented MULTISITE as backend, via livestatus -> use in distributed environments, my become the default
+# 2023-11-30: added RESTAPI backend --> this is kind of slow (20 seconds compared to 0.5 for MULTISITE)
#
# creating topology_data.json from inventory data
#
@@ -77,6 +78,7 @@ from create_topology_classes import (
HostCacheLiveStatus,
HostCacheFileSystem,
HostCacheMultiSite,
+ HostCacheRestApi,
CacheItems,
)
@@ -279,16 +281,18 @@ if __name__ == '__main__':
print()
print(f'Start time: {strftime(SETTINGS.time_format)}')
- match SETTINGS.backend:
- case 'LIVESTATUS':
- HOST_CACHE = HostCacheLiveStatus()
- case 'FILESYSTEM':
- HOST_CACHE = HostCacheFileSystem()
- case 'MULTISITE':
- HOST_CACHE = HostCacheMultiSite(debug=SETTINGS.debug)
- case _:
- print(f'Backend {SETTINGS.backend} not (yet) implemented')
- exit()
+ _backends = {
+ 'LIVESTATUS': HostCacheLiveStatus,
+ 'FILESYSTEM': HostCacheFileSystem,
+ 'MULTISITE': HostCacheMultiSite,
+ 'RESTAPI': HostCacheRestApi,
+ }
+ HOST_CACHE = _backends.get(SETTINGS.backend, None)
+ if not HOST_CACHE:
+ print(f'Backend {SETTINGS.backend} not (yet) implemented')
+ exit()
+ else:
+ HOST_CACHE = HOST_CACHE(debug=SETTINGS.debug)
user_data = get_data_from_toml(file=SETTINGS.user_data_file)
HOST_MAP = user_data.get('HOST_MAP', {})
diff --git a/mkp/create_topology_data-0.4.0-20231130.mkp b/mkp/create_topology_data-0.4.0-20231130.mkp
new file mode 100644
index 0000000000000000000000000000000000000000..47c80f9322aeac5082214ab6b980a24bd42fafff
Binary files /dev/null and b/mkp/create_topology_data-0.4.0-20231130.mkp differ
diff --git a/packages/create_topology_data b/packages/create_topology_data
index 9c199bfded9ea3c0421e6c880132febbbffc28e8..d37aa91badd352cae98a2bac0c6ec5b762ecc617 100644
--- a/packages/create_topology_data
+++ b/packages/create_topology_data
@@ -59,7 +59,7 @@
'topology_data/create_topology_args.py']},
'name': 'create_topology_data',
'title': 'Network Visualization data creation',
- 'version': '0.3.0-20231128',
+ 'version': '0.4.0-20231130',
'version.min_required': '2.2.0p1',
'version.packaged': '2.2.0p14',
'version.usable_until': '2.3.0p1'}