update project

README.md

-[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.3.0-20231128.mkp "create_topology_data-0.3.0-20231128.mkp"
+[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.4.0-20231130.mkp "create_topology_data-0.4.0-20231130.mkp"
 Network Visualization data creation tool from inventory data
 
 This script creates the topology data file needed for the [Checkmk Exchange Network visualization](https://exchange.checkmk.com/p/network-visualization) plugin by Andreas Boesl and [schnetz](https://exchange.checkmk.com/u/schnetz).\

bin/topology_data/create_topology_args.py

@@ -63,12 +63,13 @@ def parse_arguments() -> arg_Namespace:
     parser.add_argument(
         '-b', '--backend',
         # nargs='+',
-        choices=['LIVESTATUS', 'FILESYSTEM', 'MULTISITE'],
+        choices=['LIVESTATUS', 'FILESYSTEM', 'MULTISITE', 'RESTAPI'],
         default='LIVESTATUS',
         help=f'Backend used to retrieve the topology data\n'
              f'     LIVESTATUS is the default for performance reasons -> only local site\n'
              f'     FILESYSTEM fetches the data directly form the inventory files -> use in distributed environments\n'
-             f'     MULTISITE like LIVESTATUS but for distribute environments -> may become the default',
+             f'     MULTISITE like LIVESTATUS but for distribute environments -> may become the default\n'
+             f'     RESTAPI uses the CMK REST API.',
     )
 
     parser.add_argument(

bin/topology_data/create_topology_classes.py

@@ -11,11 +11,12 @@
 
 from os import environ
 from pathlib import Path
-from time import strftime
+from time import strftime, time_ns
 from typing import Dict, List, Any, NamedTuple
 from enum import Enum, unique
 from abc import abstractmethod
 from ast import literal_eval
+from requests import session
 import livestatus
 
 
@@ -193,6 +194,10 @@ class HostCache:
         self.__inventory_pre_fetch_list: List[str] = [
             PATH_INTERFACES,
         ]
+        self._count = 0
+        self._debug = debug
+        if debug:
+            print('init HOST_CACHE')
 
     @abstractmethod
     def get_inventory_data(self, host: str, debug: bool = False) -> Dict[str, str] | None:
@@ -221,7 +226,15 @@ class HostCache:
 
     def __fill_cache(self, host: str):
         # pre fill inventory data
+        if self._debug:
+            self._count += 1
+            _pre_query = time_ns()
+
         inventory = self.get_inventory_data(host=host)
+
+        if self._debug:
+            print(f'{(time_ns() - _pre_query) / 1e9}|{self._count:0>4}|inventory|{host}')
+
         if inventory:
             self.__cache[host][CacheItems.inventory.value] = {}
             self.__cache[host][CacheItems.inventory.value].update({
@@ -233,8 +246,16 @@ class HostCache:
         else:
             self.__cache[host][CacheItems.inventory.value] = None
         self.__cache[host][CacheItems.interfaces.value] = {}
+
+        if self._debug:
+            self._count += 1
+            _pre_query = time_ns()
+
         self.__cache[host][CacheItems.interfaces.value][CACHE_INTERFACES_ITEM] = self.get_interface_items(host)
 
+        if self._debug:
+            print(f'{(time_ns() - _pre_query) / 1e9}|{self._count:0>4}|items|{host}')
+
     def get_data(self, host: str, item: CacheItems, path: str):
         if host not in self.__cache.keys():
             self.__cache[host]: Dict[str, Any] = {}
@@ -262,6 +283,8 @@ class HostCacheLiveStatus(HostCache):
                     print(f'exception: {e}')
                 return
             return data
+        if debug:
+            print(f'Device: {host}: no inventory data found!')
 
     def get_interface_items(self, host: str, debug: bool = False) -> List:
         """
@@ -307,24 +330,27 @@ class HostCacheFileSystem(HostCache):
             data = literal_eval(inventory_file.read_text())
             return data
         else:
+            if debug:
+                print(f'Device: {host}: not found in inventory data path!')
             return None
 
     def get_interface_items(self, host: str, debug: bool = False) -> List:
         """
-        Sample autochecks data we keep only the item
+        Sample autochecks data, we keep only the item
         [
-         {'check_plugin_name': 'if64', 'item': 'Fa0', 'parameters': {'discovered_oper_status': ['2'], ...}},
-         {'check_plugin_name': 'if64', 'item': 'Fa1/0/1', 'parameters': {'discovered_oper_status': ['1'], ...}},
-         {'check_plugin_name': 'if64', 'item': 'Fa1/0/10', 'parameters': {'discovered_oper_status': ['2'], ...}},
-         {'check_plugin_name': 'if64', 'item': 'Fa1/0/11', 'parameters': {'discovered_oper_status': ['2'], ...}},
-         {'check_plugin_name': 'if64', 'item': 'Fa1/0/12', 'parameters': {'discovered_oper_status': ['1'], ...}}
+         {'check_plugin_name': 'if64', 'item': 'Fa0', 'parameters': {'discovered_oper_status': ['2'], ...}},\n
+         {'check_plugin_name': 'if64', 'item': 'Fa1/0/1', 'parameters': {'discovered_oper_status': ['1'], ...}},\n
+         {'check_plugin_name': 'if64', 'item': 'Fa1/0/10', 'parameters': {'discovered_oper_status': ['2'], ...}},\n
+         {'check_plugin_name': 'if64', 'item': 'Fa1/0/11', 'parameters': {'discovered_oper_status': ['2'], ...}},\n
+         {'check_plugin_name': 'if64', 'item': 'Fa1/0/12', 'parameters': {'discovered_oper_status': ['1'], ...}}\n
         ]
 
         Args:
-            host:
-            debug:
+            host: name of the host object in cmk to fetch the data for
+            debug: output debug information
 
         Returns:
+            List of interface service items
 
         """
         __autochecks_path = 'var/check_mk/autochecks'
@@ -348,12 +374,14 @@ class HostCacheMultiSite(HostCache):
         super().__init__(debug=debug)
         self.__sites = {}
         self.get_sites(debug=debug)
+        if debug:
+            print('Create livestatus connection(s)')
         self.__c = livestatus.MultiSiteConnection(self.__sites)
         # self.__c.set_prepend_site(False)  # is default
         # self.__c.parallelize = True   # is default
         self.__dead_sites = [site['site']['alias'] for site in self.__c.dead_sites().values()]
         if self.__dead_sites:
-            self.__dead_sites = ', '.join( self.__dead_sites)
+            self.__dead_sites = ', '.join(self.__dead_sites)
             print(f'WARNING: use of dead site(s) {self.__dead_sites} is disabled')
             self.__c.set_only_sites(self.__c.alive_sites())
 
@@ -416,7 +444,74 @@ class HostCacheMultiSite(HostCache):
         items = []
         for host, description, check_command in data:
             items.append(description[10:])  # remove 'Interface ' from description
-
         if debug:
-            print(f'Interfaces items found: {len(items)} an host {host}')
+            if items:
+                print(f'Interfaces items found: {len(items)} an host {host}')
+            else:
+                print(f'No Interfaces items found for host {host}')
         return items
+
+
+class HostCacheRestApi(HostCache):
+    def __init__(self, debug: bool = False):
+        super().__init__(debug=debug)
+        try:
+            self.__secret = Path(f'{OMD_ROOT}/var/check_mk/web/automation/automation.secret').read_text().strip('\n)')
+        except FileNotFoundError as e:
+            print(f'automation.secret not found, {e}')
+            exit()
+        self.__hostname = 'localhost'
+        self.__site = OMD_ROOT.split('/')[-1]
+        self.__api_url = f"http://{self.__hostname}/{self.__site}/check_mk/api/1.0"
+        self.__user = 'automation'
+        if debug:
+            print('Create REST API session')
+        self.__session = session()
+        self.__session.headers['Authorization'] = f"Bearer {self.__user} {self.__secret}"
+        self.__session.headers['Accept'] = 'application/json'
+
+    def get_inventory_data(self, host: str, debug: bool = False) -> Dict[str, str] | None:
+        # self._count += 1
+
+        __query = '{"op": "=", "left": "name", "right": "' + host + '"}'
+        resp = self.__session.get(
+            f"{self.__api_url}/domain-types/host/collections/all",
+            params={
+                "query": __query,
+                "columns": ['mk_inventory'],
+            },
+        )
+        if resp.status_code == 200:
+            # print(f'{resp.elapsed}|{self._count:0>4}|inventory|{host}')
+            try:
+                return resp.json()['value'][0]['extensions']['mk_inventory']
+            except IndexError:
+                return None
+        else:
+            if debug:
+                print(f'Device: {host}: no inventory data found!')
+            return None
+
+    def get_interface_items(self, host: str, debug: bool = False) -> List:
+        # self._count += 1
+
+        __query = '{"op": "~", "left": "description", "right": "Interface "}'
+        resp = self.__session.get(
+            f"{self.__api_url}/objects/host/NX01/collections/services",
+            params={
+                "query": __query,
+                "columns": ['host_name', 'description', 'check_command'],
+            },
+        )
+
+        if resp.status_code == 200:
+            # print(f'{resp.elapsed}|{self._count:0>4}|items|{host}')
+
+            items = [service['extensions']['description'][10:] for service in resp.json()['value']]
+            if debug:
+                print(f'Interfaces items found: {len(items)} an host {host}')
+            return items
+        else:
+            if debug:
+                print(f'No Interfaces items found for host {host}')
+            return []

bin/topology_data/create_topology_data.py

@@ -39,6 +39,7 @@
 #             LIVESTATUS is the default for performance reasons -> for now only local site
 #             FILESYSTEM fetches the data directly form the inventory files -> use in distributed environments
 # 2023-11-28: implemented MULTISITE as backend, via livestatus -> use in distributed environments, my become the default
+# 2023-11-30: added RESTAPI backend --> this is kind of slow (20 seconds compared to 0.5 for MULTISITE)
 #
 # creating topology_data.json from inventory data
 #
@@ -77,6 +78,7 @@ from create_topology_classes import (
     HostCacheLiveStatus,
     HostCacheFileSystem,
     HostCacheMultiSite,
+    HostCacheRestApi,
     CacheItems,
 )
 
@@ -279,16 +281,18 @@ if __name__ == '__main__':
     print()
     print(f'Start time: {strftime(SETTINGS.time_format)}')
 
-    match SETTINGS.backend:
-        case 'LIVESTATUS':
-            HOST_CACHE = HostCacheLiveStatus()
-        case 'FILESYSTEM':
-            HOST_CACHE = HostCacheFileSystem()
-        case 'MULTISITE':
-            HOST_CACHE = HostCacheMultiSite(debug=SETTINGS.debug)
-        case _:
-            print(f'Backend {SETTINGS.backend} not (yet) implemented')
-            exit()
+    _backends = {
+        'LIVESTATUS': HostCacheLiveStatus,
+        'FILESYSTEM': HostCacheFileSystem,
+        'MULTISITE': HostCacheMultiSite,
+        'RESTAPI': HostCacheRestApi,
+    }
+    HOST_CACHE = _backends.get(SETTINGS.backend, None)
+    if not HOST_CACHE:
+        print(f'Backend {SETTINGS.backend} not (yet) implemented')
+        exit()
+    else:
+        HOST_CACHE = HOST_CACHE(debug=SETTINGS.debug)
 
     user_data = get_data_from_toml(file=SETTINGS.user_data_file)
     HOST_MAP = user_data.get('HOST_MAP', {})

packages/create_topology_data

@@ -59,7 +59,7 @@
                    'topology_data/create_topology_args.py']},
  'name': 'create_topology_data',
  'title': 'Network Visualization data creation',
- 'version': '0.3.0-20231128',
+ 'version': '0.4.0-20231130',
  'version.min_required': '2.2.0p1',
  'version.packaged': '2.2.0p14',
  'version.usable_until': '2.3.0p1'}