update project

README.md

-[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.1.0-20231028.mkp "create_topology_data-0.1.0-20231028.mkp"
+[PACKAGE]: ../../raw/master/mkp/create_topology_data-0.2.0-20231116.mkp "create_topology_data-0.2.0-20231116.mkp"
 Network Visualization data creation tool from inventory data
 
 This script creates the topology data file needed for the [Checkmk Exchange Network visualization](https://exchange.checkmk.com/p/network-visualization) plugin by Andreas Boesl and [schnetz](https://exchange.checkmk.com/u/schnetz).\

bin/topology_data/create_topology_args.py

@@ -9,6 +9,7 @@
 
 #
 # options used
+# -b --backend
 # -d --default
 # -l --layer
 # -o --output-directory
@@ -59,6 +60,16 @@ def parse_arguments() -> arg_Namespace:
     )
     command_group = parser.add_mutually_exclusive_group()
 
+    parser.add_argument(
+        '-b', '--backend',
+        # nargs='+',
+        choices=['LIVESTATUS', 'FILESYSTEM', 'MULTISITE'],
+        default='LIVESTATUS',
+        help=f'Backend used to retrieve the topology data\n'
+             f'LIVESTATUS fetches the data only from the local site. In distributed environments use FILESYSTEM.\n'
+             f'For performance reasons LIVESTATUS is the default.',
+    )
+
     parser.add_argument(
         '-d', '--default', default=False, action='store_const', const=True,
         help='Set the created topology data as default',

bin/topology_data/create_topology_classes.py

@@ -14,26 +14,27 @@ from pathlib import Path
 from time import strftime
 from typing import Dict, List, Any, NamedTuple
 from enum import Enum, unique
+from abc import abstractmethod
+from ast import literal_eval
+
 from create_topology_utils import (
     CREATE_TOPOLOGY_VERSION,
-    PATH_CDP,
-    PATH_LLDP,
     PATH_INTERFACES,
     USER_DATA_FILE,
-    LQ_INTERFACES,
+    CACHE_INTERFACES_ITEM,
     SCRIPT,
-    get_inventory_data,
+    get_data_form_live_status,
     get_table_from_inventory,
-    get_interface_items_from_lq,
     ExitCodes,
     get_data_from_toml,
+    OMD_ROOT,
 )
 
 
 @unique
-class CacheSources(Enum):
+class CacheItems(Enum):
     inventory = 'inventory'
-    lq = 'lq'
+    interfaces = 'interfaces'
 
 
 class InventoryColumns(NamedTuple):
@@ -59,9 +60,6 @@ class Settings:
         self.__topology_save_path = 'var/topology_data'
         self.__topology_file_name = 'network_data.json'
         self.__path_to_if_table = 'networking,interfaces'
-        # self.__inventory_path = 'var/check_mk/inventory'
-        # self.__autochecks_path = 'var/check_mk/autochecks'
-
         self.__settings = {
             'layers': [],
             'seed_devices': None,
@@ -78,6 +76,7 @@ class Settings:
             'check_user_data_only': False,
             'keep': None,
             'min_age': None,
+            'backend': 'LIVESTATUS',
         }
         # args in the form {'s, __seed_devices': 'CORE01', 'p, __path_in_inventory': None, ... }}
         # we will remove 's, __'
@@ -99,6 +98,10 @@ class Settings:
                 print(f'-l/--layers options must be unique. Don~\'t use any layer more than once.')
                 exit(code=ExitCodes.BAD_OPTION_LIST.value)
 
+    @property
+    def backend(self) -> str:
+        return self.__settings['backend']
+
     @property
     def version(self) -> bool:
         return self.__settings['version']
@@ -186,36 +189,151 @@ class HostCache:
     def __init__(self):
         self.__cache = {}
         self.__inventory_pre_fetch_list: List[str] = [
-            # PATH_CDP,
-            # PATH_LLDP,
             PATH_INTERFACES,
         ]
 
+    @abstractmethod
+    def get_inventory_data(self, host: str, debug: bool = False) -> Dict[str, str] | None:
+        """
+        Args:
+            host: the host name to return the inventory data for
+            debug: enable debug output
+
+        Returns:
+            the inventory data as dictionary
+        """
+
+    @abstractmethod
+    def get_interface_items(self, host: str, debug: bool = False) -> List:
+        """
+
+        Args:
+            host:  the host name to return the interface items
+            debug: enable debug output
+
+        Returns:
+            list of the interface items
+        """
+
     def __fill_cache(self, host: str):
         # pre fill inventory data
-        inventory = get_inventory_data(host=host)
+        inventory = self.get_inventory_data(host=host)
         if inventory:
-            self.__cache[host][CacheSources.inventory.value] = {}
-            self.__cache[host][CacheSources.inventory.value].update({
+            self.__cache[host][CacheItems.inventory.value] = {}
+            self.__cache[host][CacheItems.inventory.value].update({
                 entry: get_table_from_inventory(
                     inventory=inventory,
                     path=entry
                 ) for entry in self.__inventory_pre_fetch_list
             })
         else:
-            self.__cache[host][CacheSources.inventory.value] = None
-        # prefill live status data
-        self.__cache[host][CacheSources.lq.value] = {}
-        self.__cache[host][CacheSources.lq.value][LQ_INTERFACES] = get_interface_items_from_lq(host)
+            self.__cache[host][CacheItems.inventory.value] = None
+        self.__cache[host][CacheItems.interfaces.value] = {}
+        self.__cache[host][CacheItems.interfaces.value][CACHE_INTERFACES_ITEM] = self.get_interface_items(host)
 
-    def get_data(self, host: str, source: CacheSources, path: str):
+    def get_data(self, host: str, item: CacheItems, path: str):
         if host not in self.__cache.keys():
             self.__cache[host]: Dict[str, Any] = {}
             self.__fill_cache(host=host)
         try:
-            return self.__cache[host][source.value][path]
+            return self.__cache[host][item.value][path]
         except (KeyError, TypeError) as _e:
             return None
 
     def add_inventory_prefetch_path(self, path: str):
         self.__inventory_pre_fetch_list = list(set(self.__inventory_pre_fetch_list + [path]))
+
+
+class HostCacheLiveStatus(HostCache):
+    def get_inventory_data(self, host: str, debug: bool = False) -> Dict[str, str] | None:
+        query = f'GET hosts\nColumns: mk_inventory\nOutputFormat: python3\nFilter: host_name = {host}\n'
+        data = get_data_form_live_status(query=query)
+        if data:
+            try:
+                data = literal_eval(data[0][0].decode('utf-8'))
+            except SyntaxError as e:
+                if debug:
+                    print(f'data: |{data}|')
+                    print(f'type: {type(data)}')
+                    print(f'exception: {e}')
+                return
+            return data
+
+    def get_interface_items(self, host: str, debug: bool = False) -> List:
+        """
+        Sample data from lq query, we keep only the item (description without "Interface" ).
+        [
+         ['C9540-7-1', 'Interface Vlan-999', 'check_mk-if64'],
+         ['C9540-7-1', 'Interface Vlan-998', 'check_mk-if64'],
+         ['C9540-7-1', 'Interface Vlan-997', 'check_mk-if64'],
+         ['C9540-7-1', 'Interface Vlan-996', 'check_mk-if64'],
+         ['C9540-7-1', 'Interface Vlan-8', 'check_mk-if64'],
+         ['C9540-7-1', 'Interface Te2/0/2', 'check_mk-if64'],
+         ['C9540-7-1', 'Interface Te2/0/30', 'check_mk-if64']
+         ]
+        Args:
+            host:
+            debug:
+
+        Returns:
+
+        """
+        query = (
+            'GET services\n'
+            'Columns: host_name description check_command\n'
+            'Filter: description ~ Interface\n'
+            f'Filter: host_name = {host}\n'
+            'OutputFormat: python3\n'
+        )
+        data = get_data_form_live_status(query=query)
+        items = []
+        for host, description, check_command in data:
+            items.append(description[10:])  # remove 'Interface ' from description
+
+        if debug:
+            print(f'Interfaces items found: {len(items)} an host {host}')
+        return items
+
+
+class HostCacheFileSystem(HostCache):
+    def get_inventory_data(self, host: str, debug: bool = False) -> Dict[str, str] | None:
+        __inventory_path = 'var/check_mk/inventory'
+        inventory_file = Path(f'{OMD_ROOT}/{__inventory_path}/{host}')
+        if inventory_file.exists():
+            data = literal_eval(inventory_file.read_text())
+            return data
+        else:
+            return None
+
+    def get_interface_items(self, host: str, debug: bool = False) -> List:
+        """
+        Sample autochecks data we keep only the item
+        [
+         {'check_plugin_name': 'if64', 'item': 'Fa0', 'parameters': {'discovered_oper_status': ['2'], ...}},
+         {'check_plugin_name': 'if64', 'item': 'Fa1/0/1', 'parameters': {'discovered_oper_status': ['1'], ...}},
+         {'check_plugin_name': 'if64', 'item': 'Fa1/0/10', 'parameters': {'discovered_oper_status': ['2'], ...}},
+         {'check_plugin_name': 'if64', 'item': 'Fa1/0/11', 'parameters': {'discovered_oper_status': ['2'], ...}},
+         {'check_plugin_name': 'if64', 'item': 'Fa1/0/12', 'parameters': {'discovered_oper_status': ['1'], ...}}
+        ]
+
+        Args:
+            host:
+            debug:
+
+        Returns:
+
+        """
+        __autochecks_path = 'var/check_mk/autochecks'
+        autochecks_file = Path(f'{OMD_ROOT}/{__autochecks_path}/{host}.mk')
+        __data = []
+        if autochecks_file.exists():
+            data: List[Dict[str, str]] = literal_eval(autochecks_file.read_text())
+            for service in data:
+                if service['check_plugin_name'] in ['if64']:
+                    __data.append(service['item'])
+        else:
+            if debug:
+                print(f'Device: {host}: not found in auto checks path!')
+            return []
+
+        return __data

bin/topology_data/create_topology_data.py

@@ -34,6 +34,7 @@
 #             removed option --inventory-columns, now handled in CUSTOM_LAYERS
 #             removed option --path-in-inventory, now handled in CUSTOM_LAYERS
 #             removed lower limits for --keep and --min-age
+# 2023-1ß-29: fixed missing path from custom layer in host prefetch
 
 #
 # PoC for creating topology_data.json from inventory data
@@ -61,7 +62,7 @@ from create_topology_utils import (
     get_data_from_toml,
     merge_topologies,
     save_topology,
-    LQ_INTERFACES,
+    CACHE_INTERFACES_ITEM,
     PATH_INTERFACES,
     LAYERS,
 )
@@ -70,8 +71,9 @@ from create_topology_classes import (
     InventoryColumns,
     Settings,
     StaticConnection,
-    HostCache,
-    CacheSources,
+    HostCacheLiveStatus,
+    HostCacheFileSystem,
+    CacheItems,
 )
 
 HOST_MAP: Dict[str, str] = {}
@@ -89,7 +91,7 @@ def get_list_of_devices(data) -> List[str]:
 def get_service_by_interface(host: str, interface: str, debug: bool = False) -> str:
     # try to find the item for an interface
     def _match_entry_with_item(_entry: Dict[str, str]):
-        values = [_entry.get('name'), _entry.get('description'), entry.get('alias')]
+        values = [_entry.get('name'), _entry.get('description'), _entry.get('alias')]
         for value in values:
             if value in items:
                 return value
@@ -112,24 +114,24 @@ def get_service_by_interface(host: str, interface: str, debug: bool = False) ->
         return interface
 
     # get list of interface items
-    items = HOST_CACHE.get_data(host=host, source=CacheSources.lq, path=LQ_INTERFACES)
+    items = HOST_CACHE.get_data(host=host, item=CacheItems.interfaces, path=CACHE_INTERFACES_ITEM)
 
     # the easy case
     if interface in items:
         return interface
     else:
         # try to find the interface in the host interface inventory list
-        inventory = HOST_CACHE.get_data(host=host, source=CacheSources.inventory, path=PATH_INTERFACES)
+        inventory = HOST_CACHE.get_data(host=host, item=CacheItems.inventory, path=PATH_INTERFACES)
         if inventory:
-            for entry in inventory:
+            for _entry in inventory:
                 if interface in [
-                    entry.get('name'),
-                    entry.get('description'),
-                    entry.get('alias'),
-                    str(entry.get('index')),
-                    entry.get('phys_address'),
+                    _entry.get('name'),
+                    _entry.get('description'),
+                    _entry.get('alias'),
+                    str(_entry.get('index')),
+                    _entry.get('phys_address'),
                 ]:
-                    return _match_entry_with_item(entry)
+                    return _match_entry_with_item(_entry)
         if debug:
             print(f'Device: {host}: service for interface {interface} not found')
 
@@ -242,7 +244,7 @@ def create_topology(
                 continue
 
         # topo_data = get_inventory_data_lq(host=device, path=path_in_inventory, debug=debug)
-        topo_data = HOST_CACHE.get_data(host=device, source=CacheSources.inventory, path=path_in_inventory)
+        topo_data = HOST_CACHE.get_data(host=device, item=CacheItems.inventory, path=path_in_inventory)
         if topo_data:
             topology_data.update(
                 create_device_from_inv(
@@ -252,9 +254,9 @@ def create_topology(
                     label=label,
                 ))
             devices_list = get_list_of_devices(topology_data[device]['connections'])
-            for entry in devices_list:
-                if entry not in devices_done:
-                    devices_to_go.append(entry)
+            for _entry in devices_list:
+                if _entry not in devices_done:
+                    devices_to_go.append(_entry)
 
         devices_to_go = list(set(devices_to_go))
         devices_done.append(device)
@@ -270,7 +272,14 @@ def create_topology(
 if __name__ == '__main__':
     start_time = time_ns()
     SETTINGS = Settings(vars(parse_arguments()))
-    HOST_CACHE = HostCache()
+    match SETTINGS.backend:
+        case 'LIVESTATUS':
+            HOST_CACHE = HostCacheLiveStatus()
+        case 'FILESYSTEM':
+            HOST_CACHE = HostCacheFileSystem()
+        case _:
+            print(f'Backend {SETTINGS.backend} not (yet) implemented')
+            exit()
 
     print()
     print(f'Start time: {strftime(SETTINGS.time_format)}')
@@ -287,7 +296,7 @@ if __name__ == '__main__':
             jobs.append('STATIC')
         elif layer in LAYERS.keys():
             jobs.append(LAYERS[layer])
-            HOST_CACHE.add_inventory_prefetch_path(LAYERS[layer]['path'])
+            HOST_CACHE.add_inventory_prefetch_path(path=LAYERS[layer]['path'])
         elif layer == 'CUSTOM':
             for entry in CUSTOM_LAYERS:
                 jobs.append(entry)

bin/topology_data/create_topology_utils.py

@@ -39,15 +39,16 @@ LABEL_LLDP = 'inv_LLDP'
 COLUMNS_LLDP = 'system_name,local_port_num,port_id'
 COLUMNS_CDP = 'device_id,local_port,device_port'
 USER_DATA_FILE = 'create_topology_data.toml'
-LQ_INTERFACES = 'interface_items'
+CACHE_INTERFACES_ITEM = 'interface_items'
 LAYERS = {
     'CDP': {'path': PATH_CDP, 'columns': COLUMNS_CDP, 'label': LABEL_CDP},
     'LLDP': {'path': PATH_LLDP, 'columns': COLUMNS_LLDP, 'label': LABEL_LLDP},
 }
+OMD_ROOT = environ["OMD_ROOT"]
 
 
 def get_data_form_live_status(query: str):
-    address = f'{environ.get("OMD_ROOT")}/tmp/run/live'
+    address = f'{OMD_ROOT}/tmp/run/live'
     family = socket.AF_INET if type(address) is tuple else socket.AF_UNIX
     sock = socket.socket(family, socket.SOCK_STREAM)
     sock.connect(address)
@@ -84,7 +85,7 @@ def get_data_from_toml(file: str, debug: bool = False) -> Dict:
 
 def rm_tree(root: Path):
     # safety
-    if not str(root).startswith(f'{environ["OMD_ROOT"]}/var/topology_data'):
+    if not str(root).startswith(f'{OMD_ROOT}/var/topology_data'):
         print(f"WARNING: bad path to remove, {str(root)}, don\'t delete it.")
         return
     for p in root.iterdir():
@@ -300,22 +301,6 @@ def is_equal_with_default(data: Dict, file: str) -> bool:
         return compare_dicts(data, default_data)
 
 
-def get_inventory_data(host: str, debug: bool = False, ) -> Dict[str, str] | None:
-    query = f'GET hosts\nColumns: mk_inventory\nOutputFormat: python3\nFilter: host_name = {host}\n'
-    data = get_data_form_live_status(query=query)
-
-    if data:
-        try:
-            data = literal_eval(data[0][0].decode('utf-8'))
-        except SyntaxError as e:
-            if debug:
-                print(f'data: |{data}|')
-                print(f'type: {type(data)}')
-                print(f'exception: {e}')
-            return
-        return data
-
-
 def get_table_from_inventory(inventory: Dict[str, Any], path: List[str]) -> List | None:
     path = ('Nodes,' + ',Nodes,'.join(path.split(',')) + ',Table,Rows').split(',')
     table = inventory.copy()
@@ -325,21 +310,3 @@ def get_table_from_inventory(inventory: Dict[str, Any], path: List[str]) -> List
         except KeyError:
             return
     return table
-
-
-def get_interface_items_from_lq(host: str, debug: bool = False) -> List:
-    query = (
-        'GET services\n'
-        'Columns: host_name description check_command\n'
-        'Filter: description ~ Interface\n'
-        f'Filter: host_name = {host}\n'
-        'OutputFormat: python3\n'
-    )
-    data = get_data_form_live_status(query=query)
-    items = []
-    for host, description, check_command in data:
-        items.append(description[10:])  # remove 'Interface ' from description
-
-    if debug:
-        print(f'Interfaces items found: {len(items)} an host {host}')
-    return items

packages/create_topology_data

 {'author': 'Th.L. (thl-cmk[at]outlook[dot]com)',
- 'description': 'PoC for Network Visualization data creation from inventory '
-                'data\n'
+ 'description': 'Network Visualization data creation tool from inventory data\n'
                 '\n'
                 'This tool creates the topology data file needed for the '
                 'Checkmk "Network Visualization" plugin \n'
@@ -34,6 +33,9 @@
                 'LLDP: '
                 'https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_lldp_cache\n'
                 '\n'
+                'For the latest version and documentation see:\n'
+                'https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/create_topology_data\n'
+                '\n'
                 '#########################################################################\n'
                 'Important:\n'
                 '\n'
@@ -57,7 +59,7 @@
                    'topology_data/create_topology_args.py']},
  'name': 'create_topology_data',
  'title': 'Network Visualization data creation',
- 'version': '0.1.0-20231028',
+ 'version': '0.2.0-20231116',
  'version.min_required': '2.2.0p1',
  'version.packaged': '2.2.0p11',
  'version.usable_until': '2.3.0p1'}