added option --include-l3-hosts, changed default layers to None, reworked static topology

README.md

-[PACKAGE]: ../../raw/master/mkp/nvdct-0.9.3-20241209.mkp "nvdct-0.9.3-20241209.mkp"
+[PACKAGE]: ../../raw/master/mkp/nvdct-0.9.4-20241210.mkp "nvdct-0.9.4-20241210.mkp"
 # Network Visualization Data Creation Tool (NVDCT)
 
 This script creates the topology data file needed for the [Checkmk Exchange Network visualization](https://exchange.checkmk.com/p/network-visualization) plugin.\

source/bin/nvdct/conf/nvdct.toml

@@ -13,6 +13,7 @@
 #
 
 # list of (additional to -s/--seed-devices) seed devices
+# [0-9-a-zA-Z\.\_\-]{1,253} -> host
 L2_SEED_DEVICES = [
     # "CORE01",
     # "LOCATION01",
@@ -26,6 +27,7 @@ L2_DROP_HOSTS = [
 ]
 
 # hosts will be ignored in L3v4 topology
+# [0-9-a-zA-Z\.\_\-]{1,253} -> host
 L3V4_IGNORE_HOSTS = [
     # "host1",
     # "host2",
@@ -64,29 +66,39 @@ PROTECTED_TOPOLOGIES = [
 ]
 
 # user defined static connections
-# these connections will be added from host to neighbour and in reverese
-# hosts/neighbours in this section will be added to SEED_DEVICES
+# [0-9-a-zA-Z\.\_\-]{1,253} -> host
 STATIC_CONNECTIONS = [
-    # ["cmk_host1", "local-port1", "neighbour-port1", "neighbour1", "label"],
-    # ["cmk_host1", "local-port2", "neighbour-port2", "neighbour2", "label"],
+    # valid entry formats
+    # ["left_host", "left_service", "right_service", "right_host"],
+    # connection: "left_host"<->"left_service"<->"right_service"<->"right_host"
+    # ["left_host", "", "right_service", "right_host"],
+    # connection: "left_host"<->"right_service"<->"right_host"
+    # ["left_host", "left_service", "", "right_host"],
+    # connection: "left_host"<->"left_service"<->"right_host"
+    # ["left_host", "", "", "right_host"],
+    # connection: "left_host"<->"right_host"
 ]
 
-# optional custom layers use option -l/--layers CUSTOM to include this layers
+# THIS OPTION IS DEPRECATED
+# optional custom layers use option -l/--layers CUSTOM to include these layers
 # don't use --pre-fetch without a host_label that matches all host you want to add
+# THIS OPTION IS DEPRECATED
 CUSTOM_LAYERS = [
 #    { path = "path,in,inventory", columns = "columns from inventory", label = "label for the layer", host_label = "CMK host label to find matching hosts" },
 #    { path = "networking,lldp_cache,neighbours", columns = "neighbour_name,local_port,neighbour_port", label = "custom_LLDP", host_label = "nvdct/has_lldp_neighbours" },
 #    { path = "networking,cdp_cache,neighbours", columns = "neighbour_name,local_port,neighbour_port", label = "custom_CDP", host_label = "nvdct/has_cdp_neighbours" },
 ]
 
-# list customers so include/excluse, use option --filter-costumers INCLUDE/EXCLUDE
+# list customers to include/excluse, use option --filter-costumers INCLUDE/EXCLUDE
+# [0-9-a-zA-Z\.\_\-]{1,16} -> customer
 CUSTOMERS = [
     # "customer1",
     # "customer2",
     # "customer3",
 ]
 
-# list site so include/excluse, use option --filter-sites INCLUDE/EXCLUDE
+# list site to include/excluse, use option --filter-sites INCLUDE/EXCLUDE
+# [0-9-a-zA-Z\.\_\-]{1,16} -> site
 SITES = [
     # "site1",
     # "site2",
@@ -94,6 +106,7 @@ SITES = [
 ]
 
 # map inventory neighbour name to Checkmk host name
+# [0-9-a-zA-Z\.\_\-]{1,253} -> host
 [L2_HOST_MAP]
 # inventory_neighbour1 = "cmk_host1"
 # inventory_neighbour2 = "cmk_host2"
@@ -106,6 +119,7 @@ SITES = [
 # "^Meraki.*\\s-\\s" = ""
 
 # replace network objects (takes place after summarize)
+# [0-9-a-zA-Z\.\_\-]{1,253} -> host
 [L3V4_REPLACE]
 # "10.193.172.0/24" = "MPLS"
 # "10.194.8.0/23" = "MPLS"
@@ -127,29 +141,30 @@ SITES = [
 # must be sorted from slower to faster speed
 # use only one entry to have all conections with the same thickness
 # bits per second = thickness
-# 2000000 = 1  # 2 mbit
-# 5000000 = 2  # 5 mbit
-# 1e7 = 3      # 10 mbit
-# 51e7 = 4     # 51 mbit
-1e8 = 1  # 100 mbit
-1e9 = 3  # 1 gbit
-1e10 = 5 # 10 gbit
+# "2000000" = 1  # 2 mbit
+# "5000000" = 2  # 5 mbit
+# "1e7" = 3      # 10 mbit
+# "51e7" = 4     # 51 mbit
+"1e8" = 1  # 100 mbit
+"1e9" = 3  # 1 gbit
+"1e10" = 5 # 10 gbit
 
 [SETTINGS]
-# api_port = 80
+# api_port = 5001
 # backend = "MULTISITE" | "RESTAPI" | "LIVESTATUS"
 # case = "LOWER" | "UPPER"
 # default = false
 # dont_compare = false
 # filter_customers = "INCLUDE" |"EXCLUDE"
 # filter_sites = "INCLUDE" | "EXCLUDE"
+# include_l3_hosts = false
 # keep = 0
-# layers = ["LLDP", "CDP", "STATIC", "CUSTOM", "L3v4"]
+# layers = ["LLDP", "CDP", L3v4, "STATIC", "CUSTOM"]
 # log_file = "~/var/log/nvdct.log"
 # log_level = "WARNING"
 # log_to_stdout = false
 # min_age = 0
-# output_directory = ''
+# output_directory = ''  #
 # pre_fetch = false
 # prefix = ""
 # quiet = true

source/bin/nvdct/lib/args.py

@@ -23,6 +23,7 @@
 # --dont-compare
 # --filter-customers
 # --filter-sites
+# --include-l3-hosts
 # --keep
 # --log-file
 # --log-level
@@ -43,18 +44,19 @@ from argparse import (
 )
 from pathlib import Path
 
-from lib.utils import (
-    ExitCodes,
+from lib.constants import (
     HOME_URL,
     MIN_CDP_VERSION,
+    MIN_LINUX_IP_ADDRESSES,
+    MIN_SNMP_IP_ADDRESSES,
+    MIN_WINDOWS_IP_ADDRESSES,
     MIN_LLDP_VERSION,
-    MIN_IP_ADDRESSES,
     NVDCT_VERSION,
-    # SAMPLE_SEEDS,
     SCRIPT,
     TIME_FORMAT_ARGPARSER,
     USER_DATA_FILE,
 )
+from lib.utils import ExitCodes
 
 
 def parse_arguments() -> arg_Namespace:
@@ -83,7 +85,7 @@ def parse_arguments() -> arg_Namespace:
                f' {ExitCodes.BACKEND_NOT_IMPLEMENTED.value} - Backend not implemented\n'
                f' {ExitCodes.AUTOMATION_SECRET_NOT_FOUND.value} - Automation secret not found\n'
                '\nUsage:\n'
-               f'{SCRIPT} -u ~/local/bin/nvdct/conf/{USER_DATA_FILE} \n\n'
+               f'{SCRIPT} -u ~/local/bin/nvdct/conf/my_{USER_DATA_FILE} \n\n'
     )
 
     parser.add_argument(
@@ -122,12 +124,13 @@ def parse_arguments() -> arg_Namespace:
         choices=['CDP', 'CUSTOM', 'LLDP', 'STATIC', 'L3v4'],
         # default=['CDP'],
         help=(
-            f' - CDP  : needs inv_cdp_cache package at least in version {MIN_CDP_VERSION}\n'
-            f' - LLDP : needs inv_lldp_cache package at least in version {MIN_LLDP_VERSION}\n'
-            f' - L3v4 : needs inv_ip_address package at least in version {MIN_IP_ADDRESSES}\n'
-            f'          adds, layer 3 topology fpr IPv4\n'
-            f' - STATIC (deprecated)\n'
-            f' - CUSTOM (deprecated)\n'
+            f' - CDP   : needs inv_cdp_cache package at least in version {MIN_CDP_VERSION}\n'
+            f' - LLDP  : needs inv_lldp_cache package at least in version {MIN_LLDP_VERSION}\n'
+            f' - L3v4  : needs inv_ip_address package at least in version {MIN_SNMP_IP_ADDRESSES} for SNMP based hosts\n'
+            f'           for Linux based hosts inv_lnx_ip_if in version {MIN_LINUX_IP_ADDRESSES}\n'
+            f'           for Windows based hosts inv_win_ip_if in version {MIN_WINDOWS_IP_ADDRESSES}\n'
+            f' - STATIC: creates a topology base on the "STATIC_CONNECTIONS" in the toml file\n'
+            f' - CUSTOM: (deprecated)\n'
         )
     )
     parser.add_argument(
@@ -192,6 +195,10 @@ def parse_arguments() -> arg_Namespace:
         help='INCLUDE/EXCLUDE site list from config file.\n'
              'Note: MULTISITE backend only.',
     )
+    parser.add_argument(
+        '--include-l3-hosts', action='store_const', const=True,  # default=False,
+        help='Include hosts (single IP objects) in layer 3 topology',
+    )
     parser.add_argument(
         '--remove-domain', action='store_const', const=True,  # default=False,
         help='Remove the domain name from the neighbor name',

source/bin/nvdct/lib/backends.py

@@ -23,14 +23,17 @@ from sys import exit as sys_exit
 
 from livestatus import MultiSiteConnection, SiteConfigurations, SiteId
 
-from lib.utils import (
+from lib.constants import (
     CACHE_INTERFACES_DATA,
+    OMD_ROOT,
+    PATH_INTERFACES,
+)
+from lib.utils import (
     ExitCodes,
     get_data_form_live_status,
     get_table_from_inventory,
     LOGGER,
-    OMD_ROOT,
-    PATH_INTERFACES,
+
 )
 
 
@@ -410,8 +413,8 @@ class HostCacheMultiSite(HostCacheLiveStatus):
                 'local site only. Try -b RESTAPI if you have a distributed environment.'
             )
 
-    def filter_sites(self, filter: str| None, sites:List[str]):
-        match filter:
+    def filter_sites(self, filter_: str | None, sites: List[str]):
+        match filter_:
             case 'INCLUDE':
                 self.sites = {site: data for site, data in self.sites.items() if site in sites}
             case 'EXCLUDE':
@@ -419,8 +422,8 @@ class HostCacheMultiSite(HostCacheLiveStatus):
             case _:
                 return
 
-    def filter_costumers(self, filter: str | None, costumers:List[str]):
-        match filter:
+    def filter_costumers(self, filter_: str | None, costumers: List[str]):
+        match filter_:
             case 'INCLUDE':
                 self.sites = {
                     site: data for site, data in self.sites.items() if data.get('customer') in costumers
@@ -437,10 +440,17 @@ class HostCacheMultiSite(HostCacheLiveStatus):
 
 
 class HostCacheRestApi(HostCache):
-    def __init__(self, pre_fetch: bool, api_port: int):
+    def __init__(
+            self,
+            pre_fetch: bool,
+            api_port: int,
+            filter_sites: str | None = None,
+            sites: List[str] = [],
+    ):
         super().__init__(pre_fetch, '[RESTAPI]')
         LOGGER.debug(f'{self.backend} init backend')
         self._api_port = api_port
+        self.sites = []
         try:
             self.__secret = Path(
                 f'{OMD_ROOT}/var/check_mk/web/automation/automation.secret'
@@ -458,9 +468,33 @@ class HostCacheRestApi(HostCache):
         self.__session.headers['Authorization'] = f"Bearer {self.__user} {self.__secret}"
         self.__session.headers['Accept'] = 'application/json'
 
+        self.get_sites()
+        self.filter_sites(filter_=filter_sites, sites=sites)
+        LOGGER.info(f'{self.backend} filtered sites : {self.sites}')
+
         if self.pre_fetch:
             self.pre_fetch_hosts()
 
+    def get_sites(self):
+        LOGGER.debug(f'{self.backend} get_sites')
+        resp = self.__session.get(url=f"{self.__api_url}/domain-types/site_connection/collections/all")
+        if resp.status_code == 200:
+            sites = resp.json().get("value")
+            self.sites = [site.get('id') for site in sites]
+            LOGGER.debug(f'{self.backend} sites : {self.sites}')
+        else:
+            LOGGER.warning(f'{self.backend} got no site information! status code {resp.status_code}')
+            LOGGER.debug(f'{self.backend} response text: {resp.text}')
+
+    def filter_sites(self, filter_: str | None, sites: List[str]):
+        match filter_:
+            case 'INCLUDE':
+                self.sites = [site for site in self.sites if site in sites]
+            case 'EXCLUDE':
+                self.sites = [site for site in self.sites if site not in sites]
+            case _:
+                return
+
     def get_inventory_data(self, hosts: List[str]) -> Dict[str, Dict | None]:
         LOGGER.debug(f'{self.backend} get_inventory_data {hosts}')
         host_data: Dict[str, Dict | None] = {}
@@ -475,8 +509,9 @@ class HostCacheRestApi(HostCache):
             resp = self.__session.get(
                 url=f"{self.__api_url}/domain-types/host/collections/all",
                 params={
-                    "query": query,
-                    "columns": ['name', 'mk_inventory'],
+                    'query': query,
+                    'columns': ['name', 'mk_inventory'],
+                    'sites': self.sites,
                 },
             )
             if resp.status_code == 200:
@@ -487,7 +522,7 @@ class HostCacheRestApi(HostCache):
                     if host:
                         host_data[host] = raw_host['extensions'].get('mk_inventory')
             else:
-                LOGGER.info(
+                LOGGER.warning(
                     f'{self.backend} got no inventory data found!, status code {resp.status_code}'
                 )
                 LOGGER.debug(f'{self.backend} response query: {query}')
@@ -511,10 +546,11 @@ class HostCacheRestApi(HostCache):
             query = f'{{"op": "and", "expr": [{query_item},{query_host}]}}'
 
             resp = self.__session.get(
-                url=f"{self.__api_url}/domain-types/service/collections/all",
+                url=f'{self.__api_url}/domain-types/service/collections/all',
                 params={
-                    "query": query,
-                    "columns": ['host_name', 'description', 'long_plugin_output'],
+                    'query': query,
+                    'columns': ['host_name', 'description', 'long_plugin_output'],
+                    'sites': self.sites,
                 },
             )
 
@@ -556,10 +592,11 @@ class HostCacheRestApi(HostCache):
         #     return False
         query = '{"op": "=", "left": "name", "right": "' + host + '"}'
         resp = self.__session.get(
-                url=f"{self.__api_url}/domain-types/host/collections/all",
+                url=f'{self.__api_url}/domain-types/host/collections/all',
                 params={
-                    "query": query,
-                    "columns": ['name'],
+                    'query': query,
+                    'columns': ['name'],
+                    'sites': self.sites,
                 },
             )
         if resp.status_code == 200:
@@ -579,14 +616,14 @@ class HostCacheRestApi(HostCache):
 
     def get_hosts_by_label(self, label: str) -> List[str] | None:
         LOGGER.debug(f'{self.backend} get_hosts_by_label {label}')
-        # query = '{"op": "=", "left": "label_names", "right": "' + label + '"}'
         query = '{"op": "=", "left": "labels", "right": "' + label + '"}'
 
         resp = self.__session.get(
-            url=f"{self.__api_url}/domain-types/host/collections/all",
+            url=f'{self.__api_url}/domain-types/host/collections/all',
             params={
-                "query": query,
-                "columns": ['name', 'labels'],
+                'columns': ['name', 'labels'],
+                'query': query,
+                'sites': self.sites,
             },
         )
         if resp.status_code == 200:
@@ -608,11 +645,12 @@ class HostCacheRestApi(HostCache):
 
     def pre_fetch_hosts(self):
         LOGGER.debug(f'{self.backend} pre_fetch_hosts')
-
+        LOGGER.critical(f'{self.backend} pre_fetch_hosts sites {self.sites}')
         resp = self.__session.get(
-            url=f"{self.__api_url}/domain-types/host/collections/all",
+            url=f'{self.__api_url}/domain-types/host/collections/all',
             params={
-                "columns": ['name'],
+                'columns': ['name'],
+                'sites': self.sites,
             },
         )
         if resp.status_code == 200:

source/bin/nvdct/lib/constants.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# License: GNU General Public License v2
+# Author: thl-cmk[at]outlook[dot]com
+# URL   : https://thl-cmk.hopto.org
+# Date  : 2024-12-11
+# File  : nvdct/lib/constants.py
+
+
+from logging import  getLogger
+from os import environ
+from typing import Final
+
+NVDCT_VERSION: Final[str] = '0.9.4-20241210'
+#
+OMD_ROOT: Final[str] = environ["OMD_ROOT"]
+#
+API_PORT: Final[int] = 5001
+CACHE_INTERFACES_DATA: Final[str] = 'interface_data'
+CMK_SITE_CONF: Final[str] = f'{OMD_ROOT}/etc/omd/site.conf'
+COLUMNS_CDP: Final[str] = 'neighbour_name,local_port,neighbour_port'
+COLUMNS_L3v4: Final[str] = 'address,device,cidr,network,type'
+COLUMNS_LLDP: Final[str] = 'neighbour_name,local_port,neighbour_port'
+DATAPATH: Final[str] = f'{OMD_ROOT}/var/check_mk/topology/data'
+HOME_URL: Final[str] = 'https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/nvdct'
+HOST_LABEL_CDP: Final[str] = "'nvdct/has_cdp_neighbours' 'yes'"
+HOST_LABEL_L3V4_HOSTS: Final[str] = "'nvdct/l3v4_topology' 'host'"
+HOST_LABEL_L3V4_ROUTER: Final[str] = "'nvdct/l3v4_topology' 'router'"
+HOST_LABEL_LLDP: Final[str] = "'nvdct/has_lldp_neighbours' 'yes'"
+LABEL_CDP: Final[str] = 'CDP'
+LABEL_L3v4: Final[str] = 'LAYER3v4'
+LABEL_LLDP: Final[str] = 'LLDP'
+LOGGER: Final[str] = getLogger('root)')
+LOG_FILE: Final[str] = f'{OMD_ROOT}/var/log/nvdct.log'
+MIN_CDP_VERSION: Final[str] = '0.7.1-20240320'
+MIN_LINUX_IP_ADDRESSES: Final[str] = '0.0.4-20241210'
+MIN_SNMP_IP_ADDRESSES: Final[str] = '0.0.6-20241210'
+MIN_WINDOWS_IP_ADDRESSES: Final[str] = '0.0.3-20241210'
+MIN_LLDP_VERSION: Final[str] = '0.9.3-20240320'
+PATH_CDP: Final[str] = 'networking,cdp_cache,neighbours'
+PATH_INTERFACES: Final[str] = 'networking,interfaces'
+PATH_L3v4: Final[str] = 'networking,addresses'
+PATH_LLDP: Final[str] = 'networking,lldp_cache,neighbours'
+SCRIPT: Final[str] = '~/local/bin/nvdct/nvdct.py'
+TIME_FORMAT: Final[str] = '%Y-%m-%dT%H:%M:%S.%m'
+TIME_FORMAT_ARGPARSER: Final[str] = '%%Y-%%m-%%dT%%H:%%M:%%S.%%m'
+USER_DATA_FILE: Final[str] = 'nvdct.toml'

source/bin/nvdct/lib/settings.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 #
 # License: GNU General Public License v2
-
+from cmk_addons.plugins.bgp_topology.lib.utils import OMD_ROOT
 # Author: thl-cmk[at]outlook[dot]com
 # URL   : https://thl-cmk.hopto.org
 # Date  : 2023-10-12
@@ -13,20 +13,29 @@
 from collections.abc import Mapping
 from ipaddress import AddressValueError, IPv4Address, IPv4Network, NetmaskValueError
 from logging import CRITICAL, FATAL, ERROR, WARNING, INFO, DEBUG
-from os import environ
 from sys import exit as sys_exit
 from time import strftime
 from typing import Dict, List, NamedTuple, Tuple
+from pathlib import Path
 
+from lib.constants import (
+    API_PORT,
+    LOGGER,
+    LOG_FILE,
+    OMD_ROOT,
+    TIME_FORMAT,
+    USER_DATA_FILE,
+)
 from lib.utils import (
     ExitCodes,
+    Layer,
     get_data_from_toml,
     get_local_cmk_api_port,
-    # get_local_cmk_version,
-    LOGGER,
-    TIME_FORMAT,
-    USER_DATA_FILE,
-    Layer
+    is_valid_customer_name,
+    is_valid_hostname,
+    is_valid_log_file,
+    is_valid_output_directory,
+    is_valid_site_name,
 )
 
 
@@ -45,11 +54,10 @@ class Thickness(NamedTuple):
 
 
 class StaticConnection(NamedTuple):
-    host: str
-    local_port: str
-    neighbour_port: str
-    neighbour: str
-    label: str
+    right_host: str
+    right_service: str
+    left_service: str
+    left_host: str
 
 
 class Wildcard(NamedTuple):
@@ -65,11 +73,6 @@ class Settings:
             self,
             cli_args: Mapping[str, object],
     ):
-        self.__omd_root = environ['OMD_ROOT']
-        self.__path_to_if_table = 'networking,interfaces'
-        self.__topology_file_name = 'network_data.json'
-        # self.__topology_save_path = 'var/topology_data'
-        self.__topology_save_path_cmk_2_3 = 'var/check_mk/topology/data'
         # cli defaults
         self.__settings = {
             # 'api_port': 80,
@@ -80,10 +83,11 @@ class Settings:
             'dont_compare': False,
             'filter_customers': None,
             'filter_sites': None,
+            'include_l3_hosts': False,
             'keep': False,
             'remove_domain': False,
-            'layers': ['CDP'],
-            'log_file': f'{self.omd_root}/var/log/nvdct.log',
+            'layers': [],
+            'log_file': LOG_FILE,
             'log_level': 'WARNING',
             'log_to_stdout': False,
             'min_age': 0,
@@ -91,11 +95,10 @@ class Settings:
             'prefix': None,
             'quiet': False,
             'pre_fetch': False,
-            # 'seed_devices': [],
             'skip_l3_if': False,
             'skip_l3_ip': False,
             'time_format': TIME_FORMAT,
-            'user_data_file': f'{self.omd_root}/local/bin/nvdct/conf/{USER_DATA_FILE}',
+            'user_data_file': f'{OMD_ROOT}/local/bin/nvdct/conf/{USER_DATA_FILE}',
         }
         # args in the form {'s, __seed_devices': 'CORE01', 'p, __path_in_inventory': None, ... }}
         # we will remove 's, __'
@@ -128,7 +131,6 @@ class Settings:
         self.__api_port: int | None = None
 
         # init user data with defaults
-        # self.__drop_host_regex: List[str] | None = None
         self.__custom_layers: List[StaticConnection] | None = None
         self.__customers: List[str] | None = None
         self.__emblems: Emblems | None = None
@@ -157,7 +159,7 @@ class Settings:
             else:
                 self.__api_port = get_local_cmk_api_port()
             if self.__api_port is None:
-                self.__api_port = 80
+                self.__api_port = API_PORT
 
         return self.__api_port
 
@@ -217,6 +219,10 @@ class Settings:
             )
         return None
 
+    @property  # --include-l3-hosts
+    def include_l3_hosts(self) -> bool:
+        return bool(self.__settings['include_l3_hosts'])
+
     @property  # --keep
     def keep(self) -> int | None:
         if isinstance(self.__settings['keep'], int):
@@ -233,27 +239,25 @@ class Settings:
 
     @property  # --log-file
     def log_file(self) -> str:
-        return str(self.__settings['log_file'])
+        raw_log_file = str(Path(str(self.__settings['log_file'])).expanduser())
+        if is_valid_log_file(raw_log_file):
+            return raw_log_file
+        else:
+            LOGGER.error(f'Falling back to {LOG_FILE}')
+            return LOG_FILE
 
     @property  # --log-level
     def loglevel(self) -> int:
-        match self.__settings['log_level']:
-            case 'DEBUG':
-                return DEBUG
-            case 'INFO':
-                return INFO
-            case 'WARNING':
-                return WARNING
-            case 'ERROR':
-                return ERROR
-            case 'FATAL':
-                return FATAL
-            case 'CRITICAL':
-                return CRITICAL
-            case 'OFF':
-                return -1
-            case _:
-                return WARNING
+        log_levels = {
+            'DEBUG': DEBUG,
+            'INFO': INFO,
+            'WARNING': WARNING,
+            'ERROR': ERROR,
+            'FATAL': FATAL,
+            'CRITICAL': CRITICAL,
+            'OFF': -1,
+        }
+        return log_levels.get(self.__settings['log_level'], WARNING)
 
     @property  # --log-to-stdout
     def log_to_stdtout(self) -> bool:
@@ -271,7 +275,11 @@ class Settings:
         # init output directory with current time if not set
         if not self.__settings['output_directory']:
             self.__settings['output_directory'] = f'{strftime(self.__settings["time_format"])}'
-        return str(self.__settings['output_directory'])
+        if is_valid_output_directory(str(self.__settings['output_directory'])):
+            return str(self.__settings['output_directory'])
+        else:
+            LOGGER.error('Falling back to "nvdct"')
+            return 'nvdct'
 
     @property  # --prefix
     def prefix(self) -> str | None:
@@ -311,7 +319,7 @@ class Settings:
         if self.__customers is None:
             self.__customers = [
                 str(customer) for customer in set(self.__user_data.get('CUSTOMERS', []))
-            ]
+                if is_valid_customer_name(customer)]
             LOGGER.info(f'Found {len(self.__customers)} to filter on')
         return self.__customers
 
@@ -362,9 +370,7 @@ class Settings:
     def l2_seed_devices(self) -> List[str]:
         if self.__l2_seed_devices is None:
             self.__l2_seed_devices = list(set(str(host) for host in (
-                # self.__user_data.get('L2_SEED_DEVICES', []) + self.__settings.get('seed_devices', [])
-                self.__user_data.get('L2_SEED_DEVICES', [])
-            )))
+                self.__user_data.get('L2_SEED_DEVICES', [])) if is_valid_hostname(host)))
         return self.__l2_seed_devices
 
     @property
@@ -373,7 +379,7 @@ class Settings:
             self.__l2_host_map = {
                 str(host): str(replace_host) for host, replace_host in self.__user_data.get(
                     'L2_HOST_MAP', {}
-                ).items()
+                ).items() if is_valid_hostname(host)
             }
         return self.__l2_host_map
 
@@ -392,7 +398,7 @@ class Settings:
         if self.__l3v4_ignore_hosts is None:
             self.__l3v4_ignore_hosts = [str(host) for host in set(self.__user_data.get(
                 'L3V4_IGNORE_HOSTS', []
-            ))]
+            )) if is_valid_hostname(host)]
         return self.__l3v4_ignore_hosts
 
     @property
@@ -460,9 +466,12 @@ class Settings:
                     _ip_network = IPv4Network(ip_network)  # noqa: F841
                 except (AddressValueError, NetmaskValueError):
                     LOGGER.error(
-                        f'Invalid entry in L3V4_REPLACE found: {ip_network} -> ignored'
+                        f'Invalid entry in L3V4_REPLACE found: {ip_network} -> line ignored'
                     )
                     continue
+                if not is_valid_hostname(node):
+                    LOGGER.error(f'Invalid node name found: {node} -> line ignored ')
+                    continue
                 self.__l3v4_replace[ip_network] = str(node)
             LOGGER.info(
                 f'Valid entries in L3V4_REPLACE found: {len(self.__l3v4_replace)}/'
@@ -526,18 +535,22 @@ class Settings:
             self.__static_connections = []
             for connection in self.__user_data.get('STATIC_CONNECTIONS', []):
                 try:
-                    host, local_port, neighbour_port, neighbour, label = connection
+                    left_host, left_service, right_service, right_host = connection
                 except ValueError:
                     LOGGER.error(
-                        f'Worng entry in STATIC_CONNECTIONS -> {connection} -> ignored'
+                        f'Wrong entry in STATIC_CONNECTIONS -> {connection} -> ignored'
                     )
                     continue
+                if not right_host or not left_host:
+                    LOGGER.warning(f'Both hosts must be set, got {connection}')
+                    continue
+                if not is_valid_hostname(right_host) or not is_valid_hostname(left_host):
+                    continue
                 self.__static_connections.append(StaticConnection(
-                    host=str(host),
-                    local_port=str(local_port),
-                    neighbour_port=str(neighbour_port),
-                    neighbour=str(neighbour),
-                    label=str(label),
+                    right_host=str(right_host),
+                    right_service=str(right_service),
+                    left_service=str(left_service),
+                    left_host=str(left_host),
                 ))
             LOGGER.info(
                 f'Valid entries in STATIC_CONNECTIONS found: {len(self.__static_connections)}/'
@@ -548,21 +561,6 @@ class Settings:
     @property
     def sites(self) -> List[str]:
         if self.__sites is None:
-            self.__sites = [str(site) for site in set(self.__user_data.get('SITES', []))]
-            LOGGER.info(f'fFound {len(self.__sites)} to filter on')
+            self.__sites = [str(site) for site in set(self.__user_data.get('SITES', [])) if is_valid_site_name(site)]
+            LOGGER.info(f'Found {len(self.__sites)} to filter on')
         return self.__sites
-
-    #
-    # all other settings
-    #
-    @property
-    def omd_root(self) -> str:
-        return self.__omd_root
-
-    @property
-    def topology_save_path(self) -> str:
-        return self.__topology_save_path_cmk_2_3
-
-    @property
-    def topology_file_name(self) -> str:
-        return self.__topology_file_name

source/bin/nvdct/lib/utils.py

@@ -14,7 +14,6 @@ from enum import Enum, unique
 from json import dumps
 from logging import disable as log_off, Formatter, getLogger, StreamHandler
 from logging.handlers import RotatingFileHandler
-from os import environ
 from pathlib import Path
 from re import match as re_match
 from socket import socket, AF_UNIX, AF_INET, SOCK_STREAM, SHUT_WR
@@ -23,8 +22,23 @@ from time import time as now_time
 from tomllib import loads as toml_loads, TOMLDecodeError
 from typing import List, Dict, TextIO
 
-NVDCT_VERSION = '0.9.3-20241209'
-
+from lib.constants import (
+    CMK_SITE_CONF,
+    COLUMNS_CDP,
+    COLUMNS_LLDP,
+    HOST_LABEL_CDP,
+    HOST_LABEL_L3V4_ROUTER,
+    HOST_LABEL_LLDP,
+    LABEL_CDP,
+    LABEL_L3v4,
+    LABEL_LLDP,
+    LOGGER,
+    OMD_ROOT,
+    PATH_CDP,
+    PATH_L3v4,
+    PATH_LLDP,
+    DATAPATH,
+)
 
 @unique
 class ExitCodes(Enum):
@@ -33,15 +47,7 @@ class ExitCodes(Enum):
     BAD_TOML_FORMAT = 2
     BACKEND_NOT_IMPLEMENTED = 3
     AUTOMATION_SECRET_NOT_FOUND = 4
-
-
-@dataclass(frozen=True)
-class Layer:
-    path: str
-    columns: str
-    label: str
-    host_label: str
-
+    NO_LAYER_CONFIGURED = 5
 
 @dataclass(frozen=True)
 class Ipv4Info:
@@ -53,44 +59,18 @@ class Ipv4Info:
     network: str
     type: str
 
-
 @dataclass(frozen=True)
 class InventoryColumns:
     neighbour: str
     local_port: str
     neighbour_port: str
 
-
-# constants
-OMD_ROOT = environ["OMD_ROOT"]
-
-CACHE_INTERFACES_DATA = 'interface_data'
-CMK_SITE_CONF = f'{OMD_ROOT}/etc/omd/site.conf'
-COLUMNS_CDP = 'neighbour_name,local_port,neighbour_port'
-COLUMNS_L3v4 = 'address,device,cidr,network,type'
-COLUMNS_LLDP = 'neighbour_name,local_port,neighbour_port'
-HOME_URL = 'https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/nvdct'
-HOST_LABEL_CDP = "'nvdct/has_cdp_neighbours' 'yes'"
-HOST_LABEL_L3V4 = "'nvdct/l3v4_topology' 'router'"
-HOST_LABEL_LLDP = "'nvdct/has_lldp_neighbours' 'yes'"
-LABEL_CDP = 'CDP'
-LABEL_L3v4 = 'LAYER3v4'
-LABEL_LLDP = 'LLDP'
-LOG_FILE = f'{OMD_ROOT}/var/log/nvdct.log'
-LOGGER = getLogger('root)')
-MIN_CDP_VERSION = '0.7.1-20240320'
-MIN_IP_ADDRESSES = '0.0.5-2024120'
-MIN_LLDP_VERSION = '0.9.3-20240320'
-PATH_CDP = 'networking,cdp_cache,neighbours'
-PATH_INTERFACES = 'networking,interfaces'
-PATH_L3v4 = 'networking,addresses'
-PATH_LLDP = 'networking,lldp_cache,neighbours'
-SAMPLE_SEEDS = 'Core01 Core02'
-SCRIPT = '~/local/bin/nvdct/nvdct.py'
-TIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%m'
-TIME_FORMAT_ARGPARSER = '%%Y-%%m-%%dT%%H:%%M:%%S.%%m'
-USER_DATA_FILE = 'nvdct.toml'
-
+@dataclass(frozen=True)
+class Layer:
+    path: str
+    columns: str
+    label: str
+    host_label: str
 
 LAYERS = {
     'CDP': Layer(
@@ -109,7 +89,7 @@ LAYERS = {
         path=PATH_L3v4,
         columns='',
         label=LABEL_L3v4,
-        host_label=HOST_LABEL_L3V4,
+        host_label=HOST_LABEL_L3V4_ROUTER,
     ),
 }
 
@@ -164,9 +144,7 @@ def get_data_from_toml(file: str) -> Dict:
 
 def rm_tree(root: Path) -> None:
     # safety
-    if not str(root).startswith(
-            f'{OMD_ROOT}/var/topology_data'
-    ) and not str(root).startswith(f'{OMD_ROOT}/var/check_mk/topology'):
+    if not str(root).startswith(DATAPATH):
         LOGGER.warning(msg=f"WARNING: bad path to remove, {str(root)}, don\'t delete it.")
         return
     for p in root.iterdir():
@@ -323,6 +301,47 @@ def is_list_of_str_equal(list1: List[str], list2: List[str]) -> bool:
     return tmp_list1 == tmp_list2
 
 
+def is_valid_hostname(host: str) -> bool:
+    re_host_pattern = r'^[0-9a-z-A-Z\.\-\_]{1,253}$'
+    if re_match(re_host_pattern, host):
+        return True
+    else:
+        LOGGER.error(f'Invalid hostname found: {host}')
+        return False
+
+def is_valid_site_name(site: str) -> bool:
+    re_host_pattern = r'^[0-9a-z-A-Z\.\-\_]{1,16}$'
+    if re_match(re_host_pattern, site):
+        return True
+    else:
+        LOGGER.error(f'Invalid site name found: {site}')
+        return False
+
+def is_valid_customer_name(customer: str) -> bool:
+    re_host_pattern = r'^[0-9a-z-A-Z\.\-\_]{1,16}$'
+    if re_match(re_host_pattern, customer):
+        return True
+    else:
+        LOGGER.error(f'Invalid customer name found: {customer}')
+        return False
+
+
+def is_valid_output_directory(directory: str) -> bool:
+    # 2024-12-11T17:35:08.12
+    re_host_pattern = r'^[0-9a-z-A-Z\.\-\_\:]{1,30}$'
+    if re_match(re_host_pattern, directory):
+        return True
+    else:
+        LOGGER.error(f'Invalid output directory name found: {directory}')
+        return False
+
+def is_valid_log_file(log_file: str) -> bool:
+    if not log_file.startswith(f'{OMD_ROOT}/var/log/'):
+        LOGGER.error(f'Logg file needs to be under "{OMD_ROOT}/var/log/"! Got {Path(log_file).absolute()}')
+        return False
+    return True
+
+
 # not used in cmk 2.3.x format
 def merge_topologies(topo_pri: Dict, topo_sec: Dict) -> Dict:
     """
@@ -410,8 +429,7 @@ def configure_logger(log_file: str, log_level: int, log_to_console: bool) -> Non
 
     log = getLogger()
     log_formatter = Formatter(
-        fmt='%(asctime)s :: %(levelname)s :: %(module)s :'
-            ': %(funcName)s() :: %(lineno)s :: %(message)s',
+        fmt='%(asctime)s :: %(levelname)s :: %(module)s :: %(funcName)s() :: %(lineno)s :: %(message)s',
     )
 
     log.setLevel(log_level)

source/packages/nvdct

@@ -17,8 +17,7 @@
                 'environments\n'
                 ' - Add custom connections (STATIC) for connections that are '
                 'not in the inventory\n'
-                ' - Optimized for my CDP, LLDP and IPv4 inventory plugins\n'
-                ' - Can also be used with custom inventory plugins\n'
+                ' - Optimized for my CDP, LLDP and IP inventory plugins\n'
                 '\n'
                 'For more information about the network visualization plugin '
                 'see: \n'
@@ -28,14 +27,7 @@
                 '\n'
                 'The inventory data could be created with my inventory '
                 'plugins:\n'
-                'CDP: '
-                'https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_cdp_cache\n'
-                'LLDP: '
-                'https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_lldp_cache\n'
-                'L3v4: '
-                'https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_ipv4_addresses\n'
-                'IF_name: '
-                'https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/inventory/inv_ifname\n'
+                'https://thl-cmk.hopto.org/gitlab/explore/projects/topics/Network%20Visualization\n'
                 '\n'
                 'For the latest version and documentation see:\n'
                 'https://thl-cmk.hopto.org/gitlab/checkmk/vendor-independent/nvdct\n',
@@ -47,14 +39,15 @@
                    'nvdct/lib/utils.py',
                    'nvdct/lib/__init__.py',
                    'nvdct/conf/nvdct.toml',
-                   'nvdct/lib/topologies.py'],
+                   'nvdct/lib/topologies.py',
+                   'nvdct/lib/constants.py'],
            'web': ['htdocs/images/icons/cloud_80.png',
                    'htdocs/images/icons/ip-address_80.png',
                    'htdocs/images/icons/ip-network_80.png',
                    'htdocs/images/icons/location_80.png']},
  'name': 'nvdct',
  'title': 'Network Visualization Data Creation Tool (NVDCT)',
- 'version': '0.9.3-20241209',
+ 'version': '0.9.4-20241210',
  'version.min_required': '2.3.0b1',
  'version.packaged': 'cmk-mkp-tool 0.2.0',
  'version.usable_until': '2.4.0p1'}