diff --git a/README.md b/README.md
index bbc24b316faf1e1fd64ad3291106fbc55532e7ba..e77139d4f50cc9fa69570d288208f436d7e8c1aa 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-[PACKAGE]: ../../raw/master/mkp/nvdct-0.8.12-20240702.mkp "nvdct-0.8.12-20240702.mkp"
+[PACKAGE]: ../../raw/master/mkp/nvdct-0.9.0-20240923.mkp "nvdct-0.9.0-20240923.mkp"
# Network Visualization Data Creation Tool (NVDCT)
This script creates the topology data file needed for the [Checkmk Exchange Network visualization](https://exchange.checkmk.com/p/network-visualization) plugin.\
diff --git a/mkp/nvdct-0.9.0-20240923.mkp b/mkp/nvdct-0.9.0-20240923.mkp
new file mode 100644
index 0000000000000000000000000000000000000000..e7925df7a841b83c72474e9ec4a66687335ad8bf
Binary files /dev/null and b/mkp/nvdct-0.9.0-20240923.mkp differ
diff --git a/source/bin/nvdct/conf/nvdct.toml b/source/bin/nvdct/conf/nvdct.toml
index ac4235ed8f563cd796131e056dec0a6887a23fad..48d787bc9502e2280b76208eab3e784f90a20f8e 100755
--- a/source/bin/nvdct/conf/nvdct.toml
+++ b/source/bin/nvdct/conf/nvdct.toml
@@ -84,6 +84,20 @@ CUSTOM_LAYERS = [
# { path = "networking,cdp_cache,neighbours", columns = "neighbour_name,local_port,neighbour_port", label = "custom_CDP", host_label = "nvdct/has_cdp_neighbours" },
]
+# list site so include/excluse, use option --filter-sites INCLUDE/EXCLUDE
+SITES = [
+ # "site1",
+ # "site2",
+ # "site3",
+]
+
+# list customers so include/excluse, use option --filter-costumers INCLUDE/EXCLUDE
+CUSTOMERS = [
+ # "customer1",
+ # "customer2",
+ # "customer3",
+]
+
[MAP_SPEED_TO_THICKNESS]
# must be sorted from slower to faster speed
# use only one entry to have all conections with the same thickness
@@ -122,23 +136,23 @@ CUSTOM_LAYERS = [
[SETTINGS]
# api_port = 80
-# backend = "MULTISITE"
+# backend = "MULTISITE" | "RESTAPI" | "LIVESTATUS"
+# case = "LOWER" | "UPPER"
# default = false
# dont_compare = false
+# filter_customers = "INCLUDE" |"EXCLUDE"
+# filter_sites = "INCLUDE" | "EXCLUDE"
# keep = 0
# keep_domain = false
# layers = ["LLDP", "CDP", "STATIC", "CUSTOM", "L3v4"]
# log_file = "~/var/log/nvdct.log"
# log_level = "WARNING"
# log_to_stdout = false
-# lowercase = false
# min_age = 0
-# new_format = false
# output_directory = ''
-# prefix = ""
# pre_fetch = false
+# prefix = ""
# quiet = true
# skip_l3_if = false
# skip_l3_ip = false
# time_format = "%Y-%m-%dT%H:%M:%S.%m"
-# uppercase = false
diff --git a/source/bin/nvdct/lib/args.py b/source/bin/nvdct/lib/args.py
index 9376073dacd175c9c4343b4aa075046b058256bf..080a0758e7f9aacfb4e3f50112f6cb852d996660 100755
--- a/source/bin/nvdct/lib/args.py
+++ b/source/bin/nvdct/lib/args.py
@@ -18,22 +18,22 @@
# -u --user-data-file
# -v --version
# --api-port (deprecated ?)
+# --case
# --check-user-data-only
# --dont-compare
+# --filter-customers
+# --filter-sites
# --keep
# --keep-domain
# --log-file
# --log-level
# --log-to-stdout
-# --lowercase
# --min-age
-# --new-format (deprecated)
# --pre-fetch
# --quiet
# --skip-l3-if
# --skip-l3-ip
# --time-format
-# --uppercase
from argparse import (
@@ -41,6 +41,8 @@ from argparse import (
Namespace as arg_Namespace,
RawTextHelpFormatter,
)
+from pathlib import Path
+
from lib.utils import (
ExitCodes,
HOME_URL,
@@ -83,15 +85,12 @@ def parse_arguments() -> arg_Namespace:
'\nUsage:\n'
f'{SCRIPT} -s {SAMPLE_SEEDS} -d\n\n'
)
- command_group = parser.add_mutually_exclusive_group()
parser.add_argument(
'-b', '--backend',
- # nargs='+',
- choices=['FILESYSTEM', 'LIVESTATUS', 'MULTISITE', 'RESTAPI'],
+ choices=['LIVESTATUS', 'MULTISITE', 'RESTAPI'],
# default='MULTISITE',
help='Backend used to retrieve the topology data\n'
- ' - FILESYSTEM : fetches the data directly form the inventory files (deprecatred)\n'
' - LIVESTATUS : fetches data via local Livestatus (local site only)\n'
' - MULTISITE : like LIVESTATUS but for distributed environments (default)\n'
' - RESTAPI : uses the CMK REST API.',
@@ -101,14 +100,12 @@ def parse_arguments() -> arg_Namespace:
help='Set the created topology data as default. Will be created automatically\n'
'if it doesn\'t exists.',
)
-
parser.add_argument(
'-o', '--output-directory', type=str,
help='Directory name where to save the topology data.\n'
'I.e.: my_topology. Default is the actual date/time\n'
'in "--time-format" format.\n'
- 'NOTE: the directory is a sub directory under "~/var/topology_data/" (CMK2.2.0)\n'
- 'For CMK 2.3.0 the path is under "~/var/check_mk/topology/data/".',
+ 'NOTE: the directory is a sub directory under "~/var/check_mk/topology/data/"\n',
)
parser.add_argument(
'-s', '--seed-devices', type=str, nargs='+',
@@ -125,12 +122,10 @@ def parse_arguments() -> arg_Namespace:
choices=['CDP', 'CUSTOM', 'LLDP', 'STATIC', 'L3v4'],
# default=['CDP'],
help=(
- f'Layers with least significant layer first. Listed layers\n'
- f'will be merged automatically (CMK2.2 only).\n'
f' - CDP : needs inv_cdp_cache package at least in version {MIN_CDP_VERSION}\n'
f' - LLDP : needs inv_lldp_cache package at least in version {MIN_LLDP_VERSION}\n'
f' - L3v4 : needs inv_ipv4_addresses package at least in version {MIN_IPV4_ADDRESSES}\n'
- f' adds, layer 3 topology fpr IPv4 (CMK 2.3.0 only)'
+ f' adds, layer 3 topology fpr IPv4 '
)
)
parser.add_argument(
@@ -139,7 +134,8 @@ def parse_arguments() -> arg_Namespace:
f'Default is ~/local/bin/nvdct/conf/conf/{USER_DATA_FILE}\n',
)
parser.add_argument(
- '-v', '--version', action='store_const', const=True, # default=False,
+ '-v', '--version', action='version',
+ version=f'{Path(SCRIPT).name} version: {NVDCT_VERSION}',
help='Print version of this script and exit',
)
parser.add_argument(
@@ -147,6 +143,12 @@ def parse_arguments() -> arg_Namespace:
help='TCP Port to access the REST API. Default is 80. NVDCT will try to automatically\n'
'detect the site apache port.',
)
+ parser.add_argument(
+ '--case',
+ choices=['LOWER', 'UPPER'],
+ # default='NONE',
+ help='Change neighbour name to all lower/upper case',
+ )
parser.add_argument(
'--check-user-data-only', action='store_const', const=True, # default=False,
help=f'Only tries to read/parse the user data from {USER_DATA_FILE} and exits.',
@@ -174,6 +176,20 @@ def parse_arguments() -> arg_Namespace:
'So, if you run this tool in a cron job, a new topology will be\n'
'created only if there was a change, unless you use "--dont-compare".'
)
+ parser.add_argument(
+ '--filter-customers',
+ choices=['INCLUDE', 'EXCLUDE'],
+ # default='INCLUDE',
+ help='INCLUDE/EXCLUDE customer list from config file.\n'
+ 'Note: MULTISITE backend only.',
+ )
+ parser.add_argument(
+ '--filter-sites',
+ choices=['INCLUDE', 'EXCLUDE'],
+ # default='INCLUDE',
+ help='INCLUDE/EXCLUDE site list from config file.\n'
+ 'Note: MULTISITE backend only.',
+ )
parser.add_argument(
'--keep-domain', action='store_const', const=True, # default=False,
help='Do not remove the domain name from the neighbor name',
@@ -184,19 +200,10 @@ def parse_arguments() -> arg_Namespace:
'max will be deleted.\n'
'NOTE: The default topologies will be always kept.\n'
)
- command_group.add_argument(
- '--lowercase', action='store_const', const=True, # default=False,
- help='Change neighbour names to all lower case',
- )
parser.add_argument(
'--min-age', type=int,
help='The minimum number of days before a topology is deleted by "--keep".'
)
- parser.add_argument(
- '--new-format', action='store_const', const=True, # default=False,
- help='Save data in new format. Use for CMK 2.3.x\n'
- 'NVDCT will try to automatically detect the correct format. (deprecated)',
- )
parser.add_argument(
'--pre-fetch', action='store_const', const=True, # default=False,
help='Try to fetch host data, with less API calls. Can improve RESTAPI backend\n'
@@ -218,8 +225,5 @@ def parse_arguments() -> arg_Namespace:
'--time-format', type=str,
help=f'Format string to render the time. (default: {TIME_FORMAT_ARGPARSER})',
)
- command_group.add_argument(
- '--uppercase', action='store_const', const=True, # default=False,
- help='Change neighbour names to all upper case',
- )
+
return parser.parse_args()
diff --git a/source/bin/nvdct/lib/backends.py b/source/bin/nvdct/lib/backends.py
index bfccbb2545eedbface265a5d958bd4ab75d4dcc0..6d003fd855470c40d713d9e2b6ecc50c51b90246 100755
--- a/source/bin/nvdct/lib/backends.py
+++ b/source/bin/nvdct/lib/backends.py
@@ -10,7 +10,7 @@
# 2024-06-18: fixed host_exist returns always True if host was in host_cache, even with host=None
-from _collections_abc import Mapping, Sequence
+from collections.abc import Mapping, Sequence
from abc import abstractmethod
from ast import literal_eval
from enum import Enum, unique
@@ -334,10 +334,19 @@ class HostCacheLiveStatus(HostCache):
class HostCacheMultiSite(HostCacheLiveStatus):
- def __init__(self, pre_fetch: bool):
+ def __init__(
+ self,
+ pre_fetch: bool,
+ filter_sites: str | None = None,
+ sites: List[str] = [],
+ filter_customers: str | None = None,
+ customers: List[str] = None,
+ ):
self._backend = '[MULTISITE]'
self.sites: SiteConfigurations = SiteConfigurations({})
self.get_sites()
+ self.filter_sites(filter_sites, sites)
+ self.filter_costumers(filter_customers, customers)
LOGGER.info(f'{self.backend} Create livestatus connection(s)')
self.c = MultiSiteConnection(self.sites)
# self.c.set_prepend_site(False) # is default
@@ -351,9 +360,8 @@ class HostCacheMultiSite(HostCacheLiveStatus):
if self.pre_fetch:
self.pre_fetch_hosts()
+ # https://github.com/Checkmk/checkmk/blob/master/packages/cmk-livestatus-client/example_multisite.py
def get_sites(self):
- # see: https://github.com/Checkmk/checkmk/blob/master/packages/cmk-livestatus-client/example_multisite.py
-
sites_mk = Path(f'{OMD_ROOT}/etc/check_mk/multisite.d/sites.mk')
socket_path = f'unix:{OMD_ROOT}/tmp/run/live'
if sites_mk.exists():
@@ -372,6 +380,7 @@ class HostCacheMultiSite(HostCacheLiveStatus):
self.sites.update({site: {
'alias': data['alias'],
'timeout': data['timeout'],
+ 'customer': data['customer'] # needed to filter by customer
# 'nagios_url': '/nagios/',
}})
if data['socket'] == ('local', None):
@@ -395,10 +404,32 @@ class HostCacheMultiSite(HostCacheLiveStatus):
}})
LOGGER.critical(
- f'{self.backend} file {str(sites_mk.absolute())} not found. Fallback to'
+ f'{self.backend} file {str(sites_mk.absolute())} not found. Fallback to '
'local site only. Try -b RESTAPI if you have a distributed environment.'
)
+ def filter_sites(self, filter: str| None, sites:List[str]):
+ match filter:
+ case 'INCLUDE':
+ self.sites = {site: data for site, data in self.sites.items() if site in sites}
+ case 'EXCLUDE':
+ self.sites = {site: data for site, data in self.sites.items() if site not in sites}
+ case _:
+ return
+
+ def filter_costumers(self, filter: str | None, costumers:List[str]):
+ match filter:
+ case 'INCLUDE':
+ self.sites = {
+ site: data for site, data in self.sites.items() if data.get('customer') in costumers
+ }
+ case 'EXCLUDE':
+ self.sites = {
+ site: data for site, data in self.sites.items() if data.get('customer') not in costumers
+ }
+ case _:
+ return
+
def get_raw_data(self, query: str) -> object:
return self.c.query(query=query)
@@ -589,89 +620,3 @@ class HostCacheRestApi(HostCache):
LOGGER.debug(f'{self.backend} # of host found: {len(self.cache.keys())}')
else:
LOGGER.warning(f'{self.backend} respons: {resp.text}')
-
-
-class HostCacheFileSystem(HostCache):
- def __init__(self, pre_fetch: bool):
- super().__init__(pre_fetch, '[FILESYSTEM]')
-
- def get_inventory_data(self, hosts: Sequence[str]) -> Dict[str, Dict | None]:
- host_data: Dict[str, Dict | None] = {}
- __inventory_path = 'var/check_mk/inventory'
- for host in hosts:
- # init host_data with None
- host_data[host] = None
- inventory_file: Path = Path(f'{OMD_ROOT}/{__inventory_path}/{host}')
- try:
- data = literal_eval(inventory_file.read_text())
- except FileNotFoundError:
- LOGGER.warning(
- msg=f'{self.backend} Device: {host}: not found in inventory data path!'
- )
- continue
-
- LOGGER.debug(f'{self.backend} data for host {host}: {data}')
- host_data[host] = data
-
- return host_data
-
- def get_interface_data(self, hosts: Sequence[str]) -> Dict[str, Dict | None]:
- """
- Sample autochecks data, we keep only the item
- [
- {'check_plugin_name': 'if64', 'item': 'Fa0', 'parameters': {'discovered_oper_status': ['2'], ...}},\n # noqa: E501
- {'check_plugin_name': 'if64', 'item': 'Fa1/0/1', 'parameters': {'discovered_oper_status': ['1'], ...}},\n # noqa: E501
- {'check_plugin_name': 'if64', 'item': 'Fa1/0/10', 'parameters': {'discovered_oper_status': ['2'], ...}},\n # noqa: E501
- {'check_plugin_name': 'if64', 'item': 'Fa1/0/11', 'parameters': {'discovered_oper_status': ['2'], ...}},\n # noqa: E501
- {'check_plugin_name': 'if64', 'item': 'Fa1/0/12', 'parameters': {'discovered_oper_status': ['1'], ...}}\n # noqa: E501
- ]
-
- Args:
- hosts: list of names of the host objects in cmk to fetch the data for
-
- Returns:
- List of interface service items
-
- """
- host_data: Dict[str, Dict | None] = {}
- __autochecks_path = 'var/check_mk/autochecks'
- for host in hosts:
- # init host_data with None
- host_data[host] = None
- autochecks_file: Path = Path(f'{OMD_ROOT}/{__autochecks_path}/{host}.mk')
- items: Dict[str, object] = {}
- try:
- data: Sequence[Mapping[str, str]] = literal_eval(autochecks_file.read_text())
- except FileNotFoundError:
- LOGGER.warning(
- msg=f'{self.backend} Device: {host}: not found in auto checks path!'
- )
- continue
- LOGGER.debug(f'{self.backend} data for host {host}: {data}')
-
- for service in data:
- if service['check_plugin_name'] in ['if64']:
- items[service['item']] = {}
-
- if items:
- LOGGER.info(
- msg=f'{self.backend} Interfaces items found: {len(items)} an host {host}'
- )
- host_data[host] = items
- else:
- LOGGER.warning(
- msg=f'{self.backend} No Interfaces items found for host {host}'
- )
- return host_data
-
- def host_exists(self, host: str) -> bool:
- # always True, don't works in distributed environments as autocheck files
- # are only locally available
- return False
-
- def get_hosts_by_label(self, label: str) -> List[str] | None:
- # not implemented, no (good) way to get a list of hosts from file system
- return None
-
- def pre_fetch_hosts(self):
- pass
diff --git a/source/bin/nvdct/lib/settings.py b/source/bin/nvdct/lib/settings.py
index 610abdc61e0a40aea07ba4e51053713aa5e3764b..8a55f5553528b266fbe2e180d82da55d50905afe 100755
--- a/source/bin/nvdct/lib/settings.py
+++ b/source/bin/nvdct/lib/settings.py
@@ -10,11 +10,10 @@
# fixed path to default user data file
-from _collections_abc import Mapping
+from collections.abc import Mapping
from ipaddress import AddressValueError, IPv4Address, IPv4Network, NetmaskValueError
from logging import CRITICAL, FATAL, ERROR, WARNING, INFO, DEBUG
from os import environ
-from pathlib import Path
from sys import exit as sys_exit
from time import strftime
from typing import Dict, List, NamedTuple
@@ -23,10 +22,8 @@ from lib.utils import (
ExitCodes,
get_data_from_toml,
get_local_cmk_api_port,
- get_local_cmk_version,
+ # get_local_cmk_version,
LOGGER,
- NVDCT_VERSION,
- SCRIPT,
TIME_FORMAT,
USER_DATA_FILE,
Layer
@@ -71,24 +68,25 @@ class Settings:
self.__omd_root = environ['OMD_ROOT']
self.__path_to_if_table = 'networking,interfaces'
self.__topology_file_name = 'network_data.json'
- self.__topology_save_path = 'var/topology_data'
+ # self.__topology_save_path = 'var/topology_data'
self.__topology_save_path_cmk_2_3 = 'var/check_mk/topology/data'
# cli defaults
self.__settings = {
# 'api_port': 80,
'backend': 'MULTISITE',
+ 'case': None,
'check_user_data_only': False,
'default': False,
'dont_compare': False,
+ 'filter_customers': None,
+ 'filter_sites': None,
'keep': False,
'keep_domain': False,
'layers': ['CDP'],
'log_file': f'{self.omd_root}/var/log/nvdct.log',
'log_level': 'WARNING',
'log_to_stdout': False,
- 'lowercase': False,
'min_age': 0,
- # 'new_format': False,
'output_directory': None,
'prefix': None,
'quiet': False,
@@ -97,9 +95,8 @@ class Settings:
'skip_l3_if': False,
'skip_l3_ip': False,
'time_format': TIME_FORMAT,
- 'uppercase': False,
'user_data_file': f'{self.omd_root}/local/bin/nvdct/conf/{USER_DATA_FILE}',
- 'version': False,
+ # 'version': False,
}
# args in the form {'s, __seed_devices': 'CORE01', 'p, __path_in_inventory': None, ... }}
# we will remove 's, __'
@@ -107,10 +104,6 @@ class Settings:
{k.split(',')[-1].strip(' ').strip('_'): v for k, v in cli_args.items() if v}
)
- if self.__args.get('version'):
- print(f'{Path(SCRIPT).name} version: {NVDCT_VERSION}')
- sys_exit(ExitCodes.OK.value)
-
self.__user_data = get_data_from_toml(
file=self.__args.get('user_data_file', self.user_data_file)
)
@@ -134,7 +127,6 @@ class Settings:
sys_exit(ExitCodes.BAD_OPTION_LIST.value)
self.__api_port: int | None = None
- self.__new_format: bool | None = None
# init user data with defaults
self.__custom_layers: List[StaticConnection] | None = None
@@ -151,6 +143,8 @@ class Settings:
self.__seed_devices: List[str] | None = None
self.__static_connections: List[StaticConnection] | None = None
self.__emblems: Emblems | None = None
+ self.__sites: List[str] | None = None
+ self.__customers: List[str] | None = None
#
# CLI settings
@@ -169,7 +163,25 @@ class Settings:
@property # -b --backend
def backend(self) -> str:
- return str(self.__settings['backend'])
+ if str(self.__settings['backend']) in ['LIVESTATUS', 'MULTISITE', 'RESTAPI']:
+ return str(self.__settings['backend'])
+ else: # fallback to defaukt -> exit ??
+ LOGGER.warning(
+ f'Unknown backend: {self.__settings['backend']}. Accepted backends are: '
+ 'LIVESTATUS, MULTISITE, RESTAPI. Fall back zo MULTISITE.'
+ )
+ return 'MULTISITE'
+
+ @property # --case
+ def case(self) -> str| None:
+ if self.__settings['case'] in ['LOWER', 'UPPER']:
+ return self.__settings['case']
+ elif self.__settings['case'] is not None:
+ LOGGER.warning(
+ f'Unknon case setting {self.__settings["case"]}. '
+ 'Accepted are LOWER|UPPER. Fallback to no change.'
+ )
+ return None
@property # --check-user-data-only
def check_user_data_only(self) -> bool:
@@ -183,6 +195,28 @@ class Settings:
def dont_compare(self) -> bool:
return bool(self.__settings['dont_compare'])
+ @property # --filter-customers
+ def filter_customers(self) -> str | None:
+ if self.__settings['filter_customers'] in ['INCLUDE', 'EXCLUDE']:
+ return self.__settings['filter_customers']
+ elif self.__settings['filter_customers'] is not None:
+ LOGGER.error(
+ f'Wrong setting for "filter_customers": '
+ f'{self.__settings["filter_customers"]}, supported settings INCLUDE|EXCLUDE.'
+ )
+ return None
+
+ @property # --filter-sites
+ def filter_sites(self) -> str | None:
+ if self.__settings['filter_sites'] in ['INCLUDE', 'EXCLUDE']:
+ return self.__settings['filter_sites']
+ elif self.__settings['filter_sites'] is not None:
+ LOGGER.error(
+ f'Wrong setting for "filter_sites": '
+ f'{self.__settings["filter_sites"]}, supported settings INCLUDE|EXCLUDE.'
+ )
+ return None
+
@property # --keep
def keep(self) -> int | None:
if isinstance(self.__settings['keep'], int):
@@ -225,10 +259,6 @@ class Settings:
def log_to_stdtout(self) -> bool:
return bool(self.__settings['log_to_stdout'])
- @property # --lowercase
- def lowercase(self) -> bool:
- return bool(self.__settings['lowercase'])
-
@property # --min-age
def min_age(self) -> int:
if isinstance(self.__settings['min_age'], int):
@@ -236,18 +266,6 @@ class Settings:
else:
return 0
- @property # --new-format
- def new_format(self) -> bool:
- if self.__new_format is None:
- if self.__settings.get('new_format') is True:
- self.__new_format = True
- elif get_local_cmk_version().startswith('2.2'):
- self.__new_format = False
- else:
- self.__new_format = True
-
- return self.__new_format
-
@property # --output-directory
def output_directory(self) -> str:
# init output directory with current time if not set
@@ -289,17 +307,13 @@ class Settings:
def time_format(self) -> str:
return str(self.__settings['time_format'])
- @property # --uppercase
- def uppercase(self) -> bool:
- return bool(self.__settings['uppercase'])
-
@property # --user-data-file
def user_data_file(self) -> str:
return str(self.__settings['user_data_file'])
- @property # --version
- def version(self) -> bool:
- return bool(self.__settings['version'])
+ # @property # --version
+ # def version(self) -> bool:
+ # return bool(self.__settings['version'])
#
# user data setting
@@ -330,7 +344,7 @@ class Settings:
@property
def drop_hosts(self) -> List[str]:
if self.__drop_host is None:
- self.__drop_host = [str(host) for host in self.__user_data.get('DROP_HOSTS', [])]
+ self.__drop_host = [str(host) for host in set(self.__user_data.get('DROP_HOSTS', []))]
return self.__drop_host
@property
@@ -368,9 +382,9 @@ class Settings:
@property
def l3v4_ignore_hosts(self) -> List[str]:
if self.__l3v4_ignore_hosts is None:
- self.__l3v4_ignore_hosts = [str(host) for host in self.__user_data.get(
+ self.__l3v4_ignore_hosts = [str(host) for host in set(self.__user_data.get(
'L3V4_IGNORE_HOSTS', []
- )]
+ ))]
return self.__l3v4_ignore_hosts
@property
@@ -523,6 +537,22 @@ class Settings:
)
return self.__static_connections
+ @property
+ def sites(self) -> List[str]:
+ if self.__sites is None:
+ self.__sites = [str(site) for site in set(self.__user_data.get('SITES', []))]
+ LOGGER.info(f'fFound {len(self.__sites)} to filter on')
+ return self.__sites
+
+ @property
+ def customers(self) -> List[str]:
+ if self.__customers is None:
+ self.__customers = [
+ str(customer) for customer in set(self.__user_data.get('CUSTOMERS', []))
+ ]
+ LOGGER.info(f'fFound {len(self.__customers)} to filter on')
+ return self.__customers
+
#
# all other settings
#
@@ -532,10 +562,7 @@ class Settings:
@property
def topology_save_path(self) -> str:
- if self.new_format:
- return self.__topology_save_path_cmk_2_3
-
- return self.__topology_save_path
+ return self.__topology_save_path_cmk_2_3
@property
def topology_file_name(self) -> str:
diff --git a/source/bin/nvdct/lib/topologies.py b/source/bin/nvdct/lib/topologies.py
index 712fb332c40881e9fe81edc6784cf7d8e32ded2f..8c35a45b2a6062d07f37c1198a2f994e6b748a7f 100755
--- a/source/bin/nvdct/lib/topologies.py
+++ b/source/bin/nvdct/lib/topologies.py
@@ -8,7 +8,7 @@
# Date : 2024-06-09
# File : lib/topologies.py
-from _collections_abc import Mapping, Sequence
+from collections.abc import Mapping, Sequence
from ipaddress import IPv4Address, IPv4Network
from typing import Dict, List
@@ -20,6 +20,8 @@ from lib.utils import LOGGER
class NvObjects:
def __init__(self) -> None:
self.nv_objects: Dict[str, any] = {}
+ self.host_count: int = 0
+ self.host_list: List[str] = []
def add_host_object(
self,
@@ -28,6 +30,8 @@ class NvObjects:
emblem: str | None = None
) -> None:
if host not in self.nv_objects:
+ self.host_count += 1
+ self.host_list.append(host)
link: Dict = {}
metadata: Dict = {}
# LOGGER.debug(f'host: {host}, {host_cache.host_exists(host=host)}')
diff --git a/source/bin/nvdct/lib/utils.py b/source/bin/nvdct/lib/utils.py
index 572fb9e36fd703720fc9340cc9a433c72c298ab1..d92a87de116cc4343da4454f6fd9cdc6eb350802 100755
--- a/source/bin/nvdct/lib/utils.py
+++ b/source/bin/nvdct/lib/utils.py
@@ -7,7 +7,7 @@
# Date : 2023-10-12
# File : nvdct/lib/utils.py
-from _collections_abc import Mapping, Sequence
+from collections.abc import Mapping, Sequence
from ast import literal_eval
from dataclasses import dataclass
from enum import Enum, unique
@@ -23,7 +23,7 @@ from time import time as now_time
from tomllib import loads as toml_loads, TOMLDecodeError
from typing import List, Dict, TextIO
-NVDCT_VERSION = '0.8.12-20240702'
+NVDCT_VERSION = '0.9.0-20240923'
@unique
@@ -251,6 +251,7 @@ def save_data_to_file(data: Mapping, path: str, file: str, make_default: bool) -
Path(f'{parent_path}/default').symlink_to(target=Path(path), target_is_directory=True)
+# CMK version 2.2.x format
def save_topology(
data: dict,
base_directory: str,
@@ -322,6 +323,7 @@ def is_list_of_str_equal(list1: List[str], list2: List[str]) -> bool:
return tmp_list1 == tmp_list2
+# not used in cmk 2.3.x format
def merge_topologies(topo_pri: Dict, topo_sec: Dict) -> Dict:
"""
Merge dict_prim into dict_sec
diff --git a/source/bin/nvdct/nvdct.py b/source/bin/nvdct/nvdct.py
index 39c334ac9c543fe0f559e5454baf42b18cfc83fe..12eda3b07cf50d6727c1da7c1d2445a471e63a4f 100755
--- a/source/bin/nvdct/nvdct.py
+++ b/source/bin/nvdct/nvdct.py
@@ -113,6 +113,12 @@
# moved (default) config file(s) to ./conf/
# 2024-06-14: added debug code for bad IPv4 address data
# 2024-06-17: fixed bad IPv4 address data (just drop it)
+# 2024-09-23: replaced options --lowercase/--uppercase with --case LOWER|UPPER
+# changed version output from settings to argparse action
+# removed backend FILESYSTEM -> will fallback to MULTISITE
+# removed support for CMK2.2.x file format (removed option --new-format)
+# 2024-09-24: added site filter for multisite deployments (MULTISITE only), option --filter-sites and SITES section in toml file
+# added customer filter for MSP deployments (MULTISITE only), option --filter-customers and section CUSTOMERS in toml file
# creating topology data json from inventory data
#
@@ -208,7 +214,7 @@ __data = {
"""
import sys
-from _collections_abc import Mapping, Sequence
+from collections.abc import Mapping, Sequence
from ipaddress import IPv4Network
from logging import DEBUG
from re import compile as re_compile
@@ -219,7 +225,6 @@ from lib.args import parse_arguments
from lib.backends import (
CacheItems,
HostCache,
- HostCacheFileSystem,
HostCacheLiveStatus,
HostCacheMultiSite,
HostCacheRestApi,
@@ -227,7 +232,7 @@ from lib.backends import (
from lib.topologies import (
NvConnections,
NvObjects,
- get_list_of_devices,
+ # get_list_of_devices,
get_network_summary,
get_service_by_interface,
is_ignore_ipv4,
@@ -243,12 +248,12 @@ from lib.utils import (
Layer,
LAYERS,
LOGGER,
- merge_topologies,
+ # merge_topologies,
NVDCT_VERSION,
PATH_L3v4,
remove_old_data,
save_data_to_file,
- save_topology,
+ # save_topology,
StdoutQuiet,
)
from lib.settings import (
@@ -276,9 +281,7 @@ def create_l2_device_from_inv(
inv_data: Sequence[Mapping[str, str]],
inv_columns: InventoryColumns,
label: str,
-) -> Dict[str, object] | None:
- data: Dict = {'connections': {}, "interfaces": []}
-
+) -> None:
for topo_neighbour in inv_data:
# check if required data are not empty
if not (neighbour := topo_neighbour.get(inv_columns.neighbour)):
@@ -311,10 +314,13 @@ def create_l2_device_from_inv(
LOGGER.info(msg=f'drop neighbour: {neighbour}')
continue
- if SETTINGS.uppercase:
+ if SETTINGS.case == 'UPPER':
neighbour = neighbour.upper()
- if SETTINGS.lowercase:
+ LOGGER.debug(f'Changed neighbour to upper case: {neighbour}')
+ elif SETTINGS.case == 'LOWER':
neighbour = neighbour.lower()
+ LOGGER.debug(f'Changed neighbour to lower case: {neighbour}')
+
if SETTINGS.prefix:
neighbour = f'{SETTINGS.prefix}{neighbour}'
# rewrite neighbour if inventory neighbour and checkmk host don't match
@@ -346,17 +352,11 @@ def create_l2_device_from_inv(
f'-> neighbour_port {neighbour_port}'
)
- if neighbour and local_port and neighbour_port:
- data['connections'].update({local_port: [neighbour, neighbour_port, label]})
- if local_port not in data['interfaces']:
- data['interfaces'].append(local_port)
-
metadata = {
'duplex': topo_neighbour.get('duplex'),
'native_vlan': topo_neighbour.get('native_vlan'),
}
- # add to new object list
NV_OBJECTS.add_host_object(host=host, host_cache=HOST_CACHE)
NV_OBJECTS.add_host_object(host=neighbour, host_cache=HOST_CACHE)
NV_OBJECTS.add_service_object(
@@ -387,32 +387,10 @@ def create_l2_device_from_inv(
right=f'{neighbour_port}@{neighbour}',
)
- return {host: data}
-
-def create_static_connections(connections: Sequence[StaticConnection]) -> Dict:
- data: Dict = {}
+def create_static_connections(connections: Sequence[StaticConnection]):
for connection in connections:
LOGGER.info(msg=f'connection: {connection}')
- if connection.host not in data:
- data[connection.host] = {'connections': {}, 'interfaces': []}
-
- if connection.neighbour not in data:
- data[connection.neighbour] = {'connections': {}, 'interfaces': []}
-
- # add connection from host to neighbour
- data[connection.host]['connections'].update({connection.local_port: [
- connection.neighbour, connection.neighbour_port, connection.label
- ]})
- data[connection.host]['interfaces'].append(connection.local_port)
-
- # add connection from neighbour to host
- data[connection.neighbour]['connections'].update({
- connection.neighbour_port: [connection.host, connection.local_port, connection.label]
- })
- data[connection.neighbour]['interfaces'].append(connection.neighbour_port)
-
- # new format
NV_OBJECTS.add_host_object(
host=connection.host,
host_cache=HOST_CACHE,
@@ -448,22 +426,14 @@ def create_static_connections(connections: Sequence[StaticConnection]) -> Dict:
right=f'{connection.neighbour_port}@{connection.neighbour}',
)
- LOGGER.debug(msg=data)
- LOGGER.info(msg=f'Devices added: {len(data)}, source STATIC')
- # if not SETTINGS.quiet:
- print(f'Devices added.: {len(data)}, source STATIC')
-
- return data
-
def create_l2_topology(
seed_devices: Sequence[str],
path_in_inventory: str,
inv_columns: InventoryColumns,
label: str,
-) -> Dict:
+) -> None:
devices_to_go = list(set(seed_devices)) # remove duplicates
- topology_data: Dict = {}
devices_done = []
while devices_to_go:
@@ -482,15 +452,14 @@ def create_l2_topology(
host=device, item=CacheItems.inventory, path=path_in_inventory
)
if topo_data:
- topology_data.update(
- create_l2_device_from_inv(
- host=device,
- inv_data=topo_data,
- inv_columns=inv_columns,
- label=label,
- ))
- devices_list = get_list_of_devices(topology_data[device]['connections'])
- for _entry in devices_list:
+ create_l2_device_from_inv(
+ host=device,
+ inv_data=topo_data,
+ inv_columns=inv_columns,
+ label=label,
+ )
+
+ for _entry in NV_OBJECTS.host_list:
if _entry not in devices_done:
devices_to_go.append(_entry)
@@ -499,12 +468,6 @@ def create_l2_topology(
devices_to_go.remove(device)
LOGGER.info(msg=f'Device done: {device}, source: {label}')
- LOGGER.info(f'Devices added: {len(devices_done)}, source {label}')
- # if not SETTINGS.quiet:
- print(f'Devices added.: {len(devices_done)}, source {label}')
-
- return topology_data
-
def create_l3v4_topology(
ignore_hosts: Sequence[str],
@@ -541,7 +504,7 @@ def create_l3v4_topology(
emblem = EMBLEMS.ip_network
try:
ipv4_info = Ipv4Info(**_entry)
- except TypeError as e:
+ except TypeError: # as e
LOGGER.warning(f'Drop IPv4 address data for host: {host}, data: {_entry}')
continue
@@ -550,11 +513,16 @@ def create_l3v4_topology(
continue
if ipv4_info.cidr == 32: # drop host addresses
- LOGGER.info(f'host: {host} dropped host address: {ipv4_info.address}/{ipv4_info.cidr}')
+ LOGGER.info(
+ f'host: {host} dropped host address: {ipv4_info.address}/{ipv4_info.cidr}'
+ )
continue
if ipv4_info.type.lower() != 'ipv4': # drop if not ipv4
- LOGGER.warning(f'host: {host} dropped non ipv4 address: {ipv4_info.address}, type: {ipv4_info.type}')
+ LOGGER.warning(
+ f'host: {host} dropped non ipv4 address: {ipv4_info.address},'
+ ' type: {ipv4_info.type}'
+ )
continue
if is_ignore_ipv4(ipv4_info.address, ignore_ips):
@@ -626,14 +594,11 @@ def create_l3v4_topology(
left=network, right=f'{ipv4_info.address}@{ipv4_info.device}@{host}',
)
- LOGGER.info(msg=f'Devices added: {len(host_list)}, source L3v4')
- # if not SETTINGS.quiet:
- print(f'Devices added.: {len(host_list)}, source L3v4')
-
if __name__ == '__main__':
start_time = time_ns()
SETTINGS = Settings(vars(parse_arguments()))
+
sys.stdout = StdoutQuiet(quiet=SETTINGS.quiet)
configure_logger(
@@ -643,7 +608,6 @@ if __name__ == '__main__':
)
LOGGER.info(msg='Data creation started')
- # if not SETTINGS.quiet:
print()
print(
f'Network Visualisation Data Creation Tool (NVDCT)\n'
@@ -653,25 +617,25 @@ if __name__ == '__main__':
print()
print(f'Start time....: {strftime(SETTINGS.time_format)}')
- _backends = {
- 'LIVESTATUS': HostCacheLiveStatus,
- 'FILESYSTEM': HostCacheFileSystem,
- 'MULTISITE': HostCacheMultiSite,
- 'RESTAPI': HostCacheRestApi,
- }
- host_cache_backend = _backends.get(SETTINGS.backend, None)
- if not host_cache_backend:
- LOGGER.error(msg=f'Backend {SETTINGS.backend} not (yet) implemented')
- sys.exit(ExitCodes.BACKEND_NOT_IMPLEMENTED.value)
- elif SETTINGS.backend == 'RESTAPI':
- HOST_CACHE = host_cache_backend(
- pre_fetch=SETTINGS.pre_fetch,
- api_port=SETTINGS.api_port
- )
- else:
- HOST_CACHE = host_cache_backend(
- pre_fetch=SETTINGS.pre_fetch,
- )
+ match SETTINGS.backend:
+ case 'RESTAPI':
+ HOST_CACHE = HostCacheRestApi(
+ pre_fetch=SETTINGS.pre_fetch,
+ api_port=SETTINGS.api_port
+ )
+ case 'MULTISITE':
+ HOST_CACHE = HostCacheMultiSite(
+ pre_fetch=SETTINGS.pre_fetch,
+ filter_sites=SETTINGS.filter_sites,
+ sites = SETTINGS.sites,
+ )
+ case 'LIVESTATUS':
+ HOST_CACHE = HostCacheLiveStatus(
+ pre_fetch=SETTINGS.pre_fetch,
+ )
+ case _:
+ LOGGER.error(msg=f'Backend {SETTINGS.backend} not (yet) implemented')
+ sys.exit(ExitCodes.BACKEND_NOT_IMPLEMENTED.value)
HOST_MAP = SETTINGS.host_map
DROP_HOSTS = SETTINGS.drop_hosts
@@ -717,27 +681,26 @@ if __name__ == '__main__':
for job in jobs:
match job:
case 'STATIC':
- topology: Dict | None = create_static_connections(
+ label = 'static'
+ create_static_connections(
connections=SETTINGS.static_connections
)
- label = 'static'
case 'L3v4':
topology = None
label = 'l3v4'
- if SETTINGS.new_format:
- create_l3v4_topology(
- ignore_hosts=SETTINGS.l3v4_ignore_hosts,
- ignore_ips=SETTINGS.l3v4_ignore_ips,
- ignore_wildcard=SETTINGS.l3v4_ignore_wildcard,
- summarize=SETTINGS.l3v4_summarize,
- replace=SETTINGS.l3v4_replace,
- skip_if=SETTINGS.skip_l3_if,
- skip_ip=SETTINGS.skip_l3_ip
- )
+ create_l3v4_topology(
+ ignore_hosts=SETTINGS.l3v4_ignore_hosts,
+ ignore_ips=SETTINGS.l3v4_ignore_ips,
+ ignore_wildcard=SETTINGS.l3v4_ignore_wildcard,
+ summarize=SETTINGS.l3v4_summarize,
+ replace=SETTINGS.l3v4_replace,
+ skip_if=SETTINGS.skip_l3_if,
+ skip_ip=SETTINGS.skip_l3_ip
+ )
case _:
label = job.label.lower()
columns = job.columns.split(',')
- topology = create_l2_topology(
+ create_l2_topology(
seed_devices=SETTINGS.seed_devices,
path_in_inventory=job.path,
inv_columns=InventoryColumns(
@@ -748,52 +711,42 @@ if __name__ == '__main__':
label=label,
)
- if SETTINGS.new_format:
- LOGGER.info(f'Save topology {label} in new format')
- NV_CONNECTIONS.add_meta_data_to_connections(
- nv_objects=NV_OBJECTS,
- speed_map=MAP_SPEED_TO_THICKNESS,
- )
- if not SETTINGS.loglevel == DEBUG:
- connections = NV_CONNECTIONS.nv_connections
- else:
- connections = sorted(NV_CONNECTIONS.nv_connections)
- _data = {
- 'version': 1,
- 'name': label,
- 'objects': NV_OBJECTS.nv_objects if not SETTINGS.loglevel == DEBUG else dict(
- sorted(NV_OBJECTS.nv_objects.items())
- ),
- 'connections': connections
- }
- save_data_to_file(
- data=_data,
- path=(
- f'{SETTINGS.omd_root}/{SETTINGS.topology_save_path}/'
- f'{SETTINGS.output_directory}'
- ),
- file=f'data_{label}.json',
- make_default=SETTINGS.default,
- )
+ NV_CONNECTIONS.add_meta_data_to_connections(
+ nv_objects=NV_OBJECTS,
+ speed_map=MAP_SPEED_TO_THICKNESS,
+ )
+ if not SETTINGS.loglevel == DEBUG:
+ connections = NV_CONNECTIONS.nv_connections
+ else:
+ connections = sorted(NV_CONNECTIONS.nv_connections)
+ _data = {
+ 'version': 1,
+ 'name': label,
+ 'objects': NV_OBJECTS.nv_objects if not SETTINGS.loglevel == DEBUG else dict(
+ sorted(NV_OBJECTS.nv_objects.items())
+ ),
+ 'connections': connections
+ }
+ save_data_to_file(
+ data=_data,
+ path=(
+ f'{SETTINGS.omd_root}/{SETTINGS.topology_save_path}/'
+ f'{SETTINGS.output_directory}'
+ ),
+ file=f'data_{label}.json',
+ make_default=SETTINGS.default,
+ )
+
+ message = (
+ f'Source {label:.<7s}: Devices/Objects/Connections added {NV_OBJECTS.host_count}/'
+ f'{len(NV_OBJECTS.nv_objects)}/{len(NV_CONNECTIONS.nv_connections)}'
+ )
+ LOGGER.info(msg=message)
+ print(message)
NV_OBJECTS = NvObjects()
NV_CONNECTIONS = NvConnections()
- if topology:
- final_topology = merge_topologies(topo_pri=topology, topo_sec=final_topology)
-
- if final_topology and not SETTINGS.new_format:
- LOGGER.info('Save topology in CMK 2.2.x format')
-
- save_topology(
- data=final_topology,
- base_directory=f'{SETTINGS.omd_root}/{SETTINGS.topology_save_path}',
- output_directory=SETTINGS.output_directory,
- topology_file_name=SETTINGS.topology_file_name,
- dont_compare=SETTINGS.dont_compare,
- make_default=SETTINGS.default,
- )
-
if SETTINGS.keep:
remove_old_data(
keep=SETTINGS.keep,
@@ -802,7 +755,6 @@ if __name__ == '__main__':
protected=SETTINGS.protected_topologies,
)
- # if not SETTINGS.quiet:
print(f'Time taken....: {(time_ns() - start_time) / 1e9}/s')
print(f'End time......: {strftime(SETTINGS.time_format)}')
print()
diff --git a/source/packages/nvdct b/source/packages/nvdct
index da6ca1483c51a0fa6f3cb5246b5d4fc473bbd8fe..61fa09521d0092bca60c49a0fa3e63a38ec50510 100644
--- a/source/packages/nvdct
+++ b/source/packages/nvdct
@@ -8,9 +8,8 @@
'\n'
'Features:\n'
'\n'
- ' - Ready for CMK 2.3.0\n'
' - Create Layer 2 (CDP/LLDP) topology data\n'
- ' - Create Layer 3 (IPv4) topology data (CMK2.3.0 only)\n'
+ ' - Create Layer 3 (IPv4) topology data\n'
' - Highlight connection issues (Speed, Duplex/Half duplex, '
'Native VLAN)\n'
' - Read connection data from the Checkmk HW/SW inventory.\n'
@@ -19,16 +18,13 @@
' - Add custom connections (STATIC) for connections that are '
'not in the inventory\n'
' - Optimized for my CDP, LLDP and IPv4 inventory plugins\n'
- ' - Merge CDP, LLDP, STATIC topologies (CMK 2.2.0 only)\n'
' - Can also be used with custom inventory plugins\n'
'\n'
'For more information about the network visualization plugin '
'see: \n'
'the announcement from schnetz '
'https://forum.checkmk.com/t/network-visualization/41680\n'
- 'and the plugin on the Exchange (CMK 2.2.0 only, CMK 2.3.0 has '
- 'this built-in) '
- 'https://exchange.checkmk.com/p/network-visualization\n'
+ '(CMK 2.3.0 has this plugin built-in) \n'
'\n'
'The inventory data could be created with my inventory '
'plugins:\n'
@@ -58,7 +54,7 @@
'htdocs/images/icons/location_80.png']},
'name': 'nvdct',
'title': 'Network Visualization Data Creation Tool (NVDCT)',
- 'version': '0.8.12-20240702',
- 'version.min_required': '2.2.0b1',
+ 'version': '0.9.0-20240923',
+ 'version.min_required': '2.3.0b1',
'version.packaged': 'cmk-mkp-tool 0.2.0',
'version.usable_until': '2.4.0p1'}