From 1df0c9ed39ee32b8d275f573d3a59f198d3dc528 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Wed, 9 Jun 2021 22:57:48 +0200 Subject: [PATCH] feat: improve error handling --- prometheuspvesd/cli.py | 25 ++++++++++++++++---- prometheuspvesd/discovery.py | 45 +++++++++++++++--------------------- prometheuspvesd/exception.py | 6 +++++ 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/prometheuspvesd/cli.py b/prometheuspvesd/cli.py index b00c92f..1dd0e80 100644 --- a/prometheuspvesd/cli.py +++ b/prometheuspvesd/cli.py @@ -12,6 +12,7 @@ import prometheuspvesd.exception from prometheuspvesd import __version__ from prometheuspvesd.config import SingleConfig from prometheuspvesd.discovery import Discovery +from prometheuspvesd.exception import APIError from prometheuspvesd.model import HostList from prometheuspvesd.utils import SingleLog @@ -25,7 +26,17 @@ class PrometheusSD: self.args = self._cli_args() self.config = self._get_config() - self.discovery = Discovery() + signal.signal(signal.SIGINT, self._terminate) + signal.signal(signal.SIGTERM, self._terminate) + while True: + try: + self.discovery = Discovery() + except APIError as e: + self.logger.error("Proxmoxer API error: {0}".format(str(e).strip())) + sleep(5) + continue + break + self._fetch() def _cli_args(self): @@ -89,9 +100,6 @@ class PrometheusSD: return config def _fetch(self): - signal.signal(signal.SIGINT, self._terminate) - signal.signal(signal.SIGTERM, self._terminate) - loop_delay = self.config.config["loop_delay"] output_file = self.config.config["output_file"] @@ -99,7 +107,14 @@ class PrometheusSD: self.logger.debug("Propagate from PVE") while True: - self._write(self.discovery.propagate()) + try: + inventory = self.discovery.propagate() + except APIError as e: + self.logger.error("Proxmoxer API error: {0}".format(str(e).strip())) + except Exception as e: # noqa + self.logger.error("Unknown error: {0}".format(str(e).strip())) + else: + self._write(inventory) if not self.config.config["service"]: break diff --git a/prometheuspvesd/discovery.py b/prometheuspvesd/discovery.py index e66199a..3c2474d 100644 --- a/prometheuspvesd/discovery.py +++ b/prometheuspvesd/discovery.py @@ -6,7 +6,10 @@ import re import socket from collections import defaultdict +import requests + from prometheuspvesd.config import SingleConfig +from prometheuspvesd.exception import APIError from prometheuspvesd.model import Host from prometheuspvesd.model import HostList from prometheuspvesd.utils import SingleLog @@ -24,7 +27,7 @@ class Discovery(): def __init__(self): if not HAS_PROXMOXER: - self.logger.error( + self.log.sysexit_with_message( "The Proxmox VE Prometheus SD requires proxmoxer: " "https://pypi.org/project/proxmoxer/" ) @@ -36,13 +39,16 @@ class Discovery(): self.host_list = HostList() def _auth(self): - return ProxmoxAPI( - self.config.config["pve"]["server"], - user=self.config.config["pve"]["user"], - password=self.config.config["pve"]["password"], - verify_ssl=to_bool(self.config.config["pve"]["verify_ssl"]), - timeout=self.config.config["pve"]["auth_timeout"] - ) + try: + return ProxmoxAPI( + self.config.config["pve"]["server"], + user=self.config.config["pve"]["user"], + password=self.config.config["pve"]["password"], + verify_ssl=to_bool(self.config.config["pve"]["verify_ssl"]), + timeout=self.config.config["pve"]["auth_timeout"] + ) + except requests.RequestException as e: + raise APIError(str(e)) def _get_names(self, pve_list, pve_type): names = [] @@ -87,7 +93,7 @@ class Discovery(): networks = self.client.nodes(pve_node).get( "qemu", vmid, "agent", "network-get-interfaces" )["result"] - except Exception: # noqa + except Exception: # noqa # nosec pass if networks: @@ -106,7 +112,7 @@ class Discovery(): if find and find.group(1): address = find.group(1) break - except Exception: # noqa + except Exception: # noqa # nosec pass return address @@ -135,7 +141,7 @@ class Discovery(): qemu_list = self._exclude(self.client.nodes(node).qemu.get()) container_list = self._exclude(self.client.nodes(node).lxc.get()) except Exception as e: # noqa - self.logger.error("Proxmoxer API error: {0}".format(str(e))) + raise APIError(str(e)) # Merge QEMU and Containers lists from this node instances = self._get_variables(qemu_list, "qemu").copy() @@ -158,7 +164,7 @@ class Discovery(): except KeyError: description = None except Exception as e: # noqa - self.logger.error("Proxmoxer API error: {0}".format(str(e))) + raise APIError(str(e)) try: metadata = json.loads(description) @@ -188,19 +194,4 @@ class Discovery(): self.host_list.add_host(prom_host) self.logger.debug("Discovered {}".format(prom_host)) - for pool in self._get_names(self.client.pools.get(), "pool"): - try: - pool_list = self._exclude(self.client.pool(pool).get()["members"]) - except Exception as e: # noqa - self.logger.error("Proxmoxer API error: {0}".format(str(e))) - - members = [ - member["name"] - for member in pool_list - if (member["type"] == "qemu" or member["type"] == "lxc") - ] - - for member in members: - self.inventory.add_host(group=pool, host=member) - return self.host_list diff --git a/prometheuspvesd/exception.py b/prometheuspvesd/exception.py index 05a6910..c6890b0 100644 --- a/prometheuspvesd/exception.py +++ b/prometheuspvesd/exception.py @@ -11,6 +11,12 @@ class PrometheusSDError(Exception): self.original_exception = original_exception +class APIError(PrometheusSDError): + """Errors related to API connections.""" + + pass + + class ConfigError(PrometheusSDError): """Errors related to config file handling."""