feat: improve error handling

This commit is contained in:
Robert Kaussow 2021-06-09 22:57:48 +02:00
parent fdc41dcdef
commit 1df0c9ed39
3 changed files with 44 additions and 32 deletions

View File

@ -12,6 +12,7 @@ import prometheuspvesd.exception
from prometheuspvesd import __version__ from prometheuspvesd import __version__
from prometheuspvesd.config import SingleConfig from prometheuspvesd.config import SingleConfig
from prometheuspvesd.discovery import Discovery from prometheuspvesd.discovery import Discovery
from prometheuspvesd.exception import APIError
from prometheuspvesd.model import HostList from prometheuspvesd.model import HostList
from prometheuspvesd.utils import SingleLog from prometheuspvesd.utils import SingleLog
@ -25,7 +26,17 @@ class PrometheusSD:
self.args = self._cli_args() self.args = self._cli_args()
self.config = self._get_config() self.config = self._get_config()
signal.signal(signal.SIGINT, self._terminate)
signal.signal(signal.SIGTERM, self._terminate)
while True:
try:
self.discovery = Discovery() self.discovery = Discovery()
except APIError as e:
self.logger.error("Proxmoxer API error: {0}".format(str(e).strip()))
sleep(5)
continue
break
self._fetch() self._fetch()
def _cli_args(self): def _cli_args(self):
@ -89,9 +100,6 @@ class PrometheusSD:
return config return config
def _fetch(self): def _fetch(self):
signal.signal(signal.SIGINT, self._terminate)
signal.signal(signal.SIGTERM, self._terminate)
loop_delay = self.config.config["loop_delay"] loop_delay = self.config.config["loop_delay"]
output_file = self.config.config["output_file"] output_file = self.config.config["output_file"]
@ -99,7 +107,14 @@ class PrometheusSD:
self.logger.debug("Propagate from PVE") self.logger.debug("Propagate from PVE")
while True: while True:
self._write(self.discovery.propagate()) try:
inventory = self.discovery.propagate()
except APIError as e:
self.logger.error("Proxmoxer API error: {0}".format(str(e).strip()))
except Exception as e: # noqa
self.logger.error("Unknown error: {0}".format(str(e).strip()))
else:
self._write(inventory)
if not self.config.config["service"]: if not self.config.config["service"]:
break break

View File

@ -6,7 +6,10 @@ import re
import socket import socket
from collections import defaultdict from collections import defaultdict
import requests
from prometheuspvesd.config import SingleConfig from prometheuspvesd.config import SingleConfig
from prometheuspvesd.exception import APIError
from prometheuspvesd.model import Host from prometheuspvesd.model import Host
from prometheuspvesd.model import HostList from prometheuspvesd.model import HostList
from prometheuspvesd.utils import SingleLog from prometheuspvesd.utils import SingleLog
@ -24,7 +27,7 @@ class Discovery():
def __init__(self): def __init__(self):
if not HAS_PROXMOXER: if not HAS_PROXMOXER:
self.logger.error( self.log.sysexit_with_message(
"The Proxmox VE Prometheus SD requires proxmoxer: " "The Proxmox VE Prometheus SD requires proxmoxer: "
"https://pypi.org/project/proxmoxer/" "https://pypi.org/project/proxmoxer/"
) )
@ -36,6 +39,7 @@ class Discovery():
self.host_list = HostList() self.host_list = HostList()
def _auth(self): def _auth(self):
try:
return ProxmoxAPI( return ProxmoxAPI(
self.config.config["pve"]["server"], self.config.config["pve"]["server"],
user=self.config.config["pve"]["user"], user=self.config.config["pve"]["user"],
@ -43,6 +47,8 @@ class Discovery():
verify_ssl=to_bool(self.config.config["pve"]["verify_ssl"]), verify_ssl=to_bool(self.config.config["pve"]["verify_ssl"]),
timeout=self.config.config["pve"]["auth_timeout"] timeout=self.config.config["pve"]["auth_timeout"]
) )
except requests.RequestException as e:
raise APIError(str(e))
def _get_names(self, pve_list, pve_type): def _get_names(self, pve_list, pve_type):
names = [] names = []
@ -87,7 +93,7 @@ class Discovery():
networks = self.client.nodes(pve_node).get( networks = self.client.nodes(pve_node).get(
"qemu", vmid, "agent", "network-get-interfaces" "qemu", vmid, "agent", "network-get-interfaces"
)["result"] )["result"]
except Exception: # noqa except Exception: # noqa # nosec
pass pass
if networks: if networks:
@ -106,7 +112,7 @@ class Discovery():
if find and find.group(1): if find and find.group(1):
address = find.group(1) address = find.group(1)
break break
except Exception: # noqa except Exception: # noqa # nosec
pass pass
return address return address
@ -135,7 +141,7 @@ class Discovery():
qemu_list = self._exclude(self.client.nodes(node).qemu.get()) qemu_list = self._exclude(self.client.nodes(node).qemu.get())
container_list = self._exclude(self.client.nodes(node).lxc.get()) container_list = self._exclude(self.client.nodes(node).lxc.get())
except Exception as e: # noqa except Exception as e: # noqa
self.logger.error("Proxmoxer API error: {0}".format(str(e))) raise APIError(str(e))
# Merge QEMU and Containers lists from this node # Merge QEMU and Containers lists from this node
instances = self._get_variables(qemu_list, "qemu").copy() instances = self._get_variables(qemu_list, "qemu").copy()
@ -158,7 +164,7 @@ class Discovery():
except KeyError: except KeyError:
description = None description = None
except Exception as e: # noqa except Exception as e: # noqa
self.logger.error("Proxmoxer API error: {0}".format(str(e))) raise APIError(str(e))
try: try:
metadata = json.loads(description) metadata = json.loads(description)
@ -188,19 +194,4 @@ class Discovery():
self.host_list.add_host(prom_host) self.host_list.add_host(prom_host)
self.logger.debug("Discovered {}".format(prom_host)) self.logger.debug("Discovered {}".format(prom_host))
for pool in self._get_names(self.client.pools.get(), "pool"):
try:
pool_list = self._exclude(self.client.pool(pool).get()["members"])
except Exception as e: # noqa
self.logger.error("Proxmoxer API error: {0}".format(str(e)))
members = [
member["name"]
for member in pool_list
if (member["type"] == "qemu" or member["type"] == "lxc")
]
for member in members:
self.inventory.add_host(group=pool, host=member)
return self.host_list return self.host_list

View File

@ -11,6 +11,12 @@ class PrometheusSDError(Exception):
self.original_exception = original_exception self.original_exception = original_exception
class APIError(PrometheusSDError):
"""Errors related to API connections."""
pass
class ConfigError(PrometheusSDError): class ConfigError(PrometheusSDError):
"""Errors related to config file handling.""" """Errors related to config file handling."""