feat: improve error handling

prometheuspvesd/cli.py

@@ -12,6 +12,7 @@ import prometheuspvesd.exception
 from prometheuspvesd import __version__
 from prometheuspvesd.config import SingleConfig
 from prometheuspvesd.discovery import Discovery
+from prometheuspvesd.exception import APIError
 from prometheuspvesd.model import HostList
 from prometheuspvesd.utils import SingleLog
 
@@ -25,7 +26,17 @@ class PrometheusSD:
         self.args = self._cli_args()
         self.config = self._get_config()
 
-        self.discovery = Discovery()
+        signal.signal(signal.SIGINT, self._terminate)
+        signal.signal(signal.SIGTERM, self._terminate)
+        while True:
+            try:
+                self.discovery = Discovery()
+            except APIError as e:
+                self.logger.error("Proxmoxer API error: {0}".format(str(e).strip()))
+                sleep(5)
+                continue
+            break
+
         self._fetch()
 
     def _cli_args(self):
@@ -89,9 +100,6 @@ class PrometheusSD:
         return config
 
     def _fetch(self):
-        signal.signal(signal.SIGINT, self._terminate)
-        signal.signal(signal.SIGTERM, self._terminate)
-
         loop_delay = self.config.config["loop_delay"]
         output_file = self.config.config["output_file"]
 
@@ -99,7 +107,14 @@ class PrometheusSD:
         self.logger.debug("Propagate from PVE")
 
         while True:
-            self._write(self.discovery.propagate())
+            try:
+                inventory = self.discovery.propagate()
+            except APIError as e:
+                self.logger.error("Proxmoxer API error: {0}".format(str(e).strip()))
+            except Exception as e:  # noqa
+                self.logger.error("Unknown error: {0}".format(str(e).strip()))
+            else:
+                self._write(inventory)
 
             if not self.config.config["service"]:
                 break

prometheuspvesd/discovery.py

@@ -6,7 +6,10 @@ import re
 import socket
 from collections import defaultdict
 
+import requests
+
 from prometheuspvesd.config import SingleConfig
+from prometheuspvesd.exception import APIError
 from prometheuspvesd.model import Host
 from prometheuspvesd.model import HostList
 from prometheuspvesd.utils import SingleLog
@@ -24,7 +27,7 @@ class Discovery():
 
     def __init__(self):
         if not HAS_PROXMOXER:
-            self.logger.error(
+            self.log.sysexit_with_message(
                 "The Proxmox VE Prometheus SD requires proxmoxer: "
                 "https://pypi.org/project/proxmoxer/"
             )
@@ -36,13 +39,16 @@ class Discovery():
         self.host_list = HostList()
 
     def _auth(self):
-        return ProxmoxAPI(
-            self.config.config["pve"]["server"],
-            user=self.config.config["pve"]["user"],
-            password=self.config.config["pve"]["password"],
-            verify_ssl=to_bool(self.config.config["pve"]["verify_ssl"]),
-            timeout=self.config.config["pve"]["auth_timeout"]
-        )
+        try:
+            return ProxmoxAPI(
+                self.config.config["pve"]["server"],
+                user=self.config.config["pve"]["user"],
+                password=self.config.config["pve"]["password"],
+                verify_ssl=to_bool(self.config.config["pve"]["verify_ssl"]),
+                timeout=self.config.config["pve"]["auth_timeout"]
+            )
+        except requests.RequestException as e:
+            raise APIError(str(e))
 
     def _get_names(self, pve_list, pve_type):
         names = []
@@ -87,7 +93,7 @@ class Discovery():
                     networks = self.client.nodes(pve_node).get(
                         "qemu", vmid, "agent", "network-get-interfaces"
                     )["result"]
-            except Exception:  # noqa
+            except Exception:  # noqa  # nosec
                 pass
 
             if networks:
@@ -106,7 +112,7 @@ class Discovery():
                     if find and find.group(1):
                         address = find.group(1)
                         break
-            except Exception:  # noqa
+            except Exception:  # noqa  # nosec
                 pass
 
         return address
@@ -135,7 +141,7 @@ class Discovery():
                 qemu_list = self._exclude(self.client.nodes(node).qemu.get())
                 container_list = self._exclude(self.client.nodes(node).lxc.get())
             except Exception as e:  # noqa
-                self.logger.error("Proxmoxer API error: {0}".format(str(e)))
+                raise APIError(str(e))
 
             # Merge QEMU and Containers lists from this node
             instances = self._get_variables(qemu_list, "qemu").copy()
@@ -158,7 +164,7 @@ class Discovery():
                 except KeyError:
                     description = None
                 except Exception as e:  # noqa
-                    self.logger.error("Proxmoxer API error: {0}".format(str(e)))
+                    raise APIError(str(e))
 
                 try:
                     metadata = json.loads(description)
@@ -188,19 +194,4 @@ class Discovery():
                 self.host_list.add_host(prom_host)
                 self.logger.debug("Discovered {}".format(prom_host))
 
-        for pool in self._get_names(self.client.pools.get(), "pool"):
-            try:
-                pool_list = self._exclude(self.client.pool(pool).get()["members"])
-            except Exception as e:  # noqa
-                self.logger.error("Proxmoxer API error: {0}".format(str(e)))
-
-            members = [
-                member["name"]
-                for member in pool_list
-                if (member["type"] == "qemu" or member["type"] == "lxc")
-            ]
-
-            for member in members:
-                self.inventory.add_host(group=pool, host=member)
-
         return self.host_list

prometheuspvesd/exception.py

@@ -11,6 +11,12 @@ class PrometheusSDError(Exception):
         self.original_exception = original_exception
 
 
+class APIError(PrometheusSDError):
+    """Errors related to API connections."""
+
+    pass
+
+
 class ConfigError(PrometheusSDError):
     """Errors related to config file handling."""