feat: add service discovery metrics

docs/content/configuration/defaults.md

@@ -13,6 +13,11 @@ logging:
     # supported log formats: console|json|simple
     format: console
 
+metrics:
+    enabled: true
+    address: "127.0.0.1"
+    port: 8000
+
 output_file:
 loop_delay: 300
 # Run pve sd in a loop and discover hosts every n seconds (as defined in loop_delay).

docs/content/configuration/env.md

@@ -7,17 +7,26 @@ title: Environment Variables
 <!-- spellchecker-disable -->
 {{< highlight Shell "linenos=table" >}}
 PROMETHEUS_PVE_SD_CONFIG_FILE=
+
 # supported log levels: debug|info|warning|error|critical
 PROMETHEUS_PVE_SD_LOG_LEVEL=warning
 # supported log formats: console|json|simple
 PROMETHEUS_PVE_SD_LOG_FORMAT=console
+
+METRICS_ENABLED=true
+METRICS_ADDRESS=127.0.01
+METRICS_PORT=8000
+
 PROMETHEUS_PVE_SD_OUTPUT_FILE=
 PROMETHEUS_PVE_SD_LOOP_DELAY=300
+
 # Run pve sd in a loop and discover hosts every n seconds (as defined in PROMETHEUS_PVE_SD_LOOP_DELAY).
 # Can be disabled to run disovery only once.
 PROMETHEUS_PVE_SD_SERVICE=true
+
 PROMETHEUS_PVE_SD_EXCLUDE_STATE=
 PROMETHEUS_PVE_SD_EXCLUDE_VMID=
+
 PROMETHEUS_PVE_SD_PVE_SERVER=
 PROMETHEUS_PVE_SD_PVE_USER=
 PROMETHEUS_PVE_SD_PVE_PASSWORD=

poetry.lock

@@ -404,6 +404,17 @@ importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""}
 [package.extras]
 dev = ["pre-commit", "tox"]
 
+[[package]]
+name = "prometheus-client"
+version = "0.11.0"
+description = "Python client for the Prometheus monitoring system."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+
+[package.extras]
+twisted = ["twisted"]
+
 [[package]]
 name = "proxmoxer"
 version = "1.1.1"
@@ -687,7 +698,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.6.0"
-content-hash = "d22e186f254a2c3c272eab9e38414221eec6eed25aa874f93b561faf5855a8e8"
+content-hash = "c801a2419674bd6da0ccd03908adb40371db22b727790f8e89053c39041fb4dd"
 
 [metadata.files]
 anyconfig = [
@@ -875,6 +886,10 @@ pluggy = [
     {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"},
     {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"},
 ]
+prometheus-client = [
+    {file = "prometheus_client-0.11.0-py2.py3-none-any.whl", hash = "sha256:b014bc76815eb1399da8ce5fc84b7717a3e63652b0c0f8804092c9363acab1b2"},
+    {file = "prometheus_client-0.11.0.tar.gz", hash = "sha256:3a8baade6cb80bcfe43297e33e7623f3118d660d41387593758e2fb1ea173a86"},
+]
 proxmoxer = [
     {file = "proxmoxer-1.1.1.tar.gz", hash = "sha256:684a69190129da0f102703fc9861f5eea82a7d804f9f96d35c7fd73452f1da7e"},
 ]
@@ -927,26 +942,18 @@ pyyaml = [
     {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"},
     {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"},
     {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"},
-    {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"},
-    {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"},
     {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"},
     {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"},
     {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"},
     {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"},
-    {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"},
-    {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"},
     {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"},
     {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"},
     {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"},
     {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"},
-    {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"},
-    {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"},
     {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"},
     {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"},
     {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"},
     {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"},
-    {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"},
-    {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"},
     {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"},
     {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"},
     {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"},
@@ -967,29 +974,22 @@ requests = [
     {file = "ruamel.yaml.clib-0.2.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:73b3d43e04cc4b228fa6fa5d796409ece6fcb53a6c270eb2048109cbcbc3b9c2"},
     {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:53b9dd1abd70e257a6e32f934ebc482dac5edb8c93e23deb663eac724c30b026"},
     {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:839dd72545ef7ba78fd2aa1a5dd07b33696adf3e68fae7f31327161c1093001b"},
-    {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1236df55e0f73cd138c0eca074ee086136c3f16a97c2ac719032c050f7e0622f"},
     {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win32.whl", hash = "sha256:b1e981fe1aff1fd11627f531524826a4dcc1f26c726235a52fcb62ded27d150f"},
     {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4e52c96ca66de04be42ea2278012a2342d89f5e82b4512fb6fb7134e377e2e62"},
     {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a873e4d4954f865dcb60bdc4914af7eaae48fb56b60ed6daa1d6251c72f5337c"},
     {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ab845f1f51f7eb750a78937be9f79baea4a42c7960f5a94dde34e69f3cce1988"},
-    {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:2fd336a5c6415c82e2deb40d08c222087febe0aebe520f4d21910629018ab0f3"},
     {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win32.whl", hash = "sha256:e9f7d1d8c26a6a12c23421061f9022bb62704e38211fe375c645485f38df34a2"},
     {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:2602e91bd5c1b874d6f93d3086f9830f3e907c543c7672cf293a97c3fabdcd91"},
     {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:44c7b0498c39f27795224438f1a6be6c5352f82cb887bc33d962c3a3acc00df6"},
     {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8e8fd0a22c9d92af3a34f91e8a2594eeb35cba90ab643c5e0e643567dc8be43e"},
-    {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:75f0ee6839532e52a3a53f80ce64925ed4aed697dd3fa890c4c918f3304bd4f4"},
     {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win32.whl", hash = "sha256:464e66a04e740d754170be5e740657a3b3b6d2bcc567f0c3437879a6e6087ff6"},
     {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:52ae5739e4b5d6317b52f5b040b1b6639e8af68a5b8fd606a8b08658fbd0cab5"},
     {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4df5019e7783d14b79217ad9c56edf1ba7485d614ad5a385d1b3c768635c81c0"},
     {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5254af7d8bdf4d5484c089f929cb7f5bafa59b4f01d4f48adda4be41e6d29f99"},
-    {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8be05be57dc5c7b4a0b24edcaa2f7275866d9c907725226cdde46da09367d923"},
     {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win32.whl", hash = "sha256:74161d827407f4db9072011adcfb825b5258a5ccb3d2cd518dd6c9edea9e30f1"},
     {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:058a1cc3df2a8aecc12f983a48bda99315cebf55a3b3a5463e37bb599b05727b"},
     {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c6ac7e45367b1317e56f1461719c853fd6825226f45b835df7436bb04031fd8a"},
     {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:b4b0d31f2052b3f9f9b5327024dc629a253a83d8649d4734ca7f35b60ec3e9e5"},
-    {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1f8c0a4577c0e6c99d208de5c4d3fd8aceed9574bb154d7a2b21c16bb924154c"},
-    {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win32.whl", hash = "sha256:46d6d20815064e8bb023ea8628cfb7402c0f0e83de2c2227a88097e239a7dffd"},
-    {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:6c0a5dc52fc74eb87c67374a4e554d4761fd42a4d01390b7e868b30d21f4b8bb"},
     {file = "ruamel.yaml.clib-0.2.2.tar.gz", hash = "sha256:2d24bd98af676f4990c4d715bcdc2a60b19c56a3fb3a763164d2d8ca0e806ba7"},
 ]
 six = [

prometheuspvesd/cli.py

@@ -8,6 +8,8 @@ import signal
 import tempfile
 from time import sleep
 
+from prometheus_client import start_http_server
+
 import prometheuspvesd.exception
 from prometheuspvesd import __version__
 from prometheuspvesd.config import SingleConfig
@@ -119,12 +121,20 @@ class PrometheusSD:
         return config
 
     def _fetch(self):
-        loop_delay = self.config.config["loop_delay"]
+        self.logger.info("Writes targets to {}".format(self.config.config["output_file"]))
-        output_file = self.config.config["output_file"]
-
-        self.logger.info("Writes targets to {}".format(output_file))
         self.logger.debug("Propagate from PVE")
 
+        if self.config.config["service"] and self.config.config["metrics"]["enabled"]:
+            self.logger.info(
+                "Starting metrics http endpoint on port {}".format(
+                    self.config.config["metrics"]["port"]
+                )
+            )
+            start_http_server(
+                self.config.config["metrics"]["port"],
+                addr=self.config.config["metrics"]["address"]
+            )
+
         while True:
             try:
                 inventory = self.discovery.propagate()
@@ -138,7 +148,11 @@ class PrometheusSD:
             if not self.config.config["service"]:
                 break
 
-            self.logger.info("Waiting {} seconds for next discovery loop".format(loop_delay))
+            self.logger.info(
+                "Waiting {} seconds for next discovery loop".format(
+                    self.config.config["loop_delay"]
+                )
+            )
             sleep(self.config.config["loop_delay"])
 
     def _write(self, host_list: HostList):

prometheuspvesd/config.py

@@ -32,6 +32,21 @@ class Config():
     """
 
     SETTINGS = {
+        "metrics.enabled": {
+            "default": True,
+            "env": "METRICS_ENABLED",
+            "type": environs.Env().bool
+        },
+        "metrics.address": {
+            "default": "127.0.0.1",
+            "env": "METRICS_ADDRESS",
+            "type": environs.Env().str
+        },
+        "metrics.port": {
+            "default": 8000,
+            "env": "METRICS_PORT",
+            "type": environs.Env().int
+        },
         "config_file": {
             "default": "",
             "env": "CONFIG_FILE",

prometheuspvesd/discovery.py

@@ -7,6 +7,9 @@ import socket
 from collections import defaultdict
 
 import requests
+from prometheus_client import Counter
+from prometheus_client import Gauge
+from prometheus_client import Summary
 
 from prometheuspvesd.config import SingleConfig
 from prometheuspvesd.exception import APIError
@@ -21,6 +24,15 @@ try:
 except ImportError:
     HAS_PROXMOXER = False
 
+PROPAGATION_TIME = Summary(
+    "pve_sd_propagate_seconds", "Time spent propagating the inventory from PVE"
+)
+HOST_GAUGE = Gauge("pve_sd_hosts", "Number of hosts discovered by PVE SD")
+PVE_REQUEST_COUNT_TOTAL = Counter("pve_sd_requests_total", "Total count of requests to PVE API")
+PVE_REQUEST_COUNT_ERROR_TOTAL = Counter(
+    "pve_sd_requests_error_total", "Total count of failed requests to PVE API"
+)
+
 
 class Discovery():
     """Prometheus PVE Service Discovery."""
@@ -48,6 +60,7 @@ class Discovery():
                 timeout=self.config.config["pve"]["auth_timeout"]
             )
         except requests.RequestException as e:
+            PVE_REQUEST_COUNT_ERROR_TOTAL.inc()
             raise APIError(str(e))
 
     def _get_names(self, pve_list, pve_type):
@@ -89,6 +102,7 @@ class Discovery():
         if pve_type == "qemu":
             # If qemu agent is enabled, try to gather the IP address
             try:
+                PVE_REQUEST_COUNT_TOTAL.inc()
                 if self.client.nodes(pve_node).get(pve_type, vmid, "agent", "info") is not None:
                     networks = self.client.nodes(pve_node).get(
                         "qemu", vmid, "agent", "network-get-interfaces"
@@ -104,6 +118,7 @@ class Discovery():
 
         if not address:
             try:
+                PVE_REQUEST_COUNT_TOTAL.inc()
                 config = self.client.nodes(pve_node).get(pve_type, vmid, "config")
                 sources = [config["net0"], config["ipconfig0"]]
 
@@ -133,20 +148,26 @@ class Discovery():
             filtered.append(item.copy())
         return filtered
 
+    @PROPAGATION_TIME.time()
     def propagate(self):
         self.host_list.clear()
 
+        PVE_REQUEST_COUNT_TOTAL.inc()
         for node in self._get_names(self.client.nodes.get(), "node"):
             try:
+                PVE_REQUEST_COUNT_TOTAL.inc()
                 qemu_list = self._exclude(self.client.nodes(node).qemu.get())
+                PVE_REQUEST_COUNT_TOTAL.inc()
                 container_list = self._exclude(self.client.nodes(node).lxc.get())
             except Exception as e:  # noqa
+                PVE_REQUEST_COUNT_ERROR_TOTAL.inc()
                 raise APIError(str(e))
 
             # Merge QEMU and Containers lists from this node
             instances = self._get_variables(qemu_list, "qemu").copy()
             instances.update(self._get_variables(container_list, "container"))
 
+            HOST_GAUGE.set(len(instances))
             self.logger.info("Found {} targets".format(len(instances)))
             for host in instances:
                 host_meta = instances[host]
@@ -157,6 +178,7 @@ class Discovery():
                 except KeyError:
                     pve_type = "qemu"
 
+                PVE_REQUEST_COUNT_TOTAL.inc()
                 config = self.client.nodes(node).get(pve_type, vmid, "config")
 
                 try:
@@ -164,6 +186,7 @@ class Discovery():
                 except KeyError:
                     description = None
                 except Exception as e:  # noqa
+                    PVE_REQUEST_COUNT_ERROR_TOTAL.inc()
                     raise APIError(str(e))
 
                 try:

pyproject.toml

@@ -44,6 +44,7 @@ python = "^3.6.0"
 python-json-logger = "2.0.1"
 requests = "2.25.1"
 "ruamel.yaml" = "0.17.9"
+prometheus-client = "^0.11.0"
 
 [tool.poetry.dev-dependencies]
 bandit = "1.7.0"