diff --git a/docs/content/configuration/defaults.md b/docs/content/configuration/defaults.md index 4a0394a..a8754e1 100644 --- a/docs/content/configuration/defaults.md +++ b/docs/content/configuration/defaults.md @@ -13,6 +13,11 @@ logging: # supported log formats: console|json|simple format: console +metrics: + enabled: true + address: "127.0.0.1" + port: 8000 + output_file: loop_delay: 300 # Run pve sd in a loop and discover hosts every n seconds (as defined in loop_delay). diff --git a/docs/content/configuration/env.md b/docs/content/configuration/env.md index 236481b..11dec4c 100644 --- a/docs/content/configuration/env.md +++ b/docs/content/configuration/env.md @@ -7,17 +7,26 @@ title: Environment Variables {{< highlight Shell "linenos=table" >}} PROMETHEUS_PVE_SD_CONFIG_FILE= + # supported log levels: debug|info|warning|error|critical PROMETHEUS_PVE_SD_LOG_LEVEL=warning # supported log formats: console|json|simple PROMETHEUS_PVE_SD_LOG_FORMAT=console + +METRICS_ENABLED=true +METRICS_ADDRESS=127.0.01 +METRICS_PORT=8000 + PROMETHEUS_PVE_SD_OUTPUT_FILE= PROMETHEUS_PVE_SD_LOOP_DELAY=300 + # Run pve sd in a loop and discover hosts every n seconds (as defined in PROMETHEUS_PVE_SD_LOOP_DELAY). # Can be disabled to run disovery only once. PROMETHEUS_PVE_SD_SERVICE=true + PROMETHEUS_PVE_SD_EXCLUDE_STATE= PROMETHEUS_PVE_SD_EXCLUDE_VMID= + PROMETHEUS_PVE_SD_PVE_SERVER= PROMETHEUS_PVE_SD_PVE_USER= PROMETHEUS_PVE_SD_PVE_PASSWORD= diff --git a/poetry.lock b/poetry.lock index 029922c..8e6a7b1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -404,6 +404,17 @@ importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] dev = ["pre-commit", "tox"] +[[package]] +name = "prometheus-client" +version = "0.11.0" +description = "Python client for the Prometheus monitoring system." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +twisted = ["twisted"] + [[package]] name = "proxmoxer" version = "1.1.1" @@ -687,7 +698,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt [metadata] lock-version = "1.1" python-versions = "^3.6.0" -content-hash = "d22e186f254a2c3c272eab9e38414221eec6eed25aa874f93b561faf5855a8e8" +content-hash = "c801a2419674bd6da0ccd03908adb40371db22b727790f8e89053c39041fb4dd" [metadata.files] anyconfig = [ @@ -875,6 +886,10 @@ pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] +prometheus-client = [ + {file = "prometheus_client-0.11.0-py2.py3-none-any.whl", hash = "sha256:b014bc76815eb1399da8ce5fc84b7717a3e63652b0c0f8804092c9363acab1b2"}, + {file = "prometheus_client-0.11.0.tar.gz", hash = "sha256:3a8baade6cb80bcfe43297e33e7623f3118d660d41387593758e2fb1ea173a86"}, +] proxmoxer = [ {file = "proxmoxer-1.1.1.tar.gz", hash = "sha256:684a69190129da0f102703fc9861f5eea82a7d804f9f96d35c7fd73452f1da7e"}, ] @@ -927,26 +942,18 @@ pyyaml = [ {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, @@ -967,29 +974,22 @@ requests = [ {file = "ruamel.yaml.clib-0.2.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:73b3d43e04cc4b228fa6fa5d796409ece6fcb53a6c270eb2048109cbcbc3b9c2"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:53b9dd1abd70e257a6e32f934ebc482dac5edb8c93e23deb663eac724c30b026"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:839dd72545ef7ba78fd2aa1a5dd07b33696adf3e68fae7f31327161c1093001b"}, - {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1236df55e0f73cd138c0eca074ee086136c3f16a97c2ac719032c050f7e0622f"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win32.whl", hash = "sha256:b1e981fe1aff1fd11627f531524826a4dcc1f26c726235a52fcb62ded27d150f"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4e52c96ca66de04be42ea2278012a2342d89f5e82b4512fb6fb7134e377e2e62"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a873e4d4954f865dcb60bdc4914af7eaae48fb56b60ed6daa1d6251c72f5337c"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ab845f1f51f7eb750a78937be9f79baea4a42c7960f5a94dde34e69f3cce1988"}, - {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:2fd336a5c6415c82e2deb40d08c222087febe0aebe520f4d21910629018ab0f3"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win32.whl", hash = "sha256:e9f7d1d8c26a6a12c23421061f9022bb62704e38211fe375c645485f38df34a2"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:2602e91bd5c1b874d6f93d3086f9830f3e907c543c7672cf293a97c3fabdcd91"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:44c7b0498c39f27795224438f1a6be6c5352f82cb887bc33d962c3a3acc00df6"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8e8fd0a22c9d92af3a34f91e8a2594eeb35cba90ab643c5e0e643567dc8be43e"}, - {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:75f0ee6839532e52a3a53f80ce64925ed4aed697dd3fa890c4c918f3304bd4f4"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win32.whl", hash = "sha256:464e66a04e740d754170be5e740657a3b3b6d2bcc567f0c3437879a6e6087ff6"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:52ae5739e4b5d6317b52f5b040b1b6639e8af68a5b8fd606a8b08658fbd0cab5"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4df5019e7783d14b79217ad9c56edf1ba7485d614ad5a385d1b3c768635c81c0"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5254af7d8bdf4d5484c089f929cb7f5bafa59b4f01d4f48adda4be41e6d29f99"}, - {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8be05be57dc5c7b4a0b24edcaa2f7275866d9c907725226cdde46da09367d923"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win32.whl", hash = "sha256:74161d827407f4db9072011adcfb825b5258a5ccb3d2cd518dd6c9edea9e30f1"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:058a1cc3df2a8aecc12f983a48bda99315cebf55a3b3a5463e37bb599b05727b"}, {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c6ac7e45367b1317e56f1461719c853fd6825226f45b835df7436bb04031fd8a"}, {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:b4b0d31f2052b3f9f9b5327024dc629a253a83d8649d4734ca7f35b60ec3e9e5"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1f8c0a4577c0e6c99d208de5c4d3fd8aceed9574bb154d7a2b21c16bb924154c"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win32.whl", hash = "sha256:46d6d20815064e8bb023ea8628cfb7402c0f0e83de2c2227a88097e239a7dffd"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:6c0a5dc52fc74eb87c67374a4e554d4761fd42a4d01390b7e868b30d21f4b8bb"}, {file = "ruamel.yaml.clib-0.2.2.tar.gz", hash = "sha256:2d24bd98af676f4990c4d715bcdc2a60b19c56a3fb3a763164d2d8ca0e806ba7"}, ] six = [ diff --git a/prometheuspvesd/cli.py b/prometheuspvesd/cli.py index a941888..e1e15a5 100644 --- a/prometheuspvesd/cli.py +++ b/prometheuspvesd/cli.py @@ -8,6 +8,8 @@ import signal import tempfile from time import sleep +from prometheus_client import start_http_server + import prometheuspvesd.exception from prometheuspvesd import __version__ from prometheuspvesd.config import SingleConfig @@ -119,12 +121,20 @@ class PrometheusSD: return config def _fetch(self): - loop_delay = self.config.config["loop_delay"] - output_file = self.config.config["output_file"] - - self.logger.info("Writes targets to {}".format(output_file)) + self.logger.info("Writes targets to {}".format(self.config.config["output_file"])) self.logger.debug("Propagate from PVE") + if self.config.config["service"] and self.config.config["metrics"]["enabled"]: + self.logger.info( + "Starting metrics http endpoint on port {}".format( + self.config.config["metrics"]["port"] + ) + ) + start_http_server( + self.config.config["metrics"]["port"], + addr=self.config.config["metrics"]["address"] + ) + while True: try: inventory = self.discovery.propagate() @@ -138,7 +148,11 @@ class PrometheusSD: if not self.config.config["service"]: break - self.logger.info("Waiting {} seconds for next discovery loop".format(loop_delay)) + self.logger.info( + "Waiting {} seconds for next discovery loop".format( + self.config.config["loop_delay"] + ) + ) sleep(self.config.config["loop_delay"]) def _write(self, host_list: HostList): diff --git a/prometheuspvesd/config.py b/prometheuspvesd/config.py index 0b8aff5..de975d8 100644 --- a/prometheuspvesd/config.py +++ b/prometheuspvesd/config.py @@ -32,6 +32,21 @@ class Config(): """ SETTINGS = { + "metrics.enabled": { + "default": True, + "env": "METRICS_ENABLED", + "type": environs.Env().bool + }, + "metrics.address": { + "default": "127.0.0.1", + "env": "METRICS_ADDRESS", + "type": environs.Env().str + }, + "metrics.port": { + "default": 8000, + "env": "METRICS_PORT", + "type": environs.Env().int + }, "config_file": { "default": "", "env": "CONFIG_FILE", diff --git a/prometheuspvesd/discovery.py b/prometheuspvesd/discovery.py index 52f86be..17e825f 100644 --- a/prometheuspvesd/discovery.py +++ b/prometheuspvesd/discovery.py @@ -7,6 +7,9 @@ import socket from collections import defaultdict import requests +from prometheus_client import Counter +from prometheus_client import Gauge +from prometheus_client import Summary from prometheuspvesd.config import SingleConfig from prometheuspvesd.exception import APIError @@ -21,6 +24,15 @@ try: except ImportError: HAS_PROXMOXER = False +PROPAGATION_TIME = Summary( + "pve_sd_propagate_seconds", "Time spent propagating the inventory from PVE" +) +HOST_GAUGE = Gauge("pve_sd_hosts", "Number of hosts discovered by PVE SD") +PVE_REQUEST_COUNT_TOTAL = Counter("pve_sd_requests_total", "Total count of requests to PVE API") +PVE_REQUEST_COUNT_ERROR_TOTAL = Counter( + "pve_sd_requests_error_total", "Total count of failed requests to PVE API" +) + class Discovery(): """Prometheus PVE Service Discovery.""" @@ -48,6 +60,7 @@ class Discovery(): timeout=self.config.config["pve"]["auth_timeout"] ) except requests.RequestException as e: + PVE_REQUEST_COUNT_ERROR_TOTAL.inc() raise APIError(str(e)) def _get_names(self, pve_list, pve_type): @@ -89,6 +102,7 @@ class Discovery(): if pve_type == "qemu": # If qemu agent is enabled, try to gather the IP address try: + PVE_REQUEST_COUNT_TOTAL.inc() if self.client.nodes(pve_node).get(pve_type, vmid, "agent", "info") is not None: networks = self.client.nodes(pve_node).get( "qemu", vmid, "agent", "network-get-interfaces" @@ -104,6 +118,7 @@ class Discovery(): if not address: try: + PVE_REQUEST_COUNT_TOTAL.inc() config = self.client.nodes(pve_node).get(pve_type, vmid, "config") sources = [config["net0"], config["ipconfig0"]] @@ -133,20 +148,26 @@ class Discovery(): filtered.append(item.copy()) return filtered + @PROPAGATION_TIME.time() def propagate(self): self.host_list.clear() + PVE_REQUEST_COUNT_TOTAL.inc() for node in self._get_names(self.client.nodes.get(), "node"): try: + PVE_REQUEST_COUNT_TOTAL.inc() qemu_list = self._exclude(self.client.nodes(node).qemu.get()) + PVE_REQUEST_COUNT_TOTAL.inc() container_list = self._exclude(self.client.nodes(node).lxc.get()) except Exception as e: # noqa + PVE_REQUEST_COUNT_ERROR_TOTAL.inc() raise APIError(str(e)) # Merge QEMU and Containers lists from this node instances = self._get_variables(qemu_list, "qemu").copy() instances.update(self._get_variables(container_list, "container")) + HOST_GAUGE.set(len(instances)) self.logger.info("Found {} targets".format(len(instances))) for host in instances: host_meta = instances[host] @@ -157,6 +178,7 @@ class Discovery(): except KeyError: pve_type = "qemu" + PVE_REQUEST_COUNT_TOTAL.inc() config = self.client.nodes(node).get(pve_type, vmid, "config") try: @@ -164,6 +186,7 @@ class Discovery(): except KeyError: description = None except Exception as e: # noqa + PVE_REQUEST_COUNT_ERROR_TOTAL.inc() raise APIError(str(e)) try: diff --git a/pyproject.toml b/pyproject.toml index 9053b19..ac5ecd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ colorama = "0.4.4" environs = "9.3.2" jsonschema = "3.2.0" nested-lookup = "0.2.22" +prometheus-client = "0.11.0" proxmoxer = "1.1.1" python = "^3.6.0" python-json-logger = "2.0.1"