feat: add service discovery metrics

This commit is contained in:
Robert Kaussow 2021-06-15 22:14:40 +02:00
parent 9d90c3b787
commit cbb62d791b
Signed by: xoxys
GPG Key ID: 4E692A2EAECC03C0
7 changed files with 88 additions and 21 deletions

View File

@ -13,6 +13,11 @@ logging:
# supported log formats: console|json|simple
format: console
metrics:
enabled: true
address: "127.0.0.1"
port: 8000
output_file:
loop_delay: 300
# Run pve sd in a loop and discover hosts every n seconds (as defined in loop_delay).

View File

@ -7,17 +7,26 @@ title: Environment Variables
<!-- spellchecker-disable -->
{{< highlight Shell "linenos=table" >}}
PROMETHEUS_PVE_SD_CONFIG_FILE=
# supported log levels: debug|info|warning|error|critical
PROMETHEUS_PVE_SD_LOG_LEVEL=warning
# supported log formats: console|json|simple
PROMETHEUS_PVE_SD_LOG_FORMAT=console
METRICS_ENABLED=true
METRICS_ADDRESS=127.0.01
METRICS_PORT=8000
PROMETHEUS_PVE_SD_OUTPUT_FILE=
PROMETHEUS_PVE_SD_LOOP_DELAY=300
# Run pve sd in a loop and discover hosts every n seconds (as defined in PROMETHEUS_PVE_SD_LOOP_DELAY).
# Can be disabled to run disovery only once.
PROMETHEUS_PVE_SD_SERVICE=true
PROMETHEUS_PVE_SD_EXCLUDE_STATE=
PROMETHEUS_PVE_SD_EXCLUDE_VMID=
PROMETHEUS_PVE_SD_PVE_SERVER=
PROMETHEUS_PVE_SD_PVE_USER=
PROMETHEUS_PVE_SD_PVE_PASSWORD=

32
poetry.lock generated
View File

@ -404,6 +404,17 @@ importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""}
[package.extras]
dev = ["pre-commit", "tox"]
[[package]]
name = "prometheus-client"
version = "0.11.0"
description = "Python client for the Prometheus monitoring system."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[package.extras]
twisted = ["twisted"]
[[package]]
name = "proxmoxer"
version = "1.1.1"
@ -687,7 +698,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt
[metadata]
lock-version = "1.1"
python-versions = "^3.6.0"
content-hash = "d22e186f254a2c3c272eab9e38414221eec6eed25aa874f93b561faf5855a8e8"
content-hash = "c801a2419674bd6da0ccd03908adb40371db22b727790f8e89053c39041fb4dd"
[metadata.files]
anyconfig = [
@ -875,6 +886,10 @@ pluggy = [
{file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"},
{file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"},
]
prometheus-client = [
{file = "prometheus_client-0.11.0-py2.py3-none-any.whl", hash = "sha256:b014bc76815eb1399da8ce5fc84b7717a3e63652b0c0f8804092c9363acab1b2"},
{file = "prometheus_client-0.11.0.tar.gz", hash = "sha256:3a8baade6cb80bcfe43297e33e7623f3118d660d41387593758e2fb1ea173a86"},
]
proxmoxer = [
{file = "proxmoxer-1.1.1.tar.gz", hash = "sha256:684a69190129da0f102703fc9861f5eea82a7d804f9f96d35c7fd73452f1da7e"},
]
@ -927,26 +942,18 @@ pyyaml = [
{file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"},
{file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"},
{file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"},
{file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"},
{file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"},
{file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"},
{file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"},
{file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"},
{file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"},
{file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"},
{file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"},
{file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"},
{file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"},
{file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"},
{file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"},
{file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"},
{file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"},
{file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"},
{file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"},
{file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"},
{file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"},
{file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"},
{file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"},
{file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"},
{file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"},
{file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"},
@ -967,29 +974,22 @@ requests = [
{file = "ruamel.yaml.clib-0.2.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:73b3d43e04cc4b228fa6fa5d796409ece6fcb53a6c270eb2048109cbcbc3b9c2"},
{file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:53b9dd1abd70e257a6e32f934ebc482dac5edb8c93e23deb663eac724c30b026"},
{file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:839dd72545ef7ba78fd2aa1a5dd07b33696adf3e68fae7f31327161c1093001b"},
{file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1236df55e0f73cd138c0eca074ee086136c3f16a97c2ac719032c050f7e0622f"},
{file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win32.whl", hash = "sha256:b1e981fe1aff1fd11627f531524826a4dcc1f26c726235a52fcb62ded27d150f"},
{file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4e52c96ca66de04be42ea2278012a2342d89f5e82b4512fb6fb7134e377e2e62"},
{file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a873e4d4954f865dcb60bdc4914af7eaae48fb56b60ed6daa1d6251c72f5337c"},
{file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ab845f1f51f7eb750a78937be9f79baea4a42c7960f5a94dde34e69f3cce1988"},
{file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:2fd336a5c6415c82e2deb40d08c222087febe0aebe520f4d21910629018ab0f3"},
{file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win32.whl", hash = "sha256:e9f7d1d8c26a6a12c23421061f9022bb62704e38211fe375c645485f38df34a2"},
{file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:2602e91bd5c1b874d6f93d3086f9830f3e907c543c7672cf293a97c3fabdcd91"},
{file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:44c7b0498c39f27795224438f1a6be6c5352f82cb887bc33d962c3a3acc00df6"},
{file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8e8fd0a22c9d92af3a34f91e8a2594eeb35cba90ab643c5e0e643567dc8be43e"},
{file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:75f0ee6839532e52a3a53f80ce64925ed4aed697dd3fa890c4c918f3304bd4f4"},
{file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win32.whl", hash = "sha256:464e66a04e740d754170be5e740657a3b3b6d2bcc567f0c3437879a6e6087ff6"},
{file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:52ae5739e4b5d6317b52f5b040b1b6639e8af68a5b8fd606a8b08658fbd0cab5"},
{file = "ruamel.yaml.clib-0.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4df5019e7783d14b79217ad9c56edf1ba7485d614ad5a385d1b3c768635c81c0"},
{file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5254af7d8bdf4d5484c089f929cb7f5bafa59b4f01d4f48adda4be41e6d29f99"},
{file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8be05be57dc5c7b4a0b24edcaa2f7275866d9c907725226cdde46da09367d923"},
{file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win32.whl", hash = "sha256:74161d827407f4db9072011adcfb825b5258a5ccb3d2cd518dd6c9edea9e30f1"},
{file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:058a1cc3df2a8aecc12f983a48bda99315cebf55a3b3a5463e37bb599b05727b"},
{file = "ruamel.yaml.clib-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c6ac7e45367b1317e56f1461719c853fd6825226f45b835df7436bb04031fd8a"},
{file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:b4b0d31f2052b3f9f9b5327024dc629a253a83d8649d4734ca7f35b60ec3e9e5"},
{file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1f8c0a4577c0e6c99d208de5c4d3fd8aceed9574bb154d7a2b21c16bb924154c"},
{file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win32.whl", hash = "sha256:46d6d20815064e8bb023ea8628cfb7402c0f0e83de2c2227a88097e239a7dffd"},
{file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:6c0a5dc52fc74eb87c67374a4e554d4761fd42a4d01390b7e868b30d21f4b8bb"},
{file = "ruamel.yaml.clib-0.2.2.tar.gz", hash = "sha256:2d24bd98af676f4990c4d715bcdc2a60b19c56a3fb3a763164d2d8ca0e806ba7"},
]
six = [

View File

@ -8,6 +8,8 @@ import signal
import tempfile
from time import sleep
from prometheus_client import start_http_server
import prometheuspvesd.exception
from prometheuspvesd import __version__
from prometheuspvesd.config import SingleConfig
@ -119,12 +121,20 @@ class PrometheusSD:
return config
def _fetch(self):
loop_delay = self.config.config["loop_delay"]
output_file = self.config.config["output_file"]
self.logger.info("Writes targets to {}".format(output_file))
self.logger.info("Writes targets to {}".format(self.config.config["output_file"]))
self.logger.debug("Propagate from PVE")
if self.config.config["service"] and self.config.config["metrics"]["enabled"]:
self.logger.info(
"Starting metrics http endpoint on port {}".format(
self.config.config["metrics"]["port"]
)
)
start_http_server(
self.config.config["metrics"]["port"],
addr=self.config.config["metrics"]["address"]
)
while True:
try:
inventory = self.discovery.propagate()
@ -138,7 +148,11 @@ class PrometheusSD:
if not self.config.config["service"]:
break
self.logger.info("Waiting {} seconds for next discovery loop".format(loop_delay))
self.logger.info(
"Waiting {} seconds for next discovery loop".format(
self.config.config["loop_delay"]
)
)
sleep(self.config.config["loop_delay"])
def _write(self, host_list: HostList):

View File

@ -32,6 +32,21 @@ class Config():
"""
SETTINGS = {
"metrics.enabled": {
"default": True,
"env": "METRICS_ENABLED",
"type": environs.Env().bool
},
"metrics.address": {
"default": "127.0.0.1",
"env": "METRICS_ADDRESS",
"type": environs.Env().str
},
"metrics.port": {
"default": 8000,
"env": "METRICS_PORT",
"type": environs.Env().int
},
"config_file": {
"default": "",
"env": "CONFIG_FILE",

View File

@ -7,6 +7,9 @@ import socket
from collections import defaultdict
import requests
from prometheus_client import Counter
from prometheus_client import Gauge
from prometheus_client import Summary
from prometheuspvesd.config import SingleConfig
from prometheuspvesd.exception import APIError
@ -21,6 +24,15 @@ try:
except ImportError:
HAS_PROXMOXER = False
PROPAGATION_TIME = Summary(
"pve_sd_propagate_seconds", "Time spent propagating the inventory from PVE"
)
HOST_GAUGE = Gauge("pve_sd_hosts", "Number of hosts discovered by PVE SD")
PVE_REQUEST_COUNT_TOTAL = Counter("pve_sd_requests_total", "Total count of requests to PVE API")
PVE_REQUEST_COUNT_ERROR_TOTAL = Counter(
"pve_sd_requests_error_total", "Total count of failed requests to PVE API"
)
class Discovery():
"""Prometheus PVE Service Discovery."""
@ -48,6 +60,7 @@ class Discovery():
timeout=self.config.config["pve"]["auth_timeout"]
)
except requests.RequestException as e:
PVE_REQUEST_COUNT_ERROR_TOTAL.inc()
raise APIError(str(e))
def _get_names(self, pve_list, pve_type):
@ -89,6 +102,7 @@ class Discovery():
if pve_type == "qemu":
# If qemu agent is enabled, try to gather the IP address
try:
PVE_REQUEST_COUNT_TOTAL.inc()
if self.client.nodes(pve_node).get(pve_type, vmid, "agent", "info") is not None:
networks = self.client.nodes(pve_node).get(
"qemu", vmid, "agent", "network-get-interfaces"
@ -104,6 +118,7 @@ class Discovery():
if not address:
try:
PVE_REQUEST_COUNT_TOTAL.inc()
config = self.client.nodes(pve_node).get(pve_type, vmid, "config")
sources = [config["net0"], config["ipconfig0"]]
@ -133,20 +148,26 @@ class Discovery():
filtered.append(item.copy())
return filtered
@PROPAGATION_TIME.time()
def propagate(self):
self.host_list.clear()
PVE_REQUEST_COUNT_TOTAL.inc()
for node in self._get_names(self.client.nodes.get(), "node"):
try:
PVE_REQUEST_COUNT_TOTAL.inc()
qemu_list = self._exclude(self.client.nodes(node).qemu.get())
PVE_REQUEST_COUNT_TOTAL.inc()
container_list = self._exclude(self.client.nodes(node).lxc.get())
except Exception as e: # noqa
PVE_REQUEST_COUNT_ERROR_TOTAL.inc()
raise APIError(str(e))
# Merge QEMU and Containers lists from this node
instances = self._get_variables(qemu_list, "qemu").copy()
instances.update(self._get_variables(container_list, "container"))
HOST_GAUGE.set(len(instances))
self.logger.info("Found {} targets".format(len(instances)))
for host in instances:
host_meta = instances[host]
@ -157,6 +178,7 @@ class Discovery():
except KeyError:
pve_type = "qemu"
PVE_REQUEST_COUNT_TOTAL.inc()
config = self.client.nodes(node).get(pve_type, vmid, "config")
try:
@ -164,6 +186,7 @@ class Discovery():
except KeyError:
description = None
except Exception as e: # noqa
PVE_REQUEST_COUNT_ERROR_TOTAL.inc()
raise APIError(str(e))
try:

View File

@ -44,6 +44,7 @@ python = "^3.6.0"
python-json-logger = "2.0.1"
requests = "2.25.1"
"ruamel.yaml" = "0.17.9"
prometheus-client = "^0.11.0"
[tool.poetry.dev-dependencies]
bandit = "1.7.0"