From 9c89d4e9227a49802be584781e2e6cfe32807069 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Mar 2022 20:48:39 +0200 Subject: [PATCH] feat: add options include_tags and include_vmid (#189) --- docs/content/configuration/defaults.md | 6 ++ docs/content/configuration/env.md | 2 + docs/content/usage/advanced.md | 7 ++- docs/content/usage/getting-started.md | 2 +- prometheuspvesd/config.py | 12 ++++ prometheuspvesd/discovery.py | 20 ++++++- prometheuspvesd/pve.json | 66 +++++++++++++++++++++ prometheuspvesd/test/data/config.yml | 3 + prometheuspvesd/test/fixtures/fixtures.py | 16 ++++- prometheuspvesd/test/unit/test_discovery.py | 49 ++++++++++++++- 10 files changed, 172 insertions(+), 11 deletions(-) create mode 100644 prometheuspvesd/pve.json diff --git a/docs/content/configuration/defaults.md b/docs/content/configuration/defaults.md index 940b0b3..4b8c7a9 100644 --- a/docs/content/configuration/defaults.md +++ b/docs/content/configuration/defaults.md @@ -26,9 +26,11 @@ service: true exclude_state: [] # needs to be a list of strings exclude_vmid: [] +include_vmid: [] # can be used to exclude vms by tags (proxmox 6+) exclude_tags: [] +include_tags: [] pve: server: @@ -45,3 +47,7 @@ pve: # auth_timeout: 5 # verify_ssl: true ``` + +If `include_tags` and `exclude_tags` are set at the same time, the `exclude_tags` option takes precedence. + +If `include_tags` is set, and your VM don't have any tags set, they will not show up! diff --git a/docs/content/configuration/env.md b/docs/content/configuration/env.md index ce74740..b384771 100644 --- a/docs/content/configuration/env.md +++ b/docs/content/configuration/env.md @@ -26,6 +26,8 @@ PROMETHEUS_PVE_SD_SERVICE=true PROMETHEUS_PVE_SD_EXCLUDE_STATE= PROMETHEUS_PVE_SD_EXCLUDE_VMID= +PROMETHEUS_PVE_SD_INCLUDE_VMID= + PROMETHEUS_PVE_SD_PVE_SERVER= PROMETHEUS_PVE_SD_PVE_USER= PROMETHEUS_PVE_SD_PVE_PASSWORD= diff --git a/docs/content/usage/advanced.md b/docs/content/usage/advanced.md index 5a11cad..3a3fd75 100644 --- a/docs/content/usage/advanced.md +++ b/docs/content/usage/advanced.md @@ -24,18 +24,19 @@ Tags of a node exposed by the `__meta_pve_tags` might be useful to build more co **Example:** -Extract `group` and `alert` from a list of tags like this: `__meta_pve_tags="alert:team-1,group:cluster-1,node:node-1"` +Extract `group` and `alert` from a list of tags like this: `__meta_pve_tags="alert_team-1,group_cluster-1,node_node-1"` ```YAML + relabel_configs: - source_labels: - __meta_pve_tags - regex: ".*group:([\w\-_]*)" + regex: ".*group_([\w\-_]*)" target_label: "group" replacement: "${1}" - source_labels: - __meta_pve_tags - regex: ".*alert:([\w\-_]*)" + regex: ".*alert_([\w\-_]*)" target_label: "alert" replacement: "${1}" ``` diff --git a/docs/content/usage/getting-started.md b/docs/content/usage/getting-started.md index 072c88d..acff11b 100644 --- a/docs/content/usage/getting-started.md +++ b/docs/content/usage/getting-started.md @@ -69,7 +69,7 @@ If the static file is served by a web server, e.g. while using the [Prometheus O ```YAML - http_sd_configs: - url: pve-sd-service:80/proxmox.json + - url: http://pve-sd-service:80/proxmox.json job_name: telegraf-pve metrics_path: /metrics relabel_configs: diff --git a/prometheuspvesd/config.py b/prometheuspvesd/config.py index 973c0cd..088fc70 100644 --- a/prometheuspvesd/config.py +++ b/prometheuspvesd/config.py @@ -106,6 +106,18 @@ class Config(): "file": True, "type": environs.Env().list }, + "include_vmid": { + "default": [], + "env": "INCLUDE_VMID", + "file": True, + "type": environs.Env().list + }, + "include_tags": { + "default": [], + "env": "INCLUDE_TAGS", + "file": True, + "type": environs.Env().list + }, "pve.server": { "default": "", "env": "PVE_SERVER", diff --git a/prometheuspvesd/discovery.py b/prometheuspvesd/discovery.py index f0253bf..a69dce7 100644 --- a/prometheuspvesd/discovery.py +++ b/prometheuspvesd/discovery.py @@ -144,10 +144,24 @@ class Discovery(): return ipv4_address, ipv6_address - def _exclude(self, pve_list): + def _filter(self, pve_list): filtered = [] for item in pve_list: obj = defaultdict(dict, item) + if ( + len(self.config.config["include_vmid"]) > 0 + and str(obj["vmid"]) not in self.config.config["include_vmid"] + ): + continue + + if ( + len(self.config.config["include_tags"]) > 0 and ( + bool(obj["tags"]) is False # continue if tags is not set + or set(obj["tags"].split(",")).isdisjoint(self.config.config["include_tags"]) + ) + ): + continue + if obj["template"] == 1: continue @@ -184,9 +198,9 @@ class Discovery(): for node in self._get_names(self.client.get("nodes"), "node"): try: PVE_REQUEST_COUNT_TOTAL.inc() - qemu_list = self._exclude(self.client.get("nodes", node, "qemu")) + qemu_list = self._filter(self.client.get("nodes", node, "qemu")) PVE_REQUEST_COUNT_TOTAL.inc() - container_list = self._exclude(self.client.get("nodes", node, "lxc")) + container_list = self._filter(self.client.get("nodes", node, "lxc")) except Exception as e: # noqa PVE_REQUEST_COUNT_ERROR_TOTAL.inc() raise APIError(str(e)) diff --git a/prometheuspvesd/pve.json b/prometheuspvesd/pve.json new file mode 100644 index 0000000..6e4212e --- /dev/null +++ b/prometheuspvesd/pve.json @@ -0,0 +1,66 @@ +[ + { + "targets": [ + "fhnw-syspr01" + ], + "labels": { + "__meta_pve_ipv4": "10.112.153.88", + "__meta_pve_ipv6": null, + "__meta_pve_name": "fhnw-syspr01", + "__meta_pve_type": "qemu", + "__meta_pve_vmid": "103", + "__meta_pve_cpu": "1", + "__meta_pve_cores": "2", + "__meta_pve_memory": "2048", + "__meta_pve_status": "running" + } + }, + { + "targets": [ + "admin03" + ], + "labels": { + "__meta_pve_ipv4": "10.112.153.83", + "__meta_pve_ipv6": null, + "__meta_pve_name": "admin03", + "__meta_pve_type": "qemu", + "__meta_pve_vmid": "102", + "__meta_pve_cpu": "1", + "__meta_pve_cores": "1", + "__meta_pve_memory": "1024", + "__meta_pve_status": "running" + } + }, + { + "targets": [ + "psql02" + ], + "labels": { + "__meta_pve_ipv4": "10.112.153.91", + "__meta_pve_ipv6": null, + "__meta_pve_name": "psql02", + "__meta_pve_type": "qemu", + "__meta_pve_vmid": "100", + "__meta_pve_cpu": "1", + "__meta_pve_cores": "4", + "__meta_pve_memory": "4096", + "__meta_pve_status": "running" + } + }, + { + "targets": [ + "fhnw-gitlab01" + ], + "labels": { + "__meta_pve_ipv4": "10.112.153.92", + "__meta_pve_ipv6": null, + "__meta_pve_name": "fhnw-gitlab01", + "__meta_pve_type": "qemu", + "__meta_pve_vmid": "104", + "__meta_pve_cpu": "1", + "__meta_pve_cores": "2", + "__meta_pve_memory": "2048", + "__meta_pve_status": "running" + } + } +] \ No newline at end of file diff --git a/prometheuspvesd/test/data/config.yml b/prometheuspvesd/test/data/config.yml index b295ef2..94fe5b0 100644 --- a/prometheuspvesd/test/data/config.yml +++ b/prometheuspvesd/test/data/config.yml @@ -17,6 +17,9 @@ exclude_state: [] exclude_vmid: [] exclude_tags: [] +include_vmid: [] +include_tags: [] + pve: server: proxmox.example.com user: root diff --git a/prometheuspvesd/test/fixtures/fixtures.py b/prometheuspvesd/test/fixtures/fixtures.py index 4591b71..2540403 100644 --- a/prometheuspvesd/test/fixtures/fixtures.py +++ b/prometheuspvesd/test/fixtures/fixtures.py @@ -84,6 +84,18 @@ def builtins(): "file": True, "type": environs.Env().list }, + "include_vmid": { + "default": [], + "env": "INCLUDE_VMID", + "file": True, + "type": environs.Env().list + }, + "include_tags": { + "default": [], + "env": "INCLUDE_TAGS", + "file": True, + "type": environs.Env().list + }, "pve.server": { "default": "dummyserver", "env": "PVE_SERVER", @@ -123,6 +135,8 @@ def defaults(): "exclude_state": [], "exclude_tags": [], "exclude_vmid": [], + "include_tags": [], + "include_vmid": [], "logging": { "format": "console", "level": "WARNING" @@ -166,7 +180,7 @@ def qemus(): "status": "running", "netout": 12159205236, "mem": 496179157, - "tags": "unmonitored,excluded" + "tags": "unmonitored,excluded,postgres" }, { "diskwrite": 0, diff --git a/prometheuspvesd/test/unit/test_discovery.py b/prometheuspvesd/test/unit/test_discovery.py index 4e163ad..eaa4f38 100644 --- a/prometheuspvesd/test/unit/test_discovery.py +++ b/prometheuspvesd/test/unit/test_discovery.py @@ -31,25 +31,68 @@ def get_mock(*args): def test_exclude_vmid(discovery, qemus): discovery.config.config["exclude_vmid"] = ["100", "101", "102"] - filtered = discovery._exclude(qemus) + filtered = discovery._filter(qemus) assert len(filtered) == 0 def test_exclude_state(discovery, qemus): discovery.config.config["exclude_state"] = ["prelaunch"] - filtered = discovery._exclude(qemus) + filtered = discovery._filter(qemus) assert len(filtered) == 2 def test_exclude_tags(discovery, qemus): discovery.config.config["exclude_tags"] = ["unmonitored"] - filtered = discovery._exclude(qemus) + filtered = discovery._filter(qemus) assert len(filtered) == 2 +def test_include_tags(discovery, qemus): + discovery.config.config["include_tags"] = ["monitored"] + filtered = discovery._filter(qemus) + + assert len(filtered) == 1 + + +def test_include_tags_multiple(discovery, qemus): + discovery.config.config["include_tags"] = ["monitored", "postgres"] + filtered = discovery._filter(qemus) + + assert len(filtered) == 2 + + +def test_include_tags_empty(discovery, qemus): + discovery.config.config["include_tags"] = [] + filtered = discovery._filter(qemus) + + assert len(filtered) == 3 + + +def test_include_vmid(discovery, qemus): + discovery.config.config["include_vmid"] = ["101", "100"] + filtered = discovery._filter(qemus) + + assert len(filtered) == 2 + + +def test_include_vmid_empty(discovery, qemus): + discovery.config.config["include_vmid"] = [] + filtered = discovery._filter(qemus) + + assert len(filtered) == 3 + + +def test_include_and_exclude_tags(discovery, qemus): + discovery.config.config["include_tags"] = ["postgres"] + discovery.config.config["exclude_tags"] = ["unmonitored"] + filtered = discovery._filter(qemus) + + assert len(filtered) == 0 + + def test_validate_ip(discovery, addresses): # IPv4 validation for address in addresses["ipv4_valid"]: