feat: add options include_tags and include_vmid (#189)

This commit is contained in:
Mathias Petermann 2022-03-30 20:48:39 +02:00 committed by GitHub
parent 588643e5dd
commit 9c89d4e922
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 172 additions and 11 deletions

View File

@ -26,9 +26,11 @@ service: true
exclude_state: [] exclude_state: []
# needs to be a list of strings # needs to be a list of strings
exclude_vmid: [] exclude_vmid: []
include_vmid: []
# can be used to exclude vms by tags (proxmox 6+) # can be used to exclude vms by tags (proxmox 6+)
exclude_tags: [] exclude_tags: []
include_tags: []
pve: pve:
server: server:
@ -45,3 +47,7 @@ pve:
# auth_timeout: 5 # auth_timeout: 5
# verify_ssl: true # verify_ssl: true
``` ```
If `include_tags` and `exclude_tags` are set at the same time, the `exclude_tags` option takes precedence.
If `include_tags` is set, and your VM don't have any tags set, they will not show up!

View File

@ -26,6 +26,8 @@ PROMETHEUS_PVE_SD_SERVICE=true
PROMETHEUS_PVE_SD_EXCLUDE_STATE= PROMETHEUS_PVE_SD_EXCLUDE_STATE=
PROMETHEUS_PVE_SD_EXCLUDE_VMID= PROMETHEUS_PVE_SD_EXCLUDE_VMID=
PROMETHEUS_PVE_SD_INCLUDE_VMID=
PROMETHEUS_PVE_SD_PVE_SERVER= PROMETHEUS_PVE_SD_PVE_SERVER=
PROMETHEUS_PVE_SD_PVE_USER= PROMETHEUS_PVE_SD_PVE_USER=
PROMETHEUS_PVE_SD_PVE_PASSWORD= PROMETHEUS_PVE_SD_PVE_PASSWORD=

View File

@ -24,18 +24,19 @@ Tags of a node exposed by the `__meta_pve_tags` might be useful to build more co
**Example:** **Example:**
Extract `group` and `alert` from a list of tags like this: `__meta_pve_tags="alert:team-1,group:cluster-1,node:node-1"` Extract `group` and `alert` from a list of tags like this: `__meta_pve_tags="alert_team-1,group_cluster-1,node_node-1"`
```YAML ```YAML
relabel_configs: relabel_configs:
- source_labels: - source_labels:
- __meta_pve_tags - __meta_pve_tags
regex: ".*group:([\w\-_]*)" regex: ".*group_([\w\-_]*)"
target_label: "group" target_label: "group"
replacement: "${1}" replacement: "${1}"
- source_labels: - source_labels:
- __meta_pve_tags - __meta_pve_tags
regex: ".*alert:([\w\-_]*)" regex: ".*alert_([\w\-_]*)"
target_label: "alert" target_label: "alert"
replacement: "${1}" replacement: "${1}"
``` ```

View File

@ -69,7 +69,7 @@ If the static file is served by a web server, e.g. while using the [Prometheus O
```YAML ```YAML
- http_sd_configs: - http_sd_configs:
url: pve-sd-service:80/proxmox.json - url: http://pve-sd-service:80/proxmox.json
job_name: telegraf-pve job_name: telegraf-pve
metrics_path: /metrics metrics_path: /metrics
relabel_configs: relabel_configs:

View File

@ -106,6 +106,18 @@ class Config():
"file": True, "file": True,
"type": environs.Env().list "type": environs.Env().list
}, },
"include_vmid": {
"default": [],
"env": "INCLUDE_VMID",
"file": True,
"type": environs.Env().list
},
"include_tags": {
"default": [],
"env": "INCLUDE_TAGS",
"file": True,
"type": environs.Env().list
},
"pve.server": { "pve.server": {
"default": "", "default": "",
"env": "PVE_SERVER", "env": "PVE_SERVER",

View File

@ -144,10 +144,24 @@ class Discovery():
return ipv4_address, ipv6_address return ipv4_address, ipv6_address
def _exclude(self, pve_list): def _filter(self, pve_list):
filtered = [] filtered = []
for item in pve_list: for item in pve_list:
obj = defaultdict(dict, item) obj = defaultdict(dict, item)
if (
len(self.config.config["include_vmid"]) > 0
and str(obj["vmid"]) not in self.config.config["include_vmid"]
):
continue
if (
len(self.config.config["include_tags"]) > 0 and (
bool(obj["tags"]) is False # continue if tags is not set
or set(obj["tags"].split(",")).isdisjoint(self.config.config["include_tags"])
)
):
continue
if obj["template"] == 1: if obj["template"] == 1:
continue continue
@ -184,9 +198,9 @@ class Discovery():
for node in self._get_names(self.client.get("nodes"), "node"): for node in self._get_names(self.client.get("nodes"), "node"):
try: try:
PVE_REQUEST_COUNT_TOTAL.inc() PVE_REQUEST_COUNT_TOTAL.inc()
qemu_list = self._exclude(self.client.get("nodes", node, "qemu")) qemu_list = self._filter(self.client.get("nodes", node, "qemu"))
PVE_REQUEST_COUNT_TOTAL.inc() PVE_REQUEST_COUNT_TOTAL.inc()
container_list = self._exclude(self.client.get("nodes", node, "lxc")) container_list = self._filter(self.client.get("nodes", node, "lxc"))
except Exception as e: # noqa except Exception as e: # noqa
PVE_REQUEST_COUNT_ERROR_TOTAL.inc() PVE_REQUEST_COUNT_ERROR_TOTAL.inc()
raise APIError(str(e)) raise APIError(str(e))

66
prometheuspvesd/pve.json Normal file
View File

@ -0,0 +1,66 @@
[
{
"targets": [
"fhnw-syspr01"
],
"labels": {
"__meta_pve_ipv4": "10.112.153.88",
"__meta_pve_ipv6": null,
"__meta_pve_name": "fhnw-syspr01",
"__meta_pve_type": "qemu",
"__meta_pve_vmid": "103",
"__meta_pve_cpu": "1",
"__meta_pve_cores": "2",
"__meta_pve_memory": "2048",
"__meta_pve_status": "running"
}
},
{
"targets": [
"admin03"
],
"labels": {
"__meta_pve_ipv4": "10.112.153.83",
"__meta_pve_ipv6": null,
"__meta_pve_name": "admin03",
"__meta_pve_type": "qemu",
"__meta_pve_vmid": "102",
"__meta_pve_cpu": "1",
"__meta_pve_cores": "1",
"__meta_pve_memory": "1024",
"__meta_pve_status": "running"
}
},
{
"targets": [
"psql02"
],
"labels": {
"__meta_pve_ipv4": "10.112.153.91",
"__meta_pve_ipv6": null,
"__meta_pve_name": "psql02",
"__meta_pve_type": "qemu",
"__meta_pve_vmid": "100",
"__meta_pve_cpu": "1",
"__meta_pve_cores": "4",
"__meta_pve_memory": "4096",
"__meta_pve_status": "running"
}
},
{
"targets": [
"fhnw-gitlab01"
],
"labels": {
"__meta_pve_ipv4": "10.112.153.92",
"__meta_pve_ipv6": null,
"__meta_pve_name": "fhnw-gitlab01",
"__meta_pve_type": "qemu",
"__meta_pve_vmid": "104",
"__meta_pve_cpu": "1",
"__meta_pve_cores": "2",
"__meta_pve_memory": "2048",
"__meta_pve_status": "running"
}
}
]

View File

@ -17,6 +17,9 @@ exclude_state: []
exclude_vmid: [] exclude_vmid: []
exclude_tags: [] exclude_tags: []
include_vmid: []
include_tags: []
pve: pve:
server: proxmox.example.com server: proxmox.example.com
user: root user: root

View File

@ -84,6 +84,18 @@ def builtins():
"file": True, "file": True,
"type": environs.Env().list "type": environs.Env().list
}, },
"include_vmid": {
"default": [],
"env": "INCLUDE_VMID",
"file": True,
"type": environs.Env().list
},
"include_tags": {
"default": [],
"env": "INCLUDE_TAGS",
"file": True,
"type": environs.Env().list
},
"pve.server": { "pve.server": {
"default": "dummyserver", "default": "dummyserver",
"env": "PVE_SERVER", "env": "PVE_SERVER",
@ -123,6 +135,8 @@ def defaults():
"exclude_state": [], "exclude_state": [],
"exclude_tags": [], "exclude_tags": [],
"exclude_vmid": [], "exclude_vmid": [],
"include_tags": [],
"include_vmid": [],
"logging": { "logging": {
"format": "console", "format": "console",
"level": "WARNING" "level": "WARNING"
@ -166,7 +180,7 @@ def qemus():
"status": "running", "status": "running",
"netout": 12159205236, "netout": 12159205236,
"mem": 496179157, "mem": 496179157,
"tags": "unmonitored,excluded" "tags": "unmonitored,excluded,postgres"
}, },
{ {
"diskwrite": 0, "diskwrite": 0,

View File

@ -31,25 +31,68 @@ def get_mock(*args):
def test_exclude_vmid(discovery, qemus): def test_exclude_vmid(discovery, qemus):
discovery.config.config["exclude_vmid"] = ["100", "101", "102"] discovery.config.config["exclude_vmid"] = ["100", "101", "102"]
filtered = discovery._exclude(qemus) filtered = discovery._filter(qemus)
assert len(filtered) == 0 assert len(filtered) == 0
def test_exclude_state(discovery, qemus): def test_exclude_state(discovery, qemus):
discovery.config.config["exclude_state"] = ["prelaunch"] discovery.config.config["exclude_state"] = ["prelaunch"]
filtered = discovery._exclude(qemus) filtered = discovery._filter(qemus)
assert len(filtered) == 2 assert len(filtered) == 2
def test_exclude_tags(discovery, qemus): def test_exclude_tags(discovery, qemus):
discovery.config.config["exclude_tags"] = ["unmonitored"] discovery.config.config["exclude_tags"] = ["unmonitored"]
filtered = discovery._exclude(qemus) filtered = discovery._filter(qemus)
assert len(filtered) == 2 assert len(filtered) == 2
def test_include_tags(discovery, qemus):
discovery.config.config["include_tags"] = ["monitored"]
filtered = discovery._filter(qemus)
assert len(filtered) == 1
def test_include_tags_multiple(discovery, qemus):
discovery.config.config["include_tags"] = ["monitored", "postgres"]
filtered = discovery._filter(qemus)
assert len(filtered) == 2
def test_include_tags_empty(discovery, qemus):
discovery.config.config["include_tags"] = []
filtered = discovery._filter(qemus)
assert len(filtered) == 3
def test_include_vmid(discovery, qemus):
discovery.config.config["include_vmid"] = ["101", "100"]
filtered = discovery._filter(qemus)
assert len(filtered) == 2
def test_include_vmid_empty(discovery, qemus):
discovery.config.config["include_vmid"] = []
filtered = discovery._filter(qemus)
assert len(filtered) == 3
def test_include_and_exclude_tags(discovery, qemus):
discovery.config.config["include_tags"] = ["postgres"]
discovery.config.config["exclude_tags"] = ["unmonitored"]
filtered = discovery._filter(qemus)
assert len(filtered) == 0
def test_validate_ip(discovery, addresses): def test_validate_ip(discovery, addresses):
# IPv4 validation # IPv4 validation
for address in addresses["ipv4_valid"]: for address in addresses["ipv4_valid"]: