feat: add options include_tags and include_vmid (#189)

docs/content/configuration/defaults.md

@@ -26,9 +26,11 @@ service: true
 exclude_state: []
 # needs to be a list of strings
 exclude_vmid: []
+include_vmid: []
 
 # can be used to exclude vms by tags (proxmox 6+)
 exclude_tags: []
+include_tags: []
 
 pve:
     server:
@@ -45,3 +47,7 @@ pve:
 #     auth_timeout: 5
 #     verify_ssl: true
 ```
+
+If `include_tags` and `exclude_tags` are set at the same time, the `exclude_tags` option takes precedence.
+
+If `include_tags` is set, and your VM don't have any tags set, they will not show up!

docs/content/configuration/env.md

@@ -26,6 +26,8 @@ PROMETHEUS_PVE_SD_SERVICE=true
 PROMETHEUS_PVE_SD_EXCLUDE_STATE=
 PROMETHEUS_PVE_SD_EXCLUDE_VMID=
 
+PROMETHEUS_PVE_SD_INCLUDE_VMID=
+
 PROMETHEUS_PVE_SD_PVE_SERVER=
 PROMETHEUS_PVE_SD_PVE_USER=
 PROMETHEUS_PVE_SD_PVE_PASSWORD=

docs/content/usage/advanced.md

@@ -24,18 +24,19 @@ Tags of a node exposed by the `__meta_pve_tags` might be useful to build more co
 
 **Example:**
 
-Extract `group` and `alert` from a list of tags like this: `__meta_pve_tags="alert:team-1,group:cluster-1,node:node-1"`
+Extract `group` and `alert` from a list of tags like this: `__meta_pve_tags="alert_team-1,group_cluster-1,node_node-1"`
 
 ```YAML
+
 relabel_configs:
 - source_labels:
   - __meta_pve_tags
-  regex: ".*group:([\w\-_]*)"
+  regex: ".*group_([\w\-_]*)"
   target_label: "group"
   replacement: "${1}"
 - source_labels:
   - __meta_pve_tags
-  regex: ".*alert:([\w\-_]*)"
+  regex: ".*alert_([\w\-_]*)"
   target_label: "alert"
   replacement: "${1}"
 ```

docs/content/usage/getting-started.md

@@ -69,7 +69,7 @@ If the static file is served by a web server, e.g. while using the [Prometheus O
 
 ```YAML
 - http_sd_configs:
-    url: pve-sd-service:80/proxmox.json
+    - url: http://pve-sd-service:80/proxmox.json
   job_name: telegraf-pve
   metrics_path: /metrics
   relabel_configs:

prometheuspvesd/config.py

@@ -106,6 +106,18 @@ class Config():
             "file": True,
             "type": environs.Env().list
         },
+        "include_vmid": {
+            "default": [],
+            "env": "INCLUDE_VMID",
+            "file": True,
+            "type": environs.Env().list
+        },
+        "include_tags": {
+            "default": [],
+            "env": "INCLUDE_TAGS",
+            "file": True,
+            "type": environs.Env().list
+        },
         "pve.server": {
             "default": "",
             "env": "PVE_SERVER",

prometheuspvesd/discovery.py

@@ -144,10 +144,24 @@ class Discovery():
 
         return ipv4_address, ipv6_address
 
-    def _exclude(self, pve_list):
+    def _filter(self, pve_list):
         filtered = []
         for item in pve_list:
             obj = defaultdict(dict, item)
+            if (
+                len(self.config.config["include_vmid"]) > 0
+                and str(obj["vmid"]) not in self.config.config["include_vmid"]
+            ):
+                continue
+
+            if (
+                len(self.config.config["include_tags"]) > 0 and (
+                    bool(obj["tags"]) is False  # continue if tags is not set
+                    or set(obj["tags"].split(",")).isdisjoint(self.config.config["include_tags"])
+                )
+            ):
+                continue
+
             if obj["template"] == 1:
                 continue
 
@@ -184,9 +198,9 @@ class Discovery():
         for node in self._get_names(self.client.get("nodes"), "node"):
             try:
                 PVE_REQUEST_COUNT_TOTAL.inc()
-                qemu_list = self._exclude(self.client.get("nodes", node, "qemu"))
+                qemu_list = self._filter(self.client.get("nodes", node, "qemu"))
                 PVE_REQUEST_COUNT_TOTAL.inc()
-                container_list = self._exclude(self.client.get("nodes", node, "lxc"))
+                container_list = self._filter(self.client.get("nodes", node, "lxc"))
             except Exception as e:  # noqa
                 PVE_REQUEST_COUNT_ERROR_TOTAL.inc()
                 raise APIError(str(e))

prometheuspvesd/pve.json

@@ -0,0 +1,66 @@
+[
+    {
+        "targets": [
+            "fhnw-syspr01"
+        ],
+        "labels": {
+            "__meta_pve_ipv4": "10.112.153.88",
+            "__meta_pve_ipv6": null,
+            "__meta_pve_name": "fhnw-syspr01",
+            "__meta_pve_type": "qemu",
+            "__meta_pve_vmid": "103",
+            "__meta_pve_cpu": "1",
+            "__meta_pve_cores": "2",
+            "__meta_pve_memory": "2048",
+            "__meta_pve_status": "running"
+        }
+    },
+    {
+        "targets": [
+            "admin03"
+        ],
+        "labels": {
+            "__meta_pve_ipv4": "10.112.153.83",
+            "__meta_pve_ipv6": null,
+            "__meta_pve_name": "admin03",
+            "__meta_pve_type": "qemu",
+            "__meta_pve_vmid": "102",
+            "__meta_pve_cpu": "1",
+            "__meta_pve_cores": "1",
+            "__meta_pve_memory": "1024",
+            "__meta_pve_status": "running"
+        }
+    },
+    {
+        "targets": [
+            "psql02"
+        ],
+        "labels": {
+            "__meta_pve_ipv4": "10.112.153.91",
+            "__meta_pve_ipv6": null,
+            "__meta_pve_name": "psql02",
+            "__meta_pve_type": "qemu",
+            "__meta_pve_vmid": "100",
+            "__meta_pve_cpu": "1",
+            "__meta_pve_cores": "4",
+            "__meta_pve_memory": "4096",
+            "__meta_pve_status": "running"
+        }
+    },
+    {
+        "targets": [
+            "fhnw-gitlab01"
+        ],
+        "labels": {
+            "__meta_pve_ipv4": "10.112.153.92",
+            "__meta_pve_ipv6": null,
+            "__meta_pve_name": "fhnw-gitlab01",
+            "__meta_pve_type": "qemu",
+            "__meta_pve_vmid": "104",
+            "__meta_pve_cpu": "1",
+            "__meta_pve_cores": "2",
+            "__meta_pve_memory": "2048",
+            "__meta_pve_status": "running"
+        }
+    }
+]

prometheuspvesd/test/data/config.yml

@@ -17,6 +17,9 @@ exclude_state: []
 exclude_vmid: []
 exclude_tags: []
 
+include_vmid: []
+include_tags: []
+
 pve:
   server: proxmox.example.com
   user: root

prometheuspvesd/test/fixtures/fixtures.py

@@ -84,6 +84,18 @@ def builtins():
             "file": True,
             "type": environs.Env().list
         },
+        "include_vmid": {
+            "default": [],
+            "env": "INCLUDE_VMID",
+            "file": True,
+            "type": environs.Env().list
+        },
+        "include_tags": {
+            "default": [],
+            "env": "INCLUDE_TAGS",
+            "file": True,
+            "type": environs.Env().list
+        },
         "pve.server": {
             "default": "dummyserver",
             "env": "PVE_SERVER",
@@ -123,6 +135,8 @@ def defaults():
         "exclude_state": [],
         "exclude_tags": [],
         "exclude_vmid": [],
+        "include_tags": [],
+        "include_vmid": [],
         "logging": {
             "format": "console",
             "level": "WARNING"
@@ -166,7 +180,7 @@ def qemus():
             "status": "running",
             "netout": 12159205236,
             "mem": 496179157,
-            "tags": "unmonitored,excluded"
+            "tags": "unmonitored,excluded,postgres"
         },
         {
             "diskwrite": 0,

prometheuspvesd/test/unit/test_discovery.py

@@ -31,25 +31,68 @@ def get_mock(*args):
 
 def test_exclude_vmid(discovery, qemus):
     discovery.config.config["exclude_vmid"] = ["100", "101", "102"]
-    filtered = discovery._exclude(qemus)
+    filtered = discovery._filter(qemus)
 
     assert len(filtered) == 0
 
 
 def test_exclude_state(discovery, qemus):
     discovery.config.config["exclude_state"] = ["prelaunch"]
-    filtered = discovery._exclude(qemus)
+    filtered = discovery._filter(qemus)
 
     assert len(filtered) == 2
 
 
 def test_exclude_tags(discovery, qemus):
     discovery.config.config["exclude_tags"] = ["unmonitored"]
-    filtered = discovery._exclude(qemus)
+    filtered = discovery._filter(qemus)
 
     assert len(filtered) == 2
 
 
+def test_include_tags(discovery, qemus):
+    discovery.config.config["include_tags"] = ["monitored"]
+    filtered = discovery._filter(qemus)
+
+    assert len(filtered) == 1
+
+
+def test_include_tags_multiple(discovery, qemus):
+    discovery.config.config["include_tags"] = ["monitored", "postgres"]
+    filtered = discovery._filter(qemus)
+
+    assert len(filtered) == 2
+
+
+def test_include_tags_empty(discovery, qemus):
+    discovery.config.config["include_tags"] = []
+    filtered = discovery._filter(qemus)
+
+    assert len(filtered) == 3
+
+
+def test_include_vmid(discovery, qemus):
+    discovery.config.config["include_vmid"] = ["101", "100"]
+    filtered = discovery._filter(qemus)
+
+    assert len(filtered) == 2
+
+
+def test_include_vmid_empty(discovery, qemus):
+    discovery.config.config["include_vmid"] = []
+    filtered = discovery._filter(qemus)
+
+    assert len(filtered) == 3
+
+
+def test_include_and_exclude_tags(discovery, qemus):
+    discovery.config.config["include_tags"] = ["postgres"]
+    discovery.config.config["exclude_tags"] = ["unmonitored"]
+    filtered = discovery._filter(qemus)
+
+    assert len(filtered) == 0
+
+
 def test_validate_ip(discovery, addresses):
     # IPv4 validation
     for address in addresses["ipv4_valid"]: