inital commit

This commit is contained in:
Robert Kaussow 2021-06-09 20:44:10 +02:00
commit 4586d9642b
Signed by: xoxys
GPG Key ID: 4E692A2EAECC03C0
29 changed files with 3585 additions and 0 deletions

27
.chglog/CHANGELOG.tpl.md Executable file
View File

@ -0,0 +1,27 @@
# Changelog
{{ range .Versions -}}
## {{ if .Tag.Previous }}[{{ .Tag.Name }}]({{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}){{ else }}{{ .Tag.Name }}{{ end }} ({{ datetime "2006-01-02" .Tag.Date }})
{{ range .CommitGroups -}}
### {{ .Title }}
{{ $subjects := list }}
{{ range .Commits -}}
{{ if not (has .Subject $subjects) -}}
- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
{{ $subjects = append $subjects .Subject -}}
{{ end }}
{{- end }}
{{- end -}}
{{- if .NoteGroups -}}
{{ range .NoteGroups -}}
### {{ .Title }}
{{ range .Notes }}
{{ .Body }}
{{ end }}
{{ end -}}
{{ end -}}
{{ end -}}

25
.chglog/config.yml Executable file
View File

@ -0,0 +1,25 @@
style: github
template: CHANGELOG.tpl.md
info:
title: CHANGELOG
repository_url: https://github.com/thegeeklab/prometheus-pve-sd
options:
commit_groups:
title_maps:
feat: Features
fix: Bug Fixes
perf: Performance Improvements
refactor: Code Refactoring
chore: Others
test: Testing
ci: CI Pipeline
docs: Documentation
header:
pattern: "^(\\w*)(?:\\(([\\w\\$\\.\\-\\*\\s]*)\\))?\\:\\s(.*)$"
pattern_maps:
- Type
- Scope
- Subject
notes:
keywords:
- BREAKING CHANGE

3
.dictionary Normal file
View File

@ -0,0 +1,3 @@
Kaussow
PyPI
xoxys

493
.drone.jsonnet Normal file
View File

@ -0,0 +1,493 @@
local PythonVersion(pyversion='3.6') = {
name: 'python' + std.strReplace(pyversion, '.', '') + '-pytest',
image: 'python:' + pyversion,
environment: {
PY_COLORS: 1,
},
commands: [
'pip install poetry poetry-dynamic-versioning -qq',
'poetry config experimental.new-installer false',
'poetry install',
'poetry version',
'poetry run prometheus-pve-sd --help',
],
depends_on: [
'fetch',
],
};
local PipelineLint = {
kind: 'pipeline',
name: 'lint',
platform: {
os: 'linux',
arch: 'amd64',
},
steps: [
{
name: 'yapf',
image: 'python:3.9',
environment: {
PY_COLORS: 1,
},
commands: [
'git fetch -tq',
'pip install poetry poetry-dynamic-versioning -qq',
'poetry config experimental.new-installer false',
'poetry install',
'poetry run yapf -dr ./prometheuspvesd',
],
},
{
name: 'flake8',
image: 'python:3.9',
environment: {
PY_COLORS: 1,
},
commands: [
'git fetch -tq',
'pip install poetry poetry-dynamic-versioning -qq',
'poetry config experimental.new-installer false',
'poetry install',
'poetry run flake8 ./prometheuspvesd',
],
},
],
trigger: {
ref: ['refs/heads/main', 'refs/tags/**', 'refs/pull/**'],
},
};
local PipelineTest = {
kind: 'pipeline',
name: 'test',
platform: {
os: 'linux',
arch: 'amd64',
},
steps: [
{
name: 'fetch',
image: 'python:3.9',
commands: [
'git fetch -tq',
],
},
PythonVersion(pyversion='3.6'),
PythonVersion(pyversion='3.7'),
PythonVersion(pyversion='3.8'),
PythonVersion(pyversion='3.9'),
],
depends_on: [
'lint',
],
trigger: {
ref: ['refs/heads/main', 'refs/tags/**', 'refs/pull/**'],
},
};
local PipelineSecurity = {
kind: 'pipeline',
name: 'security',
platform: {
os: 'linux',
arch: 'amd64',
},
steps: [
{
name: 'bandit',
image: 'python:3.9',
environment: {
PY_COLORS: 1,
},
commands: [
'git fetch -tq',
'pip install poetry poetry-dynamic-versioning -qq',
'poetry config experimental.new-installer false',
'poetry install',
'poetry run bandit -r ./prometheuspvesd -x ./prometheuspvesd/test',
],
},
],
depends_on: [
'test',
],
trigger: {
ref: ['refs/heads/main', 'refs/tags/**', 'refs/pull/**'],
},
};
local PipelineBuildPackage = {
kind: 'pipeline',
name: 'build-package',
platform: {
os: 'linux',
arch: 'amd64',
},
steps: [
{
name: 'build',
image: 'python:3.9',
commands: [
'git fetch -tq',
'pip install poetry poetry-dynamic-versioning -qq',
'poetry build',
],
},
{
name: 'checksum',
image: 'alpine',
commands: [
'cd dist/ && sha256sum * > ../sha256sum.txt',
],
},
{
name: 'changelog-generate',
image: 'thegeeklab/git-chglog',
commands: [
'git fetch -tq',
'git-chglog --no-color --no-emoji -o CHANGELOG.md ${DRONE_TAG:---next-tag unreleased unreleased}',
],
},
{
name: 'changelog-format',
image: 'thegeeklab/alpine-tools',
commands: [
'prettier CHANGELOG.md',
'prettier -w CHANGELOG.md',
],
},
{
name: 'publish-github',
image: 'plugins/github-release',
settings: {
overwrite: true,
api_key: { from_secret: 'github_token' },
files: ['dist/*', 'sha256sum.txt'],
title: '${DRONE_TAG}',
note: 'CHANGELOG.md',
},
when: {
ref: ['refs/tags/**'],
},
},
{
name: 'publish-pypi',
image: 'python:3.9',
commands: [
'git fetch -tq',
'pip install poetry poetry-dynamic-versioning -qq',
'poetry publish -n',
],
environment: {
POETRY_HTTP_BASIC_PYPI_USERNAME: { from_secret: 'pypi_username' },
POETRY_HTTP_BASIC_PYPI_PASSWORD: { from_secret: 'pypi_password' },
},
when: {
ref: ['refs/tags/**'],
},
},
],
depends_on: [
'security',
],
trigger: {
ref: ['refs/heads/main', 'refs/tags/**', 'refs/pull/**'],
},
};
local PipelineBuildContainer(arch='amd64') = {
local build = if arch == 'arm' then [{
name: 'build',
image: 'python:3.9-alpine',
commands: [
'apk add -Uq --no-cache build-base libressl-dev libffi-dev musl-dev python3-dev git cargo',
'git fetch -tq',
'pip install poetry poetry-dynamic-versioning -qq',
'poetry build',
],
environment: {
CARGO_NET_GIT_FETCH_WITH_CLI: true,
},
}] else [{
name: 'build',
image: 'python:3.9',
commands: [
'git fetch -tq',
'pip install poetry poetry-dynamic-versioning -qq',
'poetry build',
],
}],
kind: 'pipeline',
name: 'build-container-' + arch,
platform: {
os: 'linux',
arch: arch,
},
steps: build + [
{
name: 'dryrun',
image: 'thegeeklab/drone-docker:19',
settings: {
dry_run: true,
dockerfile: 'docker/Dockerfile.' + arch,
repo: 'thegeeklab/${DRONE_REPO_NAME}',
username: { from_secret: 'docker_username' },
password: { from_secret: 'docker_password' },
},
depends_on: ['build'],
when: {
ref: ['refs/pull/**'],
},
},
{
name: 'publish-dockerhub',
image: 'thegeeklab/drone-docker:19',
settings: {
auto_tag: true,
auto_tag_suffix: arch,
dockerfile: 'docker/Dockerfile.' + arch,
repo: 'thegeeklab/${DRONE_REPO_NAME}',
username: { from_secret: 'docker_username' },
password: { from_secret: 'docker_password' },
},
when: {
ref: ['refs/heads/main', 'refs/tags/**'],
},
depends_on: ['dryrun'],
},
{
name: 'publish-quay',
image: 'thegeeklab/drone-docker:19',
settings: {
auto_tag: true,
auto_tag_suffix: arch,
dockerfile: 'docker/Dockerfile.' + arch,
registry: 'quay.io',
repo: 'quay.io/thegeeklab/${DRONE_REPO_NAME}',
username: { from_secret: 'quay_username' },
password: { from_secret: 'quay_password' },
},
when: {
ref: ['refs/heads/main', 'refs/tags/**'],
},
depends_on: ['dryrun'],
},
],
depends_on: [
'security',
],
trigger: {
ref: ['refs/heads/main', 'refs/tags/**', 'refs/pull/**'],
},
};
local PipelineDocs = {
kind: 'pipeline',
name: 'docs',
platform: {
os: 'linux',
arch: 'amd64',
},
concurrency: {
limit: 1,
},
steps: [
{
name: 'assets',
image: 'thegeeklab/alpine-tools',
commands: [
'make doc',
],
},
{
name: 'markdownlint',
image: 'thegeeklab/markdownlint-cli',
commands: [
"markdownlint 'docs/content/**/*.md' 'README.md' 'CONTRIBUTING.md'",
],
},
{
name: 'spellcheck',
image: 'node:lts-alpine',
commands: [
'npm install -g spellchecker-cli',
"spellchecker --files 'docs/content/**/*.md' 'README.md' -d .dictionary -p spell indefinite-article syntax-urls --no-suggestions",
],
environment: {
FORCE_COLOR: true,
NPM_CONFIG_LOGLEVEL: 'error',
},
},
{
name: 'testbuild',
image: 'thegeeklab/hugo:0.83.1',
commands: [
'hugo -s docs/ -b http://localhost/',
],
},
{
name: 'link-validation',
image: 'thegeeklab/link-validator',
commands: [
'link-validator -ro',
],
environment: {
LINK_VALIDATOR_BASE_DIR: 'docs/public',
},
},
{
name: 'build',
image: 'thegeeklab/hugo:0.83.1',
commands: [
'hugo -s docs/',
],
},
{
name: 'beautify',
image: 'node:lts-alpine',
commands: [
'npm install -g js-beautify',
"html-beautify -r -f 'docs/public/**/*.html'",
],
environment: {
FORCE_COLOR: true,
NPM_CONFIG_LOGLEVEL: 'error',
},
},
{
name: 'publish',
image: 'plugins/s3-sync',
settings: {
access_key: { from_secret: 's3_access_key' },
bucket: 'geekdocs',
delete: true,
endpoint: 'https://sp.rknet.org',
path_style: true,
secret_key: { from_secret: 's3_secret_access_key' },
source: 'docs/public/',
strip_prefix: 'docs/public/',
target: '/${DRONE_REPO_NAME}',
},
when: {
ref: ['refs/heads/main', 'refs/tags/**'],
},
},
],
depends_on: [
'build-package',
'build-container-amd64',
'build-container-arm64',
'build-container-arm',
],
trigger: {
ref: ['refs/heads/main', 'refs/tags/**', 'refs/pull/**'],
},
};
local PipelineNotifications = {
kind: 'pipeline',
name: 'notifications',
platform: {
os: 'linux',
arch: 'amd64',
},
steps: [
{
image: 'plugins/manifest',
name: 'manifest-dockerhub',
settings: {
ignore_missing: true,
auto_tag: true,
username: { from_secret: 'docker_username' },
password: { from_secret: 'docker_password' },
spec: 'docker/manifest.tmpl',
},
when: {
status: ['success'],
},
},
{
image: 'plugins/manifest',
name: 'manifest-quay',
settings: {
ignore_missing: true,
auto_tag: true,
username: { from_secret: 'quay_username' },
password: { from_secret: 'quay_password' },
spec: 'docker/manifest-quay.tmpl',
},
when: {
status: ['success'],
},
},
{
name: 'pushrm-dockerhub',
pull: 'always',
image: 'chko/docker-pushrm:1',
environment: {
DOCKER_PASS: {
from_secret: 'docker_password',
},
DOCKER_USER: {
from_secret: 'docker_username',
},
PUSHRM_FILE: 'README.md',
PUSHRM_SHORT: 'Prometheus Service Discovery for Proxmox VE',
PUSHRM_TARGET: 'thegeeklab/${DRONE_REPO_NAME}',
},
when: {
status: ['success'],
},
},
{
name: 'pushrm-quay',
pull: 'always',
image: 'chko/docker-pushrm:1',
environment: {
APIKEY__QUAY_IO: {
from_secret: 'quay_token',
},
PUSHRM_FILE: 'README.md',
PUSHRM_TARGET: 'quay.io/thegeeklab/${DRONE_REPO_NAME}',
},
when: {
status: ['success'],
},
},
{
name: 'matrix',
image: 'plugins/matrix',
settings: {
homeserver: { from_secret: 'matrix_homeserver' },
roomid: { from_secret: 'matrix_roomid' },
template: 'Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}',
username: { from_secret: 'matrix_username' },
password: { from_secret: 'matrix_password' },
},
when: {
status: ['success', 'failure'],
},
},
],
depends_on: [
'docs',
],
trigger: {
ref: ['refs/heads/main', 'refs/tags/**'],
status: ['success', 'failure'],
},
};
[
PipelineLint,
PipelineTest,
PipelineSecurity,
PipelineBuildPackage,
PipelineBuildContainer(arch='amd64'),
PipelineBuildContainer(arch='arm64'),
PipelineBuildContainer(arch='arm'),
PipelineDocs,
PipelineNotifications,
]

635
.drone.yml Normal file
View File

@ -0,0 +1,635 @@
---
kind: pipeline
name: lint
platform:
os: linux
arch: amd64
steps:
- name: yapf
image: python:3.9
commands:
- git fetch -tq
- pip install poetry poetry-dynamic-versioning -qq
- poetry config experimental.new-installer false
- poetry install
- poetry run yapf -dr ./prometheuspvesd
environment:
PY_COLORS: 1
- name: flake8
image: python:3.9
commands:
- git fetch -tq
- pip install poetry poetry-dynamic-versioning -qq
- poetry config experimental.new-installer false
- poetry install
- poetry run flake8 ./prometheuspvesd
environment:
PY_COLORS: 1
trigger:
ref:
- refs/heads/main
- refs/tags/**
- refs/pull/**
---
kind: pipeline
name: test
platform:
os: linux
arch: amd64
steps:
- name: fetch
image: python:3.9
commands:
- git fetch -tq
- name: python36-pytest
image: python:3.6
commands:
- pip install poetry poetry-dynamic-versioning -qq
- poetry config experimental.new-installer false
- poetry install
- poetry version
- poetry run prometheus-pve-sd --help
environment:
PY_COLORS: 1
depends_on:
- fetch
- name: python37-pytest
image: python:3.7
commands:
- pip install poetry poetry-dynamic-versioning -qq
- poetry config experimental.new-installer false
- poetry install
- poetry version
- poetry run prometheus-pve-sd --help
environment:
PY_COLORS: 1
depends_on:
- fetch
- name: python38-pytest
image: python:3.8
commands:
- pip install poetry poetry-dynamic-versioning -qq
- poetry config experimental.new-installer false
- poetry install
- poetry version
- poetry run prometheus-pve-sd --help
environment:
PY_COLORS: 1
depends_on:
- fetch
- name: python39-pytest
image: python:3.9
commands:
- pip install poetry poetry-dynamic-versioning -qq
- poetry config experimental.new-installer false
- poetry install
- poetry version
- poetry run prometheus-pve-sd --help
environment:
PY_COLORS: 1
depends_on:
- fetch
trigger:
ref:
- refs/heads/main
- refs/tags/**
- refs/pull/**
depends_on:
- lint
---
kind: pipeline
name: security
platform:
os: linux
arch: amd64
steps:
- name: bandit
image: python:3.9
commands:
- git fetch -tq
- pip install poetry poetry-dynamic-versioning -qq
- poetry config experimental.new-installer false
- poetry install
- poetry run bandit -r ./prometheuspvesd -x ./prometheuspvesd/test
environment:
PY_COLORS: 1
trigger:
ref:
- refs/heads/main
- refs/tags/**
- refs/pull/**
depends_on:
- test
---
kind: pipeline
name: build-package
platform:
os: linux
arch: amd64
steps:
- name: build
image: python:3.9
commands:
- git fetch -tq
- pip install poetry poetry-dynamic-versioning -qq
- poetry build
- name: checksum
image: alpine
commands:
- cd dist/ && sha256sum * > ../sha256sum.txt
- name: changelog-generate
image: thegeeklab/git-chglog
commands:
- git fetch -tq
- git-chglog --no-color --no-emoji -o CHANGELOG.md ${DRONE_TAG:---next-tag unreleased unreleased}
- name: changelog-format
image: thegeeklab/alpine-tools
commands:
- prettier CHANGELOG.md
- prettier -w CHANGELOG.md
- name: publish-github
image: plugins/github-release
settings:
api_key:
from_secret: github_token
files:
- dist/*
- sha256sum.txt
note: CHANGELOG.md
overwrite: true
title: ${DRONE_TAG}
when:
ref:
- refs/tags/**
- name: publish-pypi
image: python:3.9
commands:
- git fetch -tq
- pip install poetry poetry-dynamic-versioning -qq
- poetry publish -n
environment:
POETRY_HTTP_BASIC_PYPI_PASSWORD:
from_secret: pypi_password
POETRY_HTTP_BASIC_PYPI_USERNAME:
from_secret: pypi_username
when:
ref:
- refs/tags/**
trigger:
ref:
- refs/heads/main
- refs/tags/**
- refs/pull/**
depends_on:
- security
---
kind: pipeline
name: build-container-amd64
platform:
os: linux
arch: amd64
steps:
- name: build
image: python:3.9
commands:
- git fetch -tq
- pip install poetry poetry-dynamic-versioning -qq
- poetry build
- name: dryrun
image: thegeeklab/drone-docker:19
settings:
dockerfile: docker/Dockerfile.amd64
dry_run: true
password:
from_secret: docker_password
repo: thegeeklab/${DRONE_REPO_NAME}
username:
from_secret: docker_username
when:
ref:
- refs/pull/**
depends_on:
- build
- name: publish-dockerhub
image: thegeeklab/drone-docker:19
settings:
auto_tag: true
auto_tag_suffix: amd64
dockerfile: docker/Dockerfile.amd64
password:
from_secret: docker_password
repo: thegeeklab/${DRONE_REPO_NAME}
username:
from_secret: docker_username
when:
ref:
- refs/heads/main
- refs/tags/**
depends_on:
- dryrun
- name: publish-quay
image: thegeeklab/drone-docker:19
settings:
auto_tag: true
auto_tag_suffix: amd64
dockerfile: docker/Dockerfile.amd64
password:
from_secret: quay_password
registry: quay.io
repo: quay.io/thegeeklab/${DRONE_REPO_NAME}
username:
from_secret: quay_username
when:
ref:
- refs/heads/main
- refs/tags/**
depends_on:
- dryrun
trigger:
ref:
- refs/heads/main
- refs/tags/**
- refs/pull/**
depends_on:
- security
---
kind: pipeline
name: build-container-arm64
platform:
os: linux
arch: arm64
steps:
- name: build
image: python:3.9
commands:
- git fetch -tq
- pip install poetry poetry-dynamic-versioning -qq
- poetry build
- name: dryrun
image: thegeeklab/drone-docker:19
settings:
dockerfile: docker/Dockerfile.arm64
dry_run: true
password:
from_secret: docker_password
repo: thegeeklab/${DRONE_REPO_NAME}
username:
from_secret: docker_username
when:
ref:
- refs/pull/**
depends_on:
- build
- name: publish-dockerhub
image: thegeeklab/drone-docker:19
settings:
auto_tag: true
auto_tag_suffix: arm64
dockerfile: docker/Dockerfile.arm64
password:
from_secret: docker_password
repo: thegeeklab/${DRONE_REPO_NAME}
username:
from_secret: docker_username
when:
ref:
- refs/heads/main
- refs/tags/**
depends_on:
- dryrun
- name: publish-quay
image: thegeeklab/drone-docker:19
settings:
auto_tag: true
auto_tag_suffix: arm64
dockerfile: docker/Dockerfile.arm64
password:
from_secret: quay_password
registry: quay.io
repo: quay.io/thegeeklab/${DRONE_REPO_NAME}
username:
from_secret: quay_username
when:
ref:
- refs/heads/main
- refs/tags/**
depends_on:
- dryrun
trigger:
ref:
- refs/heads/main
- refs/tags/**
- refs/pull/**
depends_on:
- security
---
kind: pipeline
name: build-container-arm
platform:
os: linux
arch: arm
steps:
- name: build
image: python:3.9-alpine
commands:
- apk add -Uq --no-cache build-base libressl-dev libffi-dev musl-dev python3-dev git cargo
- git fetch -tq
- pip install poetry poetry-dynamic-versioning -qq
- poetry build
environment:
CARGO_NET_GIT_FETCH_WITH_CLI: true
- name: dryrun
image: thegeeklab/drone-docker:19
settings:
dockerfile: docker/Dockerfile.arm
dry_run: true
password:
from_secret: docker_password
repo: thegeeklab/${DRONE_REPO_NAME}
username:
from_secret: docker_username
when:
ref:
- refs/pull/**
depends_on:
- build
- name: publish-dockerhub
image: thegeeklab/drone-docker:19
settings:
auto_tag: true
auto_tag_suffix: arm
dockerfile: docker/Dockerfile.arm
password:
from_secret: docker_password
repo: thegeeklab/${DRONE_REPO_NAME}
username:
from_secret: docker_username
when:
ref:
- refs/heads/main
- refs/tags/**
depends_on:
- dryrun
- name: publish-quay
image: thegeeklab/drone-docker:19
settings:
auto_tag: true
auto_tag_suffix: arm
dockerfile: docker/Dockerfile.arm
password:
from_secret: quay_password
registry: quay.io
repo: quay.io/thegeeklab/${DRONE_REPO_NAME}
username:
from_secret: quay_username
when:
ref:
- refs/heads/main
- refs/tags/**
depends_on:
- dryrun
trigger:
ref:
- refs/heads/main
- refs/tags/**
- refs/pull/**
depends_on:
- security
---
kind: pipeline
name: docs
platform:
os: linux
arch: amd64
concurrency:
limit: 1
steps:
- name: assets
image: thegeeklab/alpine-tools
commands:
- make doc
- name: markdownlint
image: thegeeklab/markdownlint-cli
commands:
- markdownlint 'docs/content/**/*.md' 'README.md' 'CONTRIBUTING.md'
- name: spellcheck
image: node:lts-alpine
commands:
- npm install -g spellchecker-cli
- spellchecker --files 'docs/content/**/*.md' 'README.md' -d .dictionary -p spell indefinite-article syntax-urls --no-suggestions
environment:
FORCE_COLOR: true
NPM_CONFIG_LOGLEVEL: error
- name: testbuild
image: thegeeklab/hugo:0.83.1
commands:
- hugo -s docs/ -b http://localhost/
- name: link-validation
image: thegeeklab/link-validator
commands:
- link-validator -ro
environment:
LINK_VALIDATOR_BASE_DIR: docs/public
- name: build
image: thegeeklab/hugo:0.83.1
commands:
- hugo -s docs/
- name: beautify
image: node:lts-alpine
commands:
- npm install -g js-beautify
- html-beautify -r -f 'docs/public/**/*.html'
environment:
FORCE_COLOR: true
NPM_CONFIG_LOGLEVEL: error
- name: publish
image: plugins/s3-sync
settings:
access_key:
from_secret: s3_access_key
bucket: geekdocs
delete: true
endpoint: https://sp.rknet.org
path_style: true
secret_key:
from_secret: s3_secret_access_key
source: docs/public/
strip_prefix: docs/public/
target: /${DRONE_REPO_NAME}
when:
ref:
- refs/heads/main
- refs/tags/**
trigger:
ref:
- refs/heads/main
- refs/tags/**
- refs/pull/**
depends_on:
- build-package
- build-container-amd64
- build-container-arm64
- build-container-arm
---
kind: pipeline
name: notifications
platform:
os: linux
arch: amd64
steps:
- name: manifest-dockerhub
image: plugins/manifest
settings:
auto_tag: true
ignore_missing: true
password:
from_secret: docker_password
spec: docker/manifest.tmpl
username:
from_secret: docker_username
when:
status:
- success
- name: manifest-quay
image: plugins/manifest
settings:
auto_tag: true
ignore_missing: true
password:
from_secret: quay_password
spec: docker/manifest-quay.tmpl
username:
from_secret: quay_username
when:
status:
- success
- name: pushrm-dockerhub
pull: always
image: chko/docker-pushrm:1
environment:
DOCKER_PASS:
from_secret: docker_password
DOCKER_USER:
from_secret: docker_username
PUSHRM_FILE: README.md
PUSHRM_SHORT: Prometheus Service Discovery for Proxmox VE
PUSHRM_TARGET: thegeeklab/${DRONE_REPO_NAME}
when:
status:
- success
- name: pushrm-quay
pull: always
image: chko/docker-pushrm:1
environment:
APIKEY__QUAY_IO:
from_secret: quay_token
PUSHRM_FILE: README.md
PUSHRM_TARGET: quay.io/thegeeklab/${DRONE_REPO_NAME}
when:
status:
- success
- name: matrix
image: plugins/matrix
settings:
homeserver:
from_secret: matrix_homeserver
password:
from_secret: matrix_password
roomid:
from_secret: matrix_roomid
template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}"
username:
from_secret: matrix_username
when:
status:
- success
- failure
trigger:
ref:
- refs/heads/main
- refs/tags/**
status:
- success
- failure
depends_on:
- docs
---
kind: signature
hmac: 791f2a668d1295b69cb7000c3a38b8e3ca15d99b1c8ac40559c71aa9456fc018
...

56
.github/settings.yml vendored Normal file
View File

@ -0,0 +1,56 @@
repository:
name: prometheus-pve-sd
description: Prometheus Service Discovery for Proxmox VE
topics: prometheus, proxmox, metrics, sd, python
private: false
has_issues: true
has_projects: false
has_wiki: false
has_downloads: true
default_branch: main
allow_squash_merge: true
allow_merge_commit: true
allow_rebase_merge: true
labels:
- name: bug
color: d73a4a
description: Something isn't working
- name: documentation
color: 0075ca
description: Improvements or additions to documentation
- name: duplicate
color: cfd3d7
description: This issue or pull request already exists
- name: enhancement
color: a2eeef
description: New feature or request
- name: good first issue
color: 7057ff
description: Good for newcomers
- name: help wanted
color: 008672
description: Extra attention is needed
- name: invalid
color: e4e669
description: This doesn't seem right
- name: question
color: d876e3
description: Further information is requested
- name: wontfix
color: ffffff
description: This will not be worked on
branches:
- name: main
protection:
required_pull_request_reviews: null
required_status_checks:
strict: false
contexts:
- continuous-integration/drone/pr
enforce_admins: null
restrictions: null

111
.gitignore vendored Normal file
View File

@ -0,0 +1,111 @@
# ---> Python
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# dotenv
.env
# virtualenv
.venv
venv/
ENV/
env/
env*/
# Spyder project settings
.spyderproject
# Rope project settings
.ropeproject
# Ignore ide addons
.server-script
.on-save.json
.vscode
.pytest_cache
pip-wheel-metadata
# Hugo documentation
docs/themes/
docs/public/
resources/_gen/
# Misc
CHANGELOG.md

6
.markdownlint.yml Normal file
View File

@ -0,0 +1,6 @@
---
default: True
MD013: False
MD041: False
MD004:
style: dash

2
.prettierignore Normal file
View File

@ -0,0 +1,2 @@
.drone.yml
*.tpl.md

31
CONTRIBUTING.md Normal file
View File

@ -0,0 +1,31 @@
# Contributing
## Security
If you think you have found a **security issue**, please do not mention it in this repository.
Instead, send an email to security@thegeeklab.de with as many details as possible so it can be handled confidential.
## Bug Reports and Feature Requests
If you have found a **bug** or have a **feature request** please use the search first in case a similar issue already exists.
If not, please create an issue in this repository
## Code
If you would like to fix a bug or implement a feature, please fork the repository and create a Pull Request.
Before you start any Pull Request, it is recommended that you create an issue to discuss first if you have any
doubts about requirement or implementation. That way you can be sure that the maintainer(s) agree on what to change and how,
and you can hopefully get a quick merge afterwards.
Pull Requests can only be merged once all status checks are green.
## Do not force push to your Pull Request branch
Please do not force push to your Pull Requests branch after you have created your Pull Request, as doing so makes it harder for us to review your work.
Pull Requests will always be squashed by us when we merge your work. Commit as many times as you need in your Pull Request branch.
## Re-requesting a review
Please do not ping your reviewer(s) by mentioning them in a new comment. Instead, use the re-request review functionality.
Read more about this in the [GitHub docs, Re-requesting a review](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/incorporating-feedback-in-your-pull-request#re-requesting-a-review).

21
LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2021 Robert Kaussow <mail@thegeeklab.de>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions:
The above copyright notice and this permission notice (including the next
paragraph) shall be included in all copies or substantial portions of the
Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

20
Makefile Normal file
View File

@ -0,0 +1,20 @@
# renovate: datasource=github-releases depName=thegeeklab/hugo-geekdoc
THEME_VERSION := v0.13.4
THEME := hugo-geekdoc
BASEDIR := docs
THEMEDIR := $(BASEDIR)/themes
.PHONY: all
all: doc
.PHONY: doc
doc: doc-assets
.PHONY: doc-assets
doc-assets:
mkdir -p $(THEMEDIR)/$(THEME)/ ; \
curl -sSL "https://github.com/thegeeklab/$(THEME)/releases/download/${THEME_VERSION}/$(THEME).tar.gz" | tar -xz -C $(THEMEDIR)/$(THEME)/ --strip-components=1
.PHONY: clean
clean:
rm -rf $(THEMEDIR) && \

24
README.md Normal file
View File

@ -0,0 +1,24 @@
# prometheus-pve-sd
Prometheus Service Discovery for Proxmox VE
[![Build Status](https://img.shields.io/drone/build/thegeeklab/prometheus-pve-sd?logo=drone&server=https%3A%2F%2Fdrone.thegeeklab.de)](https://drone.thegeeklab.de/thegeeklab/prometheus-pve-sd)
[![Docker Hub](https://img.shields.io/badge/dockerhub-latest-blue.svg?logo=docker&logoColor=white)](https://hub.docker.com/r/thegeeklab/prometheus-pve-sd)
[![Quay.io](https://img.shields.io/badge/quay-latest-blue.svg?logo=docker&logoColor=white)](https://quay.io/repository/thegeeklab/prometheus-pve-sd)
[![Python Version](https://img.shields.io/pypi/pyversions/prometheus-pve-sd.svg)](https://pypi.org/project/prometheus-pve-sd/)
[![PyPI Status](https://img.shields.io/pypi/status/prometheus-pve-sd.svg)](https://pypi.org/project/prometheus-pve-sd/)
[![PyPI Release](https://img.shields.io/pypi/v/prometheus-pve-sd.svg)](https://pypi.org/project/prometheus-pve-sd/)
[![GitHub contributors](https://img.shields.io/github/contributors/thegeeklab/prometheus-pve-sd)](https://github.com/thegeeklab/prometheus-pve-sd/graphs/contributors)
[![Source: GitHub](https://img.shields.io/badge/source-github-blue.svg?logo=github&logoColor=white)](https://github.com/thegeeklab/prometheus-pve-sd)
[![License: MIT](https://img.shields.io/github/license/thegeeklab/prometheus-pve-sd)](https://github.com/thegeeklab/prometheus-pve-sd/blob/main/LICENSE)
TBD
## Contributors
Special thanks goes to all [contributors](https://github.com/thegeeklab/prometheus-pve-sd/graphs/contributors). If you would like to contribute,
please see the [instructions](https://github.com/thegeeklab/prometheus-pve-sd/blob/main/CONTRIBUTING.md).
## License
This project is licensed under the MIT License - see the [LICENSE](https://github.com/thegeeklab/prometheus-pve-sd/blob/main/LICENSE) file for details.

23
docker/Dockerfile.amd64 Normal file
View File

@ -0,0 +1,23 @@
FROM amd64/python:3.9-alpine@sha256:d2dfb8f0a8b3ab3e2899bba05e53c2b16bc1b8c1fca83637266edb8d1a57dc86
LABEL maintainer="Robert Kaussow <mail@thegeeklab.de>"
LABEL org.opencontainers.image.authors="Robert Kaussow <mail@thegeeklab.de>"
LABEL org.opencontainers.image.title="prometheus-pve-sd"
LABEL org.opencontainers.image.url="https://github.com/thegeeklab/prometheus-pve-sd/"
LABEL org.opencontainers.image.source="https://github.com/thegeeklab/prometheus-pve-sd/"
LABEL org.opencontainers.image.documentation="https://github.com/thegeeklab/prometheus-pve-sd/"
ENV PY_COLORS=1
ADD dist/prometheus_pve_sd--*.whl /
RUN apk update && \
pip install --upgrade --no-cache-dir pip && \
pip install --no-cache-dir $(find / -name "prometheus_pve_sd--*.whl") && \
rm -f prometheus_pve_sd--*.whl && \
rm -rf /var/cache/apk/* && \
rm -rf /root/.cache/
USER root
CMD []
ENTRYPOINT ["/usr/local/bin/prometheus-pve-sd"]

23
docker/Dockerfile.arm Normal file
View File

@ -0,0 +1,23 @@
FROM arm32v7/python:3.9-alpine@sha256:183252dde15e315fbe570a5355be839251aa4878b20163c767dd5238de3c988e
LABEL maintainer="Robert Kaussow <mail@thegeeklab.de>"
LABEL org.opencontainers.image.authors="Robert Kaussow <mail@thegeeklab.de>"
LABEL org.opencontainers.image.title="prometheus-pve-sd"
LABEL org.opencontainers.image.url="https://github.com/thegeeklab/prometheus-pve-sd/"
LABEL org.opencontainers.image.source="https://github.com/thegeeklab/prometheus-pve-sd/"
LABEL org.opencontainers.image.documentation="https://github.com/thegeeklab/prometheus-pve-sd/"
ENV PY_COLORS=1
ADD dist/prometheus_pve_sd--*.whl /
RUN apk update && \
pip install --upgrade --no-cache-dir pip && \
pip install --no-cache-dir $(find / -name "prometheus_pve_sd--*.whl") && \
rm -f prometheus_pve_sd--*.whl && \
rm -rf /var/cache/apk/* && \
rm -rf /root/.cache/
USER root
CMD []
ENTRYPOINT ["/usr/local/bin/prometheus-pve-sd"]

23
docker/Dockerfile.arm64 Normal file
View File

@ -0,0 +1,23 @@
FROM arm64v8/python:3.9-alpine@sha256:2dd55073bf996f367008a7fad490add0343b5c151b708dcf52c133f70ab171a9
LABEL maintainer="Robert Kaussow <mail@thegeeklab.de>"
LABEL org.opencontainers.image.authors="Robert Kaussow <mail@thegeeklab.de>"
LABEL org.opencontainers.image.title="prometheus-pve-sd"
LABEL org.opencontainers.image.url="https://github.com/thegeeklab/prometheus-pve-sd/"
LABEL org.opencontainers.image.source="https://github.com/thegeeklab/prometheus-pve-sd/"
LABEL org.opencontainers.image.documentation="https://github.com/thegeeklab/prometheus-pve-sd/"
ENV PY_COLORS=1
ADD dist/prometheus_pve_sd--*.whl /
RUN apk update && \
pip install --upgrade --no-cache-dir pip && \
pip install --no-cache-dir $(find / -name "prometheus_pve_sd--*.whl") && \
rm -f prometheus_pve_sd--*.whl && \
rm -rf /var/cache/apk/* && \
rm -rf /root/.cache/
USER root
CMD []
ENTRYPOINT ["/usr/local/bin/prometheus-pve-sd"]

24
docker/manifest-quay.tmpl Normal file
View File

@ -0,0 +1,24 @@
image: quay.io/thegeeklab/prometheus-pve-sd:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
{{/if}}
manifests:
- image: quay.io/thegeeklab/prometheus-pve-sd:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}amd64
platform:
architecture: amd64
os: linux
- image: quay.io/thegeeklab/prometheus-pve-sd:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm64
platform:
architecture: arm64
os: linux
variant: v8
- image: quay.io/thegeeklab/prometheus-pve-sd:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm
platform:
architecture: arm
os: linux
variant: v7

24
docker/manifest.tmpl Normal file
View File

@ -0,0 +1,24 @@
image: thegeeklab/prometheus-pve-sd:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
{{/if}}
manifests:
- image: thegeeklab/prometheus-pve-sd:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}amd64
platform:
architecture: amd64
os: linux
- image: thegeeklab/prometheus-pve-sd:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm64
platform:
architecture: arm64
os: linux
variant: v8
- image: thegeeklab/prometheus-pve-sd:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm
platform:
architecture: arm
os: linux
variant: v7

1022
poetry.lock generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,3 @@
"""Default package."""
__version__ = "0.0.0"

111
prometheuspvesd/cli.py Normal file
View File

@ -0,0 +1,111 @@
#!/usr/bin/env python3
"""Entrypoint and CLI handler."""
import argparse
import json
import shutil
import signal
import tempfile
from time import sleep
import prometheuspvesd.exception
from prometheuspvesd import __version__
from prometheuspvesd.config import SingleConfig
from prometheuspvesd.discovery import Discovery
from prometheuspvesd.model import HostList
from prometheuspvesd.utils import SingleLog
class PrometheusSD:
"""Main Prometheus SD object."""
def __init__(self):
self.log = SingleLog()
self.logger = self.log.logger
self.args = self._cli_args()
self.config = self._get_config()
self.discovery = Discovery()
self._fetch()
def _cli_args(self):
"""
Use argparse for parsing CLI arguments.
:return: args objec
"""
parser = argparse.ArgumentParser(description="Prometheus Service Discovery for Proxmox VE")
parser.add_argument(
"-c", "--config", dest="config_file", help="location of configuration file"
)
parser.add_argument(
"-o", "--output", dest="output_file", action="store", help="output file"
)
parser.add_argument(
"-v", dest="logging.level", action="append_const", const=-1, help="increase log level"
)
parser.add_argument(
"-q", dest="logging.level", action="append_const", const=1, help="decrease log level"
)
parser.add_argument(
"--version", action="version", version="%(prog)s {}".format(__version__)
)
return parser.parse_args().__dict__
def _get_config(self):
try:
config = SingleConfig(args=self.args)
except prometheuspvesd.exception.ConfigError as e:
self.log.sysexit_with_message(e)
try:
self.log.set_level(config.config["logging"]["level"])
except ValueError as e:
self.log.sysexit_with_message("Can not set log level.\n{}".format(str(e)))
required = [("pve.server", config.config["pve"]["server"]),
("pve.user", config.config["pve"]["user"]),
("pve.password", config.config["pve"]["password"])]
for name, value in required:
if not value:
self.log.sysexit_with_message("Option '{}' is required but not set".format(name))
self.logger.info("Using config file {}".format(config.config_file))
return config
def _fetch(self):
signal.signal(signal.SIGINT, self._terminate)
signal.signal(signal.SIGTERM, self._terminate)
loop_delay = self.config.config["loop_delay"]
output_file = self.config.config["output_file"]
self.logger.info("Writes targets to {}".format(output_file))
while True:
self.logger.debug("Propagate from PVE")
self._write(self.discovery.propagate())
self.logger.info("Waiting {} seconds for next loop".format(loop_delay))
sleep(self.config.config["loop_delay"])
def _write(self, host_list: HostList):
output = []
for host in host_list.hosts:
output.append(host.to_sd_json())
# Write to tmp file and move after write
temp_file = tempfile.NamedTemporaryFile(mode="w", prefix="prometheus-pve-sd", delete=False)
with temp_file as tf:
json.dump(output, tf, indent=4)
shutil.move(temp_file.name, self.config.config["output_file"])
def _terminate(self, signal, frame):
self.log.sysexit_with_message("Terminating", code=0)
def main():
PrometheusSD()

247
prometheuspvesd/config.py Normal file
View File

@ -0,0 +1,247 @@
#!/usr/bin/env python3
"""Global settings definition."""
import os
from pathlib import Path
from pathlib import PurePath
import anyconfig
import environs
import jsonschema.exceptions
import ruamel.yaml
from appdirs import AppDirs
from jsonschema._utils import format_as_index
import prometheuspvesd.exception
from prometheuspvesd.utils import Singleton
config_dir = AppDirs("prometheus-pve-sd").user_config_dir
default_config_file = os.path.join(config_dir, "config.yml")
cache_dir = AppDirs("prometheus-pve-sd").user_cache_dir
default_output_file = os.path.join(cache_dir, "pve.json")
class Config():
"""
Create an object with all necessary settings.
Settings are loade from multiple locations in defined order (last wins):
- default settings defined by `self._get_defaults()`
- yaml config file, defaults to OS specific user config dir (https://pypi.org/project/appdirs/)
- provides cli parameters
"""
SETTINGS = {
"config_file": {
"default": "",
"env": "CONFIG_FILE",
"type": environs.Env().str
},
"logging.level": {
"default": "WARNING",
"env": "LOG_LEVEL",
"file": True,
"type": environs.Env().str
},
"logging.json": {
"default": False,
"env": "LOG_JSON",
"file": True,
"type": environs.Env().bool
},
"output_file": {
"default": default_output_file,
"env": "output_file",
"file": True,
"type": environs.Env().str
},
"loop_delay": {
"default": 300,
"env": "LOOP_DELAY",
"file": True,
"type": environs.Env().int
},
"exclude_state": {
"default": [],
"env": "EXCLUDE_STATE",
"file": True,
"type": environs.Env().list
},
"exclude_vmid": {
"default": [],
"env": "EXCLUDE_STATE",
"file": True,
"type": environs.Env().list
},
"pve.server": {
"default": "",
"env": "PVE_SERVER",
"file": True,
"type": environs.Env().str
},
"pve.user": {
"default": "",
"env": "PVE_USER",
"file": True,
"type": environs.Env().str
},
"pve.password": {
"default": "",
"env": "PVE_PASSWORD",
"file": True,
"type": environs.Env().str
},
"pve.auth_timeout": {
"default": 5,
"env": "PVE_AUTH_TIMEOUT",
"file": True,
"type": environs.Env().int
},
"pve.verify_ssl": {
"default": True,
"env": "PVE_VERIFY_SSL",
"file": True,
"type": environs.Env().bool
},
}
def __init__(self, args={}):
"""
Initialize a new settings class.
:param args: An optional dict of options, arguments and commands from the CLI.
:param config_file: An optional path to a yaml config file.
:returns: None
"""
self._args = args
self._schema = None
self.config_file = default_config_file
self.config = None
self._set_config()
def _get_args(self, args):
cleaned = dict(filter(lambda item: item[1] is not None, args.items()))
normalized = {}
for key, value in cleaned.items():
normalized = self._add_dict_branch(normalized, key.split("."), value)
# Override correct log level from argparse
levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
log_level = levels.index(self.SETTINGS["logging.level"]["default"])
if normalized.get("logging"):
for adjustment in normalized["logging"]["level"]:
log_level = min(len(levels) - 1, max(log_level + adjustment, 0))
normalized["logging"]["level"] = levels[log_level]
return normalized
def _get_defaults(self):
normalized = {}
for key, item in self.SETTINGS.items():
normalized = self._add_dict_branch(normalized, key.split("."), item["default"])
self.schema = anyconfig.gen_schema(normalized)
return normalized
def _get_envs(self):
normalized = {}
for key, item in self.SETTINGS.items():
if item.get("env"):
prefix = "PROMETHEUS_PVE_SD_"
envname = prefix + item["env"]
try:
value = item["type"](envname)
normalized = self._add_dict_branch(normalized, key.split("."), value)
except environs.EnvError as e:
if '"{}" not set'.format(envname) in str(e):
pass
else:
raise prometheuspvesd.exception.ConfigError(
"Unable to read environment variable", str(e)
)
return normalized
def _set_config(self):
args = self._get_args(self._args)
envs = self._get_envs()
defaults = self._get_defaults()
# preset config file path
if envs.get("config_file"):
self.config_file = self._normalize_path(envs.get("config_file"))
if args.get("config_file"):
self.config_file = self._normalize_path(args.get("config_file"))
source_files = []
source_files.append(self.config_file)
for config in source_files:
if config and os.path.exists(config):
with open(config, "r", encoding="utf8") as stream:
s = stream.read()
try:
file_dict = ruamel.yaml.safe_load(s)
except (
ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError
) as e:
message = "{} {}".format(e.context, e.problem)
raise prometheuspvesd.exception.ConfigError(
"Unable to read config file {}".format(config), message
)
if self._validate(file_dict):
anyconfig.merge(defaults, file_dict, ac_merge=anyconfig.MS_DICTS)
defaults["logging"]["level"] = defaults["logging"]["level"].upper()
if self._validate(envs):
anyconfig.merge(defaults, envs, ac_merge=anyconfig.MS_DICTS)
if self._validate(args):
anyconfig.merge(defaults, args, ac_merge=anyconfig.MS_DICTS)
if "config_file" in defaults:
defaults.pop("config_file")
defaults["logging"]["level"] = defaults["logging"]["level"].upper()
Path(PurePath(self.config_file).parent).mkdir(parents=True, exist_ok=True)
Path(PurePath(defaults["output_file"]).parent).mkdir(parents=True, exist_ok=True)
self.config = defaults
def _normalize_path(self, path):
if not os.path.isabs(path):
base = os.path.join(os.getcwd(), path)
return os.path.abspath(os.path.expanduser(os.path.expandvars(base)))
else:
return path
def _validate(self, config):
try:
anyconfig.validate(config, self.schema, ac_schema_safe=False)
except jsonschema.exceptions.ValidationError as e:
schema_error = "Failed validating '{validator}' in schema{schema}\n{message}".format(
validator=e.validator,
schema=format_as_index(list(e.relative_schema_path)[:-1]),
message=e.message
)
raise prometheuspvesd.exception.ConfigError("Configuration error", schema_error)
return True
def _add_dict_branch(self, tree, vector, value):
key = vector[0]
tree[key] = value \
if len(vector) == 1 \
else self._add_dict_branch(tree[key] if key in tree else {}, vector[1:], value)
return tree
class SingleConfig(Config, metaclass=Singleton):
"""Singleton config class."""
pass

View File

@ -0,0 +1,206 @@
#!/usr/bin/env python3
"""Prometheus Discovery."""
import json
import re
import socket
from collections import defaultdict
from prometheuspvesd.config import SingleConfig
from prometheuspvesd.model import Host
from prometheuspvesd.model import HostList
from prometheuspvesd.utils import SingleLog
from prometheuspvesd.utils import to_bool
try:
from proxmoxer import ProxmoxAPI
HAS_PROXMOXER = True
except ImportError:
HAS_PROXMOXER = False
class Discovery():
"""Prometheus PVE Service Discovery."""
def __init__(self):
if not HAS_PROXMOXER:
self.logger.error(
"The Proxmox VE Prometheus SD requires proxmoxer: "
"https://pypi.org/project/proxmoxer/"
)
self.config = SingleConfig()
self.log = SingleLog()
self.logger = SingleLog().logger
self.client = self._auth()
self.host_list = HostList()
def _auth(self):
return ProxmoxAPI(
self.config.config["pve"]["server"],
user=self.config.config["pve"]["user"],
password=self.config.config["pve"]["password"],
verify_ssl=to_bool(self.config.config["pve"]["verify_ssl"]),
timeout=self.config.config["pve"]["auth_timeout"]
)
def _get_names(self, pve_list, pve_type):
names = []
if pve_type == "node":
names = [node["node"] for node in pve_list]
elif pve_type == "pool":
names = [pool["poolid"] for pool in pve_list]
return names
def _get_variables(self, pve_list, pve_type):
variables = {}
if pve_type in ["qemu", "container"]:
for vm in pve_list:
nested = {}
for key, value in vm.items():
nested["proxmox_" + key] = value
variables[vm["name"]] = nested
return variables
def _get_ip_address(self, pve_type, pve_node, vmid):
def validate(address):
try:
# IP address validation
if socket.inet_aton(address):
# Ignore localhost
if address != "127.0.0.1":
return address
except socket.error:
return False
address = False
networks = False
if pve_type == "qemu":
# If qemu agent is enabled, try to gather the IP address
try:
if self.client.nodes(pve_node).get(pve_type, vmid, "agent", "info") is not None:
networks = self.client.nodes(pve_node).get(
"qemu", vmid, "agent", "network-get-interfaces"
)["result"]
except Exception: # noqa
pass
if networks:
if type(networks) is list:
for network in networks:
for ip_address in network["ip-addresses"]:
address = validate(ip_address["ip-address"])
if not address:
try:
config = self.client.nodes(pve_node).get(pve_type, vmid, "config")
sources = [config["net0"], config["ipconfig0"]]
for s in sources:
find = re.search(r"ip=(\d*\.\d*\.\d*\.\d*)", str(sources))
if find and find.group(1):
address = find.group(1)
break
except Exception: # noqa
pass
return address
def _exclude(self, pve_list):
filtered = []
for item in pve_list:
obj = defaultdict(dict, item)
if obj["template"] == 1:
continue
if obj["status"] in self.config.config["exclude_state"]:
continue
if obj["vmid"] in self.config.config["exclude_vmid"]:
continue
filtered.append(item.copy())
return filtered
def propagate(self):
self.host_list.clear()
for node in self._get_names(self.client.nodes.get(), "node"):
try:
qemu_list = self._exclude(self.client.nodes(node).qemu.get())
container_list = self._exclude(self.client.nodes(node).lxc.get())
except Exception as e: # noqa
self.logger.error("Proxmoxer API error: {0}".format(str(e)))
# Merge QEMU and Containers lists from this node
instances = self._get_variables(qemu_list, "qemu").copy()
instances.update(self._get_variables(container_list, "container"))
self.logger.info("Found {} targets".format(len(instances)))
for host in instances:
host_meta = instances[host]
vmid = host_meta["proxmox_vmid"]
try:
pve_type = host_meta["proxmox_type"]
except KeyError:
pve_type = "qemu"
config = self.client.nodes(node).get(pve_type, vmid, "config")
try:
description = (config["description"])
except KeyError:
description = None
except Exception as e: # noqa
self.logger.error("Proxmoxer API error: {0}".format(str(e)))
try:
metadata = json.loads(description)
except TypeError:
metadata = {}
except ValueError:
metadata = {"notes": description}
address = self._get_ip_address(pve_type, node, vmid) or host
prom_host = Host(vmid, host, address, pve_type)
config_flags = [("cpu", "sockets"), ("cores", "cores"), ("memory", "memory")]
meta_flags = [("status", "proxmox_status")]
for key, flag in config_flags:
if flag in config:
prom_host.add_label(key, config[flag])
for key, flag in meta_flags:
if flag in host_meta:
prom_host.add_label(key, host_meta[flag])
if "groups" in metadata:
prom_host.add_label("groups", ",".join(metadata["groups"]))
self.host_list.add_host(prom_host)
self.logger.debug("Discovered {}".format(prom_host))
for pool in self._get_names(self.client.pools.get(), "pool"):
try:
pool_list = self._exclude(self.client.pool(pool).get()["members"])
except Exception as e: # noqa
self.logger.error("Proxmoxer API error: {0}".format(str(e)))
members = [
member["name"]
for member in pool_list
if (member["type"] == "qemu" or member["type"] == "lxc")
]
for member in members:
self.inventory.add_host(group=pool, host=member)
return self.host_list

View File

@ -0,0 +1,17 @@
#!/usr/bin/env python3
"""Custom exceptions."""
class PrometheusSDError(Exception):
"""Generic exception class for promtheus-pve-sd."""
def __init__(self, msg, original_exception=""):
super(PrometheusSDError,
self).__init__("{msg}\n{org}".format(msg=msg, org=original_exception))
self.original_exception = original_exception
class ConfigError(PrometheusSDError):
"""Errors related to config file handling."""
pass

48
prometheuspvesd/model.py Normal file
View File

@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Prometheus SD object models."""
class Host:
"""Represents a virtual machine or container in PVE."""
def __init__(self, vmid, hostname, ip_address, pve_type):
self.hostname = hostname
self.ip_address = ip_address
self.vmid = vmid
self.pve_type = pve_type
self.labels = {}
self.labels["__meta_pve_ip"] = ip_address
self.labels["__meta_pve_name"] = hostname
self.labels["__meta_pve_type"] = pve_type
self.labels["__meta_pve_vmid"] = str(vmid)
def __str__(self):
return f"{self.hostname}({self.vmid}): {self.pve_type} {self.ip_address}"
def add_label(self, key, value):
key = key.replace("-", "_").replace(" ", "_")
self.labels[f"__meta_pve_{key}"] = str(value)
def to_sd_json(self):
return {"targets": [self.hostname], "labels": self.labels}
class HostList:
"""Collection of host objects."""
def __init__(self):
self.hosts = []
def clear(self):
self.hosts = []
def add_host(self, host: Host):
if not self.host_exists(host):
self.hosts.append(host)
def host_exists(self, host: Host):
"""Check if a host is already in the list by id and type."""
for current in self.hosts:
if current.pve_type == host.pve_type and current.vmid == host.vmid:
return True
return False

242
prometheuspvesd/utils.py Normal file
View File

@ -0,0 +1,242 @@
#!/usr/bin/env python3
"""Global utility methods and classes."""
import logging
import os
import sys
from distutils.util import strtobool
import colorama
from pythonjsonlogger import jsonlogger
CONSOLE_FORMAT = "{}{}[%(levelname)s]{} %(message)s"
JSON_FORMAT = "(asctime) (levelname) (message)"
def to_bool(string):
return bool(strtobool(str(string)))
def _should_do_markup():
py_colors = os.environ.get("PY_COLORS", None)
if py_colors is not None:
return to_bool(py_colors)
return sys.stdout.isatty() and os.environ.get("TERM") != "dumb"
colorama.init(autoreset=True, strip=not _should_do_markup())
class Singleton(type):
"""Meta singleton class."""
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
return cls._instances[cls]
class LogFilter(object):
"""A custom log filter which excludes log messages above the logged level."""
def __init__(self, level):
"""
Initialize a new custom log filter.
:param level: Log level limit
:returns: None
"""
self.__level = level
def filter(self, logRecord): # noqa
# https://docs.python.org/3/library/logging.html#logrecord-attributes
return logRecord.levelno <= self.__level
class MultilineFormatter(logging.Formatter):
"""Logging Formatter to reset color after newline characters."""
def format(self, record): # noqa
record.msg = record.msg.replace("\n", "\n{}... ".format(colorama.Style.RESET_ALL))
return logging.Formatter.format(self, record)
class MultilineJsonFormatter(jsonlogger.JsonFormatter):
"""Logging Formatter to remove newline characters."""
def format(self, record): # noqa
record.msg = record.msg.replace("\n", " ")
return jsonlogger.JsonFormatter.format(self, record)
class Log:
"""Handle logging."""
def __init__(self, level=logging.WARN, name="prometheuspvesd", json=False):
self.logger = logging.getLogger(name)
self.logger.setLevel(level)
self.logger.addHandler(self._get_error_handler(json=json))
self.logger.addHandler(self._get_warn_handler(json=json))
self.logger.addHandler(self._get_info_handler(json=json))
self.logger.addHandler(self._get_critical_handler(json=json))
self.logger.addHandler(self._get_debug_handler(json=json))
self.logger.propagate = False
def _get_error_handler(self, json=False):
handler = logging.StreamHandler(sys.stderr)
handler.setLevel(logging.ERROR)
handler.addFilter(LogFilter(logging.ERROR))
handler.setFormatter(
MultilineFormatter(
self.error(
CONSOLE_FORMAT.format(
colorama.Fore.RED, colorama.Style.BRIGHT, colorama.Style.RESET_ALL
)
)
)
)
if json:
handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
return handler
def _get_warn_handler(self, json=False):
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.WARN)
handler.addFilter(LogFilter(logging.WARN))
handler.setFormatter(
MultilineFormatter(
self.warn(
CONSOLE_FORMAT.format(
colorama.Fore.YELLOW, colorama.Style.BRIGHT, colorama.Style.RESET_ALL
)
)
)
)
if json:
handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
return handler
def _get_info_handler(self, json=False):
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.INFO)
handler.addFilter(LogFilter(logging.INFO))
handler.setFormatter(
MultilineFormatter(
self.info(
CONSOLE_FORMAT.format(
colorama.Fore.CYAN, colorama.Style.BRIGHT, colorama.Style.RESET_ALL
)
)
)
)
if json:
handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
return handler
def _get_critical_handler(self, json=False):
handler = logging.StreamHandler(sys.stderr)
handler.setLevel(logging.CRITICAL)
handler.addFilter(LogFilter(logging.CRITICAL))
handler.setFormatter(
MultilineFormatter(
self.critical(
CONSOLE_FORMAT.format(
colorama.Fore.RED, colorama.Style.BRIGHT, colorama.Style.RESET_ALL
)
)
)
)
if json:
handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
return handler
def _get_debug_handler(self, json=False):
handler = logging.StreamHandler(sys.stderr)
handler.setLevel(logging.DEBUG)
handler.addFilter(LogFilter(logging.DEBUG))
handler.setFormatter(
MultilineFormatter(
self.critical(
CONSOLE_FORMAT.format(
colorama.Fore.BLUE, colorama.Style.BRIGHT, colorama.Style.RESET_ALL
)
)
)
)
if json:
handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
return handler
def set_level(self, s):
self.logger.setLevel(s)
def debug(self, msg):
"""Format info messages and return string."""
return msg
def critical(self, msg):
"""Format critical messages and return string."""
return msg
def error(self, msg):
"""Format error messages and return string."""
return msg
def warn(self, msg):
"""Format warn messages and return string."""
return msg
def info(self, msg):
"""Format info messages and return string."""
return msg
def _color_text(self, color, msg):
"""
Colorize strings.
:param color: colorama color settings
:param msg: string to colorize
:returns: string
"""
return "{}{}{}".format(color, msg, colorama.Style.RESET_ALL)
def sysexit(self, code=1):
sys.exit(code)
def sysexit_with_message(self, msg, code=1):
self.logger.critical(str(msg))
self.sysexit(code)
class SingleLog(Log, metaclass=Singleton):
"""Singleton logging class."""
pass
class UnsafeTag:
"""Handle custom yaml unsafe tag."""
yaml_tag = u"!unsafe"
def __init__(self, value):
self.unsafe = value
@staticmethod
def yaml_constructor(loader, node):
return loader.construct_scalar(node)

95
pyproject.toml Normal file
View File

@ -0,0 +1,95 @@
[tool.poetry]
authors = ["Robert Kaussow <mail@thegeeklab.de>"]
classifiers = [
"Development Status :: 5 - Production/Stable",
"Environment :: Console",
"License :: OSI Approved :: MIT License",
"Intended Audience :: Developers",
"Intended Audience :: Information Technology",
"Intended Audience :: System Administrators",
"Natural Language :: English",
"Operating System :: POSIX",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Topic :: Utilities",
]
description = "Prometheus Service Discovery for Proxmox VE."
documentation = "https://github.com/thegeeklab/prometheus-pve-sd/"
homepage = "https://github.com/thegeeklab/prometheus-pve-sd/"
include = [
"LICENSE",
]
keywords = ["prometheus", "sd", "pve", "metrics"]
license = "MIT"
name = "prometheus-pve-sd"
packages = [
{include = "prometheuspvesd"},
]
readme = "README.md"
repository = "https://github.com/thegeeklab/prometheus-pve-sd/"
version = "0.0.0"
[tool.poetry.dependencies]
anyconfig = "0.11.0"
appdirs = "1.4.4"
colorama = "0.4.4"
environs = "9.3.2"
jsonschema = "3.2.0"
nested-lookup = "0.2.22"
proxmoxer = "1.1.1"
python = "^3.6.0"
python-json-logger = "2.0.1"
requests = "2.25.1"
"ruamel.yaml" = "0.17.6"
[tool.poetry.dev-dependencies]
bandit = "1.7.0"
flake8 = "3.9.2"
flake8-blind-except = "0.2.0"
flake8-builtins = "1.5.3"
flake8-docstrings = "1.6.0"
flake8-eradicate = "1.0.0"
flake8-isort = "4.0.0"
flake8-logging-format = "0.6.0"
flake8-pep3101 = "1.3.0"
flake8-polyfill = "1.0.2"
flake8-quotes = "3.2.0"
pep8-naming = "0.11.1"
pydocstyle = "6.1.1"
pytest = "6.2.4"
pytest-cov = "2.12.0"
pytest-mock = "3.6.1"
yapf = "0.31.0"
[tool.poetry.scripts]
prometheus-pve-sd = "prometheuspvesd.cli:main"
[tool.poetry-dynamic-versioning]
enable = true
style = "semver"
vcs = "git"
[tool.isort]
default_section = "THIRDPARTY"
force_single_line = true
line_length = 99
sections = ["FUTURE", "STDLIB", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"]
skip_glob = ["**/.env*", "**/env/*", "**/.venv/*", "**/docs/*"]
[tool.pytest.ini_options]
addopts = "prometheuspvesd --cov=prometheuspvesd --cov-report=xml:coverage.xml --cov-report=term --cov-append --no-cov-on-fail"
filterwarnings = [
"ignore::FutureWarning",
"ignore:.*collections.*:DeprecationWarning",
"ignore:.*pep8.*:FutureWarning",
]
[tool.coverage.run]
omit = ["**/test/*"]
[build-system]
build-backend = "poetry.core.masonry.api"
requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]

4
renovate.json Normal file
View File

@ -0,0 +1,4 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": ["github>thegeeklab/renovate-presets"]
}

19
setup.cfg Normal file
View File

@ -0,0 +1,19 @@
[flake8]
# Explanation of errors
#
# D102: Missing docstring in public method
# D103: Missing docstring in public function
# D107: Missing docstring in __init__
# D202: No blank lines allowed after function docstring
# W503:Line break occurred before a binary operator
ignore = D102, D103, D107, D202, W503
max-line-length = 99
inline-quotes = double
exclude = .git, __pycache__, build, dist, test, *.pyc, *.egg-info, .cache, .eggs, env*
[yapf]
based_on_style = google
column_limit = 99
dedent_closing_brackets = true
coalesce_brackets = true
split_before_logical_operator = true