chore(deps): lock file maintenance (#695)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

.lycheeignore

@@ -0,0 +1 @@
+https://hub.docker.com/r/thegeeklab/*

.pre-commit-hooks.yaml

@@ -2,7 +2,9 @@
 - id: ansible-doctor
   name: ansible-doctor
   description: Create annotation based documentation for your Ansible roles.
-  entry: ansible-doctor -f
+  entry: ansible-doctor -f -qqq
   language: python
   pass_filenames: False
   always_run: True
+  additional_dependencies:
+    - .[ansible-core]

.woodpecker/build-container.yml

@@ -6,15 +6,14 @@ when:
       - ${CI_REPO_DEFAULT_BRANCH}
 
 steps:
-  build:
+  - name: build
     image: docker.io/library/python:3.12
     commands:
-      - git fetch --depth=2147483647
       - pip install poetry poetry-dynamic-versioning -qq
       - poetry build
 
-  dryrun:
-    image: quay.io/thegeeklab/wp-docker-buildx:1
+  - name: dryrun
+    image: quay.io/thegeeklab/wp-docker-buildx:4
     settings:
       containerfile: Containerfile.multiarch
       dry_run: true
@@ -26,9 +25,9 @@ steps:
     when:
       - event: [pull_request]
 
-  publish-dockerhub:
+  - name: publish-dockerhub
+    image: quay.io/thegeeklab/wp-docker-buildx:4
     group: container
-    image: quay.io/thegeeklab/wp-docker-buildx:1
     settings:
       auto_tag: true
       containerfile: Containerfile.multiarch
@@ -47,9 +46,9 @@ steps:
         branch:
           - ${CI_REPO_DEFAULT_BRANCH}
 
-  publish-quay:
+  - name: publish-quay
+    image: quay.io/thegeeklab/wp-docker-buildx:4
     group: container
-    image: quay.io/thegeeklab/wp-docker-buildx:1
     settings:
       auto_tag: true
       containerfile: Containerfile.multiarch

.woodpecker/build-package.yml

@@ -6,27 +6,25 @@ when:
       - ${CI_REPO_DEFAULT_BRANCH}
 
 steps:
-  build:
+  - name: build
     image: docker.io/library/python:3.12
     commands:
-      - git fetch --depth=2147483647
       - pip install poetry poetry-dynamic-versioning -qq
       - poetry build
 
-  checksum:
+  - name: checksum
     image: quay.io/thegeeklab/alpine-tools
     commands:
       - cd dist/ && sha256sum * > ../sha256sum.txt
 
-  changelog:
+  - name: changelog
     image: quay.io/thegeeklab/git-sv
     commands:
-      - git fetch --depth=2147483647
       - git sv current-version
       - git sv release-notes -t ${CI_COMMIT_TAG:-next} -o CHANGELOG.md
       - cat CHANGELOG.md
 
-  publish-github:
+  - name: publish-github
     image: docker.io/plugins/github-release
     settings:
       api_key:
@@ -40,15 +38,14 @@ steps:
     when:
       - event: [tag]
 
-  publish-pypi:
+  - name: publish-pypi
     image: docker.io/library/python:3.12
     secrets:
-    - source: pypi_password
-      target:  POETRY_HTTP_BASIC_PYPI_PASSWORD
-    - source: pypi_username
-      target:  POETRY_HTTP_BASIC_PYPI_USERNAME
+      - source: pypi_password
+        target: POETRY_HTTP_BASIC_PYPI_PASSWORD
+      - source: pypi_username
+        target: POETRY_HTTP_BASIC_PYPI_USERNAME
     commands:
-      - git fetch --depth=2147483647
       - pip install poetry poetry-dynamic-versioning -qq
       - poetry publish -n
     when:

.woodpecker/docs.yml

@@ -6,51 +6,44 @@ when:
       - ${CI_REPO_DEFAULT_BRANCH}
 
 steps:
-  assets:
+  - name: assets
     image: quay.io/thegeeklab/alpine-tools
     commands:
       - make doc
 
-  markdownlint:
+  - name: markdownlint
     image: quay.io/thegeeklab/markdownlint-cli
     group: test
     commands:
       - markdownlint 'README.md' 'CONTRIBUTING.md'
 
-  spellcheck:
+  - name: spellcheck
     image: quay.io/thegeeklab/alpine-tools
     group: test
     commands:
-      - spellchecker --files '_docs/**/*.md' 'README.md' 'CONTRIBUTING.md' -d .dictionary -p spell indefinite-article syntax-urls
+      - spellchecker --files 'docs/**/*.md' 'README.md' 'CONTRIBUTING.md' -d .dictionary -p spell indefinite-article syntax-urls
     environment:
       FORCE_COLOR: "true"
-      NPM_CONFIG_LOGLEVEL: "error"
 
-  link-validation:
+  - name: link-validation
     image: docker.io/lycheeverse/lychee
     group: test
     commands:
       - lychee --no-progress --format detailed docs/content README.md
 
-  testbuild:
-    image: quay.io/thegeeklab/hugo:0.115.2
-    commands:
-      - hugo --panicOnWarning -s docs/ -b http://localhost:8000/
-
-  build:
-    image: quay.io/thegeeklab/hugo:0.115.2
+  - name: build
+    image: quay.io/thegeeklab/hugo:0.125.7
     commands:
       - hugo --panicOnWarning -s docs/
 
-  beautify:
+  - name: beautify
     image: quay.io/thegeeklab/alpine-tools
     commands:
       - html-beautify -r -f 'docs/public/**/*.html'
     environment:
       FORCE_COLOR: "true"
-      NPM_CONFIG_LOGLEVEL: error
 
-  publish:
+  - name: publish
     image: quay.io/thegeeklab/wp-s3-action
     settings:
       access_key:
@@ -69,15 +62,15 @@ steps:
       - event: [push, manual]
         branch:
           - ${CI_REPO_DEFAULT_BRANCH}
-        status: [success]
+        status: [success, failure]
 
-  pushrm-dockerhub:
+  - name: pushrm-dockerhub
     image: docker.io/chko/docker-pushrm:1
     secrets:
       - source: docker_password
-        target:  DOCKER_PASS
+        target: DOCKER_PASS
       - source: docker_username
-        target:  DOCKER_USER
+        target: DOCKER_USER
     environment:
       PUSHRM_FILE: README.md
       PUSHRM_SHORT: Annotation based documentation for your Ansible roles
@@ -88,7 +81,7 @@ steps:
           - ${CI_REPO_DEFAULT_BRANCH}
         status: [success]
 
-  pushrm-quay:
+  - name: pushrm-quay
     image: docker.io/chko/docker-pushrm:1
     secrets:
       - source: quay_token

.woodpecker/lint.yml

@@ -6,22 +6,20 @@ when:
       - ${CI_REPO_DEFAULT_BRANCH}
 
 steps:
-  check-format:
+  - name: check-format
     image: docker.io/library/python:3.12
     commands:
-      - git fetch --depth=2147483647
       - pip install poetry poetry-dynamic-versioning -qq
-      - poetry install
-      - poetry run yapf -dr ./${CI_REPO_NAME//-/}
+      - poetry install -E ansible-core
+      - poetry run ruff format --check --diff ./${CI_REPO_NAME//-/}
     environment:
       PY_COLORS: "1"
 
-  check-coding:
+  - name: check-coding
     image: docker.io/library/python:3.12
     commands:
-      - git fetch --depth=2147483647
       - pip install poetry poetry-dynamic-versioning -qq
-      - poetry install
-      - poetry run ruff ./${CI_REPO_NAME//-/}
+      - poetry install -E ansible-core
+      - poetry run ruff check ./${CI_REPO_NAME//-/}
     environment:
       PY_COLORS: "1"

.woodpecker/notify.yml

@@ -8,7 +8,7 @@ when:
 runs_on: [success, failure]
 
 steps:
-  matrix:
+  - name: matrix
     image: quay.io/thegeeklab/wp-matrix
     settings:
       homeserver:

.woodpecker/test.yml

@@ -5,22 +5,30 @@ when:
     branch:
       - ${CI_REPO_DEFAULT_BRANCH}
 
-matrix:
-  PYTHON_VERSION:
-    - docker.io/library/python:3.8
-    - docker.io/library/python:3.9
-    - docker.io/library/python:3.10
-    - docker.io/library/python:3.11
-    - docker.io/library/python:3.12
-
-steps:
-  pytest:
-    image: ${PYTHON_VERSION}
+variables:
+  - &pytest_base
+    group: pytest
     commands:
-      - git fetch --depth=2147483647
       - pip install poetry poetry-dynamic-versioning -qq
-      - poetry install
+      - poetry install -E ansible-core
       - poetry version
       - poetry run ${CI_REPO_NAME} --help
     environment:
       PY_COLORS: "1"
+
+steps:
+  - name: python-312
+    image: docker.io/library/python:3.12
+    <<: *pytest_base
+
+  - name: python-311
+    image: docker.io/library/python:3.11
+    <<: *pytest_base
+
+  - name: python-310
+    image: docker.io/library/python:3.10
+    <<: *pytest_base
+
+  - name: python-39
+    image: docker.io/library/python:3.9
+    <<: *pytest_base

Containerfile.multiarch

@@ -1,4 +1,4 @@
-FROM python:3.12-alpine@sha256:a5d1738d6abbdff3e81c10b7f86923ebcb340ca536e21e8c5ee7d938d263dba1
+FROM python:3.12-alpine@sha256:ef097620baf1272e38264207003b0982285da3236a20ed829bf6bbf1e85fe3cb
 
 LABEL maintainer="Robert Kaussow <mail@thegeeklab.de>"
 LABEL org.opencontainers.image.authors="Robert Kaussow <mail@thegeeklab.de>"
@@ -14,7 +14,7 @@ ADD dist/ansible_doctor-*.whl /
 
 RUN apk --update add --virtual .build-deps build-base libffi-dev openssl-dev && \
     pip install --upgrade --no-cache-dir pip && \
-    pip install --no-cache-dir $(find / -name "ansible_doctor-*.whl") && \
+    pip install --no-cache-dir $(find / -name "ansible_doctor-*.whl")[ansible-core] && \
     rm -f ansible_doctor-*.whl && \
     rm -rf /var/cache/apk/* && \
     rm -rf /root/.cache/

Makefile

@@ -1,5 +1,5 @@
 # renovate: datasource=github-releases depName=thegeeklab/hugo-geekdoc
-THEME_VERSION := v0.41.3
+THEME_VERSION := v0.45.0
 THEME := hugo-geekdoc
 BASEDIR := docs
 THEMEDIR := $(BASEDIR)/themes

ansibledoctor/__init__.py

@@ -1,3 +1,10 @@
 """Provide version information."""
 
 __version__ = "0.0.0"
+
+import sys
+
+try:
+    import ansible  # noqa
+except ImportError:
+    sys.exit("ERROR: Python requirements are missing: 'ansible-core' not found.")

ansibledoctor/cli.py

@@ -47,7 +47,7 @@ class AnsibleDoctor:
             dest="recursive",
             action="store_true",
             default=None,
-            help="run recursively over the base directory subfolders"
+            help="run recursively over the base directory subfolders",
         )
         parser.add_argument(
             "-f",
@@ -55,7 +55,7 @@ class AnsibleDoctor:
             dest="force_overwrite",
             action="store_true",
             default=None,
-            help="force overwrite output file"
+            help="force overwrite output file",
         )
         parser.add_argument(
             "-d",
@@ -63,7 +63,7 @@ class AnsibleDoctor:
             dest="dry_run",
             action="store_true",
             default=None,
-            help="dry run without writing"
+            help="dry run without writing",
         )
         parser.add_argument(
             "-n",
@@ -71,7 +71,7 @@ class AnsibleDoctor:
             dest="role_detection",
             action="store_false",
             default=None,
-            help="disable automatic role detection"
+            help="disable automatic role detection",
         )
         parser.add_argument(
             "-v", dest="logging.level", action="append_const", const=-1, help="increase log level"

ansibledoctor/config.py

@@ -32,96 +32,96 @@ class Config:
         "config_file": {
             "default": default_config_file,
             "env": "CONFIG_FILE",
-            "type": environs.Env().str
+            "type": environs.Env().str,
         },
         "base_dir": {
             "default": os.getcwd(),
             "refresh": os.getcwd,
             "env": "BASE_DIR",
-            "type": environs.Env().str
+            "type": environs.Env().str,
         },
         "role_name": {
             "default": "",
             "env": "ROLE_NAME",
-            "type": environs.Env().str
+            "type": environs.Env().str,
         },
         "dry_run": {
             "default": False,
             "env": "DRY_RUN",
             "file": True,
-            "type": environs.Env().bool
+            "type": environs.Env().bool,
         },
         "logging.level": {
             "default": "WARNING",
             "env": "LOG_LEVEL",
             "file": True,
-            "type": environs.Env().str
+            "type": environs.Env().str,
         },
         "logging.json": {
             "default": False,
             "env": "LOG_JSON",
             "file": True,
-            "type": environs.Env().bool
+            "type": environs.Env().bool,
         },
         "output_dir": {
             "default": os.getcwd(),
             "refresh": os.getcwd,
             "env": "OUTPUT_DIR",
             "file": True,
-            "type": environs.Env().str
+            "type": environs.Env().str,
         },
         "recursive": {
             "default": False,
             "env": "RECURSIVE",
-            "type": environs.Env().bool
+            "type": environs.Env().bool,
         },
         "template_dir": {
             "default": os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates"),
             "env": "TEMPLATE_DIR",
             "file": True,
-            "type": environs.Env().str
+            "type": environs.Env().str,
         },
         "template": {
             "default": "readme",
             "env": "TEMPLATE",
             "file": True,
-            "type": environs.Env().str
+            "type": environs.Env().str,
         },
         "template_autotrim": {
             "default": True,
             "env": "TEMPLATE_AUTOTRIM",
             "file": True,
-            "type": environs.Env().bool
+            "type": environs.Env().bool,
         },
         "force_overwrite": {
             "default": False,
             "env": "FORCE_OVERWRITE",
             "file": True,
-            "type": environs.Env().bool
+            "type": environs.Env().bool,
         },
         "custom_header": {
             "default": "",
             "env": "CUSTOM_HEADER",
             "file": True,
-            "type": environs.Env().str
+            "type": environs.Env().str,
         },
         "exclude_files": {
             "default": [],
             "env": "EXCLUDE_FILES",
             "file": True,
-            "type": environs.Env().list
+            "type": environs.Env().list,
         },
         "exclude_tags": {
             "default": [],
             "env": "EXCLUDE_TAGS",
             "file": True,
-            "type": environs.Env().list
+            "type": environs.Env().list,
         },
         "role_detection": {
             "default": True,
             "env": "ROLE_DETECTION",
             "file": True,
-            "type": environs.Env().bool
+            "type": environs.Env().bool,
         },
     }
 
@@ -130,31 +130,31 @@ class Config:
             "name": "meta",
             "automatic": True,
             "subtypes": ["value"],
-            "allow_multiple": False
+            "allow_multiple": False,
         },
         "todo": {
             "name": "todo",
             "automatic": True,
             "subtypes": ["value"],
-            "allow_multiple": True
+            "allow_multiple": True,
         },
         "var": {
             "name": "var",
             "automatic": True,
             "subtypes": ["value", "example", "description", "type", "deprecated"],
-            "allow_multiple": False
+            "allow_multiple": False,
         },
         "example": {
             "name": "example",
             "automatic": True,
             "subtypes": [],
-            "allow_multiple": False
+            "allow_multiple": False,
         },
         "tag": {
             "name": "tag",
             "automatic": True,
             "subtypes": ["value", "description"],
-            "allow_multiple": False
+            "allow_multiple": False,
         },
     }
 
@@ -263,14 +263,15 @@ class Config:
         source_files.append((os.path.join(os.getcwd(), ".ansibledoctor.yml"), True))
         source_files.append((os.path.join(os.getcwd(), ".ansibledoctor.yaml"), True))
 
-        for (config, first_found) in source_files:
+        for config, first_found in source_files:
             if config and os.path.exists(config):
                 with open(config, encoding="utf8") as stream:
                     s = stream.read()
                     try:
-                        file_dict = ruamel.yaml.safe_load(s)
+                        file_dict = ruamel.yaml.YAML(typ="safe", pure=True).load(s)
                     except (
-                        ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError
+                        ruamel.yaml.composer.ComposerError,
+                        ruamel.yaml.scanner.ScannerError,
                     ) as e:
                         message = f"{e.context} {e.problem}"
                         raise ansibledoctor.exception.ConfigError(
@@ -311,27 +312,26 @@ class Config:
         try:
             anyconfig.validate(config, self.schema, ac_schema_safe=False)
         except jsonschema.exceptions.ValidationError as e:
-            schema_error = "Failed validating '{validator}' in schema{schema}\n{message}".format(
-                validator=e.validator,
-                schema=format_as_index(list(e.relative_schema_path)[:-1]),
-                message=e.message
-            )
+            schema = format_as_index(list(e.relative_schema_path)[:-1])
+            schema_error = f"Failed validating '{e.validator}' in schema {schema}\n{e.message}"
             raise ansibledoctor.exception.ConfigError("Configuration error", schema_error) from e
 
         return True
 
     def _add_dict_branch(self, tree, vector, value):
         key = vector[0]
-        tree[key] = value \
-            if len(vector) == 1 \
-            else self._add_dict_branch(tree[key] if key in tree else {}, vector[1:], value)
+        tree[key] = (
+            value
+            if len(vector) == 1
+            else self._add_dict_branch(tree.get(key, {}), vector[1:], value)
+        )
         return tree
 
     def get_annotations_definition(self, automatic=True):
         annotations = {}
         if automatic:
             for k, item in self.ANNOTATIONS.items():
-                if "automatic" in item and item["automatic"]:
+                if item.get("automatic"):
                     annotations[k] = item
         return annotations
 
@@ -339,7 +339,7 @@ class Config:
         annotations = []
         if automatic:
             for k, item in self.ANNOTATIONS.items():
-                if "automatic" in item and item["automatic"]:
+                if item.get("automatic"):
                     annotations.append(k)
         return annotations
 

ansibledoctor/doc_generator.py

@@ -43,7 +43,7 @@ class Generator:
             self.log.sysexit_with_message(f"Can not open template dir {template_dir}")
 
         for file in glob.iglob(template_dir + "/**/*." + self.extension, recursive=True):
-            relative_file = file[len(template_dir) + 1:]
+            relative_file = file[len(template_dir) + 1 :]
             if ntpath.basename(file)[:1] != "_":
                 self.logger.debug(f"Found template file: {relative_file}")
                 self.template_files.append(relative_file)
@@ -63,8 +63,7 @@ class Generator:
 
         for file in self.template_files:
             doc_file = os.path.join(
-                self.config.config.get("output_dir"),
-                os.path.splitext(file)[0]
+                self.config.config.get("output_dir"), os.path.splitext(file)[0]
             )
             if os.path.isfile(doc_file):
                 files_to_overwite.append(doc_file)
@@ -81,14 +80,17 @@ class Generator:
                 self.log.sysexit_with_message(f"Can not open custom header file\n{e!s}")
 
         if (
-            len(files_to_overwite) > 0 and self.config.config.get("force_overwrite") is False
+            len(files_to_overwite) > 0
+            and self.config.config.get("force_overwrite") is False
             and not self.config.config["dry_run"]
         ):
             files_to_overwite_string = "\n".join(files_to_overwite)
-            self.logger.warning(f"This files will be overwritten:\n{files_to_overwite_string}")
+            prompt = f"These files will be overwritten:\n{files_to_overwite_string}".replace(
+                "\n", "\n... "
+            )
 
             try:
-                if not FileUtils.query_yes_no("Do you want to continue?"):
+                if not FileUtils.query_yes_no(f"{prompt}\nDo you want to continue?"):
                     self.log.sysexit_with_message("Aborted...")
             except ansibledoctor.exception.InputError as e:
                 self.logger.debug(str(e))
@@ -96,8 +98,7 @@ class Generator:
 
         for file in self.template_files:
             doc_file = os.path.join(
-                self.config.config.get("output_dir"),
-                os.path.splitext(file)[0]
+                self.config.config.get("output_dir"), os.path.splitext(file)[0]
             )
             source_file = self.config.get_template() + "/" + file
 
@@ -115,7 +116,7 @@ class Generator:
                                 loader=FileSystemLoader(self.config.get_template()),
                                 lstrip_blocks=True,
                                 trim_blocks=True,
-                                autoescape=jinja2.select_autoescape()
+                                autoescape=jinja2.select_autoescape(),
                             )
                             jenv.filters["to_nice_yaml"] = self._to_nice_yaml
                             jenv.filters["deep_get"] = self._deep_get
@@ -133,7 +134,7 @@ class Generator:
                         except (
                             jinja2.exceptions.UndefinedError,
                             jinja2.exceptions.TemplateSyntaxError,
-                            jinja2.exceptions.TemplateRuntimeError
+                            jinja2.exceptions.TemplateRuntimeError,
                         ) as e:
                             self.log.sysexit_with_message(
                                 f"Jinja2 templating error while loading file: '{file}'\n{e!s}"
@@ -154,8 +155,9 @@ class Generator:
     def _deep_get(self, _, dictionary, keys):
         default = None
         return reduce(
-            lambda d, key: d.get(key, default)
-            if isinstance(d, dict) else default, keys.split("."), dictionary
+            lambda d, key: d.get(key, default) if isinstance(d, dict) else default,
+            keys.split("."),
+            dictionary,
         )
 
     @pass_eval_context

ansibledoctor/doc_parser.py

@@ -3,17 +3,16 @@
 
 import fnmatch
 from collections import defaultdict
-from contextlib import suppress
 
 import anyconfig
-import ruamel.yaml
-from nested_lookup import nested_lookup
 
 from ansibledoctor.annotation import Annotation
 from ansibledoctor.config import SingleConfig
 from ansibledoctor.contstants import YAML_EXTENSIONS
+from ansibledoctor.exception import YAMLError
 from ansibledoctor.file_registry import Registry
-from ansibledoctor.utils import SingleLog, UnsafeTag, flatten
+from ansibledoctor.utils import SingleLog, flatten
+from ansibledoctor.utils.yamlhelper import parse_yaml, parse_yaml_ansible
 
 
 class Parser:
@@ -31,96 +30,57 @@ class Parser:
         self._parse_task_tags()
         self._populate_doc_data()
 
-    def _yaml_remove_comments(self, d):
-        if isinstance(d, dict):
-            for k, v in d.items():
-                self._yaml_remove_comments(k)
-                self._yaml_remove_comments(v)
-        elif isinstance(d, list):
-            for elem in d:
-                self._yaml_remove_comments(elem)
-
-        with suppress(AttributeError):
-            attr = "comment" if isinstance(
-                d, ruamel.yaml.scalarstring.ScalarString
-            ) else ruamel.yaml.comments.Comment.attrib
-            delattr(d, attr)
-
     def _parse_var_files(self):
         for rfile in self._files_registry.get_files():
             if any(fnmatch.fnmatch(rfile, "*/defaults/*." + ext) for ext in YAML_EXTENSIONS):
-                with open(rfile, encoding="utf8") as yaml_file:
+                with open(rfile, encoding="utf8") as yamlfile:
                     try:
-                        ruamel.yaml.add_constructor(
-                            UnsafeTag.yaml_tag,
-                            UnsafeTag.yaml_constructor,
-                            constructor=ruamel.yaml.SafeConstructor
-                        )
+                        raw = parse_yaml(yamlfile)
+                    except YAMLError as e:
+                        self.log.sysexit_with_message(f"Unable to read yaml file {rfile}\n{e}")
 
-                        raw = ruamel.yaml.YAML(typ="rt").load(yaml_file)
-                        self._yaml_remove_comments(raw)
+                    data = defaultdict(dict, raw or {})
 
-                        data = defaultdict(dict, raw or {})
-                        for key, value in data.items():
-                            self._data["var"][key] = {"value": {key: value}}
-                    except (
-                        ruamel.yaml.composer.ComposerError,
-                        ruamel.yaml.scanner.ScannerError,
-                        ruamel.yaml.constructor.ConstructorError,
-                        ruamel.yaml.constructor.DuplicateKeyError,
-                    ) as e:
-                        message = f"{e.context} {e.problem}"
-                        self.log.sysexit_with_message(
-                            f"Unable to read yaml file {rfile}\n{message}"
-                        )
+                    for key, value in data.items():
+                        self._data["var"][key] = {"value": {key: value}}
 
     def _parse_meta_file(self):
         self._data["meta"]["name"] = {"value": self.config.config["role_name"]}
 
         for rfile in self._files_registry.get_files():
             if any("meta/main." + ext in rfile for ext in YAML_EXTENSIONS):
-                with open(rfile, encoding="utf8") as yaml_file:
+                with open(rfile, encoding="utf8") as yamlfile:
                     try:
-                        raw = ruamel.yaml.YAML(typ="rt").load(yaml_file)
-                        self._yaml_remove_comments(raw)
+                        raw = parse_yaml(yamlfile)
+                    except YAMLError as e:
+                        self.log.sysexit_with_message(f"Unable to read yaml file {rfile}\n{e}")
 
-                        data = defaultdict(dict, raw)
-                        if data.get("galaxy_info"):
-                            for key, value in data.get("galaxy_info").items():
-                                self._data["meta"][key] = {"value": value}
+                    data = defaultdict(dict, raw)
+                    if data.get("galaxy_info"):
+                        for key, value in data.get("galaxy_info").items():
+                            self._data["meta"][key] = {"value": value}
 
-                        if data.get("dependencies") is not None:
-                            self._data["meta"]["dependencies"] = {
-                                "value": data.get("dependencies")
-                            }
-                    except (
-                        ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError
-                    ) as e:
-                        message = f"{e.context} {e.problem}"
-                        self.log.sysexit_with_message(
-                            f"Unable to read yaml file {rfile}\n{message}"
-                        )
+                    if data.get("dependencies") is not None:
+                        self._data["meta"]["dependencies"] = {"value": data.get("dependencies")}
 
     def _parse_task_tags(self):
         for rfile in self._files_registry.get_files():
             if any(fnmatch.fnmatch(rfile, "*/tasks/*." + ext) for ext in YAML_EXTENSIONS):
-                with open(rfile, encoding="utf8") as yaml_file:
+                with open(rfile, encoding="utf8") as yamlfile:
                     try:
-                        raw = ruamel.yaml.YAML(typ="rt").load(yaml_file)
-                        self._yaml_remove_comments(raw)
+                        raw = parse_yaml_ansible(yamlfile)
+                    except YAMLError as e:
+                        self.log.sysexit_with_message(f"Unable to read yaml file {rfile}\n{e}")
 
-                        tags = list(set(flatten(nested_lookup("tags", raw))))
-                        for tag in [
-                            x for x in tags if x not in self.config.config["exclude_tags"]
-                        ]:
-                            self._data["tag"][tag] = {"value": tag}
-                    except (
-                        ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError
-                    ) as e:
-                        message = f"{e.context} {e.problem}"
-                        self.log.sysexit_with_message(
-                            f"Unable to read yaml file {rfile}\n{message}"
-                        )
+                    tags = [
+                        task.get("tags")
+                        for task in raw
+                        if task.get("tags")
+                        and task.get("tags") not in self.config.config["exclude_tags"]
+                    ]
+
+                    for tag in flatten(tags):
+                        self._data["tag"][tag] = {"value": tag}
 
     def _populate_doc_data(self):
         """Generate the documentation data object."""

ansibledoctor/exception.py

@@ -10,6 +10,12 @@ class DoctorError(Exception):
         self.original_exception = original_exception
 
 
+class YAMLError(DoctorError):
+    """Errors while reading a yaml file."""
+
+    pass
+
+
 class ConfigError(DoctorError):
     """Errors related to config file handling."""
 

ansibledoctor/utils/__init__.py

@@ -30,7 +30,7 @@ def strtobool(value):
         "f": False,
         "false": False,
         "off": False,
-        "0": False
+        "0": False,
     }
 
     try:
@@ -128,15 +128,15 @@ class LogFilter:
 class MultilineFormatter(logging.Formatter):
     """Reset color after newline characters."""
 
-    def format(self, record):  # noqa
-        record.msg = record.msg.replace("\n", f"\n{colorama.Style.RESET_ALL}... ")
+    def format(self, record):
+        record.msg = record.msg.strip().replace("\n", f"\n{colorama.Style.RESET_ALL}... ")
         return logging.Formatter.format(self, record)
 
 
 class MultilineJsonFormatter(jsonlogger.JsonFormatter):
     """Remove newline characters."""
 
-    def format(self, record):  # noqa
+    def format(self, record):
         record.msg = record.msg.replace("\n", " ")
         return jsonlogger.JsonFormatter.format(self, record)
 
@@ -236,7 +236,7 @@ class Log:
         handler.addFilter(LogFilter(logging.DEBUG))
         handler.setFormatter(
             MultilineFormatter(
-                self.critical(
+                self.debug(
                     CONSOLE_FORMAT.format(
                         colorama.Fore.BLUE, colorama.Style.BRIGHT, colorama.Style.RESET_ALL
                     )
@@ -287,7 +287,7 @@ class Log:
         sys.exit(code)
 
     def sysexit_with_message(self, msg, code=1):
-        self.logger.critical(str(msg))
+        self.logger.critical(str(msg.strip()))
         self.sysexit(code)
 
 
@@ -297,19 +297,6 @@ class SingleLog(Log, metaclass=Singleton):
     pass
 
 
-class UnsafeTag:
-    """Handle custom yaml unsafe tag."""
-
-    yaml_tag = "!unsafe"
-
-    def __init__(self, value):
-        self.unsafe = value
-
-    @staticmethod
-    def yaml_constructor(loader, node):
-        return loader.construct_scalar(node)
-
-
 class FileUtils:
     """Mics static methods for file handling."""
 

ansibledoctor/utils/yamlhelper.py

@@ -0,0 +1,85 @@
+"""Utils for YAML file operations."""
+
+from collections import defaultdict
+from contextlib import suppress
+
+import ruamel.yaml
+from ansible.parsing.yaml.loader import AnsibleLoader
+
+import ansibledoctor.exception
+
+
+class UnsafeTag:
+    """Handle custom yaml unsafe tag."""
+
+    yaml_tag = "!unsafe"
+
+    def __init__(self, value):
+        self.unsafe = value
+
+    @staticmethod
+    def yaml_constructor(loader, node):
+        return loader.construct_scalar(node)
+
+
+def parse_yaml_ansible(yamlfile):
+    try:
+        loader = AnsibleLoader(yamlfile)
+        data = loader.get_single_data() or []
+    except (
+        ruamel.yaml.parser.ParserError,
+        ruamel.yaml.scanner.ScannerError,
+        ruamel.yaml.constructor.ConstructorError,
+        ruamel.yaml.composer.ComposerError,
+    ) as e:
+        message = (
+            f"{e.context} in line {e.context_mark.line}, column {e.context_mark.line}\n"
+            f"{e.problem} in line {e.problem_mark.line}, column {e.problem_mark.column}"
+        )
+        raise ansibledoctor.exception.YAMLError(message) from e
+
+    return data
+
+
+def parse_yaml(yamlfile):
+    try:
+        ruamel.yaml.add_constructor(
+            UnsafeTag.yaml_tag,
+            UnsafeTag.yaml_constructor,
+            constructor=ruamel.yaml.SafeConstructor,
+        )
+
+        data = ruamel.yaml.YAML(typ="rt").load(yamlfile)
+        _yaml_remove_comments(data)
+        data = defaultdict(dict, data or {})
+    except (
+        ruamel.yaml.parser.ParserError,
+        ruamel.yaml.scanner.ScannerError,
+        ruamel.yaml.constructor.ConstructorError,
+        ruamel.yaml.composer.ComposerError,
+    ) as e:
+        message = (
+            f"{e.context} in line {e.context_mark.line}, column {e.context_mark.line}\n"
+            f"{e.problem} in line {e.problem_mark.line}, column {e.problem_mark.column}"
+        )
+        raise ansibledoctor.exception.YAMLError(message) from e
+
+    return data
+
+
+def _yaml_remove_comments(d):
+    if isinstance(d, dict):
+        for k, v in d.items():
+            _yaml_remove_comments(k)
+            _yaml_remove_comments(v)
+    elif isinstance(d, list):
+        for elem in d:
+            _yaml_remove_comments(elem)
+
+    with suppress(AttributeError):
+        attr = (
+            "comment"
+            if isinstance(d, ruamel.yaml.scalarstring.ScalarString)
+            else ruamel.yaml.comments.Comment.attrib
+        )
+        delattr(d, attr)

docs/content/setup/pip.md

@@ -4,11 +4,12 @@ title: Using pip
 
 ```Shell
 # From PyPI as unprivileged user
-$ pip install ansible-doctor --user
+$ pip install ansible-doctor[ansible-core] --user
 
 # .. or as root
-$ sudo pip install ansible-doctor
+$ sudo pip install ansible-doctor[ansible-core]
 
 # From Wheel file
-$ pip install https://github.com/thegeeklab/ansible-doctor/releases/download/v0.1.1/ansible_doctor-0.1.1-py2.py3-none-any.whl
+# Please check first whether a newer version is available.
+$ pip install https://github.com/thegeeklab/ansible-doctor/releases/download/v3.1.4/ansible_doctor-3.1.4-py2.py3-none-any.whl[ansible-core]
 ```

docs/content/usage/configuration.md

@@ -119,8 +119,8 @@ To use _ansible-doctor_ with the [pre-commit](https://pre-commit.com/) framework
 
 {{< highlight yaml "linenos=table" >}}
 - repo: https://github.com/thegeeklab/ansible-doctor
-  # change ref to the latest release from https://github.com/thegeeklab/ansible-doctor/releases
-  rev: v1.4.8
+  # update version with `pre-commit autoupdate`
+  rev: v4.0.4
   hooks:
     - id: ansible-doctor
 {{< /highlight >}}

example/demo-role/meta/main.yml

@@ -18,6 +18,5 @@ galaxy_info:
     - documentation
 
 dependencies:
-  - role1
   - role: role2
   - name: namespace.role3

example/demo-role/tasks/main.yml

@@ -11,6 +11,8 @@
 - name: Demo task with a tag list
   debug:
     msg: "Demo message"
+    tags:
+      - module-tag
   tags:
     - role-tag1
     - role-tag2

pyproject.toml

@@ -10,7 +10,6 @@ classifiers = [
   "Natural Language :: English",
   "Operating System :: POSIX",
   "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
@@ -22,42 +21,40 @@ classifiers = [
 description = "Generate documentation from annotated Ansible roles using templates."
 documentation = "https://ansible-doctor.geekdocs.de/"
 homepage = "https://ansible-doctor.geekdocs.de/"
-include = [
-  "LICENSE",
-]
+include = ["LICENSE"]
 keywords = ["ansible", "role", "documentation"]
 license = "GPL-3.0-only"
 name = "ansible-doctor"
-packages = [
-  {include = "ansibledoctor"},
-]
+packages = [{ include = "ansibledoctor" }]
 readme = "README.md"
 repository = "https://github.com/thegeeklab/ansible-doctor/"
 version = "0.0.0"
 
 [tool.poetry.dependencies]
-Jinja2 = "3.1.2"
-anyconfig = "0.13.0"
+Jinja2 = "3.1.4"
+anyconfig = "0.14.0"
 appdirs = "1.4.4"
 colorama = "0.4.6"
-environs = "9.5.0"
-jsonschema = "4.19.1"
-nested-lookup = "0.2.25"
-pathspec = "0.11.2"
-python = "^3.8.0"
+environs = "11.0.0"
+jsonschema = "4.22.0"
+pathspec = "0.12.1"
+python = "^3.9.0"
 python-json-logger = "2.0.7"
-"ruamel.yaml" = "0.17.40"
+"ruamel.yaml" = "0.18.6"
+ansible-core = { version = "2.14.16", optional = true }
+
+[tool.poetry.extras]
+ansible-core = ["ansible-core"]
 
 [tool.poetry.scripts]
 ansible-doctor = "ansibledoctor.cli:main"
 
 [tool.poetry.group.dev.dependencies]
-ruff = "0.0.292"
-pytest = "7.4.2"
-pytest-mock = "3.11.1"
-pytest-cov = "4.1.0"
+ruff = "0.4.4"
+pytest = "8.2.0"
+pytest-mock = "3.14.0"
+pytest-cov = "5.0.0"
 toml = "0.10.2"
-yapf = "0.40.2"
 
 [tool.poetry-dynamic-versioning]
 enable = true
@@ -81,17 +78,22 @@ requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]
 
 [tool.ruff]
 exclude = [
-    ".git",
-    "__pycache__",
-    "build",
-    "dist",
-    "test",
-    "*.pyc",
-    "*.egg-info",
-    ".cache",
-    ".eggs",
-    "env*",
+  ".git",
+  "__pycache__",
+  "build",
+  "dist",
+  "test",
+  "*.pyc",
+  "*.egg-info",
+  ".cache",
+  ".eggs",
+  "env*",
 ]
+
+line-length = 99
+indent-width = 4
+
+[tool.ruff.lint]
 # Explanation of errors
 #
 # D102: Missing docstring in public method
@@ -102,47 +104,39 @@ exclude = [
 # D203: One blank line required before class docstring
 # D212: Multi-line docstring summary should start at the first line
 ignore = [
-    "D102",
-    "D103",
-    "D105",
-    "D107",
-    "D202",
-    "D203",
-    "D212",
-    "UP038",
-    "RUF012",
+  "D102",
+  "D103",
+  "D105",
+  "D107",
+  "D202",
+  "D203",
+  "D212",
+  "UP038",
+  "RUF012",
 ]
-line-length = 99
 select = [
-    "D",
-    "E",
-    "F",
-    "Q",
-    "W",
-    "I",
-    "S",
-    "BLE",
-    "N",
-    "UP",
-    "B",
-    "A",
-    "C4",
-    "T20",
-    "SIM",
-    "RET",
-    "ARG",
-    "ERA",
-    "RUF",
+  "D",
+  "E",
+  "F",
+  "Q",
+  "W",
+  "I",
+  "S",
+  "BLE",
+  "N",
+  "UP",
+  "B",
+  "A",
+  "C4",
+  "T20",
+  "SIM",
+  "RET",
+  "ARG",
+  "ERA",
+  "RUF",
 ]
 
-[tool.ruff.flake8-quotes]
-inline-quotes = "double"
-
-[tool.yapf]
-based_on_style = "google"
-column_limit = 99
-dedent_closing_brackets = true
-coalesce_brackets = true
-split_before_logical_operator = true
-indent_dictionary_value = true
-allow_split_before_dict_value = false
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+line-ending = "lf"

renovate.json

@@ -1,4 +1,17 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": ["github>thegeeklab/renovate-presets"]
+  "extends": ["github>thegeeklab/renovate-presets"],
+  "packageRules": [
+    {
+      "description": "Ansible base dependencies",
+      "matchPackageNames": ["ansible-core"],
+      "separateMinorPatch": true
+    },
+    {
+      "matchManagers": ["woodpecker"],
+      "matchFileNames": [".woodpecker/test.yml"],
+      "matchPackageNames": ["docker.io/library/python"],
+      "enabled": false
+    }
+  ]
 }