From 8a8647512c27a7d9da29b4713e225277c900e5fa Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Sun, 12 Feb 2023 12:57:57 +0100 Subject: [PATCH] feat: add option to run ansible-doctor recursively (#451) --- ansibledoctor/cli.py | 62 +++++++++++------ ansibledoctor/config.py | 97 ++++++++++++++++----------- ansibledoctor/file_registry.py | 12 ++-- docs/content/setup/docker.md | 5 +- docs/content/usage/configuration.md | 16 +++-- example/{ => demo-role}/README.md | 0 example/other-role/.ansibledoctor.yml | 5 ++ example/other-role/HEADER.md | 4 ++ example/other-role/README.md | 63 +++++++++++++++++ example/other-role/defaults/main.yml | 4 ++ example/other-role/meta/main.yml | 21 ++++++ example/other-role/tasks/main.yml | 16 +++++ 12 files changed, 229 insertions(+), 76 deletions(-) rename example/{ => demo-role}/README.md (100%) create mode 100644 example/other-role/.ansibledoctor.yml create mode 100644 example/other-role/HEADER.md create mode 100644 example/other-role/README.md create mode 100644 example/other-role/defaults/main.yml create mode 100644 example/other-role/meta/main.yml create mode 100644 example/other-role/tasks/main.yml diff --git a/ansibledoctor/cli.py b/ansibledoctor/cli.py index d65f7aa..3e31291 100644 --- a/ansibledoctor/cli.py +++ b/ansibledoctor/cli.py @@ -2,6 +2,7 @@ """Entrypoint and CLI handler.""" import argparse +import os import ansibledoctor.exception from ansibledoctor import __version__ @@ -19,10 +20,7 @@ class AnsibleDoctor: self.logger = self.log.logger self.args = self._cli_args() self.config = self._get_config() - - doc_parser = Parser() - doc_generator = Generator(doc_parser) - doc_generator.render() + self._execute() def _cli_args(self): """ @@ -35,13 +33,21 @@ class AnsibleDoctor: description="Generate documentation from annotated Ansible roles using templates" ) parser.add_argument( - "role_dir", nargs="?", help="role directory (default: current working dir)" + "base_dir", nargs="?", help="base directory (default: current working directory)" ) parser.add_argument( - "-c", "--config", dest="config_file", help="location of configuration file" + "-c", "--config", dest="config_file", help="path to configuration file" ) parser.add_argument( - "-o", "--output", dest="output_dir", action="store", help="output base dir" + "-o", "--output", dest="output_dir", action="store", help="output directory" + ) + parser.add_argument( + "-r", + "--recursive", + dest="recursive", + action="store_true", + default=None, + help="run recursively over the base directory subfolders" ) parser.add_argument( "-f", @@ -83,22 +89,38 @@ class AnsibleDoctor: except ansibledoctor.exception.ConfigError as e: self.log.sysexit_with_message(e) - try: - self.log.set_level(config.config["logging"]["level"]) - except ValueError as e: - self.log.sysexit_with_message(f"Can not set log level.\n{str(e)}") + return config - if config.config["role_detection"]: - if config.is_role: - self.logger.info("Ansible role detected") - else: - self.log.sysexit_with_message("No Ansible role detected") - else: - self.logger.info("Ansible role detection disabled") + def _execute(self): + cwd = self.config.base_dir + walkdirs = [cwd] - self.logger.info(f"Using config file {config.config_file}") + if self.config.recursive: + walkdirs = [f.path for f in os.scandir(cwd) if f.is_dir()] - return config + for item in walkdirs: + os.chdir(item) + + self.config.set_config(base_dir=os.getcwd()) + try: + self.log.set_level(self.config.config["logging"]["level"]) + except ValueError as e: + self.log.sysexit_with_message(f"Can not set log level.\n{str(e)}") + self.logger.info(f"Using config file: {self.config.config_file}") + + self.logger.debug(f"Using working dir: {item}") + + if self.config.config["role_detection"]: + if self.config.is_role: + self.logger.info(f"Ansible role detected: {self.config.config['role_name']}") + else: + self.log.sysexit_with_message("No Ansible role detected") + else: + self.logger.info("Ansible role detection disabled") + + doc_parser = Parser() + doc_generator = Generator(doc_parser) + doc_generator.render() def main(): diff --git a/ansibledoctor/config.py b/ansibledoctor/config.py index 54ab504..c226b8c 100644 --- a/ansibledoctor/config.py +++ b/ansibledoctor/config.py @@ -15,6 +15,7 @@ from ansibledoctor.utils import Singleton config_dir = AppDirs("ansible-doctor").user_config_dir default_config_file = os.path.join(config_dir, "config.yml") +default_envs_prefix = "ANSIBLE_DOCTOR_" class Config(): @@ -29,13 +30,14 @@ class Config(): SETTINGS = { "config_file": { - "default": "", + "default": default_config_file, "env": "CONFIG_FILE", "type": environs.Env().str }, - "role_dir": { - "default": "", - "env": "ROLE_DIR", + "base_dir": { + "default": os.getcwd(), + "refresh": os.getcwd, + "env": "BASE_DIR", "type": environs.Env().str }, "role_name": { @@ -63,10 +65,16 @@ class Config(): }, "output_dir": { "default": os.getcwd(), + "refresh": os.getcwd, "env": "OUTPUT_DIR", "file": True, "type": environs.Env().str }, + "recursive": { + "default": False, + "env": "RECURSIVE", + "type": environs.Env().bool + }, "template_dir": { "default": os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates"), "env": "TEMPLATE_DIR", @@ -158,11 +166,9 @@ class Config(): else: self._args = args self._schema = None - self.config_file = default_config_file - self.role_dir = os.getcwd() self.config = None - self._set_config() - self.is_role = self._set_is_role() or False + self.is_role = False + self.set_config() def _get_args(self, args): cleaned = dict(filter(lambda item: item[1] is not None, args.items())) @@ -184,11 +190,10 @@ class Config(): def _get_defaults(self): normalized = {} for key, item in self.SETTINGS.items(): + if item.get("refresh"): + item["default"] = item["refresh"]() normalized = self._add_dict_branch(normalized, key.split("."), item["default"]) - # compute role_name default - normalized["role_name"] = os.path.basename(self.role_dir) - self.schema = anyconfig.gen_schema(normalized) return normalized @@ -196,8 +201,7 @@ class Config(): normalized = {} for key, item in self.SETTINGS.items(): if item.get("env"): - prefix = "ANSIBLE_DOCTOR_" - envname = prefix + item["env"] + envname = f"{default_envs_prefix}{item['env']}" try: value = item["type"](envname) normalized = self._add_dict_branch(normalized, key.split("."), value) @@ -211,29 +215,49 @@ class Config(): return normalized - def _set_config(self): + def set_config(self, base_dir=None): args = self._get_args(self._args) envs = self._get_envs() defaults = self._get_defaults() - # preset config file path + self.recursive = defaults.get("recursive") + if envs.get("recursive"): + self.recursive = envs.get("recursive") + if args.get("recursive"): + self.recursive = args.get("recursive") + if "recursive" in defaults: + defaults.pop("recursive") + + self.config_file = defaults.get("config_file") if envs.get("config_file"): self.config_file = self._normalize_path(envs.get("config_file")) - if envs.get("role_dir"): - self.role_dir = self._normalize_path(envs.get("role_dir")) - if args.get("config_file"): self.config_file = self._normalize_path(args.get("config_file")) - if args.get("role_dir"): - self.role_dir = self._normalize_path(args.get("role_dir")) + if "config_file" in defaults: + defaults.pop("config_file") + + self.base_dir = defaults.get("base_dir") + if envs.get("base_dir"): + self.base_dir = self._normalize_path(envs.get("base_dir")) + if args.get("base_dir"): + self.base_dir = self._normalize_path(args.get("base_dir")) + if base_dir: + self.base_dir = base_dir + if "base_dir" in defaults: + defaults.pop("base_dir") + + self.is_role = os.path.isdir(os.path.join(self.base_dir, "tasks")) + + # compute role_name default + defaults["role_name"] = os.path.basename(self.base_dir) source_files = [] - source_files.append(self.config_file) - source_files.append(os.path.join(os.getcwd(), ".ansibledoctor")) - source_files.append(os.path.join(os.getcwd(), ".ansibledoctor.yml")) - source_files.append(os.path.join(os.getcwd(), ".ansibledoctor.yaml")) + source_files.append((self.config_file, False)) + source_files.append((os.path.join(os.getcwd(), ".ansibledoctor"), True)) + source_files.append((os.path.join(os.getcwd(), ".ansibledoctor.yml"), True)) + source_files.append((os.path.join(os.getcwd(), ".ansibledoctor.yaml"), True)) - for config in source_files: + for (config, first_found) in source_files: if config and os.path.exists(config): with open(config, encoding="utf8") as stream: s = stream.read() @@ -244,13 +268,17 @@ class Config(): ) as e: message = f"{e.context} {e.problem}" raise ansibledoctor.exception.ConfigError( - f"Unable to read config file {config}", message + f"Unable to read config file: {config}", message ) from e if self._validate(file_dict): anyconfig.merge(defaults, file_dict, ac_merge=anyconfig.MS_DICTS) defaults["logging"]["level"] = defaults["logging"]["level"].upper() + self.config_file = config + if first_found: + break + if self._validate(envs): anyconfig.merge(defaults, envs, ac_merge=anyconfig.MS_DICTS) @@ -258,14 +286,9 @@ class Config(): anyconfig.merge(defaults, args, ac_merge=anyconfig.MS_DICTS) fix_files = ["output_dir", "template_dir", "custom_header"] - for file in fix_files: - if defaults[file] and defaults[file] != "": - defaults[file] = self._normalize_path(defaults[file]) - - if "config_file" in defaults: - defaults.pop("config_file") - if "role_dir" in defaults: - defaults.pop("role_dir") + for filename in fix_files: + if defaults[filename] and defaults[filename] != "": + defaults[filename] = self._normalize_path(defaults[filename]) defaults["logging"]["level"] = defaults["logging"]["level"].upper() @@ -278,12 +301,6 @@ class Config(): return path - def _set_is_role(self): - if os.path.isdir(os.path.join(self.role_dir, "tasks")): - return True - - return False - def _validate(self, config): try: anyconfig.validate(config, self.schema, ac_schema_safe=False) diff --git a/ansibledoctor/file_registry.py b/ansibledoctor/file_registry.py index 668cca3..adbe486 100644 --- a/ansibledoctor/file_registry.py +++ b/ansibledoctor/file_registry.py @@ -35,22 +35,22 @@ class Registry: :return: None """ extensions = YAML_EXTENSIONS - role_dir = self.config.role_dir - role_name = os.path.basename(role_dir) + base_dir = self.config.base_dir + role_name = os.path.basename(base_dir) excludes = self.config.config.get("exclude_files") excludespec = pathspec.PathSpec.from_lines("gitwildmatch", excludes) - self.log.debug(f"Scan for files: {role_dir}") + self.log.debug(f"Scan for files: {base_dir}") for extension in extensions: - pattern = os.path.join(role_dir, "**/*." + extension) + pattern = os.path.join(base_dir, "**/*." + extension) for filename in glob.iglob(pattern, recursive=True): if not excludespec.match_file(filename): self.log.debug( "Adding file to '{}': {}".format( - role_name, os.path.relpath(filename, role_dir) + role_name, os.path.relpath(filename, base_dir) ) ) self._doc.append(filename) else: - self.log.debug(f"Excluding file: {os.path.relpath(filename, role_dir)}") + self.log.debug(f"Excluding file: {os.path.relpath(filename, base_dir)}") diff --git a/docs/content/setup/docker.md b/docs/content/setup/docker.md index f2e5b66..2d301dd 100644 --- a/docs/content/setup/docker.md +++ b/docs/content/setup/docker.md @@ -4,10 +4,9 @@ title: Using docker ```Shell docker run \ - -e ANSIBLE_DOCTOR_ROLE_DIR=example/demo-role/ \ - -e ANSIBLE_DOCTOR_OUTPUT_DIR=example/ \ + -e ANSIBLE_DOCTOR_BASE_DIR=example/demo-role/ \ -e ANSIBLE_DOCTOR_FORCE_OVERWRITE=true \ - -e ANSIBLE_DOCTOR_CUSTOM_HEADER=example/demo-role/HEADER.md \ + -e ANSIBLE_DOCTOR_CUSTOM_HEADER=HEADER.md \ -e ANSIBLE_DOCTOR_LOG_LEVEL=info \ -e PY_COLORS=1 \ -v $(pwd):/doctor \ diff --git a/docs/content/usage/configuration.md b/docs/content/usage/configuration.md index 71b94aa..ffc7c62 100644 --- a/docs/content/usage/configuration.md +++ b/docs/content/usage/configuration.md @@ -19,7 +19,7 @@ Configuration options can be set in different places, which are processed in the ```YAML --- # Default is the current working directory. -role_dir: +base_dir: # Default is the basename of 'role_name'. role_name: # Auto-detect if the given directory is a role, can be disabled @@ -60,19 +60,20 @@ exclude_tags: [] ```Shell $ ansible-doctor --help -usage: ansible-doctor [-h] [-c CONFIG_FILE] [-o OUTPUT_DIR] [-f] [-d] [-n] [-v] [-q] [--version] [role_dir] +usage: ansible-doctor [-h] [-c CONFIG_FILE] [-o OUTPUT_DIR] [-r] [-f] [-d] [-n] [-v] [-q] [--version] [base_dir] Generate documentation from annotated Ansible roles using templates positional arguments: - role_dir role directory (default: current working dir) + base_dir base directory (default: current working directory) -optional arguments: +options: -h, --help show this help message and exit -c CONFIG_FILE, --config CONFIG_FILE - location of configuration file + path to configuration file -o OUTPUT_DIR, --output OUTPUT_DIR - output base dir + output directory + -r, --recursive run recursively over the base directory subfolders -f, --force force overwrite output file -d, --dry-run dry run without writing -n, --no-role-detection @@ -87,7 +88,8 @@ optional arguments: ```Shell ANSIBLE_DOCTOR_CONFIG_FILE= ANSIBLE_DOCTOR_ROLE_DETECTION=true -ANSIBLE_DOCTOR_ROLE_DIR= +ANSIBLE_DOCTOR_BASE_DIR= +ANSIBLE_DOCTOR_RECURSIVE=false ANSIBLE_DOCTOR_ROLE_NAME= ANSIBLE_DOCTOR_DRY_RUN=false ANSIBLE_DOCTOR_LOG_LEVEL=warning diff --git a/example/README.md b/example/demo-role/README.md similarity index 100% rename from example/README.md rename to example/demo-role/README.md diff --git a/example/other-role/.ansibledoctor.yml b/example/other-role/.ansibledoctor.yml new file mode 100644 index 0000000..4974bc1 --- /dev/null +++ b/example/other-role/.ansibledoctor.yml @@ -0,0 +1,5 @@ +--- +custom_header: HEADER.md +logging: + level: debug +template: readme diff --git a/example/other-role/HEADER.md b/example/other-role/HEADER.md new file mode 100644 index 0000000..d3dea2c --- /dev/null +++ b/example/other-role/HEADER.md @@ -0,0 +1,4 @@ +# other-role-custom-header + +[![Build Status](https://img.shields.io/drone/build/thegeeklab/ansible-doctor?logo=drone&server=https%3A%2F%2Fdrone.thegeeklab.de)](https://drone.thegeeklab.de/thegeeklab/ansible-doctor) +[![License: GPL-3.0](https://img.shields.io/github/license/thegeeklab/ansible-doctor)](https://github.com/thegeeklab/ansible-doctor/blob/main/LICENSE) diff --git a/example/other-role/README.md b/example/other-role/README.md new file mode 100644 index 0000000..44fe188 --- /dev/null +++ b/example/other-role/README.md @@ -0,0 +1,63 @@ +# other-role-custom-header + +[![Build Status](https://img.shields.io/drone/build/thegeeklab/ansible-doctor?logo=drone&server=https%3A%2F%2Fdrone.thegeeklab.de)](https://drone.thegeeklab.de/thegeeklab/ansible-doctor) +[![License: GPL-3.0](https://img.shields.io/github/license/thegeeklab/ansible-doctor)](https://github.com/thegeeklab/ansible-doctor/blob/main/LICENSE) + +Role to demonstrate ansible-doctor. It is also possible to overwrite +the default description with an annotation. + +## Table of content + +- [Default Variables](#default-variables) + - [demo_role_unset](#demo_role_unset) +- [Discovered Tags](#discovered-tags) +- [Open Tasks](#open-tasks) +- [Dependencies](#dependencies) +- [License](#license) +- [Author](#author) + +--- + +## Default Variables + +### demo_role_unset + +Values can be plain strings, but there is no magic or autoformatting... + +#### Default value + +```YAML +demo_role_unset: +``` + +#### Example usage + +```YAML +demo_role_unset: some_value +``` + +## Discovered Tags + +**_role-tag1_** + +**_role-tag2_** + +## Open Tasks + +- Unscoped general todo. +- (bug): Some bug that is known and need to be fixed. +- (bug): Multi line description are possible as well. Some bug that is known and need to be fixed. +- (improvement): Some things that need to be improved. + +## Dependencies + +- role1 +- role2 + +## License + +MIT + +## Author + +[John Doe](https://blog.example.com) diff --git a/example/other-role/defaults/main.yml b/example/other-role/defaults/main.yml new file mode 100644 index 0000000..0453c62 --- /dev/null +++ b/example/other-role/defaults/main.yml @@ -0,0 +1,4 @@ +--- +# @var demo_role_unset:description: Values can be plain strings, but there is no magic or autoformatting... +# @var demo_role_unset:example: demo_role_unset: some_value +demo_role_unset: diff --git a/example/other-role/meta/main.yml b/example/other-role/meta/main.yml new file mode 100644 index 0000000..c42faa6 --- /dev/null +++ b/example/other-role/meta/main.yml @@ -0,0 +1,21 @@ +--- +# @meta description: > +# Role to demonstrate ansible-doctor. It is also possible to overwrite +# the default description with an annotation. +# @end +# @meta author: [John Doe](https\://blog.example.com) +galaxy_info: + description: Role to demonstrate ansible-doctor. + author: John Doe + license: MIT + min_ansible_version: 2.4 + platforms: + - name: EL + versions: + - 7 + galaxy_tags: + - demo + - documentation +dependencies: + - role1 + - role: role2 diff --git a/example/other-role/tasks/main.yml b/example/other-role/tasks/main.yml new file mode 100644 index 0000000..26f5da4 --- /dev/null +++ b/example/other-role/tasks/main.yml @@ -0,0 +1,16 @@ +--- +# @todo bug: Some bug that is known and need to be fixed. +# @todo bug: > +# Multi line description are possible as well. +# Some bug that is known and need to be fixed. +# @end + +# @todo improvement: Some things that need to be improved. +# @todo default: Unscoped general todo. + +- name: Demo task with a tag list + debug: + msg: "Demo message" + tags: + - role-tag1 + - role-tag2