diff --git a/CHANGELOG.md b/CHANGELOG.md index 5708b8e..0940cbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,5 @@ -* BREAKING - * rename `base_dir` parameter to `role_dir` * BUGFIX - * raise exception if template dir not accessable - * better log level parsing and error handling for log settings - * fix broken custom header handling - * fix handling of working dir + * add missing default for `role_dir` + * fix value mapping in jinja2 source dict +* FEATURE + * use explicit marker to convert annotation values to json diff --git a/ansibledoctor/Annotation.py b/ansibledoctor/Annotation.py index 0eb0d62..d600c23 100644 --- a/ansibledoctor/Annotation.py +++ b/ansibledoctor/Annotation.py @@ -35,7 +35,8 @@ class Annotation: self._all_items = defaultdict(dict) self._file_handler = None self.config = SingleConfig() - self.log = SingleLog().logger + self.log = SingleLog() + self.logger = self.log.logger self._files_registry = files_registry self._all_annotations = self.config.get_annotations_definition() @@ -54,6 +55,7 @@ class Annotation: for rfile in self._files_registry.get_files(): self._file_handler = open(rfile, encoding="utf8") + num = 1 while True: line = self._file_handler.readline() if not line: @@ -61,10 +63,11 @@ class Annotation: if re.match(regex, line.strip()): item = self._get_annotation_data( - line, self._annotation_definition["name"]) + num, line, self._annotation_definition["name"], rfile) if item: - self.log.info(str(item)) + self.logger.info(str(item)) self._populate_item(item.get_obj().items()) + num += 1 self._file_handler.close() @@ -73,7 +76,7 @@ class Annotation: anyconfig.merge(self._all_items[key], value, ac_merge=anyconfig.MS_DICTS) - def _get_annotation_data(self, line, name): + def _get_annotation_data(self, num, line, name, rfile): """ Make some string conversion on a line in order to get the relevant data. @@ -90,7 +93,7 @@ class Annotation: parts = [part.strip() for part in line1.split(":", 2)] key = str(parts[0]) item.data[key] = {} - multiline_char = [">"] + multiline_char = [">", "$>"] if len(parts) < 2: return @@ -98,13 +101,15 @@ class Annotation: if len(parts) == 2: parts = parts[:1] + ["value"] + parts[1:] - if name == "var": - try: - content = {key: json.loads(parts[2].strip())} - except ValueError: - content = [parts[2].strip()] - else: - content = [parts[2]] + subtypes = self.config.ANNOTATIONS.get(name)["subtypes"] + if subtypes and parts[1] not in subtypes: + return + + content = [parts[2]] + + if parts[2] not in multiline_char and parts[2].startswith("$"): + source = parts[2].replace("$", "").strip() + content = self._str_to_json(key, source, rfile, num, line) item.data[key][parts[1]] = content @@ -125,6 +130,7 @@ class Annotation: if re.match(stars_with_annotation, next_line): self._file_handler.seek(current_file_position) break + # match if empty line or commented empty line test_line = next_line.replace("#", "").strip() if len(test_line) == 0: @@ -142,6 +148,16 @@ class Annotation: final = final[1:] multiline.append(final) - item.data[key][parts[1]] = multiline + if parts[2].startswith("$"): + source = "".join([x.strip() for x in multiline]) + multiline = self._str_to_json(key, source, rfile, num, line) + item.data[key][parts[1]] = multiline return item + + def _str_to_json(self, key, string, rfile, num, line): + try: + return {key: json.loads(string)} + except ValueError: + self.log.sysexit_with_message( + "Json value error: Can't parse json in {}:{}:\n{}".format(rfile, str(num), line.strip())) diff --git a/ansibledoctor/Config.py b/ansibledoctor/Config.py index 0cf29e5..1f1d1d3 100644 --- a/ansibledoctor/Config.py +++ b/ansibledoctor/Config.py @@ -100,23 +100,32 @@ class Config(): ANNOTATIONS = { "meta": { "name": "meta", - "automatic": True + "automatic": True, + "subtypes": [] }, "todo": { "name": "todo", "automatic": True, + "subtypes": [] }, "var": { "name": "var", "automatic": True, + "subtypes": [ + "value", + "example", + "description" + ] }, "example": { "name": "example", - "regex": r"(\#\ *\@example\ *\: *.*)" + "automatic": True, + "subtypes": [] }, "tag": { "name": "tag", "automatic": True, + "subtypes": [] }, } diff --git a/ansibledoctor/DocumentationGenerator.py b/ansibledoctor/DocumentationGenerator.py index 6d0f712..f6b4141 100644 --- a/ansibledoctor/DocumentationGenerator.py +++ b/ansibledoctor/DocumentationGenerator.py @@ -14,6 +14,7 @@ import jinja2.exceptions import ruamel.yaml from jinja2 import Environment from jinja2 import FileSystemLoader +from jinja2.filters import evalcontextfilter from six import binary_type from six import text_type @@ -109,6 +110,7 @@ class Generator: jenv = Environment(loader=FileSystemLoader(self.config.get_template()), lstrip_blocks=True, trim_blocks=True) # nosec jenv.filters["to_nice_yaml"] = self._to_nice_yaml jenv.filters["deep_get"] = self._deep_get + jenv.filters["save_join"] = self._save_join data = jenv.from_string(data).render(role_data, role=role_data) if not self.config.config["dry_run"]: with open(doc_file, "wb") as outfile: @@ -136,6 +138,12 @@ class Generator: default = None return reduce(lambda d, key: d.get(key, default) if isinstance(d, dict) else default, keys.split("."), dictionary) + @evalcontextfilter + def _save_join(self, eval_ctx, value, d=u"", attribute=None): + if isinstance(value, str): + value = [value] + return jinja2.filters.do_join(eval_ctx, value, d, attribute=None) + def render(self): self.logger.info("Using output dir: " + self.config.config.get("output_dir")) self._write_doc() diff --git a/ansibledoctor/DocumentationParser.py b/ansibledoctor/DocumentationParser.py index c9db13b..e7fbf77 100644 --- a/ansibledoctor/DocumentationParser.py +++ b/ansibledoctor/DocumentationParser.py @@ -8,6 +8,7 @@ from collections import defaultdict import anyconfig import yaml +from nested_lookup import nested_lookup from ansibledoctor.Annotation import Annotation from ansibledoctor.Config import SingleConfig @@ -21,17 +22,17 @@ class Parser: self._annotation_objs = {} self._data = defaultdict(dict) self.config = SingleConfig() - self.log = SingleLog().logger + self.log = SingleLog() + self.logger = SingleLog().logger self._files_registry = Registry() self._parse_meta_file() - self._parse_vars_file() + self._parse_var_files() + # self._parse_task_tags() self._populate_doc_data() - def _parse_vars_file(self): - extensions = YAML_EXTENSIONS - + def _parse_var_files(self): for rfile in self._files_registry.get_files(): - if any(fnmatch.fnmatch(rfile, "*/defaults/*." + ext) for ext in extensions): + if any(fnmatch.fnmatch(rfile, "*/defaults/*." + ext) for ext in YAML_EXTENSIONS): with open(rfile, "r", encoding="utf8") as yaml_file: try: data = defaultdict(dict, yaml.load(yaml_file, Loader=yaml.SafeLoader)) @@ -41,10 +42,8 @@ class Parser: print(exc) def _parse_meta_file(self): - extensions = YAML_EXTENSIONS - for rfile in self._files_registry.get_files(): - if any("meta/main." + ext in rfile for ext in extensions): + if any("meta/main." + ext in rfile for ext in YAML_EXTENSIONS): with open(rfile, "r", encoding="utf8") as yaml_file: try: data = defaultdict(dict, yaml.safe_load(yaml_file)) @@ -54,17 +53,35 @@ class Parser: if data.get("dependencies") is not None: self._data["meta"]["dependencies"] = {"value": data.get("dependencies")} + self._data["meta"]["name"] = {"value": os.path.basename(self.config.role_dir)} except yaml.YAMLError as exc: print(exc) + def _parse_task_tags(self): + for rfile in self._files_registry.get_files(): + if any(fnmatch.fnmatch(rfile, "*/tasks/*." + ext) for ext in YAML_EXTENSIONS): + with open(rfile, "r", encoding="utf8") as yaml_file: + try: + data = yaml.safe_load(yaml_file) + except yaml.YAMLError as exc: + print(exc) + + tags_found = nested_lookup("tags", data) + for tag in tags_found: + self._data["tags"][tag] = {} + def _populate_doc_data(self): """Generate the documentation data object.""" tags = defaultdict(dict) for annotaion in self.config.get_annotations_names(automatic=True): - self.log.info("Finding annotations for: @" + annotaion) + self.logger.info("Finding annotations for: @" + annotaion) self._annotation_objs[annotaion] = Annotation(name=annotaion, files_registry=self._files_registry) tags[annotaion] = self._annotation_objs[annotaion].get_details() - anyconfig.merge(self._data, tags, ac_merge=anyconfig.MS_DICTS) + + try: + anyconfig.merge(self._data, tags, ac_merge=anyconfig.MS_DICTS) + except ValueError as e: + self.log.sysexit_with_message("Unable to merge annotation values:\n{}".format(e)) def get_data(self): return self._data diff --git a/ansibledoctor/templates/readme/README.md.j2 b/ansibledoctor/templates/readme/README.md.j2 index 566945b..85bd641 100644 --- a/ansibledoctor/templates/readme/README.md.j2 +++ b/ansibledoctor/templates/readme/README.md.j2 @@ -1,10 +1,10 @@ {% if not append | deep_get(role, "internal.append") %} {% set meta = role.meta | default({}) %} -# {{ name | deep_get(meta, "name.value") | default("_undefined_") }} +# {{ meta.name.value | save_join(" ") }} {% endif %} {% if description | deep_get(meta, "description.value") %} -{{ description | deep_get(meta, "description.value") }} +{{ meta.description.value | save_join(" ") }} {% endif %} {# TOC #} diff --git a/ansibledoctor/templates/readme/_meta.j2 b/ansibledoctor/templates/readme/_meta.j2 index 511f663..43092f2 100644 --- a/ansibledoctor/templates/readme/_meta.j2 +++ b/ansibledoctor/templates/readme/_meta.j2 @@ -20,6 +20,5 @@ None. ## Author {{ meta.author.value }} - {% endif %} {% endif %} diff --git a/ansibledoctor/templates/readme/_vars.j2 b/ansibledoctor/templates/readme/_vars.j2 index 2e4d4cc..9777336 100644 --- a/ansibledoctor/templates/readme/_vars.j2 +++ b/ansibledoctor/templates/readme/_vars.j2 @@ -1,12 +1,13 @@ {% set var = role.var | default({}) %} {% if var %} + ## Default Variables {% for key, item in var.items() %} ### {{ key }} {% if item.description is defined and item.description %} -{{ item.description | join(" ") | striptags }} +{{ item.description | save_join(" ") | striptags }} {% endif %} {% if item.value is defined and item.value %} @@ -29,7 +30,6 @@ {% endfor %} {% endif %} ``` - {% endif %} {% endfor %} {% endif %} diff --git a/example/README.md b/example/README.md index 47de576..5c4dbb1 100644 --- a/example/README.md +++ b/example/README.md @@ -3,7 +3,7 @@ [![Build Status](https://cloud.drone.io/api/badges/xoxys/ansible-doctor/status.svg)](https://cloud.drone.io/xoxys/ansible-doctor) ![License](https://img.shields.io/github/license/xoxys/ansible-doctor) -Role to demonstrate ansible-doctor +Role to demonstrate ansible-doctor. It is also possible to overwrite the default description with an annotation. ## Table of content @@ -20,10 +20,13 @@ Role to demonstrate ansible-doctor * [Author](#author) --- + ## Default Variables ### demo_role_unset +You can set values as string, but there is no magic or autoformatting... + #### Default value ```YAML @@ -33,10 +36,9 @@ demo_role_unset: #### Example usage ```YAML -demo_role_unset: some value +demo_role_unset: some_value ``` - ### demo_role_empty #### Default value @@ -55,6 +57,8 @@ demo_role_single: b ### demo_role_empty_dict +... or you can use a valid json. In this case, the json will be automatically prefixed with the annotation key and you can use e.g. `to_nice_yaml` filter in your templates. To get this working, you have to prefix your json with a `$` char. + #### Default value ```YAML @@ -73,7 +77,6 @@ demo_role_empty_dict: - subval2 ``` - ### demo_role_dict #### Default value @@ -96,7 +99,6 @@ demo_role_dict: - subval2 ``` - ### demo_role_other_tags If a variable need some more explanation, this is a good place to do so. @@ -115,15 +117,14 @@ demo_role_other_tags: - package2 ``` - ### demo_role_undefined_var -Test oneline desc. +If you want to add an explicit notice, that a var is not set by default, this is one option. Make sure to flag it as json value: `@var demo_role_undefined_var: $ "_unset_"` #### Default value ```YAML - - _undefined_ +demo_role_undefined_var: _unset_ ``` ## Dependencies @@ -137,4 +138,3 @@ MIT ## Author Robert Kaussow - diff --git a/example/demo-role/defaults/main.yml b/example/demo-role/defaults/main.yml index 9626596..15ca2c6 100644 --- a/example/demo-role/defaults/main.yml +++ b/example/demo-role/defaults/main.yml @@ -1,9 +1,18 @@ --- +# @var demo_role_unset:description: You can set values as string, but there is no magic or autoformatting... +# @var demo_role_unset:example: demo_role_unset: some_value demo_role_unset: -# @var demo_role_unset:example: "some value" + demo_role_empty: "" demo_role_single: 'b' -# @var demo_role_empty_dict:example: {"key1": {"sub": "some value"}, "key2": {"sublist": ["subval1", "subval2"]}} + +# @var demo_role_empty_dict:description: > +# ... or you can use a valid json. In this case, +# the json will be automatically prefixed with the annotation key +# and you can use e.g. `to_nice_yaml` filter in your templates. +# To get this working, you have to prefix your json with a `$` char. +# @end +# @var demo_role_empty_dict:example: $ {"key1": {"sub": "some value"}, "key2": {"sublist": ["subval1", "subval2"]}} demo_role_empty_dict: {} # @var demo_role_dict:example: > @@ -23,8 +32,16 @@ demo_role_dict: # If a variable need some more explanation, # this is a good place to do so. # @end -# @var demo_role_other_tags:example: ["package1", "package2"] +# @var demo_role_other_tags:example: $> +# [ +# "package1", +# "package2" +# ] +# @end demo_role_other_tags: [] -# @var demo_role_undefined_var:description: Test oneline desc. -# @var demo_role_undefined_var: _undefined_ +# @var demo_role_undefined_var:description: > +# If you want to add an explicit notice, that a var is not set by default, this is one option. +# Make sure to flag it as json value: `@var demo_role_undefined_var: $ "_unset_"` +# @end +# @var demo_role_undefined_var: $ "_unset_" diff --git a/example/demo-role/meta/main.yml b/example/demo-role/meta/main.yml index 40c1204..133222a 100644 --- a/example/demo-role/meta/main.yml +++ b/example/demo-role/meta/main.yml @@ -1,7 +1,10 @@ --- -# @meta name: demo-role +# @meta description: > +# Role to demonstrate ansible-doctor. It is also possible to overwrite +# the default description with an annotation. +# @end galaxy_info: - description: Role to demonstrate ansible-doctor + description: Role to demonstrate ansible-doctor. author: Robert Kaussow license: MIT min_ansible_version: 2.4 diff --git a/setup.py b/setup.py index 7a0fcb7..42a2bc7 100755 --- a/setup.py +++ b/setup.py @@ -67,7 +67,8 @@ setup( "python-json-logger", "jsonschema", "jinja2", - "environs" + "environs", + "nested-lookup" ], entry_points={ "console_scripts": [