use an explicit marker symbol to value2json conversion

CHANGELOG.md

@@ -1,7 +1,5 @@
-* BREAKING
-  * rename `base_dir` parameter to `role_dir`
 * BUGFIX
-  * raise exception if template dir not accessable
+  * add missing default for `role_dir`
-  * better log level parsing and error handling for log settings
+  * fix value mapping in jinja2 source dict
-  * fix broken custom header handling
+* FEATURE
-  * fix handling of working dir
+  * use explicit marker to convert annotation values to json

ansibledoctor/Annotation.py

@@ -35,7 +35,8 @@ class Annotation:
         self._all_items = defaultdict(dict)
         self._file_handler = None
         self.config = SingleConfig()
-        self.log = SingleLog().logger
+        self.log = SingleLog()
+        self.logger = self.log.logger
         self._files_registry = files_registry
 
         self._all_annotations = self.config.get_annotations_definition()
@@ -54,6 +55,7 @@ class Annotation:
         for rfile in self._files_registry.get_files():
             self._file_handler = open(rfile, encoding="utf8")
 
+            num = 1
             while True:
                 line = self._file_handler.readline()
                 if not line:
@@ -61,10 +63,11 @@ class Annotation:
 
                 if re.match(regex, line.strip()):
                     item = self._get_annotation_data(
-                        line, self._annotation_definition["name"])
+                        num, line, self._annotation_definition["name"], rfile)
                     if item:
-                        self.log.info(str(item))
+                        self.logger.info(str(item))
                         self._populate_item(item.get_obj().items())
+                num += 1
 
             self._file_handler.close()
 
@@ -73,7 +76,7 @@ class Annotation:
             anyconfig.merge(self._all_items[key],
                             value, ac_merge=anyconfig.MS_DICTS)
 
-    def _get_annotation_data(self, line, name):
+    def _get_annotation_data(self, num, line, name, rfile):
         """
         Make some string conversion on a line in order to get the relevant data.
 
@@ -90,7 +93,7 @@ class Annotation:
         parts = [part.strip() for part in line1.split(":", 2)]
         key = str(parts[0])
         item.data[key] = {}
-        multiline_char = [">"]
+        multiline_char = [">", "$>"]
 
         if len(parts) < 2:
             return
@@ -98,13 +101,15 @@ class Annotation:
         if len(parts) == 2:
             parts = parts[:1] + ["value"] + parts[1:]
 
-        if name == "var":
+        subtypes = self.config.ANNOTATIONS.get(name)["subtypes"]
-            try:
+        if subtypes and parts[1] not in subtypes:
-                content = {key: json.loads(parts[2].strip())}
+            return
-            except ValueError:
+
-                content = [parts[2].strip()]
+        content = [parts[2]]
-        else:
+
-            content = [parts[2]]
+        if parts[2] not in multiline_char and parts[2].startswith("$"):
+            source = parts[2].replace("$", "").strip()
+            content = self._str_to_json(key, source, rfile, num, line)
 
         item.data[key][parts[1]] = content
 
@@ -125,6 +130,7 @@ class Annotation:
                 if re.match(stars_with_annotation, next_line):
                     self._file_handler.seek(current_file_position)
                     break
+
                 # match if empty line or commented empty line
                 test_line = next_line.replace("#", "").strip()
                 if len(test_line) == 0:
@@ -142,6 +148,16 @@ class Annotation:
                     final = final[1:]
                 multiline.append(final)
 
-            item.data[key][parts[1]] = multiline
+            if parts[2].startswith("$"):
+                source = "".join([x.strip() for x in multiline])
+                multiline = self._str_to_json(key, source, rfile, num, line)
 
+            item.data[key][parts[1]] = multiline
         return item
+
+    def _str_to_json(self, key, string, rfile, num, line):
+        try:
+            return {key: json.loads(string)}
+        except ValueError:
+            self.log.sysexit_with_message(
+                "Json value error: Can't parse json in {}:{}:\n{}".format(rfile, str(num), line.strip()))

ansibledoctor/Config.py

@@ -100,23 +100,32 @@ class Config():
     ANNOTATIONS = {
         "meta": {
             "name": "meta",
-            "automatic": True
+            "automatic": True,
+            "subtypes": []
         },
         "todo": {
             "name": "todo",
             "automatic": True,
+            "subtypes": []
         },
         "var": {
             "name": "var",
             "automatic": True,
+            "subtypes": [
+                "value",
+                "example",
+                "description"
+            ]
         },
         "example": {
             "name": "example",
-            "regex": r"(\#\ *\@example\ *\: *.*)"
+            "automatic": True,
+            "subtypes": []
         },
         "tag": {
             "name": "tag",
             "automatic": True,
+            "subtypes": []
         },
     }
 

ansibledoctor/DocumentationGenerator.py

@@ -14,6 +14,7 @@ import jinja2.exceptions
 import ruamel.yaml
 from jinja2 import Environment
 from jinja2 import FileSystemLoader
+from jinja2.filters import evalcontextfilter
 from six import binary_type
 from six import text_type
 
@@ -109,6 +110,7 @@ class Generator:
                             jenv = Environment(loader=FileSystemLoader(self.config.get_template()), lstrip_blocks=True, trim_blocks=True)  # nosec
                             jenv.filters["to_nice_yaml"] = self._to_nice_yaml
                             jenv.filters["deep_get"] = self._deep_get
+                            jenv.filters["save_join"] = self._save_join
                             data = jenv.from_string(data).render(role_data, role=role_data)
                             if not self.config.config["dry_run"]:
                                 with open(doc_file, "wb") as outfile:
@@ -136,6 +138,12 @@ class Generator:
         default = None
         return reduce(lambda d, key: d.get(key, default) if isinstance(d, dict) else default, keys.split("."), dictionary)
 
+    @evalcontextfilter
+    def _save_join(self, eval_ctx, value, d=u"", attribute=None):
+        if isinstance(value, str):
+            value = [value]
+        return jinja2.filters.do_join(eval_ctx, value, d, attribute=None)
+
     def render(self):
         self.logger.info("Using output dir: " + self.config.config.get("output_dir"))
         self._write_doc()

ansibledoctor/DocumentationParser.py

@@ -8,6 +8,7 @@ from collections import defaultdict
 
 import anyconfig
 import yaml
+from nested_lookup import nested_lookup
 
 from ansibledoctor.Annotation import Annotation
 from ansibledoctor.Config import SingleConfig
@@ -21,17 +22,17 @@ class Parser:
         self._annotation_objs = {}
         self._data = defaultdict(dict)
         self.config = SingleConfig()
-        self.log = SingleLog().logger
+        self.log = SingleLog()
+        self.logger = SingleLog().logger
         self._files_registry = Registry()
         self._parse_meta_file()
-        self._parse_vars_file()
+        self._parse_var_files()
+        # self._parse_task_tags()
         self._populate_doc_data()
 
-    def _parse_vars_file(self):
+    def _parse_var_files(self):
-        extensions = YAML_EXTENSIONS
-
         for rfile in self._files_registry.get_files():
-            if any(fnmatch.fnmatch(rfile, "*/defaults/*." + ext) for ext in extensions):
+            if any(fnmatch.fnmatch(rfile, "*/defaults/*." + ext) for ext in YAML_EXTENSIONS):
                 with open(rfile, "r", encoding="utf8") as yaml_file:
                     try:
                         data = defaultdict(dict, yaml.load(yaml_file, Loader=yaml.SafeLoader))
@@ -41,10 +42,8 @@ class Parser:
                         print(exc)
 
     def _parse_meta_file(self):
-        extensions = YAML_EXTENSIONS
-
         for rfile in self._files_registry.get_files():
-            if any("meta/main." + ext in rfile for ext in extensions):
+            if any("meta/main." + ext in rfile for ext in YAML_EXTENSIONS):
                 with open(rfile, "r", encoding="utf8") as yaml_file:
                     try:
                         data = defaultdict(dict, yaml.safe_load(yaml_file))
@@ -54,17 +53,35 @@ class Parser:
 
                         if data.get("dependencies") is not None:
                             self._data["meta"]["dependencies"] = {"value": data.get("dependencies")}
+                            self._data["meta"]["name"] = {"value": os.path.basename(self.config.role_dir)}
                     except yaml.YAMLError as exc:
                         print(exc)
 
+    def _parse_task_tags(self):
+        for rfile in self._files_registry.get_files():
+            if any(fnmatch.fnmatch(rfile, "*/tasks/*." + ext) for ext in YAML_EXTENSIONS):
+                with open(rfile, "r", encoding="utf8") as yaml_file:
+                    try:
+                        data = yaml.safe_load(yaml_file)
+                    except yaml.YAMLError as exc:
+                        print(exc)
+
+                    tags_found = nested_lookup("tags", data)
+                    for tag in tags_found:
+                        self._data["tags"][tag] = {}
+
     def _populate_doc_data(self):
         """Generate the documentation data object."""
         tags = defaultdict(dict)
         for annotaion in self.config.get_annotations_names(automatic=True):
-            self.log.info("Finding annotations for: @" + annotaion)
+            self.logger.info("Finding annotations for: @" + annotaion)
             self._annotation_objs[annotaion] = Annotation(name=annotaion, files_registry=self._files_registry)
             tags[annotaion] = self._annotation_objs[annotaion].get_details()
-        anyconfig.merge(self._data, tags, ac_merge=anyconfig.MS_DICTS)
+
+        try:
+            anyconfig.merge(self._data, tags, ac_merge=anyconfig.MS_DICTS)
+        except ValueError as e:
+            self.log.sysexit_with_message("Unable to merge annotation values:\n{}".format(e))
 
     def get_data(self):
         return self._data

ansibledoctor/templates/readme/README.md.j2

@@ -1,10 +1,10 @@
 {% if not append | deep_get(role, "internal.append") %}
 {% set meta = role.meta | default({}) %}
-# {{ name | deep_get(meta, "name.value") | default("_undefined_") }}
+# {{ meta.name.value | save_join(" ") }}
 {% endif %}
 {% if description | deep_get(meta, "description.value") %}
 
-{{ description | deep_get(meta, "description.value") }}
+{{ meta.description.value | save_join(" ") }}
 {% endif %}
 
 {#      TOC      #}

ansibledoctor/templates/readme/_meta.j2

@@ -20,6 +20,5 @@ None.
 ## Author
 
 {{ meta.author.value }}
-
 {% endif %}
 {% endif %}

ansibledoctor/templates/readme/_vars.j2

@@ -1,12 +1,13 @@
 {% set var = role.var | default({}) %}
 {% if var %}
+
 ## Default Variables
 {% for key, item in var.items() %}
 
 ### {{ key }}
 {% if item.description is defined and item.description %}
 
-{{ item.description | join(" ") | striptags }}
+{{ item.description | save_join(" ") | striptags }}
 {% endif %}
 {% if item.value is defined and item.value %}
 
@@ -29,7 +30,6 @@
 {% endfor %}
 {% endif %}
 ```
-
 {% endif %}
 {% endfor %}
 {% endif %}

example/README.md

@@ -3,7 +3,7 @@
 [![Build Status](https://cloud.drone.io/api/badges/xoxys/ansible-doctor/status.svg)](https://cloud.drone.io/xoxys/ansible-doctor)
 ![License](https://img.shields.io/github/license/xoxys/ansible-doctor)
 
-Role to demonstrate ansible-doctor
+Role to demonstrate ansible-doctor. It is also possible to overwrite the default description with an annotation.
 
 ## Table of content
 
@@ -20,10 +20,13 @@ Role to demonstrate ansible-doctor
 * [Author](#author)
 
 ---
+
 ## Default Variables
 
 ### demo_role_unset
 
+You can set values as string, but there is no magic or autoformatting...
+
 #### Default value
 
 ```YAML
@@ -33,10 +36,9 @@ demo_role_unset:
 #### Example usage
 
 ```YAML
-demo_role_unset: some value
+demo_role_unset: some_value
 ```
 
-
 ### demo_role_empty
 
 #### Default value
@@ -55,6 +57,8 @@ demo_role_single: b
 
 ### demo_role_empty_dict
 
+... or you can use a valid json. In this case, the json will be automatically prefixed with the annotation key and you can use e.g. `to_nice_yaml` filter in your templates. To get this working, you have to prefix your json with a `$` char.
+
 #### Default value
 
 ```YAML
@@ -73,7 +77,6 @@ demo_role_empty_dict:
       - subval2
 ```
 
-
 ### demo_role_dict
 
 #### Default value
@@ -96,7 +99,6 @@ demo_role_dict:
       - subval2
 ```
 
-
 ### demo_role_other_tags
 
 If a variable need some more explanation, this is a good place to do so.
@@ -115,15 +117,14 @@ demo_role_other_tags:
   - package2
 ```
 
-
 ### demo_role_undefined_var
 
-Test oneline desc.
+If you want to add an explicit notice, that a var is not set by default, this is one option. Make sure to flag it as json value: `@var demo_role_undefined_var: $ "_unset_"`
 
 #### Default value
 
 ```YAML
-  - _undefined_
+demo_role_undefined_var: _unset_
 ```
 
 ## Dependencies
@@ -137,4 +138,3 @@ MIT
 ## Author
 
 Robert Kaussow <mail@example.com>
-

example/demo-role/defaults/main.yml

@@ -1,9 +1,18 @@
 ---
+# @var demo_role_unset:description: You can set values as string, but there is no magic or autoformatting...
+# @var demo_role_unset:example: demo_role_unset: some_value
 demo_role_unset:
-# @var demo_role_unset:example: "some value"
+
 demo_role_empty: ""
 demo_role_single: 'b'
-# @var demo_role_empty_dict:example: {"key1": {"sub": "some value"}, "key2": {"sublist": ["subval1", "subval2"]}}
+
+# @var demo_role_empty_dict:description: >
+# ... or you can use a valid json. In this case,
+# the json will be automatically prefixed with the annotation key
+# and you can use e.g. `to_nice_yaml` filter in your templates.
+# To get this working, you have to prefix your json with a `$` char.
+# @end
+# @var demo_role_empty_dict:example: $ {"key1": {"sub": "some value"}, "key2": {"sublist": ["subval1", "subval2"]}}
 demo_role_empty_dict: {}
 
 # @var demo_role_dict:example: >
@@ -23,8 +32,16 @@ demo_role_dict:
 # If a variable need some more explanation,
 # this is a good place to do so.
 # @end
-# @var demo_role_other_tags:example: ["package1", "package2"]
+# @var demo_role_other_tags:example: $>
+# [
+#   "package1",
+#   "package2"
+# ]
+# @end
 demo_role_other_tags: []
 
-# @var demo_role_undefined_var:description: Test oneline desc.
+# @var demo_role_undefined_var:description: >
-# @var demo_role_undefined_var: _undefined_
+# If you want to add an explicit notice, that a var is not set by default, this is one option.
+# Make sure to flag it as json value: `@var demo_role_undefined_var: $ "_unset_"`
+# @end
+# @var demo_role_undefined_var: $ "_unset_"

example/demo-role/meta/main.yml

@@ -1,7 +1,10 @@
 ---
-# @meta name: demo-role
+# @meta description: >
+# Role to demonstrate ansible-doctor. It is also possible to overwrite 
+# the default description with an annotation. 
+# @end
 galaxy_info:
-  description: Role to demonstrate ansible-doctor
+  description: Role to demonstrate ansible-doctor.
   author: Robert Kaussow <mail@example.com>
   license: MIT
   min_ansible_version: 2.4

setup.py

@@ -67,7 +67,8 @@ setup(
         "python-json-logger",
         "jsonschema",
         "jinja2",
-        "environs"
+        "environs",
+        "nested-lookup"
     ],
     entry_points={
         "console_scripts": [