add yapf as formatter

.drone.jsonnet

@@ -51,7 +51,7 @@ local PipelineTest = {
     PythonVersion(pyversion='3.5'),
     PythonVersion(pyversion='3.6'),
     PythonVersion(pyversion='3.7'),
-    PythonVersion(pyversion='3.8-rc'),
+    PythonVersion(pyversion='3.8'),
   ],
   depends_on: [
     'lint',

.flake8

@@ -1,8 +1,18 @@
 [flake8]
-# Temp disable Docstring checks D101, D102, D103, D107
+ignore = D102, D103, D107, D202, W503
-ignore = E501, W503, F401, N813, D101, D102, D103, D107
+max-line-length = 99
-max-line-length = 110
 inline-quotes = double
-exclude = .git,.tox,__pycache__,build,dist,tests,*.pyc,*.egg-info,.cache,.eggs,env*
+exclude =
+    .git
+    .tox
+    __pycache__
+    build
+    dist
+    tests
+    *.pyc
+    *.egg-info
+    .cache
+    .eggs
+    env*
 application-import-names = ansiblelater
 format = ${cyan}%(path)s:%(row)d:%(col)d${reset}: ${red_bold}%(code)s${reset} %(text)s

.github/settings.yml

@@ -56,5 +56,3 @@ branches:
           - continuous-integration/drone/pr
       enforce_admins: null
       restrictions: null
-
-...

ansibledoctor/Annotation.py

@@ -2,18 +2,17 @@
 """Find and parse annotations to AnnotationItem objects."""
 
 import json
-import pprint
 import re
 from collections import defaultdict
 
 import anyconfig
 
 from ansibledoctor.Config import SingleConfig
-from ansibledoctor.FileRegistry import Registry
 from ansibledoctor.Utils import SingleLog
 
 
 class AnnotationItem:
+    """Handle annotations."""
 
     # next time improve this by looping over public available attributes
     def __init__(self):
@@ -30,6 +29,8 @@ class AnnotationItem:
 
 
 class Annotation:
+    """Handle annotations."""
+
     def __init__(self, name, files_registry):
         self._all_items = defaultdict(dict)
         self._file_handler = None
@@ -62,7 +63,8 @@ class Annotation:
 
                 if re.match(regex, line.strip()):
                     item = self._get_annotation_data(
-                        num, line, self._annotation_definition["name"], rfile)
+                        num, line, self._annotation_definition["name"], rfile
+                    )
                     if item:
                         self.logger.info(str(item))
                         self._populate_item(item.get_obj().items())
@@ -166,4 +168,7 @@ class Annotation:
             return {key: json.loads(string)}
         except ValueError:
             self.log.sysexit_with_message(
-                "Json value error: Can't parse json in {}:{}:\n{}".format(rfile, str(num), line.strip()))
+                "Json value error: Can't parse json in {}:{}:\n{}".format(
+                    rfile, str(num), line.strip()
+                )
+            )

ansibledoctor/Cli.py

@@ -2,9 +2,6 @@
 """Entrypoint and CLI handler."""
 
 import argparse
-import logging
-import os
-import sys
 
 import ansibledoctor.Exception
 from ansibledoctor import __version__
@@ -15,6 +12,7 @@ from ansibledoctor.Utils import SingleLog
 
 
 class AnsibleDoctor:
+    """Main doctor object."""
 
     def __init__(self):
         self.log = SingleLog()
@@ -34,20 +32,42 @@ class AnsibleDoctor:
         """
         # TODO: add function to print to stdout instead of file
         parser = argparse.ArgumentParser(
-            description="Generate documentation from annotated Ansible roles using templates")
+            description="Generate documentation from annotated Ansible roles using templates"
-        parser.add_argument("role_dir", nargs="?", help="role directory (default: current working dir)")
+        )
-        parser.add_argument("-c", "--config", dest="config_file", help="location of configuration file")
+        parser.add_argument(
-        parser.add_argument("-o", "--output", dest="output_dir", action="store",
+            "role_dir", nargs="?", help="role directory (default: current working dir)"
-                            help="output base dir")
+        )
-        parser.add_argument("-f", "--force", dest="force_overwrite", action="store_true", default=None,
+        parser.add_argument(
-                            help="force overwrite output file")
+            "-c", "--config", dest="config_file", help="location of configuration file"
-        parser.add_argument("-d", "--dry-run", dest="dry_run", action="store_true", default=None,
+        )
-                            help="dry run without writing")
+        parser.add_argument(
-        parser.add_argument("-v", dest="logging.level", action="append_const", const=-1,
+            "-o", "--output", dest="output_dir", action="store", help="output base dir"
-                            help="increase log level")
+        )
-        parser.add_argument("-q", dest="logging.level", action="append_const",
+        parser.add_argument(
-                            const=1, help="decrease log level")
+            "-f",
-        parser.add_argument("--version", action="version", version="%(prog)s {}".format(__version__))
+            "--force",
+            dest="force_overwrite",
+            action="store_true",
+            default=None,
+            help="force overwrite output file"
+        )
+        parser.add_argument(
+            "-d",
+            "--dry-run",
+            dest="dry_run",
+            action="store_true",
+            default=None,
+            help="dry run without writing"
+        )
+        parser.add_argument(
+            "-v", dest="logging.level", action="append_const", const=-1, help="increase log level"
+        )
+        parser.add_argument(
+            "-q", dest="logging.level", action="append_const", const=1, help="decrease log level"
+        )
+        parser.add_argument(
+            "--version", action="version", version="%(prog)s {}".format(__version__)
+        )
 
         return parser.parse_args().__dict__
 

ansibledoctor/Config.py

@@ -1,9 +1,7 @@
 #!/usr/bin/env python3
 """Global settings definition."""
 
-import logging
 import os
-import sys
 
 import anyconfig
 import environs
@@ -11,7 +9,6 @@ import jsonschema.exceptions
 import ruamel.yaml
 from appdirs import AppDirs
 from jsonschema._utils import format_as_index
-from pkg_resources import resource_filename
 
 import ansibledoctor.Exception
 from ansibledoctor.Utils import Singleton
@@ -116,11 +113,7 @@ class Config():
         "var": {
             "name": "var",
             "automatic": True,
-            "subtypes": [
+            "subtypes": ["value", "example", "description"]
-                "value",
-                "example",
-                "description"
-            ]
         },
         "example": {
             "name": "example",
@@ -192,7 +185,9 @@ class Config():
                     if '"{}" not set'.format(envname) in str(e):
                         pass
                     else:
-                        raise ansibledoctor.Exception.ConfigError("Unable to read environment variable", str(e))
+                        raise ansibledoctor.Exception.ConfigError(
+                            "Unable to read environment variable", str(e)
+                        )
 
         return normalized
 
@@ -224,7 +219,9 @@ class Config():
                     s = stream.read()
                     try:
                         file_dict = ruamel.yaml.safe_load(s)
-                    except (ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError) as e:
+                    except (
+                        ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError
+                    ) as e:
                         message = "{} {}".format(e.context, e.problem)
                         raise ansibledoctor.Exception.ConfigError(
                             "Unable to read config file {}".format(config), message
@@ -313,4 +310,6 @@ class Config():
 
 
 class SingleConfig(Config, metaclass=Singleton):
+    """Singleton config class."""
+
     pass

ansibledoctor/DocumentationGenerator.py

@@ -1,13 +1,9 @@
 #!/usr/bin/env python3
 """Prepare output and write compiled jinja2 templates."""
 
-import codecs
 import glob
-import json
 import ntpath
 import os
-import pprint
-import sys
 from functools import reduce
 
 import jinja2.exceptions
@@ -15,8 +11,6 @@ import ruamel.yaml
 from jinja2 import Environment
 from jinja2 import FileSystemLoader
 from jinja2.filters import evalcontextfilter
-from six import binary_type
-from six import text_type
 
 import ansibledoctor.Exception
 from ansibledoctor.Config import SingleConfig
@@ -25,6 +19,8 @@ from ansibledoctor.Utils import SingleLog
 
 
 class Generator:
+    """Generate documentation from jinja2 templates."""
+
     def __init__(self, doc_parser):
         self.template_files = []
         self.extension = "j2"
@@ -67,7 +63,10 @@ class Generator:
         files_to_overwite = []
 
         for file in self.template_files:
-            doc_file = os.path.join(self.config.config.get("output_dir"), os.path.splitext(file)[0])
+            doc_file = os.path.join(
+                self.config.config.get("output_dir"),
+                os.path.splitext(file)[0]
+            )
             if os.path.isfile(doc_file):
                 files_to_overwite.append(doc_file)
 
@@ -95,7 +94,10 @@ class Generator:
                     self.log.sysexit_with_message("Aborted...")
 
         for file in self.template_files:
-            doc_file = os.path.join(self.config.config.get("output_dir"), os.path.splitext(file)[0])
+            doc_file = os.path.join(
+                self.config.config.get("output_dir"),
+                os.path.splitext(file)[0]
+            )
             source_file = self.config.get_template() + "/" + file
 
             self.logger.debug("Writing doc output to: " + doc_file + " from: " + source_file)
@@ -109,7 +111,11 @@ class Generator:
                     if data is not None:
                         try:
                             # print(json.dumps(role_data, indent=4, sort_keys=True))
-                            jenv = Environment(loader=FileSystemLoader(self.config.get_template()), lstrip_blocks=True, trim_blocks=True)  # nosec
+                            jenv = Environment(
+                                loader=FileSystemLoader(self.config.get_template()),
+                                lstrip_blocks=True,
+                                trim_blocks=True
+                            )  # nosec
                             jenv.filters["to_nice_yaml"] = self._to_nice_yaml
                             jenv.filters["deep_get"] = self._deep_get
                             jenv.filters["save_join"] = self._save_join
@@ -121,12 +127,18 @@ class Generator:
                                     self.logger.info("Writing to: " + doc_file)
                             else:
                                 self.logger.info("Writing to: " + doc_file)
-                        except (jinja2.exceptions.UndefinedError, jinja2.exceptions.TemplateSyntaxError)as e:
+                        except (
+                            jinja2.exceptions.UndefinedError, jinja2.exceptions.TemplateSyntaxError
+                        ) as e:
                             self.log.sysexit_with_message(
-                                "Jinja2 templating error while loading file: '{}'\n{}".format(file, str(e)))
+                                "Jinja2 templating error while loading file: '{}'\n{}".format(
+                                    file, str(e)
+                                )
+                            )
                         except UnicodeEncodeError as e:
                             self.log.sysexit_with_message(
-                                "Unable to print special characters\n{}".format(str(e)))
+                                "Unable to print special characters\n{}".format(str(e))
+                            )
 
     def _to_nice_yaml(self, a, indent=4, *args, **kw):
         """Make verbose, human readable yaml."""
@@ -138,7 +150,10 @@ class Generator:
 
     def _deep_get(self, _, dictionary, keys, *args, **kw):
         default = None
-        return reduce(lambda d, key: d.get(key, default) if isinstance(d, dict) else default, keys.split("."), dictionary)
+        return reduce(
+            lambda d, key: d.get(key, default)
+            if isinstance(d, dict) else default, keys.split("."), dictionary
+        )
 
     @evalcontextfilter
     def _save_join(self, eval_ctx, value, d=u"", attribute=None):

ansibledoctor/DocumentationParser.py

@@ -2,8 +2,6 @@
 """Parse static files."""
 
 import fnmatch
-import json
-import os
 from collections import defaultdict
 
 import anyconfig
@@ -19,6 +17,8 @@ from ansibledoctor.Utils import UnsafeTag
 
 
 class Parser:
+    """Parse yaml files."""
+
     def __init__(self):
         self._annotation_objs = {}
         self._data = defaultdict(dict)
@@ -36,13 +36,21 @@ class Parser:
             if any(fnmatch.fnmatch(rfile, "*/defaults/*." + ext) for ext in YAML_EXTENSIONS):
                 with open(rfile, "r", encoding="utf8") as yaml_file:
                     try:
-                        ruamel.yaml.add_constructor(UnsafeTag.yaml_tag, UnsafeTag.yaml_constructor, constructor=ruamel.yaml.SafeConstructor)
+                        ruamel.yaml.add_constructor(
+                            UnsafeTag.yaml_tag,
+                            UnsafeTag.yaml_constructor,
+                            constructor=ruamel.yaml.SafeConstructor
+                        )
                         data = defaultdict(dict, (ruamel.yaml.safe_load(yaml_file) or {}))
                         for key, value in data.items():
                             self._data["var"][key] = {"value": {key: value}}
-                    except (ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError) as e:
+                    except (
+                        ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError
+                    ) as e:
                         message = "{} {}".format(e.context, e.problem)
-                        self.log.sysexit_with_message("Unable to read yaml file {}\n{}".format(rfile, message))
+                        self.log.sysexit_with_message(
+                            "Unable to read yaml file {}\n{}".format(rfile, message)
+                        )
 
     def _parse_meta_file(self):
         for rfile in self._files_registry.get_files():
@@ -55,12 +63,18 @@ class Parser:
                                 self._data["meta"][key] = {"value": value}
 
                         if data.get("dependencies") is not None:
-                            self._data["meta"]["dependencies"] = {"value": data.get("dependencies")}
+                            self._data["meta"]["dependencies"] = {
+                                "value": data.get("dependencies")
+                            }
 
                         self._data["meta"]["name"] = {"value": self.config.config["role_name"]}
-                    except (ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError) as e:
+                    except (
+                        ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError
+                    ) as e:
                         message = "{} {}".format(e.context, e.problem)
-                        self.log.sysexit_with_message("Unable to read yaml file {}\n{}".format(rfile, message))
+                        self.log.sysexit_with_message(
+                            "Unable to read yaml file {}\n{}".format(rfile, message)
+                        )
 
     def _parse_task_tags(self):
         for rfile in self._files_registry.get_files():
@@ -68,9 +82,13 @@ class Parser:
                 with open(rfile, "r", encoding="utf8") as yaml_file:
                     try:
                         data = ruamel.yaml.safe_load(yaml_file)
-                    except (ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError) as e:
+                    except (
+                        ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError
+                    ) as e:
                         message = "{} {}".format(e.context, e.problem)
-                        self.log.sysexit_with_message("Unable to read yaml file {}\n{}".format(rfile, message))
+                        self.log.sysexit_with_message(
+                            "Unable to read yaml file {}\n{}".format(rfile, message)
+                        )
 
                     tags_found = nested_lookup("tags", data)
                     for tag in tags_found:
@@ -81,7 +99,9 @@ class Parser:
         tags = defaultdict(dict)
         for annotaion in self.config.get_annotations_names(automatic=True):
             self.logger.info("Finding annotations for: @" + annotaion)
-            self._annotation_objs[annotaion] = Annotation(name=annotaion, files_registry=self._files_registry)
+            self._annotation_objs[annotaion] = Annotation(
+                name=annotaion, files_registry=self._files_registry
+            )
             tags[annotaion] = self._annotation_objs[annotaion].get_details()
 
         try:

ansibledoctor/FileRegistry.py

@@ -3,7 +3,6 @@
 
 import glob
 import os
-import sys
 
 import pathspec
 
@@ -13,6 +12,7 @@ from ansibledoctor.Utils import SingleLog
 
 
 class Registry:
+    """Register all yaml files."""
 
     _doc = {}
     log = None
@@ -46,7 +46,13 @@ class Registry:
             pattern = os.path.join(role_dir, "**/*." + extension)
             for filename in glob.iglob(pattern, recursive=True):
                 if not excludespec.match_file(filename):
-                    self.log.debug("Adding file to '{}': {}".format(role_name, os.path.relpath(filename, role_dir)))
+                    self.log.debug(
+                        "Adding file to '{}': {}".format(
+                            role_name, os.path.relpath(filename, role_dir)
+                        )
+                    )
                     self._doc.append(filename)
                 else:
-                    self.log.debug("Excluding file: {}".format(os.path.relpath(filename, role_dir)))
+                    self.log.debug(
+                        "Excluding file: {}".format(os.path.relpath(filename, role_dir))
+                    )

ansibledoctor/Utils.py

@@ -3,7 +3,6 @@
 
 import logging
 import os
-import pprint
 import sys
 from distutils.util import strtobool
 
@@ -32,6 +31,8 @@ colorama.init(autoreset=True, strip=not _should_do_markup())
 
 
 class Singleton(type):
+    """Meta singleton class."""
+
     _instances = {}
 
     def __call__(cls, *args, **kwargs):
@@ -61,7 +62,7 @@ class LogFilter(object):
 class MultilineFormatter(logging.Formatter):
     """Logging Formatter to reset color after newline characters."""
 
-    def format(self, record): # noqa
+    def format(self, record):  # noqa
         record.msg = record.msg.replace("\n", "\n{}... ".format(colorama.Style.RESET_ALL))
         return logging.Formatter.format(self, record)
 
@@ -69,12 +70,14 @@ class MultilineFormatter(logging.Formatter):
 class MultilineJsonFormatter(jsonlogger.JsonFormatter):
     """Logging Formatter to remove newline characters."""
 
-    def format(self, record): # noqa
+    def format(self, record):  # noqa
         record.msg = record.msg.replace("\n", " ")
         return jsonlogger.JsonFormatter.format(self, record)
 
 
 class Log:
+    """Handle logging."""
+
     def __init__(self, level=logging.WARN, name="ansibledoctor", json=False):
         self.logger = logging.getLogger(name)
         self.logger.setLevel(level)
@@ -89,8 +92,11 @@ class Log:
         handler = logging.StreamHandler(sys.stderr)
         handler.setLevel(logging.ERROR)
         handler.addFilter(LogFilter(logging.ERROR))
-        handler.setFormatter(MultilineFormatter(
+        handler.setFormatter(
-            self.error(CONSOLE_FORMAT.format(colorama.Fore.RED, colorama.Style.RESET_ALL))))
+            MultilineFormatter(
+                self.error(CONSOLE_FORMAT.format(colorama.Fore.RED, colorama.Style.RESET_ALL))
+            )
+        )
 
         if json:
             handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
@@ -101,8 +107,11 @@ class Log:
         handler = logging.StreamHandler(sys.stdout)
         handler.setLevel(logging.WARN)
         handler.addFilter(LogFilter(logging.WARN))
-        handler.setFormatter(MultilineFormatter(
+        handler.setFormatter(
-            self.warn(CONSOLE_FORMAT.format(colorama.Fore.YELLOW, colorama.Style.RESET_ALL))))
+            MultilineFormatter(
+                self.warn(CONSOLE_FORMAT.format(colorama.Fore.YELLOW, colorama.Style.RESET_ALL))
+            )
+        )
 
         if json:
             handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
@@ -113,8 +122,11 @@ class Log:
         handler = logging.StreamHandler(sys.stdout)
         handler.setLevel(logging.INFO)
         handler.addFilter(LogFilter(logging.INFO))
-        handler.setFormatter(MultilineFormatter(
+        handler.setFormatter(
-            self.info(CONSOLE_FORMAT.format(colorama.Fore.CYAN, colorama.Style.RESET_ALL))))
+            MultilineFormatter(
+                self.info(CONSOLE_FORMAT.format(colorama.Fore.CYAN, colorama.Style.RESET_ALL))
+            )
+        )
 
         if json:
             handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
@@ -125,8 +137,11 @@ class Log:
         handler = logging.StreamHandler(sys.stderr)
         handler.setLevel(logging.CRITICAL)
         handler.addFilter(LogFilter(logging.CRITICAL))
-        handler.setFormatter(MultilineFormatter(
+        handler.setFormatter(
-            self.critical(CONSOLE_FORMAT.format(colorama.Fore.RED, colorama.Style.RESET_ALL))))
+            MultilineFormatter(
+                self.critical(CONSOLE_FORMAT.format(colorama.Fore.RED, colorama.Style.RESET_ALL))
+            )
+        )
 
         if json:
             handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
@@ -137,8 +152,11 @@ class Log:
         handler = logging.StreamHandler(sys.stderr)
         handler.setLevel(logging.DEBUG)
         handler.addFilter(LogFilter(logging.DEBUG))
-        handler.setFormatter(MultilineFormatter(
+        handler.setFormatter(
-            self.critical(CONSOLE_FORMAT.format(colorama.Fore.BLUE, colorama.Style.RESET_ALL))))
+            MultilineFormatter(
+                self.critical(CONSOLE_FORMAT.format(colorama.Fore.BLUE, colorama.Style.RESET_ALL))
+            )
+        )
 
         if json:
             handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
@@ -188,10 +206,14 @@ class Log:
 
 
 class SingleLog(Log, metaclass=Singleton):
+    """Singleton logging class."""
+
     pass
 
 
 class UnsafeTag:
+    """Handle custom yaml unsafe tag."""
+
     yaml_tag = u"!unsafe"
 
     def __init__(self, value):
@@ -203,6 +225,8 @@ class UnsafeTag:
 
 
 class FileUtils:
+    """Mics static methods for file handling."""
+
     @staticmethod
     def create_path(path):
         os.makedirs(path, exist_ok=True)

setup.cfg

@@ -10,11 +10,21 @@ default_section = THIRDPARTY
 known_first_party = ansibledoctor
 sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER
 force_single_line = true
-line_length = 110
+line_length = 99
-skip_glob = **/env/*,**/docs/*
+skip_glob = **/.env*,**/env/*,**/docs/*
+
+[yapf]
+based_on_style = google
+column_limit = 99
+dedent_closing_brackets = true
+coalesce_brackets = true
+split_before_logical_operator = true
 
 [tool:pytest]
 filterwarnings =
     ignore::FutureWarning
     ignore:.*collections.*:DeprecationWarning
     ignore:.*pep8.*:FutureWarning
+
+[coverage:run]
+omit = **/tests/*

setup.py

@@ -33,26 +33,28 @@ setup(
     keywords="ansible role documentation",
     author=get_property("__author__", PACKAGE_NAME),
     author_email=get_property("__email__", PACKAGE_NAME),
-    url="https://github.com/xoxys/ansible-doctor",
+    url=get_property("__url__", PACKAGE_NAME)
-    license=get_property("__url__", PACKAGE_NAME),
+    license=get_property("__license__", PACKAGE_NAME),
     long_description=get_readme(),
     long_description_content_type="text/markdown",
     packages=find_packages(exclude=["*.tests", "tests", "tests.*"]),
     include_package_data=True,
     zip_safe=False,
-    python_requires=">=3.5",
+    python_requires=">=3.5,<4",
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
+        "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
         "Intended Audience :: Developers",
         "Intended Audience :: Information Technology",
         "Intended Audience :: System Administrators",
-        "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
         "Natural Language :: English",
         "Operating System :: POSIX",
-        "Programming Language :: Python :: 3 :: Only",
+        "Programming Language :: Python :: 3",
-        "Topic :: System :: Installation/Setup",
+        "Programming Language :: Python :: 3.5",
-        "Topic :: System :: Systems Administration",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
         "Topic :: Utilities",
         "Topic :: Software Development",
         "Topic :: Software Development :: Documentation",

test-requirements.txt

@@ -1,11 +1,8 @@
-# open issue
-# https://gitlab.com/pycqa/flake8-docstrings/issues/36
 pydocstyle<4.0.0
 flake8
 flake8-colors
 flake8-blind-except
 flake8-builtins
-flake8-colors
 flake8-docstrings<=3.0.0
 flake8-isort
 flake8-logging-format
@@ -17,3 +14,4 @@ pytest
 pytest-mock
 pytest-cov
 bandit
+yapf