github
/
ansible-doctor
mirror of https://github.com/thegeeklab/ansible-doctor.git
0
0
Fork 0
Code Issues Projects Releases Wiki Activity

add yapf as formatter

This commit is contained in:
Robert Kaussow 2020-04-05 23:16:53 +02:00
parent ce216b41dc
commit 12aaab20d7
13 changed files with 197 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.drone.jsonnet
View File

@ -51,7 +51,7 @@ local PipelineTest = {
PythonVersion(pyversion='3.5'),
PythonVersion(pyversion='3.6'),
PythonVersion(pyversion='3.7'),
PythonVersion(pyversion='3.8-rc'),
PythonVersion(pyversion='3.8'),
],
depends_on: [
'lint',

18
.flake8
View File

@ -1,8 +1,18 @@
[flake8]
# Temp disable Docstring checks D101, D102, D103, D107
ignore = E501, W503, F401, N813, D101, D102, D103, D107
max-line-length = 110
ignore = D102, D103, D107, D202, W503
max-line-length = 99
inline-quotes = double
exclude = .git,.tox,__pycache__,build,dist,tests,*.pyc,*.egg-info,.cache,.eggs,env*
exclude =
.git
.tox
__pycache__
build
dist
tests
*.pyc
*.egg-info
.cache
.eggs
env*
application-import-names = ansiblelater
format = ${cyan}%(path)s:%(row)d:%(col)d${reset}: ${red_bold}%(code)s${reset} %(text)s

2
.github/settings.yml vendored
View File

@ -56,5 +56,3 @@ branches:
- continuous-integration/drone/pr
enforce_admins: null
restrictions: null
...

13
ansibledoctor/Annotation.py
View File

@ -2,18 +2,17 @@
"""Find and parse annotations to AnnotationItem objects."""
import json
import pprint
import re
from collections import defaultdict
import anyconfig
from ansibledoctor.Config import SingleConfig
from ansibledoctor.FileRegistry import Registry
from ansibledoctor.Utils import SingleLog
class AnnotationItem:
"""Handle annotations."""
# next time improve this by looping over public available attributes
def __init__(self):
@ -30,6 +29,8 @@ class AnnotationItem:
class Annotation:
"""Handle annotations."""
def __init__(self, name, files_registry):
self._all_items = defaultdict(dict)
self._file_handler = None
@ -62,7 +63,8 @@ class Annotation:
if re.match(regex, line.strip()):
item = self._get_annotation_data(
num, line, self._annotation_definition["name"], rfile)
num, line, self._annotation_definition["name"], rfile
)
if item:
self.logger.info(str(item))
self._populate_item(item.get_obj().items())
@ -166,4 +168,7 @@ class Annotation:
return {key: json.loads(string)}
except ValueError:
self.log.sysexit_with_message(
"Json value error: Can't parse json in {}:{}:\n{}".format(rfile, str(num), line.strip()))
"Json value error: Can't parse json in {}:{}:\n{}".format(
rfile, str(num), line.strip()
)
)

54
ansibledoctor/Cli.py
View File

@ -2,9 +2,6 @@
"""Entrypoint and CLI handler."""
import argparse
import logging
import os
import sys
import ansibledoctor.Exception
from ansibledoctor import __version__
@ -15,6 +12,7 @@ from ansibledoctor.Utils import SingleLog
class AnsibleDoctor:
"""Main doctor object."""
def __init__(self):
self.log = SingleLog()
@ -34,20 +32,42 @@ class AnsibleDoctor:
"""
# TODO: add function to print to stdout instead of file
parser = argparse.ArgumentParser(
description="Generate documentation from annotated Ansible roles using templates")
parser.add_argument("role_dir", nargs="?", help="role directory (default: current working dir)")
parser.add_argument("-c", "--config", dest="config_file", help="location of configuration file")
parser.add_argument("-o", "--output", dest="output_dir", action="store",
help="output base dir")
parser.add_argument("-f", "--force", dest="force_overwrite", action="store_true", default=None,
help="force overwrite output file")
parser.add_argument("-d", "--dry-run", dest="dry_run", action="store_true", default=None,
help="dry run without writing")
parser.add_argument("-v", dest="logging.level", action="append_const", const=-1,
help="increase log level")
parser.add_argument("-q", dest="logging.level", action="append_const",
const=1, help="decrease log level")
parser.add_argument("--version", action="version", version="%(prog)s {}".format(__version__))
description="Generate documentation from annotated Ansible roles using templates"
)
parser.add_argument(
"role_dir", nargs="?", help="role directory (default: current working dir)"
)
parser.add_argument(
"-c", "--config", dest="config_file", help="location of configuration file"
)
parser.add_argument(
"-o", "--output", dest="output_dir", action="store", help="output base dir"
)
parser.add_argument(
"-f",
"--force",
dest="force_overwrite",
action="store_true",
default=None,
help="force overwrite output file"
)
parser.add_argument(
"-d",
"--dry-run",
dest="dry_run",
action="store_true",
default=None,
help="dry run without writing"
)
parser.add_argument(
"-v", dest="logging.level", action="append_const", const=-1, help="increase log level"
)
parser.add_argument(
"-q", dest="logging.level", action="append_const", const=1, help="decrease log level"
)
parser.add_argument(
"--version", action="version", version="%(prog)s {}".format(__version__)
)
return parser.parse_args().__dict__

19
ansibledoctor/Config.py
View File

@ -1,9 +1,7 @@
#!/usr/bin/env python3
"""Global settings definition."""
import logging
import os
import sys
import anyconfig
import environs
@ -11,7 +9,6 @@ import jsonschema.exceptions
import ruamel.yaml
from appdirs import AppDirs
from jsonschema._utils import format_as_index
from pkg_resources import resource_filename
import ansibledoctor.Exception
from ansibledoctor.Utils import Singleton
@ -116,11 +113,7 @@ class Config():
"var": {
"name": "var",
"automatic": True,
"subtypes": [
"value",
"example",
"description"
]
"subtypes": ["value", "example", "description"]
},
"example": {
"name": "example",
@ -192,7 +185,9 @@ class Config():
if '"{}" not set'.format(envname) in str(e):
pass
else:
raise ansibledoctor.Exception.ConfigError("Unable to read environment variable", str(e))
raise ansibledoctor.Exception.ConfigError(
"Unable to read environment variable", str(e)
)
return normalized
@ -224,7 +219,9 @@ class Config():
s = stream.read()
try:
file_dict = ruamel.yaml.safe_load(s)
except (ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError) as e:
except (
ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError
) as e:
message = "{} {}".format(e.context, e.problem)
raise ansibledoctor.Exception.ConfigError(
"Unable to read config file {}".format(config), message
@ -313,4 +310,6 @@ class Config():
class SingleConfig(Config, metaclass=Singleton):
"""Singleton config class."""
pass

41
ansibledoctor/DocumentationGenerator.py
View File

@ -1,13 +1,9 @@
#!/usr/bin/env python3
"""Prepare output and write compiled jinja2 templates."""
import codecs
import glob
import json
import ntpath
import os
import pprint
import sys
from functools import reduce
import jinja2.exceptions
@ -15,8 +11,6 @@ import ruamel.yaml
from jinja2 import Environment
from jinja2 import FileSystemLoader
from jinja2.filters import evalcontextfilter
from six import binary_type
from six import text_type
import ansibledoctor.Exception
from ansibledoctor.Config import SingleConfig
@ -25,6 +19,8 @@ from ansibledoctor.Utils import SingleLog
class Generator:
"""Generate documentation from jinja2 templates."""
def __init__(self, doc_parser):
self.template_files = []
self.extension = "j2"
@ -67,7 +63,10 @@ class Generator:
files_to_overwite = []
for file in self.template_files:
doc_file = os.path.join(self.config.config.get("output_dir"), os.path.splitext(file)[0])
doc_file = os.path.join(
self.config.config.get("output_dir"),
os.path.splitext(file)[0]
)
if os.path.isfile(doc_file):
files_to_overwite.append(doc_file)
@ -95,7 +94,10 @@ class Generator:
self.log.sysexit_with_message("Aborted...")
for file in self.template_files:
doc_file = os.path.join(self.config.config.get("output_dir"), os.path.splitext(file)[0])
doc_file = os.path.join(
self.config.config.get("output_dir"),
os.path.splitext(file)[0]
)
source_file = self.config.get_template() + "/" + file
self.logger.debug("Writing doc output to: " + doc_file + " from: " + source_file)
@ -109,7 +111,11 @@ class Generator:
if data is not None:
try:
# print(json.dumps(role_data, indent=4, sort_keys=True))
jenv = Environment(loader=FileSystemLoader(self.config.get_template()), lstrip_blocks=True, trim_blocks=True) # nosec
jenv = Environment(
loader=FileSystemLoader(self.config.get_template()),
lstrip_blocks=True,
trim_blocks=True
) # nosec
jenv.filters["to_nice_yaml"] = self._to_nice_yaml
jenv.filters["deep_get"] = self._deep_get
jenv.filters["save_join"] = self._save_join
@ -121,12 +127,18 @@ class Generator:
self.logger.info("Writing to: " + doc_file)
else:
self.logger.info("Writing to: " + doc_file)
except (jinja2.exceptions.UndefinedError, jinja2.exceptions.TemplateSyntaxError)as e:
except (
jinja2.exceptions.UndefinedError, jinja2.exceptions.TemplateSyntaxError
) as e:
self.log.sysexit_with_message(
"Jinja2 templating error while loading file: '{}'\n{}".format(file, str(e)))
"Jinja2 templating error while loading file: '{}'\n{}".format(
file, str(e)
)
)
except UnicodeEncodeError as e:
self.log.sysexit_with_message(
"Unable to print special characters\n{}".format(str(e)))
"Unable to print special characters\n{}".format(str(e))
)
def _to_nice_yaml(self, a, indent=4, *args, **kw):
"""Make verbose, human readable yaml."""
@ -138,7 +150,10 @@ class Generator:
def _deep_get(self, _, dictionary, keys, *args, **kw):
default = None
return reduce(lambda d, key: d.get(key, default) if isinstance(d, dict) else default, keys.split("."), dictionary)
return reduce(
lambda d, key: d.get(key, default)
if isinstance(d, dict) else default, keys.split("."), dictionary
)
@evalcontextfilter
def _save_join(self, eval_ctx, value, d=u"", attribute=None):

42
ansibledoctor/DocumentationParser.py
View File

@ -2,8 +2,6 @@
"""Parse static files."""
import fnmatch
import json
import os
from collections import defaultdict
import anyconfig
@ -19,6 +17,8 @@ from ansibledoctor.Utils import UnsafeTag
class Parser:
"""Parse yaml files."""
def __init__(self):
self._annotation_objs = {}
self._data = defaultdict(dict)
@ -36,13 +36,21 @@ class Parser:
if any(fnmatch.fnmatch(rfile, "*/defaults/*." + ext) for ext in YAML_EXTENSIONS):
with open(rfile, "r", encoding="utf8") as yaml_file:
try:
ruamel.yaml.add_constructor(UnsafeTag.yaml_tag, UnsafeTag.yaml_constructor, constructor=ruamel.yaml.SafeConstructor)
ruamel.yaml.add_constructor(
UnsafeTag.yaml_tag,
UnsafeTag.yaml_constructor,
constructor=ruamel.yaml.SafeConstructor
)
data = defaultdict(dict, (ruamel.yaml.safe_load(yaml_file) or {}))
for key, value in data.items():
self._data["var"][key] = {"value": {key: value}}
except (ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError) as e:
except (
ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError
) as e:
message = "{} {}".format(e.context, e.problem)
self.log.sysexit_with_message("Unable to read yaml file {}\n{}".format(rfile, message))
self.log.sysexit_with_message(
"Unable to read yaml file {}\n{}".format(rfile, message)
)
def _parse_meta_file(self):
for rfile in self._files_registry.get_files():
@ -55,12 +63,18 @@ class Parser:
self._data["meta"][key] = {"value": value}
if data.get("dependencies") is not None:
self._data["meta"]["dependencies"] = {"value": data.get("dependencies")}
self._data["meta"]["dependencies"] = {
"value": data.get("dependencies")
}
self._data["meta"]["name"] = {"value": self.config.config["role_name"]}
except (ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError) as e:
except (
ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError
) as e:
message = "{} {}".format(e.context, e.problem)
self.log.sysexit_with_message("Unable to read yaml file {}\n{}".format(rfile, message))
self.log.sysexit_with_message(
"Unable to read yaml file {}\n{}".format(rfile, message)
)
def _parse_task_tags(self):
for rfile in self._files_registry.get_files():
@ -68,9 +82,13 @@ class Parser:
with open(rfile, "r", encoding="utf8") as yaml_file:
try:
data = ruamel.yaml.safe_load(yaml_file)
except (ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError) as e:
except (
ruamel.yaml.composer.ComposerError, ruamel.yaml.scanner.ScannerError
) as e:
message = "{} {}".format(e.context, e.problem)
self.log.sysexit_with_message("Unable to read yaml file {}\n{}".format(rfile, message))
self.log.sysexit_with_message(
"Unable to read yaml file {}\n{}".format(rfile, message)
)
tags_found = nested_lookup("tags", data)
for tag in tags_found:
@ -81,7 +99,9 @@ class Parser:
tags = defaultdict(dict)
for annotaion in self.config.get_annotations_names(automatic=True):
self.logger.info("Finding annotations for: @" + annotaion)
self._annotation_objs[annotaion] = Annotation(name=annotaion, files_registry=self._files_registry)
self._annotation_objs[annotaion] = Annotation(
name=annotaion, files_registry=self._files_registry
)
tags[annotaion] = self._annotation_objs[annotaion].get_details()
try:

12
ansibledoctor/FileRegistry.py
View File

@ -3,7 +3,6 @@
import glob
import os
import sys
import pathspec
@ -13,6 +12,7 @@ from ansibledoctor.Utils import SingleLog
class Registry:
"""Register all yaml files."""
_doc = {}
log = None
@ -46,7 +46,13 @@ class Registry:
pattern = os.path.join(role_dir, "**/*." + extension)
for filename in glob.iglob(pattern, recursive=True):
if not excludespec.match_file(filename):
self.log.debug("Adding file to '{}': {}".format(role_name, os.path.relpath(filename, role_dir)))
self.log.debug(
"Adding file to '{}': {}".format(
role_name, os.path.relpath(filename, role_dir)
)
)
self._doc.append(filename)
else:
self.log.debug("Excluding file: {}".format(os.path.relpath(filename, role_dir)))
self.log.debug(
"Excluding file: {}".format(os.path.relpath(filename, role_dir))
)

50
ansibledoctor/Utils.py
View File

@ -3,7 +3,6 @@
import logging
import os
import pprint
import sys
from distutils.util import strtobool
@ -32,6 +31,8 @@ colorama.init(autoreset=True, strip=not _should_do_markup())
class Singleton(type):
"""Meta singleton class."""
_instances = {}
def __call__(cls, *args, **kwargs):
@ -61,7 +62,7 @@ class LogFilter(object):
class MultilineFormatter(logging.Formatter):
"""Logging Formatter to reset color after newline characters."""
def format(self, record): # noqa
def format(self, record): # noqa
record.msg = record.msg.replace("\n", "\n{}... ".format(colorama.Style.RESET_ALL))
return logging.Formatter.format(self, record)
@ -69,12 +70,14 @@ class MultilineFormatter(logging.Formatter):
class MultilineJsonFormatter(jsonlogger.JsonFormatter):
"""Logging Formatter to remove newline characters."""
def format(self, record): # noqa
def format(self, record): # noqa
record.msg = record.msg.replace("\n", " ")
return jsonlogger.JsonFormatter.format(self, record)
class Log:
"""Handle logging."""
def __init__(self, level=logging.WARN, name="ansibledoctor", json=False):
self.logger = logging.getLogger(name)
self.logger.setLevel(level)
@ -89,8 +92,11 @@ class Log:
handler = logging.StreamHandler(sys.stderr)
handler.setLevel(logging.ERROR)
handler.addFilter(LogFilter(logging.ERROR))
handler.setFormatter(MultilineFormatter(
self.error(CONSOLE_FORMAT.format(colorama.Fore.RED, colorama.Style.RESET_ALL))))
handler.setFormatter(
MultilineFormatter(
self.error(CONSOLE_FORMAT.format(colorama.Fore.RED, colorama.Style.RESET_ALL))
)
)
if json:
handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
@ -101,8 +107,11 @@ class Log:
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.WARN)
handler.addFilter(LogFilter(logging.WARN))
handler.setFormatter(MultilineFormatter(
self.warn(CONSOLE_FORMAT.format(colorama.Fore.YELLOW, colorama.Style.RESET_ALL))))
handler.setFormatter(
MultilineFormatter(
self.warn(CONSOLE_FORMAT.format(colorama.Fore.YELLOW, colorama.Style.RESET_ALL))
)
)
if json:
handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
@ -113,8 +122,11 @@ class Log:
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.INFO)
handler.addFilter(LogFilter(logging.INFO))
handler.setFormatter(MultilineFormatter(
self.info(CONSOLE_FORMAT.format(colorama.Fore.CYAN, colorama.Style.RESET_ALL))))
handler.setFormatter(
MultilineFormatter(
self.info(CONSOLE_FORMAT.format(colorama.Fore.CYAN, colorama.Style.RESET_ALL))
)
)
if json:
handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
@ -125,8 +137,11 @@ class Log:
handler = logging.StreamHandler(sys.stderr)
handler.setLevel(logging.CRITICAL)
handler.addFilter(LogFilter(logging.CRITICAL))
handler.setFormatter(MultilineFormatter(
self.critical(CONSOLE_FORMAT.format(colorama.Fore.RED, colorama.Style.RESET_ALL))))
handler.setFormatter(
MultilineFormatter(
self.critical(CONSOLE_FORMAT.format(colorama.Fore.RED, colorama.Style.RESET_ALL))
)
)
if json:
handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
@ -137,8 +152,11 @@ class Log:
handler = logging.StreamHandler(sys.stderr)
handler.setLevel(logging.DEBUG)
handler.addFilter(LogFilter(logging.DEBUG))
handler.setFormatter(MultilineFormatter(
self.critical(CONSOLE_FORMAT.format(colorama.Fore.BLUE, colorama.Style.RESET_ALL))))
handler.setFormatter(
MultilineFormatter(
self.critical(CONSOLE_FORMAT.format(colorama.Fore.BLUE, colorama.Style.RESET_ALL))
)
)
if json:
handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
@ -188,10 +206,14 @@ class Log:
class SingleLog(Log, metaclass=Singleton):
"""Singleton logging class."""
pass
class UnsafeTag:
"""Handle custom yaml unsafe tag."""
yaml_tag = u"!unsafe"
def __init__(self, value):
@ -203,6 +225,8 @@ class UnsafeTag:
class FileUtils:
"""Mics static methods for file handling."""
@staticmethod
def create_path(path):
os.makedirs(path, exist_ok=True)

14
setup.cfg
View File

@ -10,11 +10,21 @@ default_section = THIRDPARTY
known_first_party = ansibledoctor
sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER
force_single_line = true
line_length = 110
skip_glob = **/env/*,**/docs/*
line_length = 99
skip_glob = **/.env*,**/env/*,**/docs/*
[yapf]
based_on_style = google
column_limit = 99
dedent_closing_brackets = true
coalesce_brackets = true
split_before_logical_operator = true
[tool:pytest]
filterwarnings =
ignore::FutureWarning
ignore:.*collections.*:DeprecationWarning
ignore:.*pep8.*:FutureWarning
[coverage:run]
omit = **/tests/*

16
setup.py
View File

@ -33,26 +33,28 @@ setup(
keywords="ansible role documentation",
author=get_property("__author__", PACKAGE_NAME),
author_email=get_property("__email__", PACKAGE_NAME),
url="https://github.com/xoxys/ansible-doctor",
license=get_property("__url__", PACKAGE_NAME),
url=get_property("__url__", PACKAGE_NAME)
license=get_property("__license__", PACKAGE_NAME),
long_description=get_readme(),
long_description_content_type="text/markdown",
packages=find_packages(exclude=["*.tests", "tests", "tests.*"]),
include_package_data=True,
zip_safe=False,
python_requires=">=3.5",
python_requires=">=3.5,<4",
classifiers=[
"Development Status :: 5 - Production/Stable",
"Environment :: Console",
"License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
"Intended Audience :: Developers",
"Intended Audience :: Information Technology",
"Intended Audience :: System Administrators",
"License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
"Natural Language :: English",
"Operating System :: POSIX",
"Programming Language :: Python :: 3 :: Only",
"Topic :: System :: Installation/Setup",
"Topic :: System :: Systems Administration",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Topic :: Utilities",
"Topic :: Software Development",
"Topic :: Software Development :: Documentation",

4
test-requirements.txt
View File

@ -1,11 +1,8 @@
# open issue
# https://gitlab.com/pycqa/flake8-docstrings/issues/36
pydocstyle<4.0.0
flake8
flake8-colors
flake8-blind-except
flake8-builtins
flake8-colors
flake8-docstrings<=3.0.0
flake8-isort
flake8-logging-format
@ -17,3 +14,4 @@ pytest
pytest-mock
pytest-cov
bandit
yapf