From 2b574970c211427ed0bd784f48138529524184e8 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Thu, 4 Apr 2019 15:35:47 +0200 Subject: [PATCH] fix json logging and add file exclude options --- ansiblelater/__main__.py | 8 ++----- ansiblelater/command/candidates.py | 35 ++++++++++++++---------------- ansiblelater/command/review.py | 5 ----- ansiblelater/exceptions.py | 5 +---- ansiblelater/logger.py | 11 ++-------- ansiblelater/settings.py | 22 +++++++++++++++++-- ansiblelater/standard.py | 1 - ansiblelater/utils/__init__.py | 31 ++++++++++---------------- ansiblelater/utils/rulehelper.py | 33 ++++++++++++++-------------- 9 files changed, 69 insertions(+), 82 deletions(-) diff --git a/ansiblelater/__main__.py b/ansiblelater/__main__.py index 1930227..b3a990a 100755 --- a/ansiblelater/__main__.py +++ b/ansiblelater/__main__.py @@ -1,14 +1,10 @@ #!/usr/bin/env python import argparse -import json import logging -from ansiblelater import __version__ -from ansiblelater import LOG -from ansiblelater import logger -from ansiblelater.command import base -from ansiblelater.command import candidates +from ansiblelater import LOG, __version__, logger +from ansiblelater.command import base, candidates def main(): diff --git a/ansiblelater/command/candidates.py b/ansiblelater/command/candidates.py index 21bbf30..1d8339a 100644 --- a/ansiblelater/command/candidates.py +++ b/ansiblelater/command/candidates.py @@ -5,18 +5,12 @@ import codecs import copy import os import re -import sys from distutils.version import LooseVersion -import ansible - -from ansiblelater import LOG -from ansiblelater import utils -from ansiblelater.logger import flag_extra +from ansiblelater import LOG, utils from ansiblelater.command.review import Error -from ansiblelater.exceptions import ( # noqa - LaterError, LaterAnsibleError -) +from ansiblelater.exceptions import LaterAnsibleError, LaterError # noqa +from ansiblelater.logger import flag_extra try: # Ansible 2.4 import of module loader @@ -78,13 +72,13 @@ class Candidate(object): if self.expected_version: if isinstance(self, RoleFile): LOG.warn("%s %s is in a role that contains a meta/main.yml without a declared " - "standards version. " - "Using latest standards version %s" % - (type(self).__name__, self.path, version)) + "standards version. " + "Using latest standards version %s" % + (type(self).__name__, self.path, version)) else: LOG.warn("%s %s does not present standards version. " - "Using latest standards version %s" % - (type(self).__name__, self.path, version)) + "Using latest standards version %s" % + (type(self).__name__, self.path, version)) LOG.info("%s %s declares standards version %s" % (type(self).__name__, self.path, version)) @@ -114,8 +108,8 @@ class Candidate(object): result = standard.check(self, settings.config) if not result: - utils.sysexit_with_message("Standard '%s' returns an empty result object." % - (standard.check.__name__)) + utils.sysexit_with_message("Standard '{}' returns an empty result object.".format( + standard.check.__name__)) labels = {"tag": "review", "standard": standard.name, "file": self.path, "passed": True} @@ -128,13 +122,16 @@ class Candidate(object): if not standard.version: LOG.warn("{id}Best practice '{name}' not met:\n{path}:{error}".format( - id=standard.id, name=standard.name, path=self.path, error=err), extra=flag_extra(err_labels)) + id=standard.id, name=standard.name, path=self.path, error=err), + extra=flag_extra(err_labels)) elif LooseVersion(standard.version) > LooseVersion(self.version): LOG.warn("{id}Future standard '{name}' not met:\n{path}:{error}".format( - id=standard.id, name=standard.name, path=self.path, error=err), extra=flag_extra(err_labels)) + id=standard.id, name=standard.name, path=self.path, error=err), + extra=flag_extra(err_labels)) else: LOG.error("{id}Standard '{name}' not met:\n{path}:{error}".format( - id=standard.id, name=standard.name, path=self.path, error=err), extra=flag_extra(err_labels)) + id=standard.id, name=standard.name, path=self.path, error=err), + extra=flag_extra(err_labels)) errors = errors + 1 if not result.errors: if not standard.version: diff --git a/ansiblelater/command/review.py b/ansiblelater/command/review.py index b746898..30e373b 100644 --- a/ansiblelater/command/review.py +++ b/ansiblelater/command/review.py @@ -1,8 +1,5 @@ """Review candidates.""" -import os -import sys - from six import iteritems @@ -44,5 +41,3 @@ class Result(object): def message(self): return "\n".join(["{0}:{1}".format(self.candidate, error) for error in self.errors]) - - diff --git a/ansiblelater/exceptions.py b/ansiblelater/exceptions.py index 4d13f19..b60df69 100644 --- a/ansiblelater/exceptions.py +++ b/ansiblelater/exceptions.py @@ -7,10 +7,7 @@ class LaterError(Exception): """Generic exception for later.""" def __init__(self, msg, original): - """ - Initialize new exception. - - """ + """Initialize new exception.""" super(LaterError, self).__init__(msg + (": %s" % original)) self.original = original diff --git a/ansiblelater/logger.py b/ansiblelater/logger.py index e197fc5..624064c 100644 --- a/ansiblelater/logger.py +++ b/ansiblelater/logger.py @@ -5,12 +5,12 @@ import os import sys import colorama -from six import iteritems from ansible.module_utils.parsing.convert_bool import boolean as to_bool from pythonjsonlogger import jsonlogger +from six import iteritems CONSOLE_FORMAT = "%(levelname)s: %(message)s" -JSON_FORMAT = "(levelname) (asctime)" +JSON_FORMAT = "(asctime) (levelname) (message)" def _should_do_markup(): @@ -24,13 +24,6 @@ def _should_do_markup(): colorama.init(autoreset=True, strip=not _should_do_markup()) -clashing_keywords = {key for key in dir(logging.LogRecord(None, None, "", 0, "", (), None, None)) if "__" not in key} -additional_clashing_keywords = { - "message", - "asctime" -} -clashing_keywords = clashing_keywords.union(additional_clashing_keywords) - def flag_extra(kwargs): """Ensure kwargs not conflict with the logging module.""" diff --git a/ansiblelater/settings.py b/ansiblelater/settings.py index 7f00fcb..e118361 100644 --- a/ansiblelater/settings.py +++ b/ansiblelater/settings.py @@ -1,14 +1,16 @@ """Global settings object definition.""" +import copy import logging import os import anyconfig from appdirs import AppDirs +from globmatch import glob_match from jsonschema._utils import format_as_index from pkg_resources import resource_filename -from ansiblelater import logger, utils +from ansiblelater import utils config_dir = AppDirs("ansible-later").user_config_dir default_config_file = os.path.join(config_dir, "config.yml") @@ -37,6 +39,7 @@ class Settings(object): self.args = self._set_args(args) self.config = self._get_config() self.schema = None + self._update_filelist() def _set_args(self, args): self.config_file = args.get("config_file") or default_config_file @@ -80,6 +83,8 @@ class Settings(object): "rules": { "standards": rules_dir, "filter": [], + "ignore_dotfiles": True, + "exclude_files": [] }, "logging": { "level": logging.WARN, @@ -98,7 +103,7 @@ class Settings(object): filelist = [] for root, dirs, files in os.walk("."): for filename in files: - filelist.append(os.path.join(root, filename)) + filelist.append(os.path.relpath(os.path.normpath(os.path.join(root, filename)))) else: filelist = args["rules"]["files"] @@ -114,3 +119,16 @@ class Settings(object): schema=format_as_index(list(e.relative_schema_path)[:-1]) ) utils.sysexit_with_message("{schema}: {msg}".format(schema=schema_error, msg=e.message)) + + def _update_filelist(self): + files = self.config["rules"]["files"] + excludes = self.config["rules"]["exclude_files"] + ignore_dotfiles = self.config["rules"]["ignore_dotfiles"] + + if ignore_dotfiles: + excludes.append(".") + + valid = copy.copy(files) + for item in valid: + if glob_match(item, excludes): + files.remove(item) diff --git a/ansiblelater/standard.py b/ansiblelater/standard.py index 0026b86..91e314a 100644 --- a/ansiblelater/standard.py +++ b/ansiblelater/standard.py @@ -28,4 +28,3 @@ class Standard(object): def __repr__(self): # noqa return "Standard: %s (version: %s, types: %s)" % ( self.name, self.version, self.types) - diff --git a/ansiblelater/utils/__init__.py b/ansiblelater/utils/__init__.py index 91ba08e..f0186ba 100644 --- a/ansiblelater/utils/__init__.py +++ b/ansiblelater/utils/__init__.py @@ -1,22 +1,19 @@ from __future__ import print_function import contextlib -import importlib -import logging import os -import sys import re -import colorama +import sys +from distutils.version import LooseVersion import yaml -from distutils.version import LooseVersion + from ansiblelater import logger -from ansible.module_utils.parsing.convert_bool import boolean as to_bool try: - import ConfigParser as configparser + import ConfigParser as configparser # noqa except ImportError: - import configparser + import configparser # noqa LOG = logger.get_logger(__name__) @@ -67,19 +64,13 @@ def is_line_in_ranges(line, ranges): return not ranges or any([line in r for r in ranges]) - -def read_config(config_file): - config = configparser.RawConfigParser({'standards': None}) - config.read(config_file) - - return Settings(config, config_file) - - def safe_load(string): """ Parse the provided string returns a dict. + :param string: A string to be parsed. - :return: dict + :returns: dict + """ try: return yaml.safe_load(string) or {} @@ -88,12 +79,14 @@ def safe_load(string): @contextlib.contextmanager -def open_file(filename, mode='r'): +def open_file(filename, mode="r"): """ Open the provide file safely and returns a file type. + :param filename: A string containing an absolute path to the file to open. :param mode: A string describing the way in which the file will be used. - :return: file type + :returns: file type + """ with open(filename, mode) as stream: yield stream diff --git a/ansiblelater/utils/rulehelper.py b/ansiblelater/utils/rulehelper.py index 9ee87d8..1ae8274 100644 --- a/ansiblelater/utils/rulehelper.py +++ b/ansiblelater/utils/rulehelper.py @@ -1,23 +1,21 @@ import codecs -import yaml - from collections import defaultdict + +import yaml from yamllint import linter from yamllint.config import YamlLintConfig -# Workaround for import errors with ansble 2.1 and 2.3 -from ansible.parsing.dataloader import DataLoader + from ansiblelater.command.review import Error -from .yamlhelper import normalize_task -from .yamlhelper import action_tasks -from .yamlhelper import parse_yaml_linenumbers -from .yamlhelper import normalized_yaml -from ansiblelater.exceptions import LaterError, LaterAnsibleError +from ansiblelater.exceptions import LaterAnsibleError, LaterError + +from .yamlhelper import (action_tasks, normalize_task, normalized_yaml, + parse_yaml_linenumbers) def get_tasks(candidate, settings): errors = [] try: - with codecs.open(candidate.path, mode='rb', encoding='utf-8') as f: + with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f: yamllines = parse_yaml_linenumbers(f, candidate.path) except LaterError as ex: @@ -33,7 +31,7 @@ def get_action_tasks(candidate, settings): tasks = [] errors = [] try: - with codecs.open(candidate.path, mode='rb', encoding='utf-8') as f: + with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f: yamllines = parse_yaml_linenumbers(f, candidate.path) if yamllines: @@ -65,19 +63,20 @@ def get_normalized_tasks(candidate, settings): normalized = [] errors = [] try: - with codecs.open(candidate.path, mode='rb', encoding='utf-8') as f: + with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f: yamllines = parse_yaml_linenumbers(f, candidate.path) if yamllines: tasks = action_tasks(yamllines, candidate) for task in tasks: # An empty `tags` block causes `None` to be returned if - # the `or []` is not present - `task.get('tags', [])` + # the `or []` is not present - `task.get("tags", [])` # does not suffice. - if 'skip_ansible_lint' in (task.get('tags') or []): + if "skip_ansible_lint" in (task.get("tags") or []): # No need to normalize_task if we are skipping it. continue - normalized.append(normalize_task(task, candidate.path, settings["ansible"]["custom_modules"])) + normalized.append( + normalize_task(task, candidate.path, settings["ansible"]["custom_modules"])) except LaterError as ex: e = ex.original @@ -112,7 +111,7 @@ def get_raw_yaml(candidate, settings): errors = [] try: - with codecs.open(candidate.path, mode='rb', encoding='utf-8') as f: + with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f: content = yaml.safe_load(f) except LaterError as ex: @@ -125,7 +124,7 @@ def get_raw_yaml(candidate, settings): def run_yamllint(candidate, settings, options="extends: default"): errors = [] try: - with codecs.open(candidate.path, mode='rb', encoding='utf-8') as f: + with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f: for problem in linter.run(f, YamlLintConfig(options)): errors.append(Error(problem.line, problem.desc)) except LaterError as ex: