fix json logging and add file exclude options

ansiblelater/__main__.py

@@ -1,14 +1,10 @@
 #!/usr/bin/env python
 
 import argparse
-import json
 import logging
 
-from ansiblelater import __version__
-from ansiblelater import LOG
-from ansiblelater import logger
-from ansiblelater.command import base
-from ansiblelater.command import candidates
+from ansiblelater import LOG, __version__, logger
+from ansiblelater.command import base, candidates
 
 
 def main():

ansiblelater/command/candidates.py

@@ -5,18 +5,12 @@ import codecs
 import copy
 import os
 import re
-import sys
 from distutils.version import LooseVersion
 
-import ansible
-
-from ansiblelater import LOG
-from ansiblelater import utils
-from ansiblelater.logger import flag_extra
+from ansiblelater import LOG, utils
 from ansiblelater.command.review import Error
-from ansiblelater.exceptions import (  # noqa
-    LaterError, LaterAnsibleError
-)
+from ansiblelater.exceptions import LaterAnsibleError, LaterError  # noqa
+from ansiblelater.logger import flag_extra
 
 try:
     # Ansible 2.4 import of module loader
@@ -114,8 +108,8 @@ class Candidate(object):
             result = standard.check(self, settings.config)
 
             if not result:
-                utils.sysexit_with_message("Standard '%s' returns an empty result object." %
-                    (standard.check.__name__))
+                utils.sysexit_with_message("Standard '{}' returns an empty result object.".format(
+                    standard.check.__name__))
 
             labels = {"tag": "review", "standard": standard.name, "file": self.path, "passed": True}
 
@@ -128,13 +122,16 @@ class Candidate(object):
 
                 if not standard.version:
                     LOG.warn("{id}Best practice '{name}' not met:\n{path}:{error}".format(
-                        id=standard.id, name=standard.name, path=self.path, error=err), extra=flag_extra(err_labels))
+                        id=standard.id, name=standard.name, path=self.path, error=err),
+                        extra=flag_extra(err_labels))
                 elif LooseVersion(standard.version) > LooseVersion(self.version):
                     LOG.warn("{id}Future standard '{name}' not met:\n{path}:{error}".format(
-                        id=standard.id, name=standard.name, path=self.path, error=err), extra=flag_extra(err_labels))
+                        id=standard.id, name=standard.name, path=self.path, error=err),
+                        extra=flag_extra(err_labels))
                 else:
                     LOG.error("{id}Standard '{name}' not met:\n{path}:{error}".format(
-                        id=standard.id, name=standard.name, path=self.path, error=err), extra=flag_extra(err_labels))
+                        id=standard.id, name=standard.name, path=self.path, error=err),
+                        extra=flag_extra(err_labels))
                     errors = errors + 1
             if not result.errors:
                 if not standard.version:

ansiblelater/command/review.py

@@ -1,8 +1,5 @@
 """Review candidates."""
 
-import os
-import sys
-
 from six import iteritems
 
 
@@ -44,5 +41,3 @@ class Result(object):
     def message(self):
         return "\n".join(["{0}:{1}".format(self.candidate, error)
                           for error in self.errors])
-
-

ansiblelater/exceptions.py

@@ -7,10 +7,7 @@ class LaterError(Exception):
     """Generic exception for later."""
 
     def __init__(self, msg, original):
-        """
-        Initialize new exception.
-
-        """
+        """Initialize new exception."""
         super(LaterError, self).__init__(msg + (": %s" % original))
         self.original = original
 

ansiblelater/logger.py

@@ -5,12 +5,12 @@ import os
 import sys
 
 import colorama
-from six import iteritems
 from ansible.module_utils.parsing.convert_bool import boolean as to_bool
 from pythonjsonlogger import jsonlogger
+from six import iteritems
 
 CONSOLE_FORMAT = "%(levelname)s: %(message)s"
-JSON_FORMAT = "(levelname) (asctime)"
+JSON_FORMAT = "(asctime) (levelname) (message)"
 
 
 def _should_do_markup():
@@ -24,13 +24,6 @@ def _should_do_markup():
 
 colorama.init(autoreset=True, strip=not _should_do_markup())
 
-clashing_keywords = {key for key in dir(logging.LogRecord(None, None, "", 0, "", (), None, None)) if "__" not in key}
-additional_clashing_keywords = {
-    "message",
-    "asctime"
-}
-clashing_keywords = clashing_keywords.union(additional_clashing_keywords)
-
 
 def flag_extra(kwargs):
     """Ensure kwargs not conflict with the logging module."""

ansiblelater/settings.py

@@ -1,14 +1,16 @@
 """Global settings object definition."""
 
+import copy
 import logging
 import os
 
 import anyconfig
 from appdirs import AppDirs
+from globmatch import glob_match
 from jsonschema._utils import format_as_index
 from pkg_resources import resource_filename
 
-from ansiblelater import logger, utils
+from ansiblelater import utils
 
 config_dir = AppDirs("ansible-later").user_config_dir
 default_config_file = os.path.join(config_dir, "config.yml")
@@ -37,6 +39,7 @@ class Settings(object):
         self.args = self._set_args(args)
         self.config = self._get_config()
         self.schema = None
+        self._update_filelist()
 
     def _set_args(self, args):
         self.config_file = args.get("config_file") or default_config_file
@@ -80,6 +83,8 @@ class Settings(object):
             "rules": {
                 "standards": rules_dir,
                 "filter": [],
+                "ignore_dotfiles": True,
+                "exclude_files": []
             },
             "logging": {
                 "level": logging.WARN,
@@ -98,7 +103,7 @@ class Settings(object):
             filelist = []
             for root, dirs, files in os.walk("."):
                 for filename in files:
-                    filelist.append(os.path.join(root, filename))
+                    filelist.append(os.path.relpath(os.path.normpath(os.path.join(root, filename))))
         else:
             filelist = args["rules"]["files"]
 
@@ -114,3 +119,16 @@ class Settings(object):
                 schema=format_as_index(list(e.relative_schema_path)[:-1])
             )
             utils.sysexit_with_message("{schema}: {msg}".format(schema=schema_error, msg=e.message))
+
+    def _update_filelist(self):
+        files = self.config["rules"]["files"]
+        excludes = self.config["rules"]["exclude_files"]
+        ignore_dotfiles = self.config["rules"]["ignore_dotfiles"]
+
+        if ignore_dotfiles:
+            excludes.append(".")
+
+        valid = copy.copy(files)
+        for item in valid:
+            if glob_match(item, excludes):
+                files.remove(item)

ansiblelater/standard.py

@@ -28,4 +28,3 @@ class Standard(object):
     def __repr__(self): # noqa
         return "Standard: %s (version: %s, types: %s)" % (
                self.name, self.version, self.types)
-

ansiblelater/utils/__init__.py

@@ -1,22 +1,19 @@
 from __future__ import print_function
 
 import contextlib
-import importlib
-import logging
 import os
-import sys
 import re
-import colorama
+import sys
+from distutils.version import LooseVersion
 
 import yaml
-from distutils.version import LooseVersion
+
 from ansiblelater import logger
-from ansible.module_utils.parsing.convert_bool import boolean as to_bool
 
 try:
-    import ConfigParser as configparser
+    import ConfigParser as configparser # noqa
 except ImportError:
-    import configparser
+    import configparser # noqa
 
 
 LOG = logger.get_logger(__name__)
@@ -67,19 +64,13 @@ def is_line_in_ranges(line, ranges):
     return not ranges or any([line in r for r in ranges])
 
 
-
-def read_config(config_file):
-    config = configparser.RawConfigParser({'standards': None})
-    config.read(config_file)
-
-    return Settings(config, config_file)
-
-
 def safe_load(string):
     """
     Parse the provided string returns a dict.
+
     :param string: A string to be parsed.
-    :return: dict
+    :returns: dict
+
     """
     try:
         return yaml.safe_load(string) or {}
@@ -88,12 +79,14 @@ def safe_load(string):
 
 
 @contextlib.contextmanager
-def open_file(filename, mode='r'):
+def open_file(filename, mode="r"):
     """
     Open the provide file safely and returns a file type.
+
     :param filename: A string containing an absolute path to the file to open.
     :param mode: A string describing the way in which the file will be used.
-    :return: file type
+    :returns: file type
+
     """
     with open(filename, mode) as stream:
         yield stream

ansiblelater/utils/rulehelper.py

@@ -1,23 +1,21 @@
 import codecs
-import yaml
-
 from collections import defaultdict
+
+import yaml
 from yamllint import linter
 from yamllint.config import YamlLintConfig
-# Workaround for import errors with ansble 2.1 and 2.3
-from ansible.parsing.dataloader import DataLoader
+
 from ansiblelater.command.review import Error
-from .yamlhelper import normalize_task
-from .yamlhelper import action_tasks
-from .yamlhelper import parse_yaml_linenumbers
-from .yamlhelper import normalized_yaml
-from ansiblelater.exceptions import LaterError, LaterAnsibleError
+from ansiblelater.exceptions import LaterAnsibleError, LaterError
+
+from .yamlhelper import (action_tasks, normalize_task, normalized_yaml,
+                         parse_yaml_linenumbers)
 
 
 def get_tasks(candidate, settings):
     errors = []
     try:
-        with codecs.open(candidate.path, mode='rb', encoding='utf-8') as f:
+        with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
             yamllines = parse_yaml_linenumbers(f, candidate.path)
 
     except LaterError as ex:
@@ -33,7 +31,7 @@ def get_action_tasks(candidate, settings):
     tasks = []
     errors = []
     try:
-        with codecs.open(candidate.path, mode='rb', encoding='utf-8') as f:
+        with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
             yamllines = parse_yaml_linenumbers(f, candidate.path)
 
         if yamllines:
@@ -65,19 +63,20 @@ def get_normalized_tasks(candidate, settings):
     normalized = []
     errors = []
     try:
-        with codecs.open(candidate.path, mode='rb', encoding='utf-8') as f:
+        with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
             yamllines = parse_yaml_linenumbers(f, candidate.path)
 
         if yamllines:
             tasks = action_tasks(yamllines, candidate)
             for task in tasks:
                 # An empty `tags` block causes `None` to be returned if
-                # the `or []` is not present - `task.get('tags', [])`
+                # the `or []` is not present - `task.get("tags", [])`
                 # does not suffice.
-                if 'skip_ansible_lint' in (task.get('tags') or []):
+                if "skip_ansible_lint" in (task.get("tags") or []):
                     # No need to normalize_task if we are skipping it.
                     continue
-                normalized.append(normalize_task(task, candidate.path, settings["ansible"]["custom_modules"]))
+                normalized.append(
+                    normalize_task(task, candidate.path, settings["ansible"]["custom_modules"]))
 
     except LaterError as ex:
         e = ex.original
@@ -112,7 +111,7 @@ def get_raw_yaml(candidate, settings):
     errors = []
 
     try:
-        with codecs.open(candidate.path, mode='rb', encoding='utf-8') as f:
+        with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
             content = yaml.safe_load(f)
 
     except LaterError as ex:
@@ -125,7 +124,7 @@ def get_raw_yaml(candidate, settings):
 def run_yamllint(candidate, settings, options="extends: default"):
     errors = []
     try:
-        with codecs.open(candidate.path, mode='rb', encoding='utf-8') as f:
+        with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
             for problem in linter.run(f, YamlLintConfig(options)):
                 errors.append(Error(problem.line, problem.desc))
     except LaterError as ex: