From 0ffdf36a0d9c2afc01f2f79526f15c4c4c8da008 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Thu, 28 Mar 2019 16:54:45 +0100 Subject: [PATCH] refactor logging and switch to python standard library --- ansiblelater/__init__.py | 35 +++++------ ansiblelater/__main__.py | 63 ++++++++----------- ansiblelater/logger.py | 111 +++++++++++++++++++++++++++++++++ ansiblelater/settings.py | 66 ++++++++++++++------ ansiblelater/utils/__init__.py | 45 +++---------- setup.cfg | 2 +- 6 files changed, 209 insertions(+), 113 deletions(-) create mode 100644 ansiblelater/logger.py diff --git a/ansiblelater/__init__.py b/ansiblelater/__init__.py index bdbfdcb..bc98a53 100644 --- a/ansiblelater/__init__.py +++ b/ansiblelater/__init__.py @@ -16,11 +16,11 @@ from distutils.version import LooseVersion import ansible from appdirs import AppDirs -from ansiblelater.utils import (abort, error, get_property, info, +from . import logger +from .settings import Settings +from ansiblelater.utils import (get_property, is_line_in_ranges, lines_ranges, - read_standards, standards_latest, warn) - - + read_standards, standards_latest) try: # Ansible 2.4 import of module loader @@ -31,13 +31,8 @@ except ImportError: except ImportError: from ansible.utils import module_finder as module_loader -try: - import ConfigParser as configparser -except ImportError: - import configparser - - - +settings = Settings() +logger = logger.get_logger(__name__, settings.config["logging"]["level"]) class Standard(object): @@ -281,16 +276,16 @@ def candidate_review(candidate, settings, lines=None): candidate.version = standards_latest(standards.standards) if candidate.expected_version: if isinstance(candidate, RoleFile): - warn("%s %s is in a role that contains a meta/main.yml without a declared " - "standards version. " - "Using latest standards version %s" % - (type(candidate).__name__, candidate.path, candidate.version), - settings) + logger.warn("%s %s is in a role that contains a meta/main.yml without a declared " + "standards version. " + "Using latest standards version %s" % + (type(candidate).__name__, candidate.path, candidate.version), + settings) else: - warn("%s %s does not present standards version. " - "Using latest standards version %s" % - (type(candidate).__name__, candidate.path, candidate.version), - settings) + logger.warn("%s %s does not present standards version. " + "Using latest standards version %s" % + (type(candidate).__name__, candidate.path, candidate.version), + settings) info("%s %s declares standards version %s" % (type(candidate).__name__, candidate.path, candidate.version), diff --git a/ansiblelater/__main__.py b/ansiblelater/__main__.py index 45738ed..6de6bdb 100755 --- a/ansiblelater/__main__.py +++ b/ansiblelater/__main__.py @@ -1,56 +1,45 @@ #!/usr/bin/env python +import argparse +import json import logging -import optparse import os import sys -from appdirs import AppDirs -from pkg_resources import resource_filename +from ansiblelater import __version__, settings, logger +from ansiblelater.utils import get_property -from ansiblelater import classify, settings -from ansiblelater.utils import get_property, info, warn - -from .settings import Settings +# from .settings import Settings def main(): - config_dir = AppDirs("ansible-later").user_config_dir - default_config_file = os.path.join(config_dir, "config.yml") - - parser = optparse.OptionParser("%prog playbook_file|role_file|inventory_file", - version="%prog " + get_property("__version__")) - parser.add_option('-c', dest='config_file', default=default_config_file, - help="Location of configuration file: [%s]" % default_config_file) - parser.add_option('-d', dest='rules_dir', - help="Location of standards rules") - parser.add_option('-q', dest='log_level', action="store_const", - const=logging.ERROR, help="Only output errors") - parser.add_option('-s', dest='standards_filter', action='append', - help="limit standards to specific names") - parser.add_option('-v', '--verbose', dest='log_level', action="count", + parser = argparse.ArgumentParser( + description="Validate ansible files against best pratice guideline") + parser.add_argument('-c', dest='config_file', + help="Location of configuration file: [%s]" % settings.config_file) + parser.add_argument('-d', dest='rules.standards', + help="Location of standards rules") + parser.add_argument('-q', dest='logging.level', action="store_const", + const=logging.ERROR, help="Only output errors") + parser.add_argument('-s', dest='rules.filter', action='append', + help="limit standards to specific names") + parser.add_argument('-v', '--verbose', dest='logging.level', action="count", help="Show more verbose output") + parser.add_argument('--version', action='version', version='%(prog)s {}'.format(__version__)) - options, args = parser.parse_args(sys.argv[1:]) + args = parser.parse_args().__dict__ - settings = Settings(options) + # Override correct log level from argparse + levels = [logging.WARNING, logging.INFO, logging.DEBUG] + if args.get("logging.level"): + args["logging.level"] = levels[min(len(levels) - 1, args["logging.level"] - 1)] - # print(settings.rulesdir) - # settings = read_config(options.configfile) + settings.set_args(args) + + # print(json.dumps(settings.config, indent=4, sort_keys=True)) + # print(settings.config["logging"]["level"]) - # # Merge CLI options with config options. CLI options override config options. - # for key, value in options.__dict__.items(): - # if value: - # setattr(settings, key, value) - # if os.path.exists(settings.configfile): - # info("Using configuration file: %s" % settings.configfile, settings) - # else: - # warn("No configuration file found at %s" % settings.configfile, settings, file=sys.stderr) - # if not settings.rulesdir: - # rules_dir = os.path.join(resource_filename('ansiblelater', 'examples')) - # warn("Using example standards found at %s" % rules_dir, settings, file=sys.stderr) - # settings.rulesdir = rules_dir # if len(args) == 0: # candidates = [] diff --git a/ansiblelater/logger.py b/ansiblelater/logger.py new file mode 100644 index 0000000..a609ac0 --- /dev/null +++ b/ansiblelater/logger.py @@ -0,0 +1,111 @@ +import logging +import os +import sys + +import colorama +from pythonjsonlogger import jsonlogger +from ansible.module_utils.parsing.convert_bool import boolean as to_bool + + +def should_do_markup(): + py_colors = os.environ.get('PY_COLORS', None) + if py_colors is not None: + return to_bool(py_colors, strict=False) + + return sys.stdout.isatty() and os.environ.get('TERM') != 'dumb' + + +colorama.init(autoreset=True, strip=not should_do_markup()) + + +class LogFilter(object): + """ + A custom log filter which excludes log messages above the logged + level. + """ + + def __init__(self, level): + self.__level = level + + def filter(self, logRecord): # pragma: no cover + # https://docs.python.org/3/library/logging.html#logrecord-attributes + return logRecord.levelno <= self.__level + + +def get_logger(name=None, level=logging.DEBUG, json=False): + """ + Build a logger with the given name and returns the logger. + :param name: The name for the logger. This is usually the module + name, ``__name__``. + :return: logger object + """ + + logger = logging.getLogger(name) + logger.setLevel(level) + #handler = logging.StreamHandler() + #formatter = jsonlogger.JsonFormatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + #formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + #handler.setFormatter(formatter) + logger.addHandler(_get_error_handler(json=json)) + logger.addHandler(_get_warn_handler(json=json)) + logger.addHandler(_get_info_handler(json=json)) + logger.propagate = False + + return logger + + +def _get_error_handler(json=False): + handler = logging.StreamHandler(sys.stderr) + handler.setLevel(logging.ERROR) + handler.addFilter(LogFilter(logging.ERROR)) + handler.setFormatter(logging.Formatter(error('%(message)s'))) + + if json: + handler.setFormatter(jsonlogger.JsonFormatter('%(message)s')) + + return handler + + +def _get_warn_handler(json=False): + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.WARN) + handler.addFilter(LogFilter(logging.WARN)) + handler.setFormatter(logging.Formatter(warn('%(message)s'))) + + if json: + handler.setFormatter(jsonlogger.JsonFormatter('%(message)s')) + + return handler + + +def _get_info_handler(json=False): + handler = logging.StreamHandler(sys.stderr) + handler.setLevel(logging.INFO) + handler.addFilter(LogFilter(logging.INFO)) + handler.setFormatter(logging.Formatter(info('%(message)s'))) + + if json: + handler.setFormatter(jsonlogger.JsonFormatter('%(message)s')) + + return handler + + +def abort(message, file=sys.stderr): + return color_text(colorama.Fore.RED, "FATAL: {}".format(message)) + sys.exit(1) + + +def error(message): + return color_text(colorama.Fore.RED, "ERROR: {}".format(message)) + + +def warn(message): + return color_text(colorama.Fore.YELLOW, "WARN: {}".format(message)) + + +def info(message): + return color_text(colorama.Fore.BLUE, "INFO: {}".format(message)) + + +def color_text(color, msg): + return '{}{}{}'.format(color, msg, colorama.Style.RESET_ALL) diff --git a/ansiblelater/settings.py b/ansiblelater/settings.py index cdf0411..ece6e67 100644 --- a/ansiblelater/settings.py +++ b/ansiblelater/settings.py @@ -1,42 +1,60 @@ -import json import logging import os +import six import anyconfig +from appdirs import AppDirs from pkg_resources import resource_filename -from ansiblelater import utils +from ansiblelater import utils, logger -try: - import ConfigParser as configparser -except ImportError: - import configparser +config_dir = AppDirs("ansible-later").user_config_dir +default_config_file = os.path.join(config_dir, "config.yml") + +logger = logger.get_logger(__name__) +class NewInitCaller(type): + def __call__(cls, *args, **kwargs): + obj = type.__call__(cls, *args, **kwargs) + obj.after_init() + return obj + + +@six.add_metaclass(NewInitCaller) class Settings(object): - def __init__(self, args={}): - self.args = self._get_args(args) + def __init__(self, args={}, config_file=default_config_file): + self.args = args + self.config_file = config_file self.config = self._get_config() - def _get_args(self, args): - # Override correct log level from argparse - levels = [logging.WARNING, logging.INFO, logging.DEBUG] - if args.log_level: - args.log_level = levels[min(len(levels) - 1, args.log_level - 1)] + def set_args(self, args={}): + self.config_file = args.get("config_file") or default_config_file - args_dict = dict(filter(lambda item: item[1] is not None, args.__dict__.items())) - return args_dict + args.pop("config_file", None) + args = dict(filter(lambda item: item[1] is not None, args.items())) + + args_dict = {} + for key, value in args.items(): + args_dict = utils.add_dict_branch(args_dict, key.split("."), value) + + self.args = args_dict + self.config = self._get_config() + self._validate() def _get_config(self): defaults = self._get_defaults() - config_file = self.args.get('config_file') + config_file = self.config_file + cli_options = self.args if config_file and os.path.exists(config_file): with utils.open_file(config_file) as stream: s = stream.read() anyconfig.merge(defaults, utils.safe_load(s), ac_merge=anyconfig.MS_DICTS) - print(json.dumps(defaults, indent=4, sort_keys=True)) + if cli_options: + anyconfig.merge(defaults, cli_options, ac_merge=anyconfig.MS_DICTS) + return defaults def _get_defaults(self): @@ -44,13 +62,21 @@ class Settings(object): return { 'rules': { - 'standards': self.args.get('rules_dir', rules_dir), - 'standards_filter': [], + 'standards': rules_dir, + 'filter': [], }, 'logging': { - 'level': self.args.get('log_level', logging.WARN), + 'level': logging.WARN, }, 'ansible': { 'custom_modules': [], } } + + def after_init(self): + self.config = self._get_config() + self._validate() + + def _validate(self): + logger.setLevel(self.config["logging"]["level"]) + diff --git a/ansiblelater/utils/__init__.py b/ansiblelater/utils/__init__.py index aeed4a2..73909d6 100644 --- a/ansiblelater/utils/__init__.py +++ b/ansiblelater/utils/__init__.py @@ -17,41 +17,6 @@ try: except ImportError: import configparser - -def should_do_markup(): - py_colors = os.environ.get('PY_COLORS', None) - if py_colors is not None: - return to_bool(py_colors, strict=False) - - return sys.stdout.isatty() and os.environ.get('TERM') != 'dumb' - - -colorama.init(autoreset=True, strip=not should_do_markup()) - - -def abort(message, file=sys.stderr): - return color_text(colorama.Fore.RED, "FATAL: {}".format(message)) - sys.exit(1) - - -def error(message, file=sys.stderr): - return color_text(colorama.Fore.RED, "ERROR: {}".format(message)) - - -def warn(message, settings, file=sys.stdout): - if settings.log_level <= logging.WARNING: - return color_text(colorama.Fore.YELLOW, "WARN: {}".format(message)) - - -def info(message, settings, file=sys.stdout): - if settings.log_level <= logging.INFO: - return color_text(colorama.Fore.BLUE, "INFO: {}".format(message)) - - -def color_text(color, msg): - print('{}{}{}'.format(color, msg, colorama.Style.RESET_ALL)) - - def count_spaces(c_string): leading_spaces = 0 trailing_spaces = 0 @@ -137,3 +102,13 @@ def open_file(filename, mode='r'): """ with open(filename, mode) as stream: yield stream + + +def add_dict_branch(tree, vector, value): + key = vector[0] + tree[key] = value \ + if len(vector) == 1 \ + else add_dict_branch(tree[key] if key in tree else {}, + vector[1:], + value) + return tree diff --git a/setup.cfg b/setup.cfg index 418f2ee..f2702d5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,7 @@ max-line-length = 100 inline-quotes = double exclude = .git,.tox,__pycache__,build,dist,tests,*.pyc,*.egg-info,.cache,.eggs application-import-names = ansiblelater -format = ${cyan}%(path)s:%(row)d:%(col)d${reset}: ${red_bold}%(code)s${reset} %(text)s +#format = ${cyan}%(path)s:%(row)d:%(col)d${reset}: ${red_bold}%(code)s${reset} %(text)s [isort] default_section = THIRDPARTY