github
/
ansible-later
mirror of https://github.com/thegeeklab/ansible-later.git
0
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: reduce duplicate error logs while processing files with parsing issues (#61)

This commit is contained in:
Robert Kaussow 2021-01-09 13:16:08 +01:00 committed by GitHub
parent 204fd46304
commit 5fc13d790e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 134 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ansiblelater/command/candidates.py
View File

@ -37,6 +37,7 @@ class Candidate(object):
self.filetype = type(self).__name__.lower()
self.expected_version = True
self.standards = self._get_standards(settings, standards)
self.faulty = False
try:
with codecs.open(filename, mode="rb", encoding="utf-8") as f:

26
ansiblelater/rules/yamlfiles.py
View File

@ -1,15 +1,13 @@
"""Checks related to generic YAML syntax (yamllint)."""
import codecs
import os
import yaml
from ansiblelater.command.candidates import Error
from ansiblelater.command.candidates import Result
from ansiblelater.utils.rulehelper import get_action_tasks
from ansiblelater.utils.rulehelper import get_normalized_task
from ansiblelater.utils.rulehelper import get_normalized_yaml
from ansiblelater.utils.rulehelper import get_raw_yaml
from ansiblelater.utils.rulehelper import run_yamllint
@ -17,7 +15,7 @@ def check_yaml_has_content(candidate, settings):
lines, errors = get_normalized_yaml(candidate, settings)
description = "the file appears to have no useful content"
if not lines and not errors:
if (lines and len(lines) == 0) and not errors:
errors.append(Error(None, description))
return Result(candidate.path, errors)
@ -53,19 +51,19 @@ def check_native_yaml(candidate, settings):
def check_yaml_empty_lines(candidate, settings):
options = "rules: {{empty-lines: {conf}}}".format(conf=settings["yamllint"]["empty-lines"])
errors = run_yamllint(candidate.path, options)
errors = run_yamllint(candidate, options)
return Result(candidate.path, errors)
def check_yaml_indent(candidate, settings):
options = "rules: {{indentation: {conf}}}".format(conf=settings["yamllint"]["indentation"])
errors = run_yamllint(candidate.path, options)
errors = run_yamllint(candidate, options)
return Result(candidate.path, errors)
def check_yaml_hyphens(candidate, settings):
options = "rules: {{hyphens: {conf}}}".format(conf=settings["yamllint"]["hyphens"])
errors = run_yamllint(candidate.path, options)
errors = run_yamllint(candidate, options)
return Result(candidate.path, errors)
@ -73,19 +71,19 @@ def check_yaml_document_start(candidate, settings):
options = "rules: {{document-start: {conf}}}".format(
conf=settings["yamllint"]["document-start"]
)
errors = run_yamllint(candidate.path, options)
errors = run_yamllint(candidate, options)
return Result(candidate.path, errors)
def check_yaml_document_end(candidate, settings):
options = "rules: {{document-end: {conf}}}".format(conf=settings["yamllint"]["document-end"])
errors = run_yamllint(candidate.path, options)
errors = run_yamllint(candidate, options)
return Result(candidate.path, errors)
def check_yaml_colons(candidate, settings):
options = "rules: {{colons: {conf}}}".format(conf=settings["yamllint"]["colons"])
errors = run_yamllint(candidate.path, options)
errors = run_yamllint(candidate, options)
return Result(candidate.path, errors)
@ -96,12 +94,6 @@ def check_yaml_file(candidate, settings):
if os.path.isfile(filename) and os.path.splitext(filename)[1][1:] != "yml":
errors.append(Error(None, "file does not have a .yml extension"))
elif os.path.isfile(filename) and os.path.splitext(filename)[1][1:] == "yml":
with codecs.open(filename, mode="rb", encoding="utf-8") as f:
try:
yaml.safe_load(f)
except Exception as e:
errors.append(
Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
)
content, errors = get_raw_yaml(candidate, settings)
return Result(candidate.path, errors)

8
ansiblelater/settings.py
View File

@ -3,6 +3,7 @@
import os
import anyconfig
import jsonschema.exceptions
import pathspec
from appdirs import AppDirs
from jsonschema._utils import format_as_index
@ -159,8 +160,11 @@ class Settings(object):
try:
anyconfig.validate(config, self.schema, ac_schema_safe=False)
return True
except Exception as e:
schema_error = "Failed validating '{validator}' in schema{schema}".format(
except jsonschema.exceptions.ValidationError as e:
schema_error = (
"Error while loading configuration:\n"
"Failed validating '{validator}' in schema{schema}"
).format(
validator=e.validator, schema=format_as_index(list(e.relative_schema_path)[:-1])
)
utils.sysexit_with_message(

192
ansiblelater/utils/rulehelper.py
View File

@ -19,15 +19,21 @@ from .yamlhelper import parse_yaml_linenumbers
def get_tasks(candidate, settings):
errors = []
try:
with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
yamllines = parse_yaml_linenumbers(f, candidate.path)
yamllines = []
except LaterError as ex:
e = ex.original
errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
except LaterAnsibleError as e:
errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
if not candidate.faulty:
try:
with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
yamllines = parse_yaml_linenumbers(f, candidate.path)
except LaterError as ex:
e = ex.original
errors.append(
Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
)
candidate.faulty = True
except LaterAnsibleError as e:
errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
candidate.faulty = True
return yamllines, errors
@ -35,17 +41,23 @@ def get_tasks(candidate, settings):
def get_action_tasks(candidate, settings):
tasks = []
errors = []
try:
with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
yamllines = parse_yaml_linenumbers(f, candidate.path)
if yamllines:
tasks = action_tasks(yamllines, candidate)
except LaterError as ex:
e = ex.original
errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
except LaterAnsibleError as e:
errors.append(Error(e.line, "syntax error: {}".format(e.message)))
if not candidate.faulty:
try:
with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
yamllines = parse_yaml_linenumbers(f, candidate.path)
if yamllines:
tasks = action_tasks(yamllines, candidate)
except LaterError as ex:
e = ex.original
errors.append(
Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
)
candidate.faulty = True
except LaterAnsibleError as e:
errors.append(Error(e.line, "syntax error: {}".format(e.message)))
candidate.faulty = True
return tasks, errors
@ -53,13 +65,21 @@ def get_action_tasks(candidate, settings):
def get_normalized_task(task, candidate, settings):
normalized = None
errors = []
try:
normalized = normalize_task(task, candidate.path, settings["ansible"]["custom_modules"])
except LaterError as ex:
e = ex.original
errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
except LaterAnsibleError as e:
errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
if not candidate.faulty:
try:
normalized = normalize_task(
task, candidate.path, settings["ansible"]["custom_modules"]
)
except LaterError as ex:
e = ex.original
errors.append(
Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
)
candidate.faulty = True
except LaterAnsibleError as e:
errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
candidate.faulty = True
return normalized, errors
@ -67,54 +87,68 @@ def get_normalized_task(task, candidate, settings):
def get_normalized_tasks(candidate, settings, full=False):
normalized = []
errors = []
try:
with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
yamllines = parse_yaml_linenumbers(f, candidate.path)
if yamllines:
tasks = action_tasks(yamllines, candidate)
for task in tasks:
# An empty `tags` block causes `None` to be returned if
# the `or []` is not present - `task.get("tags", [])`
# does not suffice.
if candidate.faulty:
try:
with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
yamllines = parse_yaml_linenumbers(f, candidate.path)
# Deprecated.
if "skip_ansible_lint" in (task.get("tags") or []) and not full:
# No need to normalize_task if we are skipping it.
continue
if yamllines:
tasks = action_tasks(yamllines, candidate)
for task in tasks:
# An empty `tags` block causes `None` to be returned if
# the `or []` is not present - `task.get("tags", [])`
# does not suffice.
if "skip_ansible_later" in (task.get("tags") or []) and not full:
# No need to normalize_task if we are skipping it.
continue
# Deprecated.
if "skip_ansible_lint" in (task.get("tags") or []) and not full:
# No need to normalize_task if we are skipping it.
continue
normalized.append(
normalize_task(task, candidate.path, settings["ansible"]["custom_modules"])
)
if "skip_ansible_later" in (task.get("tags") or []) and not full:
# No need to normalize_task if we are skipping it.
continue
except LaterError as ex:
e = ex.original
errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
except LaterAnsibleError as e:
errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
normalized.append(
normalize_task(
task, candidate.path, settings["ansible"]["custom_modules"]
)
)
except LaterError as ex:
e = ex.original
errors.append(
Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
)
candidate.faulty = True
except LaterAnsibleError as e:
errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
candidate.faulty = True
return normalized, errors
def get_normalized_yaml(candidate, settings, options=None):
errors = []
yamllines = None
if not options:
options = defaultdict(dict)
options.update(remove_empty=True)
options.update(remove_markers=True)
if not candidate.faulty:
if not options:
options = defaultdict(dict)
options.update(remove_empty=True)
options.update(remove_markers=True)
try:
yamllines = normalized_yaml(candidate.path, options)
except LaterError as ex:
e = ex.original
errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
except LaterAnsibleError as e:
errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
try:
yamllines = normalized_yaml(candidate.path, options)
except LaterError as ex:
e = ex.original
errors.append(
Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
)
candidate.faulty = True
except LaterAnsibleError as e:
errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
candidate.faulty = True
return yamllines, errors
@ -123,25 +157,33 @@ def get_raw_yaml(candidate, settings):
content = None
errors = []
try:
with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
content = yaml.safe_load(f)
except LaterError as ex:
e = ex.original
errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
if not candidate.faulty:
try:
with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
content = yaml.safe_load(f)
except yaml.YAMLError as e:
errors.append(
Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
)
candidate.faulty = True
return content, errors
def run_yamllint(path, options="extends: default"):
def run_yamllint(candidate, options="extends: default"):
errors = []
try:
with codecs.open(path, mode="rb", encoding="utf-8") as f:
for problem in linter.run(f, YamlLintConfig(options)):
errors.append(Error(problem.line, problem.desc))
except LaterError as ex:
e = ex.original
errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
if not candidate.faulty:
try:
with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
yaml.safe_load(f)
for problem in linter.run(f, YamlLintConfig(options)):
errors.append(Error(problem.line, problem.desc))
except yaml.YAMLError as e:
errors.append(
Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
)
candidate.faulty = True
return errors

2
ansiblelater/utils/yamlhelper.py
View File

@ -544,7 +544,7 @@ def parse_yaml_linenumbers(data, filename):
loader = AnsibleLoader(data, **kwargs)
loader.compose_node = compose_node
loader.construct_mapping = construct_mapping
data = loader.get_single_data()
data = loader.get_single_data() or []
except (yaml.parser.ParserError, yaml.scanner.ScannerError) as e:
raise LaterError("syntax error", e)
except (yaml.composer.ComposerError) as e: