feat: reduce duplicate error logs while processing files with parsing issues (#61)

ansiblelater/command/candidates.py

@@ -37,6 +37,7 @@ class Candidate(object):
         self.filetype = type(self).__name__.lower()
         self.expected_version = True
         self.standards = self._get_standards(settings, standards)
+        self.faulty = False
 
         try:
             with codecs.open(filename, mode="rb", encoding="utf-8") as f:

ansiblelater/rules/yamlfiles.py

@@ -1,15 +1,13 @@
 """Checks related to generic YAML syntax (yamllint)."""
 
-import codecs
 import os
 
-import yaml
-
 from ansiblelater.command.candidates import Error
 from ansiblelater.command.candidates import Result
 from ansiblelater.utils.rulehelper import get_action_tasks
 from ansiblelater.utils.rulehelper import get_normalized_task
 from ansiblelater.utils.rulehelper import get_normalized_yaml
+from ansiblelater.utils.rulehelper import get_raw_yaml
 from ansiblelater.utils.rulehelper import run_yamllint
 
 
@@ -17,7 +15,7 @@ def check_yaml_has_content(candidate, settings):
     lines, errors = get_normalized_yaml(candidate, settings)
     description = "the file appears to have no useful content"
 
-    if not lines and not errors:
+    if (lines and len(lines) == 0) and not errors:
         errors.append(Error(None, description))
 
     return Result(candidate.path, errors)
@@ -53,19 +51,19 @@ def check_native_yaml(candidate, settings):
 
 def check_yaml_empty_lines(candidate, settings):
     options = "rules: {{empty-lines: {conf}}}".format(conf=settings["yamllint"]["empty-lines"])
-    errors = run_yamllint(candidate.path, options)
+    errors = run_yamllint(candidate, options)
     return Result(candidate.path, errors)
 
 
 def check_yaml_indent(candidate, settings):
     options = "rules: {{indentation: {conf}}}".format(conf=settings["yamllint"]["indentation"])
-    errors = run_yamllint(candidate.path, options)
+    errors = run_yamllint(candidate, options)
     return Result(candidate.path, errors)
 
 
 def check_yaml_hyphens(candidate, settings):
     options = "rules: {{hyphens: {conf}}}".format(conf=settings["yamllint"]["hyphens"])
-    errors = run_yamllint(candidate.path, options)
+    errors = run_yamllint(candidate, options)
     return Result(candidate.path, errors)
 
 
@@ -73,19 +71,19 @@ def check_yaml_document_start(candidate, settings):
     options = "rules: {{document-start: {conf}}}".format(
         conf=settings["yamllint"]["document-start"]
     )
-    errors = run_yamllint(candidate.path, options)
+    errors = run_yamllint(candidate, options)
     return Result(candidate.path, errors)
 
 
 def check_yaml_document_end(candidate, settings):
     options = "rules: {{document-end: {conf}}}".format(conf=settings["yamllint"]["document-end"])
-    errors = run_yamllint(candidate.path, options)
+    errors = run_yamllint(candidate, options)
     return Result(candidate.path, errors)
 
 
 def check_yaml_colons(candidate, settings):
     options = "rules: {{colons: {conf}}}".format(conf=settings["yamllint"]["colons"])
-    errors = run_yamllint(candidate.path, options)
+    errors = run_yamllint(candidate, options)
     return Result(candidate.path, errors)
 
 
@@ -96,12 +94,6 @@ def check_yaml_file(candidate, settings):
     if os.path.isfile(filename) and os.path.splitext(filename)[1][1:] != "yml":
         errors.append(Error(None, "file does not have a .yml extension"))
     elif os.path.isfile(filename) and os.path.splitext(filename)[1][1:] == "yml":
-        with codecs.open(filename, mode="rb", encoding="utf-8") as f:
+        content, errors = get_raw_yaml(candidate, settings)
-            try:
-                yaml.safe_load(f)
-            except Exception as e:
-                errors.append(
-                    Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
-                )
 
     return Result(candidate.path, errors)

ansiblelater/settings.py

@@ -3,6 +3,7 @@
 import os
 
 import anyconfig
+import jsonschema.exceptions
 import pathspec
 from appdirs import AppDirs
 from jsonschema._utils import format_as_index
@@ -159,8 +160,11 @@ class Settings(object):
         try:
             anyconfig.validate(config, self.schema, ac_schema_safe=False)
             return True
-        except Exception as e:
+        except jsonschema.exceptions.ValidationError as e:
-            schema_error = "Failed validating '{validator}' in schema{schema}".format(
+            schema_error = (
+                "Error while loading configuration:\n"
+                "Failed validating '{validator}' in schema{schema}"
+            ).format(
                 validator=e.validator, schema=format_as_index(list(e.relative_schema_path)[:-1])
             )
             utils.sysexit_with_message(

ansiblelater/utils/rulehelper.py

@@ -19,15 +19,21 @@ from .yamlhelper import parse_yaml_linenumbers
 
 def get_tasks(candidate, settings):
     errors = []
-    try:
+    yamllines = []
-        with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
-            yamllines = parse_yaml_linenumbers(f, candidate.path)
 
-    except LaterError as ex:
+    if not candidate.faulty:
-        e = ex.original
+        try:
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+            with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
-    except LaterAnsibleError as e:
+                yamllines = parse_yaml_linenumbers(f, candidate.path)
-        errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
+        except LaterError as ex:
+            e = ex.original
+            errors.append(
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
+            )
+            candidate.faulty = True
+        except LaterAnsibleError as e:
+            errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
+            candidate.faulty = True
 
     return yamllines, errors
 
@@ -35,17 +41,23 @@ def get_tasks(candidate, settings):
 def get_action_tasks(candidate, settings):
     tasks = []
     errors = []
-    try:
-        with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
-            yamllines = parse_yaml_linenumbers(f, candidate.path)
 
-        if yamllines:
+    if not candidate.faulty:
-            tasks = action_tasks(yamllines, candidate)
+        try:
-    except LaterError as ex:
+            with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
-        e = ex.original
+                yamllines = parse_yaml_linenumbers(f, candidate.path)
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+
-    except LaterAnsibleError as e:
+            if yamllines:
-        errors.append(Error(e.line, "syntax error: {}".format(e.message)))
+                tasks = action_tasks(yamllines, candidate)
+        except LaterError as ex:
+            e = ex.original
+            errors.append(
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
+            )
+            candidate.faulty = True
+        except LaterAnsibleError as e:
+            errors.append(Error(e.line, "syntax error: {}".format(e.message)))
+            candidate.faulty = True
 
     return tasks, errors
 
@@ -53,13 +65,21 @@ def get_action_tasks(candidate, settings):
 def get_normalized_task(task, candidate, settings):
     normalized = None
     errors = []
-    try:
+
-        normalized = normalize_task(task, candidate.path, settings["ansible"]["custom_modules"])
+    if not candidate.faulty:
-    except LaterError as ex:
+        try:
-        e = ex.original
+            normalized = normalize_task(
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+                task, candidate.path, settings["ansible"]["custom_modules"]
-    except LaterAnsibleError as e:
+            )
-        errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
+        except LaterError as ex:
+            e = ex.original
+            errors.append(
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
+            )
+            candidate.faulty = True
+        except LaterAnsibleError as e:
+            errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
+            candidate.faulty = True
 
     return normalized, errors
 
@@ -67,54 +87,68 @@ def get_normalized_task(task, candidate, settings):
 def get_normalized_tasks(candidate, settings, full=False):
     normalized = []
     errors = []
-    try:
-        with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
-            yamllines = parse_yaml_linenumbers(f, candidate.path)
 
-        if yamllines:
+    if candidate.faulty:
-            tasks = action_tasks(yamllines, candidate)
+        try:
-            for task in tasks:
+            with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
-                # An empty `tags` block causes `None` to be returned if
+                yamllines = parse_yaml_linenumbers(f, candidate.path)
-                # the `or []` is not present - `task.get("tags", [])`
-                # does not suffice.
 
-                # Deprecated.
+            if yamllines:
-                if "skip_ansible_lint" in (task.get("tags") or []) and not full:
+                tasks = action_tasks(yamllines, candidate)
-                    # No need to normalize_task if we are skipping it.
+                for task in tasks:
-                    continue
+                    # An empty `tags` block causes `None` to be returned if
+                    # the `or []` is not present - `task.get("tags", [])`
+                    # does not suffice.
 
-                if "skip_ansible_later" in (task.get("tags") or []) and not full:
+                    # Deprecated.
-                    # No need to normalize_task if we are skipping it.
+                    if "skip_ansible_lint" in (task.get("tags") or []) and not full:
-                    continue
+                        # No need to normalize_task if we are skipping it.
+                        continue
 
-                normalized.append(
+                    if "skip_ansible_later" in (task.get("tags") or []) and not full:
-                    normalize_task(task, candidate.path, settings["ansible"]["custom_modules"])
+                        # No need to normalize_task if we are skipping it.
-                )
+                        continue
 
-    except LaterError as ex:
+                    normalized.append(
-        e = ex.original
+                        normalize_task(
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+                            task, candidate.path, settings["ansible"]["custom_modules"]
-    except LaterAnsibleError as e:
+                        )
-        errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
+                    )
+
+        except LaterError as ex:
+            e = ex.original
+            errors.append(
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
+            )
+            candidate.faulty = True
+        except LaterAnsibleError as e:
+            errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
+            candidate.faulty = True
 
     return normalized, errors
 
 
 def get_normalized_yaml(candidate, settings, options=None):
     errors = []
+    yamllines = None
 
-    if not options:
+    if not candidate.faulty:
-        options = defaultdict(dict)
+        if not options:
-        options.update(remove_empty=True)
+            options = defaultdict(dict)
-        options.update(remove_markers=True)
+            options.update(remove_empty=True)
+            options.update(remove_markers=True)
 
-    try:
+        try:
-        yamllines = normalized_yaml(candidate.path, options)
+            yamllines = normalized_yaml(candidate.path, options)
-    except LaterError as ex:
+        except LaterError as ex:
-        e = ex.original
+            e = ex.original
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+            errors.append(
-    except LaterAnsibleError as e:
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
-        errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
+            )
+            candidate.faulty = True
+        except LaterAnsibleError as e:
+            errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
+            candidate.faulty = True
 
     return yamllines, errors
 
@@ -123,25 +157,33 @@ def get_raw_yaml(candidate, settings):
     content = None
     errors = []
 
-    try:
+    if not candidate.faulty:
-        with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
+        try:
-            content = yaml.safe_load(f)
+            with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
-
+                content = yaml.safe_load(f)
-    except LaterError as ex:
+        except yaml.YAMLError as e:
-        e = ex.original
+            errors.append(
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
+            )
+            candidate.faulty = True
 
     return content, errors
 
 
-def run_yamllint(path, options="extends: default"):
+def run_yamllint(candidate, options="extends: default"):
     errors = []
-    try:
+
-        with codecs.open(path, mode="rb", encoding="utf-8") as f:
+    if not candidate.faulty:
-            for problem in linter.run(f, YamlLintConfig(options)):
+        try:
-                errors.append(Error(problem.line, problem.desc))
+            with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
-    except LaterError as ex:
+                yaml.safe_load(f)
-        e = ex.original
+
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+                for problem in linter.run(f, YamlLintConfig(options)):
+                    errors.append(Error(problem.line, problem.desc))
+        except yaml.YAMLError as e:
+            errors.append(
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
+            )
+            candidate.faulty = True
 
     return errors

ansiblelater/utils/yamlhelper.py

@@ -544,7 +544,7 @@ def parse_yaml_linenumbers(data, filename):
         loader = AnsibleLoader(data, **kwargs)
         loader.compose_node = compose_node
         loader.construct_mapping = construct_mapping
-        data = loader.get_single_data()
+        data = loader.get_single_data() or []
     except (yaml.parser.ParserError, yaml.scanner.ScannerError) as e:
         raise LaterError("syntax error", e)
     except (yaml.composer.ComposerError) as e: