feat: reduce duplicate error logs while processing files with parsing issues (#61)

ansiblelater/command/candidates.py

@@ -37,6 +37,7 @@ class Candidate(object):
         self.filetype = type(self).__name__.lower()
         self.expected_version = True
         self.standards = self._get_standards(settings, standards)
+        self.faulty = False
 
         try:
             with codecs.open(filename, mode="rb", encoding="utf-8") as f:

ansiblelater/rules/yamlfiles.py

@@ -1,15 +1,13 @@
 """Checks related to generic YAML syntax (yamllint)."""
 
-import codecs
 import os
 
-import yaml
-
 from ansiblelater.command.candidates import Error
 from ansiblelater.command.candidates import Result
 from ansiblelater.utils.rulehelper import get_action_tasks
 from ansiblelater.utils.rulehelper import get_normalized_task
 from ansiblelater.utils.rulehelper import get_normalized_yaml
+from ansiblelater.utils.rulehelper import get_raw_yaml
 from ansiblelater.utils.rulehelper import run_yamllint
 
 
@@ -17,7 +15,7 @@ def check_yaml_has_content(candidate, settings):
     lines, errors = get_normalized_yaml(candidate, settings)
     description = "the file appears to have no useful content"
 
-    if not lines and not errors:
+    if (lines and len(lines) == 0) and not errors:
         errors.append(Error(None, description))
 
     return Result(candidate.path, errors)
@@ -53,19 +51,19 @@ def check_native_yaml(candidate, settings):
 
 def check_yaml_empty_lines(candidate, settings):
     options = "rules: {{empty-lines: {conf}}}".format(conf=settings["yamllint"]["empty-lines"])
-    errors = run_yamllint(candidate.path, options)
+    errors = run_yamllint(candidate, options)
     return Result(candidate.path, errors)
 
 
 def check_yaml_indent(candidate, settings):
     options = "rules: {{indentation: {conf}}}".format(conf=settings["yamllint"]["indentation"])
-    errors = run_yamllint(candidate.path, options)
+    errors = run_yamllint(candidate, options)
     return Result(candidate.path, errors)
 
 
 def check_yaml_hyphens(candidate, settings):
     options = "rules: {{hyphens: {conf}}}".format(conf=settings["yamllint"]["hyphens"])
-    errors = run_yamllint(candidate.path, options)
+    errors = run_yamllint(candidate, options)
     return Result(candidate.path, errors)
 
 
@@ -73,19 +71,19 @@ def check_yaml_document_start(candidate, settings):
     options = "rules: {{document-start: {conf}}}".format(
         conf=settings["yamllint"]["document-start"]
     )
-    errors = run_yamllint(candidate.path, options)
+    errors = run_yamllint(candidate, options)
     return Result(candidate.path, errors)
 
 
 def check_yaml_document_end(candidate, settings):
     options = "rules: {{document-end: {conf}}}".format(conf=settings["yamllint"]["document-end"])
-    errors = run_yamllint(candidate.path, options)
+    errors = run_yamllint(candidate, options)
     return Result(candidate.path, errors)
 
 
 def check_yaml_colons(candidate, settings):
     options = "rules: {{colons: {conf}}}".format(conf=settings["yamllint"]["colons"])
-    errors = run_yamllint(candidate.path, options)
+    errors = run_yamllint(candidate, options)
     return Result(candidate.path, errors)
 
 
@@ -96,12 +94,6 @@ def check_yaml_file(candidate, settings):
     if os.path.isfile(filename) and os.path.splitext(filename)[1][1:] != "yml":
         errors.append(Error(None, "file does not have a .yml extension"))
     elif os.path.isfile(filename) and os.path.splitext(filename)[1][1:] == "yml":
-        with codecs.open(filename, mode="rb", encoding="utf-8") as f:
+        content, errors = get_raw_yaml(candidate, settings)
-            try:
-                yaml.safe_load(f)
-            except Exception as e:
-                errors.append(
-                    Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
-                )
 
     return Result(candidate.path, errors)

ansiblelater/settings.py

@@ -3,6 +3,7 @@
 import os
 
 import anyconfig
+import jsonschema.exceptions
 import pathspec
 from appdirs import AppDirs
 from jsonschema._utils import format_as_index
@@ -159,8 +160,11 @@ class Settings(object):
         try:
             anyconfig.validate(config, self.schema, ac_schema_safe=False)
             return True
-        except Exception as e:
+        except jsonschema.exceptions.ValidationError as e:
-            schema_error = "Failed validating '{validator}' in schema{schema}".format(
+            schema_error = (
+                "Error while loading configuration:\n"
+                "Failed validating '{validator}' in schema{schema}"
+            ).format(
                 validator=e.validator, schema=format_as_index(list(e.relative_schema_path)[:-1])
             )
             utils.sysexit_with_message(

ansiblelater/utils/rulehelper.py

@@ -19,15 +19,21 @@ from .yamlhelper import parse_yaml_linenumbers
 
 def get_tasks(candidate, settings):
     errors = []
+    yamllines = []
+
+    if not candidate.faulty:
         try:
             with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
                 yamllines = parse_yaml_linenumbers(f, candidate.path)
-
         except LaterError as ex:
             e = ex.original
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+            errors.append(
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
+            )
+            candidate.faulty = True
         except LaterAnsibleError as e:
             errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
+            candidate.faulty = True
 
     return yamllines, errors
 
@@ -35,6 +41,8 @@ def get_tasks(candidate, settings):
 def get_action_tasks(candidate, settings):
     tasks = []
     errors = []
+
+    if not candidate.faulty:
         try:
             with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
                 yamllines = parse_yaml_linenumbers(f, candidate.path)
@@ -43,9 +51,13 @@ def get_action_tasks(candidate, settings):
                 tasks = action_tasks(yamllines, candidate)
         except LaterError as ex:
             e = ex.original
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+            errors.append(
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
+            )
+            candidate.faulty = True
         except LaterAnsibleError as e:
             errors.append(Error(e.line, "syntax error: {}".format(e.message)))
+            candidate.faulty = True
 
     return tasks, errors
 
@@ -53,13 +65,21 @@ def get_action_tasks(candidate, settings):
 def get_normalized_task(task, candidate, settings):
     normalized = None
     errors = []
+
+    if not candidate.faulty:
         try:
-        normalized = normalize_task(task, candidate.path, settings["ansible"]["custom_modules"])
+            normalized = normalize_task(
+                task, candidate.path, settings["ansible"]["custom_modules"]
+            )
         except LaterError as ex:
             e = ex.original
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+            errors.append(
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
+            )
+            candidate.faulty = True
         except LaterAnsibleError as e:
             errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
+            candidate.faulty = True
 
     return normalized, errors
 
@@ -67,6 +87,8 @@ def get_normalized_task(task, candidate, settings):
 def get_normalized_tasks(candidate, settings, full=False):
     normalized = []
     errors = []
+
+    if candidate.faulty:
         try:
             with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
                 yamllines = parse_yaml_linenumbers(f, candidate.path)
@@ -88,21 +110,29 @@ def get_normalized_tasks(candidate, settings, full=False):
                         continue
 
                     normalized.append(
-                    normalize_task(task, candidate.path, settings["ansible"]["custom_modules"])
+                        normalize_task(
+                            task, candidate.path, settings["ansible"]["custom_modules"]
+                        )
                     )
 
         except LaterError as ex:
             e = ex.original
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+            errors.append(
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
+            )
+            candidate.faulty = True
         except LaterAnsibleError as e:
             errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
+            candidate.faulty = True
 
     return normalized, errors
 
 
 def get_normalized_yaml(candidate, settings, options=None):
     errors = []
+    yamllines = None
 
+    if not candidate.faulty:
         if not options:
             options = defaultdict(dict)
             options.update(remove_empty=True)
@@ -112,9 +142,13 @@ def get_normalized_yaml(candidate, settings, options=None):
             yamllines = normalized_yaml(candidate.path, options)
         except LaterError as ex:
             e = ex.original
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+            errors.append(
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
+            )
+            candidate.faulty = True
         except LaterAnsibleError as e:
             errors.append(Error(e.line, "syntax error: {msg}".format(msg=e.message)))
+            candidate.faulty = True
 
     return yamllines, errors
 
@@ -123,25 +157,33 @@ def get_raw_yaml(candidate, settings):
     content = None
     errors = []
 
+    if not candidate.faulty:
         try:
             with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
                 content = yaml.safe_load(f)
-
+        except yaml.YAMLError as e:
-    except LaterError as ex:
+            errors.append(
-        e = ex.original
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+            )
+            candidate.faulty = True
 
     return content, errors
 
 
-def run_yamllint(path, options="extends: default"):
+def run_yamllint(candidate, options="extends: default"):
     errors = []
+
+    if not candidate.faulty:
         try:
-        with codecs.open(path, mode="rb", encoding="utf-8") as f:
+            with codecs.open(candidate.path, mode="rb", encoding="utf-8") as f:
+                yaml.safe_load(f)
+
                 for problem in linter.run(f, YamlLintConfig(options)):
                     errors.append(Error(problem.line, problem.desc))
-    except LaterError as ex:
+        except yaml.YAMLError as e:
-        e = ex.original
+            errors.append(
-        errors.append(Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem)))
+                Error(e.problem_mark.line + 1, "syntax error: {msg}".format(msg=e.problem))
+            )
+            candidate.faulty = True
 
     return errors

ansiblelater/utils/yamlhelper.py

@@ -544,7 +544,7 @@ def parse_yaml_linenumbers(data, filename):
         loader = AnsibleLoader(data, **kwargs)
         loader.compose_node = compose_node
         loader.construct_mapping = construct_mapping
-        data = loader.get_single_data()
+        data = loader.get_single_data() or []
     except (yaml.parser.ParserError, yaml.scanner.ScannerError) as e:
         raise LaterError("syntax error", e)
     except (yaml.composer.ComposerError) as e: