enable multiprocessing for candidate reviews

ansiblelater/__main__.py

@@ -3,6 +3,7 @@
 
 import argparse
 import json
+import multiprocessing
 
 from ansiblelater import LOG, __version__, logger
 from ansiblelater.command import base, candidates
@@ -29,14 +30,14 @@ def main():
 
     settings = base.get_settings(args)
     config = settings.config
-    # print(json.dumps(settings.config["logging"], indent=4, sort_keys=True))
 
     logger.update_logger(LOG, config["logging"]["level"], config["logging"]["json"])
 
     files = config["rules"]["files"]
     standards = base.get_standards(config["rules"]["standards"])
 
-    errors = 0
+    workers = multiprocessing.cpu_count() - 2
+    p = multiprocessing.Pool(workers)
     for filename in files:
         lines = None
         candidate = candidates.classify(filename, settings, standards)
@@ -51,10 +52,12 @@ def main():
                 LOG.info("Reviewing %s lines %s" % (candidate, lines))
             else:
                 LOG.info("Reviewing all of %s" % candidate)
-            errors = errors + candidate.review(settings, lines)
+            p.imap(candidate.review, (settings, lines,))
         else:
             LOG.info("Couldn't classify file %s" % filename)
-    return errors
+
+    p.close()
+    p.join()
 
 
 if __name__ == "__main__":

ansiblelater/logger.py

@@ -57,7 +57,7 @@ class LogFilter(object):
 class MultilineFormatter(logging.Formatter):
     """Logging Formatter to reset color after newline characters."""
 
-    def format(self, record):
+    def format(self, record): # noqa
         record.msg = record.msg.replace("\n", "\n{}... ".format(colorama.Style.RESET_ALL))
         return logging.Formatter.format(self, record)
 
@@ -65,7 +65,7 @@ class MultilineFormatter(logging.Formatter):
 class MultilineJsonFormatter(jsonlogger.JsonFormatter):
     """Logging Formatter to remove newline characters."""
 
-    def format(self, record):
+    def format(self, record): # noqa
         record.msg = record.msg.replace("\n", " ")
         return jsonlogger.JsonFormatter.format(self, record)
 
@@ -180,5 +180,4 @@ def color_text(color, msg):
     :returns: string
 
     """
-
     return "{}{}{}".format(color, msg, colorama.Style.RESET_ALL)

ansiblelater/settings.py

@@ -1,6 +1,5 @@
 """Global settings object definition."""
 
-import copy
 import logging
 import os
 
@@ -36,6 +35,7 @@ class Settings(object):
 
         """
         self.config_file = config_file
+        self.args_files = False
         self.args = self._set_args(args)
         self.config = self._get_config()
         self.schema = None
@@ -60,7 +60,11 @@ class Settings(object):
                 log_level = min(len(levels) - 1, max(log_level + adjustment, 0))
             tmp_dict["logging"]["level"] = logging.getLevelName(levels[log_level])
 
-        tmp_dict["rules"]["files"] = self._get_files(tmp_dict)
+        if len(tmp_dict["rules"]["files"]) == 0:
+            tmp_dict["rules"]["files"] = "*"
+        else:
+            tmp_dict["rules"]["files"] = tmp_dict["rules"]["files"]
+            self.args_files = True
 
         return tmp_dict
 
@@ -101,17 +105,6 @@ class Settings(object):
 
         return defaults
 
-    def _get_files(self, args):
-        if len(args["rules"]["files"]) == 0:
-            filelist = []
-            for root, dirs, files in os.walk("."):
-                for filename in files:
-                    filelist.append(os.path.relpath(os.path.normpath(os.path.join(root, filename))))
-        else:
-            filelist = args["rules"]["files"]
-
-        return filelist
-
     def _validate(self, config):
         try:
             anyconfig.validate(config, self.schema, ac_schema_safe=False)
@@ -124,14 +117,24 @@ class Settings(object):
             utils.sysexit_with_message("{schema}: {msg}".format(schema=schema_error, msg=e.message))
 
     def _update_filelist(self):
-        files = self.config["rules"]["files"]
+        include = self.config["rules"]["files"]
         excludes = self.config["rules"]["exclude_files"]
         ignore_dotfiles = self.config["rules"]["ignore_dotfiles"]
 
-        if ignore_dotfiles:
+        if ignore_dotfiles and not self.args_files:
             excludes.append(".")
+        else:
+            del excludes[:]
 
-        valid = copy.copy(files)
+        filelist = []
-        for item in valid:
+        for root, dirs, files in os.walk("."):
-            if glob_match(item, excludes):
+            for filename in files:
-                files.remove(item)
+                filelist.append(
+                    os.path.relpath(os.path.normpath(os.path.join(root, filename))))
+
+        valid = []
+        for item in filelist:
+            if glob_match(item, include) and not glob_match(item, excludes):
+                valid.append(item)
+
+        self.config["rules"]["files"] = valid