diff --git a/CHANGELOG.md b/CHANGELOG.md index 27dfbee..e397efc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,2 +1,2 @@ - BUGFIXES - - Remove multiprocessing due to bad implementation and resulting performance issues + - Fix multiprocessing handler causing performance issues diff --git a/ansiblelater/__main__.py b/ansiblelater/__main__.py index 2dc7eea..d8ee432 100755 --- a/ansiblelater/__main__.py +++ b/ansiblelater/__main__.py @@ -2,6 +2,7 @@ """Main program.""" import argparse +import multiprocessing import sys from ansiblelater import LOG @@ -38,7 +39,9 @@ def main(): files = config["rules"]["files"] standards = base.get_standards(config["rules"]["standards"]) - errors = 0 + workers = max(multiprocessing.cpu_count() - 2, 2) + p = multiprocessing.Pool(workers) + tasks = [] for filename in files: lines = None candidate = candidates.classify(filename, settings, standards) @@ -53,10 +56,15 @@ def main(): LOG.info("Reviewing %s lines %s" % (candidate, lines)) else: LOG.info("Reviewing all of %s" % candidate) - errors = errors + candidate.review(settings, lines) + tasks.append((candidate, settings, lines)) + # errors = errors + res else: LOG.info("Couldn't classify file %s" % filename) + errors = (sum(p.map(_review_wrapper, tasks))) + p.close() + p.join() + if not errors == 0: return_code = 1 else: @@ -65,5 +73,10 @@ def main(): sys.exit(return_code) +def _review_wrapper(args): + (candidate, settings, lines) = args + return candidate.review(settings, lines) + + if __name__ == "__main__": main() diff --git a/ansiblelater/command/candidates.py b/ansiblelater/command/candidates.py index b3681da..c68a4d0 100644 --- a/ansiblelater/command/candidates.py +++ b/ansiblelater/command/candidates.py @@ -102,7 +102,7 @@ class Candidate(object): return target_standards - def review(self, settings, lines=0): + def review(self, settings, lines=None): errors = 0 for standard in self.standards: