fix: skip not parsable lines in json output

.drone.jsonnet

@@ -1,4 +1,4 @@
-local PythonVersion(pyversion='3.5') = {
+local PythonVersion(pyversion='3.7') = {
   name: 'python' + std.strReplace(pyversion, '.', '') + '-pytest',
   image: 'python:' + pyversion,
   pull: 'always',
@@ -25,7 +25,7 @@ local PipelineLint = {
   steps: [
     {
       name: 'flake8',
-      image: 'python:3.7',
+      image: 'python:3.10',
       pull: 'always',
       environment: {
         PY_COLORS: 1,
@@ -50,10 +50,10 @@ local PipelineTest = {
     arch: 'amd64',
   },
   steps: [
-    PythonVersion(pyversion='3.5'),
-    PythonVersion(pyversion='3.6'),
     PythonVersion(pyversion='3.7'),
     PythonVersion(pyversion='3.8'),
+    PythonVersion(pyversion='3.9'),
+    PythonVersion(pyversion='3.10'),
   ],
   trigger: {
     ref: ['refs/heads/master', 'refs/tags/**', 'refs/pull/**'],
@@ -73,7 +73,7 @@ local PipelineSecurity = {
   steps: [
     {
       name: 'bandit',
-      image: 'python:3.7',
+      image: 'python:3.10',
       pull: 'always',
       environment: {
         PY_COLORS: 1,
@@ -103,7 +103,7 @@ local PipelineBuild = {
   steps: [
     {
       name: 'build',
-      image: 'python:3.7',
+      image: 'python:3.10',
       pull: 'always',
       commands: [
         'python setup.py sdist bdist_wheel',

.drone.yml

@@ -9,7 +9,7 @@ platform:
 steps:
   - name: flake8
     pull: always
-    image: python:3.7
+    image: python:3.10
     commands:
       - pip install -r test-requirements.txt -qq
       - pip install -qq .
@@ -32,30 +32,6 @@ platform:
   arch: amd64
 
 steps:
-  - name: python35-pytest
-    pull: always
-    image: python:3.5
-    commands:
-      - pip install -r test-requirements.txt -qq
-      - pip install -qq .
-      - drone-cleanup-agents --help
-    environment:
-      PY_COLORS: 1
-    depends_on:
-      - clone
-
-  - name: python36-pytest
-    pull: always
-    image: python:3.6
-    commands:
-      - pip install -r test-requirements.txt -qq
-      - pip install -qq .
-      - drone-cleanup-agents --help
-    environment:
-      PY_COLORS: 1
-    depends_on:
-      - clone
-
   - name: python37-pytest
     pull: always
     image: python:3.7
@@ -80,6 +56,30 @@ steps:
     depends_on:
       - clone
 
+  - name: python39-pytest
+    pull: always
+    image: python:3.9
+    commands:
+      - pip install -r test-requirements.txt -qq
+      - pip install -qq .
+      - drone-cleanup-agents --help
+    environment:
+      PY_COLORS: 1
+    depends_on:
+      - clone
+
+  - name: python310-pytest
+    pull: always
+    image: python:3.10
+    commands:
+      - pip install -r test-requirements.txt -qq
+      - pip install -qq .
+      - drone-cleanup-agents --help
+    environment:
+      PY_COLORS: 1
+    depends_on:
+      - clone
+
 trigger:
   ref:
     - refs/heads/master
@@ -100,7 +100,7 @@ platform:
 steps:
   - name: bandit
     pull: always
-    image: python:3.7
+    image: python:3.10
     commands:
       - pip install -r test-requirements.txt -qq
       - pip install -qq .
@@ -128,7 +128,7 @@ platform:
 steps:
   - name: build
     pull: always
-    image: python:3.7
+    image: python:3.10
     commands:
       - python setup.py sdist bdist_wheel
 

.flake8

@@ -1,8 +0,0 @@
-[flake8]
-# Temp disable Docstring checks D101, D102, D103, D107 
-ignore = E501, W503, F401, N813, D101, D102, D103, D107  
-max-line-length = 100
-inline-quotes = double
-exclude = .git,.tox,__pycache__,build,dist,tests,*.pyc,*.egg-info,.cache,.eggs,env*
-application-import-names = cleanupagents
-format = ${cyan}%(path)s:%(row)d:%(col)d${reset}: ${red_bold}%(code)s${reset} %(text)s

cleanupagents/Cli.py

@@ -3,20 +3,16 @@
 
 import argparse
 import json
-import logging
 import os
 import pickle  # nosec
 import shutil
-import sys
 from collections import defaultdict
-from urllib.parse import urlparse
 
 from cleanupagents import __version__
 from cleanupagents.Logging import SingleLog
 from cleanupagents.Utils import humanize
 from cleanupagents.Utils import normalize_path
 from cleanupagents.Utils import run_command
-from cleanupagents.Utils import to_bool
 
 
 class AgentCleanup:
@@ -102,7 +98,11 @@ class AgentCleanup:
                 self.log.sysexit_with_message("Command error:\n{}".format(humanize(res.stdout)))
 
             for line in res.stdout.splitlines():
-                obj = json.loads(line)
+                try:
+                    obj = json.loads(line)
+                except json.decoder.JSONDecodeError:
+                    self.logger.debug("Unable to parse line: {}".format(line))
+                    continue
 
                 if obj["state"] == "error":
                     error.append(obj["name"])

cleanupagents/Logging.py

@@ -9,7 +9,6 @@ import sys
 import colorama
 from pythonjsonlogger import jsonlogger
 
-import cleanupagents.Utils
 from cleanupagents.Utils import Singleton
 from cleanupagents.Utils import to_bool
 
@@ -65,11 +64,11 @@ class MultilineJsonFormatter(jsonlogger.JsonFormatter):
 
 
 class Log:
-    def __init__(self, level=logging.WARN, name="cleanupagents", logfile="/var/log/drone-agents.log", json=False):
+    def __init__(self, level=logging.WARNING, name="cleanupagents", logfile="/var/log/drone-agents.log", json=False):
         self.logger = logging.getLogger(name)
         self.logger.setLevel(level)
         self.logger.addHandler(self._get_error_handler(json=json))
-        self.logger.addHandler(self._get_warn_handler(json=json))
+        self.logger.addHandler(self._get_warning_handler(json=json))
         self.logger.addHandler(self._get_info_handler(json=json))
         self.logger.addHandler(self._get_critical_handler(json=json))
         self.logger.addHandler(self._get_debug_handler(json=json))
@@ -105,12 +104,12 @@ class Log:
 
         return handler
 
-    def _get_warn_handler(self, json=False):
+    def _get_warning_handler(self, json=False):
         handler = logging.StreamHandler(sys.stdout)
-        handler.setLevel(logging.WARN)
+        handler.setLevel(logging.WARNING)
-        handler.addFilter(LogFilter(logging.WARN))
+        handler.addFilter(LogFilter(logging.WARNING))
         handler.setFormatter(MultilineFormatter(
-            self.warn(CONSOLE_FORMAT.format(colorama.Fore.YELLOW, colorama.Style.RESET_ALL))))
+            self.warning(CONSOLE_FORMAT.format(colorama.Fore.YELLOW, colorama.Style.RESET_ALL))))
 
         if json:
             handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
@@ -168,8 +167,8 @@ class Log:
         """Format error messages and return string."""
         return msg
 
-    def warn(self, msg):
+    def warning(self, msg):
-        """Format warn messages and return string."""
+        """Format warning messages and return string."""
         return msg
 
     def info(self, msg):

cleanupagents/Utils.py

@@ -4,7 +4,6 @@
 import os
 import shlex
 import subprocess  # nosec
-import sys
 from distutils.util import strtobool
 
 

setup.cfg

@@ -18,3 +18,22 @@ filterwarnings =
     ignore::FutureWarning
     ignore:.*collections.*:DeprecationWarning
     ignore:.*pep8.*:FutureWarning
+
+[flake8]
+# Explanation of errors
+#
+# D100: Missing docstring in public module
+# D101: Missing docstring in public class
+# D102: Missing docstring in public method
+# D103: Missing docstring in public function
+# D105: Missing docstring in magic method
+# D107: Missing docstring in __init__
+# D202: No blank lines allowed after function docstring
+# G001: Logging statements should not use string.format() for their first argument
+# G004: Logging statements should not use f"..." for their first argument
+# W503: Line break occurred before a binary operator
+ignore = D100, D101, D102, D103, D105, D107, D202, G001, G004, W503
+max-line-length = 200
+inline-quotes = double
+exclude = .git,.tox,__pycache__,build,dist,tests,*.pyc,*.egg-info,.cache,.eggs,env*
+application-import-names = cleanupagents