From d3e7a4ba3c9f4a8443ce9b29468393b5fdf8ab64 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Tue, 25 Oct 2022 09:41:17 +0200 Subject: [PATCH] fix: skip not parsable lines in json output --- .drone.jsonnet | 12 ++++----- .drone.yml | 54 ++++++++++++++++++++-------------------- .flake8 | 8 ------ cleanupagents/Cli.py | 10 ++++---- cleanupagents/Logging.py | 17 ++++++------- cleanupagents/Utils.py | 1 - setup.cfg | 19 ++++++++++++++ 7 files changed, 65 insertions(+), 56 deletions(-) delete mode 100644 .flake8 diff --git a/.drone.jsonnet b/.drone.jsonnet index 54c0a83..6de53ed 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -1,4 +1,4 @@ -local PythonVersion(pyversion='3.5') = { +local PythonVersion(pyversion='3.7') = { name: 'python' + std.strReplace(pyversion, '.', '') + '-pytest', image: 'python:' + pyversion, pull: 'always', @@ -25,7 +25,7 @@ local PipelineLint = { steps: [ { name: 'flake8', - image: 'python:3.7', + image: 'python:3.10', pull: 'always', environment: { PY_COLORS: 1, @@ -50,10 +50,10 @@ local PipelineTest = { arch: 'amd64', }, steps: [ - PythonVersion(pyversion='3.5'), - PythonVersion(pyversion='3.6'), PythonVersion(pyversion='3.7'), PythonVersion(pyversion='3.8'), + PythonVersion(pyversion='3.9'), + PythonVersion(pyversion='3.10'), ], trigger: { ref: ['refs/heads/master', 'refs/tags/**', 'refs/pull/**'], @@ -73,7 +73,7 @@ local PipelineSecurity = { steps: [ { name: 'bandit', - image: 'python:3.7', + image: 'python:3.10', pull: 'always', environment: { PY_COLORS: 1, @@ -103,7 +103,7 @@ local PipelineBuild = { steps: [ { name: 'build', - image: 'python:3.7', + image: 'python:3.10', pull: 'always', commands: [ 'python setup.py sdist bdist_wheel', diff --git a/.drone.yml b/.drone.yml index a4f94b2..f7b7215 100644 --- a/.drone.yml +++ b/.drone.yml @@ -9,7 +9,7 @@ platform: steps: - name: flake8 pull: always - image: python:3.7 + image: python:3.10 commands: - pip install -r test-requirements.txt -qq - pip install -qq . @@ -32,30 +32,6 @@ platform: arch: amd64 steps: - - name: python35-pytest - pull: always - image: python:3.5 - commands: - - pip install -r test-requirements.txt -qq - - pip install -qq . - - drone-cleanup-agents --help - environment: - PY_COLORS: 1 - depends_on: - - clone - - - name: python36-pytest - pull: always - image: python:3.6 - commands: - - pip install -r test-requirements.txt -qq - - pip install -qq . - - drone-cleanup-agents --help - environment: - PY_COLORS: 1 - depends_on: - - clone - - name: python37-pytest pull: always image: python:3.7 @@ -80,6 +56,30 @@ steps: depends_on: - clone + - name: python39-pytest + pull: always + image: python:3.9 + commands: + - pip install -r test-requirements.txt -qq + - pip install -qq . + - drone-cleanup-agents --help + environment: + PY_COLORS: 1 + depends_on: + - clone + + - name: python310-pytest + pull: always + image: python:3.10 + commands: + - pip install -r test-requirements.txt -qq + - pip install -qq . + - drone-cleanup-agents --help + environment: + PY_COLORS: 1 + depends_on: + - clone + trigger: ref: - refs/heads/master @@ -100,7 +100,7 @@ platform: steps: - name: bandit pull: always - image: python:3.7 + image: python:3.10 commands: - pip install -r test-requirements.txt -qq - pip install -qq . @@ -128,7 +128,7 @@ platform: steps: - name: build pull: always - image: python:3.7 + image: python:3.10 commands: - python setup.py sdist bdist_wheel diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 839a6c4..0000000 --- a/.flake8 +++ /dev/null @@ -1,8 +0,0 @@ -[flake8] -# Temp disable Docstring checks D101, D102, D103, D107 -ignore = E501, W503, F401, N813, D101, D102, D103, D107 -max-line-length = 100 -inline-quotes = double -exclude = .git,.tox,__pycache__,build,dist,tests,*.pyc,*.egg-info,.cache,.eggs,env* -application-import-names = cleanupagents -format = ${cyan}%(path)s:%(row)d:%(col)d${reset}: ${red_bold}%(code)s${reset} %(text)s diff --git a/cleanupagents/Cli.py b/cleanupagents/Cli.py index b5e96cf..a35c2b2 100644 --- a/cleanupagents/Cli.py +++ b/cleanupagents/Cli.py @@ -3,20 +3,16 @@ import argparse import json -import logging import os import pickle # nosec import shutil -import sys from collections import defaultdict -from urllib.parse import urlparse from cleanupagents import __version__ from cleanupagents.Logging import SingleLog from cleanupagents.Utils import humanize from cleanupagents.Utils import normalize_path from cleanupagents.Utils import run_command -from cleanupagents.Utils import to_bool class AgentCleanup: @@ -102,7 +98,11 @@ class AgentCleanup: self.log.sysexit_with_message("Command error:\n{}".format(humanize(res.stdout))) for line in res.stdout.splitlines(): - obj = json.loads(line) + try: + obj = json.loads(line) + except json.decoder.JSONDecodeError: + self.logger.debug("Unable to parse line: {}".format(line)) + continue if obj["state"] == "error": error.append(obj["name"]) diff --git a/cleanupagents/Logging.py b/cleanupagents/Logging.py index cdc68d8..b80d72d 100644 --- a/cleanupagents/Logging.py +++ b/cleanupagents/Logging.py @@ -9,7 +9,6 @@ import sys import colorama from pythonjsonlogger import jsonlogger -import cleanupagents.Utils from cleanupagents.Utils import Singleton from cleanupagents.Utils import to_bool @@ -65,11 +64,11 @@ class MultilineJsonFormatter(jsonlogger.JsonFormatter): class Log: - def __init__(self, level=logging.WARN, name="cleanupagents", logfile="/var/log/drone-agents.log", json=False): + def __init__(self, level=logging.WARNING, name="cleanupagents", logfile="/var/log/drone-agents.log", json=False): self.logger = logging.getLogger(name) self.logger.setLevel(level) self.logger.addHandler(self._get_error_handler(json=json)) - self.logger.addHandler(self._get_warn_handler(json=json)) + self.logger.addHandler(self._get_warning_handler(json=json)) self.logger.addHandler(self._get_info_handler(json=json)) self.logger.addHandler(self._get_critical_handler(json=json)) self.logger.addHandler(self._get_debug_handler(json=json)) @@ -105,12 +104,12 @@ class Log: return handler - def _get_warn_handler(self, json=False): + def _get_warning_handler(self, json=False): handler = logging.StreamHandler(sys.stdout) - handler.setLevel(logging.WARN) - handler.addFilter(LogFilter(logging.WARN)) + handler.setLevel(logging.WARNING) + handler.addFilter(LogFilter(logging.WARNING)) handler.setFormatter(MultilineFormatter( - self.warn(CONSOLE_FORMAT.format(colorama.Fore.YELLOW, colorama.Style.RESET_ALL)))) + self.warning(CONSOLE_FORMAT.format(colorama.Fore.YELLOW, colorama.Style.RESET_ALL)))) if json: handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT)) @@ -168,8 +167,8 @@ class Log: """Format error messages and return string.""" return msg - def warn(self, msg): - """Format warn messages and return string.""" + def warning(self, msg): + """Format warning messages and return string.""" return msg def info(self, msg): diff --git a/cleanupagents/Utils.py b/cleanupagents/Utils.py index ef4dbca..00ed4c6 100644 --- a/cleanupagents/Utils.py +++ b/cleanupagents/Utils.py @@ -4,7 +4,6 @@ import os import shlex import subprocess # nosec -import sys from distutils.util import strtobool diff --git a/setup.cfg b/setup.cfg index 46fb2da..3581314 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,3 +18,22 @@ filterwarnings = ignore::FutureWarning ignore:.*collections.*:DeprecationWarning ignore:.*pep8.*:FutureWarning + +[flake8] +# Explanation of errors +# +# D100: Missing docstring in public module +# D101: Missing docstring in public class +# D102: Missing docstring in public method +# D103: Missing docstring in public function +# D105: Missing docstring in magic method +# D107: Missing docstring in __init__ +# D202: No blank lines allowed after function docstring +# G001: Logging statements should not use string.format() for their first argument +# G004: Logging statements should not use f"..." for their first argument +# W503: Line break occurred before a binary operator +ignore = D100, D101, D102, D103, D105, D107, D202, G001, G004, W503 +max-line-length = 200 +inline-quotes = double +exclude = .git,.tox,__pycache__,build,dist,tests,*.pyc,*.egg-info,.cache,.eggs,env* +application-import-names = cleanupagents