fix: skip not parsable lines in json output

This commit is contained in:
Robert Kaussow 2022-10-25 09:41:17 +02:00
parent 9ea4b33577
commit d3e7a4ba3c
Signed by: xoxys
GPG Key ID: 4E692A2EAECC03C0
7 changed files with 65 additions and 56 deletions

View File

@ -1,4 +1,4 @@
local PythonVersion(pyversion='3.5') = { local PythonVersion(pyversion='3.7') = {
name: 'python' + std.strReplace(pyversion, '.', '') + '-pytest', name: 'python' + std.strReplace(pyversion, '.', '') + '-pytest',
image: 'python:' + pyversion, image: 'python:' + pyversion,
pull: 'always', pull: 'always',
@ -25,7 +25,7 @@ local PipelineLint = {
steps: [ steps: [
{ {
name: 'flake8', name: 'flake8',
image: 'python:3.7', image: 'python:3.10',
pull: 'always', pull: 'always',
environment: { environment: {
PY_COLORS: 1, PY_COLORS: 1,
@ -50,10 +50,10 @@ local PipelineTest = {
arch: 'amd64', arch: 'amd64',
}, },
steps: [ steps: [
PythonVersion(pyversion='3.5'),
PythonVersion(pyversion='3.6'),
PythonVersion(pyversion='3.7'), PythonVersion(pyversion='3.7'),
PythonVersion(pyversion='3.8'), PythonVersion(pyversion='3.8'),
PythonVersion(pyversion='3.9'),
PythonVersion(pyversion='3.10'),
], ],
trigger: { trigger: {
ref: ['refs/heads/master', 'refs/tags/**', 'refs/pull/**'], ref: ['refs/heads/master', 'refs/tags/**', 'refs/pull/**'],
@ -73,7 +73,7 @@ local PipelineSecurity = {
steps: [ steps: [
{ {
name: 'bandit', name: 'bandit',
image: 'python:3.7', image: 'python:3.10',
pull: 'always', pull: 'always',
environment: { environment: {
PY_COLORS: 1, PY_COLORS: 1,
@ -103,7 +103,7 @@ local PipelineBuild = {
steps: [ steps: [
{ {
name: 'build', name: 'build',
image: 'python:3.7', image: 'python:3.10',
pull: 'always', pull: 'always',
commands: [ commands: [
'python setup.py sdist bdist_wheel', 'python setup.py sdist bdist_wheel',

View File

@ -9,7 +9,7 @@ platform:
steps: steps:
- name: flake8 - name: flake8
pull: always pull: always
image: python:3.7 image: python:3.10
commands: commands:
- pip install -r test-requirements.txt -qq - pip install -r test-requirements.txt -qq
- pip install -qq . - pip install -qq .
@ -32,30 +32,6 @@ platform:
arch: amd64 arch: amd64
steps: steps:
- name: python35-pytest
pull: always
image: python:3.5
commands:
- pip install -r test-requirements.txt -qq
- pip install -qq .
- drone-cleanup-agents --help
environment:
PY_COLORS: 1
depends_on:
- clone
- name: python36-pytest
pull: always
image: python:3.6
commands:
- pip install -r test-requirements.txt -qq
- pip install -qq .
- drone-cleanup-agents --help
environment:
PY_COLORS: 1
depends_on:
- clone
- name: python37-pytest - name: python37-pytest
pull: always pull: always
image: python:3.7 image: python:3.7
@ -80,6 +56,30 @@ steps:
depends_on: depends_on:
- clone - clone
- name: python39-pytest
pull: always
image: python:3.9
commands:
- pip install -r test-requirements.txt -qq
- pip install -qq .
- drone-cleanup-agents --help
environment:
PY_COLORS: 1
depends_on:
- clone
- name: python310-pytest
pull: always
image: python:3.10
commands:
- pip install -r test-requirements.txt -qq
- pip install -qq .
- drone-cleanup-agents --help
environment:
PY_COLORS: 1
depends_on:
- clone
trigger: trigger:
ref: ref:
- refs/heads/master - refs/heads/master
@ -100,7 +100,7 @@ platform:
steps: steps:
- name: bandit - name: bandit
pull: always pull: always
image: python:3.7 image: python:3.10
commands: commands:
- pip install -r test-requirements.txt -qq - pip install -r test-requirements.txt -qq
- pip install -qq . - pip install -qq .
@ -128,7 +128,7 @@ platform:
steps: steps:
- name: build - name: build
pull: always pull: always
image: python:3.7 image: python:3.10
commands: commands:
- python setup.py sdist bdist_wheel - python setup.py sdist bdist_wheel

View File

@ -1,8 +0,0 @@
[flake8]
# Temp disable Docstring checks D101, D102, D103, D107
ignore = E501, W503, F401, N813, D101, D102, D103, D107
max-line-length = 100
inline-quotes = double
exclude = .git,.tox,__pycache__,build,dist,tests,*.pyc,*.egg-info,.cache,.eggs,env*
application-import-names = cleanupagents
format = ${cyan}%(path)s:%(row)d:%(col)d${reset}: ${red_bold}%(code)s${reset} %(text)s

View File

@ -3,20 +3,16 @@
import argparse import argparse
import json import json
import logging
import os import os
import pickle # nosec import pickle # nosec
import shutil import shutil
import sys
from collections import defaultdict from collections import defaultdict
from urllib.parse import urlparse
from cleanupagents import __version__ from cleanupagents import __version__
from cleanupagents.Logging import SingleLog from cleanupagents.Logging import SingleLog
from cleanupagents.Utils import humanize from cleanupagents.Utils import humanize
from cleanupagents.Utils import normalize_path from cleanupagents.Utils import normalize_path
from cleanupagents.Utils import run_command from cleanupagents.Utils import run_command
from cleanupagents.Utils import to_bool
class AgentCleanup: class AgentCleanup:
@ -102,7 +98,11 @@ class AgentCleanup:
self.log.sysexit_with_message("Command error:\n{}".format(humanize(res.stdout))) self.log.sysexit_with_message("Command error:\n{}".format(humanize(res.stdout)))
for line in res.stdout.splitlines(): for line in res.stdout.splitlines():
obj = json.loads(line) try:
obj = json.loads(line)
except json.decoder.JSONDecodeError:
self.logger.debug("Unable to parse line: {}".format(line))
continue
if obj["state"] == "error": if obj["state"] == "error":
error.append(obj["name"]) error.append(obj["name"])

View File

@ -9,7 +9,6 @@ import sys
import colorama import colorama
from pythonjsonlogger import jsonlogger from pythonjsonlogger import jsonlogger
import cleanupagents.Utils
from cleanupagents.Utils import Singleton from cleanupagents.Utils import Singleton
from cleanupagents.Utils import to_bool from cleanupagents.Utils import to_bool
@ -65,11 +64,11 @@ class MultilineJsonFormatter(jsonlogger.JsonFormatter):
class Log: class Log:
def __init__(self, level=logging.WARN, name="cleanupagents", logfile="/var/log/drone-agents.log", json=False): def __init__(self, level=logging.WARNING, name="cleanupagents", logfile="/var/log/drone-agents.log", json=False):
self.logger = logging.getLogger(name) self.logger = logging.getLogger(name)
self.logger.setLevel(level) self.logger.setLevel(level)
self.logger.addHandler(self._get_error_handler(json=json)) self.logger.addHandler(self._get_error_handler(json=json))
self.logger.addHandler(self._get_warn_handler(json=json)) self.logger.addHandler(self._get_warning_handler(json=json))
self.logger.addHandler(self._get_info_handler(json=json)) self.logger.addHandler(self._get_info_handler(json=json))
self.logger.addHandler(self._get_critical_handler(json=json)) self.logger.addHandler(self._get_critical_handler(json=json))
self.logger.addHandler(self._get_debug_handler(json=json)) self.logger.addHandler(self._get_debug_handler(json=json))
@ -105,12 +104,12 @@ class Log:
return handler return handler
def _get_warn_handler(self, json=False): def _get_warning_handler(self, json=False):
handler = logging.StreamHandler(sys.stdout) handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.WARN) handler.setLevel(logging.WARNING)
handler.addFilter(LogFilter(logging.WARN)) handler.addFilter(LogFilter(logging.WARNING))
handler.setFormatter(MultilineFormatter( handler.setFormatter(MultilineFormatter(
self.warn(CONSOLE_FORMAT.format(colorama.Fore.YELLOW, colorama.Style.RESET_ALL)))) self.warning(CONSOLE_FORMAT.format(colorama.Fore.YELLOW, colorama.Style.RESET_ALL))))
if json: if json:
handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT)) handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
@ -168,8 +167,8 @@ class Log:
"""Format error messages and return string.""" """Format error messages and return string."""
return msg return msg
def warn(self, msg): def warning(self, msg):
"""Format warn messages and return string.""" """Format warning messages and return string."""
return msg return msg
def info(self, msg): def info(self, msg):

View File

@ -4,7 +4,6 @@
import os import os
import shlex import shlex
import subprocess # nosec import subprocess # nosec
import sys
from distutils.util import strtobool from distutils.util import strtobool

View File

@ -18,3 +18,22 @@ filterwarnings =
ignore::FutureWarning ignore::FutureWarning
ignore:.*collections.*:DeprecationWarning ignore:.*collections.*:DeprecationWarning
ignore:.*pep8.*:FutureWarning ignore:.*pep8.*:FutureWarning
[flake8]
# Explanation of errors
#
# D100: Missing docstring in public module
# D101: Missing docstring in public class
# D102: Missing docstring in public method
# D103: Missing docstring in public function
# D105: Missing docstring in magic method
# D107: Missing docstring in __init__
# D202: No blank lines allowed after function docstring
# G001: Logging statements should not use string.format() for their first argument
# G004: Logging statements should not use f"..." for their first argument
# W503: Line break occurred before a binary operator
ignore = D100, D101, D102, D103, D105, D107, D202, G001, G004, W503
max-line-length = 200
inline-quotes = double
exclude = .git,.tox,__pycache__,build,dist,tests,*.pyc,*.egg-info,.cache,.eggs,env*
application-import-names = cleanupagents