fix: skip not parsable lines in json output
This commit is contained in:
parent
9ea4b33577
commit
d3e7a4ba3c
@ -1,4 +1,4 @@
|
||||
local PythonVersion(pyversion='3.5') = {
|
||||
local PythonVersion(pyversion='3.7') = {
|
||||
name: 'python' + std.strReplace(pyversion, '.', '') + '-pytest',
|
||||
image: 'python:' + pyversion,
|
||||
pull: 'always',
|
||||
@ -25,7 +25,7 @@ local PipelineLint = {
|
||||
steps: [
|
||||
{
|
||||
name: 'flake8',
|
||||
image: 'python:3.7',
|
||||
image: 'python:3.10',
|
||||
pull: 'always',
|
||||
environment: {
|
||||
PY_COLORS: 1,
|
||||
@ -50,10 +50,10 @@ local PipelineTest = {
|
||||
arch: 'amd64',
|
||||
},
|
||||
steps: [
|
||||
PythonVersion(pyversion='3.5'),
|
||||
PythonVersion(pyversion='3.6'),
|
||||
PythonVersion(pyversion='3.7'),
|
||||
PythonVersion(pyversion='3.8'),
|
||||
PythonVersion(pyversion='3.9'),
|
||||
PythonVersion(pyversion='3.10'),
|
||||
],
|
||||
trigger: {
|
||||
ref: ['refs/heads/master', 'refs/tags/**', 'refs/pull/**'],
|
||||
@ -73,7 +73,7 @@ local PipelineSecurity = {
|
||||
steps: [
|
||||
{
|
||||
name: 'bandit',
|
||||
image: 'python:3.7',
|
||||
image: 'python:3.10',
|
||||
pull: 'always',
|
||||
environment: {
|
||||
PY_COLORS: 1,
|
||||
@ -103,7 +103,7 @@ local PipelineBuild = {
|
||||
steps: [
|
||||
{
|
||||
name: 'build',
|
||||
image: 'python:3.7',
|
||||
image: 'python:3.10',
|
||||
pull: 'always',
|
||||
commands: [
|
||||
'python setup.py sdist bdist_wheel',
|
||||
|
54
.drone.yml
54
.drone.yml
@ -9,7 +9,7 @@ platform:
|
||||
steps:
|
||||
- name: flake8
|
||||
pull: always
|
||||
image: python:3.7
|
||||
image: python:3.10
|
||||
commands:
|
||||
- pip install -r test-requirements.txt -qq
|
||||
- pip install -qq .
|
||||
@ -32,30 +32,6 @@ platform:
|
||||
arch: amd64
|
||||
|
||||
steps:
|
||||
- name: python35-pytest
|
||||
pull: always
|
||||
image: python:3.5
|
||||
commands:
|
||||
- pip install -r test-requirements.txt -qq
|
||||
- pip install -qq .
|
||||
- drone-cleanup-agents --help
|
||||
environment:
|
||||
PY_COLORS: 1
|
||||
depends_on:
|
||||
- clone
|
||||
|
||||
- name: python36-pytest
|
||||
pull: always
|
||||
image: python:3.6
|
||||
commands:
|
||||
- pip install -r test-requirements.txt -qq
|
||||
- pip install -qq .
|
||||
- drone-cleanup-agents --help
|
||||
environment:
|
||||
PY_COLORS: 1
|
||||
depends_on:
|
||||
- clone
|
||||
|
||||
- name: python37-pytest
|
||||
pull: always
|
||||
image: python:3.7
|
||||
@ -80,6 +56,30 @@ steps:
|
||||
depends_on:
|
||||
- clone
|
||||
|
||||
- name: python39-pytest
|
||||
pull: always
|
||||
image: python:3.9
|
||||
commands:
|
||||
- pip install -r test-requirements.txt -qq
|
||||
- pip install -qq .
|
||||
- drone-cleanup-agents --help
|
||||
environment:
|
||||
PY_COLORS: 1
|
||||
depends_on:
|
||||
- clone
|
||||
|
||||
- name: python310-pytest
|
||||
pull: always
|
||||
image: python:3.10
|
||||
commands:
|
||||
- pip install -r test-requirements.txt -qq
|
||||
- pip install -qq .
|
||||
- drone-cleanup-agents --help
|
||||
environment:
|
||||
PY_COLORS: 1
|
||||
depends_on:
|
||||
- clone
|
||||
|
||||
trigger:
|
||||
ref:
|
||||
- refs/heads/master
|
||||
@ -100,7 +100,7 @@ platform:
|
||||
steps:
|
||||
- name: bandit
|
||||
pull: always
|
||||
image: python:3.7
|
||||
image: python:3.10
|
||||
commands:
|
||||
- pip install -r test-requirements.txt -qq
|
||||
- pip install -qq .
|
||||
@ -128,7 +128,7 @@ platform:
|
||||
steps:
|
||||
- name: build
|
||||
pull: always
|
||||
image: python:3.7
|
||||
image: python:3.10
|
||||
commands:
|
||||
- python setup.py sdist bdist_wheel
|
||||
|
||||
|
8
.flake8
8
.flake8
@ -1,8 +0,0 @@
|
||||
[flake8]
|
||||
# Temp disable Docstring checks D101, D102, D103, D107
|
||||
ignore = E501, W503, F401, N813, D101, D102, D103, D107
|
||||
max-line-length = 100
|
||||
inline-quotes = double
|
||||
exclude = .git,.tox,__pycache__,build,dist,tests,*.pyc,*.egg-info,.cache,.eggs,env*
|
||||
application-import-names = cleanupagents
|
||||
format = ${cyan}%(path)s:%(row)d:%(col)d${reset}: ${red_bold}%(code)s${reset} %(text)s
|
@ -3,20 +3,16 @@
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pickle # nosec
|
||||
import shutil
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from cleanupagents import __version__
|
||||
from cleanupagents.Logging import SingleLog
|
||||
from cleanupagents.Utils import humanize
|
||||
from cleanupagents.Utils import normalize_path
|
||||
from cleanupagents.Utils import run_command
|
||||
from cleanupagents.Utils import to_bool
|
||||
|
||||
|
||||
class AgentCleanup:
|
||||
@ -102,7 +98,11 @@ class AgentCleanup:
|
||||
self.log.sysexit_with_message("Command error:\n{}".format(humanize(res.stdout)))
|
||||
|
||||
for line in res.stdout.splitlines():
|
||||
obj = json.loads(line)
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.decoder.JSONDecodeError:
|
||||
self.logger.debug("Unable to parse line: {}".format(line))
|
||||
continue
|
||||
|
||||
if obj["state"] == "error":
|
||||
error.append(obj["name"])
|
||||
|
@ -9,7 +9,6 @@ import sys
|
||||
import colorama
|
||||
from pythonjsonlogger import jsonlogger
|
||||
|
||||
import cleanupagents.Utils
|
||||
from cleanupagents.Utils import Singleton
|
||||
from cleanupagents.Utils import to_bool
|
||||
|
||||
@ -65,11 +64,11 @@ class MultilineJsonFormatter(jsonlogger.JsonFormatter):
|
||||
|
||||
|
||||
class Log:
|
||||
def __init__(self, level=logging.WARN, name="cleanupagents", logfile="/var/log/drone-agents.log", json=False):
|
||||
def __init__(self, level=logging.WARNING, name="cleanupagents", logfile="/var/log/drone-agents.log", json=False):
|
||||
self.logger = logging.getLogger(name)
|
||||
self.logger.setLevel(level)
|
||||
self.logger.addHandler(self._get_error_handler(json=json))
|
||||
self.logger.addHandler(self._get_warn_handler(json=json))
|
||||
self.logger.addHandler(self._get_warning_handler(json=json))
|
||||
self.logger.addHandler(self._get_info_handler(json=json))
|
||||
self.logger.addHandler(self._get_critical_handler(json=json))
|
||||
self.logger.addHandler(self._get_debug_handler(json=json))
|
||||
@ -105,12 +104,12 @@ class Log:
|
||||
|
||||
return handler
|
||||
|
||||
def _get_warn_handler(self, json=False):
|
||||
def _get_warning_handler(self, json=False):
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setLevel(logging.WARN)
|
||||
handler.addFilter(LogFilter(logging.WARN))
|
||||
handler.setLevel(logging.WARNING)
|
||||
handler.addFilter(LogFilter(logging.WARNING))
|
||||
handler.setFormatter(MultilineFormatter(
|
||||
self.warn(CONSOLE_FORMAT.format(colorama.Fore.YELLOW, colorama.Style.RESET_ALL))))
|
||||
self.warning(CONSOLE_FORMAT.format(colorama.Fore.YELLOW, colorama.Style.RESET_ALL))))
|
||||
|
||||
if json:
|
||||
handler.setFormatter(MultilineJsonFormatter(JSON_FORMAT))
|
||||
@ -168,8 +167,8 @@ class Log:
|
||||
"""Format error messages and return string."""
|
||||
return msg
|
||||
|
||||
def warn(self, msg):
|
||||
"""Format warn messages and return string."""
|
||||
def warning(self, msg):
|
||||
"""Format warning messages and return string."""
|
||||
return msg
|
||||
|
||||
def info(self, msg):
|
||||
|
@ -4,7 +4,6 @@
|
||||
import os
|
||||
import shlex
|
||||
import subprocess # nosec
|
||||
import sys
|
||||
from distutils.util import strtobool
|
||||
|
||||
|
||||
|
19
setup.cfg
19
setup.cfg
@ -18,3 +18,22 @@ filterwarnings =
|
||||
ignore::FutureWarning
|
||||
ignore:.*collections.*:DeprecationWarning
|
||||
ignore:.*pep8.*:FutureWarning
|
||||
|
||||
[flake8]
|
||||
# Explanation of errors
|
||||
#
|
||||
# D100: Missing docstring in public module
|
||||
# D101: Missing docstring in public class
|
||||
# D102: Missing docstring in public method
|
||||
# D103: Missing docstring in public function
|
||||
# D105: Missing docstring in magic method
|
||||
# D107: Missing docstring in __init__
|
||||
# D202: No blank lines allowed after function docstring
|
||||
# G001: Logging statements should not use string.format() for their first argument
|
||||
# G004: Logging statements should not use f"..." for their first argument
|
||||
# W503: Line break occurred before a binary operator
|
||||
ignore = D100, D101, D102, D103, D105, D107, D202, G001, G004, W503
|
||||
max-line-length = 200
|
||||
inline-quotes = double
|
||||
exclude = .git,.tox,__pycache__,build,dist,tests,*.pyc,*.egg-info,.cache,.eggs,env*
|
||||
application-import-names = cleanupagents
|
||||
|
Reference in New Issue
Block a user