github
/
docker-tidy
mirror of https://github.com/thegeeklab/docker-tidy.git
0
0
Fork 0
Code Issues Projects Releases Wiki Activity

add garbage colletor run method

This commit is contained in:
Robert Kaussow 2020-03-09 01:05:17 +01:00
parent 371789c258
commit 1561264542
5 changed files with 82 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dockertidy/Autostop.py
View File

@ -68,4 +68,8 @@ class AutoStop:
def _get_docker_client(self):
config = self.config.config
return docker.APIClient(version="auto", timeout=config["timeout"])
return docker.APIClient(version="auto", timeout=config["http_timeout"])
def run(self):
"""Autostop main method."""
print("run stop")

20
dockertidy/Cli.py
View File

@ -5,7 +5,9 @@ import argparse
import dockertidy.Exception
from dockertidy import __version__
from dockertidy.Autostop import AutoStop
from dockertidy.Config import SingleConfig
from dockertidy.GarbageCollector import GarbageCollector
from dockertidy.Logger import SingleLog
from dockertidy.Parser import timedelta_validator
@ -18,6 +20,9 @@ class DockerTidy:
self.logger = self.log.logger
self.args = self._cli_args()
self.config = self._get_config()
self.gc = GarbageCollector()
self.stop = AutoStop()
self.run()
def _cli_args(self):
"""
@ -54,6 +59,7 @@ class DockerTidy:
)
subparsers = parser.add_subparsers(dest="command", help="sub-command help")
subparsers.required = True
parser_gc = subparsers.add_parser("gc", help="Run docker garbage collector.")
parser_gc.add_argument(
@ -84,7 +90,7 @@ class DockerTidy:
"--exclude-image",
action="append",
type=str,
dest="gc.exclude_image",
dest="gc.exclude_images",
metavar="EXCLUDE_IMAGE",
help="Never remove images with this tag."
)
@ -92,7 +98,7 @@ class DockerTidy:
"--exclude-container-label",
action="append",
type=str,
dest="gc.exclude_container_label",
dest="gc.exclude_container_labels",
metavar="EXCLUDE_CONTAINER_LABEL",
help="Never remove containers with this label key "
"or label key=value"
@ -127,13 +133,19 @@ class DockerTidy:
except dockertidy.Exception.ConfigError as e:
self.log.sysexit_with_message(e)
print(config.config)
try:
self.log.set_level(config.config["logging"]["level"])
except ValueError as e:
self.log.sysexit_with_message("Can not set log level.\n{}".format(str(e)))
self.logger.info("Using config file {}".format(config.config_file))
self.logger.debug("Config dump: {}".format(config.config))
return config
def run(self):
"""Cli main method."""
if self.config.config["command"] == "gc":
self.gc.run()
elif self.config.config["command"] == "stop":
self.stop.run()

16
dockertidy/Config.py
View File

@ -63,37 +63,37 @@ class Config():
"type": environs.Env().bool
},
"gc.max_container_age": {
"default": "1day",
"default": "",
"env": "GC_MAX_CONTAINER_AGE",
"file": True,
"type": env.timedelta_validator
},
"gc.max_image_age": {
"default": "1day",
"default": "",
"env": "GC_MAX_IMAGE_AGE",
"file": True,
"type": env.timedelta_validator
},
"gc.dangling_volumes": {
"default": False,
"env": "GC_EXCLUDE_IMAGE",
"env": "GC_DANGLING_VOLUMES",
"file": True,
"type": environs.Env().bool
},
"gc.exclude_image": {
"gc.exclude_images": {
"default": [],
"env": "GC_DANGLING_VOLUMES",
"env": "GC_EXCLUDE_IMAGES",
"file": True,
"type": environs.Env().list
},
"gc.exclude_container_label": {
"gc.exclude_container_labels": {
"default": [],
"env": "GC_EXCLUDE_CONTAINER_LABEL",
"env": "GC_EXCLUDE_CONTAINER_LABELS",
"file": True,
"type": environs.Env().list
},
"stop.max_run_time": {
"default": "3days",
"default": "",
"env": "STOP_MAX_RUN_TIME",
"file": True,
"type": env.timedelta_validator

79
dockertidy/GarbageCollector.py
View File

@ -11,6 +11,7 @@ import requests.exceptions
from dockertidy.Config import SingleConfig
from dockertidy.Logger import SingleLog
from dockertidy.Parser import timedelta
class GarbageCollector:
@ -30,10 +31,14 @@ class GarbageCollector:
"""Identify old containers and remove them."""
config = self.config.config
client = self.docker
all_containers = self._get_all_containers(client)
filtered_containers = self._filter_excluded_containers(
all_containers,
config["gc"]["exclude_container_labels"],
all_containers = self._get_all_containers()
filtered_containers = self._filter_excluded_containers(all_containers)
self.logger.info(
"Removing containers older than '{}'".format(
timedelta(config["gc"]["max_container_age"], dt_format="%Y-%m-%d, %H:%M:%S")
)
)
for container_summary in reversed(list(filtered_containers)):
@ -43,14 +48,14 @@ class GarbageCollector:
)
if not container or not self._should_remove_container(
container,
config["gc"]["max_container_age"],
timedelta(config["gc"]["max_container_age"]),
):
continue
self.logger.info(
"Removing container %s %s %s" % (
container["Id"][:16], container.get("Name", "").lstrip("/"),
container["State"]["FinishedAt"]
"Removing container {} {} {}".format(
container["Id"][:16],
container.get("Name", "").lstrip("/"), container["State"]["FinishedAt"]
)
)
@ -152,16 +157,19 @@ class GarbageCollector:
images = self._filter_images_in_use_by_id(images, image_ids_in_use)
images = self._filter_excluded_images(images, exclude_set)
for image_summary in reversed(list(images)):
self._rmove_image(
client, image_summary, config["gc"]["max_image_age"], config["dry_run"]
self.logger.info(
"Removing images older than '{}'".format(
timedelta(config["gc"]["max_image_age"], dt_format="%Y-%m-%d, %H:%M:%S")
)
)
for image_summary in reversed(list(images)):
self._remove_image(image_summary, timedelta(config["gc"]["max_image_age"]))
def _filter_excluded_images(self, images, exclude_set):
def include_image(image_summary):
image_tags = image_summary.get("RepoTags")
if self.no_image_tags(image_tags):
if self._no_image_tags(image_tags):
return True
for exclude_pattern in exclude_set:
if fnmatch.filter(image_tags, exclude_pattern):
@ -197,7 +205,7 @@ class GarbageCollector:
def _no_image_tags(self, image_tags):
return not image_tags or image_tags == ["<none>:<none>"]
def _remove_image(self, client, image_summary, min_date):
def _remove_image(self, image_summary, min_date):
config = self.config.config
client = self.docker
image = self._api_call(client.inspect_image, image=image_summary["Id"])
@ -219,8 +227,9 @@ class GarbageCollector:
for image_tag in image_tags:
self._api_call(client.remove_image, image=image_tag)
def _remove_volume(self, client, volume):
def _remove_volume(self, volume):
config = self.config.config
client = self.docker
if not volume:
return
@ -230,13 +239,14 @@ class GarbageCollector:
self._api_call(client.remove_volume, name=volume["Name"])
def cleanup_volumes(self, client):
def cleanup_volumes(self):
"""Identify old volumes and remove them."""
dangling_volumes = self._get_dangling_volumes(client)
dangling_volumes = self._get_dangling_volumes()
for volume in reversed(dangling_volumes):
self.logger.info("Removing dangling volume %s", volume["Name"])
self._remove_volume(client, volume)
self._remove_volume(volume)
def _api_call(self, func, **kwargs):
try:
@ -260,7 +270,7 @@ class GarbageCollector:
def _build_exclude_set(self):
config = self.config.config
exclude_set = set(config["gc"]["exclude_image"])
exclude_set = set(config["gc"]["exclude_images"])
def is_image_tag(line):
return line and not line.startswith("#")
@ -288,23 +298,26 @@ class GarbageCollector:
def _get_docker_client(self):
config = self.config.config
return docker.APIClient(version="auto", timeout=config["timeout"])
return docker.APIClient(version="auto", timeout=config["http_timeout"])
def run(self):
"""Garbage collector main method."""
self.logger.info("Start garbage collection")
config = self.config.config
# def main():
# exclude_container_labels = format_exclude_labels(args.exclude_container_label)
if config["gc"]["max_container_age"]:
self.cleanup_containers()
# if args.max_container_age:
# cleanup_containers(
# client,
# args.max_container_age,
# args.dry_run,
# exclude_container_labels,
# )
if config["gc"]["max_image_age"]:
exclude_set = self._build_exclude_set()
self.cleanup_images(exclude_set)
# if args.max_image_age:
# exclude_set = build_exclude_set(args.exclude_image, args.exclude_image_file)
# cleanup_images(client, args.max_image_age, args.dry_run, exclude_set)
if config["gc"]["dangling_volumes"]:
self.logger.info("Remove dangling volumes")
self.cleanup_volumes()
# if args.dangling_volumes:
# cleanup_volumes(client, args.dry_run)
if (
not config["gc"]["max_container_age"] or not config["gc"]["max_image_age"]
or not config["gc"]["dangling_volumes"]
):
self.logger.info("Skipped, no arguments given")

9
dockertidy/Parser.py
View File

@ -26,7 +26,7 @@ def timedelta_validator(value):
raise
def timedelta(value):
def timedelta(value, dt_format=None):
"""Return the :class:`datetime.datetime.DateTime` for a time in the past.
:param value: a string containing a time format supported by
@ -34,7 +34,12 @@ def timedelta(value):
"""
if value is None:
return None
return _datetime_seconds_ago(timeparse.timeparse(value))
timedelta = _datetime_seconds_ago(timeparse.timeparse(value))
if dt_format:
timedelta = timedelta.strftime(dt_format)
return timedelta
def _datetime_seconds_ago(seconds):