docker-tidy/dockertidy/garbage_collector.py

#!/usr/bin/env python3
"""Remove unused docker containers and images."""

import fnmatch
from collections import namedtuple

import dateutil.parser
import docker.errors
import requests.exceptions

import docker
from dockertidy.config import SingleConfig
from dockertidy.logger import SingleLog
from dockertidy.parser import timedelta


class GarbageCollector:
    """Garbage collector object to handle cleanup tasks of container, images and volumes."""

    # This seems to be something docker uses for a null/zero date
    YEAR_ZERO = "0001-01-01T00:00:00Z"
    ExcludeLabel = namedtuple("ExcludeLabel", ["key", "value"])

    def __init__(self):
        self.config = SingleConfig()
        self.log = SingleLog()
        self.logger = SingleLog().logger
        self.docker = self._get_docker_client()

    def cleanup_containers(self):
        """Identify old containers and remove them."""
        config = self.config.config
        client = self.docker
        all_containers = self._get_all_containers()

        filtered_containers = self._filter_excluded_containers(all_containers)

        self.logger.info(
            "Removing containers older than '{}'".format(
                timedelta(config["gc"]["max_container_age"], dt_format="%Y-%m-%d, %H:%M:%S")
            )
        )

        for container_summary in reversed(list(filtered_containers)):
            container = self._api_call(
                client.inspect_container,
                container=container_summary["Id"],
            )
            if not container or not self._should_remove_container(
                container,
                timedelta(config["gc"]["max_container_age"]),
            ):
                continue

            self.logger.info(
                "Removing container {} {} {}".format(
                    container["Id"][:16],
                    container.get("Name", "").lstrip("/"), container["State"]["FinishedAt"]
                )
            )

            if not config["dry_run"]:
                self._api_call(
                    client.remove_container,
                    container=container["Id"],
                    v=True,
                )

    def _filter_excluded_containers(self, containers):
        config = self.config.config

        if not config["gc"]["exclude_container_labels"]:
            return containers

        def include_container(container):
            if self._should_exclude_container_with_labels(container):
                return False
            return True

        return filter(include_container, containers)

    def _should_exclude_container_with_labels(self, container):
        config = self.config.config

        if container["Labels"]:
            for exclude_label in config["gc"]["exclude_container_labels"]:
                if exclude_label.value:
                    matching_keys = fnmatch.filter(
                        container["Labels"].keys(),
                        exclude_label.key,
                    )
                    label_values_to_check = [
                        container["Labels"][matching_key] for matching_key in matching_keys
                    ]
                    if fnmatch.filter(label_values_to_check, exclude_label.value):
                        return True
                else:
                    if fnmatch.filter(container["Labels"].keys(), exclude_label.key):
                        return True
        return False

    def _should_remove_container(self, container, min_date):
        state = container.get("State", {})

        if state.get("Running"):
            return False

        if state.get("Ghost"):
            return True

        # Container was created, but never started
        if state.get("FinishedAt") == self.YEAR_ZERO:
            created_date = dateutil.parser.parse(container["Created"])
            return created_date < min_date

        finished_date = dateutil.parser.parse(state["FinishedAt"])
        return finished_date < min_date

    def _get_all_containers(self):
        client = self.docker
        self.logger.info("Getting all containers")
        containers = client.containers(all=True)
        self.logger.info("Found %s containers", len(containers))
        return containers

    def _get_all_images(self):
        client = self.docker
        self.logger.info("Getting all images")
        images = client.images()
        self.logger.info("Found %s images", len(images))
        return images

    def _get_dangling_volumes(self):
        client = self.docker
        self.logger.info("Getting dangling volumes")
        volumes = client.volumes({"dangling": True})["Volumes"] or []
        self.logger.info("Found %s dangling volumes", len(volumes))
        return volumes

    def cleanup_images(self, exclude_set):
        """Identify old images and remove them."""
        # re-fetch container list so that we don't  include removed containers
        client = self.docker
        config = self.config.config

        containers = self._get_all_containers()
        images = self._get_all_images()
        if docker.utils.compare_version("1.21", client._version) < 0:
            image_tags_in_use = {container["Image"] for container in containers}
            images = self._filter_images_in_use(images, image_tags_in_use)
        else:
            # ImageID field was added in 1.21
            image_ids_in_use = {container["ImageID"] for container in containers}
            images = self._filter_images_in_use_by_id(images, image_ids_in_use)
        images = self._filter_excluded_images(images, exclude_set)

        self.logger.info(
            "Removing images older than '{}'".format(
                timedelta(config["gc"]["max_image_age"], dt_format="%Y-%m-%d, %H:%M:%S")
            )
        )
        for image_summary in reversed(list(images)):
            self._remove_image(image_summary, timedelta(config["gc"]["max_image_age"]))

    def _filter_excluded_images(self, images, exclude_set):

        def include_image(image_summary):
            image_tags = image_summary.get("RepoTags")
            if self._no_image_tags(image_tags):
                return True
            for exclude_pattern in exclude_set:
                if fnmatch.filter(image_tags, exclude_pattern):
                    return False
            return True

        return filter(include_image, images)

    def _filter_images_in_use(self, images, image_tags_in_use):

        def get_tag_set(image_summary):
            image_tags = image_summary.get("RepoTags")
            if self._no_image_tags(image_tags):
                # The repr of the image Id used by client.containers()
                return set(["{id}:latest".format(id=image_summary["Id"][:12])])
            return set(image_tags)

        def image_not_in_use(image_summary):
            return not get_tag_set(image_summary) & image_tags_in_use

        return filter(image_not_in_use, images)

    def _filter_images_in_use_by_id(self, images, image_ids_in_use):

        def image_not_in_use(image_summary):
            return image_summary["Id"] not in image_ids_in_use

        return filter(image_not_in_use, images)

    def _is_image_old(self, image, min_date):
        return dateutil.parser.parse(image["Created"]) < min_date

    def _no_image_tags(self, image_tags):
        return not image_tags or image_tags == ["<none>:<none>"]

    def _remove_image(self, image_summary, min_date):
        config = self.config.config
        client = self.docker
        image = self._api_call(client.inspect_image, image=image_summary["Id"])

        if not image or not self._is_image_old(image, min_date):
            return

        self.logger.info(
            "Removing image {name}".format(name=self._format_image(image, image_summary))
        )
        if config["dry_run"]:
            return

        image_tags = image_summary.get("RepoTags")
        # If there are no tags, remove the id
        if self._no_image_tags(image_tags):
            self._api_call(client.remove_image, image=image_summary["Id"])
            return

        # Remove any repository tags so we don't  hit 409 Conflict
        for image_tag in image_tags:
            self._api_call(client.remove_image, image=image_tag)

    def _remove_volume(self, volume):
        config = self.config.config
        client = self.docker
        if not volume:
            return

        self.logger.info("Removing volume {name}".format(name=volume["Name"]))
        if config["dry_run"]:
            return

        self._api_call(client.remove_volume, name=volume["Name"])

    def cleanup_volumes(self):
        """Identify old volumes and remove them."""
        dangling_volumes = self._get_dangling_volumes()

        self.logger.info("Removing dangling volumes")
        for volume in reversed(dangling_volumes):
            self.logger.info("Removing dangling volume %s", volume["Name"])
            self._remove_volume(volume)

    def _api_call(self, func, **kwargs):
        try:
            return func(**kwargs)
        except requests.exceptions.Timeout as e:
            params = ",".join("%s=%s" % item for item in kwargs.items())  # noqa
            self.logger.warn(
                "Failed to call {name} {params} {msg}".format(
                    name=func.__name__, params=params, msg=str(e)
                )
            )
        except docker.errors.APIError as e:
            params = ",".join("%s=%s" % item for item in kwargs.items())  # noqa
            self.logger.warn(
                "Error calling {name} {params} {msg}".format(
                    name=func.__name__, params=params, msg=str(e)
                )
            )

    def _format_image(self, image, image_summary):

        def get_tags():
            tags = image_summary.get("RepoTags")
            if not tags or tags == ["<none>:<none>"]:
                return ""
            return ", ".join(tags)

        return "{id} {tags}".format(id=image["Id"][:16], tags=get_tags())

    def _build_exclude_set(self):
        config = self.config.config
        exclude_set = set(config["gc"]["exclude_images"])

        def is_image_tag(line):
            return line and not line.startswith("#")

        return exclude_set

    def _format_exclude_labels(self):
        config = self.config.config
        exclude_labels = []

        for exclude_label_arg in config["gc"]["exclude_container_labels"]:
            split_exclude_label = exclude_label_arg.split("=", 1)
            exclude_label_key = split_exclude_label[0]
            if len(split_exclude_label) == 2:
                exclude_label_value = split_exclude_label[1]
            else:
                exclude_label_value = None
            exclude_labels.append(
                self.ExcludeLabel(
                    key=exclude_label_key,
                    value=exclude_label_value,
                )
            )
        config["gc"]["exclude_container_labels"] = exclude_labels

    def _get_docker_client(self):
        config = self.config.config
        try:
            return docker.APIClient(version="auto", timeout=config["http_timeout"])
        except docker.errors.DockerException as e:
            self.log.sysexit_with_message("Can't create docker client\n{}".format(e))

    def run(self):
        """Garbage collector main method."""
        self.logger.info("Start garbage collection")
        config = self.config.config
        self._format_exclude_labels()

        if config["gc"]["max_container_age"]:
            self.cleanup_containers()

        if config["gc"]["max_image_age"]:
            exclude_set = self._build_exclude_set()
            self.cleanup_images(exclude_set)

        if config["gc"]["dangling_volumes"]:
            self.cleanup_volumes()

        if (
            not config["gc"]["max_container_age"] and not config["gc"]["max_image_age"]
            and not config["gc"]["dangling_volumes"]
        ):
            self.logger.warn("Skipped, no arguments given")