From 95ded148913b83c6e345883496cfe8cfe9a8e51a Mon Sep 17 00:00:00 2001 From: Andy Tran Date: Tue, 6 Mar 2018 15:21:36 -0800 Subject: [PATCH 1/2] Add --exclude-container-label argument to dcgc Add an argument to dcgc that allows dcgc to ignore removing a container and its child images based on the container's label --- README.rst | 24 +++++++++++ docker_custodian/docker_gc.py | 76 ++++++++++++++++++++++++++++++----- tests/docker_gc_test.py | 35 +++++++++++++--- 3 files changed, 119 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index c0ad43a..21e14e6 100644 --- a/README.rst +++ b/README.rst @@ -92,6 +92,30 @@ You also can use basic pattern matching to exclude images with generic tags. user/repositoryB:?.? user/repositoryC-*:tag + +Prevent containers and associated images from being removed +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``dcgc`` also supports a container exclude list based on labels. If there are +stopped containers that you'd like to keep, then you can check the labels to +prevent them from being removed. + +:: + + --exclude-container-label + Never remove containers that have the label key=value. =value can be + omitted and in that case only the key is checked. May be specified + more than once. + +You also can use basic pattern matching to exclude generic labels. + +.. code:: + + foo* + com.docker.compose.project=test* + com.docker*=*bar* + + dcstop ------ diff --git a/docker_custodian/docker_gc.py b/docker_custodian/docker_gc.py index 51d0881..0b6faa3 100644 --- a/docker_custodian/docker_gc.py +++ b/docker_custodian/docker_gc.py @@ -23,14 +23,26 @@ log = logging.getLogger(__name__) YEAR_ZERO = "0001-01-01T00:00:00Z" -def cleanup_containers(client, max_container_age, dry_run): +def cleanup_containers( + client, + max_container_age, + dry_run, + exclude_container_labels, +): all_containers = get_all_containers(client) - - for container_summary in reversed(all_containers): - container = api_call(client.inspect_container, - container=container_summary['Id']) - if not container or not should_remove_container(container, - max_container_age): + filtered_containers = filter_excluded_containers( + all_containers, + exclude_container_labels, + ) + for container_summary in reversed(list(filtered_containers)): + container = api_call( + client.inspect_container, + container=container_summary['Id'], + ) + if not container or not should_remove_container( + container, + max_container_age, + ): continue log.info("Removing container %s %s %s" % ( @@ -39,8 +51,43 @@ def cleanup_containers(client, max_container_age, dry_run): container['State']['FinishedAt'])) if not dry_run: - api_call(client.remove_container, container=container['Id'], - v=True) + api_call( + client.remove_container, + container=container['Id'], + v=True, + ) + + +def filter_excluded_containers(containers, exclude_container_labels): + def include_container(container): + if exclude_container_labels and should_exclude_container_with_labels( + container, + exclude_container_labels, + ): + return False + return True + return filter(include_container, containers) + + +def should_exclude_container_with_labels(container, exclude_container_labels): + for exclude_container_label in exclude_container_labels: + split_exclude_label = exclude_container_label.split('=', 1) + if len(split_exclude_label) == 2: + exclude_key, exclude_value = split_exclude_label + matching_keys = fnmatch.filter( + container['Labels'].keys(), exclude_key + ) + label_values_to_check = [ + container['Labels'][matching_key] + for matching_key in matching_keys + ] + if fnmatch.filter(label_values_to_check, exclude_value): + return True + else: + exclude_key = split_exclude_label[0] + if fnmatch.filter(container['Labels'].keys(), exclude_key): + return True + return False def should_remove_container(container, min_date): @@ -226,7 +273,12 @@ def main(): **kwargs_from_env()) if args.max_container_age: - cleanup_containers(client, args.max_container_age, args.dry_run) + cleanup_containers( + client, + args.max_container_age, + args.dry_run, + args.exclude_container_label, + ) if args.max_image_age: exclude_set = build_exclude_set( @@ -271,6 +323,10 @@ def get_args(args=None): type=argparse.FileType('r'), help="Path to a file which contains a list of images to exclude, one " "image tag per line.") + parser.add_argument( + '--exclude-container-label', + action='append', + help="Never remove containers with this label key or label key=value") return parser.parse_args(args=args) diff --git a/tests/docker_gc_test.py b/tests/docker_gc_test.py index 54cbe6f..fe47aca 100644 --- a/tests/docker_gc_test.py +++ b/tests/docker_gc_test.py @@ -51,24 +51,47 @@ def test_cleanup_containers(mock_client, now): 'Name': 'one', 'State': { 'Running': False, - 'FinishedAt': '2014-01-01T01:01:01Z' - } + 'FinishedAt': '2014-01-01T01:01:01Z', + }, }, { 'Id': 'abbb', 'Name': 'two', 'State': { 'Running': True, - 'FinishedAt': '2014-01-01T01:01:01Z' - } - } + 'FinishedAt': '2014-01-01T01:01:01Z', + }, + }, ] mock_client.inspect_container.side_effect = iter(mock_containers) - docker_gc.cleanup_containers(mock_client, max_container_age, False) + docker_gc.cleanup_containers(mock_client, max_container_age, False, None) mock_client.remove_container.assert_called_once_with(container='abcd', v=True) +def test_filter_excluded_containers(): + mock_containers = [ + {'Labels': {'toot': ''}}, + {'Labels': {'too': 'lol'}}, + {'Labels': {'toots': 'lol'}}, + {'Labels': {'foo': 'bar'}}, + ] + result = docker_gc.filter_excluded_containers(mock_containers, None) + assert mock_containers == list(result) + exclude_labels = ['too', 'foo'] + result = docker_gc.filter_excluded_containers( + mock_containers, + exclude_labels, + ) + assert [mock_containers[0], mock_containers[2]] == list(result) + exclude_labels = ['too*=lol'] + result = docker_gc.filter_excluded_containers( + mock_containers, + exclude_labels, + ) + assert [mock_containers[0], mock_containers[3]] == list(result) + + def test_cleanup_images(mock_client, now): max_image_age = now mock_client.images.return_value = images = [ From b84b30052a3de20f86ee59f840d6fed7138734de Mon Sep 17 00:00:00 2001 From: Andy Tran Date: Tue, 20 Mar 2018 15:30:38 -0700 Subject: [PATCH 2/2] Refactor container exclude labels --- docker_custodian/docker_gc.py | 48 +++++++++++++++++++++++++++-------- tests/docker_gc_test.py | 23 +++++++++++++++-- 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/docker_custodian/docker_gc.py b/docker_custodian/docker_gc.py index 0b6faa3..4258ff5 100644 --- a/docker_custodian/docker_gc.py +++ b/docker_custodian/docker_gc.py @@ -13,6 +13,7 @@ import docker import docker.errors import requests.exceptions +from collections import namedtuple from docker_custodian.args import timedelta_type from docker.utils import kwargs_from_env @@ -22,6 +23,8 @@ log = logging.getLogger(__name__) # This seems to be something docker uses for a null/zero date YEAR_ZERO = "0001-01-01T00:00:00Z" +ExcludeLabel = namedtuple('ExcludeLabel', ['key', 'value']) + def cleanup_containers( client, @@ -59,8 +62,11 @@ def cleanup_containers( def filter_excluded_containers(containers, exclude_container_labels): + if not exclude_container_labels: + return containers + def include_container(container): - if exclude_container_labels and should_exclude_container_with_labels( + if should_exclude_container_with_labels( container, exclude_container_labels, ): @@ -70,22 +76,20 @@ def filter_excluded_containers(containers, exclude_container_labels): def should_exclude_container_with_labels(container, exclude_container_labels): - for exclude_container_label in exclude_container_labels: - split_exclude_label = exclude_container_label.split('=', 1) - if len(split_exclude_label) == 2: - exclude_key, exclude_value = split_exclude_label + for exclude_label in exclude_container_labels: + if exclude_label.value: matching_keys = fnmatch.filter( - container['Labels'].keys(), exclude_key + container['Labels'].keys(), + exclude_label.key, ) label_values_to_check = [ container['Labels'][matching_key] for matching_key in matching_keys ] - if fnmatch.filter(label_values_to_check, exclude_value): + if fnmatch.filter(label_values_to_check, exclude_label.value): return True else: - exclude_key = split_exclude_label[0] - if fnmatch.filter(container['Labels'].keys(), exclude_key): + if fnmatch.filter(container['Labels'].keys(), exclude_label.key): return True return False @@ -261,6 +265,24 @@ def build_exclude_set(image_tags, exclude_file): return exclude_set +def format_exclude_labels(exclude_label_args): + exclude_labels = [] + for exclude_label_arg in exclude_label_args: + split_exclude_label = exclude_label_arg.split('=', 1) + exclude_label_key = split_exclude_label[0] + if len(split_exclude_label) == 2: + exclude_label_value = split_exclude_label[1] + else: + exclude_label_value = None + exclude_labels.append( + ExcludeLabel( + key=exclude_label_key, + value=exclude_label_value, + ) + ) + return exclude_labels + + def main(): logging.basicConfig( level=logging.INFO, @@ -272,12 +294,16 @@ def main(): timeout=args.timeout, **kwargs_from_env()) + exclude_container_labels = format_exclude_labels( + args.exclude_container_label + ) + if args.max_container_age: cleanup_containers( client, args.max_container_age, args.dry_run, - args.exclude_container_label, + exclude_container_labels, ) if args.max_image_age: @@ -325,7 +351,7 @@ def get_args(args=None): "image tag per line.") parser.add_argument( '--exclude-container-label', - action='append', + action='append', type=str, default=[], help="Never remove containers with this label key or label key=value") return parser.parse_args(args=args) diff --git a/tests/docker_gc_test.py b/tests/docker_gc_test.py index fe47aca..e6a31e4 100644 --- a/tests/docker_gc_test.py +++ b/tests/docker_gc_test.py @@ -78,13 +78,18 @@ def test_filter_excluded_containers(): ] result = docker_gc.filter_excluded_containers(mock_containers, None) assert mock_containers == list(result) - exclude_labels = ['too', 'foo'] + exclude_labels = [ + docker_gc.ExcludeLabel(key='too', value=None), + docker_gc.ExcludeLabel(key='foo', value=None), + ] result = docker_gc.filter_excluded_containers( mock_containers, exclude_labels, ) assert [mock_containers[0], mock_containers[2]] == list(result) - exclude_labels = ['too*=lol'] + exclude_labels = [ + docker_gc.ExcludeLabel(key='too*', value='lol'), + ] result = docker_gc.filter_excluded_containers( mock_containers, exclude_labels, @@ -482,6 +487,19 @@ def test_build_exclude_set(): assert exclude_set == expected +def test_format_exclude_labels(): + exclude_label_args = [ + 'voo*', + 'doo=poo', + ] + expected = [ + docker_gc.ExcludeLabel(key='voo*', value=None), + docker_gc.ExcludeLabel(key='doo', value='poo'), + ] + exclude_labels = docker_gc.format_exclude_labels(exclude_label_args) + assert expected == exclude_labels + + def test_build_exclude_set_empty(): exclude_set = docker_gc.build_exclude_set(None, None) assert exclude_set == set() @@ -500,5 +518,6 @@ def test_main(mock_client): max_container_age=200, exclude_image=[], exclude_image_file=None, + exclude_container_label=[], ) docker_gc.main()