From cfcb46e1de87e691c5a376d62590de69fee907ac Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 30 Jun 2015 15:33:43 -0700 Subject: [PATCH] Adding project files --- .gitignore | 9 ++ .pre-commit-config.yaml | 13 ++ LICENSE | 202 +++++++++++++++++++++++ Makefile | 18 +++ debian/.gitignore | 8 + debian/changelog | 44 +++++ debian/compat | 1 + debian/control | 9 ++ debian/docker-custodian.links | 2 + debian/rules | 21 +++ docker_custodian/__about__.py | 4 + docker_custodian/__init__.py | 1 + docker_custodian/args.py | 19 +++ docker_custodian/docker_autostop.py | 101 ++++++++++++ docker_custodian/docker_gc.py | 182 +++++++++++++++++++++ requirements.txt | 9 ++ setup.py | 35 ++++ tests/__init__.py | 0 tests/args_test.py | 33 ++++ tests/conftest.py | 51 ++++++ tests/docker_autostop_test.py | 83 ++++++++++ tests/docker_gc_test.py | 239 ++++++++++++++++++++++++++++ tox.ini | 23 +++ 23 files changed, 1107 insertions(+) create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 debian/.gitignore create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/docker-custodian.links create mode 100755 debian/rules create mode 100644 docker_custodian/__about__.py create mode 100644 docker_custodian/__init__.py create mode 100644 docker_custodian/args.py create mode 100644 docker_custodian/docker_autostop.py create mode 100644 docker_custodian/docker_gc.py create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/args_test.py create mode 100644 tests/conftest.py create mode 100644 tests/docker_autostop_test.py create mode 100644 tests/docker_gc_test.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..28a584d --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +*.py? +.*.swp +.tox +dist +build/ +*.pyc +__pycache__ + +*.egg-info/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..513b174 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +- repo: git://github.com/pre-commit/pre-commit-hooks + sha: 'v0.4.2' + hooks: + - id: check-added-large-files + - id: check-docstring-first + - id: check-merge-conflict + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: flake8 + - id: name-tests-test + - id: requirements-txt-fixer + - id: trailing-whitespace diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a51bf47 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +.PHONY: all clean tag test + +PACKAGE_VERSION=$(shell python setup.py --version) + +all: test + +clean: + git clean -fdx -- debian + rm -f ./dist + find . -iname '*.pyc' -delete + +tag: + git tag v${PACKAGE_VERSION} + +test: + tox + +tests: test diff --git a/debian/.gitignore b/debian/.gitignore new file mode 100644 index 0000000..643487d --- /dev/null +++ b/debian/.gitignore @@ -0,0 +1,8 @@ +* +!.gitignore +!changelog +!compat +!control +!copyright +!rules +!docker-custodian.links diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..48d5d1e --- /dev/null +++ b/debian/changelog @@ -0,0 +1,44 @@ +docker-custodian (0.4.0) lucid; urgency=low + + * Renamed to docker-custodian + * Changed defaults of dcgc to not remove anything + + -- Daniel Nephin Mon, 29 Jun 2015 18:48:22 -0700 + +docker-custodian (0.3.3) lucid; urgency=low + + * Bug fixes for removing images by Id and with multiple tags + + -- Daniel Nephin Thu, 04 Jun 2015 13:24:14 -0700 + +docker-custodian (0.3.2) lucid; urgency=low + + * docker-custodian should now remove image names before trying to remove + by id, so that images tagged with more than one name are removed + correctly + + -- Daniel Nephin Tue, 02 Jun 2015 13:26:56 -0700 + +docker-custodian (0.3.1) lucid; urgency=low + + * Fix broken commands + + -- Daniel Nephin Mon, 09 Mar 2015 17:58:03 -0700 + +docker-custodian (0.3.0) lucid; urgency=low + + * Change age and time options to support pytimeparse formats + + -- Daniel Nephin Fri, 06 Mar 2015 13:30:36 -0800 + +docker-custodian (0.2.0) lucid; urgency=low + + * Add docker-autostop + + -- Daniel Nephin Wed, 28 Jan 2015 15:37:40 -0800 + +docker-custodian (0.1.0) lucid; urgency=low + + * Initial release + + -- Daniel Nephin Thu, 02 Oct 2014 11:13:43 -0700 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..7f8f011 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +7 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..04ecacd --- /dev/null +++ b/debian/control @@ -0,0 +1,9 @@ +Source: docker-custodian +Maintainer: Daniel Nephin +Build-Depends: + dh-virtualenv, + +Depends: python (>=2.6) +Package: docker-custodian +Architecture: any +Description: Remove old Docker containers and images that are no longer in use diff --git a/debian/docker-custodian.links b/debian/docker-custodian.links new file mode 100644 index 0000000..814e1fc --- /dev/null +++ b/debian/docker-custodian.links @@ -0,0 +1,2 @@ +usr/share/python/docker-custodian/bin/dcgc usr/bin/dcgc +usr/share/python/docker-custodian/bin/dcstop usr/bin/dcstop diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..f99ac62 --- /dev/null +++ b/debian/rules @@ -0,0 +1,21 @@ +#!/usr/bin/make -f +# -*- makefile -*- + +export DH_OPTIONS + +%: + dh $@ --with python-virtualenv + +override_dh_virtualenv: + dh_virtualenv + +# do not call `make clean` as part of packaging +override_dh_auto_clean: + true + +# do not call `make` as part of packaging +override_dh_auto_build: + true + +override_dh_auto_test: + tox -e py27 diff --git a/docker_custodian/__about__.py b/docker_custodian/__about__.py new file mode 100644 index 0000000..1b3d3fd --- /dev/null +++ b/docker_custodian/__about__.py @@ -0,0 +1,4 @@ +# -*- coding: utf8 -*- + +__version_info__ = (0, 4, 0) +__version__ = '%d.%d.%d' % __version_info__ diff --git a/docker_custodian/__init__.py b/docker_custodian/__init__.py new file mode 100644 index 0000000..d2aca59 --- /dev/null +++ b/docker_custodian/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf8 -*- diff --git a/docker_custodian/args.py b/docker_custodian/args.py new file mode 100644 index 0000000..674266d --- /dev/null +++ b/docker_custodian/args.py @@ -0,0 +1,19 @@ +import datetime + +from dateutil import tz +from pytimeparse import timeparse + + +def timedelta_type(value): + """Retrun the :class:`datetime.datetime.DateTime` for a time in the past. + + :param value: a string containing a time format supported by :mod:`pytimeparse` + """ + if value is None: + return None + return datetime_seconds_ago(timeparse.timeparse(value)) + + +def datetime_seconds_ago(seconds): + now = datetime.datetime.now(tz.tzutc()) + return now - datetime.timedelta(seconds=seconds) diff --git a/docker_custodian/docker_autostop.py b/docker_custodian/docker_autostop.py new file mode 100644 index 0000000..0e56536 --- /dev/null +++ b/docker_custodian/docker_autostop.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +""" +Stop docker container that have been running longer than the max_run_time and +match some prefix. +""" +import argparse +import logging +import sys + +import dateutil.parser +import docker +import docker.errors +import requests.exceptions + +from docker_custodian.args import timedelta_type + + +log = logging.getLogger(__name__) + + +def stop_containers(client, max_run_time, matcher, dry_run): + for container_summary in client.containers(): + container = client.inspect_container(container_summary['Id']) + name = container['Name'].lstrip('/') + if ( + matcher(name) and + has_been_running_since(container, max_run_time) + ): + + log.info("Stopping container %s %s: running since %s" % ( + container['Id'][:16], + name, + container['State']['StartedAt'])) + + if not dry_run: + stop_container(client, container['Id']) + + +def stop_container(client, id): + try: + client.stop(id) + except requests.exceptions.Timeout as e: + log.warn("Failed to stop container %s: %s" % (id, e)) + except docker.errors.APIError as ae: + log.warn("Error stopping %s: %s" % (id, ae)) + + +def build_container_matcher(prefixes): + def matcher(name): + return any(name.startswith(prefix) for prefix in prefixes) + return matcher + + +def has_been_running_since(container, min_time): + started_at = container.get('State', {}).get('StartedAt') + if not started_at: + return False + + return dateutil.parser.parse(started_at) <= min_time + + +def main(): + logging.basicConfig( + level=logging.INFO, + format="%(message)s", + stream=sys.stdout) + + opts = get_opts() + client = docker.Client(timeout=opts.timeout) + + matcher = build_container_matcher(opts.prefix) + stop_containers(client, opts.max_run_time, matcher, opts.dry_run) + + +def get_opts(args=None): + parser = argparse.ArgumentParser() + parser.add_argument( + '--max-run-time', + type=timedelta_type, + help="Maximum time a container is allows to run. Time may be specified " + "in any pytimeparse supported format.") + parser.add_argument( + '--prefix', action="append", default=[], + help="Only stop containers which match one of the " + "prefix.") + parser.add_argument( + '--dry-run', action="store_true", + help="Only log actions, don't stop anything.") + parser.add_argument( + '-t', '--timeout', type=int, default=60, + help="HTTP timeout in seconds for making docker API calls.") + opts = parser.parse_args(args=args) + + if not opts.prefix: + parser.error("Running with no --prefix will match nothing.") + + return opts + + +if __name__ == "__main__": + main() diff --git a/docker_custodian/docker_gc.py b/docker_custodian/docker_gc.py new file mode 100644 index 0000000..e35406f --- /dev/null +++ b/docker_custodian/docker_gc.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python +""" +Remove old docker containers and images that are no longer in use. + +""" +import argparse +import logging +import sys + +import dateutil.parser +import docker +import docker.errors +import requests.exceptions + +from docker_custodian.args import timedelta_type + +log = logging.getLogger(__name__) + + +# This seems to be something docker uses for a null/zero date +YEAR_ZERO = "0001-01-01T00:00:00Z" + + +def cleanup_containers(client, max_container_age, dry_run): + all_containers = get_all_containers(client) + + for container_summary in reversed(all_containers): + container = api_call(client.inspect_container, container_summary['Id']) + if not container or not is_container_old(container, max_container_age): + continue + + log.info("Removing container %s %s %s" % ( + container['Id'][:16], + container.get('Name', '').lstrip('/'), + container['State']['FinishedAt'])) + + if not dry_run: + api_call(client.remove_container, container['Id']) + + +def is_container_old(container, min_date): + state = container.get('State', {}) + if state.get('Running'): + return False + + if state.get('Ghost'): + return True + + # Container was created, but never used + if (state.get('FinishedAt') == YEAR_ZERO and + dateutil.parser.parse(container['Created']) < min_date): + return True + + return dateutil.parser.parse(state['FinishedAt']) < min_date + + +def get_all_containers(client): + log.info("Getting all continers") + containers = client.containers(all=True) + log.info("Found %s containers", len(containers)) + return containers + + +def get_all_images(client): + log.info("Getting all images") + images = client.images() + log.info("Found %s images", len(images)) + return images + + +def cleanup_images(client, max_image_age, dry_run): + # re-fetch container list so that we don't include removed containers + image_tags_in_use = set( + container['Image'] for container in get_all_containers(client)) + + images = filter_images_in_use(get_all_images(client), image_tags_in_use) + + for image_summary in reversed(images): + remove_image(client, image_summary, max_image_age, dry_run) + + +def filter_images_in_use(images, image_tags_in_use): + def get_tag_set(image_summary): + image_tags = image_summary.get('RepoTags') + if no_image_tags(image_tags): + # The repr of the image Id used by client.containers() + return set(['%s:latest' % image_summary['Id'][:12]]) + return set(image_tags) + + def image_is_in_use(image_summary): + return not get_tag_set(image_summary) & image_tags_in_use + + return list(filter(image_is_in_use, images)) + + +def is_image_old(image, min_date): + return dateutil.parser.parse(image['Created']) < min_date + + +def no_image_tags(image_tags): + return not image_tags or image_tags == [':'] + + +def remove_image(client, image_summary, min_date, dry_run): + image = api_call(client.inspect_image, image_summary['Id']) + if not image or not is_image_old(image, min_date): + return + + log.info("Removing image %s" % format_image(image, image_summary)) + if dry_run: + return + + image_tags = image_summary.get('RepoTags') + # If there are no tags, remove the id + if no_image_tags(image_tags): + api_call(client.remove_image, image_summary['Id']) + return + + # Remove any repository tags so we don't hit 409 Conflict + for image_tag in image_tags: + api_call(client.remove_image, image_tag) + + +def api_call(func, id): + try: + return func(id) + except requests.exceptions.Timeout as e: + log.warn("Failed to call %s %s %s" % (func.__name__, id, e)) + except docker.errors.APIError as ae: + log.warn("Error calling %s %s %s" % (func.__name__, id, ae)) + + +def format_image(image, image_summary): + def get_tags(): + tags = image_summary.get('RepoTags') + if not tags or tags == [':']: + return '' + return ', '.join(tags) + + return "%s %s" % (image['Id'][:16], get_tags()) + + +def main(): + logging.basicConfig( + level=logging.INFO, + format="%(message)s", + stream=sys.stdout) + + opts = get_opts() + client = docker.Client(timeout=opts.timeout) + + if opts.max_container_age: + cleanup_containers(client, opts.max_container_age, opts.dry_run) + if opts.max_image_age: + cleanup_images(client, opts.max_image_age, opts.dry_run) + + +def get_opts(args=None): + parser = argparse.ArgumentParser() + parser.add_argument( + '--max-container-age', + type=timedelta_type, + help="Maximum age for a container. Containers older than this age " + "will be removed. Age can be specified in any pytimeparse " + "supported format.") + parser.add_argument( + '--max-image-age', + type=timedelta_type, + help="Maxium age for an image. Images older than this age will be " + "removed. Age can be specified in any pytimeparse supported " + "format.") + parser.add_argument( + '--dry-run', action="store_true", + help="Only log actions, don't remove anything.") + parser.add_argument( + '-t', '--timeout', type=int, default=60, + help="HTTP timeout in seconds for making docker API calls.") + return parser.parse_args(args=args) + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8c90b77 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +argparse==1.3.0 +backports.ssl-match-hostname==3.4.0.2 +docker-py==0.6.0 +future==0.14.3 +python-dateutil==2.4.0 +pytimeparse==1.1.2 +requests==2.5.1 +six==1.9.0 +websocket-client==0.23.0 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..b743652 --- /dev/null +++ b/setup.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +from setuptools import setup, find_packages +import sys + +from docker_custodian.__about__ import __version__ + + +install_requires = [ + 'python-dateutil', + 'docker-py >= 0.5', + 'pytimeparse', +] + +if sys.version_info < (2, 7): + install_requires.append('argparse') + + +setup( + name='docker_custodian', + version=__version__, + provides=['docker_custodian'], + author='Daniel Nephin', + author_email='dnephin@yelp.com', + description='Keep docker hosts clean and tidy.', + packages=find_packages(exclude=['tests*']), + include_package_data=True, + install_requires=install_requires, + license="Apache License 2.0", + entry_points={ + 'console_scripts': [ + 'dcstop = docker_custodian.docker_autostop:main', + 'dcgc = docker_custodian.docker_gc:main', + ], + }, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/args_test.py b/tests/args_test.py new file mode 100644 index 0000000..378bd81 --- /dev/null +++ b/tests/args_test.py @@ -0,0 +1,33 @@ +import datetime +try: + from unittest import mock +except ImportError: + import mock + +from dateutil import tz + +from docker_custodian import args + + +def test_datetime_seconds_ago(now): + expected = datetime.datetime(2014, 1, 15, 10, 10, tzinfo=tz.tzutc()) + with mock.patch( + 'docker_custodian.args.datetime.datetime', + autospec=True, + ) as mock_datetime: + mock_datetime.now.return_value = now + assert args.datetime_seconds_ago(24 * 60 * 60 * 5) == expected + + +def test_timedelta_type_none(): + assert args.timedelta_type(None) is None + + +def test_timedelta_type(now): + expected = datetime.datetime(2014, 1, 15, 10, 10, tzinfo=tz.tzutc()) + with mock.patch( + 'docker_custodian.args.datetime.datetime', + autospec=True, + ) as mock_datetime: + mock_datetime.now.return_value = now + assert args.timedelta_type('5 days') == expected diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..075d13c --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,51 @@ +import datetime + +from dateutil import tz +import docker +try: + from unittest import mock +except ImportError: + import mock +import pytest + + +@pytest.fixture +def container(): + return { + 'Id': 'abcdabcdabcdabcd', + 'Created': '2013-12-20T17:00:00Z', + 'Name': '/container_name', + 'State': { + 'Running': False, + 'FinishedAt': '2014-01-01T17:30:00Z', + 'StartedAt': '2014-01-01T17:01:00Z', + } + } + + +@pytest.fixture +def image(): + return { + 'Id': 'abcdabcdabcdabcd', + 'Created': '2014-01-20T05:00:00Z', + } + + +@pytest.fixture +def now(): + return datetime.datetime(2014, 1, 20, 10, 10, tzinfo=tz.tzutc()) + + +@pytest.fixture +def earlier_time(): + return datetime.datetime(2014, 1, 1, 0, 0, tzinfo=tz.tzutc()) + + +@pytest.fixture +def later_time(): + return datetime.datetime(2014, 1, 20, 0, 10, tzinfo=tz.tzutc()) + + +@pytest.fixture +def mock_client(): + return mock.create_autospec(docker.Client) diff --git a/tests/docker_autostop_test.py b/tests/docker_autostop_test.py new file mode 100644 index 0000000..83b428a --- /dev/null +++ b/tests/docker_autostop_test.py @@ -0,0 +1,83 @@ +try: + from unittest import mock +except ImportError: + import mock + +from docker_custodian.docker_autostop import ( + build_container_matcher, + get_opts, + has_been_running_since, + main, + stop_container, + stop_containers, +) + + +def test_stop_containers(mock_client, container, now): + matcher = mock.Mock() + mock_client.containers.return_value = [container] + mock_client.inspect_container.return_value = container + + stop_containers(mock_client, now, matcher, False) + matcher.assert_called_once_with('container_name') + mock_client.stop.assert_called_once_with(container['Id']) + + +def test_stop_container(mock_client): + id = 'asdb' + stop_container(mock_client, id) + mock_client.stop.assert_called_once_with(id) + + +def test_build_container_matcher(): + prefixes = ['one_', 'two_'] + matcher = build_container_matcher(prefixes) + + assert matcher('one_container') + assert matcher('two_container') + assert not matcher('three_container') + assert not matcher('one') + + +def test_has_been_running_since_true(container, later_time): + assert has_been_running_since(container, later_time) + + +def test_has_been_running_since_false(container, earlier_time): + assert not has_been_running_since(container, earlier_time) + + +@mock.patch('docker_custodian.docker_autostop.build_container_matcher', + autospec=True) +@mock.patch('docker_custodian.docker_autostop.stop_containers', + autospec=True) +@mock.patch('docker_custodian.docker_autostop.get_opts', + autospec=True) +def test_main(mock_get_opts, mock_stop_containers, mock_build_matcher): + main() + mock_get_opts.assert_called_once_with() + mock_build_matcher.assert_called_once_with( + mock_get_opts.return_value.prefix) + mock_stop_containers.assert_called_once_with( + mock.ANY, + mock_get_opts.return_value.max_run_time, + mock_build_matcher.return_value, + mock_get_opts.return_value.dry_run) + + +def test_get_opts_with_defaults(): + opts = get_opts(args=['--prefix', 'one', '--prefix', 'two']) + assert opts.timeout == 60 + assert opts.dry_run is False + assert opts.prefix == ['one', 'two'] + assert opts.max_run_time is None + + +def test_get_opts_with_args(now): + with mock.patch( + 'docker_custodian.docker_autostop.timedelta_type', + autospec=True + ) as mock_timedelta_type: + opts = get_opts(args=['--prefix', 'one', '--max-run-time', '24h']) + assert opts.max_run_time == mock_timedelta_type.return_value + mock_timedelta_type.assert_called_once_with('24h') diff --git a/tests/docker_gc_test.py b/tests/docker_gc_test.py new file mode 100644 index 0000000..8dc933c --- /dev/null +++ b/tests/docker_gc_test.py @@ -0,0 +1,239 @@ +import docker.errors +try: + from unittest import mock +except ImportError: + import mock +import requests.exceptions + +from docker_custodian import docker_gc + + +class TestIsOldContainer(object): + + def test_is_running(self, container, now): + container['State']['Running'] = True + assert not docker_gc.is_container_old(container, now) + + def test_is_ghost(self, container, now): + container['State']['Ghost'] = True + assert docker_gc.is_container_old(container, now) + + def test_old_never_run(self, container, now): + container['State']['FinishedAt'] = docker_gc.YEAR_ZERO + assert docker_gc.is_container_old(container, now) + + def test_old_stopped(self, container, now): + assert docker_gc.is_container_old(container, now) + + def test_not_old(self, container, now): + container['State']['FinishedAt'] = '2014-01-21T00:00:00Z' + assert not docker_gc.is_container_old(container, now) + + +def test_cleanup_containers(mock_client, now): + max_container_age = now + mock_client.containers.return_value = [ + dict(Id='abcd'), + dict(Id='abbb'), + ] + mock_containers = [ + dict( + Id='abcd', + Name='one', + State=dict(Running=False, FinishedAt='2014-01-01T01:01:01Z')), + dict( + Id='abbb', + Name='two', + State=dict(Running=True, FinishedAt='2014-01-01T01:01:01Z')) + ] + mock_client.inspect_container.side_effect = iter(mock_containers) + docker_gc.cleanup_containers(mock_client, max_container_age, False) + mock_client.remove_container.assert_called_once_with('abcd') + + +def test_cleanup_images(mock_client, now): + max_image_age = now + mock_client.images.return_value = images = [ + dict(Id='abcd'), + dict(Id='abbb'), + ] + mock_images = [ + dict(Id='abcd', Created='2014-01-01T01:01:01Z'), + dict(Id='abbb', Created='2014-01-01T01:01:01Z'), + ] + mock_client.inspect_image.side_effect = iter(mock_images) + + docker_gc.cleanup_images(mock_client, max_image_age, False) + assert mock_client.remove_image.mock_calls == [ + mock.call(image['Id']) for image in reversed(images) + ] + + +def test_filter_images_in_use(): + image_tags_in_use = set([ + 'user/one:latest', + 'user/foo:latest', + 'other:12345', + '2471708c19be:latest', + ]) + images = [ + dict(RepoTags=[':'], Id='2471708c19beabababab'), + dict(RepoTags=[':'], Id='babababababaabababab'), + dict(RepoTags=['user/one:latest', 'user/one:abcd']), + dict(RepoTags=['other:abcda']), + dict(RepoTags=['other:12345']), + dict(RepoTags=['new_image:latest', 'new_image:123']), + ] + expected = [ + dict(RepoTags=[':'], Id='babababababaabababab'), + dict(RepoTags=['other:abcda']), + dict(RepoTags=['new_image:latest', 'new_image:123']), + ] + actual = docker_gc.filter_images_in_use(images, image_tags_in_use) + assert actual == expected + + +def test_is_image_old(image, now): + assert docker_gc.is_image_old(image, now) + + +def test_is_image_old_false(image, later_time): + assert not docker_gc.is_image_old(image, later_time) + + +def test_remove_image_no_tags(mock_client, image, now): + image_id = 'abcd' + image_summary = dict(Id=image_id) + mock_client.inspect_image.return_value = image + docker_gc.remove_image(mock_client, image_summary, now, False) + + mock_client.remove_image.assert_called_once_with(image_id) + + +def test_remove_image_new_image_not_removed(mock_client, image, later_time): + image_id = 'abcd' + image_summary = dict(Id=image_id) + mock_client.inspect_image.return_value = image + docker_gc.remove_image(mock_client, image_summary, later_time, False) + + assert not mock_client.remove_image.mock_calls + + +def test_remove_image_with_tags(mock_client, image, now): + image_id = 'abcd' + repo_tags = ['user/one:latest', 'user/one:12345'] + image_summary = dict(Id=image_id, RepoTags=repo_tags) + mock_client.inspect_image.return_value = image + docker_gc.remove_image(mock_client, image_summary, now, False) + + assert mock_client.remove_image.mock_calls == [ + mock.call(tag) for tag in repo_tags + ] + + +def test_api_call_success(): + func = mock.Mock() + id = "abcd" + result = docker_gc.api_call(func, id) + func.assert_called_once_with(id) + assert result == func.return_value + + +def test_api_call_with_timeout(): + func = mock.Mock( + side_effect=requests.exceptions.ReadTimeout("msg"), + __name__="remove_image") + id = "abcd" + + with mock.patch( + 'docker_custodian.docker_gc.log', + autospec=True) as mock_log: + docker_gc.api_call(func, id) + + func.assert_called_once_with(id) + mock_log.warn.assert_called_once_with('Failed to call remove_image abcd msg') + + +def test_api_call_with_api_error(): + func = mock.Mock( + side_effect=docker.errors.APIError( + "Ooops", + mock.Mock(status_code=409, reason="Conflict"), + explanation="failed"), + __name__="remove_image") + id = "abcd" + + with mock.patch( + 'docker_custodian.docker_gc.log', + autospec=True) as mock_log: + docker_gc.api_call(func, id) + + func.assert_called_once_with(id) + mock_log.warn.assert_called_once_with( + 'Error calling remove_image abcd ' + '409 Client Error: Conflict ("failed")') + + +def days_as_seconds(num): + return num * 60 * 60 * 24 + + +def test_get_opts_with_defaults(): + opts = docker_gc.get_opts(args=[]) + assert opts.timeout == 60 + assert opts.dry_run is False + assert opts.max_container_age is None + assert opts.max_image_age is None + + +def test_get_opts_with_args(): + with mock.patch( + 'docker_custodian.docker_gc.timedelta_type', + autospec=True + ) as mock_timedelta_type: + opts = docker_gc.get_opts(args=[ + '--max-image-age', '30 days', + '--max-container-age', '3d', + ]) + assert mock_timedelta_type.mock_calls == [ + mock.call('30 days'), + mock.call('3d'), + ] + assert opts.max_container_age == mock_timedelta_type.return_value + assert opts.max_image_age == mock_timedelta_type.return_value + + +def test_get_all_containers(mock_client): + count = 10 + mock_client.containers.return_value = [mock.Mock() for _ in range(count)] + with mock.patch('docker_custodian.docker_gc.log', + autospec=True) as mock_log: + containers = docker_gc.get_all_containers(mock_client) + assert containers == mock_client.containers.return_value + mock_client.containers.assert_called_once_with(all=True) + mock_log.info.assert_called_with("Found %s containers", count) + + +def test_get_all_images(mock_client): + count = 7 + mock_client.images.return_value = [mock.Mock() for _ in range(count)] + with mock.patch('docker_custodian.docker_gc.log', + autospec=True) as mock_log: + images = docker_gc.get_all_images(mock_client) + assert images == mock_client.images.return_value + mock_log.info.assert_called_with("Found %s images", count) + + +def test_main(mock_client): + with mock.patch( + 'docker_custodian.docker_gc.docker.Client', + return_value=mock_client): + + with mock.patch( + 'docker_custodian.docker_gc.get_opts', + autospec=True) as mock_get_opts: + mock_get_opts.return_value = mock.Mock( + max_image_age=100, + max_container_age=200, + ) + docker_gc.main() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..fcb7048 --- /dev/null +++ b/tox.ini @@ -0,0 +1,23 @@ +[tox] +envlist = py26,py27,py33,py34 + +[testenv] +deps = + -rrequirements.txt + flake8 + pytest + mock +commands = + py.test {posargs:tests} + flake8 . + +[testenv:install-hooks] +deps = + pre-commit +commands = + pre-commit install + +[flake8] +ignore = +exclude = .git/*,.tox/*,debian/* +max_line_length = 85