From 8135f73ee21366a5ee3faad018c9051a156c9f38 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Mon, 16 Jan 2023 09:10:35 +0100 Subject: [PATCH] refactor: use buildx for multiarch container builds (#459) --- .drone.jsonnet | 67 +++------ .drone.yml | 139 +++--------------- .../Dockerfile.amd64 => Dockerfile.multiarch | 0 docker/Dockerfile.arm64 | 26 ---- docker/manifest-quay.tmpl | 18 --- docker/manifest.tmpl | 18 --- dockertidy/autostop.py | 2 +- dockertidy/garbage_collector.py | 2 +- dockertidy/test/unit/test_autostop.py | 2 +- dockertidy/test/unit/test_garbagecollector.py | 2 +- 10 files changed, 45 insertions(+), 231 deletions(-) rename docker/Dockerfile.amd64 => Dockerfile.multiarch (100%) delete mode 100644 docker/Dockerfile.arm64 delete mode 100644 docker/manifest-quay.tmpl delete mode 100644 docker/manifest.tmpl diff --git a/.drone.jsonnet b/.drone.jsonnet index f253f19..6ec11ea 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -217,12 +217,12 @@ local PipelineBuildPackage = { }, }; -local PipelineBuildContainer(arch='amd64') = { +local PipelineBuildContainer = { kind: 'pipeline', - name: 'build-container-' + arch, + name: 'build-container', platform: { os: 'linux', - arch: arch, + arch: 'amd64', }, steps: [ { @@ -239,10 +239,13 @@ local PipelineBuildContainer(arch='amd64') = { image: 'thegeeklab/drone-docker-buildx:20', settings: { dry_run: true, - dockerfile: 'docker/Dockerfile.' + arch, + dockerfile: 'Dockerfile.multiarch', repo: 'thegeeklab/${DRONE_REPO_NAME}', - username: { from_secret: 'docker_username' }, - password: { from_secret: 'docker_password' }, + platforms: [ + 'linux/amd64', + 'linux/arm64', + ], + provenance: false, }, depends_on: ['build'], when: { @@ -254,11 +257,15 @@ local PipelineBuildContainer(arch='amd64') = { image: 'thegeeklab/drone-docker-buildx:20', settings: { auto_tag: true, - auto_tag_suffix: arch, - dockerfile: 'docker/Dockerfile.' + arch, + dockerfile: 'Dockerfile.multiarch', repo: 'thegeeklab/${DRONE_REPO_NAME}', username: { from_secret: 'docker_username' }, password: { from_secret: 'docker_password' }, + platforms: [ + 'linux/amd64', + 'linux/arm64', + ], + provenance: false, }, when: { ref: ['refs/heads/main', 'refs/tags/**'], @@ -270,12 +277,16 @@ local PipelineBuildContainer(arch='amd64') = { image: 'thegeeklab/drone-docker-buildx:20', settings: { auto_tag: true, - auto_tag_suffix: arch, - dockerfile: 'docker/Dockerfile.' + arch, + dockerfile: 'Dockerfile.multiarch', registry: 'quay.io', repo: 'quay.io/thegeeklab/${DRONE_REPO_NAME}', username: { from_secret: 'quay_username' }, password: { from_secret: 'quay_password' }, + platforms: [ + 'linux/amd64', + 'linux/arm64', + ], + provenance: false, }, when: { ref: ['refs/heads/main', 'refs/tags/**'], @@ -384,8 +395,7 @@ local PipelineDocs = { ], depends_on: [ 'build-package', - 'build-container-amd64', - 'build-container-arm64', + 'build-container', ], trigger: { ref: ['refs/heads/main', 'refs/tags/**', 'refs/pull/**'], @@ -400,37 +410,8 @@ local PipelineNotifications = { arch: 'amd64', }, steps: [ - { - image: 'plugins/manifest', - name: 'manifest-dockerhub', - settings: { - ignore_missing: true, - auto_tag: true, - username: { from_secret: 'docker_username' }, - password: { from_secret: 'docker_password' }, - spec: 'docker/manifest.tmpl', - }, - when: { - status: ['success'], - }, - }, - { - image: 'plugins/manifest', - name: 'manifest-quay', - settings: { - ignore_missing: true, - auto_tag: true, - username: { from_secret: 'quay_username' }, - password: { from_secret: 'quay_password' }, - spec: 'docker/manifest-quay.tmpl', - }, - when: { - status: ['success'], - }, - }, { name: 'pushrm-dockerhub', - pull: 'always', image: 'chko/docker-pushrm:1', environment: { DOCKER_PASS: { @@ -449,7 +430,6 @@ local PipelineNotifications = { }, { name: 'pushrm-quay', - pull: 'always', image: 'chko/docker-pushrm:1', environment: { APIKEY__QUAY_IO: { @@ -491,8 +471,7 @@ local PipelineNotifications = { PipelineTest, PipelineSecurity, PipelineBuildPackage, - PipelineBuildContainer(arch='amd64'), - PipelineBuildContainer(arch='arm64'), + PipelineBuildContainer, PipelineDocs, PipelineNotifications, ] diff --git a/.drone.yml b/.drone.yml index 6c5a438..23be34b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -247,7 +247,7 @@ depends_on: --- kind: pipeline -name: build-container-amd64 +name: build-container platform: os: linux @@ -264,13 +264,13 @@ steps: - name: dryrun image: thegeeklab/drone-docker-buildx:20 settings: - dockerfile: docker/Dockerfile.amd64 + dockerfile: Dockerfile.multiarch dry_run: true - password: - from_secret: docker_password + platforms: + - linux/amd64 + - linux/arm64 + provenance: false repo: thegeeklab/${DRONE_REPO_NAME} - username: - from_secret: docker_username when: ref: - refs/pull/** @@ -281,10 +281,13 @@ steps: image: thegeeklab/drone-docker-buildx:20 settings: auto_tag: true - auto_tag_suffix: amd64 - dockerfile: docker/Dockerfile.amd64 + dockerfile: Dockerfile.multiarch password: from_secret: docker_password + platforms: + - linux/amd64 + - linux/arm64 + provenance: false repo: thegeeklab/${DRONE_REPO_NAME} username: from_secret: docker_username @@ -299,88 +302,13 @@ steps: image: thegeeklab/drone-docker-buildx:20 settings: auto_tag: true - auto_tag_suffix: amd64 - dockerfile: docker/Dockerfile.amd64 - password: - from_secret: quay_password - registry: quay.io - repo: quay.io/thegeeklab/${DRONE_REPO_NAME} - username: - from_secret: quay_username - when: - ref: - - refs/heads/main - - refs/tags/** - depends_on: - - dryrun - -trigger: - ref: - - refs/heads/main - - refs/tags/** - - refs/pull/** - -depends_on: - - security - ---- -kind: pipeline -name: build-container-arm64 - -platform: - os: linux - arch: arm64 - -steps: - - name: build - image: python:3.11 - commands: - - git fetch -tq - - pip install poetry poetry-dynamic-versioning -qq - - poetry build - - - name: dryrun - image: thegeeklab/drone-docker-buildx:20 - settings: - dockerfile: docker/Dockerfile.arm64 - dry_run: true - password: - from_secret: docker_password - repo: thegeeklab/${DRONE_REPO_NAME} - username: - from_secret: docker_username - when: - ref: - - refs/pull/** - depends_on: - - build - - - name: publish-dockerhub - image: thegeeklab/drone-docker-buildx:20 - settings: - auto_tag: true - auto_tag_suffix: arm64 - dockerfile: docker/Dockerfile.arm64 - password: - from_secret: docker_password - repo: thegeeklab/${DRONE_REPO_NAME} - username: - from_secret: docker_username - when: - ref: - - refs/heads/main - - refs/tags/** - depends_on: - - dryrun - - - name: publish-quay - image: thegeeklab/drone-docker-buildx:20 - settings: - auto_tag: true - auto_tag_suffix: arm64 - dockerfile: docker/Dockerfile.arm64 + dockerfile: Dockerfile.multiarch password: from_secret: quay_password + platforms: + - linux/amd64 + - linux/arm64 + provenance: false registry: quay.io repo: quay.io/thegeeklab/${DRONE_REPO_NAME} username: @@ -484,8 +412,7 @@ trigger: depends_on: - build-package - - build-container-amd64 - - build-container-arm64 + - build-container --- kind: pipeline @@ -496,36 +423,7 @@ platform: arch: amd64 steps: - - name: manifest-dockerhub - image: plugins/manifest - settings: - auto_tag: true - ignore_missing: true - password: - from_secret: docker_password - spec: docker/manifest.tmpl - username: - from_secret: docker_username - when: - status: - - success - - - name: manifest-quay - image: plugins/manifest - settings: - auto_tag: true - ignore_missing: true - password: - from_secret: quay_password - spec: docker/manifest-quay.tmpl - username: - from_secret: quay_username - when: - status: - - success - - name: pushrm-dockerhub - pull: always image: chko/docker-pushrm:1 environment: DOCKER_PASS: @@ -540,7 +438,6 @@ steps: - success - name: pushrm-quay - pull: always image: chko/docker-pushrm:1 environment: APIKEY__QUAY_IO: @@ -581,6 +478,6 @@ depends_on: --- kind: signature -hmac: 737d9df344ce1a6f625be620e006a7adf6290e0402c0a1d742e066236c733a22 +hmac: d82784ee9ae352d39c3ac6b68caee218c41c15765f7d90ba8748931f75eeb258 ... diff --git a/docker/Dockerfile.amd64 b/Dockerfile.multiarch similarity index 100% rename from docker/Dockerfile.amd64 rename to Dockerfile.multiarch diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 deleted file mode 100644 index da66f2c..0000000 --- a/docker/Dockerfile.arm64 +++ /dev/null @@ -1,26 +0,0 @@ -FROM arm64v8/python:3.11-alpine@sha256:941d62a7bcc71f7d25b3fa14a8121a549a72afdc4c4e06a7636b77dca1228620 - -LABEL maintainer="Robert Kaussow " -LABEL org.opencontainers.image.authors="Robert Kaussow " -LABEL org.opencontainers.image.title="docker-tidy" -LABEL org.opencontainers.image.url="https://docker-tidy.geekdocs.de/" -LABEL org.opencontainers.image.source="https://github.com/thegeeklab/docker-tidy" -LABEL org.opencontainers.image.documentation="https://docker-tidy.geekdocs.de/" - -ENV PY_COLORS=1 -ENV TZ=UTC - -ADD dist/docker_tidy-*.whl / - -RUN apk --update add --virtual .build-deps build-base libffi-dev openssl-dev && \ - pip install --upgrade --no-cache-dir pip && \ - pip install --no-cache-dir $(find / -name "docker_tidy-*.whl") && \ - apk del .build-deps && \ - rm -f docker_tidy-*.whl && \ - rm -rf /var/cache/apk/* && \ - rm -rf /root/.cache/ && \ - rm -rf /tmp/* - -USER root -CMD [] -ENTRYPOINT ["/usr/local/bin/docker-tidy", "gc"] diff --git a/docker/manifest-quay.tmpl b/docker/manifest-quay.tmpl deleted file mode 100644 index 4202a36..0000000 --- a/docker/manifest-quay.tmpl +++ /dev/null @@ -1,18 +0,0 @@ -image: quay.io/thegeeklab/docker-tidy:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}} -{{#if build.tags}} -tags: -{{#each build.tags}} - - {{this}} -{{/each}} -{{/if}} -manifests: - - image: quay.io/thegeeklab/docker-tidy:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}amd64 - platform: - architecture: amd64 - os: linux - - - image: quay.io/thegeeklab/docker-tidy:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm64 - platform: - architecture: arm64 - os: linux - variant: v8 diff --git a/docker/manifest.tmpl b/docker/manifest.tmpl deleted file mode 100644 index b5cd2dc..0000000 --- a/docker/manifest.tmpl +++ /dev/null @@ -1,18 +0,0 @@ -image: thegeeklab/docker-tidy:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}} -{{#if build.tags}} -tags: -{{#each build.tags}} - - {{this}} -{{/each}} -{{/if}} -manifests: - - image: thegeeklab/docker-tidy:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}amd64 - platform: - architecture: amd64 - os: linux - - - image: thegeeklab/docker-tidy:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm64 - platform: - architecture: arm64 - os: linux - variant: v8 diff --git a/dockertidy/autostop.py b/dockertidy/autostop.py index 25396e3..6a74e67 100644 --- a/dockertidy/autostop.py +++ b/dockertidy/autostop.py @@ -2,10 +2,10 @@ """Stop long running docker iamges.""" import dateutil.parser +import docker import docker.errors import requests.exceptions -import docker from dockertidy.config import SingleConfig from dockertidy.logger import SingleLog from dockertidy.parser import timedelta diff --git a/dockertidy/garbage_collector.py b/dockertidy/garbage_collector.py index f45dd59..02409f6 100644 --- a/dockertidy/garbage_collector.py +++ b/dockertidy/garbage_collector.py @@ -5,10 +5,10 @@ import fnmatch from collections import namedtuple import dateutil.parser +import docker import docker.errors import requests.exceptions -import docker from dockertidy.config import SingleConfig from dockertidy.logger import SingleLog from dockertidy.parser import timedelta diff --git a/dockertidy/test/unit/test_autostop.py b/dockertidy/test/unit/test_autostop.py index 12a0d5b..d3fbe26 100644 --- a/dockertidy/test/unit/test_autostop.py +++ b/dockertidy/test/unit/test_autostop.py @@ -1,8 +1,8 @@ """Test Autostop class.""" +import docker import pytest -import docker from dockertidy import autostop pytest_plugins = [ diff --git a/dockertidy/test/unit/test_garbagecollector.py b/dockertidy/test/unit/test_garbagecollector.py index 1f2f39b..12296f3 100644 --- a/dockertidy/test/unit/test_garbagecollector.py +++ b/dockertidy/test/unit/test_garbagecollector.py @@ -1,9 +1,9 @@ """Test GarbageCollector class.""" +import docker import pytest import requests -import docker from dockertidy import garbage_collector pytest_plugins = [