From 4023d446745e1926fad3d5c29079780501126e31 Mon Sep 17 00:00:00 2001 From: Martin Honermeyer Date: Sat, 2 Jan 2016 18:02:08 +0100 Subject: [PATCH 1/5] Use /drone/docker for the Docker daemon's graph directory This allows caching the whole Docker graph with the cache plugin. (Caching works efficiently with the overlay storage driver at least. Probably shouldn't be used with other storage drivers.) --- DOCS.md | 32 +++++++++++++++++++++++++++++--- main.go | 2 +- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/DOCS.md b/DOCS.md index 8c549d2..3049624 100644 --- a/DOCS.md +++ b/DOCS.md @@ -77,10 +77,36 @@ publish: build_args: - HTTP_PROXY=http://yourproxy.com ``` - -## Layer Caching -The Drone build environment is, by default, ephemeral meaning that you layers are not saved between builds. The below example combines Drone's caching feature and Docker's `save` and `load` capabilities to cache and restore image layers between builds: +## Caching + +The Drone build environment is, by default, ephemeral meaning that you layers are not saved between builds. There are two methods for caching layers. + +### Graph directory caching + +This is the preferred method when using the `overlay` storage driver. Just use Drone's caching feature to backup and restore the directory `/drone/docker`, as shown in the following example: + +```yaml +publish: + docker: + username: kevinbacon + password: pa55word + email: kevin.bacon@mail.com + repo: foo/bar + tag: + - latest + - "1.0.1" + +cache: + mount: + - /drone/docker +``` + +NOTE: This probably won't work correctly with the `btrfs` driver, and it will be very inefficient with the `devicemapper` driver. Please make sure to use the `overlay` storage driver with this method. + +### Layer Caching + +The below example combines Drone's caching feature and Docker's `save` and `load` capabilities to cache and restore image layers between builds: ```yaml publish: diff --git a/main.go b/main.go index 3e22df8..3abc89e 100644 --- a/main.go +++ b/main.go @@ -93,7 +93,7 @@ func main() { } go func() { - args := []string{"daemon"} + args := []string{"daemon", "-g", "/drone/docker"} if len(vargs.Storage) != 0 { args = append(args, "-s", vargs.Storage) From 2bf7ebf219fe9bba4bf4f6418e8c17601a588c88 Mon Sep 17 00:00:00 2001 From: Martin Honermeyer Date: Sat, 2 Jan 2016 20:01:59 +0100 Subject: [PATCH 2/5] Clean out untagged Docker images after build --- main.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/main.go b/main.go index 3abc89e..8b2426d 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "fmt" "io/ioutil" "os" @@ -228,6 +229,29 @@ func main() { } } + // Remove untagged images, if any + var outbuf bytes.Buffer + cmd = exec.Command("sh", "-c", "docker images | grep '^' | awk '{print $3}'") + cmd.Stdout = &outbuf + cmd.Stderr = os.Stderr + trace(cmd) + err = cmd.Run() + if err != nil { + os.Exit(1) + } + + if outbuf.Len() > 0 { + images := strings.Split(strings.TrimSpace(outbuf.String()), "\n") + cmd = exec.Command("docker", append([]string{"rmi"}, images...)...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + trace(cmd) + err := cmd.Run() + if err != nil { + os.Exit(1) + } + } + // Save to tarred image repository if len(vargs.Save.File) != 0 { // if the destination directory does not exist, create it From 2ab987b917c07957f49f6d4adf30eb409b61f601 Mon Sep 17 00:00:00 2001 From: Martin Honermeyer Date: Sun, 3 Jan 2016 14:17:15 +0100 Subject: [PATCH 3/5] Use "dangling" filter instead of grepping when cleaning up images --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 8b2426d..b4fb348 100644 --- a/main.go +++ b/main.go @@ -231,7 +231,7 @@ func main() { // Remove untagged images, if any var outbuf bytes.Buffer - cmd = exec.Command("sh", "-c", "docker images | grep '^' | awk '{print $3}'") + cmd = exec.Command("docker", "images", "-q", "-f", "dangling=true") cmd.Stdout = &outbuf cmd.Stderr = os.Stderr trace(cmd) From b11030ead33a6a533f1614b85728a6aafe227f72 Mon Sep 17 00:00:00 2001 From: Martin Honermeyer Date: Sun, 3 Jan 2016 16:55:01 +0100 Subject: [PATCH 4/5] Adjust wording in caching docs --- DOCS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCS.md b/DOCS.md index 3049624..e6f4ace 100644 --- a/DOCS.md +++ b/DOCS.md @@ -80,7 +80,7 @@ publish: ## Caching -The Drone build environment is, by default, ephemeral meaning that you layers are not saved between builds. There are two methods for caching layers. +The Drone build environment is, by default, ephemeral meaning that you layers are not saved between builds. There are two methods for caching your layers. ### Graph directory caching From f2316d74bcf51a7a57c823f90e364e7918ef94c9 Mon Sep 17 00:00:00 2001 From: Martin Honermeyer Date: Wed, 20 Jan 2016 00:19:44 +0100 Subject: [PATCH 5/5] Adjust docs to state AUFS can be used with graph caching as well --- DOCS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DOCS.md b/DOCS.md index e6f4ace..8d8d34b 100644 --- a/DOCS.md +++ b/DOCS.md @@ -84,7 +84,7 @@ The Drone build environment is, by default, ephemeral meaning that you layers ar ### Graph directory caching -This is the preferred method when using the `overlay` storage driver. Just use Drone's caching feature to backup and restore the directory `/drone/docker`, as shown in the following example: +This is the preferred method when using the `overlay` or `aufs` storage drivers. Just use Drone's caching feature to backup and restore the directory `/drone/docker`, as shown in the following example: ```yaml publish: @@ -102,7 +102,7 @@ cache: - /drone/docker ``` -NOTE: This probably won't work correctly with the `btrfs` driver, and it will be very inefficient with the `devicemapper` driver. Please make sure to use the `overlay` storage driver with this method. +NOTE: This probably won't work correctly with the `btrfs` driver, and it will be very inefficient with the `devicemapper` driver. Please make sure to use the `overlay` or `aufs` storage driver with this method. ### Layer Caching