From b4190363223cb1a5cf8e3a2597c9cb139b5d065f Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Mon, 4 Sep 2023 20:56:19 +0200 Subject: [PATCH] feat: add backoff retry to docker version to avoid transient failure (#19) --- .golangci.yml | 4 ++++ go.mod | 1 + go.sum | 2 ++ plugin/impl.go | 45 +++++++++++++++++++++++++++++++++++++-------- 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 7e7ed7d..3bdcdcc 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -95,3 +95,7 @@ linters-settings: gofumpt: extra-rules: true lang-version: "1.21" + ireturn: + allow: + - error + - backoff.BackOff diff --git a/go.mod b/go.mod index c5bc380..5264321 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/thegeeklab/wp-docker-buildx go 1.21 require ( + github.com/cenkalti/backoff v2.2.1+incompatible github.com/coreos/go-semver v0.3.1 github.com/rs/zerolog v1.30.0 github.com/thegeeklab/wp-plugin-go v1.0.1 diff --git a/go.sum b/go.sum index ac0aeb7..4300e99 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= diff --git a/plugin/impl.go b/plugin/impl.go index 90ae968..f0ac654 100644 --- a/plugin/impl.go +++ b/plugin/impl.go @@ -8,6 +8,7 @@ import ( "path/filepath" "time" + "github.com/cenkalti/backoff" "github.com/rs/zerolog/log" "github.com/thegeeklab/wp-plugin-go/types" "github.com/urfave/cli/v2" @@ -16,7 +17,12 @@ import ( var ErrTypeAssertionFailed = errors.New("type assertion failed") -const strictFilePerm = 0o600 +const ( + strictFilePerm = 0o600 + daemonBackoffMaxRetries = 3 + daemonBackoffInitialInterval = 2 * time.Second + daemonBackoffMultiplier = 3.5 +) //nolint:revive func (p *Plugin) run(ctx context.Context) error { @@ -149,16 +155,31 @@ func (p *Plugin) Execute() error { // add proxy build args addProxyBuildArgs(&p.Settings.Build) - var cmds []*execabs.Cmd - cmds = append(cmds, commandVersion()) // docker version - cmds = append(cmds, commandInfo()) // docker info - cmds = append(cmds, commandBuilder(p.Settings.Daemon)) - cmds = append(cmds, commandBuildx()) + versionCmd := commandVersion() // docker version - cmds = append(cmds, commandBuild(p.Settings.Build, p.Settings.Dryrun)) // docker build + versionCmd.Stdout = os.Stdout + versionCmd.Stderr = os.Stderr + trace(versionCmd) + + backoffOps := func() error { + return versionCmd.Run() + } + backoffLog := func(err error, delay time.Duration) { + log.Error().Msgf("failed to exec docker version: %v: retry in %s", err, delay.Truncate(time.Second)) + } + + if err := backoff.RetryNotify(backoffOps, newBackoff(daemonBackoffMaxRetries), backoffLog); err != nil { + return err + } + + var batchCmd []*execabs.Cmd + batchCmd = append(batchCmd, commandInfo()) // docker info + batchCmd = append(batchCmd, commandBuilder(p.Settings.Daemon)) + batchCmd = append(batchCmd, commandBuildx()) + batchCmd = append(batchCmd, commandBuild(p.Settings.Build, p.Settings.Dryrun)) // docker build // execute all commands in batch mode. - for _, cmd := range cmds { + for _, cmd := range batchCmd { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr trace(cmd) @@ -189,3 +210,11 @@ func (p *Plugin) FlagsFromContext() error { return nil } + +func newBackoff(maxRetries uint64) backoff.BackOff { + b := backoff.NewExponentialBackOff() + b.InitialInterval = daemonBackoffInitialInterval + b.Multiplier = daemonBackoffMultiplier + + return backoff.WithMaxRetries(b, maxRetries) +}