diff --git a/.drone.yml b/.drone.yml index 1ecaf75..50854eb 100644 --- a/.drone.yml +++ b/.drone.yml @@ -8,7 +8,7 @@ platform: steps: - name: deps - image: golang:1.19 + image: golang:1.20 commands: - make deps volumes: @@ -16,7 +16,7 @@ steps: path: /go - name: lint - image: golang:1.19 + image: golang:1.20 commands: - make lint volumes: @@ -24,7 +24,7 @@ steps: path: /go - name: test - image: golang:1.19 + image: golang:1.20 commands: - make test volumes: @@ -51,7 +51,7 @@ platform: steps: - name: build - image: techknowlogick/xgo:go-1.19.x + image: techknowlogick/xgo:go-1.20.x commands: - ln -s /drone/src /source - make release @@ -292,6 +292,6 @@ depends_on: --- kind: signature -hmac: 944cf1fa4e35a0f1a4634335ec63ed97dd0f9059d05311fd2211595a7f7626b7 +hmac: 915f60726a0be195ae835611939da462c8ce7c6c0ae17f0d6de00750197ba751 ... diff --git a/.golangci.yml b/.golangci.yml index 7bb18ea..2faa799 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,25 +1,92 @@ linters: - enable: - - gosimple - - deadcode - - typecheck - - govet - - errcheck - - staticcheck - - unused - - structcheck - - varcheck - - dupl - - gofmt - - misspell - - gocritic - - bidichk - - ineffassign - - revive - - gofumpt - - depguard enable-all: false disable-all: true + enable: + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - typecheck + - unused + - asasalint + - asciicheck + - bidichk + - bodyclose + - containedctx + - contextcheck + - decorder + - depguard + - dogsled + - dupl + - dupword + - durationcheck + - errchkjson + - errname + - errorlint + - execinquery + - exhaustive + - exportloopref + - forcetypeassert + - ginkgolinter + - gocheckcompilerdirectives + - gochecknoglobals + - gochecknoinits + - gocognit + - goconst + - gocritic + - gocyclo + - godot + - godox + - goerr113 + - gofmt + - gofumpt + - goheader + - goimports + - gomnd + - gomoddirectives + - gomodguard + - goprintffuncname + - gosec + - grouper + - importas + - interfacebloat + - ireturn + - lll + - loggercheck + - maintidx + - makezero + - misspell + - musttag + - nakedret + - nestif + - nilerr + - nilnil + - nlreturn + - noctx + - nolintlint + - nonamedreturns + - nosprintfhostport + - prealloc + - predeclared + - promlinter + - reassign + - revive + # - rowserrcheck + # - sqlclosecheck + # - structcheck + - stylecheck + - tagliatelle + - tenv + - testableexamples + - thelper + - tparallel + - unconvert + - unparam + - usestdlibvars + # - wastedassign + - whitespace + - wsl fast: false run: @@ -28,4 +95,4 @@ run: linters-settings: gofumpt: extra-rules: true - lang-version: "1.18" + lang-version: "1.20" diff --git a/Makefile b/Makefile index 87058d6..15b56c4 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/cmd/golangci-lint@$(G XGO_PACKAGE ?= src.techknowlogick.com/xgo@latest GENERATE ?= -XGO_VERSION := go-1.19.x +XGO_VERSION := go-1.20.x XGO_TARGETS ?= linux/amd64,linux/arm-6,linux/arm-7,linux/arm64 TARGETOS ?= linux diff --git a/cmd/drone-s3-sync/config.go b/cmd/drone-s3-sync/config.go index ecdb7d5..ed62955 100644 --- a/cmd/drone-s3-sync/config.go +++ b/cmd/drone-s3-sync/config.go @@ -134,8 +134,9 @@ func settingsFlags(settings *plugin.Settings, category string) []cli.Flag { Category: category, }, &cli.IntFlag{ - Name: "max-concurrency", - Usage: "customize number concurrent files to process", + Name: "max-concurrency", + Usage: "customize number concurrent files to process", + //nolint:gomnd Value: 100, EnvVars: []string{"PLUGIN_MAX_CONCURRENCY"}, Destination: &settings.MaxConcurrency, diff --git a/cmd/drone-s3-sync/main.go b/cmd/drone-s3-sync/main.go index f39ecd0..0057979 100644 --- a/cmd/drone-s3-sync/main.go +++ b/cmd/drone-s3-sync/main.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "os" @@ -11,11 +12,14 @@ import ( "github.com/urfave/cli/v2" ) +//nolint:gochecknoglobals var ( BuildVersion = "devel" BuildDate = "00000000" ) +var ErrTypeAssertionFailed = errors.New("type assertion failed") + func main() { settings := &plugin.Settings{} @@ -44,12 +48,42 @@ func run(settings *plugin.Settings) cli.ActionFunc { return func(ctx *cli.Context) error { urfave.LoggingFromContext(ctx) - settings.ACL = ctx.Generic("acl").(*StringMapFlag).Get() - settings.CacheControl = ctx.Generic("cache-control").(*StringMapFlag).Get() - settings.ContentType = ctx.Generic("content-type").(*StringMapFlag).Get() - settings.ContentEncoding = ctx.Generic("content-encoding").(*StringMapFlag).Get() - settings.Metadata = ctx.Generic("metadata").(*DeepStringMapFlag).Get() - settings.Redirects = ctx.Generic("redirects").(*MapFlag).Get() + acl, ok := ctx.Generic("acl").(*StringMapFlag) + if !ok { + return fmt.Errorf("%w: failed to read acl input", ErrTypeAssertionFailed) + } + + cacheControl, ok := ctx.Generic("cache-control").(*StringMapFlag) + if !ok { + return fmt.Errorf("%w: failed to read cache-control input", ErrTypeAssertionFailed) + } + + contentType, ok := ctx.Generic("content-type").(*StringMapFlag) + if !ok { + return fmt.Errorf("%w: failed to read content-type input", ErrTypeAssertionFailed) + } + + contentEncoding, ok := ctx.Generic("content-encoding").(*StringMapFlag) + if !ok { + return fmt.Errorf("%w: failed to read content-encoding input", ErrTypeAssertionFailed) + } + + metadata, ok := ctx.Generic("metadata").(*DeepStringMapFlag) + if !ok { + return fmt.Errorf("%w: failed to read metadata input", ErrTypeAssertionFailed) + } + + redirects, ok := ctx.Generic("redirects").(*MapFlag) + if !ok { + return fmt.Errorf("%w: failed to read redirects input", ErrTypeAssertionFailed) + } + + settings.ACL = acl.Get() + settings.CacheControl = cacheControl.Get() + settings.ContentType = contentType.Get() + settings.ContentEncoding = contentEncoding.Get() + settings.Metadata = metadata.Get() + settings.Redirects = redirects.Get() plugin := plugin.New( *settings, diff --git a/cmd/drone-s3-sync/types.go b/cmd/drone-s3-sync/types.go index 51114ac..a4bfdb9 100644 --- a/cmd/drone-s3-sync/types.go +++ b/cmd/drone-s3-sync/types.go @@ -18,9 +18,11 @@ func (d *DeepStringMapFlag) Get() map[string]map[string]string { func (d *DeepStringMapFlag) Set(value string) error { d.parts = map[string]map[string]string{} + err := json.Unmarshal([]byte(value), &d.parts) if err != nil { single := map[string]string{} + err := json.Unmarshal([]byte(value), &single) if err != nil { return err @@ -46,10 +48,12 @@ func (s *StringMapFlag) Get() map[string]string { func (s *StringMapFlag) Set(value string) error { s.parts = map[string]string{} + err := json.Unmarshal([]byte(value), &s.parts) if err != nil { s.parts["*"] = value } + return nil } @@ -67,5 +71,6 @@ func (m *MapFlag) Get() map[string]string { func (m *MapFlag) Set(value string) error { m.parts = map[string]string{} + return json.Unmarshal([]byte(value), &m.parts) } diff --git a/go.mod b/go.mod index dd99c86..c297657 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/thegeeklab/drone-s3-sync -go 1.19 +go 1.20 require ( github.com/aws/aws-sdk-go v1.44.192 diff --git a/plugin/aws.go b/plugin/aws.go index 27bed9c..db0a47c 100644 --- a/plugin/aws.go +++ b/plugin/aws.go @@ -1,7 +1,9 @@ package plugin import ( + //nolint:gosec "crypto/md5" + "errors" "fmt" "io" "mime" @@ -28,20 +30,20 @@ type AWS struct { plugin *Plugin } -func NewAWS(p *Plugin) AWS { +func NewAWS(plugin *Plugin) AWS { sessCfg := &aws.Config{ - S3ForcePathStyle: aws.Bool(p.settings.PathStyle), - Region: aws.String(p.settings.Region), + S3ForcePathStyle: aws.Bool(plugin.settings.PathStyle), + Region: aws.String(plugin.settings.Region), } - if p.settings.Endpoint != "" { - sessCfg.Endpoint = &p.settings.Endpoint - sessCfg.DisableSSL = aws.Bool(strings.HasPrefix(p.settings.Endpoint, "http://")) + if plugin.settings.Endpoint != "" { + sessCfg.Endpoint = &plugin.settings.Endpoint + sessCfg.DisableSSL = aws.Bool(strings.HasPrefix(plugin.settings.Endpoint, "http://")) } // allowing to use the instance role or provide a key and secret - if p.settings.AccessKey != "" && p.settings.SecretKey != "" { - sessCfg.Credentials = credentials.NewStaticCredentials(p.settings.AccessKey, p.settings.SecretKey, "") + if plugin.settings.AccessKey != "" && plugin.settings.SecretKey != "" { + sessCfg.Credentials = credentials.NewStaticCredentials(plugin.settings.AccessKey, plugin.settings.SecretKey, "") } sess, _ := session.NewSession(sessCfg) @@ -51,11 +53,13 @@ func NewAWS(p *Plugin) AWS { r := make([]string, 1) l := make([]string, 1) - return AWS{c, cf, r, l, p} + return AWS{c, cf, r, l, plugin} } +//nolint:gocognit,gocyclo,maintidx func (a *AWS) Upload(local, remote string) error { - p := a.plugin + plugin := a.plugin + if local == "" { return nil } @@ -68,9 +72,11 @@ func (a *AWS) Upload(local, remote string) error { defer file.Close() var acl string - for pattern := range p.settings.ACL { + + for pattern := range plugin.settings.ACL { if match := glob.Glob(pattern, local); match { - acl = p.settings.ACL[pattern] + acl = plugin.settings.ACL[pattern] + break } } @@ -82,9 +88,11 @@ func (a *AWS) Upload(local, remote string) error { fileExt := filepath.Ext(local) var contentType string - for patternExt := range p.settings.ContentType { + + for patternExt := range plugin.settings.ContentType { if patternExt == fileExt { - contentType = p.settings.ContentType[patternExt] + contentType = plugin.settings.ContentType[patternExt] + break } } @@ -94,43 +102,58 @@ func (a *AWS) Upload(local, remote string) error { } var contentEncoding string - for patternExt := range p.settings.ContentEncoding { + + for patternExt := range plugin.settings.ContentEncoding { if patternExt == fileExt { - contentEncoding = p.settings.ContentEncoding[patternExt] + contentEncoding = plugin.settings.ContentEncoding[patternExt] + break } } var cacheControl string - for pattern := range p.settings.CacheControl { + + for pattern := range plugin.settings.CacheControl { if match := glob.Glob(pattern, local); match { - cacheControl = p.settings.CacheControl[pattern] + cacheControl = plugin.settings.CacheControl[pattern] + break } } metadata := map[string]*string{} - for pattern := range p.settings.Metadata { + + for pattern := range plugin.settings.Metadata { if match := glob.Glob(pattern, local); match { - for k, v := range p.settings.Metadata[pattern] { + for k, v := range plugin.settings.Metadata[pattern] { metadata[k] = aws.String(v) } + break } } + var AWSErr awserr.Error + head, err := a.client.HeadObject(&s3.HeadObjectInput{ - Bucket: aws.String(p.settings.Bucket), + Bucket: aws.String(plugin.settings.Bucket), Key: aws.String(remote), }) - if err != nil && err.(awserr.Error).Code() != "404" { + if err != nil && errors.As(err, &AWSErr) { + //nolint:errorlint,forcetypeassert if err.(awserr.Error).Code() == "404" { return err } - logrus.Debugf("'%s' not found in bucket, uploading with content-type '%s' and permissions '%s'", local, contentType, acl) + logrus.Debugf( + "'%s' not found in bucket, uploading with content-type '%s' and permissions '%s'", + local, + contentType, + acl, + ) + putObject := &s3.PutObjectInput{ - Bucket: aws.String(p.settings.Bucket), + Bucket: aws.String(plugin.settings.Bucket), Key: aws.String(remote), Body: file, ContentType: aws.String(contentType), @@ -152,48 +175,58 @@ func (a *AWS) Upload(local, remote string) error { } _, err = a.client.PutObject(putObject) + return err } + //nolint:gosec hash := md5.New() _, _ = io.Copy(hash, file) sum := fmt.Sprintf("'%x'", hash.Sum(nil)) + //nolint:nestif if sum == *head.ETag { shouldCopy := false if head.ContentType == nil && contentType != "" { logrus.Debugf("content-type has changed from unset to %s", contentType) + shouldCopy = true } if !shouldCopy && head.ContentType != nil && contentType != *head.ContentType { logrus.Debugf("content-type has changed from %s to %s", *head.ContentType, contentType) + shouldCopy = true } if !shouldCopy && head.ContentEncoding == nil && contentEncoding != "" { logrus.Debugf("Content-Encoding has changed from unset to %s", contentEncoding) + shouldCopy = true } if !shouldCopy && head.ContentEncoding != nil && contentEncoding != *head.ContentEncoding { logrus.Debugf("Content-Encoding has changed from %s to %s", *head.ContentEncoding, contentEncoding) + shouldCopy = true } if !shouldCopy && head.CacheControl == nil && cacheControl != "" { logrus.Debugf("cache-control has changed from unset to %s", cacheControl) + shouldCopy = true } if !shouldCopy && head.CacheControl != nil && cacheControl != *head.CacheControl { logrus.Debugf("cache-control has changed from %s to %s", *head.CacheControl, cacheControl) + shouldCopy = true } if !shouldCopy && len(head.Metadata) != len(metadata) { logrus.Debugf("count of metadata values has changed for %s", local) + shouldCopy = true } @@ -202,7 +235,9 @@ func (a *AWS) Upload(local, remote string) error { if hv, ok := head.Metadata[k]; ok { if *v != *hv { logrus.Debugf("metadata values have changed for %s", local) + shouldCopy = true + break } } @@ -211,7 +246,7 @@ func (a *AWS) Upload(local, remote string) error { if !shouldCopy { grant, err := a.client.GetObjectAcl(&s3.GetObjectAclInput{ - Bucket: aws.String(p.settings.Bucket), + Bucket: aws.String(plugin.settings.Bucket), Key: aws.String(remote), }) if err != nil { @@ -219,18 +254,20 @@ func (a *AWS) Upload(local, remote string) error { } previousACL := "private" - for _, g := range grant.Grants { - gt := *g.Grantee - if gt.URI != nil { - if *gt.URI == "http://acs.amazonaws.com/groups/global/AllUsers" { - if *g.Permission == "READ" { + + for _, grant := range grant.Grants { + grantee := *grant.Grantee + if grantee.URI != nil { + if *grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" { + if *grant.Permission == "READ" { previousACL = "public-read" - } else if *g.Permission == "WRITE" { + } else if *grant.Permission == "WRITE" { previousACL = "public-read-write" } } - if *gt.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" { - if *g.Permission == "READ" { + + if *grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" { + if *grant.Permission == "READ" { previousACL = "authenticated-read" } } @@ -239,20 +276,23 @@ func (a *AWS) Upload(local, remote string) error { if previousACL != acl { logrus.Debugf("permissions for '%s' have changed from '%s' to '%s'", remote, previousACL, acl) + shouldCopy = true } } if !shouldCopy { logrus.Debugf("skipping '%s' because hashes and metadata match", local) + return nil } logrus.Debugf("updating metadata for '%s' content-type: '%s', ACL: '%s'", local, contentType, acl) + copyObject := &s3.CopyObjectInput{ - Bucket: aws.String(p.settings.Bucket), + Bucket: aws.String(plugin.settings.Bucket), Key: aws.String(remote), - CopySource: aws.String(fmt.Sprintf("%s/%s", p.settings.Bucket, remote)), + CopySource: aws.String(fmt.Sprintf("%s/%s", plugin.settings.Bucket, remote)), ACL: aws.String(acl), ContentType: aws.String(contentType), Metadata: metadata, @@ -273,6 +313,7 @@ func (a *AWS) Upload(local, remote string) error { } _, err = a.client.CopyObject(copyObject) + return err } @@ -282,8 +323,9 @@ func (a *AWS) Upload(local, remote string) error { } logrus.Debugf("uploading '%s' with content-type '%s' and permissions '%s'", local, contentType, acl) + putObject := &s3.PutObjectInput{ - Bucket: aws.String(p.settings.Bucket), + Bucket: aws.String(plugin.settings.Bucket), Key: aws.String(remote), Body: file, ContentType: aws.String(contentType), @@ -305,11 +347,13 @@ func (a *AWS) Upload(local, remote string) error { } _, err = a.client.PutObject(putObject) + return err } func (a *AWS) Redirect(path, location string) error { - p := a.plugin + plugin := a.plugin + logrus.Debugf("adding redirect from '%s' to '%s'", path, location) if a.plugin.settings.DryRun { @@ -317,16 +361,18 @@ func (a *AWS) Redirect(path, location string) error { } _, err := a.client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(p.settings.Bucket), + Bucket: aws.String(plugin.settings.Bucket), Key: aws.String(path), ACL: aws.String("public-read"), WebsiteRedirectLocation: aws.String(location), }) + return err } func (a *AWS) Delete(remote string) error { - p := a.plugin + plugin := a.plugin + logrus.Debugf("removing remote file '%s'", remote) if a.plugin.settings.DryRun { @@ -334,17 +380,20 @@ func (a *AWS) Delete(remote string) error { } _, err := a.client.DeleteObject(&s3.DeleteObjectInput{ - Bucket: aws.String(p.settings.Bucket), + Bucket: aws.String(plugin.settings.Bucket), Key: aws.String(remote), }) + return err } func (a *AWS) List(path string) ([]string, error) { - p := a.plugin - remote := make([]string, 1) + plugin := a.plugin + + remote := make([]string, 0) + resp, err := a.client.ListObjects(&s3.ListObjectsInput{ - Bucket: aws.String(p.settings.Bucket), + Bucket: aws.String(plugin.settings.Bucket), Prefix: aws.String(path), }) if err != nil { @@ -357,7 +406,7 @@ func (a *AWS) List(path string) ([]string, error) { for *resp.IsTruncated { resp, err = a.client.ListObjects(&s3.ListObjectsInput{ - Bucket: aws.String(p.settings.Bucket), + Bucket: aws.String(plugin.settings.Bucket), Prefix: aws.String(path), Marker: aws.String(remote[len(remote)-1]), }) @@ -376,7 +425,9 @@ func (a *AWS) List(path string) ([]string, error) { func (a *AWS) Invalidate(invalidatePath string) error { p := a.plugin + logrus.Debugf("invalidating '%s'", invalidatePath) + _, err := a.cfClient.CreateInvalidation(&cloudfront.CreateInvalidationInput{ DistributionId: aws.String(p.settings.CloudFrontDistribution), InvalidationBatch: &cloudfront.InvalidationBatch{ @@ -389,5 +440,6 @@ func (a *AWS) Invalidate(invalidatePath string) error { }, }, }) + return err } diff --git a/plugin/impl.go b/plugin/impl.go index d1536a9..fe40b50 100644 --- a/plugin/impl.go +++ b/plugin/impl.go @@ -44,14 +44,13 @@ type Result struct { err error } -var MissingAwsValuesMessage = "Must set 'bucket'" - // Validate handles the settings validation of the plugin. func (p *Plugin) Validate() error { wd, err := os.Getwd() if err != nil { return fmt.Errorf("error while retrieving working directory: %w", err) } + p.settings.Source = filepath.Join(wd, p.settings.Source) p.settings.Target = strings.TrimPrefix(p.settings.Target, "/") @@ -88,7 +87,7 @@ func (p *Plugin) createSyncJobs() error { return err } - local := make([]string, 1) + local := make([]string, 0) err = filepath.Walk(p.settings.Source, func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() { @@ -122,13 +121,16 @@ func (p *Plugin) createSyncJobs() error { action: "redirect", }) } + if p.settings.Delete { - for _, r := range remote { + for _, remote := range remote { found := false - rPath := strings.TrimPrefix(r, p.settings.Target+"/") + remotePath := strings.TrimPrefix(remote, p.settings.Target+"/") + for _, l := range local { - if l == rPath { + if l == remotePath { found = true + break } } @@ -136,7 +138,7 @@ func (p *Plugin) createSyncJobs() error { if !found { p.settings.Jobs = append(p.settings.Jobs, Job{ local: "", - remote: r, + remote: remote, action: "delete", }) } @@ -150,41 +152,46 @@ func (p *Plugin) runJobs() error { client := p.settings.Client jobChan := make(chan struct{}, p.settings.MaxConcurrency) results := make(chan *Result, len(p.settings.Jobs)) + var invalidateJob *Job logrus.Infof("Synchronizing with bucket '%s'", p.settings.Bucket) - for _, j := range p.settings.Jobs { + + for _, job := range p.settings.Jobs { jobChan <- struct{}{} - go func(j Job) { + + go func(job Job) { var err error - switch j.action { + + switch job.action { case "upload": - err = client.Upload(j.local, j.remote) + err = client.Upload(job.local, job.remote) case "redirect": - err = client.Redirect(j.local, j.remote) + err = client.Redirect(job.local, job.remote) case "delete": - err = client.Delete(j.remote) + err = client.Delete(job.remote) case "invalidateCloudFront": - invalidateJob = &j + invalidateJob = &job default: err = nil } - results <- &Result{j, err} + results <- &Result{job, err} + <-jobChan - }(j) + }(job) } for range p.settings.Jobs { r := <-results if r.err != nil { - return fmt.Errorf("failed to %s %s to %s: %+v", r.j.action, r.j.local, r.j.remote, r.err) + return fmt.Errorf("failed to %s %s to %s: %w", r.j.action, r.j.local, r.j.remote, r.err) } } if invalidateJob != nil { err := client.Invalidate(invalidateJob.remote) if err != nil { - return fmt.Errorf("failed to %s %s to %s: %+v", invalidateJob.action, invalidateJob.local, invalidateJob.remote, err) + return fmt.Errorf("failed to %s %s to %s: %w", invalidateJob.action, invalidateJob.local, invalidateJob.remote, err) } } diff --git a/plugin/plugin.go b/plugin/plugin.go index 65eada6..85551fc 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -12,7 +12,7 @@ type Plugin struct { } // New initializes a plugin from the given Settings, Pipeline, and Network. -func New(settings Settings, pipeline drone.Pipeline, network drone.Network) drone.Plugin { +func New(settings Settings, pipeline drone.Pipeline, network drone.Network) *Plugin { return &Plugin{ settings: settings, pipeline: pipeline,