github/wp-s3-action
0
0
Fork 0
mirror of https://github.com/thegeeklab/wp-s3-action.git synced 2024-11-23 17:40:40 +00:00
Code Issues Releases Activity

refactor: rewrite plugin to use drone plugin lib (#3)

Browse Source
This commit is contained in:
Robert Kaussow 2022-05-03 11:45:54 +02:00 committed by GitHub
parent bae966b706
commit 4681e2d02b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 508 additions and 443 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.dictionary
View File

@ -3,3 +3,4 @@ github
url url
gh gh
drone-s3-sync drone-s3-sync
S3

129
cmd/drone-s3-sync/config.go Normal file
View File

@ -0,0 +1,129 @@
package main
import (
"github.com/thegeeklab/drone-s3-sync/plugin"
"github.com/urfave/cli/v2"
)
// settingsFlags has the cli.Flags for the plugin.Settings.
func settingsFlags(settings *plugin.Settings) []cli.Flag {
return []cli.Flag{
&cli.StringFlag{
Name: "endpoint",
Usage: "endpoint for the s3 connection",
EnvVars: []string{"PLUGIN_ENDPOINT", "S3_SYNC_ENDPOINT", "S3_ENDPOINT"},
Destination: &settings.Endpoint,
},
&cli.StringFlag{
Name: "access-key",
Usage: "aws access key",
EnvVars: []string{"PLUGIN_ACCESS_KEY", "AWS_ACCESS_KEY_ID"},
Destination: &settings.AccessKey,
},
&cli.StringFlag{
Name: "secret-key",
Usage: "aws secret key",
EnvVars: []string{"PLUGIN_SECRET_KEY", "AWS_SECRET_ACCESS_KEY"},
Destination: &settings.SecretKey,
},
&cli.BoolFlag{
Name: "path-style",
Usage: "use path style for bucket paths",
EnvVars: []string{"PLUGIN_PATH_STYLE"},
Destination: &settings.PathStyle,
},
&cli.StringFlag{
Name: "bucket",
Usage: "name of bucket",
EnvVars: []string{"PLUGIN_BUCKET"},
Destination: &settings.Bucket,
},
&cli.StringFlag{
Name: "region",
Usage: "aws region",
Value: "us-east-1",
EnvVars: []string{"PLUGIN_REGION"},
Destination: &settings.Region,
},
&cli.StringFlag{
Name: "source",
Usage: "upload source path",
Value: ".",
EnvVars: []string{"PLUGIN_SOURCE"},
Destination: &settings.Source,
},
&cli.StringFlag{
Name: "target",
Usage: "target path",
Value: "/",
EnvVars: []string{"PLUGIN_TARGET"},
Destination: &settings.Target,
},
&cli.BoolFlag{
Name: "delete",
Usage: "delete locally removed files from the target",
EnvVars: []string{"PLUGIN_DELETE"},
Destination: &settings.Delete,
},
&cli.GenericFlag{
Name: "access",
Usage: "access control settings",
EnvVars: []string{"PLUGIN_ACCESS", "PLUGIN_ACL"},
Value: &StringMapFlag{},
},
&cli.GenericFlag{
Name: "content-type",
Usage: "content-type settings for uploads",
EnvVars: []string{"PLUGIN_CONTENT_TYPE"},
Value: &StringMapFlag{},
},
&cli.GenericFlag{
Name: "content-encoding",
Usage: "content-encoding settings for uploads",
EnvVars: []string{"PLUGIN_CONTENT_ENCODING"},
Value: &StringMapFlag{},
},
&cli.GenericFlag{
Name: "cache-control",
Usage: "cache-control settings for uploads",
EnvVars: []string{"PLUGIN_CACHE_CONTROL"},
Value: &StringMapFlag{},
},
&cli.GenericFlag{
Name: "metadata",
Usage: "additional metadata for uploads",
EnvVars: []string{"PLUGIN_METADATA"},
Value: &DeepStringMapFlag{},
},
&cli.GenericFlag{
Name: "redirects",
Usage: "redirects to create",
EnvVars: []string{"PLUGIN_REDIRECTS"},
Value: &MapFlag{},
},
&cli.StringFlag{
Name: "cloudfront-distribution",
Usage: "id of cloudfront distribution to invalidate",
EnvVars: []string{"PLUGIN_CLOUDFRONT_DISTRIBUTION"},
Destination: &settings.CloudFrontDistribution,
},
&cli.BoolFlag{
Name: "dry-run",
Usage: "dry run disables api calls",
EnvVars: []string{"DRY_RUN", "PLUGIN_DRY_RUN"},
Destination: &settings.DryRun,
},
&cli.StringFlag{
Name: "env-file",
Usage: "source env file",
Destination: &settings.EnvFile,
},
&cli.IntFlag{
Name: "max-concurrency",
Usage: "customize number concurrent files to process",
Value: 100,
EnvVars: []string{"PLUGIN_MAX_CONCURRENCY"},
Destination: &settings.MaxConcurrency,
},
}
}

79
cmd/drone-s3-sync/main.go Normal file
View File

@ -0,0 +1,79 @@
package main
import (
"fmt"
"os"
"github.com/drone-plugins/drone-plugin-lib/errors"
"github.com/drone-plugins/drone-plugin-lib/urfave"
"github.com/joho/godotenv"
"github.com/thegeeklab/drone-s3-sync/plugin"
"github.com/urfave/cli/v2"
)
var (
BuildVersion = "devel"
BuildDate = "00000000"
)
func main() {
settings := &plugin.Settings{}
if _, err := os.Stat("/run/drone/env"); err == nil {
_ = godotenv.Overload("/run/drone/env")
}
cli.VersionPrinter = func(c *cli.Context) {
fmt.Printf("%s version=%s date=%s\n", c.App.Name, c.App.Version, BuildDate)
}
app := &cli.App{
Name: "drone-s3-sync",
Usage: "synchronize a directory with an S3 bucket",
Version: BuildVersion,
Flags: append(settingsFlags(settings), urfave.Flags()...),
Action: run(settings),
}
if err := app.Run(os.Args); err != nil {
errors.HandleExit(err)
}
}
func run(settings *plugin.Settings) cli.ActionFunc {
return func(ctx *cli.Context) error {
urfave.LoggingFromContext(ctx)
settings.Access = ctx.Generic("access").(*StringMapFlag).Get()
settings.CacheControl = ctx.Generic("cache-control").(*StringMapFlag).Get()
settings.ContentType = ctx.Generic("content-type").(*StringMapFlag).Get()
settings.ContentEncoding = ctx.Generic("content-encoding").(*StringMapFlag).Get()
settings.Metadata = ctx.Generic("metadata").(*DeepStringMapFlag).Get()
settings.Redirects = ctx.Generic("redirects").(*MapFlag).Get()
plugin := plugin.New(
*settings,
urfave.PipelineFromContext(ctx),
urfave.NetworkFromContext(ctx),
)
if err := plugin.Validate(); err != nil {
if e, ok := err.(errors.ExitCoder); ok {
return e
}
return errors.ExitMessagef("validation failed: %w", err)
}
if err := plugin.Execute(); err != nil {
if e, ok := err.(errors.ExitCoder); ok {
return e
}
return errors.ExitMessagef("execution failed: %w", err)
}
return nil
}
}

0
types.go → cmd/drone-s3-sync/types.go
View File

8
go.mod
View File

@ -4,20 +4,18 @@ go 1.18
require ( require (
github.com/aws/aws-sdk-go v1.44.5 github.com/aws/aws-sdk-go v1.44.5
github.com/drone-plugins/drone-plugin-lib v0.4.0
github.com/joho/godotenv v1.4.0 github.com/joho/godotenv v1.4.0
github.com/ryanuber/go-glob v1.0.0 github.com/ryanuber/go-glob v1.0.0
github.com/sirupsen/logrus v1.8.1 github.com/sirupsen/logrus v1.8.1
github.com/urfave/cli v1.22.8 github.com/urfave/cli/v2 v2.5.1
) )
require ( require (
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/stretchr/testify v1.4.0 // indirect
github.com/stretchr/testify v1.3.0 // indirect
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba // indirect golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba // indirect
) )

48
go.sum
View File

@ -1,6 +1,4 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/aws/aws-sdk-go v1.16.15 h1:kQyxfRyjAwIYjf0225sn/pn+WAlncKyI8dmT3+ItMFE=
github.com/aws/aws-sdk-go v1.16.15/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.44.5 h1:T4mckpWUfplPG4GA3FDWDCM1QaCzisjGzzeCVBhHKwQ= github.com/aws/aws-sdk-go v1.44.5 h1:T4mckpWUfplPG4GA3FDWDCM1QaCzisjGzzeCVBhHKwQ=
github.com/aws/aws-sdk-go v1.44.5/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.5/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@ -9,62 +7,58 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/drone-plugins/drone-plugin-lib v0.4.0 h1:qywEYGhquUuid6zNLmKia8CWY1TUa8jPQQ/G9ozfAmc=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/drone-plugins/drone-plugin-lib v0.4.0/go.mod h1:EgqogX38GoJFtckeSQyhBJYX8P+KWBPhdprAVvyRxF8=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 h1:7YvPJVmEeFHR1Tj9sZEYsmarJEQfMVYpd/Vyy/A8dqE=
github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli/v2 v2.5.1 h1:YKwdkyA0xTBzOaP2G0DVxBnCheHGP+Y9VbKAs4K1Ess=
github.com/urfave/cli v1.22.8 h1:9ic0a+f2TCJ5tSbVRX/FSSCIHJacFLYxcuNexNMJF8Q= github.com/urfave/cli/v2 v2.5.1/go.mod h1:oDzoM7pVwz6wHn5ogWgFUU1s4VJayeQS+aEZDqXIEJs=
github.com/urfave/cli v1.22.8/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc=
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba h1:AyHWHCBVlIYI5rgEM3o+1PLd0sLPcIAoaUckGQMaWtw= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba h1:AyHWHCBVlIYI5rgEM3o+1PLd0sLPcIAoaUckGQMaWtw=
golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=

157
main.go
View File

@ -1,157 +0,0 @@
package main
import (
"os"
"github.com/joho/godotenv"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
var version = "unknown"
func main() {
app := cli.NewApp()
app.Name = "s3 sync plugin"
app.Usage = "s3 sync plugin"
app.Action = run
app.Version = version
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "endpoint",
Usage: "endpoint for the s3 connection",
EnvVar: "PLUGIN_ENDPOINT,S3_SYNC_ENDPOINT,S3_ENDPOINT",
},
cli.StringFlag{
Name: "access-key",
Usage: "aws access key",
EnvVar: "PLUGIN_ACCESS_KEY,AWS_ACCESS_KEY_ID",
},
cli.StringFlag{
Name: "secret-key",
Usage: "aws secret key",
EnvVar: "PLUGIN_SECRET_KEY,AWS_SECRET_ACCESS_KEY",
},
cli.BoolFlag{
Name: "path-style",
Usage: "use path style for bucket paths",
EnvVar: "PLUGIN_PATH_STYLE",
},
cli.StringFlag{
Name: "bucket",
Usage: "name of bucket",
EnvVar: "PLUGIN_BUCKET",
},
cli.StringFlag{
Name: "region",
Usage: "aws region",
Value: "us-east-1",
EnvVar: "PLUGIN_REGION",
},
cli.StringFlag{
Name: "source",
Usage: "upload source path",
Value: ".",
EnvVar: "PLUGIN_SOURCE",
},
cli.StringFlag{
Name: "target",
Usage: "target path",
Value: "/",
EnvVar: "PLUGIN_TARGET",
},
cli.BoolFlag{
Name: "delete",
Usage: "delete locally removed files from the target",
EnvVar: "PLUGIN_DELETE",
},
cli.GenericFlag{
Name: "access",
Usage: "access control settings",
EnvVar: "PLUGIN_ACCESS,PLUGIN_ACL",
Value: &StringMapFlag{},
},
cli.GenericFlag{
Name: "content-type",
Usage: "content-type settings for uploads",
EnvVar: "PLUGIN_CONTENT_TYPE",
Value: &StringMapFlag{},
},
cli.GenericFlag{
Name: "content-encoding",
Usage: "content-encoding settings for uploads",
EnvVar: "PLUGIN_CONTENT_ENCODING",
Value: &StringMapFlag{},
},
cli.GenericFlag{
Name: "cache-control",
Usage: "cache-control settings for uploads",
EnvVar: "PLUGIN_CACHE_CONTROL",
Value: &StringMapFlag{},
},
cli.GenericFlag{
Name: "metadata",
Usage: "additional metadata for uploads",
EnvVar: "PLUGIN_METADATA",
Value: &DeepStringMapFlag{},
},
cli.GenericFlag{
Name: "redirects",
Usage: "redirects to create",
EnvVar: "PLUGIN_REDIRECTS",
Value: &MapFlag{},
},
cli.StringFlag{
Name: "cloudfront-distribution",
Usage: "id of cloudfront distribution to invalidate",
EnvVar: "PLUGIN_CLOUDFRONT_DISTRIBUTION",
},
cli.BoolFlag{
Name: "dry-run",
Usage: "dry run disables api calls",
EnvVar: "DRY_RUN,PLUGIN_DRY_RUN",
},
cli.StringFlag{
Name: "env-file",
Usage: "source env file",
},
cli.IntFlag{
Name: "max-concurrency",
Usage: "customize number concurrent files to process",
Value: 100,
EnvVar: "PLUGIN_MAX_CONCURRENCY",
},
}
if err := app.Run(os.Args); err != nil {
logrus.Fatal(err)
}
}
func run(c *cli.Context) error {
if c.String("env-file") != "" {
_ = godotenv.Load(c.String("env-file"))
}
plugin := Plugin{
Endpoint: c.String("endpoint"),
PathStyle: c.Bool("path-style"),
Key: c.String("access-key"),
Secret: c.String("secret-key"),
Bucket: c.String("bucket"),
Region: c.String("region"),
Source: c.String("source"),
Target: c.String("target"),
Delete: c.Bool("delete"),
Access: c.Generic("access").(*StringMapFlag).Get(),
CacheControl: c.Generic("cache-control").(*StringMapFlag).Get(),
ContentType: c.Generic("content-type").(*StringMapFlag).Get(),
ContentEncoding: c.Generic("content-encoding").(*StringMapFlag).Get(),
Metadata: c.Generic("metadata").(*DeepStringMapFlag).Get(),
Redirects: c.Generic("redirects").(*MapFlag).Get(),
CloudFrontDistribution: c.String("cloudfront-distribution"),
DryRun: c.Bool("dry-run"),
MaxConcurrency: c.Int("max-concurrency"),
}
return plugin.Exec()
}

203
plugin.go
View File

@ -1,203 +0,0 @@
package main
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
)
type Plugin struct {
Endpoint string
Key string
Secret string
Bucket string
Region string
Source string
Target string
Delete bool
Access map[string]string
CacheControl map[string]string
ContentType map[string]string
ContentEncoding map[string]string
Metadata map[string]map[string]string
Redirects map[string]string
CloudFrontDistribution string
DryRun bool
PathStyle bool
client AWS
jobs []job
MaxConcurrency int
}
type job struct {
local string
remote string
action string
}
type result struct {
j job
err error
}
var MissingAwsValuesMessage = "Must set 'bucket'"
func (p *Plugin) Exec() error {
err := p.sanitizeInputs()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
p.jobs = make([]job, 1)
p.client = NewAWS(p)
p.createSyncJobs()
p.createInvalidateJob()
p.runJobs()
return nil
}
func (p *Plugin) sanitizeInputs() error {
if len(p.Bucket) == 0 {
return errors.New(MissingAwsValuesMessage)
}
wd, err := os.Getwd()
if err != nil {
return err
}
p.Source = filepath.Join(wd, p.Source)
p.Target = strings.TrimPrefix(p.Target, "/")
return nil
}
func (p *Plugin) createSyncJobs() {
remote, err := p.client.List(p.Target)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
local := make([]string, 1)
err = filepath.Walk(p.Source, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return err
}
localPath := path
if p.Source != "." {
localPath = strings.TrimPrefix(path, p.Source)
localPath = strings.TrimPrefix(localPath, "/")
}
local = append(local, localPath)
p.jobs = append(p.jobs, job{
local: filepath.Join(p.Source, localPath),
remote: filepath.Join(p.Target, localPath),
action: "upload",
})
return nil
})
if err != nil {
fmt.Println(err)
os.Exit(1)
}
for path, location := range p.Redirects {
path = strings.TrimPrefix(path, "/")
local = append(local, path)
p.jobs = append(p.jobs, job{
local: path,
remote: location,
action: "redirect",
})
}
if p.Delete {
for _, r := range remote {
found := false
rPath := strings.TrimPrefix(r, p.Target+"/")
for _, l := range local {
if l == rPath {
found = true
break
}
}
if !found {
p.jobs = append(p.jobs, job{
local: "",
remote: r,
action: "delete",
})
}
}
}
}
func (p *Plugin) createInvalidateJob() {
if len(p.CloudFrontDistribution) > 0 {
p.jobs = append(p.jobs, job{
local: "",
remote: filepath.Join("/", p.Target, "*"),
action: "invalidateCloudFront",
})
}
}
func (p *Plugin) runJobs() {
client := p.client
jobChan := make(chan struct{}, p.MaxConcurrency)
results := make(chan *result, len(p.jobs))
var invalidateJob *job
fmt.Printf("Synchronizing with bucket \"%s\"\n", p.Bucket)
for _, j := range p.jobs {
jobChan <- struct{}{}
go func(j job) {
var err error
switch j.action {
case "upload":
err = client.Upload(j.local, j.remote)
case "redirect":
err = client.Redirect(j.local, j.remote)
case "delete":
err = client.Delete(j.remote)
case "invalidateCloudFront":
invalidateJob = &j
default:
err = nil
}
results <- &result{j, err}
<-jobChan
}(j)
}
for range p.jobs {
r := <-results
if r.err != nil {
fmt.Printf("ERROR: failed to %s %s to %s: %+v\n", r.j.action, r.j.local, r.j.remote, r.err)
os.Exit(1)
}
}
if invalidateJob != nil {
err := client.Invalidate(invalidateJob.remote)
if err != nil {
fmt.Printf("ERROR: failed to %s %s to %s: %+v\n", invalidateJob.action, invalidateJob.local, invalidateJob.remote, err)
os.Exit(1)
}
}
}
func debug(format string, args ...interface{}) {
if os.Getenv("DEBUG") != "" {
fmt.Printf(format+"\n", args...)
} else {
fmt.Printf(".")
}
}

103
aws.go → plugin/aws.go
View File

@ -1,4 +1,4 @@
package main package plugin
import ( import (
"crypto/md5" "crypto/md5"
@ -17,6 +17,7 @@ import (
"github.com/aws/aws-sdk-go/service/cloudfront" "github.com/aws/aws-sdk-go/service/cloudfront"
"github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3"
"github.com/ryanuber/go-glob" "github.com/ryanuber/go-glob"
"github.com/sirupsen/logrus"
) )
type AWS struct { type AWS struct {
@ -29,18 +30,18 @@ type AWS struct {
func NewAWS(p *Plugin) AWS { func NewAWS(p *Plugin) AWS {
sessCfg := &aws.Config{ sessCfg := &aws.Config{
S3ForcePathStyle: aws.Bool(p.PathStyle), S3ForcePathStyle: aws.Bool(p.settings.PathStyle),
Region: aws.String(p.Region), Region: aws.String(p.settings.Region),
} }
if p.Endpoint != "" { if p.settings.Endpoint != "" {
sessCfg.Endpoint = &p.Endpoint sessCfg.Endpoint = &p.settings.Endpoint
sessCfg.DisableSSL = aws.Bool(strings.HasPrefix(p.Endpoint, "http://")) sessCfg.DisableSSL = aws.Bool(strings.HasPrefix(p.settings.Endpoint, "http://"))
} }
// allowing to use the instance role or provide a key and secret // allowing to use the instance role or provide a key and secret
if p.Key != "" && p.Secret != "" { if p.settings.AccessKey != "" && p.settings.SecretKey != "" {
sessCfg.Credentials = credentials.NewStaticCredentials(p.Key, p.Secret, "") sessCfg.Credentials = credentials.NewStaticCredentials(p.settings.AccessKey, p.settings.SecretKey, "")
} }
sess, _ := session.NewSession(sessCfg) sess, _ := session.NewSession(sessCfg)
@ -67,9 +68,9 @@ func (a *AWS) Upload(local, remote string) error {
defer file.Close() defer file.Close()
var access string var access string
for pattern := range p.Access { for pattern := range p.settings.Access {
if match := glob.Glob(pattern, local); match { if match := glob.Glob(pattern, local); match {
access = p.Access[pattern] access = p.settings.Access[pattern]
break break
} }
} }
@ -81,9 +82,9 @@ func (a *AWS) Upload(local, remote string) error {
fileExt := filepath.Ext(local) fileExt := filepath.Ext(local)
var contentType string var contentType string
for patternExt := range p.ContentType { for patternExt := range p.settings.ContentType {
if patternExt == fileExt { if patternExt == fileExt {
contentType = p.ContentType[patternExt] contentType = p.settings.ContentType[patternExt]
break break
} }
} }
@ -93,25 +94,25 @@ func (a *AWS) Upload(local, remote string) error {
} }
var contentEncoding string var contentEncoding string
for patternExt := range p.ContentEncoding { for patternExt := range p.settings.ContentEncoding {
if patternExt == fileExt { if patternExt == fileExt {
contentEncoding = p.ContentEncoding[patternExt] contentEncoding = p.settings.ContentEncoding[patternExt]
break break
} }
} }
var cacheControl string var cacheControl string
for pattern := range p.CacheControl { for pattern := range p.settings.CacheControl {
if match := glob.Glob(pattern, local); match { if match := glob.Glob(pattern, local); match {
cacheControl = p.CacheControl[pattern] cacheControl = p.settings.CacheControl[pattern]
break break
} }
} }
metadata := map[string]*string{} metadata := map[string]*string{}
for pattern := range p.Metadata { for pattern := range p.settings.Metadata {
if match := glob.Glob(pattern, local); match { if match := glob.Glob(pattern, local); match {
for k, v := range p.Metadata[pattern] { for k, v := range p.settings.Metadata[pattern] {
metadata[k] = aws.String(v) metadata[k] = aws.String(v)
} }
break break
@ -119,7 +120,7 @@ func (a *AWS) Upload(local, remote string) error {
} }
head, err := a.client.HeadObject(&s3.HeadObjectInput{ head, err := a.client.HeadObject(&s3.HeadObjectInput{
Bucket: aws.String(p.Bucket), Bucket: aws.String(p.settings.Bucket),
Key: aws.String(remote), Key: aws.String(remote),
}) })
if err != nil && err.(awserr.Error).Code() != "404" { if err != nil && err.(awserr.Error).Code() != "404" {
@ -127,9 +128,9 @@ func (a *AWS) Upload(local, remote string) error {
return err return err
} }
debug("\"%s\" not found in bucket, uploading with Content-Type \"%s\" and permissions \"%s\"", local, contentType, access) logrus.Debugf("'%s' not found in bucket, uploading with content-type '%s' and permissions '%s'", local, contentType, access)
putObject := &s3.PutObjectInput{ putObject := &s3.PutObjectInput{
Bucket: aws.String(p.Bucket), Bucket: aws.String(p.settings.Bucket),
Key: aws.String(remote), Key: aws.String(remote),
Body: file, Body: file,
ContentType: aws.String(contentType), ContentType: aws.String(contentType),
@ -146,7 +147,7 @@ func (a *AWS) Upload(local, remote string) error {
} }
// skip upload during dry run // skip upload during dry run
if a.plugin.DryRun { if a.plugin.settings.DryRun {
return nil return nil
} }
@ -156,43 +157,43 @@ func (a *AWS) Upload(local, remote string) error {
hash := md5.New() hash := md5.New()
_, _ = io.Copy(hash, file) _, _ = io.Copy(hash, file)
sum := fmt.Sprintf("\"%x\"", hash.Sum(nil)) sum := fmt.Sprintf("'%x'", hash.Sum(nil))
if sum == *head.ETag { if sum == *head.ETag {
shouldCopy := false shouldCopy := false
if head.ContentType == nil && contentType != "" { if head.ContentType == nil && contentType != "" {
debug("Content-Type has changed from unset to %s", contentType) logrus.Debugf("content-type has changed from unset to %s", contentType)
shouldCopy = true shouldCopy = true
} }
if !shouldCopy && head.ContentType != nil && contentType != *head.ContentType { if !shouldCopy && head.ContentType != nil && contentType != *head.ContentType {
debug("Content-Type has changed from %s to %s", *head.ContentType, contentType) logrus.Debugf("content-type has changed from %s to %s", *head.ContentType, contentType)
shouldCopy = true shouldCopy = true
} }
if !shouldCopy && head.ContentEncoding == nil && contentEncoding != "" { if !shouldCopy && head.ContentEncoding == nil && contentEncoding != "" {
debug("Content-Encoding has changed from unset to %s", contentEncoding) logrus.Debugf("Content-Encoding has changed from unset to %s", contentEncoding)
shouldCopy = true shouldCopy = true
} }
if !shouldCopy && head.ContentEncoding != nil && contentEncoding != *head.ContentEncoding { if !shouldCopy && head.ContentEncoding != nil && contentEncoding != *head.ContentEncoding {
debug("Content-Encoding has changed from %s to %s", *head.ContentEncoding, contentEncoding) logrus.Debugf("Content-Encoding has changed from %s to %s", *head.ContentEncoding, contentEncoding)
shouldCopy = true shouldCopy = true
} }
if !shouldCopy && head.CacheControl == nil && cacheControl != "" { if !shouldCopy && head.CacheControl == nil && cacheControl != "" {
debug("Cache-Control has changed from unset to %s", cacheControl) logrus.Debugf("cache-control has changed from unset to %s", cacheControl)
shouldCopy = true shouldCopy = true
} }
if !shouldCopy && head.CacheControl != nil && cacheControl != *head.CacheControl { if !shouldCopy && head.CacheControl != nil && cacheControl != *head.CacheControl {
debug("Cache-Control has changed from %s to %s", *head.CacheControl, cacheControl) logrus.Debugf("cache-control has changed from %s to %s", *head.CacheControl, cacheControl)
shouldCopy = true shouldCopy = true
} }
if !shouldCopy && len(head.Metadata) != len(metadata) { if !shouldCopy && len(head.Metadata) != len(metadata) {
debug("Count of metadata values has changed for %s", local) logrus.Debugf("count of metadata values has changed for %s", local)
shouldCopy = true shouldCopy = true
} }
@ -200,7 +201,7 @@ func (a *AWS) Upload(local, remote string) error {
for k, v := range metadata { for k, v := range metadata {
if hv, ok := head.Metadata[k]; ok { if hv, ok := head.Metadata[k]; ok {
if *v != *hv { if *v != *hv {
debug("Metadata values have changed for %s", local) logrus.Debugf("metadata values have changed for %s", local)
shouldCopy = true shouldCopy = true
break break
} }
@ -210,7 +211,7 @@ func (a *AWS) Upload(local, remote string) error {
if !shouldCopy { if !shouldCopy {
grant, err := a.client.GetObjectAcl(&s3.GetObjectAclInput{ grant, err := a.client.GetObjectAcl(&s3.GetObjectAclInput{
Bucket: aws.String(p.Bucket), Bucket: aws.String(p.settings.Bucket),
Key: aws.String(remote), Key: aws.String(remote),
}) })
if err != nil { if err != nil {
@ -237,21 +238,21 @@ func (a *AWS) Upload(local, remote string) error {
} }
if previousAccess != access { if previousAccess != access {
debug("Permissions for \"%s\" have changed from \"%s\" to \"%s\"", remote, previousAccess, access) logrus.Debugf("permissions for '%s' have changed from '%s' to '%s'", remote, previousAccess, access)
shouldCopy = true shouldCopy = true
} }
} }
if !shouldCopy { if !shouldCopy {
debug("Skipping \"%s\" because hashes and metadata match", local) logrus.Debugf("skipping '%s' because hashes and metadata match", local)
return nil return nil
} }
debug("Updating metadata for \"%s\" Content-Type: \"%s\", ACL: \"%s\"", local, contentType, access) logrus.Debugf("updating metadata for '%s' content-type: '%s', ACL: '%s'", local, contentType, access)
copyObject := &s3.CopyObjectInput{ copyObject := &s3.CopyObjectInput{
Bucket: aws.String(p.Bucket), Bucket: aws.String(p.settings.Bucket),
Key: aws.String(remote), Key: aws.String(remote),
CopySource: aws.String(fmt.Sprintf("%s/%s", p.Bucket, remote)), CopySource: aws.String(fmt.Sprintf("%s/%s", p.settings.Bucket, remote)),
ACL: aws.String(access), ACL: aws.String(access),
ContentType: aws.String(contentType), ContentType: aws.String(contentType),
Metadata: metadata, Metadata: metadata,
@ -267,7 +268,7 @@ func (a *AWS) Upload(local, remote string) error {
} }
// skip update if dry run // skip update if dry run
if a.plugin.DryRun { if a.plugin.settings.DryRun {
return nil return nil
} }
@ -280,9 +281,9 @@ func (a *AWS) Upload(local, remote string) error {
return err return err
} }
debug("Uploading \"%s\" with Content-Type \"%s\" and permissions \"%s\"", local, contentType, access) logrus.Debugf("uploading '%s' with content-type '%s' and permissions '%s'", local, contentType, access)
putObject := &s3.PutObjectInput{ putObject := &s3.PutObjectInput{
Bucket: aws.String(p.Bucket), Bucket: aws.String(p.settings.Bucket),
Key: aws.String(remote), Key: aws.String(remote),
Body: file, Body: file,
ContentType: aws.String(contentType), ContentType: aws.String(contentType),
@ -299,7 +300,7 @@ func (a *AWS) Upload(local, remote string) error {
} }
// skip upload if dry run // skip upload if dry run
if a.plugin.DryRun { if a.plugin.settings.DryRun {
return nil return nil
} }
@ -309,14 +310,14 @@ func (a *AWS) Upload(local, remote string) error {
func (a *AWS) Redirect(path, location string) error { func (a *AWS) Redirect(path, location string) error {
p := a.plugin p := a.plugin
debug("Adding redirect from \"%s\" to \"%s\"", path, location) logrus.Debugf("adding redirect from '%s' to '%s'", path, location)
if a.plugin.DryRun { if a.plugin.settings.DryRun {
return nil return nil
} }
_, err := a.client.PutObject(&s3.PutObjectInput{ _, err := a.client.PutObject(&s3.PutObjectInput{
Bucket: aws.String(p.Bucket), Bucket: aws.String(p.settings.Bucket),
Key: aws.String(path), Key: aws.String(path),
ACL: aws.String("public-read"), ACL: aws.String("public-read"),
WebsiteRedirectLocation: aws.String(location), WebsiteRedirectLocation: aws.String(location),
@ -326,14 +327,14 @@ func (a *AWS) Redirect(path, location string) error {
func (a *AWS) Delete(remote string) error { func (a *AWS) Delete(remote string) error {
p := a.plugin p := a.plugin
debug("Removing remote file \"%s\"", remote) logrus.Debugf("removing remote file '%s'", remote)
if a.plugin.DryRun { if a.plugin.settings.DryRun {
return nil return nil
} }
_, err := a.client.DeleteObject(&s3.DeleteObjectInput{ _, err := a.client.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(p.Bucket), Bucket: aws.String(p.settings.Bucket),
Key: aws.String(remote), Key: aws.String(remote),
}) })
return err return err
@ -343,7 +344,7 @@ func (a *AWS) List(path string) ([]string, error) {
p := a.plugin p := a.plugin
remote := make([]string, 1) remote := make([]string, 1)
resp, err := a.client.ListObjects(&s3.ListObjectsInput{ resp, err := a.client.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(p.Bucket), Bucket: aws.String(p.settings.Bucket),
Prefix: aws.String(path), Prefix: aws.String(path),
}) })
if err != nil { if err != nil {
@ -356,7 +357,7 @@ func (a *AWS) List(path string) ([]string, error) {
for *resp.IsTruncated { for *resp.IsTruncated {
resp, err = a.client.ListObjects(&s3.ListObjectsInput{ resp, err = a.client.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(p.Bucket), Bucket: aws.String(p.settings.Bucket),
Prefix: aws.String(path), Prefix: aws.String(path),
Marker: aws.String(remote[len(remote)-1]), Marker: aws.String(remote[len(remote)-1]),
}) })
@ -375,9 +376,9 @@ func (a *AWS) List(path string) ([]string, error) {
func (a *AWS) Invalidate(invalidatePath string) error { func (a *AWS) Invalidate(invalidatePath string) error {
p := a.plugin p := a.plugin
debug("Invalidating \"%s\"", invalidatePath) logrus.Debugf("invalidating '%s'", invalidatePath)
_, err := a.cfClient.CreateInvalidation(&cloudfront.CreateInvalidationInput{ _, err := a.cfClient.CreateInvalidation(&cloudfront.CreateInvalidationInput{
DistributionId: aws.String(p.CloudFrontDistribution), DistributionId: aws.String(p.settings.CloudFrontDistribution),
InvalidationBatch: &cloudfront.InvalidationBatch{ InvalidationBatch: &cloudfront.InvalidationBatch{
CallerReference: aws.String(time.Now().Format(time.RFC3339Nano)), CallerReference: aws.String(time.Now().Format(time.RFC3339Nano)),
Paths: &cloudfront.Paths{ Paths: &cloudfront.Paths{

202
plugin/impl.go Normal file
View File

@ -0,0 +1,202 @@
package plugin
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/joho/godotenv"
"github.com/sirupsen/logrus"
)
// Settings for the Plugin.
type Settings struct {
Endpoint string
AccessKey string
SecretKey string
Bucket string
Region string
Source string
Target string
Delete bool
Access map[string]string
CacheControl map[string]string
ContentType map[string]string
ContentEncoding map[string]string
Metadata map[string]map[string]string
Redirects map[string]string
CloudFrontDistribution string
DryRun bool
PathStyle bool
Client AWS
Jobs []Job
MaxConcurrency int
EnvFile string
}
type Job struct {
local string
remote string
action string
}
type Result struct {
j Job
err error
}
var MissingAwsValuesMessage = "Must set 'bucket'"
// Validate handles the settings validation of the plugin.
func (p *Plugin) Validate() error {
if len(p.settings.Bucket) == 0 {
return fmt.Errorf("no bucket name provided")
}
wd, err := os.Getwd()
if err != nil {
return fmt.Errorf("error while retrieving working directory: %w", err)
}
p.settings.Source = filepath.Join(wd, p.settings.Source)
p.settings.Target = strings.TrimPrefix(p.settings.Target, "/")
return nil
}
// Execute provides the implementation of the plugin.
func (p *Plugin) Execute() error {
if p.settings.EnvFile != "" {
_ = godotenv.Load(p.settings.EnvFile)
}
p.settings.Jobs = make([]Job, 1)
p.settings.Client = NewAWS(p)
if err := p.createSyncJobs(); err != nil {
return fmt.Errorf("error while creating sync job: %w", err)
}
if len(p.settings.CloudFrontDistribution) > 0 {
p.settings.Jobs = append(p.settings.Jobs, Job{
local: "",
remote: filepath.Join("/", p.settings.Target, "*"),
action: "invalidateCloudFront",
})
}
if err := p.runJobs(); err != nil {
return fmt.Errorf("error while creating sync job: %w", err)
}
return nil
}
func (p *Plugin) createSyncJobs() error {
remote, err := p.settings.Client.List(p.settings.Target)
if err != nil {
return err
}
local := make([]string, 1)
err = filepath.Walk(p.settings.Source, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return err
}
localPath := path
if p.settings.Source != "." {
localPath = strings.TrimPrefix(path, p.settings.Source)
localPath = strings.TrimPrefix(localPath, "/")
}
local = append(local, localPath)
p.settings.Jobs = append(p.settings.Jobs, Job{
local: filepath.Join(p.settings.Source, localPath),
remote: filepath.Join(p.settings.Target, localPath),
action: "upload",
})
return nil
})
if err != nil {
return err
}
for path, location := range p.settings.Redirects {
path = strings.TrimPrefix(path, "/")
local = append(local, path)
p.settings.Jobs = append(p.settings.Jobs, Job{
local: path,
remote: location,
action: "redirect",
})
}
if p.settings.Delete {
for _, r := range remote {
found := false
rPath := strings.TrimPrefix(r, p.settings.Target+"/")
for _, l := range local {
if l == rPath {
found = true
break
}
}
if !found {
p.settings.Jobs = append(p.settings.Jobs, Job{
local: "",
remote: r,
action: "delete",
})
}
}
}
return nil
}
func (p *Plugin) runJobs() error {
client := p.settings.Client
jobChan := make(chan struct{}, p.settings.MaxConcurrency)
results := make(chan *Result, len(p.settings.Jobs))
var invalidateJob *Job
logrus.Infof("Synchronizing with bucket '%s'", p.settings.Bucket)
for _, j := range p.settings.Jobs {
jobChan <- struct{}{}
go func(j Job) {
var err error
switch j.action {
case "upload":
err = client.Upload(j.local, j.remote)
case "redirect":
err = client.Redirect(j.local, j.remote)
case "delete":
err = client.Delete(j.remote)
case "invalidateCloudFront":
invalidateJob = &j
default:
err = nil
}
results <- &Result{j, err}
<-jobChan
}(j)
}
for range p.settings.Jobs {
r := <-results
if r.err != nil {
return fmt.Errorf("failed to %s %s to %s: %+v", r.j.action, r.j.local, r.j.remote, r.err)
}
}
if invalidateJob != nil {
err := client.Invalidate(invalidateJob.remote)
if err != nil {
return fmt.Errorf("failed to %s %s to %s: %+v", invalidateJob.action, invalidateJob.local, invalidateJob.remote, err)
}
}
return nil
}

21
plugin/plugin.go Normal file
View File

@ -0,0 +1,21 @@
package plugin
import (
"github.com/drone-plugins/drone-plugin-lib/drone"
)
// Plugin implements drone.Plugin to provide the plugin implementation.
type Plugin struct {
settings Settings
pipeline drone.Pipeline
network drone.Network
}
// New initializes a plugin from the given Settings, Pipeline, and Network.
func New(settings Settings, pipeline drone.Pipeline, network drone.Network) drone.Plugin {
return &Plugin{
settings: settings,
pipeline: pipeline,
network: network,
}
}