From 4a6f734689274105aae078f515ec3bd18a56b479 Mon Sep 17 00:00:00 2001 From: Nathan LaFreniere Date: Wed, 18 Nov 2015 16:32:44 -0800 Subject: [PATCH 1/3] support custom metadata, only upload changed files, make sure permissions and metadata are synced --- aws.go | 289 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 219 ++--------------------------------------- types.go | 97 +++++++++++++++++++ 3 files changed, 394 insertions(+), 211 deletions(-) create mode 100644 aws.go create mode 100644 types.go diff --git a/aws.go b/aws.go new file mode 100644 index 0000000..71a0084 --- /dev/null +++ b/aws.go @@ -0,0 +1,289 @@ +package main + +import ( + "crypto/md5" + "fmt" + "io" + "mime" + "os" + "path/filepath" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/ryanuber/go-glob" +) + +type AWS struct { + client *s3.S3 + remote []string + local []string + vargs PluginArgs +} + +func NewAWS(vargs PluginArgs) AWS { + sess := session.New(&aws.Config{ + Credentials: credentials.NewStaticCredentials(vargs.Key, vargs.Secret, ""), + Region: aws.String(vargs.Region), + }) + c := s3.New(sess) + r := make([]string, 1, 1) + l := make([]string, 1, 1) + + return AWS{c, r, l, vargs} +} + +func (a *AWS) visit(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if path == "." { + return nil + } + + if info.IsDir() { + return nil + } + + localPath := strings.TrimPrefix(path, a.vargs.Source) + if strings.HasPrefix(localPath, "/") { + localPath = localPath[1:] + } + + remotePath := filepath.Join(a.vargs.Target, localPath) + + a.local = append(a.local, localPath) + file, err := os.Open(path) + if err != nil { + return err + } + + defer file.Close() + + access := "" + if a.vargs.Access.IsString() { + access = a.vargs.Access.String() + } else if !a.vargs.Access.IsEmpty() { + accessMap := a.vargs.Access.Map() + for pattern := range accessMap { + if match := glob.Glob(pattern, localPath); match == true { + access = accessMap[pattern] + break + } + } + } + + if access == "" { + access = "private" + } + + fileExt := filepath.Ext(localPath) + var contentType string + if a.vargs.ContentType.IsString() { + contentType = a.vargs.ContentType.String() + } else if !a.vargs.ContentType.IsEmpty() { + contentMap := a.vargs.ContentType.Map() + for patternExt := range contentMap { + if patternExt == fileExt { + contentType = contentMap[patternExt] + break + } + } + } + + metadata := map[string]*string{} + vmap := a.vargs.Metadata.Map() + if len(vmap) > 0 { + for pattern := range vmap { + if match := glob.Glob(pattern, localPath); match == true { + for k, v := range vmap[pattern] { + metadata[k] = aws.String(v) + } + break + } + } + } + + if contentType == "" { + contentType = mime.TypeByExtension(fileExt) + } + + exists := false + for _, remoteFile := range a.remote { + if remoteFile == localPath { + exists = true + break + } + } + + if exists { + hash := md5.New() + io.Copy(hash, file) + sum := fmt.Sprintf("\"%x\"", hash.Sum(nil)) + + head, err := a.client.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(a.vargs.Bucket), + Key: aws.String(remotePath), + }) + if err != nil { + return err + } + + if sum == *head.ETag { + shouldCopy := false + + if head.ContentType == nil && contentType != "" { + debug("Content-Type has changed from unset to %s\n", contentType) + shouldCopy = true + } + + if !shouldCopy && head.ContentType != nil && contentType != *head.ContentType { + debug("Content-Type has changed from %s to %s\n", *head.ContentType, contentType) + shouldCopy = true + } + + if !shouldCopy && len(head.Metadata) != len(metadata) { + debug("Count of metadata values has changed for %s\n", localPath) + shouldCopy = true + } + + if !shouldCopy && len(metadata) > 0 { + for k, v := range metadata { + if hv, ok := head.Metadata[k]; ok { + if *v != *hv { + debug("Metadata values have changed for %s\n", localPath) + shouldCopy = true + break + } + } + } + } + + if !shouldCopy { + grant, err := a.client.GetObjectAcl(&s3.GetObjectAclInput{ + Bucket: aws.String(a.vargs.Bucket), + Key: aws.String(remotePath), + }) + if err != nil { + return err + } + + previousAccess := "private" + for _, g := range grant.Grants { + gt := *g.Grantee + if gt.URI != nil { + if *gt.URI == "http://acs.amazonaws.com/groups/global/AllUsers" { + if *g.Permission == "READ" { + previousAccess = "public-read" + } else if *g.Permission == "WRITE" { + previousAccess = "public-read-write" + } + } else if *gt.URI == "http://acs.amazonaws.com/groups/global/AllUsers" { + if *g.Permission == "READ" { + previousAccess = "authenticated-read" + } + } + } + } + + if previousAccess != access { + debug("Permissions for \"%s\" have changed from \"%s\" to \"%s\"\n", remotePath, previousAccess, access) + shouldCopy = true + } + } + + if !shouldCopy { + debug("Skipping \"%s\" because hashes and metadata match\n", localPath) + return nil + } + + fmt.Printf("Updating metadata for \"%s\" Content-Type: \"%s\", ACL: \"%s\"\n", localPath, contentType, access) + _, err = a.client.CopyObject(&s3.CopyObjectInput{ + Bucket: aws.String(a.vargs.Bucket), + Key: aws.String(remotePath), + CopySource: aws.String(fmt.Sprintf("%s/%s", a.vargs.Bucket, remotePath)), + ACL: aws.String(access), + ContentType: aws.String(contentType), + Metadata: metadata, + MetadataDirective: aws.String("REPLACE"), + }) + return err + } + + _, err = file.Seek(0, 0) + if err != nil { + return err + } + } + + fmt.Printf("Uploading \"%s\" with Content-Type \"%s\" and permissions \"%s\"\n", localPath, contentType, access) + _, err = a.client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(a.vargs.Bucket), + Key: aws.String(remotePath), + Body: file, + ContentType: aws.String(contentType), + ACL: aws.String(access), + Metadata: metadata, + }) + return err +} + +func (a *AWS) List(path string) error { + resp, err := a.client.ListObjects(&s3.ListObjectsInput{ + Bucket: aws.String(a.vargs.Bucket), + Prefix: aws.String(path), + }) + if err != nil { + return err + } + + for _, item := range resp.Contents { + a.remote = append(a.remote, *item.Key) + } + + for *resp.IsTruncated { + resp, err = a.client.ListObjects(&s3.ListObjectsInput{ + Bucket: aws.String(a.vargs.Bucket), + Prefix: aws.String(path), + Marker: aws.String(a.remote[len(a.remote)-1]), + }) + + if err != nil { + return err + } + + for _, item := range resp.Contents { + a.remote = append(a.remote, *item.Key) + } + } + + return nil +} + +func (a *AWS) Cleanup() error { + for _, remote := range a.remote { + found := false + for _, local := range a.local { + if local == remote { + found = true + break + } + } + + if !found { + fmt.Printf("Removing remote file \"%s\"\n", remote) + _, err := a.client.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(a.vargs.Bucket), + Key: aws.String(remote), + }) + if err != nil { + return err + } + } + } + + return nil +} diff --git a/main.go b/main.go index bd2c53e..1978d73 100644 --- a/main.go +++ b/main.go @@ -1,219 +1,15 @@ package main import ( - "encoding/json" "fmt" - "mime" "os" "path/filepath" "strings" "github.com/drone/drone-go/drone" "github.com/drone/drone-go/plugin" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" ) -type AWS struct { - client *s3.S3 - uploader *s3manager.Uploader - remote []string - local []string - vargs PluginArgs -} - -type StringMap struct { - parts map[string]string -} - -func (e *StringMap) UnmarshalJSON(b []byte) error { - if len(b) == 0 { - return nil - } - - p := map[string]string{} - if err := json.Unmarshal(b, &p); err != nil { - var s string - if err := json.Unmarshal(b, &s); err != nil { - return err - } - p["_string_"] = s - } - - e.parts = p - return nil -} - -func (e *StringMap) IsEmpty() bool { - if e == nil || len(e.parts) == 0 { - return true - } - - return false -} - -func (e *StringMap) IsString() bool { - if e.IsEmpty() || len(e.parts) != 1 { - return false - } - - _, ok := e.parts["_string_"] - return ok -} - -func (e *StringMap) String() string { - if e.IsEmpty() || !e.IsString() { - return "" - } - - return e.parts["_string_"] -} - -func (e *StringMap) Map() map[string]string { - if e.IsEmpty() || e.IsString() { - return map[string]string{} - } - - return e.parts -} - -type PluginArgs struct { - Key string `json:"access_key"` - Secret string `json:"secret_key"` - Bucket string `json:"bucket"` - Region string `json:"region"` - Source string `json:"source"` - Target string `json:"target"` - Delete bool `json:"delete"` - Access StringMap `json:"acl"` - ContentType StringMap `json:"content_type"` -} - -func NewClient(vargs PluginArgs) AWS { - sess := session.New(&aws.Config{ - Credentials: credentials.NewStaticCredentials(vargs.Key, vargs.Secret, ""), - Region: aws.String(vargs.Region), - }) - client := s3.New(sess) - uploader := s3manager.NewUploader(sess) - remote := make([]string, 1, 1) - local := make([]string, 1, 1) - - a := AWS{client, uploader, remote, local, vargs} - return a -} - -func (a *AWS) visit(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if path == "." { - return nil - } - - if info.IsDir() { - return nil - } - - localPath := strings.TrimPrefix(path, a.vargs.Source) - if strings.HasPrefix(localPath, "/") { - localPath = localPath[1:] - } - - a.local = append(a.local, localPath) - file, err := os.Open(path) - if err != nil { - return err - } - - defer file.Close() - - access := "" - if a.vargs.Access.IsString() { - access = a.vargs.Access.String() - } else if !a.vargs.Access.IsEmpty() { - accessMap := a.vargs.Access.Map() - for pattern := range accessMap { - if match, _ := filepath.Match(pattern, localPath); match == true { - access = accessMap[pattern] - break - } - } - } - - if access == "" { - access = "private" - } - - fileExt := filepath.Ext(localPath) - var contentType string - if a.vargs.ContentType.IsString() { - contentType = a.vargs.ContentType.String() - } else if !a.vargs.ContentType.IsEmpty() { - contentMap := a.vargs.ContentType.Map() - for patternExt := range contentMap { - if patternExt == fileExt { - contentType = contentMap[patternExt] - break - } - } - } - - if contentType == "" { - contentType = mime.TypeByExtension(fileExt) - } - - fmt.Printf("Uploading \"%s\" with Content-Type \"%s\" and permissions \"%s\"\n", localPath, contentType, access) - _, err = a.uploader.Upload(&s3manager.UploadInput{ - Bucket: aws.String(a.vargs.Bucket), - Key: aws.String(filepath.Join(a.vargs.Target, localPath)), - Body: file, - ContentType: aws.String(contentType), - ACL: aws.String(access), - }) - if err != nil { - return err - } - - return nil -} - -func (a *AWS) List(path string) (*s3.ListObjectsOutput, error) { - return a.client.ListObjects(&s3.ListObjectsInput{ - Bucket: aws.String(a.vargs.Bucket), - Prefix: aws.String(path), - }) -} - -func (a *AWS) Cleanup() error { - for _, remote := range a.remote { - found := false - for _, local := range a.local { - if local == remote { - found = true - break - } - } - - if !found { - fmt.Printf("Removing remote file \"%s\"\n", remote) - _, err := a.client.DeleteObject(&s3.DeleteObjectInput{ - Bucket: aws.String(a.vargs.Bucket), - Key: aws.String(remote), - }) - if err != nil { - return err - } - } - } - - return nil -} - func main() { vargs := PluginArgs{} workspace := drone.Workspace{} @@ -242,18 +38,13 @@ func main() { vargs.Target = vargs.Target[1:] } - client := NewClient(vargs) - - resp, err := client.List(vargs.Target) + client := NewAWS(vargs) + err := client.List(vargs.Target) if err != nil { fmt.Println(err) os.Exit(1) } - for _, item := range resp.Contents { - client.remote = append(client.remote, *item.Key) - } - err = filepath.Walk(vargs.Source, client.visit) if err != nil { fmt.Println(err) @@ -268,3 +59,9 @@ func main() { } } } + +func debug(format string, args ...interface{}) { + if os.Getenv("DEBUG") != "" { + fmt.Printf(format, args...) + } +} diff --git a/types.go b/types.go new file mode 100644 index 0000000..e45c7d2 --- /dev/null +++ b/types.go @@ -0,0 +1,97 @@ +package main + +import "encoding/json" + +type PluginArgs struct { + Key string `json:"access_key"` + Secret string `json:"secret_key"` + Bucket string `json:"bucket"` + Region string `json:"region"` + Source string `json:"source"` + Target string `json:"target"` + Delete bool `json:"delete"` + Access StringMap `json:"acl"` + ContentType StringMap `json:"content_type"` + Metadata DeepStringMap `json:"metadata"` +} + +type DeepStringMap struct { + parts map[string]map[string]string +} + +func (e *DeepStringMap) UnmarshalJSON(b []byte) error { + if len(b) == 0 { + return nil + } + + p := map[string]map[string]string{} + if err := json.Unmarshal(b, &p); err != nil { + s := map[string]string{} + if err := json.Unmarshal(b, &s); err != nil { + return err + } + p["*"] = s + } + + e.parts = p + return nil +} + +func (e *DeepStringMap) Map() map[string]map[string]string { + return e.parts +} + +type StringMap struct { + parts map[string]string +} + +func (e *StringMap) UnmarshalJSON(b []byte) error { + if len(b) == 0 { + return nil + } + + p := map[string]string{} + if err := json.Unmarshal(b, &p); err != nil { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + p["_string_"] = s + } + + e.parts = p + return nil +} + +func (e *StringMap) IsEmpty() bool { + if e == nil || len(e.parts) == 0 { + return true + } + + return false +} + +func (e *StringMap) IsString() bool { + if e.IsEmpty() || len(e.parts) != 1 { + return false + } + + _, ok := e.parts["_string_"] + return ok +} + +func (e *StringMap) String() string { + if e.IsEmpty() || !e.IsString() { + return "" + } + + return e.parts["_string_"] +} + +func (e *StringMap) Map() map[string]string { + if e.IsEmpty() || e.IsString() { + return map[string]string{} + } + + return e.parts +} From 733ecf96a4b64bd9b44821ceb95d6fc324f5c0f6 Mon Sep 17 00:00:00 2001 From: Nathan LaFreniere Date: Wed, 18 Nov 2015 16:39:05 -0800 Subject: [PATCH 2/3] update docs --- DOCS.md | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/DOCS.md b/DOCS.md index a8342f9..bd4660e 100644 --- a/DOCS.md +++ b/DOCS.md @@ -8,7 +8,8 @@ Use the S3 sync plugin to synchronize files and folders with an Amazon S3 bucket * `source` - location of folder to sync * `target` - target folder in your S3 bucket * `delete` - deletes files in the target not found in the source -* `content_type` - override default mime-tpyes to use this value +* `content_type` - override default mime-types to use this value +* `metadata` - set custom metadata The following is a sample S3 configuration in your .drone.yml file: @@ -46,6 +47,43 @@ publish: delete: true ``` -In the case of `acl` the key of the map is a glob as matched by [filepath.Match](https://golang.org/pkg/path/filepath/#Match). If there are no matches in your settings for a given file, the default is `"private"`. +In the case of `acl` the key of the map is a glob. If there are no matches in your settings for a given file, the default is `"private"`. The `content_type` field the key is an extension including the leading dot `.`. If you want to set a content type for files with no extension, set the key to the empty string `""`. If there are no matches for the `content_type` of any file, one will automatically be determined for you. + +The `metadata` field can be set as either an object where the keys are the metadata headers: + +```yaml +publish: + s3_sync: + acl: public-read + region: "us-east-1" + bucket: "my-bucket.s3-website-us-east-1.amazonaws.com" + access_key: "970d28f4dd477bc184fbd10b376de753" + secret_key: "9c5785d3ece6a9cdefa42eb99b58986f9095ff1c" + source: folder/to/archive + target: /target/location + delete: true + metadata: + Cache-Control: max-age: 10000 +``` + +Or you can specify metadata for file patterns by using a glob: + +```yaml +publish: + s3_sync: + acl: public-read + region: "us-east-1" + bucket: "my-bucket.s3-website-us-east-1.amazonaws.com" + access_key: "970d28f4dd477bc184fbd10b376de753" + secret_key: "9c5785d3ece6a9cdefa42eb99b58986f9095ff1c" + source: folder/to/archive + target: /target/location + delete: true + metadata: + "*.png": + Cache-Control: max-age: 10000000 + "*.html": + Cache-Control: max-age: 1000 +``` From 8d58de7f0489a9fd4e3e4582d870277710f7792e Mon Sep 17 00:00:00 2001 From: Nathan LaFreniere Date: Wed, 18 Nov 2015 16:52:54 -0800 Subject: [PATCH 3/3] add quotes --- DOCS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DOCS.md b/DOCS.md index bd4660e..00b379d 100644 --- a/DOCS.md +++ b/DOCS.md @@ -65,7 +65,7 @@ publish: target: /target/location delete: true metadata: - Cache-Control: max-age: 10000 + Cache-Control: "max-age: 10000" ``` Or you can specify metadata for file patterns by using a glob: @@ -83,7 +83,7 @@ publish: delete: true metadata: "*.png": - Cache-Control: max-age: 10000000 + Cache-Control: "max-age: 10000000" "*.html": - Cache-Control: max-age: 1000 + Cache-Control: "max-age: 1000" ```