drone-s3-sync/plugin/aws.go

394 lines
9.5 KiB
Go
Raw Normal View History

package plugin
import (
"crypto/md5"
"fmt"
"io"
"mime"
"os"
"path/filepath"
2017-10-22 07:24:40 +00:00
"strings"
2016-03-10 01:22:54 +00:00
"time"
"github.com/aws/aws-sdk-go/aws"
2015-12-31 20:19:34 +00:00
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
2016-07-22 00:27:25 +00:00
"github.com/aws/aws-sdk-go/service/cloudfront"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/ryanuber/go-glob"
"github.com/sirupsen/logrus"
)
type AWS struct {
2016-03-10 01:22:54 +00:00
client *s3.S3
cfClient *cloudfront.CloudFront
remote []string
local []string
2016-07-22 00:27:25 +00:00
plugin *Plugin
}
2016-07-22 00:27:25 +00:00
func NewAWS(p *Plugin) AWS {
2017-10-28 08:00:06 +00:00
sessCfg := &aws.Config{
S3ForcePathStyle: aws.Bool(p.settings.PathStyle),
Region: aws.String(p.settings.Region),
2017-10-28 08:00:06 +00:00
}
if p.settings.Endpoint != "" {
sessCfg.Endpoint = &p.settings.Endpoint
sessCfg.DisableSSL = aws.Bool(strings.HasPrefix(p.settings.Endpoint, "http://"))
2017-10-28 08:00:06 +00:00
}
// allowing to use the instance role or provide a key and secret
if p.settings.AccessKey != "" && p.settings.SecretKey != "" {
sessCfg.Credentials = credentials.NewStaticCredentials(p.settings.AccessKey, p.settings.SecretKey, "")
}
2022-05-02 20:43:40 +00:00
sess, _ := session.NewSession(sessCfg)
2017-10-28 08:00:06 +00:00
c := s3.New(sess)
2016-03-10 01:22:54 +00:00
cf := cloudfront.New(sess)
2022-05-02 20:43:40 +00:00
r := make([]string, 1)
l := make([]string, 1)
2016-07-22 00:27:25 +00:00
return AWS{c, cf, r, l, p}
}
2015-12-20 00:15:04 +00:00
func (a *AWS) Upload(local, remote string) error {
2016-07-22 00:27:25 +00:00
p := a.plugin
2015-12-20 00:15:04 +00:00
if local == "" {
return nil
}
2015-12-20 00:15:04 +00:00
file, err := os.Open(local)
if err != nil {
return err
}
defer file.Close()
var acl string
for pattern := range p.settings.ACL {
2022-05-02 20:43:40 +00:00
if match := glob.Glob(pattern, local); match {
acl = p.settings.ACL[pattern]
2016-07-22 00:27:25 +00:00
break
}
}
if acl == "" {
acl = "private"
}
2015-12-20 00:15:04 +00:00
fileExt := filepath.Ext(local)
2016-07-22 00:27:25 +00:00
var contentType string
for patternExt := range p.settings.ContentType {
2016-07-22 00:27:25 +00:00
if patternExt == fileExt {
contentType = p.settings.ContentType[patternExt]
2016-07-22 00:27:25 +00:00
break
}
}
2016-07-22 00:27:25 +00:00
if contentType == "" {
contentType = mime.TypeByExtension(fileExt)
}
var contentEncoding string
for patternExt := range p.settings.ContentEncoding {
2016-07-22 00:27:25 +00:00
if patternExt == fileExt {
contentEncoding = p.settings.ContentEncoding[patternExt]
2016-07-22 00:27:25 +00:00
break
}
}
2016-10-19 17:56:28 +00:00
var cacheControl string
for pattern := range p.settings.CacheControl {
2022-05-02 20:43:40 +00:00
if match := glob.Glob(pattern, local); match {
cacheControl = p.settings.CacheControl[pattern]
2016-10-19 17:56:28 +00:00
break
}
}
metadata := map[string]*string{}
for pattern := range p.settings.Metadata {
2022-05-02 20:43:40 +00:00
if match := glob.Glob(pattern, local); match {
for k, v := range p.settings.Metadata[pattern] {
2016-07-22 00:27:25 +00:00
metadata[k] = aws.String(v)
}
2016-07-22 00:27:25 +00:00
break
}
}
head, err := a.client.HeadObject(&s3.HeadObjectInput{
Bucket: aws.String(p.settings.Bucket),
2015-12-20 00:15:04 +00:00
Key: aws.String(remote),
})
2016-11-19 13:34:46 +00:00
if err != nil && err.(awserr.Error).Code() != "404" {
if err.(awserr.Error).Code() == "404" {
2015-12-31 20:19:34 +00:00
return err
}
logrus.Debugf("'%s' not found in bucket, uploading with content-type '%s' and permissions '%s'", local, contentType, acl)
2022-05-02 20:43:40 +00:00
putObject := &s3.PutObjectInput{
Bucket: aws.String(p.settings.Bucket),
2015-12-31 20:19:34 +00:00
Key: aws.String(remote),
Body: file,
ContentType: aws.String(contentType),
ACL: aws.String(acl),
2015-12-31 20:19:34 +00:00
Metadata: metadata,
}
2016-10-19 17:56:28 +00:00
if len(cacheControl) > 0 {
putObject.CacheControl = aws.String(cacheControl)
}
2016-07-22 00:27:25 +00:00
if len(contentEncoding) > 0 {
putObject.ContentEncoding = aws.String(contentEncoding)
}
2016-11-19 13:34:46 +00:00
// skip upload during dry run
if a.plugin.settings.DryRun {
2016-11-19 13:34:46 +00:00
return nil
}
_, err = a.client.PutObject(putObject)
return err
}
2015-12-31 20:19:34 +00:00
hash := md5.New()
2022-05-02 20:43:40 +00:00
_, _ = io.Copy(hash, file)
sum := fmt.Sprintf("'%x'", hash.Sum(nil))
2015-12-31 20:19:34 +00:00
if sum == *head.ETag {
shouldCopy := false
2015-12-31 20:19:34 +00:00
if head.ContentType == nil && contentType != "" {
logrus.Debugf("content-type has changed from unset to %s", contentType)
2015-12-31 20:19:34 +00:00
shouldCopy = true
}
2015-12-31 20:19:34 +00:00
if !shouldCopy && head.ContentType != nil && contentType != *head.ContentType {
logrus.Debugf("content-type has changed from %s to %s", *head.ContentType, contentType)
2015-12-31 20:19:34 +00:00
shouldCopy = true
}
if !shouldCopy && head.ContentEncoding == nil && contentEncoding != "" {
logrus.Debugf("Content-Encoding has changed from unset to %s", contentEncoding)
shouldCopy = true
}
if !shouldCopy && head.ContentEncoding != nil && contentEncoding != *head.ContentEncoding {
logrus.Debugf("Content-Encoding has changed from %s to %s", *head.ContentEncoding, contentEncoding)
shouldCopy = true
}
2016-10-19 17:56:28 +00:00
if !shouldCopy && head.CacheControl == nil && cacheControl != "" {
logrus.Debugf("cache-control has changed from unset to %s", cacheControl)
2016-10-19 17:56:28 +00:00
shouldCopy = true
}
if !shouldCopy && head.CacheControl != nil && cacheControl != *head.CacheControl {
logrus.Debugf("cache-control has changed from %s to %s", *head.CacheControl, cacheControl)
2016-10-19 17:56:28 +00:00
shouldCopy = true
}
2015-12-31 20:19:34 +00:00
if !shouldCopy && len(head.Metadata) != len(metadata) {
logrus.Debugf("count of metadata values has changed for %s", local)
2015-12-31 20:19:34 +00:00
shouldCopy = true
}
2015-12-31 20:19:34 +00:00
if !shouldCopy && len(metadata) > 0 {
for k, v := range metadata {
if hv, ok := head.Metadata[k]; ok {
if *v != *hv {
logrus.Debugf("metadata values have changed for %s", local)
2015-12-31 20:19:34 +00:00
shouldCopy = true
break
}
}
}
2015-12-31 20:19:34 +00:00
}
2015-12-31 20:19:34 +00:00
if !shouldCopy {
grant, err := a.client.GetObjectAcl(&s3.GetObjectAclInput{
Bucket: aws.String(p.settings.Bucket),
2015-12-31 20:19:34 +00:00
Key: aws.String(remote),
})
if err != nil {
return err
}
previousACL := "private"
2015-12-31 20:19:34 +00:00
for _, g := range grant.Grants {
gt := *g.Grantee
if gt.URI != nil {
if *gt.URI == "http://acs.amazonaws.com/groups/global/AllUsers" {
if *g.Permission == "READ" {
previousACL = "public-read"
2015-12-31 20:19:34 +00:00
} else if *g.Permission == "WRITE" {
previousACL = "public-read-write"
2015-12-31 20:19:34 +00:00
}
2022-05-02 20:43:40 +00:00
}
if *gt.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" {
2015-12-31 20:19:34 +00:00
if *g.Permission == "READ" {
previousACL = "authenticated-read"
}
}
}
}
if previousACL != acl {
logrus.Debugf("permissions for '%s' have changed from '%s' to '%s'", remote, previousACL, acl)
2015-12-31 20:19:34 +00:00
shouldCopy = true
}
}
2015-12-31 20:19:34 +00:00
if !shouldCopy {
logrus.Debugf("skipping '%s' because hashes and metadata match", local)
2015-12-31 20:19:34 +00:00
return nil
}
2015-12-31 20:19:34 +00:00
logrus.Debugf("updating metadata for '%s' content-type: '%s', ACL: '%s'", local, contentType, acl)
2022-05-02 20:43:40 +00:00
copyObject := &s3.CopyObjectInput{
Bucket: aws.String(p.settings.Bucket),
2015-12-31 20:19:34 +00:00
Key: aws.String(remote),
CopySource: aws.String(fmt.Sprintf("%s/%s", p.settings.Bucket, remote)),
ACL: aws.String(acl),
2015-12-31 20:19:34 +00:00
ContentType: aws.String(contentType),
Metadata: metadata,
MetadataDirective: aws.String("REPLACE"),
}
2016-10-19 17:56:28 +00:00
if len(cacheControl) > 0 {
copyObject.CacheControl = aws.String(cacheControl)
}
2016-07-22 00:27:25 +00:00
if len(contentEncoding) > 0 {
copyObject.ContentEncoding = aws.String(contentEncoding)
}
2016-11-19 13:34:46 +00:00
// skip update if dry run
if a.plugin.settings.DryRun {
2016-11-19 13:34:46 +00:00
return nil
}
_, err = a.client.CopyObject(copyObject)
2015-12-31 20:19:34 +00:00
return err
2022-05-02 20:43:40 +00:00
}
2015-12-31 21:19:41 +00:00
2022-05-02 20:43:40 +00:00
_, err = file.Seek(0, 0)
if err != nil {
return err
}
logrus.Debugf("uploading '%s' with content-type '%s' and permissions '%s'", local, contentType, acl)
2022-05-02 20:43:40 +00:00
putObject := &s3.PutObjectInput{
Bucket: aws.String(p.settings.Bucket),
2022-05-02 20:43:40 +00:00
Key: aws.String(remote),
Body: file,
ContentType: aws.String(contentType),
ACL: aws.String(acl),
2022-05-02 20:43:40 +00:00
Metadata: metadata,
}
2016-10-19 17:56:28 +00:00
2022-05-02 20:43:40 +00:00
if len(cacheControl) > 0 {
putObject.CacheControl = aws.String(cacheControl)
}
2022-05-02 20:43:40 +00:00
if len(contentEncoding) > 0 {
putObject.ContentEncoding = aws.String(contentEncoding)
}
2016-11-19 13:34:46 +00:00
2022-05-02 20:43:40 +00:00
// skip upload if dry run
if a.plugin.settings.DryRun {
2022-05-02 20:43:40 +00:00
return nil
}
2022-05-02 20:43:40 +00:00
_, err = a.client.PutObject(putObject)
return err
}
2015-12-20 00:15:04 +00:00
func (a *AWS) Redirect(path, location string) error {
2016-07-22 00:27:25 +00:00
p := a.plugin
logrus.Debugf("adding redirect from '%s' to '%s'", path, location)
2016-11-19 13:34:46 +00:00
if a.plugin.settings.DryRun {
2016-11-19 13:34:46 +00:00
return nil
}
2015-12-20 00:15:04 +00:00
_, err := a.client.PutObject(&s3.PutObjectInput{
Bucket: aws.String(p.settings.Bucket),
Key: aws.String(path),
ACL: aws.String("public-read"),
2015-12-20 00:15:04 +00:00
WebsiteRedirectLocation: aws.String(location),
})
return err
}
2015-12-04 19:24:34 +00:00
2015-12-20 00:15:04 +00:00
func (a *AWS) Delete(remote string) error {
2016-07-22 00:27:25 +00:00
p := a.plugin
logrus.Debugf("removing remote file '%s'", remote)
2016-11-19 13:34:46 +00:00
if a.plugin.settings.DryRun {
2016-11-19 13:34:46 +00:00
return nil
}
2015-12-20 00:15:04 +00:00
_, err := a.client.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(p.settings.Bucket),
2015-12-20 00:15:04 +00:00
Key: aws.String(remote),
})
return err
2015-12-04 19:24:34 +00:00
}
2015-12-20 00:15:04 +00:00
func (a *AWS) List(path string) ([]string, error) {
2016-07-22 00:27:25 +00:00
p := a.plugin
2022-05-02 20:43:40 +00:00
remote := make([]string, 1)
resp, err := a.client.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(p.settings.Bucket),
Prefix: aws.String(path),
})
if err != nil {
2015-12-20 00:15:04 +00:00
return remote, err
}
for _, item := range resp.Contents {
2015-12-20 00:15:04 +00:00
remote = append(remote, *item.Key)
}
for *resp.IsTruncated {
resp, err = a.client.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(p.settings.Bucket),
Prefix: aws.String(path),
2015-12-20 00:15:04 +00:00
Marker: aws.String(remote[len(remote)-1]),
})
if err != nil {
2015-12-20 00:15:04 +00:00
return remote, err
}
for _, item := range resp.Contents {
2015-12-20 00:15:04 +00:00
remote = append(remote, *item.Key)
}
}
2015-12-20 00:15:04 +00:00
return remote, nil
}
2016-03-10 01:22:54 +00:00
func (a *AWS) Invalidate(invalidatePath string) error {
2016-07-22 00:27:25 +00:00
p := a.plugin
logrus.Debugf("invalidating '%s'", invalidatePath)
2016-03-10 01:22:54 +00:00
_, err := a.cfClient.CreateInvalidation(&cloudfront.CreateInvalidationInput{
DistributionId: aws.String(p.settings.CloudFrontDistribution),
2016-03-10 01:22:54 +00:00
InvalidationBatch: &cloudfront.InvalidationBatch{
CallerReference: aws.String(time.Now().Format(time.RFC3339Nano)),
Paths: &cloudfront.Paths{
Quantity: aws.Int64(1),
Items: []*string{
aws.String(invalidatePath),
},
},
},
})
return err
}