drone-s3-sync/plugin/aws.go

package plugin

import (
	"crypto/md5"
	"fmt"
	"io"
	"mime"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudfront"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/ryanuber/go-glob"
	"github.com/sirupsen/logrus"
)

type AWS struct {
	client   *s3.S3
	cfClient *cloudfront.CloudFront
	remote   []string
	local    []string
	plugin   *Plugin
}

func NewAWS(p *Plugin) AWS {
	sessCfg := &aws.Config{
		S3ForcePathStyle: aws.Bool(p.settings.PathStyle),
		Region:           aws.String(p.settings.Region),
	}

	if p.settings.Endpoint != "" {
		sessCfg.Endpoint = &p.settings.Endpoint
		sessCfg.DisableSSL = aws.Bool(strings.HasPrefix(p.settings.Endpoint, "http://"))
	}

	// allowing to use the instance role or provide a key and secret
	if p.settings.AccessKey != "" && p.settings.SecretKey != "" {
		sessCfg.Credentials = credentials.NewStaticCredentials(p.settings.AccessKey, p.settings.SecretKey, "")
	}

	sess, _ := session.NewSession(sessCfg)

	c := s3.New(sess)
	cf := cloudfront.New(sess)
	r := make([]string, 1)
	l := make([]string, 1)

	return AWS{c, cf, r, l, p}
}

func (a *AWS) Upload(local, remote string) error {
	p := a.plugin
	if local == "" {
		return nil
	}

	file, err := os.Open(local)
	if err != nil {
		return err
	}

	defer file.Close()

	var access string
	for pattern := range p.settings.Access {
		if match := glob.Glob(pattern, local); match {
			access = p.settings.Access[pattern]
			break
		}
	}

	if access == "" {
		access = "private"
	}

	fileExt := filepath.Ext(local)

	var contentType string
	for patternExt := range p.settings.ContentType {
		if patternExt == fileExt {
			contentType = p.settings.ContentType[patternExt]
			break
		}
	}

	if contentType == "" {
		contentType = mime.TypeByExtension(fileExt)
	}

	var contentEncoding string
	for patternExt := range p.settings.ContentEncoding {
		if patternExt == fileExt {
			contentEncoding = p.settings.ContentEncoding[patternExt]
			break
		}
	}

	var cacheControl string
	for pattern := range p.settings.CacheControl {
		if match := glob.Glob(pattern, local); match {
			cacheControl = p.settings.CacheControl[pattern]
			break
		}
	}

	metadata := map[string]*string{}
	for pattern := range p.settings.Metadata {
		if match := glob.Glob(pattern, local); match {
			for k, v := range p.settings.Metadata[pattern] {
				metadata[k] = aws.String(v)
			}
			break
		}
	}

	head, err := a.client.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(p.settings.Bucket),
		Key:    aws.String(remote),
	})
	if err != nil && err.(awserr.Error).Code() != "404" {
		if err.(awserr.Error).Code() == "404" {
			return err
		}

		logrus.Debugf("'%s' not found in bucket, uploading with content-type '%s' and permissions '%s'", local, contentType, access)
		putObject := &s3.PutObjectInput{
			Bucket:      aws.String(p.settings.Bucket),
			Key:         aws.String(remote),
			Body:        file,
			ContentType: aws.String(contentType),
			ACL:         aws.String(access),
			Metadata:    metadata,
		}

		if len(cacheControl) > 0 {
			putObject.CacheControl = aws.String(cacheControl)
		}

		if len(contentEncoding) > 0 {
			putObject.ContentEncoding = aws.String(contentEncoding)
		}

		// skip upload during dry run
		if a.plugin.settings.DryRun {
			return nil
		}

		_, err = a.client.PutObject(putObject)
		return err
	}

	hash := md5.New()
	_, _ = io.Copy(hash, file)
	sum := fmt.Sprintf("'%x'", hash.Sum(nil))

	if sum == *head.ETag {
		shouldCopy := false

		if head.ContentType == nil && contentType != "" {
			logrus.Debugf("content-type has changed from unset to %s", contentType)
			shouldCopy = true
		}

		if !shouldCopy && head.ContentType != nil && contentType != *head.ContentType {
			logrus.Debugf("content-type has changed from %s to %s", *head.ContentType, contentType)
			shouldCopy = true
		}

		if !shouldCopy && head.ContentEncoding == nil && contentEncoding != "" {
			logrus.Debugf("Content-Encoding has changed from unset to %s", contentEncoding)
			shouldCopy = true
		}

		if !shouldCopy && head.ContentEncoding != nil && contentEncoding != *head.ContentEncoding {
			logrus.Debugf("Content-Encoding has changed from %s to %s", *head.ContentEncoding, contentEncoding)
			shouldCopy = true
		}

		if !shouldCopy && head.CacheControl == nil && cacheControl != "" {
			logrus.Debugf("cache-control has changed from unset to %s", cacheControl)
			shouldCopy = true
		}

		if !shouldCopy && head.CacheControl != nil && cacheControl != *head.CacheControl {
			logrus.Debugf("cache-control has changed from %s to %s", *head.CacheControl, cacheControl)
			shouldCopy = true
		}

		if !shouldCopy && len(head.Metadata) != len(metadata) {
			logrus.Debugf("count of metadata values has changed for %s", local)
			shouldCopy = true
		}

		if !shouldCopy && len(metadata) > 0 {
			for k, v := range metadata {
				if hv, ok := head.Metadata[k]; ok {
					if *v != *hv {
						logrus.Debugf("metadata values have changed for %s", local)
						shouldCopy = true
						break
					}
				}
			}
		}

		if !shouldCopy {
			grant, err := a.client.GetObjectAcl(&s3.GetObjectAclInput{
				Bucket: aws.String(p.settings.Bucket),
				Key:    aws.String(remote),
			})
			if err != nil {
				return err
			}

			previousAccess := "private"
			for _, g := range grant.Grants {
				gt := *g.Grantee
				if gt.URI != nil {
					if *gt.URI == "http://acs.amazonaws.com/groups/global/AllUsers" {
						if *g.Permission == "READ" {
							previousAccess = "public-read"
						} else if *g.Permission == "WRITE" {
							previousAccess = "public-read-write"
						}
					}
					if *gt.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" {
						if *g.Permission == "READ" {
							previousAccess = "authenticated-read"
						}
					}
				}
			}

			if previousAccess != access {
				logrus.Debugf("permissions for '%s' have changed from '%s' to '%s'", remote, previousAccess, access)
				shouldCopy = true
			}
		}

		if !shouldCopy {
			logrus.Debugf("skipping '%s' because hashes and metadata match", local)
			return nil
		}

		logrus.Debugf("updating metadata for '%s' content-type: '%s', ACL: '%s'", local, contentType, access)
		copyObject := &s3.CopyObjectInput{
			Bucket:            aws.String(p.settings.Bucket),
			Key:               aws.String(remote),
			CopySource:        aws.String(fmt.Sprintf("%s/%s", p.settings.Bucket, remote)),
			ACL:               aws.String(access),
			ContentType:       aws.String(contentType),
			Metadata:          metadata,
			MetadataDirective: aws.String("REPLACE"),
		}

		if len(cacheControl) > 0 {
			copyObject.CacheControl = aws.String(cacheControl)
		}

		if len(contentEncoding) > 0 {
			copyObject.ContentEncoding = aws.String(contentEncoding)
		}

		// skip update if dry run
		if a.plugin.settings.DryRun {
			return nil
		}

		_, err = a.client.CopyObject(copyObject)
		return err
	}

	_, err = file.Seek(0, 0)
	if err != nil {
		return err
	}

	logrus.Debugf("uploading '%s' with content-type '%s' and permissions '%s'", local, contentType, access)
	putObject := &s3.PutObjectInput{
		Bucket:      aws.String(p.settings.Bucket),
		Key:         aws.String(remote),
		Body:        file,
		ContentType: aws.String(contentType),
		ACL:         aws.String(access),
		Metadata:    metadata,
	}

	if len(cacheControl) > 0 {
		putObject.CacheControl = aws.String(cacheControl)
	}

	if len(contentEncoding) > 0 {
		putObject.ContentEncoding = aws.String(contentEncoding)
	}

	// skip upload if dry run
	if a.plugin.settings.DryRun {
		return nil
	}

	_, err = a.client.PutObject(putObject)
	return err
}

func (a *AWS) Redirect(path, location string) error {
	p := a.plugin
	logrus.Debugf("adding redirect from '%s' to '%s'", path, location)

	if a.plugin.settings.DryRun {
		return nil
	}

	_, err := a.client.PutObject(&s3.PutObjectInput{
		Bucket:                  aws.String(p.settings.Bucket),
		Key:                     aws.String(path),
		ACL:                     aws.String("public-read"),
		WebsiteRedirectLocation: aws.String(location),
	})
	return err
}

func (a *AWS) Delete(remote string) error {
	p := a.plugin
	logrus.Debugf("removing remote file '%s'", remote)

	if a.plugin.settings.DryRun {
		return nil
	}

	_, err := a.client.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(p.settings.Bucket),
		Key:    aws.String(remote),
	})
	return err
}

func (a *AWS) List(path string) ([]string, error) {
	p := a.plugin
	remote := make([]string, 1)
	resp, err := a.client.ListObjects(&s3.ListObjectsInput{
		Bucket: aws.String(p.settings.Bucket),
		Prefix: aws.String(path),
	})
	if err != nil {
		return remote, err
	}

	for _, item := range resp.Contents {
		remote = append(remote, *item.Key)
	}

	for *resp.IsTruncated {
		resp, err = a.client.ListObjects(&s3.ListObjectsInput{
			Bucket: aws.String(p.settings.Bucket),
			Prefix: aws.String(path),
			Marker: aws.String(remote[len(remote)-1]),
		})

		if err != nil {
			return remote, err
		}

		for _, item := range resp.Contents {
			remote = append(remote, *item.Key)
		}
	}

	return remote, nil
}

func (a *AWS) Invalidate(invalidatePath string) error {
	p := a.plugin
	logrus.Debugf("invalidating '%s'", invalidatePath)
	_, err := a.cfClient.CreateInvalidation(&cloudfront.CreateInvalidationInput{
		DistributionId: aws.String(p.settings.CloudFrontDistribution),
		InvalidationBatch: &cloudfront.InvalidationBatch{
			CallerReference: aws.String(time.Now().Format(time.RFC3339Nano)),
			Paths: &cloudfront.Paths{
				Quantity: aws.Int64(1),
				Items: []*string{
					aws.String(invalidatePath),
				},
			},
		},
	})
	return err
}