S3 IAM support (#7058)

* add s3 dependencly

* rewrite image uploader by aws-sdk-go

* improve backward compatibility of s3 image uploading

* set default for expires option

* remove acl and expires

pkg/components/imguploader/imguploader.go

@@ -2,6 +2,7 @@ package imguploader
 
 import (
 	"fmt"
+	"regexp"
 
 	"github.com/grafana/grafana/pkg/setting"
 )
@@ -30,19 +31,21 @@ func NewImageUploader() (ImageUploader, error) {
 		accessKey := s3sec.Key("access_key").MustString("")
 		secretKey := s3sec.Key("secret_key").MustString("")
 
-		if bucket == "" {
+		region := ""
+		rBucket := regexp.MustCompile(`https?:\/\/(.*)\.s3(-([^.]+))?\.amazonaws\.com\/?`)
+		matches := rBucket.FindStringSubmatch(bucket)
+		if len(matches) == 0 {
 			return nil, fmt.Errorf("Could not find bucket setting for image.uploader.s3")
+		} else {
+			bucket = matches[1]
+			if matches[3] != "" {
+				region = matches[3]
+			} else {
+				region = "us-east-1"
+			}
 		}
 
-		if accessKey == "" {
-			return nil, fmt.Errorf("Could not find accessKey setting for image.uploader.s3")
-		}
-
-		if secretKey == "" {
-			return nil, fmt.Errorf("Could not find secretKey setting for image.uploader.s3")
-		}
-
-		return NewS3Uploader(bucket, accessKey, secretKey), nil
+		return NewS3Uploader(region, bucket, "public-read", accessKey, secretKey), nil
 	case "webdav":
 		webdavSec, err := setting.Cfg.GetSection("external_image_storage.webdav")
 		if err != nil {

pkg/components/imguploader/imguploader_test.go

@@ -19,7 +19,7 @@ func TestImageUploaderFactory(t *testing.T) {
 			setting.ImageUploadProvider = "s3"
 
 			s3sec, err := setting.Cfg.GetSection("external_image_storage.s3")
-			s3sec.NewKey("bucket_url", "bucket_url")
+			s3sec.NewKey("bucket_url", "https://foo.bar.baz.s3-us-east-2.amazonaws.com")
 			s3sec.NewKey("access_key", "access_key")
 			s3sec.NewKey("secret_key", "secret_key")
 
@@ -29,9 +29,10 @@ func TestImageUploaderFactory(t *testing.T) {
 			original, ok := uploader.(*S3Uploader)
 
 			So(ok, ShouldBeTrue)
+			So(original.region, ShouldEqual, "us-east-2")
+			So(original.bucket, ShouldEqual, "foo.bar.baz")
 			So(original.accessKey, ShouldEqual, "access_key")
 			So(original.secretKey, ShouldEqual, "secret_key")
-			So(original.bucket, ShouldEqual, "bucket_url")
 		})
 
 		Convey("Webdav uploader", func() {

pkg/components/imguploader/s3uploader.go

@@ -1,26 +1,33 @@
 package imguploader
 
 import (
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"path"
+	"os"
+	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/util"
-	"github.com/kr/s3/s3util"
 )
 
 type S3Uploader struct {
+	region    string
 	bucket    string
+	acl       string
 	secretKey string
 	accessKey string
 	log       log.Logger
 }
 
-func NewS3Uploader(bucket, accessKey, secretKey string) *S3Uploader {
+func NewS3Uploader(region, bucket, acl, accessKey, secretKey string) *S3Uploader {
 	return &S3Uploader{
+		region:    region,
 		bucket:    bucket,
+		acl:       acl,
 		accessKey: accessKey,
 		secretKey: secretKey,
 		log:       log.New("s3uploader"),
@@ -28,42 +35,41 @@ func NewS3Uploader(bucket, accessKey, secretKey string) *S3Uploader {
 }
 
 func (u *S3Uploader) Upload(imageDiskPath string) (string, error) {
-
-	s3util.DefaultConfig.AccessKey = u.accessKey
-	s3util.DefaultConfig.SecretKey = u.secretKey
-
-	header := make(http.Header)
-	header.Add("x-amz-acl", "public-read")
-	header.Add("Content-Type", "image/png")
-
-	var imageUrl *url.URL
-	var err error
-
-	if imageUrl, err = url.Parse(u.bucket); err != nil {
-		return "", err
+	sess := session.New()
+	creds := credentials.NewChainCredentials(
+		[]credentials.Provider{
+			&credentials.StaticProvider{Value: credentials.Value{
+				AccessKeyID:     u.accessKey,
+				SecretAccessKey: u.secretKey,
+			}},
+			&credentials.EnvProvider{},
+			&ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(sess), ExpiryWindow: 5 * time.Minute},
+		})
+	cfg := &aws.Config{
+		Region:      aws.String(u.region),
+		Credentials: creds,
 	}
 
-	// add image to url
-	imageUrl.Path = path.Join(imageUrl.Path, util.GetRandomString(20)+".png")
-	imageUrlString := imageUrl.String()
-	log.Debug("Uploading image to s3", "url", imageUrlString)
+	key := util.GetRandomString(20) + ".png"
+	log.Debug("Uploading image to s3", "bucket = ", u.bucket, ", key = ", key)
 
-	writer, err := s3util.Create(imageUrlString, header, nil)
+	file, err := os.Open(imageDiskPath)
 	if err != nil {
 		return "", err
 	}
 
-	defer writer.Close()
-
-	imgData, err := ioutil.ReadFile(imageDiskPath)
+	svc := s3.New(session.New(cfg), cfg)
+	params := &s3.PutObjectInput{
+		Bucket:      aws.String(u.bucket),
+		Key:         aws.String(key),
+		ACL:         aws.String(u.acl),
+		Body:        file,
+		ContentType: aws.String("image/png"),
+	}
+	_, err = svc.PutObject(params)
 	if err != nil {
 		return "", err
 	}
 
-	_, err = writer.Write(imgData)
-	if err != nil {
-		return "", err
-	}
-
-	return imageUrlString, nil
+	return "https://" + u.bucket + ".s3.amazonaws.com/" + key, nil
 }

pkg/setting/setting.go

@@ -14,9 +14,10 @@ import (
 	"runtime"
 	"strings"
 
-	"github.com/go-macaron/session"
 	"gopkg.in/ini.v1"
 
+	"github.com/go-macaron/session"
+
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/util"
 )

vendor/github.com/aws/aws-sdk-go/service/s3/bucket_location.go

@@ -0,0 +1,43 @@
+package s3
+
+import (
+	"io/ioutil"
+	"regexp"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/awsutil"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+var reBucketLocation = regexp.MustCompile(`>([^<>]+)<\/Location`)
+
+func buildGetBucketLocation(r *request.Request) {
+	if r.DataFilled() {
+		out := r.Data.(*GetBucketLocationOutput)
+		b, err := ioutil.ReadAll(r.HTTPResponse.Body)
+		if err != nil {
+			r.Error = awserr.New("SerializationError", "failed reading response body", err)
+			return
+		}
+
+		match := reBucketLocation.FindSubmatch(b)
+		if len(match) > 1 {
+			loc := string(match[1])
+			out.LocationConstraint = &loc
+		}
+	}
+}
+
+func populateLocationConstraint(r *request.Request) {
+	if r.ParamsFilled() && aws.StringValue(r.Config.Region) != "us-east-1" {
+		in := r.Params.(*CreateBucketInput)
+		if in.CreateBucketConfiguration == nil {
+			r.Params = awsutil.CopyOf(r.Params)
+			in = r.Params.(*CreateBucketInput)
+			in.CreateBucketConfiguration = &CreateBucketConfiguration{
+				LocationConstraint: r.Config.Region,
+			}
+		}
+	}
+}

vendor/github.com/aws/aws-sdk-go/service/s3/content_md5.go

@@ -0,0 +1,36 @@
+package s3
+
+import (
+	"crypto/md5"
+	"encoding/base64"
+	"io"
+
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+// contentMD5 computes and sets the HTTP Content-MD5 header for requests that
+// require it.
+func contentMD5(r *request.Request) {
+	h := md5.New()
+
+	// hash the body.  seek back to the first position after reading to reset
+	// the body for transmission.  copy errors may be assumed to be from the
+	// body.
+	_, err := io.Copy(h, r.Body)
+	if err != nil {
+		r.Error = awserr.New("ContentMD5", "failed to read body", err)
+		return
+	}
+	_, err = r.Body.Seek(0, 0)
+	if err != nil {
+		r.Error = awserr.New("ContentMD5", "failed to seek body", err)
+		return
+	}
+
+	// encode the md5 checksum in base64 and set the request header.
+	sum := h.Sum(nil)
+	sum64 := make([]byte, base64.StdEncoding.EncodedLen(len(sum)))
+	base64.StdEncoding.Encode(sum64, sum)
+	r.HTTPRequest.Header.Set("Content-MD5", string(sum64))
+}

vendor/github.com/aws/aws-sdk-go/service/s3/customizations.go

@@ -0,0 +1,46 @@
+package s3
+
+import (
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+func init() {
+	initClient = defaultInitClientFn
+	initRequest = defaultInitRequestFn
+}
+
+func defaultInitClientFn(c *client.Client) {
+	// Support building custom endpoints based on config
+	c.Handlers.Build.PushFront(updateEndpointForS3Config)
+
+	// Require SSL when using SSE keys
+	c.Handlers.Validate.PushBack(validateSSERequiresSSL)
+	c.Handlers.Build.PushBack(computeSSEKeys)
+
+	// S3 uses custom error unmarshaling logic
+	c.Handlers.UnmarshalError.Clear()
+	c.Handlers.UnmarshalError.PushBack(unmarshalError)
+}
+
+func defaultInitRequestFn(r *request.Request) {
+	// Add reuest handlers for specific platforms.
+	// e.g. 100-continue support for PUT requests using Go 1.6
+	platformRequestHandlers(r)
+
+	switch r.Operation.Name {
+	case opPutBucketCors, opPutBucketLifecycle, opPutBucketPolicy,
+		opPutBucketTagging, opDeleteObjects, opPutBucketLifecycleConfiguration,
+		opPutBucketReplication:
+		// These S3 operations require Content-MD5 to be set
+		r.Handlers.Build.PushBack(contentMD5)
+	case opGetBucketLocation:
+		// GetBucketLocation has custom parsing logic
+		r.Handlers.Unmarshal.PushFront(buildGetBucketLocation)
+	case opCreateBucket:
+		// Auto-populate LocationConstraint with current region
+		r.Handlers.Validate.PushFront(populateLocationConstraint)
+	case opCopyObject, opUploadPartCopy, opCompleteMultipartUpload:
+		r.Handlers.Unmarshal.PushFront(copyMultipartStatusOKUnmarhsalError)
+	}
+}

vendor/github.com/aws/aws-sdk-go/service/s3/host_style_bucket.go

@@ -0,0 +1,186 @@
+package s3
+
+import (
+	"bytes"
+	"fmt"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/awsutil"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+// an operationBlacklist is a list of operation names that should a
+// request handler should not be executed with.
+type operationBlacklist []string
+
+// Continue will return true of the Request's operation name is not
+// in the blacklist. False otherwise.
+func (b operationBlacklist) Continue(r *request.Request) bool {
+	for i := 0; i < len(b); i++ {
+		if b[i] == r.Operation.Name {
+			return false
+		}
+	}
+	return true
+}
+
+var accelerateOpBlacklist = operationBlacklist{
+	opListBuckets, opCreateBucket, opDeleteBucket,
+}
+
+// Request handler to automatically add the bucket name to the endpoint domain
+// if possible. This style of bucket is valid for all bucket names which are
+// DNS compatible and do not contain "."
+func updateEndpointForS3Config(r *request.Request) {
+	forceHostStyle := aws.BoolValue(r.Config.S3ForcePathStyle)
+	accelerate := aws.BoolValue(r.Config.S3UseAccelerate)
+
+	if accelerate && accelerateOpBlacklist.Continue(r) {
+		if forceHostStyle {
+			if r.Config.Logger != nil {
+				r.Config.Logger.Log("ERROR: aws.Config.S3UseAccelerate is not compatible with aws.Config.S3ForcePathStyle, ignoring S3ForcePathStyle.")
+			}
+		}
+		updateEndpointForAccelerate(r)
+	} else if !forceHostStyle && r.Operation.Name != opGetBucketLocation {
+		updateEndpointForHostStyle(r)
+	}
+}
+
+func updateEndpointForHostStyle(r *request.Request) {
+	bucket, ok := bucketNameFromReqParams(r.Params)
+	if !ok {
+		// Ignore operation requests if the bucketname was not provided
+		// if this is an input validation error the validation handler
+		// will report it.
+		return
+	}
+
+	if !hostCompatibleBucketName(r.HTTPRequest.URL, bucket) {
+		// bucket name must be valid to put into the host
+		return
+	}
+
+	moveBucketToHost(r.HTTPRequest.URL, bucket)
+}
+
+var (
+	accelElem = []byte("s3-accelerate.dualstack.")
+)
+
+func updateEndpointForAccelerate(r *request.Request) {
+	bucket, ok := bucketNameFromReqParams(r.Params)
+	if !ok {
+		// Ignore operation requests if the bucketname was not provided
+		// if this is an input validation error the validation handler
+		// will report it.
+		return
+	}
+
+	if !hostCompatibleBucketName(r.HTTPRequest.URL, bucket) {
+		r.Error = awserr.New("InvalidParameterException",
+			fmt.Sprintf("bucket name %s is not compatibile with S3 Accelerate", bucket),
+			nil)
+		return
+	}
+
+	// Change endpoint from s3(-[a-z0-1-])?.amazonaws.com to s3-accelerate.amazonaws.com
+	r.HTTPRequest.URL.Host = replaceHostRegion(r.HTTPRequest.URL.Host, "accelerate")
+
+	if aws.BoolValue(r.Config.UseDualStack) {
+		host := []byte(r.HTTPRequest.URL.Host)
+
+		// Strip region from hostname
+		if idx := bytes.Index(host, accelElem); idx >= 0 {
+			start := idx + len(accelElem)
+			if end := bytes.IndexByte(host[start:], '.'); end >= 0 {
+				end += start + 1
+				copy(host[start:], host[end:])
+				host = host[:len(host)-(end-start)]
+				r.HTTPRequest.URL.Host = string(host)
+			}
+		}
+	}
+
+	moveBucketToHost(r.HTTPRequest.URL, bucket)
+}
+
+// Attempts to retrieve the bucket name from the request input parameters.
+// If no bucket is found, or the field is empty "", false will be returned.
+func bucketNameFromReqParams(params interface{}) (string, bool) {
+	b, _ := awsutil.ValuesAtPath(params, "Bucket")
+	if len(b) == 0 {
+		return "", false
+	}
+
+	if bucket, ok := b[0].(*string); ok {
+		if bucketStr := aws.StringValue(bucket); bucketStr != "" {
+			return bucketStr, true
+		}
+	}
+
+	return "", false
+}
+
+// hostCompatibleBucketName returns true if the request should
+// put the bucket in the host. This is false if S3ForcePathStyle is
+// explicitly set or if the bucket is not DNS compatible.
+func hostCompatibleBucketName(u *url.URL, bucket string) bool {
+	// Bucket might be DNS compatible but dots in the hostname will fail
+	// certificate validation, so do not use host-style.
+	if u.Scheme == "https" && strings.Contains(bucket, ".") {
+		return false
+	}
+
+	// if the bucket is DNS compatible
+	return dnsCompatibleBucketName(bucket)
+}
+
+var reDomain = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`)
+var reIPAddress = regexp.MustCompile(`^(\d+\.){3}\d+$`)
+
+// dnsCompatibleBucketName returns true if the bucket name is DNS compatible.
+// Buckets created outside of the classic region MUST be DNS compatible.
+func dnsCompatibleBucketName(bucket string) bool {
+	return reDomain.MatchString(bucket) &&
+		!reIPAddress.MatchString(bucket) &&
+		!strings.Contains(bucket, "..")
+}
+
+// moveBucketToHost moves the bucket name from the URI path to URL host.
+func moveBucketToHost(u *url.URL, bucket string) {
+	u.Host = bucket + "." + u.Host
+	u.Path = strings.Replace(u.Path, "/{Bucket}", "", -1)
+	if u.Path == "" {
+		u.Path = "/"
+	}
+}
+
+const s3HostPrefix = "s3"
+
+// replaceHostRegion replaces the S3 region string in the host with the
+// value provided. If v is empty the host prefix returned will be s3.
+func replaceHostRegion(host, v string) string {
+	if !strings.HasPrefix(host, s3HostPrefix) {
+		return host
+	}
+
+	suffix := host[len(s3HostPrefix):]
+	for i := len(s3HostPrefix); i < len(host); i++ {
+		if host[i] == '.' {
+			// Trim until '.' leave the it in place.
+			suffix = host[i:]
+			break
+		}
+	}
+
+	if len(v) == 0 {
+		return fmt.Sprintf("s3%s", suffix)
+	}
+
+	return fmt.Sprintf("s3-%s%s", v, suffix)
+}

vendor/github.com/aws/aws-sdk-go/service/s3/platform_handlers.go

@@ -0,0 +1,8 @@
+// +build !go1.6
+
+package s3
+
+import "github.com/aws/aws-sdk-go/aws/request"
+
+func platformRequestHandlers(r *request.Request) {
+}

vendor/github.com/aws/aws-sdk-go/service/s3/platform_handlers_go1.6.go

@@ -0,0 +1,28 @@
+// +build go1.6
+
+package s3
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+func platformRequestHandlers(r *request.Request) {
+	if r.Operation.HTTPMethod == "PUT" {
+		// 100-Continue should only be used on put requests.
+		r.Handlers.Sign.PushBack(add100Continue)
+	}
+}
+
+func add100Continue(r *request.Request) {
+	if aws.BoolValue(r.Config.S3Disable100Continue) {
+		return
+	}
+	if r.HTTPRequest.ContentLength < 1024*1024*2 {
+		// Ignore requests smaller than 2MB. This helps prevent delaying
+		// requests unnecessarily.
+		return
+	}
+
+	r.HTTPRequest.Header.Set("Expect", "100-Continue")
+}

vendor/github.com/aws/aws-sdk-go/service/s3/service.go

@@ -0,0 +1,86 @@
+// THIS FILE IS AUTOMATICALLY GENERATED. DO NOT EDIT.
+
+package s3
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/client/metadata"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/aws/aws-sdk-go/private/protocol/restxml"
+)
+
+// S3 is a client for Amazon S3.
+//The service client's operations are safe to be used concurrently.
+// It is not safe to mutate any of the client's properties though.
+type S3 struct {
+	*client.Client
+}
+
+// Used for custom client initialization logic
+var initClient func(*client.Client)
+
+// Used for custom request initialization logic
+var initRequest func(*request.Request)
+
+// A ServiceName is the name of the service the client will make API calls to.
+const ServiceName = "s3"
+
+// New creates a new instance of the S3 client with a session.
+// If additional configuration is needed for the client instance use the optional
+// aws.Config parameter to add your extra config.
+//
+// Example:
+//     // Create a S3 client from just a session.
+//     svc := s3.New(mySession)
+//
+//     // Create a S3 client with additional configuration
+//     svc := s3.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
+func New(p client.ConfigProvider, cfgs ...*aws.Config) *S3 {
+	c := p.ClientConfig(ServiceName, cfgs...)
+	return newClient(*c.Config, c.Handlers, c.Endpoint, c.SigningRegion)
+}
+
+// newClient creates, initializes and returns a new service client instance.
+func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegion string) *S3 {
+	svc := &S3{
+		Client: client.New(
+			cfg,
+			metadata.ClientInfo{
+				ServiceName:   ServiceName,
+				SigningRegion: signingRegion,
+				Endpoint:      endpoint,
+				APIVersion:    "2006-03-01",
+			},
+			handlers,
+		),
+	}
+
+	// Handlers
+	svc.Handlers.Sign.PushBackNamed(v4.SignRequestHandler)
+	svc.Handlers.Build.PushBackNamed(restxml.BuildHandler)
+	svc.Handlers.Unmarshal.PushBackNamed(restxml.UnmarshalHandler)
+	svc.Handlers.UnmarshalMeta.PushBackNamed(restxml.UnmarshalMetaHandler)
+	svc.Handlers.UnmarshalError.PushBackNamed(restxml.UnmarshalErrorHandler)
+
+	// Run custom client initialization if present
+	if initClient != nil {
+		initClient(svc.Client)
+	}
+
+	return svc
+}
+
+// newRequest creates a new request for a S3 operation and runs any
+// custom request initialization.
+func (c *S3) newRequest(op *request.Operation, params, data interface{}) *request.Request {
+	req := c.NewRequest(op, params, data)
+
+	// Run custom request initialization if present
+	if initRequest != nil {
+		initRequest(req)
+	}
+
+	return req
+}

vendor/github.com/aws/aws-sdk-go/service/s3/sse.go

@@ -0,0 +1,44 @@
+package s3
+
+import (
+	"crypto/md5"
+	"encoding/base64"
+
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/awsutil"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+var errSSERequiresSSL = awserr.New("ConfigError", "cannot send SSE keys over HTTP.", nil)
+
+func validateSSERequiresSSL(r *request.Request) {
+	if r.HTTPRequest.URL.Scheme != "https" {
+		p, _ := awsutil.ValuesAtPath(r.Params, "SSECustomerKey||CopySourceSSECustomerKey")
+		if len(p) > 0 {
+			r.Error = errSSERequiresSSL
+		}
+	}
+}
+
+func computeSSEKeys(r *request.Request) {
+	headers := []string{
+		"x-amz-server-side-encryption-customer-key",
+		"x-amz-copy-source-server-side-encryption-customer-key",
+	}
+
+	for _, h := range headers {
+		md5h := h + "-md5"
+		if key := r.HTTPRequest.Header.Get(h); key != "" {
+			// Base64-encode the value
+			b64v := base64.StdEncoding.EncodeToString([]byte(key))
+			r.HTTPRequest.Header.Set(h, b64v)
+
+			// Add MD5 if it wasn't computed
+			if r.HTTPRequest.Header.Get(md5h) == "" {
+				sum := md5.Sum([]byte(key))
+				b64sum := base64.StdEncoding.EncodeToString(sum[:])
+				r.HTTPRequest.Header.Set(md5h, b64sum)
+			}
+		}
+	}
+}

vendor/github.com/aws/aws-sdk-go/service/s3/statusok_error.go

@@ -0,0 +1,36 @@
+package s3
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+func copyMultipartStatusOKUnmarhsalError(r *request.Request) {
+	b, err := ioutil.ReadAll(r.HTTPResponse.Body)
+	if err != nil {
+		r.Error = awserr.New("SerializationError", "unable to read response body", err)
+		return
+	}
+	body := bytes.NewReader(b)
+	r.HTTPResponse.Body = aws.ReadSeekCloser(body)
+	defer r.HTTPResponse.Body.(aws.ReaderSeekerCloser).Seek(0, 0)
+
+	if body.Len() == 0 {
+		// If there is no body don't attempt to parse the body.
+		return
+	}
+
+	unmarshalError(r)
+	if err, ok := r.Error.(awserr.Error); ok && err != nil {
+		if err.Code() == "SerializationError" {
+			r.Error = nil
+			return
+		}
+		r.HTTPResponse.StatusCode = http.StatusServiceUnavailable
+	}
+}

vendor/github.com/aws/aws-sdk-go/service/s3/unmarshal_error.go

@@ -0,0 +1,65 @@
+package s3
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+type xmlErrorResponse struct {
+	XMLName xml.Name `xml:"Error"`
+	Code    string   `xml:"Code"`
+	Message string   `xml:"Message"`
+}
+
+func unmarshalError(r *request.Request) {
+	defer r.HTTPResponse.Body.Close()
+	defer io.Copy(ioutil.Discard, r.HTTPResponse.Body)
+
+	// Bucket exists in a different region, and request needs
+	// to be made to the correct region.
+	if r.HTTPResponse.StatusCode == http.StatusMovedPermanently {
+		r.Error = awserr.NewRequestFailure(
+			awserr.New("BucketRegionError",
+				fmt.Sprintf("incorrect region, the bucket is not in '%s' region",
+					aws.StringValue(r.Config.Region)),
+				nil),
+			r.HTTPResponse.StatusCode,
+			r.RequestID,
+		)
+		return
+	}
+
+	var errCode, errMsg string
+
+	// Attempt to parse error from body if it is known
+	resp := &xmlErrorResponse{}
+	err := xml.NewDecoder(r.HTTPResponse.Body).Decode(resp)
+	if err != nil && err != io.EOF {
+		errCode = "SerializationError"
+		errMsg = "failed to decode S3 XML error response"
+	} else {
+		errCode = resp.Code
+		errMsg = resp.Message
+	}
+
+	// Fallback to status code converted to message if still no error code
+	if len(errCode) == 0 {
+		statusText := http.StatusText(r.HTTPResponse.StatusCode)
+		errCode = strings.Replace(statusText, " ", "", -1)
+		errMsg = statusText
+	}
+
+	r.Error = awserr.NewRequestFailure(
+		awserr.New(errCode, errMsg, nil),
+		r.HTTPResponse.StatusCode,
+		r.RequestID,
+	)
+}

vendor/github.com/aws/aws-sdk-go/service/s3/waiters.go

@@ -0,0 +1,139 @@
+// THIS FILE IS AUTOMATICALLY GENERATED. DO NOT EDIT.
+
+package s3
+
+import (
+	"github.com/aws/aws-sdk-go/private/waiter"
+)
+
+// WaitUntilBucketExists uses the Amazon S3 API operation
+// HeadBucket to wait for a condition to be met before returning.
+// If the condition is not meet within the max attempt window an error will
+// be returned.
+func (c *S3) WaitUntilBucketExists(input *HeadBucketInput) error {
+	waiterCfg := waiter.Config{
+		Operation:   "HeadBucket",
+		Delay:       5,
+		MaxAttempts: 20,
+		Acceptors: []waiter.WaitAcceptor{
+			{
+				State:    "success",
+				Matcher:  "status",
+				Argument: "",
+				Expected: 200,
+			},
+			{
+				State:    "success",
+				Matcher:  "status",
+				Argument: "",
+				Expected: 301,
+			},
+			{
+				State:    "success",
+				Matcher:  "status",
+				Argument: "",
+				Expected: 403,
+			},
+			{
+				State:    "retry",
+				Matcher:  "status",
+				Argument: "",
+				Expected: 404,
+			},
+		},
+	}
+
+	w := waiter.Waiter{
+		Client: c,
+		Input:  input,
+		Config: waiterCfg,
+	}
+	return w.Wait()
+}
+
+// WaitUntilBucketNotExists uses the Amazon S3 API operation
+// HeadBucket to wait for a condition to be met before returning.
+// If the condition is not meet within the max attempt window an error will
+// be returned.
+func (c *S3) WaitUntilBucketNotExists(input *HeadBucketInput) error {
+	waiterCfg := waiter.Config{
+		Operation:   "HeadBucket",
+		Delay:       5,
+		MaxAttempts: 20,
+		Acceptors: []waiter.WaitAcceptor{
+			{
+				State:    "success",
+				Matcher:  "status",
+				Argument: "",
+				Expected: 404,
+			},
+		},
+	}
+
+	w := waiter.Waiter{
+		Client: c,
+		Input:  input,
+		Config: waiterCfg,
+	}
+	return w.Wait()
+}
+
+// WaitUntilObjectExists uses the Amazon S3 API operation
+// HeadObject to wait for a condition to be met before returning.
+// If the condition is not meet within the max attempt window an error will
+// be returned.
+func (c *S3) WaitUntilObjectExists(input *HeadObjectInput) error {
+	waiterCfg := waiter.Config{
+		Operation:   "HeadObject",
+		Delay:       5,
+		MaxAttempts: 20,
+		Acceptors: []waiter.WaitAcceptor{
+			{
+				State:    "success",
+				Matcher:  "status",
+				Argument: "",
+				Expected: 200,
+			},
+			{
+				State:    "retry",
+				Matcher:  "status",
+				Argument: "",
+				Expected: 404,
+			},
+		},
+	}
+
+	w := waiter.Waiter{
+		Client: c,
+		Input:  input,
+		Config: waiterCfg,
+	}
+	return w.Wait()
+}
+
+// WaitUntilObjectNotExists uses the Amazon S3 API operation
+// HeadObject to wait for a condition to be met before returning.
+// If the condition is not meet within the max attempt window an error will
+// be returned.
+func (c *S3) WaitUntilObjectNotExists(input *HeadObjectInput) error {
+	waiterCfg := waiter.Config{
+		Operation:   "HeadObject",
+		Delay:       5,
+		MaxAttempts: 20,
+		Acceptors: []waiter.WaitAcceptor{
+			{
+				State:    "success",
+				Matcher:  "status",
+				Argument: "",
+				Expected: 404,
+			},
+		},
+	}
+
+	w := waiter.Waiter{
+		Client: c,
+		Input:  input,
+		Config: waiterCfg,
+	}
+	return w.Wait()
+}

vendor/vendor.json

@@ -298,6 +298,14 @@
 			"version": "v1.5.8",
 			"versionExact": "v1.5.8"
 		},
+		{
+			"checksumSHA1": "HtKiIAPKsBg2s1c5ytRkdZ/lqO8=",
+			"path": "github.com/aws/aws-sdk-go/service/s3",
+			"revision": "898c81ba64b9a467379d35e3fabad133beae0ee4",
+			"revisionTime": "2016-11-18T23:08:35Z",
+			"version": "v1.5.8",
+			"versionExact": "v1.5.8"
+		},
 		{
 			"checksumSHA1": "ouwhxcAsIYQ6oJbMRdLW/Ys/iyg=",
 			"path": "github.com/aws/aws-sdk-go/service/sts",