IntenseWebs/opentofu
3
0
Fork 0
mirror of https://github.com/opentofu/opentofu.git synced 2024-12-28 18:01:01 -06:00
Code Issues Packages Projects Releases Wiki Activity

retry on s3 backend internal errors

Browse Source 
Internal errors from S3 are usually transient, and can be immediately retried.
Make 2 attempts at retreiving the state object before returning an error.
This commit is contained in:
bclodius 2017-10-01 08:22:04 -04:00 committed by James Bardin
parent 71e47e56a0
commit fba02f0bea
1 changed files with 27 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
backend/remote-state/s3/client.go
View File

@ -22,7 +22,10 @@ import (
)
// Store the last saved serial in dynamo with this suffix for consistency checks.
const stateIDSuffix = "-md5"
const (
stateIDSuffix = "-md5"
s3ErrCodeInternalError = "InternalError"
)
type RemoteClient struct {
s3Client *s3.S3
@ -92,21 +95,33 @@ func (c *RemoteClient) Get() (payload *remote.Payload, err error) {
}
func (c *RemoteClient) get() (*remote.Payload, error) {
output, err := c.s3Client.GetObject(&s3.GetObjectInput{
Bucket: &c.bucketName,
Key: &c.path,
})
var output *s3.GetObjectOutput
var err error
if err != nil {
if awserr := err.(awserr.Error); awserr != nil {
if awserr.Code() == "NoSuchKey" {
return nil, nil
} else {
return nil, err
// we immediately retry on an internal error, as those are usually transient
maxRetries := 2
for retryCount := 0; ; retryCount++ {
output, err = c.s3Client.GetObject(&s3.GetObjectInput{
Bucket: &c.bucketName,
Key: &c.path,
})
if err != nil {
if awserr, ok := err.(awserr.Error); ok {
switch awserr.Code() {
case s3.ErrCodeNoSuchKey:
return nil, nil
case s3ErrCodeInternalError:
if retryCount > maxRetries {
return nil, err
}
log.Println("[WARN] s3 internal error, retrying...")
continue
}
}
} else {
return nil, err
}
break
}
defer output.Body.Close()