Backend/S3: Allow specifying retry mode for AWS API requests (#769)

Signed-off-by: Marcin Białoń <mbialon@spacelift.io>

CHANGELOG.md

@@ -58,6 +58,7 @@ S3 BACKEND:
 * Adds the `custom_ca_bundle` argument. ([#689](https://github.com/opentofu/opentofu/issues/689))
 * Adds support for the `sts_region` argument. ([#695](https://github.com/opentofu/opentofu/issues/695))
 * Adds support for `ec2_metadata_service_endpoint` and `ec2_metadata_service_endpoint_mode` arguments to enable overriding the EC2 metadata service (IMDS) endpoint. ([#693](https://github.com/opentofu/opentofu/issues/693))
+* Adds support for the `retry_mode` attribute. ([#698](https://github.com/opentofu/opentofu/issues/698))
 
 ## Previous Releases
 

internal/backend/remote-state/s3/backend.go

@@ -225,6 +225,11 @@ func (b *Backend) ConfigSchema(context.Context) *configschema.Block {
 				Optional:    true,
 				Description: "Force s3 to use path style api.",
 			},
+			"retry_mode": {
+				Type:        cty.String,
+				Optional:    true,
+				Description: "Specifies how retries are attempted. Valid values are `standard` and `adaptive`.",
+			},
 			"max_retries": {
 				Type:        cty.Number,
 				Optional:    true,
@@ -507,6 +512,18 @@ func (b *Backend) PrepareConfig(ctx context.Context, obj cty.Value) (cty.Value,
 		cty.GetAttrPath("forbidden_account_ids"),
 	)(obj, cty.Path{}, &diags)
 
+	if val := obj.GetAttr("retry_mode"); !val.IsNull() {
+		s := val.AsString()
+		if _, err := aws.ParseRetryMode(s); err != nil {
+			diags = diags.Append(tfdiags.AttributeValue(
+				tfdiags.Error,
+				"Invalid retry mode",
+				fmt.Sprintf("Valid values are %q and %q.", aws.RetryModeStandard, aws.RetryModeAdaptive),
+				cty.Path{cty.GetAttrStep{Name: "retry_mode"}},
+			))
+		}
+	}
+
 	return obj, diags
 }
 
@@ -651,6 +668,14 @@ func (b *Backend) Configure(ctx context.Context, obj cty.Value) tfdiags.Diagnost
 		cfg.ForbiddenAccountIds = val
 	}
 
+	if val, ok := stringAttrOk(obj, "retry_mode"); ok {
+		mode, err := aws.ParseRetryMode(val)
+		if err != nil {
+			panic(fmt.Sprintf("invalid retry mode %q: %s", val, err))
+		}
+		cfg.RetryMode = mode
+	}
+
 	_, awsConfig, awsDiags := awsbase.GetAwsConfig(ctx, cfg)
 
 	for _, d := range awsDiags {

internal/backend/remote-state/s3/backend_complete_test.go

@@ -2119,6 +2119,88 @@ region = us-west-2
 	}
 }
 
+func TestBackendConfig_RetryMode(t *testing.T) {
+	testCases := map[string]struct {
+		config               map[string]any
+		EnvironmentVariables map[string]string
+		ExpectedMode         aws.RetryMode
+	}{
+		"no config": {
+			config: map[string]any{
+				"access_key": servicemocks.MockStaticAccessKey,
+				"secret_key": servicemocks.MockStaticSecretKey,
+			},
+			ExpectedMode: "",
+		},
+
+		"config": {
+			config: map[string]any{
+				"access_key": servicemocks.MockStaticAccessKey,
+				"secret_key": servicemocks.MockStaticSecretKey,
+				"retry_mode": "standard",
+			},
+			ExpectedMode: aws.RetryModeStandard,
+		},
+
+		"AWS_RETRY_MODE": {
+			config: map[string]any{
+				"access_key": servicemocks.MockStaticAccessKey,
+				"secret_key": servicemocks.MockStaticSecretKey,
+			},
+			EnvironmentVariables: map[string]string{
+				"AWS_RETRY_MODE": "adaptive",
+			},
+			ExpectedMode: aws.RetryModeAdaptive,
+		},
+		"config overrides AWS_RETRY_MODE": {
+			config: map[string]any{
+				"access_key": servicemocks.MockStaticAccessKey,
+				"secret_key": servicemocks.MockStaticSecretKey,
+				"retry_mode": "standard",
+			},
+			EnvironmentVariables: map[string]string{
+				"AWS_RETRY_MODE": "adaptive",
+			},
+			ExpectedMode: aws.RetryModeStandard,
+		},
+	}
+
+	for name, tc := range testCases {
+		tc := tc
+
+		t.Run(name, func(t *testing.T) {
+			oldEnv := servicemocks.InitSessionTestEnv()
+			defer servicemocks.PopEnv(oldEnv)
+
+			// Populate required fields
+			tc.config["bucket"] = "bucket"
+			tc.config["key"] = "key"
+			tc.config["region"] = "us-east-1"
+
+			for k, v := range tc.EnvironmentVariables {
+				os.Setenv(k, v)
+			}
+
+			sts := servicemocks.MockAwsApiServer("STS", []*servicemocks.MockEndpoint{
+				servicemocks.MockStsGetCallerIdentityValidEndpoint,
+			})
+			defer sts.Close()
+
+			tc.config["sts_endpoint"] = sts.URL
+			tc.config["skip_credentials_validation"] = true
+
+			b, diags := configureBackend(t, tc.config)
+			if diags.HasErrors() {
+				t.Fatalf("configuring backend: %s", diagnosticsString(diags))
+			}
+
+			if a, e := b.awsConfig.RetryMode, tc.ExpectedMode; a != e {
+				t.Errorf("expected mode %q, got: %q", e, a)
+			}
+		})
+	}
+}
+
 func setSharedConfigFile(filename string) {
 	os.Setenv("AWS_SDK_LOAD_CONFIG", "1")
 	os.Setenv("AWS_CONFIG_FILE", filename)

internal/backend/remote-state/s3/backend_test.go

@@ -729,6 +729,15 @@ func TestBackendConfig_PrepareConfigValidation(t *testing.T) {
 			}),
 			expectedErr: "Invalid Attribute Combination: Only one of allowed_account_ids, forbidden_account_ids can be set.",
 		},
+		"invalid retry mode": {
+			config: cty.ObjectVal(map[string]cty.Value{
+				"bucket":     cty.StringVal("test"),
+				"key":        cty.StringVal("test"),
+				"region":     cty.StringVal("us-west-2"),
+				"retry_mode": cty.StringVal("xyz"),
+			}),
+			expectedErr: `Invalid retry mode: Valid values are "standard" and "adaptive".`,
+		},
 	}
 
 	for name, tc := range cases {

website/docs/language/settings/backends/s3.mdx

@@ -161,6 +161,7 @@ The following configuration is optional:
 * `secret_key` - (Optional) AWS access key. If configured, must also configure `access_key`. This can also be sourced from the `AWS_SECRET_ACCESS_KEY` environment variable, AWS shared credentials file (e.g. `~/.aws/credentials`), or AWS shared configuration file (e.g. `~/.aws/config`).
 * `iam_endpoint` - (Optional) Custom endpoint for the AWS Identity and Access Management (IAM) API. This can also be sourced from the `AWS_IAM_ENDPOINT` environment variable.
 * `max_retries` - (Optional) The maximum number of times an AWS API request is retried on retryable failure. Defaults to 5.
+* `retry_mode` - (Optional) Specifies how retries are attempted. Valid values are `standard` and `adaptive`. This can also be sourced from the `AWS_RETRY_MODE` environment variable.
 * `profile` - (Optional) Name of AWS profile in AWS shared credentials file (e.g. `~/.aws/credentials`) or AWS shared configuration file (e.g. `~/.aws/config`) to use for credentials and/or configuration. This can also be sourced from the `AWS_PROFILE` environment variable.
 * `shared_credentials_file`  - (Optional) **Deprecated** Path to the AWS shared credentials file. Defaults to `~/.aws/credentials`.
 * `shared_credentials_files`  - (Optional) List of paths to AWS shared credentials files. Defaults to `~/.aws/credentials`. This can also be sourced from the `AWS_SHARED_CREDENTIALS_FILE` environment variable.