From 6b700ff1fb31d2f88fcf80d32887ebc618110886 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Thu, 25 May 2017 19:12:20 -0400 Subject: [PATCH 1/2] replace lock_table with dynamodb_table in s3 cfg Since the DynamoDB table used by the S3 backend is no longer only used for locks, rename it in the config to remove any confusion about it being lock-specific. --- backend/remote-state/s3/backend.go | 17 +++++++-- backend/remote-state/s3/backend_state.go | 2 +- backend/remote-state/s3/backend_test.go | 28 +++++++-------- backend/remote-state/s3/client.go | 24 ++++++------- backend/remote-state/s3/client_test.go | 46 ++++++++++++------------ 5 files changed, 65 insertions(+), 52 deletions(-) diff --git a/backend/remote-state/s3/backend.go b/backend/remote-state/s3/backend.go index 846746b981..ae027f2200 100644 --- a/backend/remote-state/s3/backend.go +++ b/backend/remote-state/s3/backend.go @@ -81,6 +81,14 @@ func New() backend.Backend { Optional: true, Description: "DynamoDB table for state locking", Default: "", + Deprecated: "please use the dynamodb_table attribute", + }, + + "dynamodb_table": { + Type: schema.TypeString, + Optional: true, + Description: "DynamoDB table for state locking and consistency", + Default: "", }, "profile": { @@ -151,7 +159,7 @@ type Backend struct { serverSideEncryption bool acl string kmsKeyID string - lockTable string + ddbTable string } func (b *Backend) configure(ctx context.Context) error { @@ -167,7 +175,12 @@ func (b *Backend) configure(ctx context.Context) error { b.serverSideEncryption = data.Get("encrypt").(bool) b.acl = data.Get("acl").(string) b.kmsKeyID = data.Get("kms_key_id").(string) - b.lockTable = data.Get("lock_table").(string) + + b.ddbTable = data.Get("dynamodb_table").(string) + if b.ddbTable == "" { + // try the depracted field + b.ddbTable = data.Get("lock_table").(string) + } cfg := &terraformAWS.Config{ AccessKey: data.Get("access_key").(string), diff --git a/backend/remote-state/s3/backend_state.go b/backend/remote-state/s3/backend_state.go index e35a11884a..1835e7402d 100644 --- a/backend/remote-state/s3/backend_state.go +++ b/backend/remote-state/s3/backend_state.go @@ -96,7 +96,7 @@ func (b *Backend) State(name string) (state.State, error) { serverSideEncryption: b.serverSideEncryption, acl: b.acl, kmsKeyID: b.kmsKeyID, - lockTable: b.lockTable, + ddbTable: b.ddbTable, } stateMgr := &remote.State{Client: client} diff --git a/backend/remote-state/s3/backend_test.go b/backend/remote-state/s3/backend_test.go index d90c76a6c6..bdc81e8994 100644 --- a/backend/remote-state/s3/backend_test.go +++ b/backend/remote-state/s3/backend_test.go @@ -34,11 +34,11 @@ func TestBackend_impl(t *testing.T) { func TestBackendConfig(t *testing.T) { testACC(t) config := map[string]interface{}{ - "region": "us-west-1", - "bucket": "tf-test", - "key": "state", - "encrypt": true, - "lock_table": "dynamoTable", + "region": "us-west-1", + "bucket": "tf-test", + "key": "state", + "encrypt": true, + "dynamodb_table": "dynamoTable", } b := backend.TestBackendConfig(t, New(), config).(*Backend) @@ -90,17 +90,17 @@ func TestBackendLocked(t *testing.T) { keyName := "test/state" b1 := backend.TestBackendConfig(t, New(), map[string]interface{}{ - "bucket": bucketName, - "key": keyName, - "encrypt": true, - "lock_table": bucketName, + "bucket": bucketName, + "key": keyName, + "encrypt": true, + "dynamodb_table": bucketName, }).(*Backend) b2 := backend.TestBackendConfig(t, New(), map[string]interface{}{ - "bucket": bucketName, - "key": keyName, - "encrypt": true, - "lock_table": bucketName, + "bucket": bucketName, + "key": keyName, + "encrypt": true, + "dynamodb_table": bucketName, }).(*Backend) createS3Bucket(t, b1.s3Client, bucketName) @@ -139,7 +139,7 @@ func TestBackendExtraPaths(t *testing.T) { serverSideEncryption: b.serverSideEncryption, acl: b.acl, kmsKeyID: b.kmsKeyID, - lockTable: b.lockTable, + ddbTable: b.ddbTable, } stateMgr := &remote.State{Client: client} diff --git a/backend/remote-state/s3/client.go b/backend/remote-state/s3/client.go index 25ff529303..87ddf0ff5a 100644 --- a/backend/remote-state/s3/client.go +++ b/backend/remote-state/s3/client.go @@ -32,7 +32,7 @@ type RemoteClient struct { serverSideEncryption bool acl string kmsKeyID string - lockTable string + ddbTable string } var ( @@ -191,7 +191,7 @@ func (c *RemoteClient) Delete() error { } func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) { - if c.lockTable == "" { + if c.ddbTable == "" { return "", nil } @@ -211,7 +211,7 @@ func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) { "LockID": {S: aws.String(c.lockPath())}, "Info": {S: aws.String(string(info.Marshal()))}, }, - TableName: aws.String(c.lockTable), + TableName: aws.String(c.ddbTable), ConditionExpression: aws.String("attribute_not_exists(LockID)"), } _, err := c.dynClient.PutItem(putParams) @@ -233,7 +233,7 @@ func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) { } func (c *RemoteClient) getMD5() ([]byte, error) { - if c.lockTable == "" { + if c.ddbTable == "" { return nil, nil } @@ -242,7 +242,7 @@ func (c *RemoteClient) getMD5() ([]byte, error) { "LockID": {S: aws.String(c.lockPath() + stateIDSuffix)}, }, ProjectionExpression: aws.String("LockID, Digest"), - TableName: aws.String(c.lockTable), + TableName: aws.String(c.ddbTable), } resp, err := c.dynClient.GetItem(getParams) @@ -265,7 +265,7 @@ func (c *RemoteClient) getMD5() ([]byte, error) { // store the hash of the state to that clients can check for stale state files. func (c *RemoteClient) putMD5(sum []byte) error { - if c.lockTable == "" { + if c.ddbTable == "" { return nil } @@ -278,7 +278,7 @@ func (c *RemoteClient) putMD5(sum []byte) error { "LockID": {S: aws.String(c.lockPath() + stateIDSuffix)}, "Digest": {S: aws.String(hex.EncodeToString(sum))}, }, - TableName: aws.String(c.lockTable), + TableName: aws.String(c.ddbTable), } _, err := c.dynClient.PutItem(putParams) if err != nil { @@ -290,7 +290,7 @@ func (c *RemoteClient) putMD5(sum []byte) error { // remove the hash value for a deleted state func (c *RemoteClient) deleteMD5() error { - if c.lockTable == "" { + if c.ddbTable == "" { return nil } @@ -298,7 +298,7 @@ func (c *RemoteClient) deleteMD5() error { Key: map[string]*dynamodb.AttributeValue{ "LockID": {S: aws.String(c.lockPath() + stateIDSuffix)}, }, - TableName: aws.String(c.lockTable), + TableName: aws.String(c.ddbTable), } if _, err := c.dynClient.DeleteItem(params); err != nil { return err @@ -312,7 +312,7 @@ func (c *RemoteClient) getLockInfo() (*state.LockInfo, error) { "LockID": {S: aws.String(c.lockPath())}, }, ProjectionExpression: aws.String("LockID, Info"), - TableName: aws.String(c.lockTable), + TableName: aws.String(c.ddbTable), } resp, err := c.dynClient.GetItem(getParams) @@ -335,7 +335,7 @@ func (c *RemoteClient) getLockInfo() (*state.LockInfo, error) { } func (c *RemoteClient) Unlock(id string) error { - if c.lockTable == "" { + if c.ddbTable == "" { return nil } @@ -360,7 +360,7 @@ func (c *RemoteClient) Unlock(id string) error { Key: map[string]*dynamodb.AttributeValue{ "LockID": {S: aws.String(c.lockPath())}, }, - TableName: aws.String(c.lockTable), + TableName: aws.String(c.ddbTable), } _, err = c.dynClient.DeleteItem(params) diff --git a/backend/remote-state/s3/client_test.go b/backend/remote-state/s3/client_test.go index d93711c9fe..e981924b71 100644 --- a/backend/remote-state/s3/client_test.go +++ b/backend/remote-state/s3/client_test.go @@ -47,17 +47,17 @@ func TestRemoteClientLocks(t *testing.T) { keyName := "testState" b1 := backend.TestBackendConfig(t, New(), map[string]interface{}{ - "bucket": bucketName, - "key": keyName, - "encrypt": true, - "lock_table": bucketName, + "bucket": bucketName, + "key": keyName, + "encrypt": true, + "dynamodb_table": bucketName, }).(*Backend) b2 := backend.TestBackendConfig(t, New(), map[string]interface{}{ - "bucket": bucketName, - "key": keyName, - "encrypt": true, - "lock_table": bucketName, + "bucket": bucketName, + "key": keyName, + "encrypt": true, + "dynamodb_table": bucketName, }).(*Backend) createS3Bucket(t, b1.s3Client, bucketName) @@ -85,17 +85,17 @@ func TestForceUnlock(t *testing.T) { keyName := "testState" b1 := backend.TestBackendConfig(t, New(), map[string]interface{}{ - "bucket": bucketName, - "key": keyName, - "encrypt": true, - "lock_table": bucketName, + "bucket": bucketName, + "key": keyName, + "encrypt": true, + "dynamodb_table": bucketName, }).(*Backend) b2 := backend.TestBackendConfig(t, New(), map[string]interface{}{ - "bucket": bucketName, - "key": keyName, - "encrypt": true, - "lock_table": bucketName, + "bucket": bucketName, + "key": keyName, + "encrypt": true, + "dynamodb_table": bucketName, }).(*Backend) createS3Bucket(t, b1.s3Client, bucketName) @@ -162,9 +162,9 @@ func TestRemoteClient_clientMD5(t *testing.T) { keyName := "testState" b := backend.TestBackendConfig(t, New(), map[string]interface{}{ - "bucket": bucketName, - "key": keyName, - "lock_table": bucketName, + "bucket": bucketName, + "key": keyName, + "dynamodb_table": bucketName, }).(*Backend) createS3Bucket(t, b.s3Client, bucketName) @@ -210,9 +210,9 @@ func TestRemoteClient_stateChecksum(t *testing.T) { keyName := "testState" b1 := backend.TestBackendConfig(t, New(), map[string]interface{}{ - "bucket": bucketName, - "key": keyName, - "lock_table": bucketName, + "bucket": bucketName, + "key": keyName, + "dynamodb_table": bucketName, }).(*Backend) createS3Bucket(t, b1.s3Client, bucketName) @@ -238,7 +238,7 @@ func TestRemoteClient_stateChecksum(t *testing.T) { t.Fatal(err) } - // Use b2 without a lock_table to bypass the lock table to write the state directly. + // Use b2 without a dynamodb_table to bypass the lock table to write the state directly. // client2 will write the "incorrect" state, simulating s3 eventually consistency delays b2 := backend.TestBackendConfig(t, New(), map[string]interface{}{ "bucket": bucketName, From eba5093115e1626bd1ca122cf354205b12a3cccb Mon Sep 17 00:00:00 2001 From: James Bardin Date: Tue, 30 May 2017 18:11:38 -0400 Subject: [PATCH 2/2] Update S3 backend docs Add dynamodb_table and deprecation notice on lock_table. Add missing parameters for the S3 backends: assume_role_policy, external_id, and session_name. --- website/source/docs/backends/types/s3.html.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/website/source/docs/backends/types/s3.html.md b/website/source/docs/backends/types/s3.html.md index ae6a386cda..51419cd777 100644 --- a/website/source/docs/backends/types/s3.html.md +++ b/website/source/docs/backends/types/s3.html.md @@ -93,9 +93,10 @@ The following configuration options or environment variables are supported: * `secret_key` / `AWS_SECRET_ACCESS_KEY` - (Optional) AWS secret access key. * `kms_key_id` - (Optional) The ARN of a KMS Key to use for encrypting the state. - * `lock_table` - (Optional) The name of a DynamoDB table to use for state - locking. The table must have a primary key named LockID. If not present, - locking will be disabled. + * `lock_table` - (Optional, Deprecated) Use `dynamodb_table` instead. + * `dynamodb_table` - (Optional) The name of a DynamoDB table to use for state + locking and consistency. The table must have a primary key named LockID. If + not present, locking will be disabled. * `profile` - (Optional) This is the AWS profile name as set in the shared credentials file. * `shared_credentials_file` - (Optional) This is the path to the @@ -103,4 +104,7 @@ The following configuration options or environment variables are supported: `~/.aws/credentials` will be used. * `token` - (Optional) Use this to set an MFA token. It can also be sourced from the `AWS_SESSION_TOKEN` environment variable. - * `role_arn` - (Optional) The role to be assumed + * `role_arn` - (Optional) The role to be assumed. + * `assume_role_policy` - (Optional) The permissions applied when assuming a role. + * `external_id` - (Optional) The external ID to use when assuming the role. + * `session_name` - (Optional) The session name to use when assuming the role.