mirror of
https://github.com/opentofu/opentofu.git
synced 2024-12-25 08:21:07 -06:00
Add customer-managed encryption key (KMS) support to GCS backend (#31786)
* Add ability to use customer-managed KMS key to encrypt state, add acceptance tests * Change test names for different encrpytion methods * Commit files updated by `go mod tidy` * Add guard against missing ENVs to `setupKmsKey` func * Update KMS setup function to get credentials from ENVs * Update tests to not include zero-values in config This means that default values are supplied later by TF instead of supplied as config from the user This also avoids issues related to making field conflicts explicit with `ConflictsWith` * Make `encryption_key` & `kms_encryption_key` conflicting fields Removing the Default from `encryption_key` does not appear to be a breaking change when tested manually * Add ability to set `kms_encryption_key` via ENV * Refactor `encryption_key` to use `DefaultFunc` to access ENV, if set * Remove comments * Update `gcs` backend docs & descriptions in schema * Update `gcs` backend docs to include information on encryption methods * Apply technical writing suggestions from code review Co-authored-by: Matthew Garrell <69917312+mgarrell777@users.noreply.github.com> * Update documentation to remove passive voice * Change use of context in tests, add inline comment, update logs * Remove use of `ReadPathOrContents` for new field Co-authored-by: Matthew Garrell <69917312+mgarrell777@users.noreply.github.com>
This commit is contained in:
parent
e7fb895c46
commit
d43ec0f30f
4
go.mod
4
go.mod
@ -1,6 +1,7 @@
|
||||
module github.com/hashicorp/terraform
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.81.0
|
||||
cloud.google.com/go/storage v1.10.0
|
||||
github.com/Azure/azure-sdk-for-go v59.2.0+incompatible
|
||||
github.com/Azure/go-autorest/autorest v0.11.24
|
||||
@ -86,6 +87,7 @@ require (
|
||||
golang.org/x/text v0.3.7
|
||||
golang.org/x/tools v0.1.11
|
||||
google.golang.org/api v0.44.0-impersonate-preview
|
||||
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c
|
||||
google.golang.org/grpc v1.47.0
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0
|
||||
google.golang.org/protobuf v1.27.1
|
||||
@ -97,7 +99,6 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.81.0 // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
|
||||
github.com/Azure/go-autorest/autorest/azure/cli v0.4.4 // indirect
|
||||
@ -174,7 +175,6 @@ require (
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
|
||||
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/ini.v1 v1.66.2 // indirect
|
||||
|
@ -31,6 +31,7 @@ type Backend struct {
|
||||
prefix string
|
||||
|
||||
encryptionKey []byte
|
||||
kmsKeyName string
|
||||
}
|
||||
|
||||
func New() backend.Backend {
|
||||
@ -83,10 +84,23 @@ func New() backend.Backend {
|
||||
},
|
||||
|
||||
"encryption_key": {
|
||||
Type: schema.TypeString,
|
||||
Optional: true,
|
||||
Description: "A 32 byte base64 encoded 'customer supplied encryption key' used to encrypt all state.",
|
||||
Default: "",
|
||||
Type: schema.TypeString,
|
||||
Optional: true,
|
||||
DefaultFunc: schema.MultiEnvDefaultFunc([]string{
|
||||
"GOOGLE_ENCRYPTION_KEY",
|
||||
}, nil),
|
||||
Description: "A 32 byte base64 encoded 'customer supplied encryption key' used when reading and writing state files in the bucket.",
|
||||
ConflictsWith: []string{"kms_encryption_key"},
|
||||
},
|
||||
|
||||
"kms_encryption_key": {
|
||||
Type: schema.TypeString,
|
||||
Optional: true,
|
||||
DefaultFunc: schema.MultiEnvDefaultFunc([]string{
|
||||
"GOOGLE_KMS_ENCRYPTION_KEY",
|
||||
}, nil),
|
||||
Description: "A Cloud KMS key ('customer managed encryption key') used when reading and writing state files in the bucket. Format should be 'projects/{{project}}/locations/{{location}}/keyRings/{{keyRing}}/cryptoKeys/{{name}}'.",
|
||||
ConflictsWith: []string{"encryption_key"},
|
||||
},
|
||||
},
|
||||
}
|
||||
@ -188,11 +202,8 @@ func (b *Backend) configure(ctx context.Context) error {
|
||||
|
||||
b.storageClient = client
|
||||
|
||||
// Customer-supplied encryption
|
||||
key := data.Get("encryption_key").(string)
|
||||
if key == "" {
|
||||
key = os.Getenv("GOOGLE_ENCRYPTION_KEY")
|
||||
}
|
||||
|
||||
if key != "" {
|
||||
kc, err := backend.ReadPathOrContents(key)
|
||||
if err != nil {
|
||||
@ -212,5 +223,11 @@ func (b *Backend) configure(ctx context.Context) error {
|
||||
b.encryptionKey = k
|
||||
}
|
||||
|
||||
// Customer-managed encryption
|
||||
kmsName := data.Get("kms_encryption_key").(string)
|
||||
if kmsName != "" {
|
||||
b.kmsKeyName = kmsName
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -81,6 +81,7 @@ func (b *Backend) client(name string) (*remoteClient, error) {
|
||||
stateFilePath: b.stateFile(name),
|
||||
lockFilePath: b.lockFile(name),
|
||||
encryptionKey: b.encryptionKey,
|
||||
kmsKeyName: b.kmsKeyName,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
@ -1,6 +1,8 @@
|
||||
package gcs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
@ -8,18 +10,34 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
kms "cloud.google.com/go/kms/apiv1"
|
||||
"cloud.google.com/go/storage"
|
||||
"github.com/hashicorp/terraform/internal/backend"
|
||||
"github.com/hashicorp/terraform/internal/httpclient"
|
||||
"github.com/hashicorp/terraform/internal/states/remote"
|
||||
"google.golang.org/api/option"
|
||||
kmspb "google.golang.org/genproto/googleapis/cloud/kms/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
noPrefix = ""
|
||||
noEncryptionKey = ""
|
||||
noKmsKeyName = ""
|
||||
)
|
||||
|
||||
// See https://cloud.google.com/storage/docs/using-encryption-keys#generating_your_own_encryption_key
|
||||
var encryptionKey = "yRyCOikXi1ZDNE0xN3yiFsJjg7LGimoLrGFcLZgQoVk="
|
||||
const encryptionKey = "yRyCOikXi1ZDNE0xN3yiFsJjg7LGimoLrGFcLZgQoVk="
|
||||
|
||||
// KMS key ring name and key name are hardcoded here and re-used because key rings (and keys) cannot be deleted
|
||||
// Test code asserts their presence and creates them if they're absent. They're not deleted at the end of tests.
|
||||
// See: https://cloud.google.com/kms/docs/faq#cannot_delete
|
||||
const (
|
||||
keyRingName = "tf-gcs-backend-acc-tests"
|
||||
keyName = "tf-test-key-1"
|
||||
kmsRole = "roles/cloudkms.cryptoKeyEncrypterDecrypter" // GCS service account needs this binding on the created key
|
||||
)
|
||||
|
||||
var keyRingLocation = os.Getenv("GOOGLE_REGION")
|
||||
|
||||
func TestStateFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
@ -54,7 +72,7 @@ func TestRemoteClient(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
bucket := bucketName(t)
|
||||
be := setupBackend(t, bucket, noPrefix, noEncryptionKey)
|
||||
be := setupBackend(t, bucket, noPrefix, noEncryptionKey, noKmsKeyName)
|
||||
defer teardownBackend(t, be, noPrefix)
|
||||
|
||||
ss, err := be.StateMgr(backend.DefaultStateName)
|
||||
@ -73,7 +91,7 @@ func TestRemoteClientWithEncryption(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
bucket := bucketName(t)
|
||||
be := setupBackend(t, bucket, noPrefix, encryptionKey)
|
||||
be := setupBackend(t, bucket, noPrefix, encryptionKey, noKmsKeyName)
|
||||
defer teardownBackend(t, be, noPrefix)
|
||||
|
||||
ss, err := be.StateMgr(backend.DefaultStateName)
|
||||
@ -93,7 +111,7 @@ func TestRemoteLocks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
bucket := bucketName(t)
|
||||
be := setupBackend(t, bucket, noPrefix, noEncryptionKey)
|
||||
be := setupBackend(t, bucket, noPrefix, noEncryptionKey, noKmsKeyName)
|
||||
defer teardownBackend(t, be, noPrefix)
|
||||
|
||||
remoteClient := func() (remote.Client, error) {
|
||||
@ -127,10 +145,10 @@ func TestBackend(t *testing.T) {
|
||||
|
||||
bucket := bucketName(t)
|
||||
|
||||
be0 := setupBackend(t, bucket, noPrefix, noEncryptionKey)
|
||||
be0 := setupBackend(t, bucket, noPrefix, noEncryptionKey, noKmsKeyName)
|
||||
defer teardownBackend(t, be0, noPrefix)
|
||||
|
||||
be1 := setupBackend(t, bucket, noPrefix, noEncryptionKey)
|
||||
be1 := setupBackend(t, bucket, noPrefix, noEncryptionKey, noKmsKeyName)
|
||||
|
||||
backend.TestBackendStates(t, be0)
|
||||
backend.TestBackendStateLocks(t, be0, be1)
|
||||
@ -143,30 +161,55 @@ func TestBackendWithPrefix(t *testing.T) {
|
||||
prefix := "test/prefix"
|
||||
bucket := bucketName(t)
|
||||
|
||||
be0 := setupBackend(t, bucket, prefix, noEncryptionKey)
|
||||
be0 := setupBackend(t, bucket, prefix, noEncryptionKey, noKmsKeyName)
|
||||
defer teardownBackend(t, be0, prefix)
|
||||
|
||||
be1 := setupBackend(t, bucket, prefix+"/", noEncryptionKey)
|
||||
be1 := setupBackend(t, bucket, prefix+"/", noEncryptionKey, noKmsKeyName)
|
||||
|
||||
backend.TestBackendStates(t, be0)
|
||||
backend.TestBackendStateLocks(t, be0, be1)
|
||||
}
|
||||
func TestBackendWithEncryption(t *testing.T) {
|
||||
func TestBackendWithCustomerSuppliedEncryption(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
bucket := bucketName(t)
|
||||
|
||||
be0 := setupBackend(t, bucket, noPrefix, encryptionKey)
|
||||
be0 := setupBackend(t, bucket, noPrefix, encryptionKey, noKmsKeyName)
|
||||
defer teardownBackend(t, be0, noPrefix)
|
||||
|
||||
be1 := setupBackend(t, bucket, noPrefix, encryptionKey)
|
||||
be1 := setupBackend(t, bucket, noPrefix, encryptionKey, noKmsKeyName)
|
||||
|
||||
backend.TestBackendStates(t, be0)
|
||||
backend.TestBackendStateLocks(t, be0, be1)
|
||||
}
|
||||
|
||||
func TestBackendWithCustomerManagedKMSEncryption(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
projectID := os.Getenv("GOOGLE_PROJECT")
|
||||
bucket := bucketName(t)
|
||||
|
||||
// Taken from global variables in test file
|
||||
kmsDetails := map[string]string{
|
||||
"project": projectID,
|
||||
"location": keyRingLocation,
|
||||
"ringName": keyRingName,
|
||||
"keyName": keyName,
|
||||
}
|
||||
|
||||
kmsName := setupKmsKey(t, kmsDetails)
|
||||
|
||||
be0 := setupBackend(t, bucket, noPrefix, noEncryptionKey, kmsName)
|
||||
defer teardownBackend(t, be0, noPrefix)
|
||||
|
||||
be1 := setupBackend(t, bucket, noPrefix, noEncryptionKey, kmsName)
|
||||
|
||||
backend.TestBackendStates(t, be0)
|
||||
backend.TestBackendStateLocks(t, be0, be1)
|
||||
}
|
||||
|
||||
// setupBackend returns a new GCS backend.
|
||||
func setupBackend(t *testing.T, bucket, prefix, key string) backend.Backend {
|
||||
func setupBackend(t *testing.T, bucket, prefix, key, kmsName string) backend.Backend {
|
||||
t.Helper()
|
||||
|
||||
projectID := os.Getenv("GOOGLE_PROJECT")
|
||||
@ -177,9 +220,16 @@ func setupBackend(t *testing.T, bucket, prefix, key string) backend.Backend {
|
||||
}
|
||||
|
||||
config := map[string]interface{}{
|
||||
"bucket": bucket,
|
||||
"prefix": prefix,
|
||||
"encryption_key": key,
|
||||
"bucket": bucket,
|
||||
"prefix": prefix,
|
||||
}
|
||||
// Only add encryption keys to config if non-zero value set
|
||||
// If not set here, default values are supplied in `TestBackendConfig` by `PrepareConfig` function call
|
||||
if len(key) > 0 {
|
||||
config["encryption_key"] = key
|
||||
}
|
||||
if len(kmsName) > 0 {
|
||||
config["kms_encryption_key"] = kmsName
|
||||
}
|
||||
|
||||
b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(config))
|
||||
@ -205,6 +255,120 @@ func setupBackend(t *testing.T, bucket, prefix, key string) backend.Backend {
|
||||
return b
|
||||
}
|
||||
|
||||
// setupKmsKey asserts that a KMS key chain and key exist and necessary IAM bindings are in place
|
||||
// If the key ring or key do not exist they are created and permissions are given to the GCS Service account
|
||||
func setupKmsKey(t *testing.T, keyDetails map[string]string) string {
|
||||
t.Helper()
|
||||
|
||||
projectID := os.Getenv("GOOGLE_PROJECT")
|
||||
if projectID == "" || os.Getenv("TF_ACC") == "" {
|
||||
t.Skip("This test creates a KMS key ring and key in Cloud KMS. " +
|
||||
"Since this may incur costs, it will only run if " +
|
||||
"the TF_ACC and GOOGLE_PROJECT environment variables are set.")
|
||||
}
|
||||
|
||||
// KMS Client
|
||||
ctx := context.Background()
|
||||
opts, err := testGetClientOptions(t)
|
||||
if err != nil {
|
||||
e := fmt.Errorf("testGetClientOptions() failed: %s", err)
|
||||
t.Fatal(e)
|
||||
}
|
||||
c, err := kms.NewKeyManagementClient(ctx, opts...)
|
||||
if err != nil {
|
||||
e := fmt.Errorf("kms.NewKeyManagementClient() failed: %v", err)
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
// Get KMS key ring, create if doesn't exist
|
||||
reqGetKeyRing := &kmspb.GetKeyRingRequest{
|
||||
Name: fmt.Sprintf("projects/%s/locations/%s/keyRings/%s", keyDetails["project"], keyDetails["location"], keyDetails["ringName"]),
|
||||
}
|
||||
var keyRing *kmspb.KeyRing
|
||||
keyRing, err = c.GetKeyRing(ctx, reqGetKeyRing)
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "NotFound") {
|
||||
// Handle unexpected error that isn't related to the key ring not being made yet
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create key ring that doesn't exist
|
||||
t.Logf("Cloud KMS key ring `%s` not found: creating key ring",
|
||||
fmt.Sprintf("projects/%s/locations/%s/keyRings/%s", keyDetails["project"], keyDetails["location"], keyDetails["ringName"]),
|
||||
)
|
||||
reqCreateKeyRing := &kmspb.CreateKeyRingRequest{
|
||||
Parent: fmt.Sprintf("projects/%s/locations/%s", keyDetails["project"], keyDetails["location"]),
|
||||
KeyRingId: keyDetails["ringName"],
|
||||
}
|
||||
keyRing, err = c.CreateKeyRing(ctx, reqCreateKeyRing)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("Cloud KMS key ring `%s` created successfully", keyRing.Name)
|
||||
}
|
||||
|
||||
// Get KMS key, create if doesn't exist (and give GCS service account permission to use)
|
||||
reqGetKey := &kmspb.GetCryptoKeyRequest{
|
||||
Name: fmt.Sprintf("%s/cryptoKeys/%s", keyRing.Name, keyDetails["keyName"]),
|
||||
}
|
||||
var key *kmspb.CryptoKey
|
||||
key, err = c.GetCryptoKey(ctx, reqGetKey)
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "NotFound") {
|
||||
// Handle unexpected error that isn't related to the key not being made yet
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create key that doesn't exist
|
||||
t.Logf("Cloud KMS key `%s` not found: creating key",
|
||||
fmt.Sprintf("%s/cryptoKeys/%s", keyRing.Name, keyDetails["keyName"]),
|
||||
)
|
||||
reqCreateKey := &kmspb.CreateCryptoKeyRequest{
|
||||
Parent: keyRing.Name,
|
||||
CryptoKeyId: keyDetails["keyName"],
|
||||
CryptoKey: &kmspb.CryptoKey{
|
||||
Purpose: kmspb.CryptoKey_ENCRYPT_DECRYPT,
|
||||
},
|
||||
}
|
||||
key, err = c.CreateCryptoKey(ctx, reqCreateKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("Cloud KMS key `%s` created successfully", key.Name)
|
||||
}
|
||||
|
||||
// Get GCS Service account email, check has necessary permission on key
|
||||
// Note: we cannot reuse the backend's storage client (like in the setupBackend function)
|
||||
// because the KMS key needs to exist before the backend buckets are made in the test.
|
||||
sc, err := storage.NewClient(ctx, opts...) //reuse opts from KMS client
|
||||
if err != nil {
|
||||
e := fmt.Errorf("storage.NewClient() failed: %v", err)
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer sc.Close()
|
||||
gcsServiceAccount, err := sc.ServiceAccount(ctx, keyDetails["project"])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Assert Cloud Storage service account has permission to use this key.
|
||||
member := fmt.Sprintf("serviceAccount:%s", gcsServiceAccount)
|
||||
iamHandle := c.ResourceIAM(key.Name)
|
||||
policy, err := iamHandle.Policy(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok := policy.HasRole(member, kmsRole); !ok {
|
||||
// Add the missing permissions
|
||||
t.Logf("Granting GCS service account %s %s role on key %s", gcsServiceAccount, kmsRole, key.Name)
|
||||
policy.Add(member, kmsRole)
|
||||
err = iamHandle.SetPolicy(ctx, policy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return key.Name
|
||||
}
|
||||
|
||||
// teardownBackend deletes all states from be except the default state.
|
||||
func teardownBackend(t *testing.T, be backend.Backend, prefix string) {
|
||||
t.Helper()
|
||||
@ -242,3 +406,36 @@ func bucketName(t *testing.T) string {
|
||||
|
||||
return strings.ToLower(name)
|
||||
}
|
||||
|
||||
// getClientOptions returns the []option.ClientOption needed to configure Google API clients
|
||||
// that are required in acceptance tests but are not part of the gcs backend itself
|
||||
func testGetClientOptions(t *testing.T) ([]option.ClientOption, error) {
|
||||
t.Helper()
|
||||
|
||||
var creds string
|
||||
if v := os.Getenv("GOOGLE_BACKEND_CREDENTIALS"); v != "" {
|
||||
creds = v
|
||||
} else {
|
||||
creds = os.Getenv("GOOGLE_CREDENTIALS")
|
||||
}
|
||||
if creds == "" {
|
||||
t.Skip("This test required credentials to be supplied via" +
|
||||
"the GOOGLE_CREDENTIALS or GOOGLE_BACKEND_CREDENTIALS environment variables.")
|
||||
}
|
||||
|
||||
var opts []option.ClientOption
|
||||
var credOptions []option.ClientOption
|
||||
|
||||
contents, err := backend.ReadPathOrContents(creds)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error loading credentials: %s", err)
|
||||
}
|
||||
if !json.Valid([]byte(contents)) {
|
||||
return nil, fmt.Errorf("the string provided in credentials is neither valid json nor a valid file path")
|
||||
}
|
||||
credOptions = append(credOptions, option.WithCredentialsJSON([]byte(contents)))
|
||||
opts = append(opts, credOptions...)
|
||||
opts = append(opts, option.WithUserAgent(httpclient.UserAgentString()))
|
||||
|
||||
return opts, nil
|
||||
}
|
||||
|
@ -23,6 +23,7 @@ type remoteClient struct {
|
||||
stateFilePath string
|
||||
lockFilePath string
|
||||
encryptionKey []byte
|
||||
kmsKeyName string
|
||||
}
|
||||
|
||||
func (c *remoteClient) Get() (payload *remote.Payload, err error) {
|
||||
@ -57,6 +58,9 @@ func (c *remoteClient) Get() (payload *remote.Payload, err error) {
|
||||
func (c *remoteClient) Put(data []byte) error {
|
||||
err := func() error {
|
||||
stateFileWriter := c.stateFile().NewWriter(c.storageContext)
|
||||
if len(c.kmsKeyName) > 0 {
|
||||
stateFileWriter.KMSKeyName = c.kmsKeyName
|
||||
}
|
||||
if _, err := stateFileWriter.Write(data); err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -73,9 +73,31 @@ the path of the service account key. Terraform will use that key for authenticat
|
||||
|
||||
Terraform can impersonate a Google Service Account as described [here](https://cloud.google.com/iam/docs/creating-short-lived-service-account-credentials). A valid credential must be provided as mentioned in the earlier section and that identity must have the `roles/iam.serviceAccountTokenCreator` role on the service account you are impersonating.
|
||||
|
||||
## Encryption
|
||||
|
||||
!> **Warning:** Take care of your encryption keys because state data encrypted with a lost or deleted key is not recoverable. If you use customer-supplied encryption keys, you must securely manage your keys and ensure you do not lose them. You must not delete customer-managed encryption keys in Cloud KMS used to encrypt state. However, if you accidentally delete a key, there is a time window where [you can recover it](https://cloud.google.com/kms/docs/destroy-restore#restore).
|
||||
|
||||
### Customer-supplied encryption keys
|
||||
|
||||
To get started, follow this guide: [Use customer-supplied encryption keys](https://cloud.google.com/storage/docs/encryption/using-customer-supplied-keys)
|
||||
|
||||
If you want to remove customer-supplied keys from your backend configuration or change to a different customer-supplied key, Terraform cannot perform a state migration automatically and manual intervention is necessary instead. This intervention is necessary because Google does not store customer-supplied encryption keys, any requests sent to the Cloud Storage API must supply them instead (see [Customer-supplied Encryption Keys](https://cloud.google.com/storage/docs/encryption/customer-supplied-keys)). At the time of state migration, the backend configuration loses the old key's details and Terraform cannot use the key during the migration process.
|
||||
|
||||
~> **Important:** To migrate your state away from using customer-supplied encryption keys or change the key used by your backend, you need to perform a [rewrite (gsutil CLI)](https://cloud.google.com/storage/docs/gsutil/commands/rewrite) or [cp (gcloud CLI)](https://cloud.google.com/sdk/gcloud/reference/storage/cp#--decryption-keys) operation to remove use of the old customer-supplied encryption key on your state file. Once you remove the encryption, you can successfully run `terraform init -migrate-state` with your new backend configuration.
|
||||
|
||||
### Customer-managed encryption keys (Cloud KMS)
|
||||
|
||||
To get started, follow this guide: [Use customer-managed encryption keys](https://cloud.google.com/storage/docs/encryption/using-customer-managed-keys)
|
||||
|
||||
If you want to remove customer-managed keys from your backend configuration or change to a different customer-managed key, Terraform _can_ manage a state migration without manual intervention. This ability is because GCP stores customer-managed encryption keys and are accessible during the state migration process. However, these changes do not fully come into effect until the first write operation occurs on the state file after state migration occurs. In the first write operation after state migration, the file decrypts with the old key and then writes with the new encryption method. This method is equivalent to the [rewrite](https://cloud.google.com/storage/docs/gsutil/commands/rewrite) operation described in the customer-supplied encryption keys section. Because of the importance of the first write to state after state migration, you should not delete old KMS keys until any state file(s) encrypted with that key update.
|
||||
|
||||
Customer-managed keys do not need to be sent in requests to read files from GCS buckets because decryption occurs automatically within GCS. This process means that if you use the `terraform_remote_state` [data source](https://www.terraform.io/language/state/remote-state-data) to access KMS-encrypted state, you do not need to specify the KMS key in the data source's `config` object.
|
||||
|
||||
~> **Important:** To use customer-managed encryption keys, you need to create a key and give your project's GCS service agent permission to use it with the Cloud KMS CryptoKey Encrypter/Decrypter predefined role.
|
||||
|
||||
## Configuration Variables
|
||||
|
||||
!> **Warning:** We recommend using environment variables to supply credentials and other sensitive data. If you use `-backend-config` or hardcode these values directly in your configuration, Terraform will include these values in both the `.terraform` subdirectory and in plan files. Refer to [Credentials and Sensitive Data](/language/settings/backends/configuration#credentials-and-sensitive-data) for details.
|
||||
!> **Warning:** We recommend using environment variables to supply credentials and other sensitive data. If you use `-backend-config` or hardcode these values directly in your configuration, Terraform includes these values in both the `.terraform` subdirectory and in plan files. Refer to [Credentials and Sensitive Data](/language/settings/backends/configuration#credentials-and-sensitive-data) for details.
|
||||
|
||||
The following configuration options are supported:
|
||||
|
||||
@ -84,9 +106,7 @@ The following configuration options are supported:
|
||||
Guidelines](https://cloud.google.com/storage/docs/bucketnaming.html#requirements).
|
||||
- `credentials` / `GOOGLE_BACKEND_CREDENTIALS` / `GOOGLE_CREDENTIALS` -
|
||||
(Optional) Local path to Google Cloud Platform account credentials in JSON
|
||||
format. If unset, [Google Application Default
|
||||
Credentials](https://developers.google.com/identity/protocols/application-default-credentials)
|
||||
are used. The provided credentials must have Storage Object Admin role on the bucket.
|
||||
format. If unset, the path uses [Google Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials). The provided credentials must have the Storage Object Admin role on the bucket.
|
||||
**Warning**: if using the Google Cloud Platform provider as well, it will
|
||||
also pick up the `GOOGLE_CREDENTIALS` environment variable.
|
||||
- `impersonate_service_account` - (Optional) The service account to impersonate for accessing the State Bucket.
|
||||
@ -103,6 +123,11 @@ The following configuration options are supported:
|
||||
- `prefix` - (Optional) GCS prefix inside the bucket. Named states for
|
||||
workspaces are stored in an object called `<prefix>/<name>.tfstate`.
|
||||
- `encryption_key` / `GOOGLE_ENCRYPTION_KEY` - (Optional) A 32 byte base64
|
||||
encoded 'customer supplied encryption key' used to encrypt all state. For
|
||||
more information see [Customer Supplied Encryption
|
||||
Keys](https://cloud.google.com/storage/docs/encryption#customer-supplied).
|
||||
encoded 'customer-supplied encryption key' used when reading and writing state files in the bucket. For
|
||||
more information see [Customer-supplied Encryption
|
||||
Keys](https://cloud.google.com/storage/docs/encryption/customer-supplied-keys).
|
||||
- `kms_encryption_key` / `GOOGLE_KMS_ENCRYPTION_KEY` - (Optional) A Cloud KMS key ('customer-managed encryption key')
|
||||
used when reading and writing state files in the bucket.
|
||||
Format should be `projects/{{project}}/locations/{{location}}/keyRings/{{keyRing}}/cryptoKeys/{{name}}`.
|
||||
For more information, including IAM requirements, see [Customer-managed Encryption
|
||||
Keys](https://cloud.google.com/storage/docs/encryption/customer-managed-keys).
|
Loading…
Reference in New Issue
Block a user