Implemented GCloud backend supporting remote locking and multiple workspaces.

backend/init/init.go

@@ -14,6 +14,7 @@ import (
 	backendAzure "github.com/hashicorp/terraform/backend/remote-state/azure"
 	backendconsul "github.com/hashicorp/terraform/backend/remote-state/consul"
 	backendetcdv3 "github.com/hashicorp/terraform/backend/remote-state/etcdv3"
+	backendGCloud "github.com/hashicorp/terraform/backend/remote-state/gcloud"
 	backendinmem "github.com/hashicorp/terraform/backend/remote-state/inmem"
 	backendS3 "github.com/hashicorp/terraform/backend/remote-state/s3"
 	backendSwift "github.com/hashicorp/terraform/backend/remote-state/swift"
@@ -47,6 +48,7 @@ func init() {
 			`Warning: "azure" name is deprecated, please use "azurerm"`),
 		"azurerm": func() backend.Backend { return backendAzure.New() },
 		"etcdv3":  func() backend.Backend { return backendetcdv3.New() },
+		"gcloud":  func() backend.Backend { return backendGCloud.New() },
 	}
 
 	// Add the legacy remote backends that haven't yet been convertd to

backend/remote-state/gcloud/backend.go

@@ -0,0 +1,101 @@
+package gcloud
+
+import (
+	"cloud.google.com/go/storage"
+	"context"
+	"fmt"
+	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/helper/pathorcontents"
+	"github.com/hashicorp/terraform/helper/schema"
+	googleContext "golang.org/x/net/context"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/option"
+)
+
+func New() backend.Backend {
+	s := &schema.Backend{
+		Schema: map[string]*schema.Schema{
+			"bucket": {
+				Type:        schema.TypeString,
+				Required:    true,
+				Description: "The name of the Google Cloud Storage bucket",
+			},
+
+			"state_dir": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "The directory where state files will be saved inside the bucket",
+			},
+
+			"credentials": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "Google Cloud JSON Account Key",
+				Default:     "",
+			},
+		},
+	}
+
+	result := &Backend{Backend: s}
+	result.Backend.ConfigureFunc = result.configure
+	return result
+}
+
+type Backend struct {
+	*schema.Backend
+
+	storageClient  *storage.Client
+	storageContext googleContext.Context
+
+	bucketName string
+	stateDir   string
+}
+
+func (b *Backend) configure(ctx context.Context) error {
+	if b.storageClient != nil {
+		return nil
+	}
+
+	storageOAuth2Scope := "https://www.googleapis.com/auth/devstorage.read_write"
+
+	data := schema.FromContextBackendConfig(ctx)
+
+	b.bucketName = data.Get("bucket").(string)
+	b.stateDir = data.Get("state_dir").(string)
+	b.storageContext = googleContext.Background()
+
+	credentials := data.Get("credentials").(string)
+
+	var tokenSource oauth2.TokenSource
+
+	if credentials != "" {
+		credentialsJson, _, err := pathorcontents.Read(data.Get("credentials").(string))
+		if err != nil {
+			return fmt.Errorf("Error loading credentials: %v", err)
+		}
+
+		jwtConfig, err := google.JWTConfigFromJSON([]byte(credentialsJson), storageOAuth2Scope)
+		if err != nil {
+			return fmt.Errorf("Failed to get Google OAuth2 token: %v", err)
+		}
+
+		tokenSource = jwtConfig.TokenSource(b.storageContext)
+	} else {
+		defaultTokenSource, err := google.DefaultTokenSource(b.storageContext, storageOAuth2Scope)
+		if err != nil {
+			return fmt.Errorf("Failed to get Google Application Default Credentials: %v", err)
+		}
+
+		tokenSource = defaultTokenSource
+	}
+
+	client, err := storage.NewClient(b.storageContext, option.WithTokenSource(tokenSource))
+	if err != nil {
+		return fmt.Errorf("Failed to create Google Storage client: %v", err)
+	}
+
+	b.storageClient = client
+
+	return nil
+}

backend/remote-state/gcloud/backend_state.go

@@ -0,0 +1,165 @@
+package gcloud
+
+import (
+	"cloud.google.com/go/storage"
+	"errors"
+	"fmt"
+	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/state"
+	"github.com/hashicorp/terraform/state/remote"
+	"github.com/hashicorp/terraform/terraform"
+	"google.golang.org/api/iterator"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+func (b *Backend) States() ([]string, error) {
+	workspaces := []string{backend.DefaultStateName}
+	var stateRegex *regexp.Regexp
+	var err error
+	if b.stateDir == "" {
+		stateRegex = regexp.MustCompile(`^(.+)\.tfstate$`)
+	} else {
+		stateRegex, err = regexp.Compile(fmt.Sprintf("^%v/(.+)\\.tfstate$", regexp.QuoteMeta(b.stateDir)))
+		if err != nil {
+			return []string{}, fmt.Errorf("Failed to compile regex for querying states: %v", err)
+		}
+	}
+
+	bucket := b.storageClient.Bucket(b.bucketName)
+	query := &storage.Query{
+		Prefix: b.stateDir,
+	}
+
+	files := bucket.Objects(b.storageContext, query)
+	for {
+		attrs, err := files.Next()
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return []string{}, fmt.Errorf("Failed to query remote states: %v", err)
+		}
+
+		matches := stateRegex.FindStringSubmatch(attrs.Name)
+		if len(matches) == 2 && matches[1] != backend.DefaultStateName {
+			workspaces = append(workspaces, matches[1])
+		}
+	}
+
+	sort.Strings(workspaces[1:])
+	return workspaces, nil
+}
+
+func (b *Backend) DeleteState(name string) error {
+	if name == backend.DefaultStateName || name == "" {
+		return fmt.Errorf("Can't delete default state")
+	}
+
+	client, err := b.remoteClient(name)
+	if err != nil {
+		return fmt.Errorf("Failed to create Google Storage client: %v", err)
+	}
+
+	err = client.Delete()
+	if err != nil {
+		return fmt.Errorf("Failed to delete state file %v: %v", client.stateFileUrl(), err)
+	}
+
+	return nil
+}
+
+// get a remote client configured for this state
+func (b *Backend) remoteClient(name string) (*RemoteClient, error) {
+	if name == "" {
+		return nil, errors.New("Missing state name")
+	}
+
+	client := &RemoteClient{
+		storageContext: b.storageContext,
+		storageClient:  b.storageClient,
+		bucketName:     b.bucketName,
+		stateFilePath:  b.stateFileName(name),
+		lockFilePath:   b.lockFileName(name),
+	}
+
+	return client, nil
+}
+
+func (b *Backend) State(name string) (state.State, error) {
+	client, err := b.remoteClient(name)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to create Google Storage client: %v", err)
+	}
+
+	stateMgr := &remote.State{Client: client}
+	lockInfo := state.NewLockInfo()
+	lockInfo.Operation = "init"
+	lockId, err := stateMgr.Lock(lockInfo)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to lock state in Google Cloud Storage: %v", err)
+	}
+
+	// Local helper function so we can call it multiple places
+	lockUnlock := func(parent error) error {
+		if err := stateMgr.Unlock(lockId); err != nil {
+			return fmt.Errorf(strings.TrimSpace(errStateUnlock), lockId, client.lockFileUrl(), err)
+		}
+
+		return parent
+	}
+
+	// Grab the value
+	if err := stateMgr.RefreshState(); err != nil {
+		err = lockUnlock(err)
+		return nil, err
+	}
+
+	// If we have no state, we have to create an empty state
+	if v := stateMgr.State(); v == nil {
+		if err := stateMgr.WriteState(terraform.NewState()); err != nil {
+			err = lockUnlock(err)
+			return nil, err
+		}
+		if err := stateMgr.PersistState(); err != nil {
+			err = lockUnlock(err)
+			return nil, err
+		}
+	}
+
+	// Unlock, the state should now be initialized
+	if err := lockUnlock(nil); err != nil {
+		return nil, err
+	}
+
+	return stateMgr, nil
+}
+
+func (b *Backend) stateFileName(stateName string) string {
+	if b.stateDir == "" {
+		return fmt.Sprintf("%v.tfstate", stateName)
+	} else {
+		return fmt.Sprintf("%v/%v.tfstate", b.stateDir, stateName)
+	}
+}
+
+func (b *Backend) lockFileName(stateName string) string {
+	if b.stateDir == "" {
+		return fmt.Sprintf("%v.tflock", stateName)
+	} else {
+		return fmt.Sprintf("%v/%v.tflock", b.stateDir, stateName)
+	}
+}
+
+const errStateUnlock = `
+Error unlocking Google Cloud Storage state.
+
+Lock ID: %v
+Lock file URL: %v
+Error: %v
+
+You may have to force-unlock this state in order to use it again.
+The GCloud backend acquires a lock during initialization to ensure
+the initial state file is created.
+`

backend/remote-state/gcloud/client.go

@@ -0,0 +1,168 @@
+package gcloud
+
+import (
+	"cloud.google.com/go/storage"
+	"encoding/json"
+	"fmt"
+	uuid "github.com/hashicorp/go-uuid"
+	"github.com/hashicorp/terraform/state"
+	"github.com/hashicorp/terraform/state/remote"
+	"golang.org/x/net/context"
+	"io/ioutil"
+)
+
+type RemoteClient struct {
+	storageContext context.Context
+	storageClient  *storage.Client
+	bucketName     string
+	stateFilePath  string
+	lockFilePath   string
+}
+
+func (c *RemoteClient) Get() (payload *remote.Payload, err error) {
+	bucket := c.storageClient.Bucket(c.bucketName)
+	stateFile := bucket.Object(c.stateFilePath)
+	stateFileUrl := c.stateFileUrl()
+
+	stateFileReader, err := stateFile.NewReader(c.storageContext)
+	if err != nil {
+		if err == storage.ErrObjectNotExist {
+			return nil, nil
+		} else {
+			return nil, fmt.Errorf("Failed to open state file at %v: %v", stateFileUrl, err)
+		}
+	}
+	defer stateFileReader.Close()
+
+	stateFileContents, err := ioutil.ReadAll(stateFileReader)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to read state file from %v: %v", stateFileUrl, err)
+	}
+
+	stateFileAttrs, err := stateFile.Attrs(c.storageContext)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to read state file attrs from %v: %v", stateFileUrl, err)
+	}
+
+	result := &remote.Payload{
+		Data: stateFileContents,
+		MD5:  stateFileAttrs.MD5,
+	}
+
+	return result, nil
+}
+
+func (c *RemoteClient) Put(data []byte) error {
+	bucket := c.storageClient.Bucket(c.bucketName)
+	stateFile := bucket.Object(c.stateFilePath)
+
+	stateFileWriter := stateFile.NewWriter(c.storageContext)
+
+	stateFileWriter.Write(data)
+	err := stateFileWriter.Close()
+
+	if err != nil {
+		return fmt.Errorf("Failed to upload state to %v: %v", c.stateFileUrl(), err)
+	}
+
+	return nil
+}
+
+func (c *RemoteClient) Delete() error {
+	bucket := c.storageClient.Bucket(c.bucketName)
+	stateFile := bucket.Object(c.stateFilePath)
+
+	err := stateFile.Delete(c.storageContext)
+
+	if err != nil {
+		return fmt.Errorf("Failed to delete state file %v: %v", c.stateFileUrl(), err)
+	}
+
+	return nil
+}
+
+func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) {
+	if info.ID == "" {
+		lockID, err := uuid.GenerateUUID()
+		if err != nil {
+			return "", err
+		}
+
+		info.ID = lockID
+	}
+
+	info.Path = c.lockFileUrl()
+
+	infoJson, err := json.Marshal(info)
+	if err != nil {
+		return "", err
+	}
+
+	bucket := c.storageClient.Bucket(c.bucketName)
+	lockFile := bucket.Object(c.lockFilePath)
+
+	writer := lockFile.If(storage.Conditions{DoesNotExist: true}).NewWriter(c.storageContext)
+	writer.Write(infoJson)
+	if err := writer.Close(); err != nil {
+		return "", fmt.Errorf("Error while saving lock file (%v): %v", info.Path, err)
+	}
+
+	return info.ID, nil
+}
+
+func (c *RemoteClient) Unlock(id string) error {
+	lockErr := &state.LockError{}
+
+	bucket := c.storageClient.Bucket(c.bucketName)
+	lockFile := bucket.Object(c.lockFilePath)
+	lockFileUrl := c.lockFileUrl()
+
+	lockFileReader, err := lockFile.NewReader(c.storageContext)
+	if err != nil {
+		lockErr.Err = fmt.Errorf("Failed to retrieve lock info (%v): %v", lockFileUrl, err)
+		return lockErr
+	}
+	defer lockFileReader.Close()
+
+	lockFileContents, err := ioutil.ReadAll(lockFileReader)
+	if err != nil {
+		lockErr.Err = fmt.Errorf("Failed to retrieve lock info (%v): %v", lockFileUrl, err)
+		return lockErr
+	}
+
+	lockInfo := &state.LockInfo{}
+	err = json.Unmarshal(lockFileContents, lockInfo)
+	if err != nil {
+		lockErr.Err = fmt.Errorf("Failed to unmarshal lock info (%v): %v", lockFileUrl, err)
+		return lockErr
+	}
+
+	lockErr.Info = lockInfo
+
+	if lockInfo.ID != id {
+		lockErr.Err = fmt.Errorf("Lock id %q does not match existing lock", id)
+		return lockErr
+	}
+
+	lockFileAttrs, err := lockFile.Attrs(c.storageContext)
+	if err != nil {
+		lockErr.Err = fmt.Errorf("Failed to fetch lock file attrs (%v): %v", lockFileUrl, err)
+		return lockErr
+	}
+
+	err = lockFile.If(storage.Conditions{GenerationMatch: lockFileAttrs.Generation}).Delete(c.storageContext)
+	if err != nil {
+		lockErr.Err = fmt.Errorf("Failed to delete lock file (%v): %v", lockFileUrl, err)
+		return lockErr
+	}
+
+	return nil
+}
+
+func (c *RemoteClient) stateFileUrl() string {
+	return fmt.Sprintf("gs://%v/%v", c.bucketName, c.stateFilePath)
+}
+
+func (c *RemoteClient) lockFileUrl() string {
+	return fmt.Sprintf("gs://%v/%v", c.bucketName, c.lockFilePath)
+}