remote: when saving state, create a pending state version then upload

internal/backend/remote/backend_state.go

@@ -9,7 +9,9 @@ import (
 	"crypto/md5"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"log"
 
 	tfe "github.com/hashicorp/go-tfe"
 
@@ -61,32 +63,14 @@ func (r *remoteClient) Get() (*remote.Payload, error) {
 	}, nil
 }
 
-// Put the remote state.
+func (r *remoteClient) uploadStateFallback(ctx context.Context, stateFile *statefile.File, state []byte, jsonStateOutputs []byte) error {
-func (r *remoteClient) Put(state []byte) error {
-	ctx := context.Background()
-
-	// Read the raw state into a Terraform state.
-	stateFile, err := statefile.Read(bytes.NewReader(state))
-	if err != nil {
-		return fmt.Errorf("Error reading state: %s", err)
-	}
-
-	ov, err := jsonstate.MarshalOutputs(stateFile.State.RootModule().OutputValues)
-	if err != nil {
-		return fmt.Errorf("Error reading output values: %s", err)
-	}
-	o, err := json.Marshal(ov)
-	if err != nil {
-		return fmt.Errorf("Error converting output values to json: %s", err)
-	}
-
 	options := tfe.StateVersionCreateOptions{
 		Lineage:          tfe.String(stateFile.Lineage),
 		Serial:           tfe.Int64(int64(stateFile.Serial)),
 		MD5:              tfe.String(fmt.Sprintf("%x", md5.Sum(state))),
-		State:            tfe.String(base64.StdEncoding.EncodeToString(state)),
 		Force:            tfe.Bool(r.forcePush),
-		JSONStateOutputs: tfe.String(base64.StdEncoding.EncodeToString(o)),
+		State:            tfe.String(base64.StdEncoding.EncodeToString(state)),
+		JSONStateOutputs: tfe.String(base64.StdEncoding.EncodeToString(jsonStateOutputs)),
 	}
 
 	// If we have a run ID, make sure to add it to the options
@@ -96,10 +80,61 @@ func (r *remoteClient) Put(state []byte) error {
 	}
 
 	// Create the new state.
-	_, err = r.client.StateVersions.Create(ctx, r.workspace.ID, options)
+	_, err := r.client.StateVersions.Create(ctx, r.workspace.ID, options)
 	if err != nil {
 		r.stateUploadErr = true
-		return fmt.Errorf("Error uploading state: %v", err)
+		return fmt.Errorf("error uploading state in compatibility mode: %v", err)
+	}
+	return err
+}
+
+// Put the remote state.
+func (r *remoteClient) Put(state []byte) error {
+	ctx := context.Background()
+
+	// Read the raw state into a Terraform state.
+	stateFile, err := statefile.Read(bytes.NewReader(state))
+	if err != nil {
+		return fmt.Errorf("error reading state: %s", err)
+	}
+
+	ov, err := jsonstate.MarshalOutputs(stateFile.State.RootModule().OutputValues)
+	if err != nil {
+		return fmt.Errorf("error reading output values: %s", err)
+	}
+	o, err := json.Marshal(ov)
+	if err != nil {
+		return fmt.Errorf("error converting output values to json: %s", err)
+	}
+
+	options := tfe.StateVersionUploadOptions{
+		StateVersionCreateOptions: tfe.StateVersionCreateOptions{
+			Lineage:          tfe.String(stateFile.Lineage),
+			Serial:           tfe.Int64(int64(stateFile.Serial)),
+			MD5:              tfe.String(fmt.Sprintf("%x", md5.Sum(state))),
+			Force:            tfe.Bool(r.forcePush),
+			JSONStateOutputs: tfe.String(base64.StdEncoding.EncodeToString(o)),
+		},
+		RawState: state,
+	}
+
+	// If we have a run ID, make sure to add it to the options
+	// so the state will be properly associated with the run.
+	if r.runID != "" {
+		options.Run = &tfe.Run{ID: r.runID}
+	}
+
+	// Create the new state.
+	// Create the new state.
+	_, err = r.client.StateVersions.Upload(ctx, r.workspace.ID, options)
+	if errors.Is(err, tfe.ErrStateVersionUploadNotSupported) {
+		// Create the new state with content included in the request (Terraform Enterprise v202306-1 and below)
+		log.Println("[INFO] Detected that state version upload is not supported. Retrying using compatibility state upload.")
+		return r.uploadStateFallback(ctx, stateFile, state, o)
+	}
+	if err != nil {
+		r.stateUploadErr = true
+		return fmt.Errorf("error uploading state: %v", err)
 	}
 
 	return nil
@@ -109,7 +144,7 @@ func (r *remoteClient) Put(state []byte) error {
 func (r *remoteClient) Delete() error {
 	err := r.client.Workspaces.Delete(context.Background(), r.organization, r.workspace.Name)
 	if err != nil && err != tfe.ErrResourceNotFound {
-		return fmt.Errorf("Error deleting workspace %s: %v", r.workspace.Name, err)
+		return fmt.Errorf("error deleting workspace %s: %v", r.workspace.Name, err)
 	}
 
 	return nil

internal/backend/remote/backend_state_test.go

@@ -41,7 +41,7 @@ func TestRemoteClient_stateLock(t *testing.T) {
 	remote.TestRemoteLocks(t, s1.(*remote.State).Client, s2.(*remote.State).Client)
 }
 
-func TestRemoteClient_withRunID(t *testing.T) {
+func TestRemoteClient_Put_withRunID(t *testing.T) {
 	// Set the TFE_RUN_ID environment variable before creating the client!
 	if err := os.Setenv("TFE_RUN_ID", cloud.GenerateID("run-")); err != nil {
 		t.Fatalf("error setting env var TFE_RUN_ID: %v", err)