mirror of
https://github.com/opentofu/opentofu.git
synced 2025-02-25 18:45:20 -06:00
cloud: when saving state, create a pending state version then upload
Create a pending state version followed by a separate state upload When this version of the endpoint fails (It is not yet generally available, or when using with Terraform Enterprise) Fall back to the original call with state content included in the request. This strategy will reduce the amount of save failures due to network latency and gateway timeouts.
This commit is contained in:
Brandon Croft
parent
5ed38eb3fa
commit
9fe3f7a7b4
6
go.mod
6
go.mod
@ -39,8 +39,8 @@ require (
|
||||
github.com/hashicorp/go-hclog v1.4.0
|
||||
github.com/hashicorp/go-multierror v1.1.1
|
||||
github.com/hashicorp/go-plugin v1.4.3
|
||||
github.com/hashicorp/go-retryablehttp v0.7.2
|
||||
github.com/hashicorp/go-tfe v1.26.0
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4
|
||||
github.com/hashicorp/go-tfe v1.28.0
|
||||
github.com/hashicorp/go-uuid v1.0.3
|
||||
github.com/hashicorp/go-version v1.6.0
|
||||
github.com/hashicorp/hcl v1.0.0
|
||||
@ -208,7 +208,7 @@ require (
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
|
||||
golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect
|
||||
golang.org/x/sync v0.1.0 // indirect
|
||||
golang.org/x/sync v0.3.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
|
||||
14
go.sum
14
go.sum
@ -603,8 +603,8 @@ github.com/hashicorp/go-plugin v1.4.3 h1:DXmvivbWD5qdiBts9TpBC7BYL1Aia5sxbRgQB+v
|
||||
github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ=
|
||||
github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
|
||||
github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
|
||||
github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
|
||||
@ -616,8 +616,8 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
|
||||
github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
|
||||
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
|
||||
github.com/hashicorp/go-tfe v1.26.0 h1:aacguqCENg6Z7ttfhAxdbbY2vm/jKrntl5sUUY0h6EM=
|
||||
github.com/hashicorp/go-tfe v1.26.0/go.mod h1:1Y6nsdMuJ14lYdc1VMLl/erlthvMzUsJn+WYWaAdSc4=
|
||||
github.com/hashicorp/go-tfe v1.28.0 h1:YQNfHz5UPMiOD2idad4GCjzG3R2ExPww741PBPqMOIU=
|
||||
github.com/hashicorp/go-tfe v1.28.0/go.mod h1:z0182DGE/63AKUaWblUVBIrt+xdSmsuuXg5AoxGqDF4=
|
||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
@ -931,7 +931,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.588 h1:DYtBXB7sVc3EOW5horg8j55cLZynhsLYhHrvQ/jXKKM=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.588/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
|
||||
@ -1167,8 +1167,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
||||
25
internal/cloud/errored.tfstate
Normal file
25
internal/cloud/errored.tfstate
Normal file
@ -0,0 +1,25 @@
|
||||
{
|
||||
"version": 4,
|
||||
"terraform_version": "0.14.0",
|
||||
"serial": 1,
|
||||
"lineage": "30a4d634-f765-186a-f411-7dfa798a767e",
|
||||
"outputs": {},
|
||||
"resources": [
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "null_resource",
|
||||
"name": "foo",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/null\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"id": "yes"
|
||||
},
|
||||
"sensitive_attributes": []
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"check_results": null
|
||||
}
|
||||
@ -238,6 +238,7 @@ func (s *State) PersistState(schemas *terraform.Schemas) error {
|
||||
s.readState = s.state.DeepCopy()
|
||||
s.readLineage = s.lineage
|
||||
s.readSerial = s.serial
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -262,15 +263,13 @@ func (s *State) ShouldPersistIntermediateState(info *local.IntermediateStatePers
|
||||
return currentInterval >= wantInterval
|
||||
}
|
||||
|
||||
func (s *State) uploadState(lineage string, serial uint64, isForcePush bool, state, jsonState, jsonStateOutputs []byte) error {
|
||||
ctx := context.Background()
|
||||
|
||||
func (s *State) uploadStateFallback(ctx context.Context, lineage string, serial uint64, isForcePush bool, state, jsonState, jsonStateOutputs []byte) error {
|
||||
options := tfe.StateVersionCreateOptions{
|
||||
Lineage: tfe.String(lineage),
|
||||
Serial: tfe.Int64(int64(serial)),
|
||||
MD5: tfe.String(fmt.Sprintf("%x", md5.Sum(state))),
|
||||
State: tfe.String(base64.StdEncoding.EncodeToString(state)),
|
||||
Force: tfe.Bool(isForcePush),
|
||||
State: tfe.String(base64.StdEncoding.EncodeToString(state)),
|
||||
JSONState: tfe.String(base64.StdEncoding.EncodeToString(jsonState)),
|
||||
JSONStateOutputs: tfe.String(base64.StdEncoding.EncodeToString(jsonStateOutputs)),
|
||||
}
|
||||
@ -282,13 +281,46 @@ func (s *State) uploadState(lineage string, serial uint64, isForcePush bool, sta
|
||||
options.Run = &tfe.Run{ID: runID}
|
||||
}
|
||||
|
||||
// Create the new state.
|
||||
_, err := s.tfeClient.StateVersions.Create(ctx, s.workspace.ID, options)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *State) uploadState(lineage string, serial uint64, isForcePush bool, state, jsonState, jsonStateOutputs []byte) error {
|
||||
ctx := context.Background()
|
||||
|
||||
options := tfe.StateVersionUploadOptions{
|
||||
StateVersionCreateOptions: tfe.StateVersionCreateOptions{
|
||||
Lineage: tfe.String(lineage),
|
||||
Serial: tfe.Int64(int64(serial)),
|
||||
MD5: tfe.String(fmt.Sprintf("%x", md5.Sum(state))),
|
||||
Force: tfe.Bool(isForcePush),
|
||||
JSONStateOutputs: tfe.String(base64.StdEncoding.EncodeToString(jsonStateOutputs)),
|
||||
},
|
||||
RawState: state,
|
||||
RawJSONState: jsonState,
|
||||
}
|
||||
|
||||
// If we have a run ID, make sure to add it to the options
|
||||
// so the state will be properly associated with the run.
|
||||
runID := os.Getenv("TFE_RUN_ID")
|
||||
if runID != "" {
|
||||
options.StateVersionCreateOptions.Run = &tfe.Run{ID: runID}
|
||||
}
|
||||
|
||||
// The server is allowed to dynamically request a different time interval
|
||||
// than we'd normally use, for example if it's currently under heavy load
|
||||
// and needs clients to backoff for a while.
|
||||
ctx = tfe.ContextWithResponseHeaderHook(ctx, s.readSnapshotIntervalHeader)
|
||||
|
||||
// Create the new state.
|
||||
_, err := s.tfeClient.StateVersions.Create(ctx, s.workspace.ID, options)
|
||||
_, err := s.tfeClient.StateVersions.Upload(ctx, s.workspace.ID, options)
|
||||
if errors.Is(err, tfe.ErrStateVersionUploadNotSupported) {
|
||||
// Create the new state with content included in the request (Terraform Enterprise v202306-1 and below)
|
||||
log.Println("[INFO] Detected that state version upload is not supported. Retrying using compatibility state upload.")
|
||||
return s.uploadStateFallback(ctx, lineage, serial, isForcePush, state, jsonState, jsonStateOutputs)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@ -298,7 +298,6 @@ func TestState_PersistState(t *testing.T) {
|
||||
// since HTTP-level concerns like headers are out of scope for the
|
||||
// mock client we typically use in other tests in this package, which
|
||||
// aim to abstract away HTTP altogether.
|
||||
var serverURL string
|
||||
|
||||
// Didn't want to repeat myself here
|
||||
for _, testCase := range []struct {
|
||||
@ -314,10 +313,9 @@ func TestState_PersistState(t *testing.T) {
|
||||
snapshotsEnabled: false,
|
||||
},
|
||||
} {
|
||||
server := testServerWithSnapshotsEnabled(t, serverURL, testCase.snapshotsEnabled)
|
||||
server := testServerWithSnapshotsEnabled(t, testCase.snapshotsEnabled)
|
||||
|
||||
defer server.Close()
|
||||
serverURL = server.URL
|
||||
cfg := &tfe.Config{
|
||||
Address: server.URL,
|
||||
BasePath: "api",
|
||||
|
||||
@ -383,8 +383,9 @@ func testServerWithHandlers(handlers map[string]func(http.ResponseWriter, *http.
|
||||
return httptest.NewServer(mux)
|
||||
}
|
||||
|
||||
func testServerWithSnapshotsEnabled(t *testing.T, serverURL string, enabled bool) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
func testServerWithSnapshotsEnabled(t *testing.T, enabled bool) *httptest.Server {
|
||||
var serverURL string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Log(r.Method, r.URL.String())
|
||||
|
||||
if r.URL.Path == "/state-json" {
|
||||
@ -400,6 +401,7 @@ func testServerWithSnapshotsEnabled(t *testing.T, serverURL string, enabled bool
|
||||
w.Write(respBody)
|
||||
return
|
||||
}
|
||||
|
||||
if r.URL.Path == "/api/ping" {
|
||||
t.Log("pretending to be Ping")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
@ -409,8 +411,10 @@ func testServerWithSnapshotsEnabled(t *testing.T, serverURL string, enabled bool
|
||||
fakeBody := map[string]any{
|
||||
"data": map[string]any{
|
||||
"type": "state-versions",
|
||||
"id": GenerateID("sv-"),
|
||||
"attributes": map[string]any{
|
||||
"hosted-state-download-url": serverURL + "/state-json",
|
||||
"hosted-state-upload-url": serverURL + "/state-json",
|
||||
},
|
||||
},
|
||||
}
|
||||
@ -435,6 +439,8 @@ func testServerWithSnapshotsEnabled(t *testing.T, serverURL string, enabled bool
|
||||
w.Header().Set("x-terraform-snapshot-interval", "300")
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case "PUT":
|
||||
t.Log("pretending to be Archivist")
|
||||
default:
|
||||
t.Fatal("don't know what API operation this was supposed to be")
|
||||
}
|
||||
@ -442,6 +448,8 @@ func testServerWithSnapshotsEnabled(t *testing.T, serverURL string, enabled bool
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(fakeBodyRaw)
|
||||
}))
|
||||
serverURL = server.URL
|
||||
return server
|
||||
}
|
||||
|
||||
// testDefaultRequestHandlers is a map of request handlers intended to be used in a request
|
||||
|
||||
@ -1254,6 +1254,7 @@ func (m *MockStateVersions) Create(ctx context.Context, workspaceID string, opti
|
||||
sv := &tfe.StateVersion{
|
||||
ID: id,
|
||||
DownloadURL: url,
|
||||
UploadURL: fmt.Sprintf("/_archivist/upload/%s", id),
|
||||
Serial: *options.Serial,
|
||||
}
|
||||
|
||||
@ -1261,7 +1262,6 @@ func (m *MockStateVersions) Create(ctx context.Context, workspaceID string, opti
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m.states[sv.DownloadURL] = state
|
||||
m.outputStates[sv.ID] = []byte(*options.JSONStateOutputs)
|
||||
m.stateVersions[sv.ID] = sv
|
||||
@ -1270,6 +1270,13 @@ func (m *MockStateVersions) Create(ctx context.Context, workspaceID string, opti
|
||||
return sv, nil
|
||||
}
|
||||
|
||||
func (m *MockStateVersions) Upload(ctx context.Context, workspaceID string, options tfe.StateVersionUploadOptions) (*tfe.StateVersion, error) {
|
||||
createOptions := options.StateVersionCreateOptions
|
||||
createOptions.State = tfe.String(base64.StdEncoding.EncodeToString(options.RawState))
|
||||
|
||||
return m.Create(ctx, workspaceID, createOptions)
|
||||
}
|
||||
|
||||
func (m *MockStateVersions) Read(ctx context.Context, svID string) (*tfe.StateVersion, error) {
|
||||
return m.ReadWithOptions(ctx, svID, nil)
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user