diff --git a/backend/local/backend_plan_test.go b/backend/local/backend_plan_test.go index a5643b3216..74791456f8 100644 --- a/backend/local/backend_plan_test.go +++ b/backend/local/backend_plan_test.go @@ -320,8 +320,7 @@ func testPlanState() *states.State { Name: "foo", }.Instance(addrs.IntKey(0)), &states.ResourceInstanceObjectSrc{ - Status: states.ObjectReady, - SchemaVersion: 1, + Status: states.ObjectReady, AttrsJSON: []byte(`{ "ami": "bar", "network_interface": [{ diff --git a/states/instance_object_src.go b/states/instance_object_src.go index 9b45a8735a..6cb3c27e2a 100644 --- a/states/instance_object_src.go +++ b/states/instance_object_src.go @@ -89,3 +89,25 @@ func (os *ResourceInstanceObjectSrc) Decode(ty cty.Type) (*ResourceInstanceObjec Private: os.Private, }, nil } + +// CompleteUpgrade creates a new ResourceInstanceObjectSrc by copying the +// metadata from the receiver and writing in the given new schema version +// and attribute value that are presumed to have resulted from upgrading +// from an older schema version. +func (os *ResourceInstanceObjectSrc) CompleteUpgrade(newAttrs cty.Value, newType cty.Type, newSchemaVersion uint64) (*ResourceInstanceObjectSrc, error) { + new := os.DeepCopy() + new.AttrsFlat = nil // We always use JSON after an upgrade, even if the source used flatmap + + // This is the same principle as ResourceInstanceObject.Encode, but + // avoiding a decode/re-encode cycle because we don't have type info + // available for the "old" attributes. + newAttrs = cty.UnknownAsNull(newAttrs) + src, err := ctyjson.Marshal(newAttrs, newType) + if err != nil { + return nil, err + } + + new.AttrsJSON = src + new.SchemaVersion = newSchemaVersion + return new, nil +} diff --git a/terraform/context_refresh_test.go b/terraform/context_refresh_test.go index 72d019eab5..d2a5190aa3 100644 --- a/terraform/context_refresh_test.go +++ b/terraform/context_refresh_test.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/terraform/config/hcl2shim" "github.com/hashicorp/terraform/configs/configschema" "github.com/hashicorp/terraform/providers" + "github.com/hashicorp/terraform/states" ) func TestContext2Refresh(t *testing.T) { @@ -1567,3 +1568,169 @@ aws_instance.bar: t.Fatalf("expected:\n%s\n\ngot:\n%s", expected, actual) } } + +func TestContext2Refresh_schemaUpgradeFlatmap(t *testing.T) { + m := testModule(t, "empty") + p := testProvider("test") + p.GetSchemaReturn = &ProviderSchema{ + ResourceTypes: map[string]*configschema.Block{ + "test_thing": { + Attributes: map[string]*configschema.Attribute{ + "name": { // imagining we renamed this from "id" + Type: cty.String, + Optional: true, + }, + }, + }, + }, + ResourceTypeSchemaVersions: map[string]uint64{ + "test_thing": 5, + }, + } + p.UpgradeResourceStateResponse = providers.UpgradeResourceStateResponse{ + UpgradedState: cty.ObjectVal(map[string]cty.Value{ + "name": cty.StringVal("foo"), + }), + } + + s := states.BuildState(func(s *states.SyncState) { + s.SetResourceInstanceCurrent( + addrs.Resource{ + Mode: addrs.ManagedResourceMode, + Type: "test_thing", + Name: "bar", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + &states.ResourceInstanceObjectSrc{ + Status: states.ObjectReady, + SchemaVersion: 3, + AttrsFlat: map[string]string{ + "id": "foo", + }, + }, + addrs.ProviderConfig{Type: "test"}.Absolute(addrs.RootModuleInstance), + ) + }) + + ctx := testContext2(t, &ContextOpts{ + Config: m, + ProviderResolver: providers.ResolverFixed( + map[string]providers.Factory{ + "test": testProviderFuncFixed(p), + }, + ), + State: s, + }) + + state, diags := ctx.Refresh() + if diags.HasErrors() { + t.Fatal(diags.Err()) + } + + { + got := p.UpgradeResourceStateRequest + want := providers.UpgradeResourceStateRequest{ + TypeName: "test_thing", + Version: 3, + RawStateFlatmap: map[string]string{ + "id": "foo", + }, + } + if !cmp.Equal(got, want) { + t.Errorf("wrong upgrade request\n%s", cmp.Diff(want, got)) + } + } + + { + got := state.String() + want := strings.TrimSpace(` +test_thing.bar: + ID = + provider = provider.test + name = foo +`) + if got != want { + t.Fatalf("wrong result state\ngot:\n%s\n\nwant:\n%s", got, want) + } + } +} + +func TestContext2Refresh_schemaUpgradeJSON(t *testing.T) { + m := testModule(t, "empty") + p := testProvider("test") + p.GetSchemaReturn = &ProviderSchema{ + ResourceTypes: map[string]*configschema.Block{ + "test_thing": { + Attributes: map[string]*configschema.Attribute{ + "name": { // imagining we renamed this from "id" + Type: cty.String, + Optional: true, + }, + }, + }, + }, + ResourceTypeSchemaVersions: map[string]uint64{ + "test_thing": 5, + }, + } + p.UpgradeResourceStateResponse = providers.UpgradeResourceStateResponse{ + UpgradedState: cty.ObjectVal(map[string]cty.Value{ + "name": cty.StringVal("foo"), + }), + } + + s := states.BuildState(func(s *states.SyncState) { + s.SetResourceInstanceCurrent( + addrs.Resource{ + Mode: addrs.ManagedResourceMode, + Type: "test_thing", + Name: "bar", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + &states.ResourceInstanceObjectSrc{ + Status: states.ObjectReady, + SchemaVersion: 3, + AttrsJSON: []byte(`{"id":"foo"}`), + }, + addrs.ProviderConfig{Type: "test"}.Absolute(addrs.RootModuleInstance), + ) + }) + + ctx := testContext2(t, &ContextOpts{ + Config: m, + ProviderResolver: providers.ResolverFixed( + map[string]providers.Factory{ + "test": testProviderFuncFixed(p), + }, + ), + State: s, + }) + + state, diags := ctx.Refresh() + if diags.HasErrors() { + t.Fatal(diags.Err()) + } + + { + got := p.UpgradeResourceStateRequest + want := providers.UpgradeResourceStateRequest{ + TypeName: "test_thing", + Version: 3, + RawStateJSON: []byte(`{"id":"foo"}`), + } + if !cmp.Equal(got, want) { + t.Errorf("wrong upgrade request\n%s", cmp.Diff(want, got)) + } + } + + { + got := state.String() + want := strings.TrimSpace(` +test_thing.bar: + ID = + provider = provider.test + name = foo +`) + if got != want { + t.Fatalf("wrong result state\ngot:\n%s\n\nwant:\n%s", got, want) + } + } +} diff --git a/terraform/eval_state.go b/terraform/eval_state.go index 82cf7838cc..d861c07e4d 100644 --- a/terraform/eval_state.go +++ b/terraform/eval_state.go @@ -4,13 +4,11 @@ import ( "fmt" "log" - "github.com/hashicorp/terraform/tfdiags" - - "github.com/hashicorp/terraform/configs" - "github.com/hashicorp/terraform/addrs" + "github.com/hashicorp/terraform/configs" "github.com/hashicorp/terraform/providers" "github.com/hashicorp/terraform/states" + "github.com/hashicorp/terraform/tfdiags" ) // EvalReadState is an EvalNode implementation that reads the @@ -50,9 +48,18 @@ func (n *EvalReadState) Eval(ctx EvalContext) (interface{}, error) { } schema, currentVersion := (*n.ProviderSchema).SchemaForResourceAddr(n.Addr.ContainingResource()) - if src.SchemaVersion < currentVersion { - // TODO: Implement schema upgrades - return nil, fmt.Errorf("schema upgrading is not yet implemented to take state from version %d to version %d", src.SchemaVersion, currentVersion) + if schema == nil { + // Shouldn't happen since we should've failed long ago if no schema is present + return nil, fmt.Errorf("no schema available for %s while reading state; this is a bug in Terraform and should be reported", absAddr) + } + var diags tfdiags.Diagnostics + src, diags = UpgradeResourceState(absAddr, *n.Provider, src, schema, currentVersion) + if diags.HasErrors() { + // Note that we don't have any channel to return warnings here. We'll + // accept that for now since warnings during a schema upgrade would + // be pretty weird anyway, since this operation is supposed to seem + // invisible to the user. + return nil, diags.Err() } obj, err := src.Decode(schema.ImpliedType()) @@ -110,9 +117,18 @@ func (n *EvalReadStateDeposed) Eval(ctx EvalContext) (interface{}, error) { } schema, currentVersion := (*n.ProviderSchema).SchemaForResourceAddr(n.Addr.ContainingResource()) - if src.SchemaVersion < currentVersion { - // TODO: Implement schema upgrades - return nil, fmt.Errorf("schema upgrading is not yet implemented to take state from version %d to version %d", src.SchemaVersion, currentVersion) + if schema == nil { + // Shouldn't happen since we should've failed long ago if no schema is present + return nil, fmt.Errorf("no schema available for %s while reading state; this is a bug in Terraform and should be reported", absAddr) + } + var diags tfdiags.Diagnostics + src, diags = UpgradeResourceState(absAddr, *n.Provider, src, schema, currentVersion) + if diags.HasErrors() { + // Note that we don't have any channel to return warnings here. We'll + // accept that for now since warnings during a schema upgrade would + // be pretty weird anyway, since this operation is supposed to seem + // invisible to the user. + return nil, diags.Err() } obj, err := src.Decode(schema.ImpliedType()) diff --git a/terraform/eval_state_upgrade.go b/terraform/eval_state_upgrade.go new file mode 100644 index 0000000000..e78cc206b7 --- /dev/null +++ b/terraform/eval_state_upgrade.go @@ -0,0 +1,104 @@ +package terraform + +import ( + "fmt" + "log" + + "github.com/hashicorp/terraform/addrs" + "github.com/hashicorp/terraform/configs/configschema" + "github.com/hashicorp/terraform/providers" + "github.com/hashicorp/terraform/states" + "github.com/hashicorp/terraform/tfdiags" +) + +// UpgradeResourceState will, if necessary, run the provider-defined upgrade +// logic against the given state object to make it compliant with the +// current schema version. This is a no-op if the given state object is +// already at the latest version. +// +// If any errors occur during upgrade, error diagnostics are returned. In that +// case it is not safe to proceed with using the original state object. +func UpgradeResourceState(addr addrs.AbsResourceInstance, provider providers.Interface, src *states.ResourceInstanceObjectSrc, currentSchema *configschema.Block, currentVersion uint64) (*states.ResourceInstanceObjectSrc, tfdiags.Diagnostics) { + if src.SchemaVersion == currentVersion { + // No upgrading required, then. + return src, nil + } + if addr.Resource.Resource.Mode != addrs.ManagedResourceMode { + // We only do state upgrading for managed resources. + return src, nil + } + + providerType := addr.Resource.Resource.DefaultProviderConfig().Type + if src.SchemaVersion > currentVersion { + log.Printf("[TRACE] UpgradeResourceState: can't downgrade state for %s from version %d to %d", addr, src.SchemaVersion, currentVersion) + var diags tfdiags.Diagnostics + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Resource instance managed by newer provider version", + // This is not a very good error message, but we don't retain enough + // information in state to give good feedback on what provider + // version might be required here. :( + fmt.Sprintf("The current state of %s was created by a newer provider version than is currently selected. Upgrade the %s provider to work with this state.", addr, providerType), + )) + return nil, diags + } + + // If we get down here then we need to upgrade the state, with the + // provider's help. + // If this state was originally created by a version of Terraform prior to + // v0.12, this also includes translating from legacy flatmap to new-style + // representation, since only the provider has enough information to + // understand a flatmap built against an older schema. + log.Printf("[TRACE] UpgradeResourceState: upgrading state for %s from version %d to %d using provider %q", addr, src.SchemaVersion, currentVersion, providerType) + + req := providers.UpgradeResourceStateRequest{ + TypeName: addr.Resource.Resource.Type, + + // TODO: The internal schema version representations are all using + // uint64 instead of int64, but unsigned integers aren't friendly + // to all protobuf target languages so in practice we use int64 + // on the wire. In future we will change all of our internal + // representations to int64 too. + Version: int64(src.SchemaVersion), + } + + if len(src.AttrsJSON) != 0 { + req.RawStateJSON = src.AttrsJSON + } else { + req.RawStateFlatmap = src.AttrsFlat + } + + resp := provider.UpgradeResourceState(req) + diags := resp.Diagnostics + if diags.HasErrors() { + return nil, diags + } + + // After upgrading, the new value must conform to the current schema. When + // going over RPC this is actually already ensured by the + // marshaling/unmarshaling of the new value, but we'll check it here + // anyway for robustness, e.g. for in-process providers. + newValue := resp.UpgradedState + if errs := newValue.Type().TestConformance(currentSchema.ImpliedType()); len(errs) > 0 { + for _, err := range errs { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid resource state upgrade", + fmt.Sprintf("The %s provider upgraded the state for %s from a previous version, but produced an invalid result: %s.", providerType, addr, tfdiags.FormatError(err)), + )) + } + return nil, diags + } + + new, err := src.CompleteUpgrade(newValue, currentSchema.ImpliedType(), uint64(currentVersion)) + if err != nil { + // We already checked for type conformance above, so getting into this + // codepath should be rare and is probably a bug somewhere under CompleteUpgrade. + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Failed to encode result of resource state upgrade", + fmt.Sprintf("Failed to encode state for %s after resource schema upgrade: %s.", addr, tfdiags.FormatError(err)), + )) + } + return new, diags +} diff --git a/terraform/provider_mock.go b/terraform/provider_mock.go index a2c5de3377..8c58240895 100644 --- a/terraform/provider_mock.go +++ b/terraform/provider_mock.go @@ -125,7 +125,8 @@ func (p *MockProvider) getSchema() providers.GetSchemaResponse { } for n, s := range p.GetSchemaReturn.ResourceTypes { ret.ResourceTypes[n] = providers.Schema{ - Block: s, + Version: int64(p.GetSchemaReturn.ResourceTypeSchemaVersions[n]), + Block: s, } } } diff --git a/terraform/schemas.go b/terraform/schemas.go index 7e9237fc73..62991c82d0 100644 --- a/terraform/schemas.go +++ b/terraform/schemas.go @@ -141,7 +141,7 @@ func loadProviderSchemas(schemas map[string]*ProviderSchema, config *configs.Con for t, r := range resp.ResourceTypes { s.ResourceTypes[t] = r.Block - s.ResourceTypeSchemaVersions[t] = r.Version + s.ResourceTypeSchemaVersions[t] = uint64(r.Version) if r.Version < 0 { diags = diags.Append( fmt.Errorf("invalid negative schema version for resource type %s in provider %q", t, typeName),