Stop reading providers from the HashiCorp registry based on the statefile if not specified in code. (#773)

Signed-off-by: Jakub Martin <kubam@spacelift.io>

CHANGELOG.md

@@ -30,6 +30,7 @@ ENHANCEMENTS:
 * cloud: Remote plans on cloud backends can now be saved using the `-out` flag, referenced in the `show` command, and applied by specifying the plan file name. ([#33492](https://github.com/hashicorp/terraform/issues/33492))
 * config: The `import` block `id` field now accepts an expression referencing other values such as resource attributes, as long as the value is a string known at plan time. ([#33618](https://github.com/hashicorp/terraform/issues/33618))
 * telemetry: All checkpoint telemetry was removed ([#151](https://github.com/opentofu/opentofu/pull/151))
+* state: Provider addresses in the statefile referring to registry.terraform.io will be treated as referring to registry.opentofu.org unless the full provider address is specified in the config or `OPENTOFU_STATEFILE_PROVIDER_ADDRESS_TRANSLATION` is set to `0`. ([#773](https://github.com/opentofu/opentofu/pull/773))
 
 BUG FIXES:
 

internal/backend/local/backend_local.go

@@ -10,6 +10,8 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/zclconf/go-cty/cty"
+
 	"github.com/opentofu/opentofu/internal/backend"
 	"github.com/opentofu/opentofu/internal/configs"
 	"github.com/opentofu/opentofu/internal/configs/configload"
@@ -17,7 +19,7 @@ import (
 	"github.com/opentofu/opentofu/internal/states/statemgr"
 	"github.com/opentofu/opentofu/internal/tfdiags"
 	"github.com/opentofu/opentofu/internal/tofu"
-	"github.com/zclconf/go-cty/cty"
+	"github.com/opentofu/opentofu/internal/tofumigrate"
 )
 
 // backend.Local implementation.
@@ -208,7 +210,16 @@ func (b *Local) localRunDirect(op *backend.Operation, run *backend.LocalRun, cor
 
 	// For a "direct" local run, the input state is the most recently stored
 	// snapshot, from the previous run.
-	run.InputState = s.State()
+	state := s.State()
+	if state != nil {
+		migratedState, migrateDiags := tofumigrate.MigrateStateProviderAddresses(config, state)
+		diags = diags.Append(migrateDiags)
+		if migrateDiags.HasErrors() {
+			return nil, nil, diags
+		}
+		state = migratedState
+	}
+	run.InputState = state
 
 	tfCtx, moreDiags := tofu.NewContext(coreOpts)
 	diags = diags.Append(moreDiags)

internal/command/init.go

@@ -31,6 +31,7 @@ import (
 	"github.com/opentofu/opentofu/internal/states"
 	"github.com/opentofu/opentofu/internal/tfdiags"
 	"github.com/opentofu/opentofu/internal/tofu"
+	"github.com/opentofu/opentofu/internal/tofumigrate"
 	tfversion "github.com/opentofu/opentofu/version"
 )
 
@@ -302,6 +303,18 @@ func (c *InitCommand) Run(args []string) int {
 		}
 	}
 
+	if state != nil {
+		// Since we now have the full configuration loaded, we can use it to migrate the in-memory state view
+		// prior to fetching providers.
+		migratedState, migrateDiags := tofumigrate.MigrateStateProviderAddresses(config, state)
+		diags = diags.Append(migrateDiags)
+		if migrateDiags.HasErrors() {
+			c.showDiagnostics(diags)
+			return 1
+		}
+		state = migratedState
+	}
+
 	// Now that we have loaded all modules, check the module tree for missing providers.
 	providersOutput, providersAbort, providerDiags := c.getProviders(ctx, config, state, flagUpgrade, flagPluginPath, flagLockfile)
 	diags = diags.Append(providerDiags)

internal/tofumigrate/testdata/mention/main.tf

@@ -0,0 +1,19 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "registry.terraform.io/hashicorp/aws"
+    }
+  }
+}
+
+provider "random" {}
+provider "aws" {}
+
+resource "random_id" "example" {
+  byte_length = 8
+}
+
+resource "aws_instance" "example" {
+  ami           = "abc"
+  instance_type = "t2.micro"
+}

internal/tofumigrate/testdata/nomention/main.tf

@@ -0,0 +1,11 @@
+provider "random" {}
+provider "aws" {}
+
+resource "random_id" "example" {
+  byte_length = 8
+}
+
+resource "aws_instance" "example" {
+  ami           = "abc"
+  instance_type = "t2.micro"
+}

internal/tofumigrate/tofumigrate.go

@@ -0,0 +1,60 @@
+package tofumigrate
+
+import (
+	"os"
+
+	tfaddr "github.com/opentofu/registry-address"
+
+	"github.com/opentofu/opentofu/internal/configs"
+	"github.com/opentofu/opentofu/internal/states"
+	"github.com/opentofu/opentofu/internal/tfdiags"
+)
+
+// MigrateStateProviderAddresses can be used to update the in-memory view of the state to use registry.opentofu.org
+// provider addresses. This only applies for providers which are *not* explicitly referenced in the configuration in full form.
+// For example, if the configuration contains a provider block like this:
+//
+//	terraform {
+//	 required_providers {
+//	   random = {}
+//	 }
+//	}
+//
+// we will migrate the in-memory view of the statefile to use registry.opentofu.org/hashicorp/random.
+// However, if the configuration contains a provider block like this:
+//
+//	terraform {
+//	 required_providers {
+//	   random = {
+//	     source = "registry.terraform.io/hashicorp/random"
+//	   }
+//	 }
+//	}
+//
+// then we keep the old address.
+func MigrateStateProviderAddresses(config *configs.Config, state *states.State) (*states.State, tfdiags.Diagnostics) {
+	if os.Getenv("OPENTOFU_STATEFILE_PROVIDER_ADDRESS_TRANSLATION") == "0" {
+		return state, nil
+	}
+
+	var diags tfdiags.Diagnostics
+
+	stateCopy := state.DeepCopy()
+
+	providers, hclDiags := config.ProviderRequirements()
+	diags = diags.Append(hclDiags)
+	if hclDiags.HasErrors() {
+		return nil, diags
+	}
+
+	for _, module := range stateCopy.Modules {
+		for _, resource := range module.Resources {
+			_, referencedInConfig := providers[resource.ProviderConfig.Provider]
+			if resource.ProviderConfig.Provider.Hostname == "registry.terraform.io" && !referencedInConfig {
+				resource.ProviderConfig.Provider.Hostname = tfaddr.DefaultProviderRegistryHost
+			}
+		}
+	}
+
+	return stateCopy, diags
+}

internal/tofumigrate/tofumigrate_test.go

@@ -0,0 +1,183 @@
+package tofumigrate
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/opentofu/opentofu/internal/addrs"
+	"github.com/opentofu/opentofu/internal/configs/configload"
+	"github.com/opentofu/opentofu/internal/states"
+)
+
+func TestMigrateStateProviderAddresses(t *testing.T) {
+	loader, close := configload.NewLoaderForTests(t)
+	defer close()
+
+	mustParseInstAddr := func(s string) addrs.AbsResourceInstance {
+		addr, err := addrs.ParseAbsResourceInstanceStr(s)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return addr
+	}
+
+	makeRootProviderAddr := func(s string) addrs.AbsProviderConfig {
+		return addrs.AbsProviderConfig{
+			Module:   addrs.RootModule,
+			Provider: addrs.MustParseProviderSourceString(s),
+		}
+	}
+
+	type args struct {
+		configDir string
+		state     *states.State
+	}
+	tests := []struct {
+		name string
+		args args
+		want *states.State
+	}{
+		{
+			name: "if there are no code references, migrate",
+			args: args{
+				configDir: "testdata/nomention",
+				state: states.BuildState(func(s *states.SyncState) {
+					s.SetResourceInstanceCurrent(
+						mustParseInstAddr("random_id.example"),
+						&states.ResourceInstanceObjectSrc{
+							Status:    states.ObjectReady,
+							AttrsJSON: []byte(`{}`),
+						},
+						makeRootProviderAddr("registry.terraform.io/hashicorp/random"),
+					)
+					s.SetResourceInstanceCurrent(
+						mustParseInstAddr("aws_instance.example"),
+						&states.ResourceInstanceObjectSrc{
+							Status:    states.ObjectReady,
+							AttrsJSON: []byte(`{}`),
+						},
+						makeRootProviderAddr("registry.terraform.io/hashicorp/aws"),
+					)
+				}),
+			},
+			want: states.BuildState(func(s *states.SyncState) {
+				s.SetResourceInstanceCurrent(
+					mustParseInstAddr("random_id.example"),
+					&states.ResourceInstanceObjectSrc{
+						Status:    states.ObjectReady,
+						AttrsJSON: []byte(`{}`),
+					},
+					makeRootProviderAddr("registry.opentofu.org/hashicorp/random"),
+				)
+				s.SetResourceInstanceCurrent(
+					mustParseInstAddr("aws_instance.example"),
+					&states.ResourceInstanceObjectSrc{
+						Status:    states.ObjectReady,
+						AttrsJSON: []byte(`{}`),
+					},
+					makeRootProviderAddr("registry.opentofu.org/hashicorp/aws"),
+				)
+			}),
+		},
+		{
+			name: "if there are some full-form references in the code, only migrate the ones not referenced",
+			args: args{
+				configDir: "testdata/mention",
+				state: states.BuildState(func(s *states.SyncState) {
+					s.SetResourceInstanceCurrent(
+						mustParseInstAddr("random_id.example"),
+						&states.ResourceInstanceObjectSrc{
+							Status:    states.ObjectReady,
+							AttrsJSON: []byte(`{}`),
+						},
+						makeRootProviderAddr("registry.terraform.io/hashicorp/random"),
+					)
+					s.SetResourceInstanceCurrent(
+						mustParseInstAddr("aws_instance.example"),
+						&states.ResourceInstanceObjectSrc{
+							Status:    states.ObjectReady,
+							AttrsJSON: []byte(`{}`),
+						},
+						makeRootProviderAddr("registry.terraform.io/hashicorp/aws"),
+					)
+				}),
+			},
+			want: states.BuildState(func(s *states.SyncState) {
+				s.SetResourceInstanceCurrent(
+					mustParseInstAddr("random_id.example"),
+					&states.ResourceInstanceObjectSrc{
+						Status:    states.ObjectReady,
+						AttrsJSON: []byte(`{}`),
+					},
+					makeRootProviderAddr("registry.opentofu.org/hashicorp/random"),
+				)
+				s.SetResourceInstanceCurrent(
+					mustParseInstAddr("aws_instance.example"),
+					&states.ResourceInstanceObjectSrc{
+						Status:    states.ObjectReady,
+						AttrsJSON: []byte(`{}`),
+					},
+					makeRootProviderAddr("registry.terraform.io/hashicorp/aws"),
+				)
+			}),
+		},
+		{
+			name: "if the state file contains no legacy references, return statefile unchanged",
+			args: args{
+				configDir: "testdata/nomention",
+				state: states.BuildState(func(s *states.SyncState) {
+					s.SetResourceInstanceCurrent(
+						mustParseInstAddr("random_id.example"),
+						&states.ResourceInstanceObjectSrc{
+							Status:    states.ObjectReady,
+							AttrsJSON: []byte(`{}`),
+						},
+						makeRootProviderAddr("registry.opentofu.org/hashicorp/random"),
+					)
+					s.SetResourceInstanceCurrent(
+						mustParseInstAddr("aws_instance.example"),
+						&states.ResourceInstanceObjectSrc{
+							Status:    states.ObjectReady,
+							AttrsJSON: []byte(`{}`),
+						},
+						makeRootProviderAddr("registry.opentofu.org/hashicorp/aws"),
+					)
+				}),
+			},
+			want: states.BuildState(func(s *states.SyncState) {
+				s.SetResourceInstanceCurrent(
+					mustParseInstAddr("random_id.example"),
+					&states.ResourceInstanceObjectSrc{
+						Status:    states.ObjectReady,
+						AttrsJSON: []byte(`{}`),
+					},
+					makeRootProviderAddr("registry.opentofu.org/hashicorp/random"),
+				)
+				s.SetResourceInstanceCurrent(
+					mustParseInstAddr("aws_instance.example"),
+					&states.ResourceInstanceObjectSrc{
+						Status:    states.ObjectReady,
+						AttrsJSON: []byte(`{}`),
+					},
+					makeRootProviderAddr("registry.opentofu.org/hashicorp/aws"),
+				)
+			}),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg, hclDiags := loader.LoadConfig(tt.args.configDir)
+			if hclDiags.HasErrors() {
+				t.Fatalf("invalid configuration: %s", hclDiags.Error())
+			}
+
+			got, err := MigrateStateProviderAddresses(cfg, tt.args.state)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("MigrateStateProviderAddresses() got = %v, want %v", got, tt.want)
+			}
+			if err != nil {
+				t.Errorf("MigrateStateProviderAddresses() err = %v, want %v", err, nil)
+			}
+		})
+	}
+}