terraform init: add warning and guidance when lock file is incomplete (#31399)

* terraform init: add warning and guidance when lock file is incomplete

* make the provider list in the warning deterministic

* create installer event for tracking provider lock hashes (#31406)

* create installer event for tracking provider lock hashes

* address comments

* fix tests

* improve error message

* Update internal/command/init.go

Co-authored-by: Martin Atkins <mart@degeneration.co.uk>

Co-authored-by: Martin Atkins <mart@degeneration.co.uk>

internal/command/init.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"reflect"
+	"sort"
 	"strings"
 
 	"github.com/hashicorp/hcl/v2"
@@ -548,6 +550,11 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 	ctx, done := c.InterruptibleContext()
 	defer done()
 
+	// We want to print out a nice warning if we don't manage to pull
+	// checksums for all our providers. This is tracked via callbacks
+	// and incomplete providers are stored here for later analysis.
+	var incompleteProviders []string
+
 	// Because we're currently just streaming a series of events sequentially
 	// into the terminal, we're showing only a subset of the events to keep
 	// things relatively concise. Later it'd be nice to have a progress UI
@@ -789,6 +796,41 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 
 			c.Ui.Info(fmt.Sprintf("- Installed %s v%s (%s%s)", provider.ForDisplay(), version, authResult, keyID))
 		},
+		ProvidersLockUpdated: func(provider addrs.Provider, version getproviders.Version, localHashes []getproviders.Hash, signedHashes []getproviders.Hash, priorHashes []getproviders.Hash) {
+			// We're going to use this opportunity to track if we have any
+			// "incomplete" installs of providers. An incomplete install is
+			// when we are only going to write the local hashes into our lock
+			// file which means a `terraform init` command will fail in future
+			// when used on machines of a different architecture.
+			//
+			// We want to print a warning about this.
+
+			if len(signedHashes) > 0 {
+				// If we have any signedHashes hashes then we don't worry - as
+				// we know we retrieved all available hashes for this version
+				// anyway.
+				return
+			}
+
+			// If local hashes and prior hashes are exactly the same then
+			// it means we didn't record any signed hashes previously, and
+			// we know we're not adding any extra in now (because we already
+			// checked the signedHashes), so that's a problem.
+			//
+			// In the actual check here, if we have any priorHashes and those
+			// hashes are not the same as the local hashes then we're going to
+			// accept that this provider has been configured correctly.
+			if len(priorHashes) > 0 && !reflect.DeepEqual(localHashes, priorHashes) {
+				return
+			}
+
+			// Now, either signedHashes is empty, or priorHashes is exactly the
+			// same as our localHashes which means we never retrieved the
+			// signedHashes previously.
+			//
+			// Either way, this is bad. Let's complain/warn.
+			incompleteProviders = append(incompleteProviders, provider.ForDisplay())
+		},
 		ProvidersFetched: func(authResults map[addrs.Provider]*getproviders.PackageAuthenticationResult) {
 			thirdPartySigned := false
 			for _, authResult := range authResults {
@@ -803,18 +845,6 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 					"https://www.terraform.io/docs/cli/plugins/signing.html"))
 			}
 		},
-		HashPackageFailure: func(provider addrs.Provider, version getproviders.Version, err error) {
-			diags = diags.Append(tfdiags.Sourceless(
-				tfdiags.Error,
-				"Failed to validate installed provider",
-				fmt.Sprintf(
-					"Validating provider %s v%s failed: %s",
-					provider.ForDisplay(),
-					version,
-					err,
-				),
-			))
-		},
 	}
 	ctx = evts.OnContext(ctx)
 
@@ -874,6 +904,22 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 			return true, false, diags
 		}
 
+		// Jump in here and add a warning if any of the providers are incomplete.
+		if len(incompleteProviders) > 0 {
+			// We don't really care about the order here, we just want the
+			// output to be deterministic.
+			sort.Slice(incompleteProviders, func(i, j int) bool {
+				return incompleteProviders[i] < incompleteProviders[j]
+			})
+			diags = diags.Append(tfdiags.Sourceless(
+				tfdiags.Warning,
+				incompleteLockFileInformationHeader,
+				fmt.Sprintf(
+					incompleteLockFileInformationBody,
+					strings.Join(incompleteProviders, "\n  - "),
+					getproviders.CurrentPlatform.String())))
+		}
+
 		if previousLocks.Empty() {
 			// A change from empty to non-empty is special because it suggests
 			// we're running "terraform init" for the first time against a
@@ -1195,3 +1241,18 @@ Alternatively, upgrade to the latest version of Terraform for compatibility with
 
 // No version of the provider is compatible.
 const errProviderVersionIncompatible = `No compatible versions of provider %s were found.`
+
+// incompleteLockFileInformationHeader is the summary displayed to users when
+// the lock file has only recorded local hashes.
+const incompleteLockFileInformationHeader = `Incomplete lock file information for providers`
+
+// incompleteLockFileInformationBody is the body of text displayed to users when
+// the lock file has only recorded local hashes.
+const incompleteLockFileInformationBody = `Due to your customized provider installation methods, Terraform was forced to calculate lock file checksums locally for the following providers:
+  - %s
+
+The current .terraform.lock.hcl file only includes checksums for %s, so Terraform running on another platform will fail to install these providers.
+
+To calculate additional checksums for another platform, run:
+  terraform providers lock -platform=linux_amd64
+(where linux_amd64 is the platform to generate)`

internal/command/init_test.go

@@ -1644,7 +1644,7 @@ func TestInit_providerSource(t *testing.T) {
 	})
 	defer close()
 
-	ui := new(cli.MockUi)
+	ui := cli.NewMockUi()
 	view, _ := testView(t)
 	m := Meta{
 		testingOverrides: metaOverridesForProvider(testProvider()),
@@ -1726,13 +1726,16 @@ func TestInit_providerSource(t *testing.T) {
 			},
 		),
 	}
+
 	if diff := cmp.Diff(gotProviderLocks, wantProviderLocks, depsfile.ProviderLockComparer); diff != "" {
 		t.Errorf("wrong version selections after upgrade\n%s", diff)
 	}
 
-	outputStr := ui.OutputWriter.String()
+	if got, want := ui.OutputWriter.String(), "Installed hashicorp/test v1.2.3 (verified checksum)"; !strings.Contains(got, want) {
-	if want := "Installed hashicorp/test v1.2.3 (verified checksum)"; !strings.Contains(outputStr, want) {
+		t.Fatalf("unexpected output: %s\nexpected to include %q", got, want)
-		t.Fatalf("unexpected output: %s\nexpected to include %q", outputStr, want)
+	}
+	if got, want := ui.ErrorWriter.String(), "\n  - hashicorp/source\n  - hashicorp/test\n  - hashicorp/test-beta"; !strings.Contains(got, want) {
+		t.Fatalf("wrong error message\nshould contain: %s\ngot:\n%s", want, got)
 	}
 }
 

internal/providercache/installer.go

@@ -385,14 +385,14 @@ NeedProvider:
 				// calculated from the package we just linked, which allows
 				// the lock file to gradually transition to recording newer hash
 				// schemes when they become available.
-				var newHashes []getproviders.Hash
+				var priorHashes []getproviders.Hash
 				if lock != nil && lock.Version() == version {
 					// If the version we're installing is identical to the
 					// one we previously locked then we'll keep all of the
 					// hashes we saved previously and add to it. Otherwise
 					// we'll be starting fresh, because each version has its
 					// own set of packages and thus its own hashes.
-					newHashes = append(newHashes, preferredHashes...)
+					priorHashes = append(priorHashes, preferredHashes...)
 
 					// NOTE: The behavior here is unfortunate when a particular
 					// provider version was already cached on the first time
@@ -423,8 +423,17 @@ NeedProvider:
 				// The hashes slice gets deduplicated in the lock file
 				// implementation, so we don't worry about potentially
 				// creating a duplicate here.
+				var newHashes []getproviders.Hash
+				newHashes = append(newHashes, priorHashes...)
 				newHashes = append(newHashes, newHash)
 				locks.SetProvider(provider, version, reqs[provider], newHashes)
+				if cb := evts.ProvidersLockUpdated; cb != nil {
+					// We want to ensure that newHash and priorHashes are
+					// sorted. newHash is a single value, so it's definitely
+					// sorted. priorHashes are pulled from the lock file, so
+					// are also already sorted.
+					cb(provider, version, []getproviders.Hash{newHash}, nil, priorHashes)
+				}
 
 				if cb := evts.LinkFromCacheSuccess; cb != nil {
 					cb(provider, version, new.PackageDir)
@@ -530,14 +539,14 @@ NeedProvider:
 		// The hashes slice gets deduplicated in the lock file
 		// implementation, so we don't worry about potentially
 		// creating duplicates here.
-		var newHashes []getproviders.Hash
+		var priorHashes []getproviders.Hash
 		if lock != nil && lock.Version() == version {
 			// If the version we're installing is identical to the
 			// one we previously locked then we'll keep all of the
 			// hashes we saved previously and add to it. Otherwise
 			// we'll be starting fresh, because each version has its
 			// own set of packages and thus its own hashes.
-			newHashes = append(newHashes, preferredHashes...)
+			priorHashes = append(priorHashes, preferredHashes...)
 		}
 		newHash, err := new.Hash()
 		if err != nil {
@@ -548,15 +557,32 @@ NeedProvider:
 			}
 			continue
 		}
-		newHashes = append(newHashes, newHash)
+
+		var signedHashes []getproviders.Hash
 		if authResult.SignedByAnyParty() {
 			// We'll trust new hashes from upstream only if they were verified
 			// as signed by a suitable key. Otherwise, we'd record only
 			// a new hash we just calculated ourselves from the bytes on disk,
 			// and so the hashes would cover only the current platform.
-			newHashes = append(newHashes, meta.AcceptableHashes()...)
+			signedHashes = append(signedHashes, meta.AcceptableHashes()...)
 		}
+
+		var newHashes []getproviders.Hash
+		newHashes = append(newHashes, newHash)
+		newHashes = append(newHashes, priorHashes...)
+		newHashes = append(newHashes, signedHashes...)
+
 		locks.SetProvider(provider, version, reqs[provider], newHashes)
+		if cb := evts.ProvidersLockUpdated; cb != nil {
+			// newHash and priorHashes are already sorted.
+			// But we do need to sort signedHashes so we can reason about it
+			// sensibly.
+			sort.Slice(signedHashes, func(i, j int) bool {
+				return string(signedHashes[i]) < string(signedHashes[j])
+			})
+
+			cb(provider, version, []getproviders.Hash{newHash}, signedHashes, priorHashes)
+		}
 
 		if cb := evts.FetchPackageSuccess; cb != nil {
 			cb(provider, version, new.PackageDir, authResult)

internal/providercache/installer_events.go

@@ -106,15 +106,28 @@ type InstallerEvents struct {
 	FetchPackageSuccess func(provider addrs.Provider, version getproviders.Version, localDir string, authResult *getproviders.PackageAuthenticationResult)
 	FetchPackageFailure func(provider addrs.Provider, version getproviders.Version, err error)
 
+	// The ProvidersLockUpdated event is called whenever the lock file will be
+	// updated. It provides the following information:
+	//
+	//   - `localHashes`: Hashes computed on the local system by analyzing
+	//                    files on disk.
+	//   - `signedHashes`: Hashes signed by the private key that the origin
+	//                     registry claims is the owner of this provider.
+	//   - `priorHashes`: Hashes already present in the lock file before we
+	//                    made any changes.
+	//
+	// The final lock file will be updated with a union of all the provided
+	// hashes. It not just likely, but expected that there will be duplicates
+	// shared between all three collections of hashes i.e. the local hash and
+	// remote hashes could already be in the cached hashes.
+	//
+	// In addition, we place a guarantee that the hash slices will be ordered
+	// in the same manner enforced by the lock file within NewProviderLock.
+	ProvidersLockUpdated func(provider addrs.Provider, version getproviders.Version, localHashes []getproviders.Hash, signedHashes []getproviders.Hash, priorHashes []getproviders.Hash)
+
 	// The ProvidersFetched event is called after all fetch operations if at
 	// least one provider was fetched successfully.
 	ProvidersFetched func(authResults map[addrs.Provider]*getproviders.PackageAuthenticationResult)
-
-	// HashPackageFailure is called if the installer is unable to determine
-	// the hash of the contents of an installed package after installation.
-	// In that case, the selection will not be recorded in the target cache
-	// directory's lock file.
-	HashPackageFailure func(provider addrs.Provider, version getproviders.Version, err error)
 }
 
 // OnContext produces a context with all of the same behaviors as the given

internal/providercache/installer_events_test.go

@@ -164,21 +164,23 @@ func installerLogEventsForTests(into chan<- *testInstallerEventLogItem) *Install
 				}{version.String(), err.Error()},
 			}
 		},
+		ProvidersLockUpdated: func(provider addrs.Provider, version getproviders.Version, localHashes []getproviders.Hash, signedHashes []getproviders.Hash, priorHashes []getproviders.Hash) {
+			into <- &testInstallerEventLogItem{
+				Event:    "ProvidersLockUpdated",
+				Provider: provider,
+				Args: struct {
+					Version string
+					Local   []getproviders.Hash
+					Signed  []getproviders.Hash
+					Prior   []getproviders.Hash
+				}{version.String(), localHashes, signedHashes, priorHashes},
+			}
+		},
 		ProvidersFetched: func(authResults map[addrs.Provider]*getproviders.PackageAuthenticationResult) {
 			into <- &testInstallerEventLogItem{
 				Event: "ProvidersFetched",
 				Args:  authResults,
 			}
 		},
-		HashPackageFailure: func(provider addrs.Provider, version getproviders.Version, err error) {
-			into <- &testInstallerEventLogItem{
-				Event:    "HashPackageFailure",
-				Provider: provider,
-				Args: struct {
-					Version string
-					Error   string
-				}{version.String(), err.Error()},
-			}
-		},
 	}
 }

internal/providercache/installer_test.go

@@ -171,6 +171,21 @@ func TestEnsureProviderVersions(t *testing.T) {
 								Location getproviders.PackageLocation
 							}{"2.1.0", beepProviderDir},
 						},
+						{
+							Event:    "ProvidersLockUpdated",
+							Provider: beepProvider,
+							Args: struct {
+								Version string
+								Local   []getproviders.Hash
+								Signed  []getproviders.Hash
+								Prior   []getproviders.Hash
+							}{
+								"2.1.0",
+								[]getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="},
+								nil,
+								nil,
+							},
+						},
 						{
 							Event:    "FetchPackageSuccess",
 							Provider: beepProvider,
@@ -287,6 +302,21 @@ func TestEnsureProviderVersions(t *testing.T) {
 								Location getproviders.PackageLocation
 							}{"2.1.0", beepProviderDir},
 						},
+						{
+							Event:    "ProvidersLockUpdated",
+							Provider: beepProvider,
+							Args: struct {
+								Version string
+								Local   []getproviders.Hash
+								Signed  []getproviders.Hash
+								Prior   []getproviders.Hash
+							}{
+								"2.1.0",
+								[]getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="},
+								nil,
+								nil,
+							},
+						},
 						{
 							Event:    "FetchPackageSuccess",
 							Provider: beepProvider,
@@ -411,6 +441,21 @@ func TestEnsureProviderVersions(t *testing.T) {
 								inst.globalCacheDir.BasePath(),
 							},
 						},
+						{
+							Event:    "ProvidersLockUpdated",
+							Provider: beepProvider,
+							Args: struct {
+								Version string
+								Local   []getproviders.Hash
+								Signed  []getproviders.Hash
+								Prior   []getproviders.Hash
+							}{
+								"2.1.0",
+								[]getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="},
+								nil,
+								nil,
+							},
+						},
 						{
 							Event:    "LinkFromCacheSuccess",
 							Provider: beepProvider,
@@ -535,6 +580,21 @@ func TestEnsureProviderVersions(t *testing.T) {
 								Location getproviders.PackageLocation
 							}{"2.0.0", beepProviderDir},
 						},
+						{
+							Event:    "ProvidersLockUpdated",
+							Provider: beepProvider,
+							Args: struct {
+								Version string
+								Local   []getproviders.Hash
+								Signed  []getproviders.Hash
+								Prior   []getproviders.Hash
+							}{
+								"2.0.0",
+								[]getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="},
+								nil,
+								[]getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="},
+							},
+						},
 						{
 							Event:    "FetchPackageSuccess",
 							Provider: beepProvider,
@@ -763,6 +823,21 @@ func TestEnsureProviderVersions(t *testing.T) {
 								Location getproviders.PackageLocation
 							}{"2.1.0", beepProviderDir},
 						},
+						{
+							Event:    "ProvidersLockUpdated",
+							Provider: beepProvider,
+							Args: struct {
+								Version string
+								Local   []getproviders.Hash
+								Signed  []getproviders.Hash
+								Prior   []getproviders.Hash
+							}{
+								"2.1.0",
+								[]getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="},
+								nil,
+								nil,
+							},
+						},
 						{
 							Event:    "FetchPackageSuccess",
 							Provider: beepProvider,
@@ -929,6 +1004,21 @@ func TestEnsureProviderVersions(t *testing.T) {
 								Location getproviders.PackageLocation
 							}{"1.0.0", beepProviderDir},
 						},
+						{
+							Event:    "ProvidersLockUpdated",
+							Provider: beepProvider,
+							Args: struct {
+								Version string
+								Local   []getproviders.Hash
+								Signed  []getproviders.Hash
+								Prior   []getproviders.Hash
+							}{
+								"1.0.0",
+								[]getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="},
+								nil,
+								[]getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="},
+							},
+						},
 						{
 							Event:    "FetchPackageSuccess",
 							Provider: beepProvider,