command/init: Hint about providers in other namespaces

If a user forgets to specify the source address for a provider, Terraform
will assume they meant a provider in the registry.terraform.io/hashicorp/
namespace. If that ultimately doesn't exist, we'll now try to see if
there's some other provider source address recorded in the registry's
legacy provider lookup table, and suggest it if so.

The error message here is a terse one addressed primarily to folks who are
already somewhat familiar with provider source addresses and how to
specify them. Terraform v0.13 had a more elaborate version of this error
message which directed the user to try the v0.13 automatic upgrade tool,
but we no longer have that available in v0.14 and later so the user must
make the fix themselves.

command/init.go

@@ -474,6 +474,10 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 		log.Printf("[DEBUG] will search for provider plugins in %s", pluginDirs)
 	}
 
+	// Installation can be aborted by interruption signals
+	ctx, done := c.InterruptibleContext()
+	defer done()
+
 	// Because we're currently just streaming a series of events sequentially
 	// into the terminal, we're showing only a subset of the events to keep
 	// things relatively concise. Later it'd be nice to have a progress UI
@@ -536,11 +540,22 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 					),
 				))
 			case getproviders.ErrRegistryProviderNotKnown:
+				// We might be able to suggest an alternative provider to use
+				// instead of this one.
+				var suggestion string
+				alternative := getproviders.MissingProviderSuggestion(ctx, provider, inst.ProviderSource())
+				if alternative != provider {
+					suggestion = fmt.Sprintf(
+						"\n\nDid you intend to use %s? If so, you must specify that source address in each module which requires that provider. To see which modules are currently depending on %s, run the following command:\n    terraform providers",
+						alternative.ForDisplay(), provider.ForDisplay(),
+					)
+				}
+
 				diags = diags.Append(tfdiags.Sourceless(
 					tfdiags.Error,
 					"Failed to query available provider packages",
-					fmt.Sprintf("Could not retrieve the list of available versions for provider %s: %s",
-						provider.ForDisplay(), err,
+					fmt.Sprintf("Could not retrieve the list of available versions for provider %s: %s%s",
+						provider.ForDisplay(), err, suggestion,
 					),
 				))
 			case getproviders.ErrHostNoProviders:
@@ -736,6 +751,7 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 			))
 		},
 	}
+	ctx = evts.OnContext(ctx)
 
 	// Dev overrides cause the result of "terraform init" to be irrelevant for
 	// any overridden providers, so we'll warn about it to avoid later
@@ -747,10 +763,6 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 	if upgrade {
 		mode = providercache.InstallUpgrades
 	}
-	// Installation can be aborted by interruption signals
-	ctx, done := c.InterruptibleContext()
-	defer done()
-	ctx = evts.OnContext(ctx)
 	newLocks, err := inst.EnsureProviderVersions(ctx, previousLocks, reqs, mode)
 	if ctx.Err() == context.Canceled {
 		c.showDiagnostics(diags)

internal/getproviders/didyoumean.go

@@ -0,0 +1,253 @@
+package getproviders
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+
+	"github.com/hashicorp/go-retryablehttp"
+	svchost "github.com/hashicorp/terraform-svchost"
+	"github.com/hashicorp/terraform/addrs"
+)
+
+// MissingProviderSuggestion takes a provider address that failed installation
+// due to the remote registry reporting that it didn't exist, and attempts
+// to find another provider that the user might have meant to select.
+//
+// If the result is equal to the given address then that indicates that there
+// is no suggested alternative to offer, either because the function
+// successfully determined there is no recorded alternative or because the
+// lookup failed somehow. We don't consider a failure to find a suggestion
+// as an installation failure, because the caller should already be reporting
+// that the provider didn't exist anyway and this is only extra context for
+// that error message.
+//
+// The result of this is a best effort, so any UI presenting it should be
+// careful to give it only as a possibility and not necessarily a suitable
+// replacement for the given provider.
+//
+// In practice today this function only knows how to suggest alternatives for
+// "default" providers, which is to say ones that are in the hashicorp
+// namespace in the Terraform registry. It will always return no result for
+// any other provider. That might change in future if we introduce other ways
+// to discover provider suggestions.
+//
+// If the given context is cancelled then this function might not return a
+// renaming suggestion even if one would've been available for a completed
+// request.
+func MissingProviderSuggestion(ctx context.Context, addr addrs.Provider, source Source) addrs.Provider {
+	if !addr.IsDefault() {
+		return addr
+	}
+
+	// Our strategy here, for a default provider, is to use the default
+	// registry's special API for looking up "legacy" providers and try looking
+	// for a legacy provider whose type name matches the type of the given
+	// provider. This should then find a suitable answer for any provider
+	// that was originally auto-installable in v0.12 and earlier but moved
+	// into a non-default namespace as part of introducing the heirarchical
+	// provider namespace.
+	//
+	// To achieve that, we need to find the direct registry client in
+	// particular from the given source, because that is the only Source
+	// implementation that can actually handle a legacy provider lookup.
+	regSource := findLegacyProviderLookupSource(addr.Hostname, source)
+	if regSource == nil {
+		// If there's no direct registry source in the installation config
+		// then we can't provide a renaming suggestion.
+		return addr
+	}
+
+	defaultNS, redirectNS, err := regSource.lookupLegacyProviderNamespace(ctx, addr.Hostname, addr.Type)
+	if err != nil {
+		return addr
+	}
+
+	switch {
+	case redirectNS != "":
+		return addrs.Provider{
+			Hostname:  addr.Hostname,
+			Namespace: redirectNS,
+			Type:      addr.Type,
+		}
+	default:
+		return addrs.Provider{
+			Hostname:  addr.Hostname,
+			Namespace: defaultNS,
+			Type:      addr.Type,
+		}
+	}
+}
+
+// findLegacyProviderLookupSource tries to find a *RegistrySource that can talk
+// to the given registry host in the given Source. It might be given directly,
+// or it might be given indirectly via a MultiSource where the selector
+// includes a wildcard for registry.terraform.io.
+//
+// Returns nil if the given source does not have any configured way to talk
+// directly to the given host.
+//
+// If the given source contains multiple sources that can talk to the given
+// host directly, the first one in the sequence takes preference. In practice
+// it's pointless to have two direct installation sources that match the same
+// hostname anyway, so this shouldn't arise in normal use.
+func findLegacyProviderLookupSource(host svchost.Hostname, source Source) *RegistrySource {
+	switch source := source.(type) {
+
+	case *RegistrySource:
+		// Easy case: the source is a registry source directly, and so we'll
+		// just use it.
+		return source
+
+	case *MemoizeSource:
+		// Also easy: the source is a memoize wrapper, so defer to its
+		// underlying source.
+		return findLegacyProviderLookupSource(host, source.underlying)
+
+	case MultiSource:
+		// Trickier case: if it's a multisource then we need to scan over
+		// its selectors until we find one that is a *RegistrySource _and_
+		// that is configured to accept arbitrary providers from the
+		// given hostname.
+
+		// For our matching purposes we'll use an address that would not be
+		// valid as a real provider FQN and thus can only match a selector
+		// that has no filters at all or a selector that wildcards everything
+		// except the hostname, like "registry.terraform.io/*/*"
+		matchAddr := addrs.Provider{
+			Hostname: host,
+			// Other fields are intentionally left empty, to make this invalid
+			// as a specific provider address.
+		}
+
+		for _, selector := range source {
+			// If this source has suitable matching patterns to install from
+			// the given hostname then we'll recursively search inside it
+			// for *RegistrySource objects.
+			if selector.CanHandleProvider(matchAddr) {
+				ret := findLegacyProviderLookupSource(host, selector.Source)
+				if ret != nil {
+					return ret
+				}
+			}
+		}
+
+		// If we get here then there were no selectors that are both configured
+		// to handle modules from the given hostname and that are registry
+		// sources, so we fail.
+		return nil
+
+	default:
+		// This source cannot be and cannot contain a *RegistrySource, so
+		// we fail.
+		return nil
+	}
+}
+
+// lookupLegacyProviderNamespace is a special method available only on
+// RegistrySource which can deal with legacy provider addresses that contain
+// only a type and leave the namespace implied.
+//
+// It asks the registry at the given hostname to provide a default namespace
+// for the given provider type, which can be combined with the given hostname
+// and type name to produce a fully-qualified provider address.
+//
+// Not all unqualified type names can be resolved to a default namespace. If
+// the request fails, this method returns an error describing the failure.
+//
+// This method exists only to allow compatibility with unqualified names
+// in older configurations. New configurations should be written so as not to
+// depend on it, and this fallback mechanism will likely be removed altogether
+// in a future Terraform version.
+func (s *RegistrySource) lookupLegacyProviderNamespace(ctx context.Context, hostname svchost.Hostname, typeName string) (string, string, error) {
+	client, err := s.registryClient(hostname)
+	if err != nil {
+		return "", "", err
+	}
+	return client.legacyProviderDefaultNamespace(ctx, typeName)
+}
+
+// legacyProviderDefaultNamespace returns the raw address strings produced by
+// the registry when asked about the given unqualified provider type name.
+// The returned namespace string is taken verbatim from the registry's response.
+//
+// This method exists only to allow compatibility with unqualified names
+// in older configurations. New configurations should be written so as not to
+// depend on it.
+func (c *registryClient) legacyProviderDefaultNamespace(ctx context.Context, typeName string) (string, string, error) {
+	endpointPath, err := url.Parse(path.Join("-", typeName, "versions"))
+	if err != nil {
+		// Should never happen because we're constructing this from
+		// already-validated components.
+		return "", "", err
+	}
+	endpointURL := c.baseURL.ResolveReference(endpointPath)
+
+	req, err := retryablehttp.NewRequest("GET", endpointURL.String(), nil)
+	if err != nil {
+		return "", "", err
+	}
+	req = req.WithContext(ctx)
+	c.addHeadersToRequest(req.Request)
+
+	// This is just to give us something to return in error messages. It's
+	// not a proper provider address.
+	placeholderProviderAddr := addrs.NewLegacyProvider(typeName)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return "", "", c.errQueryFailed(placeholderProviderAddr, err)
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		// Great!
+	case http.StatusNotFound:
+		return "", "", ErrProviderNotFound{
+			Provider: placeholderProviderAddr,
+		}
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return "", "", c.errUnauthorized(placeholderProviderAddr.Hostname)
+	default:
+		return "", "", c.errQueryFailed(placeholderProviderAddr, errors.New(resp.Status))
+	}
+
+	type ResponseBody struct {
+		Id      string `json:"id"`
+		MovedTo string `json:"moved_to"`
+	}
+	var body ResponseBody
+
+	dec := json.NewDecoder(resp.Body)
+	if err := dec.Decode(&body); err != nil {
+		return "", "", c.errQueryFailed(placeholderProviderAddr, err)
+	}
+
+	provider, diags := addrs.ParseProviderSourceString(body.Id)
+	if diags.HasErrors() {
+		return "", "", fmt.Errorf("Error parsing provider ID from Registry: %s", diags.Err())
+	}
+
+	if provider.Type != typeName {
+		return "", "", fmt.Errorf("Registry returned provider with type %q, expected %q", provider.Type, typeName)
+	}
+
+	var movedTo addrs.Provider
+	if body.MovedTo != "" {
+		movedTo, diags = addrs.ParseProviderSourceString(body.MovedTo)
+		if diags.HasErrors() {
+			return "", "", fmt.Errorf("Error parsing provider ID from Registry: %s", diags.Err())
+		}
+
+		if movedTo.Type != typeName {
+			return "", "", fmt.Errorf("Registry returned provider with type %q, expected %q", movedTo.Type, typeName)
+		}
+	}
+
+	return provider.Namespace, movedTo.Namespace, nil
+}

internal/getproviders/didyoumean_test.go

@@ -0,0 +1,128 @@
+package getproviders
+
+import (
+	"context"
+	"testing"
+
+	svchost "github.com/hashicorp/terraform-svchost"
+	"github.com/hashicorp/terraform/addrs"
+)
+
+func TestMissingProviderSuggestion(t *testing.T) {
+	// Most of these test cases rely on specific "magic" provider addresses
+	// that are implemented by the fake registry source returned by
+	// testRegistrySource. Refer to that function for more details on how
+	// they work.
+
+	t.Run("happy path", func(t *testing.T) {
+		ctx := context.Background()
+		source, _, close := testRegistrySource(t)
+		defer close()
+
+		// testRegistrySource handles -/legacy as a valid legacy provider
+		// lookup mapping to legacycorp/legacy.
+		got := MissingProviderSuggestion(
+			ctx,
+			addrs.NewDefaultProvider("legacy"),
+			source,
+		)
+
+		want := addrs.Provider{
+			Hostname:  defaultRegistryHost,
+			Namespace: "legacycorp",
+			Type:      "legacy",
+		}
+		if got != want {
+			t.Errorf("wrong result\ngot:  %s\nwant: %s", got, want)
+		}
+	})
+	t.Run("provider moved", func(t *testing.T) {
+		ctx := context.Background()
+		source, _, close := testRegistrySource(t)
+		defer close()
+
+		// testRegistrySource handles -/moved as a valid legacy provider
+		// lookup mapping to hashicorp/moved but with an additional "redirect"
+		// to acme/moved. This mimics how for some providers there is both
+		// a copy under terraform-providers for v0.12 compatibility _and_ a
+		// copy in some other namespace for v0.13 or later to use. Our naming
+		// suggestions ignore the v0.12-compatible one and suggest the
+		// other one.
+		got := MissingProviderSuggestion(
+			ctx,
+			addrs.NewDefaultProvider("moved"),
+			source,
+		)
+
+		want := addrs.Provider{
+			Hostname:  defaultRegistryHost,
+			Namespace: "acme",
+			Type:      "moved",
+		}
+		if got != want {
+			t.Errorf("wrong result\ngot:  %s\nwant: %s", got, want)
+		}
+	})
+	t.Run("invalid response", func(t *testing.T) {
+		ctx := context.Background()
+		source, _, close := testRegistrySource(t)
+		defer close()
+
+		// testRegistrySource handles -/invalid by returning an invalid
+		// provider address, which MissingProviderSuggestion should reject
+		// and behave as if there was no suggestion available.
+		want := addrs.NewDefaultProvider("invalid")
+		got := MissingProviderSuggestion(
+			ctx,
+			want,
+			source,
+		)
+		if got != want {
+			t.Errorf("wrong result\ngot:  %s\nwant: %s", got, want)
+		}
+	})
+	t.Run("another registry", func(t *testing.T) {
+		ctx := context.Background()
+		source, _, close := testRegistrySource(t)
+		defer close()
+
+		// Because this provider address isn't on registry.terraform.io,
+		// MissingProviderSuggestion won't even attempt to make a suggestion
+		// for it.
+		want := addrs.Provider{
+			Hostname:  svchost.Hostname("example.com"),
+			Namespace: "whatever",
+			Type:      "foo",
+		}
+		got := MissingProviderSuggestion(
+			ctx,
+			want,
+			source,
+		)
+		if got != want {
+			t.Errorf("wrong result\ngot:  %s\nwant: %s", got, want)
+		}
+	})
+	t.Run("another namespace", func(t *testing.T) {
+		ctx := context.Background()
+		source, _, close := testRegistrySource(t)
+		defer close()
+
+		// Because this provider address isn't in
+		// registry.terraform.io/hashicorp/..., MissingProviderSuggestion won't
+		// even attempt to make a suggestion for it.
+		want := addrs.Provider{
+			Hostname:  defaultRegistryHost,
+			Namespace: "whatever",
+			Type:      "foo",
+		}
+		got := MissingProviderSuggestion(
+			ctx,
+			want,
+			source,
+		)
+		if got != want {
+			t.Errorf("wrong result\ngot:  %s\nwant: %s", got, want)
+		}
+	})
+}

internal/providercache/installer.go

@@ -65,6 +65,12 @@ func NewInstaller(targetDir *Dir, source getproviders.Source) *Installer {
 	}
 }
 
+// ProviderSource returns the getproviders.Source that the installer would
+// use for installing any new providers.
+func (i *Installer) ProviderSource() getproviders.Source {
+	return i.source
+}
+
 // SetGlobalCacheDir activates a second tier of caching for the receiving
 // installer, with the given directory used as a read-through cache for
 // installation operations that need to retrieve new packages.