diff --git a/command/e2etest/init_test.go b/command/e2etest/init_test.go index e013f6291e..f59ce0b05d 100644 --- a/command/e2etest/init_test.go +++ b/command/e2etest/init_test.go @@ -263,7 +263,7 @@ func TestInitProviders_pluginCache(t *testing.T) { t.Errorf("unexpected stderr output:\n%s\n", stderr) } - path := filepath.FromSlash(fmt.Sprintf(".terraform/plugins/registry.terraform.io/hashicorp/template/2.1.0/%s_%s/terraform-provider-template_v2.1.0_x4", runtime.GOOS, runtime.GOARCH)) + path := filepath.FromSlash(fmt.Sprintf(".terraform/providers/registry.terraform.io/hashicorp/template/2.1.0/%s_%s/terraform-provider-template_v2.1.0_x4", runtime.GOOS, runtime.GOARCH)) content, err := tf.ReadFile(path) if err != nil { t.Fatalf("failed to read installed plugin from %s: %s", path, err) @@ -272,7 +272,7 @@ func TestInitProviders_pluginCache(t *testing.T) { t.Errorf("template plugin was not installed from local cache") } - nullLinkPath := filepath.FromSlash(fmt.Sprintf(".terraform/plugins/registry.terraform.io/hashicorp/null/2.1.0/%s_%s/terraform-provider-null_v2.1.0_x4", runtime.GOOS, runtime.GOARCH)) + nullLinkPath := filepath.FromSlash(fmt.Sprintf(".terraform/providers/registry.terraform.io/hashicorp/null/2.1.0/%s_%s/terraform-provider-null_v2.1.0_x4", runtime.GOOS, runtime.GOARCH)) if runtime.GOOS == "windows" { nullLinkPath = nullLinkPath + ".exe" } diff --git a/command/init.go b/command/init.go index fd9e90d1e9..2639a6b085 100644 --- a/command/init.go +++ b/command/init.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os" + "path/filepath" "strings" "github.com/hashicorp/hcl/v2" @@ -475,6 +476,7 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State, // things relatively concise. Later it'd be nice to have a progress UI // where statuses update in-place, but we can't do that as long as we // are shimming our vt100 output to the legacy console API on Windows. + missingProviders := make(map[addrs.Provider]struct{}) evts := &providercache.InstallerEvents{ PendingProviders: func(reqs map[addrs.Provider]getproviders.VersionConstraints) { c.Ui.Output(c.Colorize().Color( @@ -512,6 +514,10 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State, c.Ui.Info(fmt.Sprintf("- Installing %s v%s...", provider.ForDisplay(), version)) }, QueryPackagesFailure: func(provider addrs.Provider, err error) { + // We track providers that had missing metadata because we might + // generate additional hints for some of them at the end. + missingProviders[provider] = struct{}{} + switch errorTy := err.(type) { case getproviders.ErrProviderNotFound: sources := errorTy.Sources @@ -742,6 +748,60 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State, c.Ui.Error("Provider installation was canceled by an interrupt signal.") return true, true, diags } + if len(missingProviders) > 0 { + // If we encountered requirements for one or more providers where we + // weren't able to find any metadata, that _might_ be because a + // user had previously (before 0.14) been incorrectly using the + // .terraform/plugins directory as if it were a local filesystem + // mirror, rather than as the main cache directory. + // + // We no longer allow that because it'd be ambiguous whether plugins in + // there are explictly intended to be a local mirror or if they are + // just leftover cache entries from provider installation in + // Terraform 0.13. + // + // To help those users migrate we have a specialized warning message + // for it, which we'll produce only if one of the missing providers can + // be seen in the "legacy" cache directory, which is what we're now + // considering .terraform/plugins to be. (The _current_ cache directory + // is .terraform/providers.) + // + // This is only a heuristic, so it might potentially produce false + // positives if a user happens to encounter another sort of error + // while they are upgrading from Terraform 0.13 to 0.14. Aside from + // upgrading users should not end up in here because they won't + // have a legacy cache directory at all. + legacyDir := c.providerLegacyCacheDir() + if legacyDir != nil { // if the legacy directory is present at all + for missingProvider := range missingProviders { + if missingProvider.IsDefault() { + // If we get here for a default provider then it's more + // likely that something _else_ went wrong, like a network + // problem, so we'll skip the warning in this case to + // avoid potentially misleading the user into creating an + // unnecessary local mirror for an official provider. + continue + } + entry := legacyDir.ProviderLatestVersion(missingProvider) + if entry == nil { + continue + } + // If we get here then the missing provider was cached, which + // implies that it might be an in-house provider the user + // placed manually to try to make Terraform use it as if it + // were a local mirror directory. + wantDir := filepath.FromSlash(fmt.Sprintf("terraform.d/plugins/%s/%s/%s", missingProvider, entry.Version, getproviders.CurrentPlatform)) + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Warning, + "Missing provider is in legacy cache directory", + fmt.Sprintf( + "Terraform supports a number of local directories that can serve as automatic local filesystem mirrors, but .terraform/plugins is not one of them because Terraform v0.13 and earlier used this directory to cache copies of provider plugins retrieved from elsewhere.\n\nIf you intended to use this directory as a filesystem mirror for %s, place it instead in the following directory:\n %s", + missingProvider, wantDir, + ), + )) + } + } + } if err != nil { // The errors captured in "err" should be redundant with what we // received via the InstallerEvents callbacks above, so we'll diff --git a/command/init_test.go b/command/init_test.go index 23f076980f..96d853d36f 100644 --- a/command/init_test.go +++ b/command/init_test.go @@ -941,15 +941,15 @@ func TestInit_getProvider(t *testing.T) { } // check that we got the providers for our config - exactPath := fmt.Sprintf(".terraform/plugins/registry.terraform.io/hashicorp/exact/1.2.3/%s", getproviders.CurrentPlatform) + exactPath := fmt.Sprintf(".terraform/providers/registry.terraform.io/hashicorp/exact/1.2.3/%s", getproviders.CurrentPlatform) if _, err := os.Stat(exactPath); os.IsNotExist(err) { t.Fatal("provider 'exact' not downloaded") } - greaterThanPath := fmt.Sprintf(".terraform/plugins/registry.terraform.io/hashicorp/greater-than/2.3.4/%s", getproviders.CurrentPlatform) + greaterThanPath := fmt.Sprintf(".terraform/providers/registry.terraform.io/hashicorp/greater-than/2.3.4/%s", getproviders.CurrentPlatform) if _, err := os.Stat(greaterThanPath); os.IsNotExist(err) { t.Fatal("provider 'greater-than' not downloaded") } - betweenPath := fmt.Sprintf(".terraform/plugins/registry.terraform.io/hashicorp/between/2.3.4/%s", getproviders.CurrentPlatform) + betweenPath := fmt.Sprintf(".terraform/providers/registry.terraform.io/hashicorp/between/2.3.4/%s", getproviders.CurrentPlatform) if _, err := os.Stat(betweenPath); os.IsNotExist(err) { t.Fatal("provider 'between' not downloaded") } @@ -1024,17 +1024,96 @@ func TestInit_getProviderSource(t *testing.T) { } // check that we got the providers for our config - exactPath := fmt.Sprintf(".terraform/plugins/registry.terraform.io/acme/alpha/1.2.3/%s", getproviders.CurrentPlatform) + exactPath := fmt.Sprintf(".terraform/providers/registry.terraform.io/acme/alpha/1.2.3/%s", getproviders.CurrentPlatform) if _, err := os.Stat(exactPath); os.IsNotExist(err) { - t.Fatal("provider 'alpha' not downloaded") + t.Error("provider 'alpha' not downloaded") } - greaterThanPath := fmt.Sprintf(".terraform/plugins/registry.example.com/acme/beta/1.0.0/%s", getproviders.CurrentPlatform) + greaterThanPath := fmt.Sprintf(".terraform/providers/registry.example.com/acme/beta/1.0.0/%s", getproviders.CurrentPlatform) if _, err := os.Stat(greaterThanPath); os.IsNotExist(err) { - t.Fatal("provider 'beta' not downloaded") + t.Error("provider 'beta' not downloaded") } - betweenPath := fmt.Sprintf(".terraform/plugins/registry.terraform.io/hashicorp/gamma/2.0.0/%s", getproviders.CurrentPlatform) + betweenPath := fmt.Sprintf(".terraform/providers/registry.terraform.io/hashicorp/gamma/2.0.0/%s", getproviders.CurrentPlatform) if _, err := os.Stat(betweenPath); os.IsNotExist(err) { - t.Fatal("provider 'gamma' not downloaded") + t.Error("provider 'gamma' not downloaded") + } +} + +func TestInit_getProviderInLegacyPluginCacheDir(t *testing.T) { + // Create a temporary working directory that is empty + td := tempDir(t) + testCopyDir(t, testFixturePath("init-legacy-provider-cache"), td) + defer os.RemoveAll(td) + defer testChdir(t, td)() + + // The test fixture has placeholder os_arch directories which we must + // now rename to match the current platform, or else the entries inside + // will be ignored. + platformStr := getproviders.CurrentPlatform.String() + if err := os.Rename( + ".terraform/plugins/example.com/test/b/1.1.0/os_arch", + ".terraform/plugins/example.com/test/b/1.1.0/"+platformStr, + ); err != nil { + t.Fatal(err) + } + if err := os.Rename( + ".terraform/plugins/registry.terraform.io/hashicorp/c/2.0.0/os_arch", + ".terraform/plugins/registry.terraform.io/hashicorp/c/2.0.0/"+platformStr, + ); err != nil { + t.Fatal(err) + } + + // An empty MultiSource serves as a way to make sure no providers are + // actually available for installation, which suits us here because + // we're testing an error case. + providerSource := getproviders.MultiSource{} + + ui := cli.NewMockUi() + m := Meta{ + Ui: ui, + ProviderSource: providerSource, + } + + c := &InitCommand{ + Meta: m, + } + + args := []string{ + "-backend=false", + } + if code := c.Run(args); code == 0 { + t.Fatalf("succeeded; want error\n%s", ui.OutputWriter.String()) + } + + // We remove all of the newlines so that we don't need to contend with + // the automatic word wrapping that our diagnostic printer does. + stderr := strings.Replace(ui.ErrorWriter.String(), "\n", " ", -1) + + if got, want := stderr, `example.com/test/a: no available releases match the given constraints`; !strings.Contains(got, want) { + t.Errorf("missing error about example.com/test/a\nwant substring: %s\n%s", want, got) + } + if got, want := stderr, `example.com/test/b: no available releases match the given constraints`; !strings.Contains(got, want) { + t.Errorf("missing error about example.com/test/b\nwant substring: %s\n%s", want, got) + } + if got, want := stderr, `hashicorp/c: no available releases match the given constraints`; !strings.Contains(got, want) { + t.Errorf("missing error about registry.terraform.io/hashicorp/c\nwant substring: %s\n%s", want, got) + } + + if got, want := stderr, `terraform.d/plugins/example.com/test/a`; strings.Contains(got, want) { + // We _don't_ expect to see a warning about the "a" provider, because + // there's no copy of that in the legacy plugin cache dir. + t.Errorf("unexpected suggested path for local example.com/test/a\ndon't want substring: %s\n%s", want, got) + } + if got, want := stderr, `terraform.d/plugins/example.com/test/b/1.1.0/`+platformStr; !strings.Contains(got, want) { + // ...but we should see a warning about the "b" provider, because + // there's an entry for that in the legacy cache dir. + t.Errorf("missing suggested path for local example.com/test/b 1.0.0 on %s\nwant substring: %s\n%s", platformStr, want, got) + } + if got, want := stderr, `terraform.d/plugins/registry.terraform.io/hashicorp/c`; strings.Contains(got, want) { + // We _don't_ expect to see a warning about the "a" provider, even + // though it's in the cache dir, because it's an official provider + // and so we assume it ended up there as a result of normal provider + // installation in Terraform 0.13. + t.Errorf("unexpected suggested path for local hashicorp/c\ndon't want substring: %s\n%s", want, got) } } @@ -1122,7 +1201,7 @@ func TestInit_getProviderInvalidPackage(t *testing.T) { } // invalid provider should be installed - packagePath := fmt.Sprintf(".terraform/plugins/registry.terraform.io/invalid/package/1.0.0/%s/terraform-package", getproviders.CurrentPlatform) + packagePath := fmt.Sprintf(".terraform/providers/registry.terraform.io/invalid/package/1.0.0/%s/terraform-package", getproviders.CurrentPlatform) if _, err := os.Stat(packagePath); os.IsNotExist(err) { t.Fatal("provider 'invalid/package' not downloaded") } @@ -1180,12 +1259,12 @@ func TestInit_getProviderDetectedLegacy(t *testing.T) { } // foo should be installed - fooPath := fmt.Sprintf(".terraform/plugins/registry.terraform.io/hashicorp/foo/1.2.3/%s", getproviders.CurrentPlatform) + fooPath := fmt.Sprintf(".terraform/providers/registry.terraform.io/hashicorp/foo/1.2.3/%s", getproviders.CurrentPlatform) if _, err := os.Stat(fooPath); os.IsNotExist(err) { t.Error("provider 'foo' not installed") } // baz should not be installed - bazPath := fmt.Sprintf(".terraform/plugins/registry.terraform.io/terraform-providers/baz/2.3.4/%s", getproviders.CurrentPlatform) + bazPath := fmt.Sprintf(".terraform/providers/registry.terraform.io/terraform-providers/baz/2.3.4/%s", getproviders.CurrentPlatform) if _, err := os.Stat(bazPath); !os.IsNotExist(err) { t.Error("provider 'baz' installed, but should not be") } @@ -2038,7 +2117,7 @@ func installFakeProviderPackagesElsewhere(t *testing.T, cacheDir *providercache. // with how the getproviders and providercache packages build paths. func expectedPackageInstallPath(name, version string, exe bool) string { platform := getproviders.CurrentPlatform - baseDir := ".terraform/plugins" + baseDir := ".terraform/providers" if exe { p := fmt.Sprintf("registry.terraform.io/hashicorp/%s/%s/%s/terraform-provider-%s_%s", name, version, platform, name, version) if platform.OS == "windows" { diff --git a/command/meta_providers.go b/command/meta_providers.go index e15ca46779..aad6382f7d 100644 --- a/command/meta_providers.go +++ b/command/meta_providers.go @@ -103,10 +103,7 @@ func (m *Meta) providerCustomLocalDirectorySource(dirs []string) getproviders.So // Only one object returned from this method should be live at any time, // because objects inside contain caches that must be maintained properly. func (m *Meta) providerLocalCacheDir() *providercache.Dir { - dir := filepath.Join(m.DataDir(), "plugins") - if dir == "" { - return nil // cache disabled - } + dir := filepath.Join(m.DataDir(), "providers") return providercache.NewDir(dir) } @@ -128,6 +125,31 @@ func (m *Meta) providerGlobalCacheDir() *providercache.Dir { return providercache.NewDir(dir) } +// providerLegacyCacheDir returns an object representing the former location +// of the local cache directory from Terraform 0.13 and earlier. +// +// This is no longer viable for use as a real cache directory because some +// incorrect documentation called for Terraform Cloud users to use it as if it +// were an implied local filesystem mirror directory. Therefore we now use it +// only to generate some hopefully-helpful migration guidance during +// "terraform init" for anyone who _was_ trying to use it as a local filesystem +// mirror directory. +// +// providerLegacyCacheDir returns nil if the legacy cache directory isn't +// present or isn't a directory, so that callers can more easily skip over +// any backward compatibility behavior that applies only when the directory +// is present. +// +// Callers must use the resulting object in a read-only mode only. Don't +// install any new providers into this directory. +func (m *Meta) providerLegacyCacheDir() *providercache.Dir { + dir := filepath.Join(m.DataDir(), "plugins") + if info, err := os.Stat(dir); err != nil || !info.IsDir() { + return nil + } + return providercache.NewDir(dir) +} + // providerInstallSource returns an object that knows how to consult one or // more external sources to determine the availability of and package // locations for versions of Terraform providers that are available for diff --git a/command/testdata/init-legacy-provider-cache/.terraform/plugins/example.com/test/b/1.1.0/os_arch/terraform-provider-b b/command/testdata/init-legacy-provider-cache/.terraform/plugins/example.com/test/b/1.1.0/os_arch/terraform-provider-b new file mode 100644 index 0000000000..7e9920ef48 --- /dev/null +++ b/command/testdata/init-legacy-provider-cache/.terraform/plugins/example.com/test/b/1.1.0/os_arch/terraform-provider-b @@ -0,0 +1,2 @@ +# This is not a real provider executable. It's just here to be discovered +# during installation and produce a warning about it being in the wrong place. diff --git a/command/testdata/init-legacy-provider-cache/.terraform/plugins/registry.terraform.io/hashicorp/c/2.0.0/os_arch/terraform-provider-c b/command/testdata/init-legacy-provider-cache/.terraform/plugins/registry.terraform.io/hashicorp/c/2.0.0/os_arch/terraform-provider-c new file mode 100644 index 0000000000..7e9920ef48 --- /dev/null +++ b/command/testdata/init-legacy-provider-cache/.terraform/plugins/registry.terraform.io/hashicorp/c/2.0.0/os_arch/terraform-provider-c @@ -0,0 +1,2 @@ +# This is not a real provider executable. It's just here to be discovered +# during installation and produce a warning about it being in the wrong place. diff --git a/command/testdata/init-legacy-provider-cache/versions.tf b/command/testdata/init-legacy-provider-cache/versions.tf new file mode 100644 index 0000000000..7829b08cac --- /dev/null +++ b/command/testdata/init-legacy-provider-cache/versions.tf @@ -0,0 +1,21 @@ +terraform { + required_providers { + a = { + # This one is just not available at all + source = "example.com/test/a" + } + b = { + # This one is unavailable but happens to be cached in the legacy + # cache directory, under .terraform/plugins + source = "example.com/test/b" + } + c = { + # This one is also cached in the legacy cache directory, but it's + # an official provider so init will assume it got there via normal + # automatic installation and not generate a warning about it. + # This one is also not available at all, but it's an official + # provider so we don't expect to see a warning about it. + source = "hashicorp/c" + } + } +} diff --git a/website/upgrade-guides/0-14.html.markdown b/website/upgrade-guides/0-14.html.markdown index 93473a2727..79ecbadbd1 100644 --- a/website/upgrade-guides/0-14.html.markdown +++ b/website/upgrade-guides/0-14.html.markdown @@ -223,6 +223,52 @@ modules being altered in-place without your knowledge, we recommend using modules only from sources directly under your control, such as a private Terraform module registry. +### The local provider cache directory + +As an implementation detail of automatic provider installation, Terraform +has historically unpacked auto-installed plugins under the local cache +directory in `.terraform/plugins`. That directory was only intended for +Terraform's internal use, but unfortunately due to a miscommunication within +our team it was inadvertently documented as if it were a "filesystem mirror" +directory that you could place local providers in to upload them to +Terraform Cloud. + +Unfortunately the implementation details have changed in Terraform v0.14 in +order to move the authority for provider version selection to the new dependency +lock file, and so manually placing extra plugins into that local cache directory +is no longer effective in Terraform v0.14. + +We've included a heuristic in `terraform init` for Terraform v0.14 which should +detect situations where you're relying on an unofficial provider manually +installed into the cache directory and generate a warning like the following: + +``` +Warning: Missing provider is in legacy cache directory + +Terraform supports a number of local directories that can serve as automatic +local filesystem mirrors, but .terraform/plugins is not one of them because +Terraform v0.13 and earlier used this directory to cache copies of provider +plugins retrieved from elsewhere. + +If you intended to use this directory as a filesystem mirror for +tf.example.com/awesomecorp/happycloud, place it instead in the following +directory: + terraform.d/plugins/tf.example.com/awesomecorp/happycloud/1.1.0/linux_amd64 +``` + +The error message suggests using the `terraform.d` directory, which is a +local search directory originally introduced in Terraform v0.10 in order to +allow sending bundled providers along with your configuration up to Terraform +Cloud. The error message assumes that use-case because it was for Terraform +Cloud in particular that this approach was previously mis-documented. + +If you aren't intending to upload the provider plugin to Terraform Cloud as +part of your configuration, we recommend instead installing to one of +[the other implied mirror directories](/docs/commands/cli-config.html#implied-local-mirror-directories), +or you can explicitly configure some +[custom provider installation methods](/docs/commands/cli-config.html#provider-installation) +if your needs are more complicated. + ## Concise Terraform Plan Output In Terraform v0.11 and earlier, the output from `terraform plan` was designed