testing framework: add validation for provider blocks in test files (#33542)

* testing framework: add validation for provider blocks in test files

* .tftest -> .tftest.hcl

internal/configs/config.go

@@ -594,7 +594,7 @@ func (c *Config) addProviderRequirementsFromProviderBlock(reqs getproviders.Requ
 // resolveProviderTypes walks through the providers in the module and ensures
 // the true types are assigned based on the provider requirements for the
 // module.
-func (c *Config) resolveProviderTypes() {
+func (c *Config) resolveProviderTypes() map[string]addrs.Provider {
 	for _, child := range c.Children {
 		child.resolveProviderTypes()
 	}
@@ -636,6 +636,147 @@ func (c *Config) resolveProviderTypes() {
 			}
 		}
 	}
+
+	return providers
+}
+
+// resolveProviderTypesForTests matches resolveProviderTypes except it uses
+// the information from resolveProviderTypes to resolve the provider types for
+// providers defined within the configs test files.
+func (c *Config) resolveProviderTypesForTests(providers map[string]addrs.Provider) {
+
+	for _, test := range c.Module.Tests {
+
+		// testProviders contains the configuration blocks for all the providers
+		// defined by this test file. It is keyed by the name of the provider
+		// and the values are a slice of provider configurations which contains
+		// all the definitions of a named provider of which there can be
+		// multiple because of aliases.
+		testProviders := make(map[string][]*Provider)
+		for _, provider := range test.Providers {
+			testProviders[provider.Name] = append(testProviders[provider.Name], provider)
+		}
+
+		// matchedProviders maps the names of providers from testProviders to
+		// the provider type we have identified for them so far. If during the
+		// course of resolving the types we find a run block is attempting to
+		// reuse a provider that has already been assigned a different type,
+		// then this is an error that we can raise now.
+		matchedProviders := make(map[string]addrs.Provider)
+
+		// First, we primarily draw our provider types from the main
+		// configuration under test. The providers for the main configuration
+		// are provided to us in the argument.
+
+		// We've now set provider types for all the providers required by the
+		// main configuration. But we can have modules with their own required
+		// providers referenced by the run blocks. We also have passed provider
+		// configs that can affect the types of providers when the names don't
+		// match, so we'll do that here.
+
+		for _, run := range test.Runs {
+
+			// If this run block is executing against our main configuration, we
+			// want to use the external providers passed in. If we are executing
+			// against a different module then we need to resolve the provider
+			// types for that first, and then use those providers.
+			providers := providers
+			if run.ConfigUnderTest != nil {
+				providers = run.ConfigUnderTest.resolveProviderTypes()
+			}
+
+			// We now check to see what providers this run block is actually
+			// using, and we can then assign types back to the
+
+			if len(run.Providers) > 0 {
+				// This provider is only using the subset of providers specified
+				// within the provider block.
+
+				for _, p := range run.Providers {
+					addr, exists := providers[p.InChild.Name]
+					if !exists {
+						// If this provider wasn't explicitly defined in the
+						// target module, then we'll set it to the default.
+						addr = addrs.NewDefaultProvider(p.InChild.Name)
+					}
+
+					// The child type is always just derived from the providers
+					// within the config this run block is using.
+					p.InChild.providerType = addr
+
+					// If we have previously assigned a type to the provider
+					// for the parent reference, then we use that for the
+					// parent type.
+					if addr, exists := matchedProviders[p.InParent.Name]; exists {
+						p.InParent.providerType = addr
+						continue
+					}
+
+					// Otherwise, we'll define the parent type based on the
+					// child and reference that backwards.
+					p.InParent.providerType = p.InChild.providerType
+
+					if aliases, exists := testProviders[p.InParent.Name]; exists {
+						matchedProviders[p.InParent.Name] = p.InParent.providerType
+						for _, alias := range aliases {
+							alias.providerType = p.InParent.providerType
+						}
+					}
+				}
+
+			} else {
+				// This provider is going to load all the providers it can using
+				// simple name matching.
+
+				for name, addr := range providers {
+
+					if _, exists := matchedProviders[name]; exists {
+						// Then we've already handled providers of this type
+						// previously.
+						continue
+					}
+
+					if aliases, exists := testProviders[name]; exists {
+						// Then this provider has been defined within our test
+						// config. Let's give it the appropriate type.
+						matchedProviders[name] = addr
+						for _, alias := range aliases {
+							alias.providerType = addr
+						}
+
+						continue
+					}
+
+					// If we get here then it means we don't actually have a
+					// provider block for this provider name within our test
+					// file. This is fine, it just means we don't have to do
+					// anything and the test will use the default provider for
+					// that name.
+
+				}
+			}
+
+		}
+
+		// Now, we've analysed all the test runs for this file. If any providers
+		// have not been claimed then we'll just give them the default provider
+		// for their name.
+		for name, aliases := range testProviders {
+			if _, exists := matchedProviders[name]; exists {
+				// Then this provider has a type already.
+				continue
+			}
+
+			addr := addrs.NewDefaultProvider(name)
+			matchedProviders[name] = addr
+
+			for _, alias := range aliases {
+				alias.providerType = addr
+			}
+		}
+
+	}
+
 }
 
 // ProviderTypes returns the FQNs of each distinct provider type referenced

internal/configs/config_build.go

@@ -36,10 +36,12 @@ func BuildConfig(root *Module, walker ModuleWalker) (*Config, hcl.Diagnostics) {
 	if !diags.HasErrors() {
 		// Now that the config is built, we can connect the provider names to all
 		// the known types for validation.
-		cfg.resolveProviderTypes()
+		providers := cfg.resolveProviderTypes()
+		cfg.resolveProviderTypesForTests(providers)
 	}
 
 	diags = append(diags, validateProviderConfigs(nil, cfg, nil)...)
+	diags = append(diags, validateProviderConfigsForTests(cfg)...)
 
 	return cfg, diags
 }

internal/configs/config_build_test.go

@@ -172,7 +172,7 @@ func TestBuildConfigInvalidModules(t *testing.T) {
 			parser := NewParser(nil)
 			path := filepath.Join(testDir, name)
 
-			mod, diags := parser.LoadConfigDir(path)
+			mod, diags := parser.LoadConfigDirWithTests(path, "tests")
 			if diags.HasErrors() {
 				// these tests should only trigger errors that are caught in
 				// the config loader.

internal/configs/provider_validation.go

@@ -9,9 +9,268 @@ import (
 	"strings"
 
 	"github.com/hashicorp/hcl/v2"
+
 	"github.com/hashicorp/terraform/internal/addrs"
 )
 
+// validateProviderConfigsForTests performs the same role as
+// validateProviderConfigs except it validates the providers configured within
+// test files.
+//
+// To do this is calls out to validateProviderConfigs for each run block that
+// has ConfigUnderTest set.
+//
+// In addition, for each run block that executes against the main config it
+// validates the providers the run block wants to use match the providers
+// specified in the main configuration. It does this without reaching out to
+// validateProviderConfigs because the main configuration has already been
+// validated, and we don't want to redo all the work that happens in that
+// function. So, we only validate the providers our test files define match
+// the providers required by the main configuration.
+//
+// This function does some fairly controversial conversions into structures
+// expected by validateProviderConfigs but since we're just using it for
+// validation we'll still get the correct error messages, and we can make the
+// declaration ranges line up sensibly so we'll even get good diagnostics.
+func validateProviderConfigsForTests(cfg *Config) (diags hcl.Diagnostics) {
+
+	for name, test := range cfg.Module.Tests {
+		for _, run := range test.Runs {
+
+			if run.ConfigUnderTest == nil {
+				// Then we're calling out to the main configuration under test.
+				//
+				// We just need to make sure that the providers we are setting
+				// actually match the providers in the configuration. The main
+				// configuration has already been validated, so we don't need to
+				// do the whole thing again.
+
+				if len(run.Providers) > 0 {
+					// This is the easy case, we can just validate that the
+					// provider types match.
+					for _, provider := range run.Providers {
+
+						parentType, childType := provider.InParent.providerType, provider.InChild.providerType
+						if parentType.IsZero() {
+							parentType = addrs.NewDefaultProvider(provider.InParent.Name)
+						}
+						if childType.IsZero() {
+							childType = addrs.NewDefaultProvider(provider.InChild.Name)
+						}
+
+						if !childType.Equals(parentType) {
+							diags = append(diags, &hcl.Diagnostic{
+								Severity: hcl.DiagError,
+								Summary:  "Provider type mismatch",
+								Detail: fmt.Sprintf(
+									"The local name %q in %s represents provider %q, but %q in the root module represents %q.\n\nThis means the provider definition for %q within %s, or other provider definitions with the same name, have been referenced by multiple run blocks and assigned to different provider types.",
+									provider.InParent.Name, name, parentType, provider.InChild.Name, childType, provider.InParent.Name, name),
+								Subject: provider.InParent.NameRange.Ptr(),
+							})
+						}
+					}
+
+					// Skip to the next file, we only need to verify the types
+					// specified here.
+					continue
+				}
+
+				// Otherwise, we need to verify that the providers required by
+				// the configuration match the types defined by our test file.
+
+				for _, requirement := range cfg.Module.ProviderRequirements.RequiredProviders {
+					if provider, exists := test.Providers[requirement.Name]; exists {
+
+						providerType := provider.providerType
+						if providerType.IsZero() {
+							providerType = addrs.NewDefaultProvider(provider.Name)
+						}
+
+						if !providerType.Equals(requirement.Type) {
+							diags = append(diags, &hcl.Diagnostic{
+								Severity: hcl.DiagError,
+								Summary:  "Provider type mismatch",
+								Detail: fmt.Sprintf(
+									"The provider %q in %s represents provider %q, but %q in the root module represents %q.\n\nThis means the provider definition for %q within %s, or other provider definitions with the same name, have been referenced by multiple run blocks and assigned to different provider types.",
+									provider.moduleUniqueKey(), name, providerType, requirement.Name, requirement.Type, provider.moduleUniqueKey(), name),
+								Subject: provider.DeclRange.Ptr(),
+							})
+						}
+					}
+
+					for _, alias := range requirement.Aliases {
+						if provider, exists := test.Providers[alias.StringCompact()]; exists {
+
+							providerType := provider.providerType
+							if providerType.IsZero() {
+								providerType = addrs.NewDefaultProvider(provider.Name)
+							}
+
+							if !providerType.Equals(requirement.Type) {
+								diags = append(diags, &hcl.Diagnostic{
+									Severity: hcl.DiagError,
+									Summary:  "Provider type mismatch",
+									Detail: fmt.Sprintf(
+										"The provider %q in %s represents provider %q, but %q in the root module represents %q.\n\nThis means the provider definition for %q within %s, or other provider definitions with the same name, have been referenced by multiple run blocks and assigned to different provider types.",
+										provider.moduleUniqueKey(), name, providerType, alias.StringCompact(), requirement.Type, provider.moduleUniqueKey(), name),
+									Subject: provider.DeclRange.Ptr(),
+								})
+							}
+						}
+					}
+				}
+
+				for _, provider := range cfg.Module.ProviderConfigs {
+
+					providerType := provider.providerType
+					if providerType.IsZero() {
+						providerType = addrs.NewDefaultProvider(provider.Name)
+					}
+
+					if testProvider, exists := test.Providers[provider.moduleUniqueKey()]; exists {
+
+						testProviderType := testProvider.providerType
+						if testProviderType.IsZero() {
+							testProviderType = addrs.NewDefaultProvider(testProvider.Name)
+						}
+
+						if !providerType.Equals(testProviderType) {
+							diags = append(diags, &hcl.Diagnostic{
+								Severity: hcl.DiagError,
+								Summary:  "Provider type mismatch",
+								Detail: fmt.Sprintf(
+									"The provider %q in %s represents provider %q, but %q in the root module represents %q.\n\nThis means the provider definition for %q within %s has been referenced by multiple run blocks and assigned to different provider types.",
+									testProvider.moduleUniqueKey(), name, testProviderType, provider.moduleUniqueKey(), providerType, testProvider.moduleUniqueKey(), name),
+								Subject: testProvider.DeclRange.Ptr(),
+							})
+						}
+					}
+				}
+
+			} else {
+				// Then we're executing another module. We'll just call out to
+				// validateProviderConfigs and let it do the whole thing.
+
+				providers := run.Providers
+				if len(providers) == 0 {
+					// If the test run didn't provide us a subset of providers
+					// to use, we'll build our own. This is so that we can fit
+					// into the schema expected by validateProviderConfigs.
+
+					matchedProviders := make(map[string]PassedProviderConfig)
+
+					// We'll go over all the requirements in the module first
+					// and see if we have defined any providers for that
+					// requirement. If we have, then we'll take not of that.
+
+					for _, requirement := range cfg.Module.ProviderRequirements.RequiredProviders {
+
+						if provider, exists := test.Providers[requirement.Name]; exists {
+							matchedProviders[requirement.Name] = PassedProviderConfig{
+								InChild: &ProviderConfigRef{
+									Name:         requirement.Name,
+									NameRange:    requirement.DeclRange,
+									providerType: requirement.Type,
+								},
+								InParent: &ProviderConfigRef{
+									Name:         provider.Name,
+									NameRange:    provider.NameRange,
+									Alias:        provider.Alias,
+									AliasRange:   provider.AliasRange,
+									providerType: provider.providerType,
+								},
+							}
+						}
+
+						// Also, remember to check for any aliases the module
+						// expects.
+
+						for _, alias := range requirement.Aliases {
+							key := alias.StringCompact()
+
+							if provider, exists := test.Providers[key]; exists {
+								matchedProviders[key] = PassedProviderConfig{
+									InChild: &ProviderConfigRef{
+										Name:         requirement.Name,
+										NameRange:    requirement.DeclRange,
+										Alias:        alias.Alias,
+										AliasRange:   requirement.DeclRange.Ptr(),
+										providerType: requirement.Type,
+									},
+									InParent: &ProviderConfigRef{
+										Name:         provider.Name,
+										NameRange:    provider.NameRange,
+										Alias:        provider.Alias,
+										AliasRange:   provider.AliasRange,
+										providerType: provider.providerType,
+									},
+								}
+							}
+
+						}
+
+					}
+
+					// Next, we'll look at any providers the module has defined
+					// directly. If we have an equivalent provider in the test
+					// file then we'll add that in to override it. If the module
+					// has both built a required providers block and a provider
+					// block for the same provider, we'll overwrite the one we
+					// made for the requirement provider. We get more precise
+					// DeclRange objects from provider blocks so it makes for
+					// better error messages to use these.
+
+					for _, provider := range cfg.Module.ProviderConfigs {
+						key := provider.moduleUniqueKey()
+
+						if testProvider, exists := test.Providers[key]; exists {
+							matchedProviders[key] = PassedProviderConfig{
+								InChild: &ProviderConfigRef{
+									Name:         provider.Name,
+									NameRange:    provider.DeclRange,
+									Alias:        provider.Alias,
+									AliasRange:   provider.DeclRange.Ptr(),
+									providerType: provider.providerType,
+								},
+								InParent: &ProviderConfigRef{
+									Name:         testProvider.Name,
+									NameRange:    testProvider.NameRange,
+									Alias:        testProvider.Alias,
+									AliasRange:   testProvider.AliasRange,
+									providerType: testProvider.providerType,
+								},
+							}
+						}
+					}
+
+					// Last thing to do here is add them into the actual
+					// providers list that is going into the module call below.
+					for _, provider := range matchedProviders {
+						providers = append(providers, provider)
+					}
+
+				}
+
+				// Let's make a little fake module call that we can use to call
+				// into validateProviderConfigs.
+				mc := &ModuleCall{
+					Name:            run.Name,
+					SourceAddr:      run.Module.Source,
+					SourceAddrRange: run.Module.SourceDeclRange,
+					SourceSet:       true,
+					Version:         run.Module.Version,
+					Providers:       providers,
+					DeclRange:       run.Module.DeclRange,
+				}
+
+				diags = append(diags, validateProviderConfigs(mc, run.ConfigUnderTest, nil)...)
+			}
+		}
+	}
+
+	return diags
+}
+
 // validateProviderConfigs walks the full configuration tree from the root
 // module outward, static validation rules to the various combinations of
 // provider configuration, required_providers values, and module call providers

internal/configs/testdata/config-diagnostics/tests-provider-mismatch-with-module/errors

@@ -0,0 +1,3 @@
+testdata/config-diagnostics/tests-provider-mismatch-with-module/main.tftest.hcl:2,1-15: Provider type mismatch; The provider "foo" in main.tftest.hcl represents provider "registry.terraform.io/hashicorp/bar", but "foo" in the root module represents "registry.terraform.io/hashicorp/foo".\n\nThis means the provider definition for "foo" within main.tftest.hcl, or other provider definitions with the same name, have been referenced by multiple run blocks and assigned to different provider types.
+testdata/config-diagnostics/tests-provider-mismatch-with-module/main.tftest.hcl:4,1-15: Provider type mismatch; The provider "foo.bar" in main.tftest.hcl represents provider "registry.terraform.io/hashicorp/bar", but "foo.bar" in the root module represents "registry.terraform.io/hashicorp/foo".\n\nThis means the provider definition for "foo.bar" within main.tftest.hcl, or other provider definitions with the same name, have been referenced by multiple run blocks and assigned to different provider types.
+testdata/config-diagnostics/tests-provider-mismatch-with-module/main.tftest.hcl:8,1-15: Provider type mismatch; The provider "bar" in main.tftest.hcl represents provider "registry.terraform.io/hashicorp/foo", but "bar" in the root module represents "registry.terraform.io/hashicorp/bar".\n\nThis means the provider definition for "bar" within main.tftest.hcl has been referenced by multiple run blocks and assigned to different provider types.

internal/configs/testdata/config-diagnostics/tests-provider-mismatch-with-module/main.tf

@@ -0,0 +1,14 @@
+terraform {
+  required_providers {
+    foo = {
+      source = "hashicorp/foo"
+      configuration_aliases = [foo.bar]
+    }
+  }
+}
+
+provider "bar" {}
+
+resource "foo_resource" "resource" {}
+
+resource "bar_resource" "resource" {}

internal/configs/testdata/config-diagnostics/tests-provider-mismatch-with-module/main.tftest.hcl

@@ -0,0 +1,18 @@
+
+provider "foo" {}
+
+provider "foo" {
+  alias = "bar"
+}
+
+provider "bar" {}
+
+run "setup_module" {
+
+  module {
+    source = "./setup"
+  }
+
+}
+
+run "main_module" {}

internal/configs/testdata/config-diagnostics/tests-provider-mismatch-with-module/setup/main.tf

@@ -0,0 +1,15 @@
+terraform {
+    required_providers {
+        foo = {
+            source = "hashicorp/bar"
+            configuration_aliases = [ foo.bar ]
+        }
+        bar = {
+            source = "hashicorp/foo"
+        }
+    }
+}
+
+resource "foo_resource" "resource" {}
+
+resource "bar_resource" "resource" {}

internal/configs/testdata/config-diagnostics/tests-provider-mismatch/errors

@@ -0,0 +1 @@
+testdata/config-diagnostics/tests-provider-mismatch/main.tftest.hcl:27,11-14: Provider type mismatch; The local name "bar" in main.tftest.hcl represents provider "registry.terraform.io/hashicorp/bar", but "foo" in the root module represents "registry.terraform.io/hashicorp/foo".\n\nThis means the provider definition for "bar" within main.tftest.hcl, or other provider definitions with the same name, have been referenced by multiple run blocks and assigned to different provider types.

internal/configs/testdata/config-diagnostics/tests-provider-mismatch/main.tf

@@ -0,0 +1,14 @@
+terraform {
+  required_providers {
+    foo = {
+      source = "hashicorp/foo"
+      configuration_aliases = [foo.bar]
+    }
+  }
+}
+
+provider "bar" {}
+
+resource "foo_resource" "resource" {}
+
+resource "bar_resource" "resource" {}

internal/configs/testdata/config-diagnostics/tests-provider-mismatch/main.tftest.hcl

@@ -0,0 +1,32 @@
+
+provider "foo" {}
+
+provider "foo" {
+  alias = "bar"
+}
+
+provider "bar" {
+  alias = "foo"
+}
+
+run "default_should_be_fine" {}
+
+run "bit_complicated_still_okay "{
+
+  providers = {
+    foo = foo
+    foo.bar = foo.bar
+    bar = bar.foo
+  }
+
+}
+
+run "mismatched_foo_direct" {
+
+  providers = {
+    foo = bar // bad!
+    foo.bar = foo.bar
+    bar = bar.foo
+  }
+
+}