core: build a module dependency tree from config+state

This new private function takes a configuration tree and a state structure
and finds all of the explicit and implied provider dependencies
represented, returning them as a moduledeps.Module tree structure.

It annotates each dependency with a "reason", which is intended to be
useful to a user trying to figure out where a particular dependency is
coming from, though we don't yet have any UI to view this.

Nothing calls this yet, but a subsequent commit will use the result of
this to produce a constraint-conforming map of provider factories during
context initialization.

terraform/module_dependencies.go

@@ -0,0 +1,156 @@
+package terraform
+
+import (
+	"github.com/hashicorp/terraform/config"
+	"github.com/hashicorp/terraform/config/module"
+	"github.com/hashicorp/terraform/moduledeps"
+	"github.com/hashicorp/terraform/plugin/discovery"
+)
+
+// moduleTreeDependencies returns the dependencies of the tree of modules
+// described by the given configuration tree and state.
+//
+// Both configuration and state are required because there can be resources
+// implied by instances in the state that no longer exist in config.
+//
+// This function will panic if any invalid version constraint strings are
+// present in the configuration. This is guaranteed not to happen for any
+// configuration that has passed a call to Config.Validate().
+func moduleTreeDependencies(root *module.Tree, state *State) *moduledeps.Module {
+
+	// First we walk the configuration tree to build the overall structure
+	// and capture the explicit/implicit/inherited provider dependencies.
+	deps := moduleTreeConfigDependencies(root, nil)
+
+	// Next we walk over the resources in the state to catch any additional
+	// dependencies created by existing resources that are no longer in config.
+	// Most things we find in state will already be present in 'deps', but
+	// we're interested in the rare thing that isn't.
+	moduleTreeMergeStateDependencies(deps, state)
+
+	return deps
+}
+
+func moduleTreeConfigDependencies(root *module.Tree, inheritProviders map[string]*config.ProviderConfig) *moduledeps.Module {
+	if root == nil {
+		// If no config is provided, we'll make a synthetic root.
+		// This isn't necessarily correct if we're called with a nil that
+		// *isn't* at the root, but in practice that can never happen.
+		return &moduledeps.Module{
+			Name: "root",
+		}
+	}
+
+	ret := &moduledeps.Module{
+		Name: root.Name(),
+	}
+
+	cfg := root.Config()
+	providerConfigs := cfg.ProviderConfigsByFullName()
+
+	// Provider dependencies
+	{
+		providers := make(moduledeps.Providers, len(providerConfigs))
+
+		// Any providerConfigs elements are *explicit* provider dependencies,
+		// which is the only situation where the user might provide an actual
+		// version constraint. We'll take care of these first.
+		for fullName, pCfg := range providerConfigs {
+			inst := moduledeps.ProviderInstance(fullName)
+			versionSet := discovery.AllVersions
+			if pCfg.Version != "" {
+				versionSet = discovery.ConstraintStr(pCfg.Version).MustParse()
+			}
+			providers[inst] = moduledeps.ProviderDependency{
+				Versions: versionSet,
+				Reason:   moduledeps.ProviderDependencyExplicit,
+			}
+		}
+
+		// Each resource in the configuration creates an *implicit* provider
+		// dependency, though we'll only record it if there isn't already
+		// an explicit dependency on the same provider.
+		for _, rc := range cfg.Resources {
+			fullName := rc.ProviderFullName()
+			inst := moduledeps.ProviderInstance(fullName)
+			if _, exists := providers[inst]; exists {
+				// Explicit dependency already present
+				continue
+			}
+
+			reason := moduledeps.ProviderDependencyImplicit
+			if _, inherited := inheritProviders[fullName]; inherited {
+				reason = moduledeps.ProviderDependencyInherited
+			}
+
+			providers[inst] = moduledeps.ProviderDependency{
+				Versions: discovery.AllVersions,
+				Reason:   reason,
+			}
+		}
+
+		ret.Providers = providers
+	}
+
+	childInherit := make(map[string]*config.ProviderConfig)
+	for k, v := range inheritProviders {
+		childInherit[k] = v
+	}
+	for k, v := range providerConfigs {
+		childInherit[k] = v
+	}
+	for _, c := range root.Children() {
+		ret.Children = append(ret.Children, moduleTreeConfigDependencies(c, childInherit))
+	}
+
+	return ret
+}
+
+func moduleTreeMergeStateDependencies(root *moduledeps.Module, state *State) {
+	if state == nil {
+		return
+	}
+
+	findModule := func(path []string) *moduledeps.Module {
+		module := root
+		for _, name := range path[1:] { // skip initial "root"
+			var next *moduledeps.Module
+			for _, cm := range module.Children {
+				if cm.Name == name {
+					next = cm
+					break
+				}
+			}
+
+			if next == nil {
+				// If we didn't find a next node, we'll need to make one
+				next = &moduledeps.Module{
+					Name: name,
+				}
+				module.Children = append(module.Children, next)
+			}
+
+			module = next
+		}
+		return module
+	}
+
+	for _, ms := range state.Modules {
+		module := findModule(ms.Path)
+
+		for _, is := range ms.Resources {
+			fullName := config.ResourceProviderFullName(is.Type, is.Provider)
+			inst := moduledeps.ProviderInstance(fullName)
+			if _, exists := module.Providers[inst]; !exists {
+				if module.Providers == nil {
+					module.Providers = make(moduledeps.Providers)
+				}
+				module.Providers[inst] = moduledeps.ProviderDependency{
+					Versions: discovery.AllVersions,
+					Reason:   moduledeps.ProviderDependencyFromState,
+				}
+			}
+		}
+	}
+
+}

terraform/module_dependencies_test.go

@@ -0,0 +1,261 @@
+package terraform
+
+import (
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/hashicorp/terraform/config/module"
+	"github.com/hashicorp/terraform/moduledeps"
+	"github.com/hashicorp/terraform/plugin/discovery"
+)
+
+func TestModuleTreeDependencies(t *testing.T) {
+	tests := map[string]struct {
+		ConfigDir string // directory name from test-fixtures dir
+		State     *State
+		Want      *moduledeps.Module
+	}{
+		"no config or state": {
+			"",
+			nil,
+			&moduledeps.Module{
+				Name:      "root",
+				Providers: moduledeps.Providers{},
+				Children:  nil,
+			},
+		},
+		"empty config no state": {
+			"empty",
+			nil,
+			&moduledeps.Module{
+				Name:      "root",
+				Providers: moduledeps.Providers{},
+				Children:  nil,
+			},
+		},
+		"explicit provider": {
+			"module-deps-explicit-provider",
+			nil,
+			&moduledeps.Module{
+				Name: "root",
+				Providers: moduledeps.Providers{
+					"foo": moduledeps.ProviderDependency{
+						Versions: discovery.ConstraintStr(">=1.0.0").MustParse(),
+						Reason:   moduledeps.ProviderDependencyExplicit,
+					},
+					"foo.bar": moduledeps.ProviderDependency{
+						Versions: discovery.ConstraintStr(">=2.0.0").MustParse(),
+						Reason:   moduledeps.ProviderDependencyExplicit,
+					},
+				},
+				Children: nil,
+			},
+		},
+		"explicit provider unconstrained": {
+			"module-deps-explicit-provider-unconstrained",
+			nil,
+			&moduledeps.Module{
+				Name: "root",
+				Providers: moduledeps.Providers{
+					"foo": moduledeps.ProviderDependency{
+						Versions: discovery.AllVersions,
+						Reason:   moduledeps.ProviderDependencyExplicit,
+					},
+				},
+				Children: nil,
+			},
+		},
+		"implicit provider": {
+			"module-deps-implicit-provider",
+			nil,
+			&moduledeps.Module{
+				Name: "root",
+				Providers: moduledeps.Providers{
+					"foo": moduledeps.ProviderDependency{
+						Versions: discovery.AllVersions,
+						Reason:   moduledeps.ProviderDependencyImplicit,
+					},
+					"foo.baz": moduledeps.ProviderDependency{
+						Versions: discovery.AllVersions,
+						Reason:   moduledeps.ProviderDependencyImplicit,
+					},
+				},
+				Children: nil,
+			},
+		},
+		"explicit provider with resource": {
+			"module-deps-explicit-provider-resource",
+			nil,
+			&moduledeps.Module{
+				Name: "root",
+				Providers: moduledeps.Providers{
+					"foo": moduledeps.ProviderDependency{
+						Versions: discovery.ConstraintStr(">=1.0.0").MustParse(),
+						Reason:   moduledeps.ProviderDependencyExplicit,
+					},
+				},
+				Children: nil,
+			},
+		},
+		"inheritance of providers": {
+			"module-deps-inherit-provider",
+			nil,
+			&moduledeps.Module{
+				Name: "root",
+				Providers: moduledeps.Providers{
+					"foo": moduledeps.ProviderDependency{
+						Versions: discovery.AllVersions,
+						Reason:   moduledeps.ProviderDependencyExplicit,
+					},
+					"bar": moduledeps.ProviderDependency{
+						Versions: discovery.AllVersions,
+						Reason:   moduledeps.ProviderDependencyExplicit,
+					},
+				},
+				Children: []*moduledeps.Module{
+					{
+						Name: "child",
+						Providers: moduledeps.Providers{
+							"foo": moduledeps.ProviderDependency{
+								Versions: discovery.AllVersions,
+								Reason:   moduledeps.ProviderDependencyInherited,
+							},
+							"baz": moduledeps.ProviderDependency{
+								Versions: discovery.AllVersions,
+								Reason:   moduledeps.ProviderDependencyImplicit,
+							},
+						},
+						Children: []*moduledeps.Module{
+							{
+								Name: "grandchild",
+								Providers: moduledeps.Providers{
+									"foo": moduledeps.ProviderDependency{
+										Versions: discovery.AllVersions,
+										Reason:   moduledeps.ProviderDependencyExplicit,
+									},
+									"bar": moduledeps.ProviderDependency{
+										Versions: discovery.AllVersions,
+										Reason:   moduledeps.ProviderDependencyInherited,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		"provider from state": {
+			"empty",
+			&State{
+				Modules: []*ModuleState{
+					{
+						Path: []string{"root"},
+						Resources: map[string]*ResourceState{
+							"foo_bar.baz": {
+								Type:     "foo_bar",
+								Provider: "",
+							},
+						},
+					},
+				},
+			},
+			&moduledeps.Module{
+				Name: "root",
+				Providers: moduledeps.Providers{
+					"foo": moduledeps.ProviderDependency{
+						Versions: discovery.AllVersions,
+						Reason:   moduledeps.ProviderDependencyFromState,
+					},
+				},
+				Children: nil,
+			},
+		},
+		"providers in both config and state": {
+			"module-deps-explicit-provider",
+			&State{
+				Modules: []*ModuleState{
+					{
+						Path: []string{"root"},
+						Resources: map[string]*ResourceState{
+							"foo_bar.test1": {
+								Type:     "foo_bar",
+								Provider: "",
+							},
+							"foo_bar.test2": {
+								Type:     "foo_bar",
+								Provider: "foo.bar",
+							},
+							"baz_bar.test": {
+								Type:     "baz_bar",
+								Provider: "",
+							},
+						},
+					},
+					// note that we've skipped root.child intentionally here,
+					// to verify that we'll infer it based on the following
+					// module rather than crashing.
+					{
+						Path: []string{"root", "child", "grandchild"},
+						Resources: map[string]*ResourceState{
+							"banana_skin.test": {
+								Type:     "banana_skin",
+								Provider: "",
+							},
+						},
+					},
+				},
+			},
+			&moduledeps.Module{
+				Name: "root",
+				Providers: moduledeps.Providers{
+					"foo": moduledeps.ProviderDependency{
+						Versions: discovery.ConstraintStr(">=1.0.0").MustParse(),
+						Reason:   moduledeps.ProviderDependencyExplicit,
+					},
+					"foo.bar": moduledeps.ProviderDependency{
+						Versions: discovery.ConstraintStr(">=2.0.0").MustParse(),
+						Reason:   moduledeps.ProviderDependencyExplicit,
+					},
+					"baz": moduledeps.ProviderDependency{
+						Versions: discovery.AllVersions,
+						Reason:   moduledeps.ProviderDependencyFromState,
+					},
+				},
+				Children: []*moduledeps.Module{
+					{
+						Name: "child",
+						Children: []*moduledeps.Module{
+							{
+								Name: "grandchild",
+								Providers: moduledeps.Providers{
+									"banana": moduledeps.ProviderDependency{
+										Versions: discovery.AllVersions,
+										Reason:   moduledeps.ProviderDependencyFromState,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			var root *module.Tree
+			if test.ConfigDir != "" {
+				root = testModule(t, test.ConfigDir)
+			}
+
+			got := moduleTreeDependencies(root, test.State)
+			if !got.Equal(test.Want) {
+				t.Errorf(
+					"wrong dependency tree\ngot:  %s\nwant: %s",
+					spew.Sdump(got),
+					spew.Sdump(test.Want),
+				)
+			}
+		})
+	}
+}

terraform/test-fixtures/module-deps-explicit-provider-resource/main.tf

@@ -0,0 +1,7 @@
+provider "foo" {
+  version = ">=1.0.0"
+}
+
+resource "foo_bar" "test1" {
+
+}

terraform/test-fixtures/module-deps-explicit-provider-unconstrained/main.tf

@@ -0,0 +1,2 @@
+provider "foo" {
+}

terraform/test-fixtures/module-deps-explicit-provider/main.tf

@@ -0,0 +1,8 @@
+provider "foo" {
+  version = ">=1.0.0"
+}
+
+provider "foo" {
+  version = ">=2.0.0"
+  alias = "bar"
+}

terraform/test-fixtures/module-deps-implicit-provider/main.tf

@@ -0,0 +1,8 @@
+
+resource "foo_bar" "test1" {
+
+}
+
+resource "foo_bar" "test2" {
+    provider = "foo.baz"
+}

terraform/test-fixtures/module-deps-inherit-provider/child/child.tf

@@ -0,0 +1,17 @@
+
+# "foo" is inherited from the parent module
+resource "foo_bar" "test" {
+
+}
+
+# but we don't use the "bar" provider inherited from the parent
+
+# "baz" is introduced here for the first time, so it's an implicit
+# dependency
+resource "baz_bar" "test" {
+
+}
+
+module "grandchild" {
+  source = "../grandchild"
+}

terraform/test-fixtures/module-deps-inherit-provider/grandchild/grandchild.tf

@@ -0,0 +1,11 @@
+
+# Here we *override* the foo from the parent
+provider "foo" {
+
+}
+
+# We also use the "bar" provider defined at the root, which was
+# completely ignored by the child module in between.
+resource "bar_thing" "test" {
+
+}

terraform/test-fixtures/module-deps-inherit-provider/main.tf

@@ -0,0 +1,11 @@
+
+provider "foo" {
+}
+
+provider "bar" {
+
+}
+
+module "child" {
+    source = "./child"
+}