moduledeps: new package for representing module dependencies

As we add support for versioned providers, it's getting more complex to
track the dependencies of each module and of the configuration as a whole,
so this new package is intended to give us some room to model that
nicely as a building block for the various aspects of dependency
management.

This package is not responsible for *building* the dependency data
structure, since that requires knowledge of core Terraform and that would
create cyclic package dependencies. A later change will add some logic
in Terraform to create a Module tree based on the combination of a given
configuration and state, returning an instance of this package's Module
type.

The Module.PluginRequirements method flattens the provider-oriented
requirements into a set of plugin-oriented requirements (flattening any
provider aliases) giving us what we need to work with the plugin/discovery
package to find matching installed plugins.

Other later uses of this package will include selecting plugin archives
to auto-install from releases.hashicorp.com as part of "terraform init",
where the module-oriented level of abstraction here should be useful for
giving users good, specific feedback when constraints cannot be met.

A "reason" is tracked for each provider dependency with the intent that
this would later drive a UI for users to see and understand why a given
dependency is present, to aid in debugging sticky issues with
dependency resolution.

moduledeps/dependencies.go

@@ -0,0 +1,43 @@
+package moduledeps
+
+import (
+	"github.com/hashicorp/terraform/plugin/discovery"
+)
+
+// Providers describes a set of provider dependencies for a given module.
+//
+// Each named provider instance can have one version constraint.
+type Providers map[ProviderInstance]ProviderDependency
+
+// ProviderDependency describes the dependency for a particular provider
+// instance, including both the set of allowed versions and the reason for
+// the dependency.
+type ProviderDependency struct {
+	Versions discovery.VersionSet
+	Reason   ProviderDependencyReason
+}
+
+// ProviderDependencyReason is an enumeration of reasons why a dependency might be
+// present.
+type ProviderDependencyReason int
+
+const (
+	// ProviderDependencyExplicit means that there is an explicit "provider"
+	// block in the configuration for this module.
+	ProviderDependencyExplicit ProviderDependencyReason = iota
+
+	// ProviderDependencyImplicit means that there is no explicit "provider"
+	// block but there is at least one resource that uses this provider.
+	ProviderDependencyImplicit
+
+	// ProviderDependencyInherited is a special case of
+	// ProviderDependencyImplicit where a parent module has defined a
+	// configuration for the provider that has been inherited by at least one
+	// resource in this module.
+	ProviderDependencyInherited
+
+	// ProviderDependencyFromState means that this provider is not currently
+	// referenced by configuration at all, but some existing instances in
+	// the state still depend on it.
+	ProviderDependencyFromState
+)

moduledeps/doc.go

@@ -0,0 +1,7 @@
+// Package moduledeps contains types that can be used to describe the
+// providers required for all of the modules in a module tree.
+//
+// It does not itself contain the functionality for populating such
+// data structures; that's in Terraform core, since this package intentionally
+// does not depend on terraform core to avoid package dependency cycles.
+package moduledeps

moduledeps/module.go

@@ -0,0 +1,135 @@
+package moduledeps
+
+import (
+	"sort"
+	"strings"
+
+	"github.com/hashicorp/terraform/plugin/discovery"
+)
+
+// Module represents the dependencies of a single module, as well being
+// a node in a tree of such structures representing the dependencies of
+// an entire configuration.
+type Module struct {
+	Name      string
+	Providers Providers
+	Children  []*Module
+}
+
+// WalkFunc is a callback type for use with Module.WalkTree
+type WalkFunc func(path []string, parent *Module, current *Module) error
+
+// WalkTree calls the given callback once for the receiver and then
+// once for each descendent, in an order such that parents are called
+// before their children and siblings are called in the order they
+// appear in the Children slice.
+//
+// When calling the callback, parent will be nil for the first call
+// for the receiving module, and then set to the direct parent of
+// each module for the subsequent calls.
+//
+// The path given to the callback is valid only until the callback
+// returns, after which it will be mutated and reused. Callbacks must
+// therefore copy the path slice if they wish to retain it.
+//
+// If the given callback returns an error, the walk will be aborted at
+// that point and that error returned to the caller.
+//
+// This function is not thread-safe for concurrent modifications of the
+// data structure, so it's the caller's responsibility to arrange for that
+// should it be needed.
+//
+// It is safe for a callback to modify the descendents of the "current"
+// module, including the ordering of the Children slice itself, but the
+// callback MUST NOT modify the parent module.
+func (m *Module) WalkTree(cb WalkFunc) error {
+	return walkModuleTree(make([]string, 0, 1), nil, m, cb)
+}
+
+func walkModuleTree(path []string, parent *Module, current *Module, cb WalkFunc) error {
+	path = append(path, current.Name)
+	err := cb(path, parent, current)
+	if err != nil {
+		return err
+	}
+
+	for _, child := range current.Children {
+		err := walkModuleTree(path, current, child, cb)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// SortChildren sorts the Children slice into lexicographic order by
+// name, in-place.
+//
+// This is primarily useful prior to calling WalkTree so that the walk
+// will proceed in a consistent order.
+func (m *Module) SortChildren() {
+	sort.Sort(sortModules{m.Children})
+}
+
+// SortDescendents is a convenience wrapper for calling SortChildren on
+// the receiver and all of its descendent modules.
+func (m *Module) SortDescendents() {
+	m.WalkTree(func(path []string, parent *Module, current *Module) error {
+		current.SortChildren()
+		return nil
+	})
+}
+
+type sortModules struct {
+	modules []*Module
+}
+
+func (s sortModules) Len() int {
+	return len(s.modules)
+}
+
+func (s sortModules) Less(i, j int) bool {
+	cmp := strings.Compare(s.modules[i].Name, s.modules[j].Name)
+	return cmp < 0
+}
+
+func (s sortModules) Swap(i, j int) {
+	s.modules[i], s.modules[j] = s.modules[j], s.modules[i]
+}
+
+// PluginRequirements produces a PluginRequirements structure that can
+// be used with discovery.PluginMetaSet.ConstrainVersions to identify
+// suitable plugins to satisfy the module's provider dependencies.
+//
+// This method only considers the direct requirements of the receiver.
+// Use AllPluginRequirements to flatten the dependencies for the
+// entire tree of modules.
+func (m *Module) PluginRequirements() discovery.PluginRequirements {
+	ret := make(discovery.PluginRequirements)
+	for inst, dep := range m.Providers {
+		// m.Providers is keyed on provider names, such as "aws.foo".
+		// a PluginRequirements wants keys to be provider *types*, such
+		// as "aws". If there are multiple aliases for the same
+		// provider then we will flatten them into a single requirement
+		// by using Intersection to merge the version sets.
+		pty := inst.Type()
+		if existing, exists := ret[pty]; exists {
+			ret[pty] = existing.Intersection(dep.Versions)
+		} else {
+			ret[pty] = dep.Versions
+		}
+	}
+	return ret
+}
+
+// AllPluginRequirements calls PluginRequirements for the receiver and all
+// of its descendents, and merges the result into a single PluginRequirements
+// structure that would satisfy all of the modules together.
+func (m *Module) AllPluginRequirements() discovery.PluginRequirements {
+	var ret discovery.PluginRequirements
+	m.WalkTree(func(path []string, parent *Module, current *Module) error {
+		ret = ret.Merge(current.PluginRequirements())
+		return nil
+	})
+	return ret
+}

moduledeps/module_test.go

@@ -0,0 +1,216 @@
+package moduledeps
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/hashicorp/terraform/plugin/discovery"
+)
+
+func TestModuleWalkTree(t *testing.T) {
+	type walkStep struct {
+		Path       []string
+		ParentName string
+	}
+
+	tests := []struct {
+		Root      *Module
+		WalkOrder []walkStep
+	}{
+		{
+			&Module{
+				Name:     "root",
+				Children: nil,
+			},
+			[]walkStep{
+				{
+					Path:       []string{"root"},
+					ParentName: "",
+				},
+			},
+		},
+		{
+			&Module{
+				Name: "root",
+				Children: []*Module{
+					{
+						Name: "child",
+					},
+				},
+			},
+			[]walkStep{
+				{
+					Path:       []string{"root"},
+					ParentName: "",
+				},
+				{
+					Path:       []string{"root", "child"},
+					ParentName: "root",
+				},
+			},
+		},
+		{
+			&Module{
+				Name: "root",
+				Children: []*Module{
+					{
+						Name: "child",
+						Children: []*Module{
+							{
+								Name: "grandchild",
+							},
+						},
+					},
+				},
+			},
+			[]walkStep{
+				{
+					Path:       []string{"root"},
+					ParentName: "",
+				},
+				{
+					Path:       []string{"root", "child"},
+					ParentName: "root",
+				},
+				{
+					Path:       []string{"root", "child", "grandchild"},
+					ParentName: "child",
+				},
+			},
+		},
+		{
+			&Module{
+				Name: "root",
+				Children: []*Module{
+					{
+						Name: "child1",
+						Children: []*Module{
+							{
+								Name: "grandchild1",
+							},
+						},
+					},
+					{
+						Name: "child2",
+						Children: []*Module{
+							{
+								Name: "grandchild2",
+							},
+						},
+					},
+				},
+			},
+			[]walkStep{
+				{
+					Path:       []string{"root"},
+					ParentName: "",
+				},
+				{
+					Path:       []string{"root", "child1"},
+					ParentName: "root",
+				},
+				{
+					Path:       []string{"root", "child1", "grandchild1"},
+					ParentName: "child1",
+				},
+				{
+					Path:       []string{"root", "child2"},
+					ParentName: "root",
+				},
+				{
+					Path:       []string{"root", "child2", "grandchild2"},
+					ParentName: "child2",
+				},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) {
+			wo := test.WalkOrder
+			test.Root.WalkTree(func(path []string, parent *Module, current *Module) error {
+				if len(wo) == 0 {
+					t.Fatalf("ran out of walk steps while expecting one for %#v", path)
+				}
+				step := wo[0]
+				wo = wo[1:]
+				if got, want := path, step.Path; !reflect.DeepEqual(got, want) {
+					t.Errorf("wrong path %#v; want %#v", got, want)
+				}
+				parentName := ""
+				if parent != nil {
+					parentName = parent.Name
+				}
+				if got, want := parentName, step.ParentName; got != want {
+					t.Errorf("wrong parent name %q; want %q", got, want)
+				}
+
+				if got, want := current.Name, path[len(path)-1]; got != want {
+					t.Errorf("mismatching current.Name %q and final path element %q", got, want)
+				}
+				return nil
+			})
+		})
+	}
+}
+
+func TestModuleSortChildren(t *testing.T) {
+	m := &Module{
+		Name: "root",
+		Children: []*Module{
+			{
+				Name: "apple",
+			},
+			{
+				Name: "zebra",
+			},
+			{
+				Name: "xylophone",
+			},
+			{
+				Name: "pig",
+			},
+		},
+	}
+
+	m.SortChildren()
+
+	want := []string{"apple", "pig", "xylophone", "zebra"}
+	var got []string
+	for _, c := range m.Children {
+		got = append(got, c.Name)
+	}
+
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("wrong order %#v; want %#v", want, got)
+	}
+}
+
+func TestModulePluginRequirements(t *testing.T) {
+	m := &Module{
+		Name: "root",
+		Providers: Providers{
+			"foo": ProviderDependency{
+				Versions: discovery.ConstraintStr(">=1.0.0").MustParse(),
+			},
+			"foo.bar": ProviderDependency{
+				Versions: discovery.ConstraintStr(">=2.0.0").MustParse(),
+			},
+			"baz": ProviderDependency{
+				Versions: discovery.ConstraintStr(">=3.0.0").MustParse(),
+			},
+		},
+	}
+
+	reqd := m.PluginRequirements()
+	if len(reqd) != 2 {
+		t.Errorf("wrong number of elements in %#v; want 2", reqd)
+	}
+	if got, want := reqd["foo"].String(), ">=1.0.0,>=2.0.0"; got != want {
+		t.Errorf("wrong combination of versions for 'foo' %q; want %q", got, want)
+	}
+	if got, want := reqd["baz"].String(), ">=3.0.0"; got != want {
+		t.Errorf("wrong combination of versions for 'baz' %q; want %q", got, want)
+	}
+}

moduledeps/provider.go

@@ -0,0 +1,30 @@
+package moduledeps
+
+import (
+	"strings"
+)
+
+// ProviderInstance describes a particular provider instance by its full name,
+// like "null" or "aws.foo".
+type ProviderInstance string
+
+// Type returns the provider type of this instance. For example, for an instance
+// named "aws.foo" the type is "aws".
+func (p ProviderInstance) Type() string {
+	t := string(p)
+	if dotPos := strings.Index(t, "."); dotPos != -1 {
+		t = t[:dotPos]
+	}
+	return t
+}
+
+// Alias returns the alias of this provider, if any. An instance named "aws.foo"
+// has the alias "foo", while an instance named just "docker" has no alias,
+// so the empty string would be returned.
+func (p ProviderInstance) Alias() string {
+	t := string(p)
+	if dotPos := strings.Index(t, "."); dotPos != -1 {
+		return t[dotPos+1:]
+	}
+	return ""
+}

moduledeps/provider_test.go

@@ -0,0 +1,36 @@
+package moduledeps
+
+import (
+	"testing"
+)
+
+func TestProviderInstance(t *testing.T) {
+	tests := []struct {
+		Name      string
+		WantType  string
+		WantAlias string
+	}{
+		{
+			Name:      "aws",
+			WantType:  "aws",
+			WantAlias: "",
+		},
+		{
+			Name:      "aws.foo",
+			WantType:  "aws",
+			WantAlias: "foo",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			inst := ProviderInstance(test.Name)
+			if got, want := inst.Type(), test.WantType; got != want {
+				t.Errorf("got type %q; want %q", got, want)
+			}
+			if got, want := inst.Alias(), test.WantAlias; got != want {
+				t.Errorf("got alias %q; want %q", got, want)
+			}
+		})
+	}
+}