diff --git a/configs/config.go b/configs/config.go index 07040fa8b7..e9b23b7a72 100644 --- a/configs/config.go +++ b/configs/config.go @@ -1,8 +1,6 @@ package configs import ( - "fmt" - version "github.com/hashicorp/go-version" "github.com/hashicorp/hcl2/hcl" ) @@ -26,6 +24,17 @@ type Config struct { // this module. If this is the root module then this field is nil. Parent *Config + // Path is a sequence of module logical names that traverse from the root + // module to this config. Path is empty for the root module. + // + // This should not be used to display a path to the end-user, since + // our UI conventions call for us to return a module address string in that + // case, and a module address string ought to be built from the dynamic + // module tree (resulting from evaluating "count" and "for_each" arguments + // on our calls to produce potentially multiple child instances per call) + // rather than from our static module tree. + Path []string + // ChildModules points to the Config for each of the direct child modules // called from this module. The keys in this map match the keys in // Module.ModuleCalls. @@ -64,56 +73,6 @@ type Config struct { Version *version.Version } -// Path returns the path of logical names that lead to this Config from its -// root. -// -// This function should not be used to display a path to the end-user, since -// our UI conventions call for us to return a module address string in that -// case, and a module address string ought to be built from the dynamic -// module tree (resulting from evaluating "count" and "for_each" arguments -// on our calls to produce potentially multiple child instances per call) -// rather than from our static module tree. -// -// This function will panic if called on a config that is not part of a -// wholesome config tree, e.g. because it has incorrectly-built Children -// maps, missing node pointers, etc. However, it should work as expected -// for any tree constructed by BuildConfig and not subsequently modified. -func (c *Config) Path() []string { - // The implementation here is not especially efficient, but we don't - // care too much because module trees are shallow and narrow in all - // reasonable configurations. - - // We'll build our path in reverse here, since we're starting at the - // leafiest node, and then we'll flip it before we return. - path := make([]string, 0, c.Depth()) - - this := c - for this.Parent != nil { - parent := this.Parent - var name string - for candidate, ref := range parent.Children { - if ref == this { - name = candidate - } - } - if name == "" { - panic(fmt.Errorf( - "Config %p does not appear in the child table for its parent %p: %#v", - this, parent, parent.Children, - )) - } - path = append(path, name) - this = parent - } - - // reverse the items - for i := 0; i < len(path)/2; i++ { - j := len(path) - i - 1 - path[i], path[j] = path[j], path[i] - } - return path -} - // Depth returns the number of "hops" the receiver is from the root of its // module tree, with the root module having a depth of zero. func (c *Config) Depth() int { diff --git a/configs/config_build.go b/configs/config_build.go index 0182426c18..7aca4f1c5c 100644 --- a/configs/config_build.go +++ b/configs/config_build.go @@ -29,8 +29,13 @@ func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config, calls := parent.Module.ModuleCalls for _, call := range calls { + path := make([]string, len(parent.Path)+1) + copy(path, parent.Path) + path[len(path)-1] = call.Name + req := ModuleRequest{ Name: call.Name, + Path: path, SourceAddr: call.SourceAddr, SourceAddrRange: call.SourceAddrRange, VersionConstraint: call.Version, @@ -50,6 +55,7 @@ func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config, child := &Config{ Parent: parent, Root: parent.Root, + Path: path, Module: mod, CallRange: call.DeclRange, SourceAddr: call.SourceAddr, @@ -102,6 +108,12 @@ type ModuleRequest struct { // HCL identifier and UTF-8 encoded. Name string + // Path is a list of logical names that traverse from the root module to + // this module. This can be used, for example, to form a lookup key for + // each distinct module call in a configuration, allowing for multiple + // calls with the same name at different points in the tree. + Path []string + // SourceAddr is the source address string provided by the user in // configuration. SourceAddr string diff --git a/configs/config_build_test.go b/configs/config_build_test.go index 3ac46e35cc..1e409d8469 100644 --- a/configs/config_build_test.go +++ b/configs/config_build_test.go @@ -44,7 +44,7 @@ func TestBuildConfig(t *testing.T) { var got []string cfg.DeepEach(func(c *Config) { - got = append(got, fmt.Sprintf("%s %s", strings.Join(c.Path(), "."), c.Version)) + got = append(got, fmt.Sprintf("%s %s", strings.Join(c.Path, "."), c.Version)) }) sort.Strings(got) want := []string{ diff --git a/configs/configload/doc.go b/configs/configload/doc.go new file mode 100644 index 0000000000..8b615f9026 --- /dev/null +++ b/configs/configload/doc.go @@ -0,0 +1,4 @@ +// Package configload knows how to install modules into the .terraform/modules +// directory and to load modules from those installed locations. It is used +// in conjunction with the LoadConfig function in the parent package. +package configload diff --git a/configs/configload/loader.go b/configs/configload/loader.go new file mode 100644 index 0000000000..fba2441736 --- /dev/null +++ b/configs/configload/loader.go @@ -0,0 +1,91 @@ +package configload + +import ( + "fmt" + + "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry" + "github.com/hashicorp/terraform/svchost/auth" + "github.com/hashicorp/terraform/svchost/disco" + "github.com/spf13/afero" +) + +// A Loader instance is the main entry-point for loading configurations via +// this package. +// +// It extends the general config-loading functionality in the parent package +// "configs" to support installation of modules from remote sources and +// loading full configurations using modules that were previously installed. +type Loader struct { + // parser is used to read configuration + parser *configs.Parser + + // modules is used to install and locate descendent modules that are + // referenced (directly or indirectly) from the root module. + modules moduleMgr +} + +// Config is used with NewLoader to specify configuration arguments for the +// loader. +type Config struct { + // ModulesDir is a path to a directory where descendent modules are + // (or should be) installed. (This is usually the + // .terraform/modules directory, in the common case where this package + // is being loaded from the main Terraform CLI package.) + ModulesDir string + + // Services is the service discovery client to use when locating remote + // module registry endpoints. If this is nil then registry sources are + // not supported, which should be true only in specialized circumstances + // such as in tests. + Services *disco.Disco + + // Creds is a credentials store for communicating with remote module + // registry endpoints. If this is nil then no credentials will be used. + Creds auth.CredentialsSource +} + +// NewLoader creates and returns a loader that reads configuration from the +// real OS filesystem. +// +// The loader has some internal state about the modules that are currently +// installed, which is read from disk as part of this function. If that +// manifest cannot be read then an error will be returned. +func NewLoader(config *Config) (*Loader, error) { + fs := afero.NewOsFs() + parser := configs.NewParser(fs) + reg := registry.NewClient(config.Services, config.Creds, nil) + + ret := &Loader{ + parser: parser, + modules: moduleMgr{ + FS: afero.Afero{fs}, + Dir: config.ModulesDir, + Services: config.Services, + Creds: config.Creds, + Registry: reg, + }, + } + + err := ret.modules.readModuleManifestSnapshot() + if err != nil { + return nil, fmt.Errorf("failed to read module manifest: %s", err) + } + + return ret, nil +} + +// Parser returns the underlying parser for this loader. +// +// This is useful for loading other sorts of files than the module directories +// that a loader deals with, since then they will share the source code cache +// for this loader and can thus be shown as snippets in diagnostic messages. +func (l *Loader) Parser() *configs.Parser { + return l.parser +} + +// Sources returns the source code cache for the underlying parser of this +// loader. This is a shorthand for l.Parser().Sources(). +func (l *Loader) Sources() map[string][]byte { + return l.parser.Sources() +} diff --git a/configs/configload/loader_load.go b/configs/configload/loader_load.go new file mode 100644 index 0000000000..104a31d201 --- /dev/null +++ b/configs/configload/loader_load.go @@ -0,0 +1,97 @@ +package configload + +import ( + "fmt" + + version "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/terraform/configs" +) + +// LoadConfig reads the Terraform module in the given directory and uses it as the +// root module to build the static module tree that represents a configuration, +// assuming that all required descendent modules have already been installed. +// +// If error diagnostics are returned, the returned configuration may be either +// nil or incomplete. In the latter case, cautious static analysis is possible +// in spite of the errors. +// +// LoadConfig performs the basic syntax and uniqueness validations that are +// required to process the individual modules, and also detects +func (l *Loader) LoadConfig(rootDir string) (*configs.Config, hcl.Diagnostics) { + rootMod, diags := l.parser.LoadConfigDir(rootDir) + if rootMod == nil { + return nil, diags + } + + cfg, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc(l.moduleWalkerLoad)) + diags = append(diags, cDiags...) + + return cfg, diags +} + +// moduleWalkerLoad is a configs.ModuleWalkerFunc for loading modules that +// are presumed to have already been installed. A different function +// (moduleWalkerInstall) is used for installation. +func (l *Loader) moduleWalkerLoad(req *configs.ModuleRequest) (*configs.Module, *version.Version, hcl.Diagnostics) { + // Since we're just loading here, we expect that all referenced modules + // will be already installed and described in our manifest. However, we + // do verify that the manifest and the configuration are in agreement + // so that we can prompt the user to run "terraform init" if not. + + key := manifestKey(req.Path) + record, exists := l.modules.manifest[key] + + if !exists { + return nil, nil, hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Module not installed", + Detail: "This module is not yet installed. Run \"terraform init\" to install all modules required by this configuration.", + Subject: &req.CallRange, + }, + } + } + + var diags hcl.Diagnostics + + // Check for inconsistencies between manifest and config + if req.SourceAddr != record.SourceAddr { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Module source has changed", + Detail: "The source address was changed since this module was installed. Run \"terraform init\" to install all modules required by this configuration.", + Subject: &req.SourceAddrRange, + }) + } + if !req.VersionConstraint.Required.Check(record.Version) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Module version requirements have changed", + Detail: fmt.Sprintf( + "The version requirements have changed since this module was installed and the installed version (%s) is no longer acceptable. Run \"terraform init\" to install all modules required by this configuration.", + record.Version, + ), + Subject: &req.SourceAddrRange, + }) + } + + mod, mDiags := l.parser.LoadConfigDir(record.Dir) + diags = append(diags, mDiags...) + if mod == nil { + // nil specifically indicates that the directory does not exist or + // cannot be read, so in this case we'll discard any generic diagnostics + // returned from LoadConfigDir and produce our own context-sensitive + // error message. + return nil, nil, hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Module not installed", + Detail: fmt.Sprintf("This module's local cache directory %s could not be read. Run \"terraform init\" to install all modules required by this configuration.", record.Dir), + Subject: &req.CallRange, + }, + } + } + + return mod, record.Version, diags +} diff --git a/configs/configload/loader_load_test.go b/configs/configload/loader_load_test.go new file mode 100644 index 0000000000..6c3a0a9408 --- /dev/null +++ b/configs/configload/loader_load_test.go @@ -0,0 +1,60 @@ +package configload + +import ( + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/zclconf/go-cty/cty" + + "github.com/hashicorp/terraform/configs" +) + +func TestLoaderLoadConfig_okay(t *testing.T) { + fixtureDir := filepath.Clean("test-fixtures/already-installed") + loader, err := NewLoader(&Config{ + ModulesDir: filepath.Join(fixtureDir, ".terraform/modules"), + }) + if err != nil { + t.Fatalf("unexpected error from NewLoader: %s", err) + } + + cfg, diags := loader.LoadConfig(fixtureDir) + assertNoDiagnostics(t, diags) + if cfg == nil { + t.Fatalf("config is nil; want non-nil") + } + + var gotPaths []string + cfg.DeepEach(func(c *configs.Config) { + gotPaths = append(gotPaths, strings.Join(c.Path, ".")) + }) + sort.Strings(gotPaths) + wantPaths := []string{ + "", // root module + "child_a", + "child_a.child_c", + "child_b", + "child_b.child_d", + } + + if !reflect.DeepEqual(gotPaths, wantPaths) { + t.Fatalf("wrong module paths\ngot: %swant %s", spew.Sdump(gotPaths), spew.Sdump(wantPaths)) + } + + t.Run("child_a.child_c output", func(t *testing.T) { + output := cfg.Children["child_a"].Children["child_c"].Module.Outputs["hello"] + got, diags := output.Expr.Value(nil) + assertNoDiagnostics(t, diags) + assertResultCtyEqual(t, got, cty.StringVal("Hello from child_c")) + }) + t.Run("child_b.child_d output", func(t *testing.T) { + output := cfg.Children["child_b"].Children["child_d"].Module.Outputs["hello"] + got, diags := output.Expr.Value(nil) + assertNoDiagnostics(t, diags) + assertResultCtyEqual(t, got, cty.StringVal("Hello from child_d")) + }) +} diff --git a/configs/configload/loader_test.go b/configs/configload/loader_test.go new file mode 100644 index 0000000000..512b99f3fa --- /dev/null +++ b/configs/configload/loader_test.go @@ -0,0 +1,61 @@ +package configload + +import ( + "reflect" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/hashicorp/hcl2/hcl" + "github.com/zclconf/go-cty/cty" +) + +func assertNoDiagnostics(t *testing.T, diags hcl.Diagnostics) bool { + t.Helper() + return assertDiagnosticCount(t, diags, 0) +} + +func assertDiagnosticCount(t *testing.T, diags hcl.Diagnostics, want int) bool { + t.Helper() + if len(diags) != 0 { + t.Errorf("wrong number of diagnostics %d; want %d", len(diags), want) + for _, diag := range diags { + t.Logf("- %s", diag) + } + return true + } + return false +} + +func assertDiagnosticSummary(t *testing.T, diags hcl.Diagnostics, want string) bool { + t.Helper() + + for _, diag := range diags { + if diag.Summary == want { + return false + } + } + + t.Errorf("missing diagnostic summary %q", want) + for _, diag := range diags { + t.Logf("- %s", diag) + } + return true +} + +func assertResultDeepEqual(t *testing.T, got, want interface{}) bool { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Errorf("wrong result\ngot: %swant: %s", spew.Sdump(got), spew.Sdump(want)) + return true + } + return false +} + +func assertResultCtyEqual(t *testing.T, got, want cty.Value) bool { + t.Helper() + if !got.RawEquals(want) { + t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, want) + return true + } + return false +} diff --git a/configs/configload/module_manifest.go b/configs/configload/module_manifest.go new file mode 100644 index 0000000000..8ecc720a74 --- /dev/null +++ b/configs/configload/module_manifest.go @@ -0,0 +1,128 @@ +package configload + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + version "github.com/hashicorp/go-version" +) + +// moduleRecord represents some metadata about an installed module, as part +// of a moduleManifest. +type moduleRecord struct { + // Key is a unique identifier for this particular module, based on its + // position within the static module tree. + Key string `json:"Key"` + + // SourceAddr is the source address given for this module in configuration. + // This is used only to detect if the source was changed in configuration + // since the module was last installed, which means that the installer + // must re-install it. + SourceAddr string `json:"Source"` + + // Version is the exact version of the module, which results from parsing + // VersionStr. nil for un-versioned modules. + Version *version.Version `json:"-"` + + // VersionStr is the version specifier string. This is used only for + // serialization in snapshots and should not be accessed or updated + // by any other codepaths; use "Version" instead. + VersionStr string `json:"Version"` + + // Dir is the path to the local directory where the module is installed. + Dir string `json:"Dir"` +} + +// moduleManifest is a map used to keep track of the filesystem locations +// and other metadata about installed modules. +// +// The configuration loader refers to this, while the module installer updates +// it to reflect any changes to the installed modules. +type moduleManifest map[string]moduleRecord + +func manifestKey(path []string) string { + return strings.Join(path, ".") +} + +// manifestSnapshotFile is an internal struct used only to assist in our JSON +// serializtion of manifest snapshots. It should not be used for any other +// purposes. +type manifestSnapshotFile struct { + Records []moduleRecord `json:"Modules"` +} + +const manifestFilename = "modules.json" + +func (m *moduleMgr) manifestSnapshotPath() string { + return filepath.Join(m.Dir, manifestFilename) +} + +// readModuleManifestSnapshot loads a manifest snapshot from the filesystem. +func (m *moduleMgr) readModuleManifestSnapshot() error { + src, err := m.FS.ReadFile(m.manifestSnapshotPath()) + if err != nil { + if os.IsNotExist(err) { + // We'll treat a missing file as an empty manifest + m.manifest = make(moduleManifest) + return nil + } + return err + } + + if len(src) == 0 { + // This should never happen, but we'll tolerate it as if it were + // a valid empty JSON object. + m.manifest = make(moduleManifest) + return nil + } + + var read manifestSnapshotFile + err = json.Unmarshal(src, &read) + + new := make(moduleManifest) + for _, record := range read.Records { + if record.VersionStr != "" { + record.Version, err = version.NewVersion(record.VersionStr) + if err != nil { + return fmt.Errorf("invalid version %q for %s: %s", record.VersionStr, record.Key, err) + } + } + if _, exists := new[record.Key]; exists { + // This should never happen in any valid file, so we'll catch it + // and report it to avoid confusing/undefined behavior if the + // snapshot file was edited incorrectly outside of Terraform. + return fmt.Errorf("snapshot file contains two records for path %s", record.Key) + } + new[record.Key] = record + } + + m.manifest = new + + return nil +} + +// writeModuleManifestSnapshot writes a snapshot of the current manifest +// to the filesystem. +// +// The caller must guarantee no concurrent modifications of the manifest for +// the duration of a call to this function, or the behavior is undefined. +func (m *moduleMgr) writeModuleManifestSnapshot() error { + var write manifestSnapshotFile + + for _, record := range m.manifest { + // Make sure VersionStr is in sync with Version, since we encourage + // callers to manipulate Version and ignore VersionStr. + record.VersionStr = record.Version.String() + write.Records = append(write.Records, record) + } + + src, err := json.Marshal(write) + if err != nil { + return err + } + + return m.FS.WriteFile(m.manifestSnapshotPath(), src, os.ModePerm) +} diff --git a/configs/configload/module_mgr.go b/configs/configload/module_mgr.go new file mode 100644 index 0000000000..5856c2ec02 --- /dev/null +++ b/configs/configload/module_mgr.go @@ -0,0 +1,35 @@ +package configload + +import ( + "github.com/hashicorp/terraform/registry" + "github.com/hashicorp/terraform/svchost/auth" + "github.com/hashicorp/terraform/svchost/disco" + "github.com/spf13/afero" +) + +type moduleMgr struct { + FS afero.Afero + + // Dir is the path where descendent modules are (or will be) installed. + Dir string + + // Services is a service discovery client that will be used to find + // remote module registry endpoints. This object may be pre-loaded with + // cached discovery information. + Services *disco.Disco + + // Creds provides optional credentials for communicating with service hosts. + Creds auth.CredentialsSource + + // Registry is a client for the module registry protocol, which is used + // when a module is requested from a registry source. + Registry *registry.Client + + // manifest tracks the currently-installed modules for this manager. + // + // The loader may read this. Only the installer may write to it, and + // after a set of updates are completed the installer must call + // writeModuleManifestSnapshot to persist a snapshot of the manifest + // to disk for use on subsequent runs. + manifest moduleManifest +} diff --git a/configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_a.tf b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_a.tf new file mode 100644 index 0000000000..2f4d0f1a0b --- /dev/null +++ b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_a.tf @@ -0,0 +1,4 @@ + +module "child_c" { + source = "./child_c" +} diff --git a/configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_c/child_c.tf b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_c/child_c.tf new file mode 100644 index 0000000000..785d98d98a --- /dev/null +++ b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_c/child_c.tf @@ -0,0 +1,4 @@ + +output "hello" { + value = "Hello from child_c" +} diff --git a/configs/configload/test-fixtures/already-installed/.terraform/modules/child_b.child_d/child_d.tf b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_b.child_d/child_d.tf new file mode 100644 index 0000000000..145576a365 --- /dev/null +++ b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_b.child_d/child_d.tf @@ -0,0 +1,4 @@ + +output "hello" { + value = "Hello from child_d" +} diff --git a/configs/configload/test-fixtures/already-installed/.terraform/modules/child_b/child_b.tf b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_b/child_b.tf new file mode 100644 index 0000000000..4a1b247d39 --- /dev/null +++ b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_b/child_b.tf @@ -0,0 +1,5 @@ + +module "child_d" { + source = "example.com/foo/bar_d/baz" + # Intentionally no version here +} diff --git a/configs/configload/test-fixtures/already-installed/.terraform/modules/modules.json b/configs/configload/test-fixtures/already-installed/.terraform/modules/modules.json new file mode 100644 index 0000000000..454e719ab6 --- /dev/null +++ b/configs/configload/test-fixtures/already-installed/.terraform/modules/modules.json @@ -0,0 +1 @@ +{"Modules":[{"Key":"","Source":"","Dir":"test-fixtures/already-installed"},{"Key":"child_a","Source":"example.com/foo/bar_a/baz","Version":"1.0.1","Dir":"test-fixtures/already-installed/.terraform/modules/child_a"},{"Key":"child_b","Source":"example.com/foo/bar_b/baz","Version":"1.0.0","Dir":"test-fixtures/already-installed/.terraform/modules/child_b"},{"Key":"child_a.child_c","Source":"./child_c","Dir":"test-fixtures/already-installed/.terraform/modules/child_a/child_c"},{"Key":"child_b.child_d","Source":"example.com/foo/bar_d/baz","Version":"1.2.0","Dir":"test-fixtures/already-installed/.terraform/modules/child_b.child_d"}]} \ No newline at end of file diff --git a/configs/configload/test-fixtures/already-installed/root.tf b/configs/configload/test-fixtures/already-installed/root.tf new file mode 100644 index 0000000000..8a4473942d --- /dev/null +++ b/configs/configload/test-fixtures/already-installed/root.tf @@ -0,0 +1,10 @@ + +module "child_a" { + source = "example.com/foo/bar_a/baz" + version = ">= 1.0.0" +} + +module "child_b" { + source = "example.com/foo/bar_b/baz" + version = ">= 1.0.0" +}