From a5c86aeff6af533d6f295c806eadc432d4a2cd9c Mon Sep 17 00:00:00 2001 From: James Bardin Date: Thu, 19 Oct 2017 16:32:19 -0400 Subject: [PATCH] Use the new regsrc and response packages Adds basic detector for registry module source strings. While this isn't a thorough validation, this will eliminate anything that is definitely not a registry module, and split out our host and module id strings. lookupModuleVersions interrogates the registry for the available versions of a particular module and the tree of dependencies. --- config/module/detector_test.go | 112 +++++++++++++++++++++++++++++++++ config/module/get.go | 7 +-- config/module/get_test.go | 2 +- config/module/registry.go | 95 ++++++++++++++++++++++++++++ config/module/versions.go | 95 ++++++++++++++++++++++++++++ config/module/versions_test.go | 60 ++++++++++++++++++ 6 files changed, 366 insertions(+), 5 deletions(-) create mode 100644 config/module/detector_test.go create mode 100644 config/module/registry.go create mode 100644 config/module/versions.go create mode 100644 config/module/versions_test.go diff --git a/config/module/detector_test.go b/config/module/detector_test.go new file mode 100644 index 0000000000..10cc057fcc --- /dev/null +++ b/config/module/detector_test.go @@ -0,0 +1,112 @@ +package module + +import ( + "fmt" + "testing" + + "github.com/hashicorp/terraform/registry/regsrc" +) + +func TestParseRegistrySource(t *testing.T) { + for _, tc := range []struct { + source string + host string + id string + err bool + notRegistry bool + }{ + { // simple source id + source: "namespace/id/provider", + id: "namespace/id/provider", + }, + { // source with hostname + source: "registry.com/namespace/id/provider", + host: "registry.com", + id: "namespace/id/provider", + }, + { // source with hostname and port + source: "registry.com:4443/namespace/id/provider", + host: "registry.com:4443", + id: "namespace/id/provider", + }, + { // too many parts + source: "registry.com/namespace/id/provider/extra", + notRegistry: true, + }, + { // local path + source: "./local/file/path", + notRegistry: true, + }, + { // local path with hostname + source: "./registry.com/namespace/id/provider", + notRegistry: true, + }, + { // full URL + source: "https://example.com/foo/bar/baz", + notRegistry: true, + }, + { // punycode host not allowed in source + source: "xn--80akhbyknj4f.com/namespace/id/provider", + err: true, + }, + { // simple source id with subdir + source: "namespace/id/provider//subdir", + id: "namespace/id/provider", + }, + { // source with hostname and subdir + source: "registry.com/namespace/id/provider//subdir", + host: "registry.com", + id: "namespace/id/provider", + }, + { // source with hostname + source: "registry.com/namespace/id/provider", + host: "registry.com", + id: "namespace/id/provider", + }, + { // we special case github + source: "github.com/namespace/id/provider", + notRegistry: true, + }, + { // we special case github ssh + source: "git@github.com:namespace/id/provider", + notRegistry: true, + }, + { // we special case bitbucket + source: "bitbucket.org/namespace/id/provider", + notRegistry: true, + }, + } { + t.Run(tc.source, func(t *testing.T) { + mod, err := regsrc.ParseModuleSource(tc.source) + if tc.notRegistry { + if err != regsrc.ErrInvalidModuleSource { + t.Fatalf("%q should not be a registry source, got err %v", tc.source, err) + } + return + } + + if tc.err { + if err == nil { + t.Fatal("expected error") + } + return + } + + if err != nil { + t.Fatal(err) + } + + id := fmt.Sprintf("%s/%s/%s", mod.RawNamespace, mod.RawName, mod.RawProvider) + + if tc.host != "" { + if mod.RawHost.Normalized() != tc.host { + t.Fatalf("expected host %q, got %q", tc.host, mod.RawHost) + } + } + + if tc.id != id { + t.Fatalf("expected id %q, got %q", tc.id, id) + } + }) + } +} diff --git a/config/module/get.go b/config/module/get.go index ac078666d8..26a31fa845 100644 --- a/config/module/get.go +++ b/config/module/get.go @@ -65,8 +65,7 @@ func GetCopy(dst, src string) error { } const ( - registryAPI = "https://registry.terraform.io/v1/modules" - xTerraformGet = "X-Terraform-Get" + registryAPI = "https://registry.terraform.io/v1/modules" ) var detectors = []getter.Detector{ @@ -79,7 +78,7 @@ var detectors = []getter.Detector{ // these prefixes can't be registry IDs // "http", "../", "./", "/", "getter::", etc -var skipRegistry = regexp.MustCompile(`^(http|[.]{1,2}/|/|[A-Za-z0-9]+::)`).MatchString +var oldSkipRegistry = regexp.MustCompile(`^(http|[.]{1,2}/|/|[A-Za-z0-9]+::)`).MatchString // registryDetector implements getter.Detector to detect Terraform Registry modules. // If a path looks like a registry module identifier, attempt to locate it in @@ -95,7 +94,7 @@ type registryDetector struct { func (d registryDetector) Detect(src, _ string) (string, bool, error) { // the namespace can't start with "http", a relative or absolute path, or // contain a go-getter "forced getter" - if skipRegistry(src) { + if oldSkipRegistry(src) { return "", false, nil } diff --git a/config/module/get_test.go b/config/module/get_test.go index b73520c43e..de2a13bab5 100644 --- a/config/module/get_test.go +++ b/config/module/get_test.go @@ -91,7 +91,7 @@ func mockRegistry() *httptest.Server { location = fmt.Sprintf("file://%s/%s", wd, location) } - w.Header().Set(xTerraformGet, location) + w.Header().Set("X-Terraform-Get", location) w.WriteHeader(http.StatusNoContent) // no body return diff --git a/config/module/registry.go b/config/module/registry.go new file mode 100644 index 0000000000..2753871944 --- /dev/null +++ b/config/module/registry.go @@ -0,0 +1,95 @@ +package module + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "net/url" + "time" + + cleanhttp "github.com/hashicorp/go-cleanhttp" + + "github.com/hashicorp/terraform/registry/regsrc" + "github.com/hashicorp/terraform/registry/response" + "github.com/hashicorp/terraform/svchost" + "github.com/hashicorp/terraform/svchost/disco" + "github.com/hashicorp/terraform/version" +) + +const ( + defaultRegistry = "registry.terraform.io" + defaultApiPath = "/v1/modules" + registryServiceID = "registry.v1" + xTerraformGet = "X-Terraform-Get" + xTerraformVersion = "X-Terraform-Version" + requestTimeout = 10 * time.Second + serviceID = "modules.v1" +) + +var ( + client *http.Client + tfVersion = version.String() + regDisco = disco.NewDisco() +) + +func init() { + client = cleanhttp.DefaultPooledClient() + client.Timeout = requestTimeout +} + +type errModuleNotFound string + +func (e errModuleNotFound) Error() string { + return `module "` + string(e) + `" not found` +} + +// Lookup module versions in the registry. +func lookupModuleVersions(module *regsrc.Module) (*response.ModuleVersions, error) { + if module.RawHost == nil { + module.RawHost = regsrc.NewFriendlyHost(defaultRegistry) + } + + regUrl := regDisco.DiscoverServiceURL(svchost.Hostname(module.RawHost.Normalized()), serviceID) + if regUrl == nil { + regUrl = &url.URL{ + Scheme: "https", + Host: module.RawHost.String(), + Path: defaultApiPath, + } + } + + location := fmt.Sprintf("%s/%s/%s/%s/versions", regUrl, module.RawNamespace, module.RawName, module.RawProvider) + log.Printf("[DEBUG] fetching module versions from %q", location) + + req, err := http.NewRequest("GET", location, nil) + if err != nil { + return nil, err + } + + req.Header.Set(xTerraformVersion, tfVersion) + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + // OK + case http.StatusNotFound: + return nil, errModuleNotFound(module.String()) + default: + return nil, fmt.Errorf("error looking up module versions: %s", resp.Status) + } + + var versions response.ModuleVersions + + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&versions); err != nil { + return nil, err + } + + return &versions, nil +} diff --git a/config/module/versions.go b/config/module/versions.go new file mode 100644 index 0000000000..8348d4b195 --- /dev/null +++ b/config/module/versions.go @@ -0,0 +1,95 @@ +package module + +import ( + "errors" + "fmt" + "sort" + + version "github.com/hashicorp/go-version" + "github.com/hashicorp/terraform/registry/response" +) + +const anyVersion = ">=0.0.0" + +// return the newest version that satisfies the provided constraint +func newest(versions []string, constraint string) (string, error) { + if constraint == "" { + constraint = anyVersion + } + cs, err := version.NewConstraint(constraint) + if err != nil { + return "", err + } + + switch len(versions) { + case 0: + return "", errors.New("no versions found") + case 1: + v, err := version.NewVersion(versions[0]) + if err != nil { + return "", err + } + + if !cs.Check(v) { + return "", fmt.Errorf("no version found matching constraint %q", constraint) + } + return versions[0], nil + } + + sort.Slice(versions, func(i, j int) bool { + // versions should have already been validated + // sort invalid version strings to the end + iv, err := version.NewVersion(versions[i]) + if err != nil { + return true + } + jv, err := version.NewVersion(versions[j]) + if err != nil { + return true + } + return iv.GreaterThan(jv) + }) + + // versions are now in order, so just find the first which satisfies the + // constraint + for i := range versions { + v, err := version.NewVersion(versions[i]) + if err != nil { + continue + } + if cs.Check(v) { + return versions[i], nil + } + } + + return "", nil +} + +// return the newest *moduleVersion that matches the given constraint +// TODO: reconcile these two types and newest* functions +func newestVersion(moduleVersions []*response.ModuleVersion, constraint string) (*response.ModuleVersion, error) { + var versions []string + modules := make(map[string]*response.ModuleVersion) + + for _, m := range moduleVersions { + versions = append(versions, m.Version) + modules[m.Version] = m + } + + match, err := newest(versions, constraint) + return modules[match], err +} + +// return the newest moduleRecord that matches the given constraint +func newestRecord(moduleVersions []moduleRecord, constraint string) (moduleRecord, error) { + var versions []string + modules := make(map[string]moduleRecord) + + for _, m := range moduleVersions { + versions = append(versions, m.Version) + modules[m.Version] = m + } + + match, err := newest(versions, constraint) + return modules[match], err +} diff --git a/config/module/versions_test.go b/config/module/versions_test.go new file mode 100644 index 0000000000..b7ff6e6271 --- /dev/null +++ b/config/module/versions_test.go @@ -0,0 +1,60 @@ +package module + +import ( + "testing" + + "github.com/hashicorp/terraform/registry/response" +) + +func TestNewestModuleVersion(t *testing.T) { + mpv := &response.ModuleProviderVersions{ + Source: "registry/test/module", + Versions: []*response.ModuleVersion{ + {Version: "0.0.4"}, + {Version: "0.3.1"}, + {Version: "2.0.1"}, + {Version: "1.2.0"}, + }, + } + + m, err := newestVersion(mpv.Versions, "") + if err != nil { + t.Fatal(err) + } + + expected := "2.0.1" + if m.Version != expected { + t.Fatalf("expected version %q, got %q", expected, m.Version) + } + + // now with a constraint + m, err = newestVersion(mpv.Versions, "~>1.0") + if err != nil { + t.Fatal(err) + } + + expected = "1.2.0" + if m.Version != expected { + t.Fatalf("expected version %q, got %q", expected, m.Version) + } +} + +func TestNewestInvalidModuleVersion(t *testing.T) { + mpv := &response.ModuleProviderVersions{ + Source: "registry/test/module", + Versions: []*response.ModuleVersion{ + {Version: "WTF"}, + {Version: "2.0.1"}, + }, + } + + m, err := newestVersion(mpv.Versions, "") + if err != nil { + t.Fatal(err) + } + + expected := "2.0.1" + if m.Version != expected { + t.Fatalf("expected version %q, got %q", expected, m.Version) + } +}