config/module: support HTTP protocol

config/module/get.go

@@ -30,10 +30,14 @@ var Getters map[string]Getter
 var forcedRegexp = regexp.MustCompile(`^([A-Za-z]+)::(.+)$`)
 
 func init() {
+	httpGetter := new(HttpGetter)
+
 	Getters = map[string]Getter{
-		"file": new(FileGetter),
-		"git":  new(GitGetter),
-		"hg":   new(HgGetter),
+		"file":  new(FileGetter),
+		"git":   new(GitGetter),
+		"hg":    new(HgGetter),
+		"http":  httpGetter,
+		"https": httpGetter,
 	}
 }
 

config/module/get_http.go

@@ -0,0 +1,128 @@
+package module
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+// HttpGetter is a Getter implementation that will download a module from
+// an HTTP endpoint. The protocol for downloading a module from an HTTP
+// endpoing is as follows:
+//
+// An HTTP GET request is made to the URL with the additional GET parameter
+// "terraform-get=1". This lets you handle that scenario specially if you
+// wish. The response must be a 2xx.
+//
+// First, a header is looked for "X-Terraform-Get" which should contain
+// a source URL to download.
+//
+// If the header is not present, then a meta tag is searched for named
+// "terraform-get" and the content should be a source URL.
+//
+// The source URL, whether from the header or meta tag, must be a fully
+// formed URL. The shorthand syntax of "github.com/foo/bar" or relative
+// paths are not allowed.
+type HttpGetter struct{}
+
+func (g *HttpGetter) Get(dst string, u *url.URL) error {
+	// Copy the URL so we can modify it
+	var newU url.URL = *u
+	u = &newU
+
+	// Add terraform-get to the parameter.
+	q := u.Query()
+	q.Add("terraform-get", "1")
+	u.RawQuery = q.Encode()
+
+	// Get the URL
+	resp, err := http.Get(u.String())
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return fmt.Errorf("bad response code: %d", resp.StatusCode)
+	}
+
+	// Extract the source URL
+	var source string
+	if v := resp.Header.Get("X-Terraform-Get"); v != "" {
+		source = v
+	} else {
+		source, err = g.parseMeta(resp.Body)
+		if err != nil {
+			return err
+		}
+	}
+	if source == "" {
+		return fmt.Errorf("no source URL was returned")
+	}
+
+	// Get it!
+	return Get(dst, source)
+}
+
+// parseMeta looks for the first meta tag in the given reader that
+// will give us the source URL.
+func (g *HttpGetter) parseMeta(r io.Reader) (string, error) {
+	d := xml.NewDecoder(r)
+	d.CharsetReader = charsetReader
+	d.Strict = false
+	var err error
+	var t xml.Token
+	for {
+		t, err = d.Token()
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			return "", err
+		}
+		if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
+			return "", nil
+		}
+		if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
+			return "", nil
+		}
+		e, ok := t.(xml.StartElement)
+		if !ok || !strings.EqualFold(e.Name.Local, "meta") {
+			continue
+		}
+		if attrValue(e.Attr, "name") != "terraform-get" {
+			continue
+		}
+		if f := attrValue(e.Attr, "content"); f != "" {
+			return f, nil
+		}
+	}
+}
+
+// attrValue returns the attribute value for the case-insensitive key
+// `name', or the empty string if nothing is found.
+func attrValue(attrs []xml.Attr, name string) string {
+	for _, a := range attrs {
+		if strings.EqualFold(a.Name.Local, name) {
+			return a.Value
+		}
+	}
+	return ""
+}
+
+// charsetReader returns a reader for the given charset. Currently
+// it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful
+// error which is printed by go get, so the user can find why the package
+// wasn't downloaded if the encoding is not supported. Note that, in
+// order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters
+// greater than 0x7f are not rejected).
+func charsetReader(charset string, input io.Reader) (io.Reader, error) {
+	switch strings.ToLower(charset) {
+	case "ascii":
+		return input, nil
+	default:
+		return nil, fmt.Errorf("can't decode XML document using charset %q", charset)
+	}
+}

config/module/get_http_test.go

@@ -0,0 +1,126 @@
+package module
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestHttpGetter_impl(t *testing.T) {
+	var _ Getter = new(HttpGetter)
+}
+
+func TestHttpGetter_header(t *testing.T) {
+	ln := testHttpServer(t)
+	defer ln.Close()
+
+	g := new(HttpGetter)
+	dst := tempDir(t)
+
+	var u url.URL
+	u.Scheme = "http"
+	u.Host = ln.Addr().String()
+	u.Path = "/header"
+
+	// Get it!
+	if err := g.Get(dst, &u); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Verify the main file exists
+	mainPath := filepath.Join(dst, "main.tf")
+	if _, err := os.Stat(mainPath); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
+func TestHttpGetter_meta(t *testing.T) {
+	ln := testHttpServer(t)
+	defer ln.Close()
+
+	g := new(HttpGetter)
+	dst := tempDir(t)
+
+	var u url.URL
+	u.Scheme = "http"
+	u.Host = ln.Addr().String()
+	u.Path = "/meta"
+
+	// Get it!
+	if err := g.Get(dst, &u); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Verify the main file exists
+	mainPath := filepath.Join(dst, "main.tf")
+	if _, err := os.Stat(mainPath); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
+func TestHttpGetter_none(t *testing.T) {
+	ln := testHttpServer(t)
+	defer ln.Close()
+
+	g := new(HttpGetter)
+	dst := tempDir(t)
+
+	var u url.URL
+	u.Scheme = "http"
+	u.Host = ln.Addr().String()
+	u.Path = "/none"
+
+	// Get it!
+	if err := g.Get(dst, &u); err == nil {
+		t.Fatal("should error")
+	}
+}
+
+func testHttpServer(t *testing.T) net.Listener {
+	ln, err := net.Listen("tcp", ":0")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/header", testHttpHandlerHeader)
+	mux.HandleFunc("/meta", testHttpHandlerMeta)
+
+	var server http.Server
+	server.Handler = mux
+	go server.Serve(ln)
+
+	return ln
+}
+
+func testHttpHandlerHeader(w http.ResponseWriter, r *http.Request) {
+	w.Header().Add("X-Terraform-Get", testModuleURL("basic").String())
+	w.WriteHeader(200)
+}
+
+func testHttpHandlerMeta(w http.ResponseWriter, r *http.Request) {
+	w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic").String())))
+}
+
+func testHttpHandlerNone(w http.ResponseWriter, r *http.Request) {
+	w.Write([]byte(testHttpNoneStr))
+}
+
+const testHttpMetaStr = `
+<html>
+<head>
+<meta name="terraform-get" content="%s">
+</head>
+</html>
+`
+
+const testHttpNoneStr = `
+<html>
+<head>
+</head>
+</html>
+`