From a35a9262d4b4c5a8fe2fa01e3712f2b8f0e7fef8 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Sep 2014 16:17:29 -0700 Subject: [PATCH] config/module: detectors, some more work on Tree --- config/module/detect.go | 50 +++++++++++++++++++ config/module/detect_file.go | 28 +++++++++++ config/module/detect_file_test.go | 32 +++++++++++++ config/module/module.go | 1 + config/module/module_test.go | 4 ++ config/module/tree.go | 79 ++++++++++++++++++++++++++++--- config/module/tree_test.go | 9 +++- 7 files changed, 196 insertions(+), 7 deletions(-) create mode 100644 config/module/detect.go create mode 100644 config/module/detect_file.go create mode 100644 config/module/detect_file_test.go diff --git a/config/module/detect.go b/config/module/detect.go new file mode 100644 index 0000000000..855b34991f --- /dev/null +++ b/config/module/detect.go @@ -0,0 +1,50 @@ +package module + +import ( + "fmt" + "net/url" +) + +// Detector defines the interface that an invalid URL or a URL with a blank +// scheme is passed through in order to determine if its shorthand for +// something else well-known. +type Detector interface { + // Detect will detect whether the string matches a known pattern to + // turn it into a proper URL. + Detect(string, string) (string, bool, error) +} + +// Detectors is the list of detectors that are tried on an invalid URL. +// This is also the order they're tried (index 0 is first). +var Detectors []Detector + +func init() { + Detectors = []Detector{ + new(FileDetector), + } +} + +// Detect turns a source string into another source string if it is +// detected to be of a known pattern. +// +// This is safe to be called with an already valid source string: Detect +// will just return it. +func Detect(src string, pwd string) (string, error) { + u, err := url.Parse(src) + if err == nil && u.Scheme != "" { + // Valid URL + return src, nil + } + + for _, d := range Detectors { + result, ok, err := d.Detect(src, pwd) + if err != nil { + return "", err + } + if ok { + return result, nil + } + } + + return "", fmt.Errorf("invalid source string: %s", src) +} diff --git a/config/module/detect_file.go b/config/module/detect_file.go new file mode 100644 index 0000000000..7f1632faa5 --- /dev/null +++ b/config/module/detect_file.go @@ -0,0 +1,28 @@ +package module + +import ( + "fmt" + "path/filepath" +) + +// FileDetector implements Detector to detect file paths. +type FileDetector struct{} + +func (d *FileDetector) Detect(src, pwd string) (string, bool, error) { + if len(src) == 0 { + return "", false, nil + } + + // Make sure we're using "/" even on Windows. URLs are "/"-based. + src = filepath.ToSlash(src) + if !filepath.IsAbs(src) { + src = filepath.Join(pwd, src) + } + + // Make sure that we don't start with "/" since we add that below + if src[0] == '/' { + src = src[1:] + } + + return fmt.Sprintf("file:///%s", src), true, nil +} diff --git a/config/module/detect_file_test.go b/config/module/detect_file_test.go new file mode 100644 index 0000000000..0f76bc05bb --- /dev/null +++ b/config/module/detect_file_test.go @@ -0,0 +1,32 @@ +package module + +import ( + "testing" +) + +func TestFileDetector(t *testing.T) { + cases := []struct { + Input string + Output string + }{ + {"./foo", "file:///pwd/foo"}, + {"foo", "file:///pwd/foo"}, + {"/foo", "file:///foo"}, + } + + pwd := "/pwd" + f := new(FileDetector) + for i, tc := range cases { + output, ok, err := f.Detect(tc.Input, pwd) + if err != nil { + t.Fatalf("err: %s", err) + } + if !ok { + t.Fatal("not ok") + } + + if output != tc.Output { + t.Fatalf("%d: bad: %#v", i, output) + } + } +} diff --git a/config/module/module.go b/config/module/module.go index f8649f6e9d..6d01ed85de 100644 --- a/config/module/module.go +++ b/config/module/module.go @@ -4,4 +4,5 @@ package module type Module struct { Name string Source string + Dir string } diff --git a/config/module/module_test.go b/config/module/module_test.go index 2ce7229de5..41f15e315f 100644 --- a/config/module/module_test.go +++ b/config/module/module_test.go @@ -45,3 +45,7 @@ func testModule(n string) string { url.Path = p return url.String() } + +func testStorage(t *testing.T) Storage { + return &FolderStorage{StorageDir: tempDir(t)} +} diff --git a/config/module/tree.go b/config/module/tree.go index 9a64a7dc8b..8cb5b967ae 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -1,6 +1,9 @@ package module import ( + "fmt" + "sync" + "github.com/hashicorp/terraform/config" ) @@ -10,8 +13,9 @@ import ( // all the modules without getting, flatten the tree into something // Terraform can use, etc. type Tree struct { - Config *config.Config - Children []*Tree + config *config.Config + children []*Tree + lock sync.Mutex } // GetMode is an enum that describes how modules are loaded. @@ -35,7 +39,15 @@ const ( // NewTree returns a new Tree for the given config structure. func NewTree(c *config.Config) *Tree { - return &Tree{Config: c} + return &Tree{config: c} +} + +// Children returns the children of this tree (the modules that are +// imported by this root). +// +// This will only return a non-nil value after Load is called. +func (t *Tree) Children() []*Tree { + return nil } // Flatten takes the entire module tree and flattens it into a single @@ -54,10 +66,10 @@ func (t *Tree) Flatten() (*config.Config, error) { // This is only the imports of _this_ level of the tree. To retrieve the // full nested imports, you'll have to traverse the tree. func (t *Tree) Modules() []*Module { - result := make([]*Module, len(t.Config.Modules)) - for i, m := range t.Config.Modules { + result := make([]*Module, len(t.config.Modules)) + for i, m := range t.config.Modules { result[i] = &Module{ - Name: m.Name, + Name: m.Name, Source: m.Source, } } @@ -70,11 +82,66 @@ func (t *Tree) Modules() []*Module { // The parameters are used to tell the tree where to find modules and // whether it can download/update modules along the way. // +// Calling this multiple times will reload the tree. +// // Various semantic-like checks are made along the way of loading since // module trees inherently require the configuration to be in a reasonably // sane state: no circular dependencies, proper module sources, etc. A full // suite of validations can be done by running Validate (after loading). func (t *Tree) Load(s Storage, mode GetMode) error { + t.lock.Lock() + defer t.lock.Unlock() + + // Reset the children if we have any + t.children = nil + + modules := t.Modules() + children := make([]*Tree, len(modules)) + + // Go through all the modules and get the directory for them. + update := mode == GetModeUpdate + for i, m := range modules { + source, err := Detect(m.Source, m.Dir) + if err != nil { + return fmt.Errorf("module %s: %s", m.Name, err) + } + + if mode > GetModeNone { + // Get the module since we specified we should + if err := s.Get(source, update); err != nil { + return err + } + } + + // Get the directory where this module is so we can load it + dir, ok, err := s.Dir(source) + if err != nil { + return err + } + if !ok { + return fmt.Errorf( + "module %s: not found, may need to be downloaded", m.Name) + } + + // Load the configuration + c, err := config.LoadDir(dir) + if err != nil { + return fmt.Errorf( + "module %s: %s", m.Name, err) + } + children[i] = NewTree(c) + } + + // Go through all the children and load them. + for _, c := range children { + if err := c.Load(s, mode); err != nil { + return err + } + } + + // Set our tree up + t.children = children + return nil } diff --git a/config/module/tree_test.go b/config/module/tree_test.go index 27fd1f06d4..fbb5e73fed 100644 --- a/config/module/tree_test.go +++ b/config/module/tree_test.go @@ -5,7 +5,14 @@ import ( "testing" ) -func TestTree(t *testing.T) { +func TestTree_Load(t *testing.T) { + tree := NewTree(testConfig(t, "basic")) + if err := tree.Load(testStorage(t), GetModeGet); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestTree_Modules(t *testing.T) { tree := NewTree(testConfig(t, "basic")) actual := tree.Modules()