Merge pull request #15828 from hashicorp/e2e-decoupling

tools/terraform-bundle: Add e2e tests

command/e2etest/init_test.go

@@ -1,8 +1,11 @@
 package e2etest
 
 import (
+	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/hashicorp/terraform/e2e"
 )
 
 func TestInitProviders(t *testing.T) {
@@ -15,7 +18,8 @@ func TestInitProviders(t *testing.T) {
 	// is to test the interaction with the real repository.
 	skipIfCannotAccessNetwork(t)
 
-	tf := newTerraform("template-provider")
+	fixturePath := filepath.Join("test-fixtures", "template-provider")
+	tf := e2e.NewBinary(terraformBin, fixturePath)
 	defer tf.Close()
 
 	stdout, stderr, err := tf.Run("init")

command/e2etest/main_test.go

@@ -1,16 +1,12 @@
 package e2etest
 
 import (
-	"bytes"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"testing"
 
-	tfcore "github.com/hashicorp/terraform/terraform"
+	"github.com/hashicorp/terraform/e2e"
 )
 
 var terraformBin string
@@ -37,29 +33,7 @@ func setup() func() {
 		return func() {}
 	}
 
-	tmpFile, err := ioutil.TempFile("", "terraform")
+	tmpFilename := e2e.GoBuild("github.com/hashicorp/terraform", "terraform")
-	if err != nil {
-		panic(err)
-	}
-	tmpFilename := tmpFile.Name()
-	if err = tmpFile.Close(); err != nil {
-		panic(err)
-	}
-
-	cmd := exec.Command(
-		"go", "build",
-		"-o", tmpFilename,
-		"github.com/hashicorp/terraform",
-	)
-	cmd.Stderr = os.Stderr
-	cmd.Stdout = os.Stdout
-
-	err = cmd.Run()
-	if err != nil {
-		// The go compiler will have already produced some error messages
-		// on stderr by the time we get here.
-		panic(fmt.Sprintf("failed to build terraform executable: %s", err))
-	}
 
 	// Make the executable available for use in tests
 	terraformBin = tmpFilename
@@ -81,209 +55,3 @@ func skipIfCannotAccessNetwork(t *testing.T) {
 		t.Skip("network access not allowed; use TF_ACC=1 to enable")
 	}
 }
-
-// Type terraform represents the combination of a compiled Terraform binary
-// and a temporary working directory to run it in.
-//
-// This is the main harness for tests in this package.
-type terraform struct {
-	bin string
-	dir string
-}
-
-// newTerraform prepares a temporary directory containing the files from the
-// given fixture and returns an instance of type terraform that can run
-// the generated Terraform binary in that directory.
-//
-// If the temporary directory cannot be created, a fixture of the given name
-// cannot be found, or if an error occurs while _copying_ the fixture files,
-// this function will panic. Tests should be written to assume that this
-// function always succeeds.
-func newTerraform(fixtureName string) *terraform {
-	tmpDir, err := ioutil.TempDir("", "terraform-e2etest")
-	if err != nil {
-		panic(err)
-	}
-
-	// For our purposes here we do a very simplistic file copy that doesn't
-	// attempt to preserve file permissions, attributes, alternate data
-	// streams, etc. Since we only have to deal with our own fixtures in
-	// the test-fixtures subdir, we know we don't need to deal with anything
-	// of this nature.
-	srcDir := filepath.Join("test-fixtures", fixtureName)
-	err = filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if path == srcDir {
-			// nothing to do at the root
-			return nil
-		}
-
-		srcFn := path
-
-		path, err = filepath.Rel(srcDir, path)
-		if err != nil {
-			return err
-		}
-
-		dstFn := filepath.Join(tmpDir, path)
-
-		if info.IsDir() {
-			return os.Mkdir(dstFn, os.ModePerm)
-		}
-
-		src, err := os.Open(srcFn)
-		if err != nil {
-			return err
-		}
-		dst, err := os.OpenFile(dstFn, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm)
-		if err != nil {
-			return err
-		}
-
-		_, err = io.Copy(dst, src)
-		if err != nil {
-			return err
-		}
-
-		if err := src.Close(); err != nil {
-			return err
-		}
-		if err := dst.Close(); err != nil {
-			return err
-		}
-
-		return nil
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	return &terraform{
-		bin: terraformBin,
-		dir: tmpDir,
-	}
-}
-
-// Cmd returns an exec.Cmd pre-configured to run the generated Terraform
-// binary with the given arguments in the temporary working directory.
-//
-// The returned object can be mutated by the caller to customize how the
-// process will be run, before calling Run.
-func (t *terraform) Cmd(args ...string) *exec.Cmd {
-	cmd := exec.Command(t.bin, args...)
-	cmd.Dir = t.dir
-	cmd.Env = os.Environ()
-
-	// Disable checkpoint since we don't want to harass that service when
-	// our tests run. (This does, of course, mean we can't actually do
-	// end-to-end testing of our Checkpoint interactions.)
-	cmd.Env = append(cmd.Env, "CHECKPOINT_DISABLE=1")
-
-	return cmd
-}
-
-// Run executes the generated Terraform binary with the given arguments
-// and returns the bytes that it wrote to both stdout and stderr.
-//
-// This is a simple way to run Terraform for non-interactive commands
-// that don't need any special environment variables. For more complex
-// situations, use Cmd and customize the command before running it.
-func (t *terraform) Run(args ...string) (stdout, stderr string, err error) {
-	cmd := t.Cmd(args...)
-	cmd.Stdin = nil
-	cmd.Stdout = &bytes.Buffer{}
-	cmd.Stderr = &bytes.Buffer{}
-	err = cmd.Run()
-	stdout = cmd.Stdout.(*bytes.Buffer).String()
-	stderr = cmd.Stderr.(*bytes.Buffer).String()
-	return
-}
-
-// Path returns a file path within the temporary working directory by
-// appending the given arguments as path segments.
-func (t *terraform) Path(parts ...string) string {
-	args := make([]string, len(parts)+1)
-	args[0] = t.dir
-	args = append(args, parts...)
-	return filepath.Join(args...)
-}
-
-// OpenFile is a helper for easily opening a file from the working directory
-// for reading.
-func (t *terraform) OpenFile(path ...string) (*os.File, error) {
-	flatPath := t.Path(path...)
-	return os.Open(flatPath)
-}
-
-// ReadFile is a helper for easily reading a whole file from the working
-// directory.
-func (t *terraform) ReadFile(path ...string) ([]byte, error) {
-	flatPath := t.Path(path...)
-	return ioutil.ReadFile(flatPath)
-}
-
-// FileExists is a helper for easily testing whether a particular file
-// exists in the working directory.
-func (t *terraform) FileExists(path ...string) bool {
-	flatPath := t.Path(path...)
-	_, err := os.Stat(flatPath)
-	return !os.IsNotExist(err)
-}
-
-// LocalState is a helper for easily reading the local backend's state file
-// terraform.tfstate from the working directory.
-func (t *terraform) LocalState() (*tfcore.State, error) {
-	f, err := t.OpenFile("terraform.tfstate")
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-	return tfcore.ReadState(f)
-}
-
-// Plan is a helper for easily reading a plan file from the working directory.
-func (t *terraform) Plan(path ...string) (*tfcore.Plan, error) {
-	f, err := t.OpenFile(path...)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-	return tfcore.ReadPlan(f)
-}
-
-// SetLocalState is a helper for easily writing to the file the local backend
-// uses for state in the working directory. This does not go through the
-// actual local backend code, so processing such as management of serials
-// does not apply and the given state will simply be written verbatim.
-func (t *terraform) SetLocalState(state *tfcore.State) error {
-	path := t.Path("terraform.tfstate")
-	f, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, os.ModePerm)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		err := f.Close()
-		if err != nil {
-			panic(fmt.Sprintf("failed to close state file after writing: %s", err))
-		}
-	}()
-
-	return tfcore.WriteState(state, f)
-}
-
-// Close cleans up the temporary resources associated with the object,
-// including its working directory. It is not valid to call Cmd or Run
-// after Close returns.
-//
-// This method does _not_ stop any running child processes. It's the
-// caller's responsibility to also terminate those _before_ closing the
-// underlying terraform object.
-//
-// This function is designed to run under "defer", so it doesn't actually
-// do any error handling and will leave dangling temporary files on disk
-// if any errors occur while cleaning up.
-func (t *terraform) Close() {
-	os.RemoveAll(t.dir)
-}

command/e2etest/primary_test.go

@@ -1,12 +1,14 @@
 package e2etest
 
 import (
+	"path/filepath"
 	"reflect"
 	"sort"
 	"strings"
 	"testing"
 
 	"github.com/davecgh/go-spew/spew"
+	"github.com/hashicorp/terraform/e2e"
 )
 
 // The tests in this file are for the "primary workflow", which includes
@@ -24,7 +26,8 @@ func TestPrimarySeparatePlan(t *testing.T) {
 	// allowed.
 	skipIfCannotAccessNetwork(t)
 
-	tf := newTerraform("full-workflow-null")
+	fixturePath := filepath.Join("test-fixtures", "full-workflow-null")
+	tf := e2e.NewBinary(terraformBin, fixturePath)
 	defer tf.Close()
 
 	//// INIT

command/e2etest/version_test.go

@@ -2,9 +2,11 @@ package e2etest
 
 import (
 	"fmt"
+	"path/filepath"
 	"strings"
 	"testing"
 
+	"github.com/hashicorp/terraform/e2e"
 	tfcore "github.com/hashicorp/terraform/terraform"
 )
 
@@ -16,7 +18,8 @@ func TestVersion(t *testing.T) {
 
 	t.Parallel()
 
-	tf := newTerraform("empty")
+	fixturePath := filepath.Join("test-fixtures", "empty")
+	tf := e2e.NewBinary(terraformBin, fixturePath)
 	defer tf.Close()
 
 	stdout, stderr, err := tf.Run("version")

e2e/e2e.go

@@ -0,0 +1,244 @@
+package e2e
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	tfcore "github.com/hashicorp/terraform/terraform"
+)
+
+// Type binary represents the combination of a compiled binary
+// and a temporary working directory to run it in.
+type binary struct {
+	binPath string
+	workDir string
+}
+
+// NewBinary prepares a temporary directory containing the files from the
+// given fixture and returns an instance of type binary that can run
+// the generated binary in that directory.
+//
+// If the temporary directory cannot be created, a fixture of the given name
+// cannot be found, or if an error occurs while _copying_ the fixture files,
+// this function will panic. Tests should be written to assume that this
+// function always succeeds.
+func NewBinary(binaryPath, workingDir string) *binary {
+	tmpDir, err := ioutil.TempDir("", "binary-e2etest")
+	if err != nil {
+		panic(err)
+	}
+
+	// For our purposes here we do a very simplistic file copy that doesn't
+	// attempt to preserve file permissions, attributes, alternate data
+	// streams, etc. Since we only have to deal with our own fixtures in
+	// the test-fixtures subdir, we know we don't need to deal with anything
+	// of this nature.
+	err = filepath.Walk(workingDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if path == workingDir {
+			// nothing to do at the root
+			return nil
+		}
+
+		srcFn := path
+
+		path, err = filepath.Rel(workingDir, path)
+		if err != nil {
+			return err
+		}
+
+		dstFn := filepath.Join(tmpDir, path)
+
+		if info.IsDir() {
+			return os.Mkdir(dstFn, os.ModePerm)
+		}
+
+		src, err := os.Open(srcFn)
+		if err != nil {
+			return err
+		}
+		dst, err := os.OpenFile(dstFn, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm)
+		if err != nil {
+			return err
+		}
+
+		_, err = io.Copy(dst, src)
+		if err != nil {
+			return err
+		}
+
+		if err := src.Close(); err != nil {
+			return err
+		}
+		if err := dst.Close(); err != nil {
+			return err
+		}
+
+		return nil
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return &binary{
+		binPath: binaryPath,
+		workDir: tmpDir,
+	}
+}
+
+// Cmd returns an exec.Cmd pre-configured to run the generated Terraform
+// binary with the given arguments in the temporary working directory.
+//
+// The returned object can be mutated by the caller to customize how the
+// process will be run, before calling Run.
+func (b *binary) Cmd(args ...string) *exec.Cmd {
+	cmd := exec.Command(b.binPath, args...)
+	cmd.Dir = b.workDir
+	cmd.Env = os.Environ()
+
+	// Disable checkpoint since we don't want to harass that service when
+	// our tests run. (This does, of course, mean we can't actually do
+	// end-to-end testing of our Checkpoint interactions.)
+	cmd.Env = append(cmd.Env, "CHECKPOINT_DISABLE=1")
+
+	return cmd
+}
+
+// Run executes the generated Terraform binary with the given arguments
+// and returns the bytes that it wrote to both stdout and stderr.
+//
+// This is a simple way to run Terraform for non-interactive commands
+// that don't need any special environment variables. For more complex
+// situations, use Cmd and customize the command before running it.
+func (b *binary) Run(args ...string) (stdout, stderr string, err error) {
+	cmd := b.Cmd(args...)
+	cmd.Stdin = nil
+	cmd.Stdout = &bytes.Buffer{}
+	cmd.Stderr = &bytes.Buffer{}
+	err = cmd.Run()
+	stdout = cmd.Stdout.(*bytes.Buffer).String()
+	stderr = cmd.Stderr.(*bytes.Buffer).String()
+	return
+}
+
+// Path returns a file path within the temporary working directory by
+// appending the given arguments as path segments.
+func (b *binary) Path(parts ...string) string {
+	args := make([]string, len(parts)+1)
+	args[0] = b.workDir
+	args = append(args, parts...)
+	return filepath.Join(args...)
+}
+
+// OpenFile is a helper for easily opening a file from the working directory
+// for reading.
+func (b *binary) OpenFile(path ...string) (*os.File, error) {
+	flatPath := b.Path(path...)
+	return os.Open(flatPath)
+}
+
+// ReadFile is a helper for easily reading a whole file from the working
+// directory.
+func (b *binary) ReadFile(path ...string) ([]byte, error) {
+	flatPath := b.Path(path...)
+	return ioutil.ReadFile(flatPath)
+}
+
+// FileExists is a helper for easily testing whether a particular file
+// exists in the working directory.
+func (b *binary) FileExists(path ...string) bool {
+	flatPath := b.Path(path...)
+	_, err := os.Stat(flatPath)
+	return !os.IsNotExist(err)
+}
+
+// LocalState is a helper for easily reading the local backend's state file
+// terraform.tfstate from the working directory.
+func (b *binary) LocalState() (*tfcore.State, error) {
+	f, err := b.OpenFile("terraform.tfstate")
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return tfcore.ReadState(f)
+}
+
+// Plan is a helper for easily reading a plan file from the working directory.
+func (b *binary) Plan(path ...string) (*tfcore.Plan, error) {
+	f, err := b.OpenFile(path...)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return tfcore.ReadPlan(f)
+}
+
+// SetLocalState is a helper for easily writing to the file the local backend
+// uses for state in the working directory. This does not go through the
+// actual local backend code, so processing such as management of serials
+// does not apply and the given state will simply be written verbatim.
+func (b *binary) SetLocalState(state *tfcore.State) error {
+	path := b.Path("terraform.tfstate")
+	f, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := f.Close()
+		if err != nil {
+			panic(fmt.Sprintf("failed to close state file after writing: %s", err))
+		}
+	}()
+
+	return tfcore.WriteState(state, f)
+}
+
+// Close cleans up the temporary resources associated with the object,
+// including its working directory. It is not valid to call Cmd or Run
+// after Close returns.
+//
+// This method does _not_ stop any running child processes. It's the
+// caller's responsibility to also terminate those _before_ closing the
+// underlying binary object.
+//
+// This function is designed to run under "defer", so it doesn't actually
+// do any error handling and will leave dangling temporary files on disk
+// if any errors occur while cleaning up.
+func (b *binary) Close() {
+	os.RemoveAll(b.workDir)
+}
+
+func GoBuild(pkgPath, tmpPrefix string) string {
+	tmpFile, err := ioutil.TempFile("", tmpPrefix)
+	if err != nil {
+		panic(err)
+	}
+	tmpFilename := tmpFile.Name()
+	if err = tmpFile.Close(); err != nil {
+		panic(err)
+	}
+
+	cmd := exec.Command(
+		"go", "build",
+		"-o", tmpFilename,
+		pkgPath,
+	)
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+
+	err = cmd.Run()
+	if err != nil {
+		// The go compiler will have already produced some error messages
+		// on stderr by the time we get here.
+		panic(fmt.Sprintf("failed to build executable: %s", err))
+	}
+
+	return tmpFilename
+}

tools/terraform-bundle/e2etest/main_test.go

@@ -0,0 +1,39 @@
+package e2etest
+
+import (
+	"os"
+	"testing"
+
+	"github.com/hashicorp/terraform/e2e"
+)
+
+var bundleBin string
+
+func TestMain(m *testing.M) {
+	teardown := setup()
+	code := m.Run()
+	teardown()
+	os.Exit(code)
+}
+
+func setup() func() {
+	tmpFilename := e2e.GoBuild("github.com/hashicorp/terraform/tools/terraform-bundle", "terraform-bundle")
+	bundleBin = tmpFilename
+
+	return func() {
+		os.Remove(tmpFilename)
+	}
+}
+
+func canAccessNetwork() bool {
+	// We re-use the flag normally used for acceptance tests since that's
+	// established as a way to opt-in to reaching out to real systems that
+	// may suffer transient errors.
+	return os.Getenv("TF_ACC") != ""
+}
+
+func skipIfCannotAccessNetwork(t *testing.T) {
+	if !canAccessNetwork() {
+		t.Skip("network access not allowed; use TF_ACC=1 to enable")
+	}
+}

tools/terraform-bundle/e2etest/package_test.go

@@ -0,0 +1,103 @@
+package e2etest
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/hashicorp/terraform/e2e"
+)
+
+func TestPackage_empty(t *testing.T) {
+	t.Parallel()
+
+	// This test reaches out to releases.hashicorp.com to download the
+	// template provider, so it can only run if network access is allowed.
+	// We intentionally don't try to stub this here, because there's already
+	// a stubbed version of this in the "command" package and so the goal here
+	// is to test the interaction with the real repository.
+	skipIfCannotAccessNetwork(t)
+
+	fixturePath := filepath.Join("test-fixtures", "empty")
+	tfBundle := e2e.NewBinary(bundleBin, fixturePath)
+	defer tfBundle.Close()
+
+	stdout, stderr, err := tfBundle.Run("package", "terraform-bundle.hcl")
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+
+	if stderr != "" {
+		t.Errorf("unexpected stderr output:\n%s", stderr)
+	}
+
+	if !strings.Contains(stdout, "Fetching Terraform 0.10.1 core package...") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+	if !strings.Contains(stdout, "Creating terraform_0.10.1-bundle") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+	if !strings.Contains(stdout, "All done!") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+}
+
+func TestPackage_manyProviders(t *testing.T) {
+	t.Parallel()
+
+	// This test reaches out to releases.hashicorp.com to download the
+	// template provider, so it can only run if network access is allowed.
+	// We intentionally don't try to stub this here, because there's already
+	// a stubbed version of this in the "command" package and so the goal here
+	// is to test the interaction with the real repository.
+	skipIfCannotAccessNetwork(t)
+
+	fixturePath := filepath.Join("test-fixtures", "many-providers")
+	tfBundle := e2e.NewBinary(bundleBin, fixturePath)
+	defer tfBundle.Close()
+
+	stdout, stderr, err := tfBundle.Run("package", "terraform-bundle.hcl")
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+
+	if stderr != "" {
+		t.Errorf("unexpected stderr output:\n%s", stderr)
+	}
+
+	if !strings.Contains(stdout, "Checking for available provider plugins on ") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+
+	// Here we have to check each provider separately
+	// because it's internally held in a map (i.e. not guaranteed order)
+
+	if !strings.Contains(stdout, `- Resolving "aws" provider (~> 0.1)...
+- Downloading plugin for provider "aws" (0.1.4)...`) {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+
+	if !strings.Contains(stdout, `- Resolving "kubernetes" provider (0.1.0)...
+- Downloading plugin for provider "kubernetes" (0.1.0)...
+- Resolving "kubernetes" provider (0.1.1)...
+- Downloading plugin for provider "kubernetes" (0.1.1)...
+- Resolving "kubernetes" provider (0.1.2)...
+- Downloading plugin for provider "kubernetes" (0.1.2)...`) {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+
+	if !strings.Contains(stdout, `- Resolving "null" provider (0.1.0)...
+- Downloading plugin for provider "null" (0.1.0)...`) {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+
+	if !strings.Contains(stdout, "Fetching Terraform 0.10.1 core package...") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+	if !strings.Contains(stdout, "Creating terraform_0.10.1-bundle") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+	if !strings.Contains(stdout, "All done!") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+}

tools/terraform-bundle/e2etest/test-fixtures/empty/terraform-bundle.hcl

@@ -0,0 +1,3 @@
+terraform {
+  version = "0.10.1"
+}

tools/terraform-bundle/e2etest/test-fixtures/many-providers/terraform-bundle.hcl

@@ -0,0 +1,9 @@
+terraform {
+  version = "0.10.1"
+}
+
+providers {
+  aws = ["~> 0.1"]
+  kubernetes = ["0.1.0", "0.1.1", "0.1.2"]
+  null = ["0.1.0"]
+}