Merge pull request #17 from hashicorp/f-acceptance

Acceptance Test Framework

Makefile

@@ -2,7 +2,6 @@ CGO_CFLAGS:=-I$(CURDIR)/vendor/libucl/include
 CGO_LDFLAGS:=-L$(CURDIR)/vendor/libucl
 LIBUCL_NAME=libucl.a
 TEST?=./...
-TESTARGS?=-timeout=5s
 
 # Windows-only
 ifeq ($(OS), Windows_NT)
@@ -23,7 +22,14 @@ dev: libucl
 libucl: vendor/libucl/$(LIBUCL_NAME)
 
 test: libucl
-	go test $(TEST) $(TESTARGS)
+	go test $(TEST) $(TESTARGS) -timeout=10s
+
+testacc: libucl
+	@if [ "$(TEST)" = "./..." ]; then \
+		echo "ERROR: Set TEST to a specific package"; \
+		exit 1; \
+	fi
+	TF_ACC=1 go test $(TEST) -v $(TESTARGS)
 
 testrace: libucl
 	go test -race $(TEST) $(TESTARGS)

builtin/providers/aws/config.go

@@ -1,6 +1,7 @@
 package aws
 
 import (
+	"os"
 	"strings"
 	"unicode"
 
@@ -36,6 +37,10 @@ func (c *Config) AWSRegion() (aws.Region, error) {
 		return aws.Regions[c.Region], nil
 	}
 
+	if v := os.Getenv("AWS_REGION"); v != "" {
+		return aws.Regions[v], nil
+	}
+
 	md, err := aws.GetMetaData("placement/availability-zone")
 	if err != nil {
 		return aws.Region{}, err

builtin/providers/aws/resource_aws_vpc_test.go

@@ -0,0 +1,86 @@
+package aws
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/terraform/helper/resource"
+	"github.com/hashicorp/terraform/terraform"
+	"github.com/mitchellh/goamz/ec2"
+)
+
+func TestAccVpc(t *testing.T) {
+	testAccPreCheck(t)
+
+	resource.Test(t, resource.TestCase{
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckVpcDestroy,
+		Steps: []resource.TestStep{
+			resource.TestStep{
+				Config: testAccVpcConfig,
+				Check:  testAccCheckVpcExists("aws_vpc.foo"),
+			},
+		},
+	})
+}
+
+func testAccCheckVpcDestroy(s *terraform.State) error {
+	conn := testAccProvider.ec2conn
+
+	for _, rs := range s.Resources {
+		if rs.Type != "aws_vpc" {
+			continue
+		}
+
+		// Try to find the VPC
+		resp, err := conn.DescribeVpcs([]string{rs.ID}, ec2.NewFilter())
+		if err == nil {
+			if len(resp.VPCs) > 0 {
+				return fmt.Errorf("VPCs still exist.")
+			}
+
+			return nil
+		}
+
+		// Verify the error is what we want
+		ec2err, ok := err.(*ec2.Error)
+		if !ok {
+			return err
+		}
+		if ec2err.Code != "InvalidVpcID.NotFound" {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func testAccCheckVpcExists(n string) resource.TestCheckFunc {
+	return func(s *terraform.State) error {
+		rs, ok := s.Resources[n]
+		if !ok {
+			return fmt.Errorf("Not found: %s", n)
+		}
+
+		if rs.ID == "" {
+			return fmt.Errorf("No VPC ID is set")
+		}
+
+		conn := testAccProvider.ec2conn
+		resp, err := conn.DescribeVpcs([]string{rs.ID}, ec2.NewFilter())
+		if err != nil {
+			return err
+		}
+		if len(resp.VPCs) == 0 {
+			return fmt.Errorf("VPC not found")
+		}
+
+		return nil
+	}
+}
+
+const testAccVpcConfig = `
+resource "aws_vpc" "foo" {
+	cidr_block = "10.1.0.0/16"
+}
+`

builtin/providers/aws/resource_provider.go

@@ -20,7 +20,15 @@ type ResourceProvider struct {
 }
 
 func (p *ResourceProvider) Validate(c *terraform.ResourceConfig) ([]string, []error) {
-	return nil, nil
+	v := &config.Validator{
+		Optional: []string{
+			"access_key",
+			"secret_key",
+			"region",
+		},
+	}
+
+	return v.Validate(c)
 }
 
 func (p *ResourceProvider) ValidateResource(
@@ -36,24 +44,24 @@ func (p *ResourceProvider) Configure(c *terraform.ResourceConfig) error {
 	// Get the auth and region. This can fail if keys/regions were not
 	// specified and we're attempting to use the environment.
 	var errs []error
-	log.Println("Building AWS auth structure")
+	log.Println("[INFO] Building AWS auth structure")
 	auth, err := p.Config.AWSAuth()
 	if err != nil {
 		errs = append(errs, err)
 	}
 
-	log.Println("Building AWS region structure")
+	log.Println("[INFO] Building AWS region structure")
 	region, err := p.Config.AWSRegion()
 	if err != nil {
 		errs = append(errs, err)
 	}
 
 	if len(errs) == 0 {
-		log.Println("Initializing EC2 connection")
+		log.Println("[INFO] Initializing EC2 connection")
 		p.ec2conn = ec2.New(auth, region)
-		log.Println("Initializing ELB connection")
+		log.Println("[INFO] Initializing ELB connection")
 		p.elbconn = elb.New(auth, region)
-		log.Println("Initializing AutoScaling connection")
+		log.Println("[INFO] Initializing AutoScaling connection")
 		p.autoscalingconn = autoscaling.New(auth, region)
 	}
 

builtin/providers/aws/resource_provider_test.go

@@ -1,6 +1,8 @@
 package aws
 
 import (
+	"os"
+	"log"
 	"reflect"
 	"testing"
 
@@ -8,6 +10,16 @@ import (
 	"github.com/hashicorp/terraform/terraform"
 )
 
+var testAccProviders map[string]terraform.ResourceProvider
+var testAccProvider *ResourceProvider
+
+func init() {
+	testAccProvider = new(ResourceProvider)
+	testAccProviders = map[string]terraform.ResourceProvider{
+		"aws": testAccProvider,
+	}
+}
+
 func TestResourceProvider_impl(t *testing.T) {
 	var _ terraform.ResourceProvider = new(ResourceProvider)
 }
@@ -41,3 +53,16 @@ func TestResourceProvider_Configure(t *testing.T) {
 		t.Fatalf("bad: %#v", rp.Config)
 	}
 }
+
+func testAccPreCheck(t *testing.T) {
+	if v := os.Getenv("AWS_ACCESS_KEY"); v == "" {
+		t.Fatal("AWS_ACCESS_KEY must be set for acceptance tests")
+	}
+	if v := os.Getenv("AWS_SECRET_KEY"); v == "" {
+		t.Fatal("AWS_SECRET_KEY must be set for acceptance tests")
+	}
+	if v := os.Getenv("AWS_REGION"); v == "" {
+		log.Println("[INFO] Test: Using us-west-2 as test region")
+		os.Setenv("AWS_REGION", "us-west-2")
+	}
+}

helper/resource/testing.go

@@ -0,0 +1,214 @@
+package resource
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/hashicorp/terraform/config"
+	"github.com/hashicorp/terraform/terraform"
+)
+
+const TestEnvVar = "TF_ACC"
+
+// TestCheckFunc is the callback type used with acceptance tests to check
+// the state of a resource. The state passed in is the latest state known,
+// or in the case of being after a destroy, it is the last known state when
+// it was created.
+type TestCheckFunc func(*terraform.State) error
+
+// TestCase is a single acceptance test case used to test the apply/destroy
+// lifecycle of a resource in a specific configuration.
+//
+// When the destroy plan is executed, the config from the last TestStep
+// is used to plan it.
+type TestCase struct {
+	// Provider is the ResourceProvider that will be under test.
+	Providers map[string]terraform.ResourceProvider
+
+	// CheckDestroy is called after the resource is finally destroyed
+	// to allow the tester to test that the resource is truly gone.
+	CheckDestroy TestCheckFunc
+
+	// Steps are the apply sequences done within the context of the
+	// same state. Each step can have its own check to verify correctness.
+	Steps []TestStep
+}
+
+// TestStep is a single apply sequence of a test, done within the
+// context of a state.
+//
+// Multiple TestSteps can be sequenced in a Test to allow testing
+// potentially complex update logic. In general, simply create/destroy
+// tests will only need one step.
+type TestStep struct {
+	// Config a string of the configuration to give to Terraform.
+	Config string
+
+	// Check is called after the Config is applied. Use this step to
+	// make your own API calls to check the status of things, and to
+	// inspect the format of the ResourceState itself.
+	//
+	// If an error is returned, the test will fail. In this case, a
+	// destroy plan will still be attempted.
+	//
+	// If this is nil, no check is done on this step.
+	Check TestCheckFunc
+
+	// Destroy will create a destroy plan if set to true.
+	Destroy bool
+}
+
+// Test performs an acceptance test on a resource.
+//
+// Tests are not run unless an environmental variable "TF_ACC" is
+// set to some non-empty value. This is to avoid test cases surprising
+// a user by creating real resources.
+//
+// Tests will fail unless the verbose flag (`go test -v`, or explicitly
+// the "-test.v" flag) is set. Because some acceptance tests take quite
+// long, we require the verbose flag so users are able to see progress
+// output.
+func Test(t TestT, c TestCase) {
+	// We only run acceptance tests if an env var is set because they're
+	// slow and generally require some outside configuration.
+	if os.Getenv(TestEnvVar) == "" {
+		t.Skip(fmt.Sprintf(
+			"Acceptance tests skipped unless env '%s' set",
+			TestEnvVar))
+		return
+	}
+
+	// We require verbose mode so that the user knows what is going on.
+	if !testTesting && !testing.Verbose() {
+		t.Fatal("Acceptance tests must be run with the -v flag on tests")
+		return
+	}
+
+	// Build our context options that we can
+	ctxProviders := make(map[string]terraform.ResourceProviderFactory)
+	for k, p := range c.Providers {
+		ctxProviders[k] = terraform.ResourceProviderFactoryFixed(p)
+	}
+	opts := terraform.ContextOpts{Providers: ctxProviders}
+
+	// A single state variable to track the lifecycle, starting with no state
+	var state *terraform.State
+
+	// Go through each step and run it
+	for i, step := range c.Steps {
+		var err error
+		log.Printf("[WARN] Test: Executing step %d", i)
+		state, err = testStep(opts, state, step)
+		if err != nil {
+			t.Error(fmt.Sprintf(
+				"Step %d error: %s", i, err))
+			break
+		}
+	}
+
+	// If we have a state, then run the destroy
+	if state != nil {
+		destroyStep := TestStep{
+			Config:  c.Steps[len(c.Steps)-1].Config,
+			Check:   c.CheckDestroy,
+			Destroy: true,
+		}
+
+		log.Printf("[WARN] Test: Executing destroy step")
+		state, err := testStep(opts, state, destroyStep)
+		if err != nil {
+			t.Error(fmt.Sprintf(
+				"Error destroying resource! WARNING: Dangling resources\n"+
+					"may exist. The full state and error is shown below.\n\n"+
+					"Error: %s\n\nState: %s",
+				err,
+				state))
+		}
+	} else {
+		log.Printf("[WARN] Skipping destroy test since there is no state.")
+	}
+}
+
+func testStep(
+	opts terraform.ContextOpts,
+	state *terraform.State,
+	step TestStep) (*terraform.State, error) {
+	// Write the configuration
+	cfgF, err := ioutil.TempFile("", "tf-test")
+	if err != nil {
+		return state, fmt.Errorf(
+			"Error creating temporary file for config: %s", err)
+	}
+	cfgPath := cfgF.Name() + ".tf"
+	cfgF.Close()
+	os.Remove(cfgF.Name())
+
+	cfgF, err = os.Create(cfgPath)
+	if err != nil {
+		return state, fmt.Errorf(
+			"Error creating temporary file for config: %s", err)
+	}
+	defer os.Remove(cfgPath)
+
+	_, err = io.Copy(cfgF, strings.NewReader(step.Config))
+	cfgF.Close()
+	if err != nil {
+		return state, fmt.Errorf(
+			"Error creating temporary file for config: %s", err)
+	}
+
+	// Parse the configuration
+	config, err := config.Load(cfgPath)
+	if err != nil {
+		return state, fmt.Errorf(
+			"Error parsing configuration: %s", err)
+	}
+
+	// Build the context
+	opts.Config = config
+	opts.State = state
+	ctx := terraform.NewContext(&opts)
+	if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 {
+		return state, fmt.Errorf(
+			"Configuration is invalid.\n\nWarnings: %#v\n\nErrors: %#v",
+			ws, es)
+	}
+
+	// Plan!
+	if _, err := ctx.Plan(&terraform.PlanOpts{Destroy: step.Destroy}); err != nil {
+		return state, fmt.Errorf(
+			"Error planning: %s", err)
+	}
+
+	// Apply!
+	state, err = ctx.Apply()
+	if err != nil {
+		return state, fmt.Errorf("Error applying: %s", err)
+	}
+
+	// Check! Excitement!
+	if step.Check != nil {
+		if err = step.Check(state); err != nil {
+			err = fmt.Errorf("Check failed: %s", err)
+		}
+	}
+
+	return state, err
+}
+
+// TestT is the interface used to handle the test lifecycle of a test.
+//
+// Users should just use a *testing.T object, which implements this.
+type TestT interface {
+	Error(args ...interface{})
+	Fatal(args ...interface{})
+	Skip(args ...interface{})
+}
+
+// This is set to true by unit tests to alter some behavior
+var testTesting = false

helper/resource/testing_test.go

@@ -0,0 +1,213 @@
+package resource
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/hashicorp/terraform/terraform"
+)
+
+func init() {
+	testTesting = true
+
+	if err := os.Setenv(TestEnvVar, "1"); err != nil {
+		panic(err)
+	}
+}
+
+func TestTest(t *testing.T) {
+	mp := testProvider()
+	mp.ApplyReturn = &terraform.ResourceState{
+		ID: "foo",
+	}
+
+	checkDestroy := false
+	checkStep := false
+
+	checkDestroyFn := func(*terraform.State) error {
+		checkDestroy = true
+		return nil
+	}
+
+	checkStepFn := func(s *terraform.State) error {
+		checkStep = true
+
+		rs, ok := s.Resources["test_instance.foo"]
+		if !ok {
+			t.Error("test_instance.foo is not present")
+			return nil
+		}
+		if rs.ID != "foo" {
+			t.Errorf("bad check ID: %s", rs.ID)
+		}
+
+		return nil
+	}
+
+	mt := new(mockT)
+	Test(mt, TestCase{
+		Providers: map[string]terraform.ResourceProvider{
+			"test": mp,
+		},
+		CheckDestroy: checkDestroyFn,
+		Steps: []TestStep{
+			TestStep{
+				Config: testConfigStr,
+				Check:  checkStepFn,
+			},
+		},
+	})
+
+	if mt.failed() {
+		t.Fatalf("test failed: %s", mt.failMessage())
+	}
+	if !checkStep {
+		t.Fatal("didn't call check for step")
+	}
+	if !checkDestroy {
+		t.Fatal("didn't call check for destroy")
+	}
+}
+
+func TestTest_empty(t *testing.T) {
+	destroyCalled := false
+	checkDestroyFn := func(*terraform.State) error {
+		destroyCalled = true
+		return nil
+	}
+
+	mt := new(mockT)
+	Test(mt, TestCase{
+		CheckDestroy: checkDestroyFn,
+	})
+
+	if mt.failed() {
+		t.Fatal("test failed")
+	}
+	if destroyCalled {
+		t.Fatal("should not call check destroy if there is no steps")
+	}
+}
+
+func TestTest_noEnv(t *testing.T) {
+	// Unset the variable
+	if err := os.Setenv(TestEnvVar, ""); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer os.Setenv(TestEnvVar, "1")
+
+	mt := new(mockT)
+	Test(mt, TestCase{})
+
+	if !mt.SkipCalled {
+		t.Fatal("skip not called")
+	}
+}
+
+func TestTest_stepError(t *testing.T) {
+	mp := testProvider()
+	mp.ApplyReturn = &terraform.ResourceState{
+		ID: "foo",
+	}
+
+	checkDestroy := false
+
+	checkDestroyFn := func(*terraform.State) error {
+		checkDestroy = true
+		return nil
+	}
+
+	checkStepFn := func(*terraform.State) error {
+		return fmt.Errorf("error")
+	}
+
+	mt := new(mockT)
+	Test(mt, TestCase{
+		Providers: map[string]terraform.ResourceProvider{
+			"test": mp,
+		},
+		CheckDestroy: checkDestroyFn,
+		Steps: []TestStep{
+			TestStep{
+				Config: testConfigStr,
+				Check:  checkStepFn,
+			},
+		},
+	})
+
+	if !mt.failed() {
+		t.Fatal("test should've failed")
+	}
+	t.Logf("Fail message: %s", mt.failMessage())
+
+	if !checkDestroy {
+		t.Fatal("didn't call check for destroy")
+	}
+}
+
+// mockT implements TestT for testing
+type mockT struct {
+	ErrorCalled bool
+	ErrorArgs   []interface{}
+	FatalCalled bool
+	FatalArgs   []interface{}
+	SkipCalled  bool
+	SkipArgs    []interface{}
+
+	f bool
+}
+
+func (t *mockT) Error(args ...interface{}) {
+	t.ErrorCalled = true
+	t.ErrorArgs = args
+	t.f = true
+}
+
+func (t *mockT) Fatal(args ...interface{}) {
+	t.FatalCalled = true
+	t.FatalArgs = args
+	t.f = true
+}
+
+func (t *mockT) Skip(args ...interface{}) {
+	t.SkipCalled = true
+	t.SkipArgs = args
+	t.f = true
+}
+
+func (t *mockT) failed() bool {
+	return t.f
+}
+
+func (t *mockT) failMessage() string {
+	if t.FatalCalled {
+		return t.FatalArgs[0].(string)
+	} else if t.ErrorCalled {
+		return t.ErrorArgs[0].(string)
+	} else if t.SkipCalled {
+		return t.SkipArgs[0].(string)
+	}
+
+	return "unknown"
+}
+
+func testProvider() *terraform.MockResourceProvider {
+	mp := new(terraform.MockResourceProvider)
+	mp.DiffReturn = &terraform.ResourceDiff{
+		Attributes: map[string]*terraform.ResourceAttrDiff{
+			"foo": &terraform.ResourceAttrDiff{
+				New: "bar",
+			},
+		},
+	}
+	mp.ResourcesReturn = []terraform.ResourceType{
+		terraform.ResourceType{Name: "test_instance"},
+	}
+
+	return mp
+}
+
+const testConfigStr = `
+resource "test_instance" "foo" {}
+`

terraform/context.go

@@ -271,6 +271,11 @@ func (c *Context) Validate() ([]string, []error) {
 // the variables. This dynamically discovers the attributes instead of
 // using a static map[string]string that the genericWalkFn uses.
 func (c *Context) computeVars(raw *config.RawConfig) error {
+	// If there isn't a raw configuration, don't do anything
+	if raw == nil {
+		return nil
+	}
+
 	// If there are on variables, then we're done
 	if len(raw.Variables) == 0 {
 		return nil
@@ -735,14 +740,14 @@ func (c *Context) genericWalkFn(cb genericWalkFunc) depgraph.WalkFunc {
 			return nil
 		case *GraphNodeResourceProvider:
 			// Interpolate in the variables and configure all the providers
-			var rc *ResourceConfig
+			var raw *config.RawConfig
 			if m.Config != nil {
-				if err := c.computeVars(m.Config.RawConfig); err != nil {
-					panic(err)
-				}
-				rc = NewResourceConfig(m.Config.RawConfig)
+				raw = m.Config.RawConfig
 			}
 
+			rc := NewResourceConfig(raw)
+			rc.interpolate(c)
+
 			for k, p := range m.Providers {
 				log.Printf("[INFO] Configuring provider: %s", k)
 				err := p.Configure(rc)

terraform/resource_provider.go

@@ -69,6 +69,14 @@ type ResourceType struct {
 // of a resource provider.
 type ResourceProviderFactory func() (ResourceProvider, error)
 
+// ResourceProviderFactoryFixed is a helper that creates a
+// ResourceProviderFactory that just returns some fixed provider.
+func ResourceProviderFactoryFixed(p ResourceProvider) ResourceProviderFactory {
+	return func() (ResourceProvider, error) {
+		return p, nil
+	}
+}
+
 func ProviderSatisfies(p ResourceProvider, n string) bool {
 	for _, rt := range p.Resources() {
 		if rt.Name == n {

terraform/resource_provider_test.go

@@ -175,3 +175,15 @@ func TestResourceConfig_IsSet_nil(t *testing.T) {
 		t.Fatal("bad")
 	}
 }
+
+func TestResourceProviderFactoryFixed(t *testing.T) {
+	p := new(MockResourceProvider)
+	var f ResourceProviderFactory = ResourceProviderFactoryFixed(p)
+	actual, err := f()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if actual != p {
+		t.Fatal("should be identical")
+	}
+}