[Testing Framework] Add test command to Terraform CLI (#33327)

* Add test structure to views package for rendering test output

* Add test file HCL configuration and parser functionality

* Adds a TestContext structure for evaluating assertions against the state and plan

* Add test command to Terraform CLI

commands.go

@@ -13,6 +13,7 @@ import (
 	svchost "github.com/hashicorp/terraform-svchost"
 	"github.com/hashicorp/terraform-svchost/auth"
 	"github.com/hashicorp/terraform-svchost/disco"
+
 	"github.com/hashicorp/terraform/internal/addrs"
 	"github.com/hashicorp/terraform/internal/command"
 	"github.com/hashicorp/terraform/internal/command/cliconfig"
@@ -284,6 +285,12 @@ func initCommands(
 			}, nil
 		},
 
+		"test": func() (cli.Command, error) {
+			return &command.TestCommand{
+				Meta: meta,
+			}, nil
+		},
+
 		"validate": func() (cli.Command, error) {
 			return &command.ValidateCommand{
 				Meta: meta,

internal/command/test.go

@@ -0,0 +1,365 @@
+package command
+
+import (
+	"sort"
+	"strings"
+
+	"github.com/hashicorp/hcl/v2"
+
+	"github.com/hashicorp/terraform/internal/addrs"
+	"github.com/hashicorp/terraform/internal/backend"
+	"github.com/hashicorp/terraform/internal/command/arguments"
+	"github.com/hashicorp/terraform/internal/command/views"
+	"github.com/hashicorp/terraform/internal/configs"
+	"github.com/hashicorp/terraform/internal/configs/configload"
+	"github.com/hashicorp/terraform/internal/moduletest"
+	"github.com/hashicorp/terraform/internal/plans"
+	"github.com/hashicorp/terraform/internal/states"
+	"github.com/hashicorp/terraform/internal/terraform"
+	"github.com/hashicorp/terraform/internal/tfdiags"
+)
+
+type TestCommand struct {
+	Meta
+
+	loader *configload.Loader
+}
+
+func (c *TestCommand) Help() string {
+	helpText := `
+Usage: terraform [global options] test [options]
+
+  Executes automated integration tests against the current Terraform 
+  configuration.
+
+  Terraform will search for .tftest files within the current configuration and 
+  testing directories. Terraform will then execute the testing run blocks within
+  any testing files in order, and verify conditional checks and assertions 
+  against the created infrastructure. 
+
+  This command creates real infrastructure and will attempt to clean up the
+  testing infrastructure on completion. Monitor the output carefully to ensure
+  this cleanup process is successful.
+
+Options:
+
+  TODO: implement optional arguments.
+`
+	return strings.TrimSpace(helpText)
+}
+
+func (c *TestCommand) Synopsis() string {
+	return "Execute integration tests for Terraform modules"
+}
+
+func (c *TestCommand) Run(rawArgs []string) int {
+	var diags tfdiags.Diagnostics
+
+	common, _ := arguments.ParseView(rawArgs)
+	c.View.Configure(common)
+
+	view := views.NewTest(arguments.ViewHuman, c.View)
+
+	loader, err := c.initConfigLoader()
+	diags = diags.Append(err)
+	if err != nil {
+		c.View.Diagnostics(diags)
+		return 1
+	}
+	c.loader = loader
+
+	config, configDiags := loader.LoadConfigWithTests(".", "tests")
+	diags = diags.Append(configDiags)
+	if configDiags.HasErrors() {
+		c.View.Diagnostics(diags)
+		return 1
+	}
+
+	suite := moduletest.Suite{
+		Files: func() map[string]*moduletest.File {
+			files := make(map[string]*moduletest.File)
+			for name, file := range config.Module.Tests {
+				var runs []*moduletest.Run
+				for _, run := range file.Runs {
+					runs = append(runs, &moduletest.Run{
+						Config: run,
+						Name:   run.Name,
+					})
+				}
+				files[name] = &moduletest.File{
+					Config: file,
+					Name:   name,
+					Runs:   runs,
+				}
+			}
+			return files
+		}(),
+	}
+
+	view.Abstract(&suite)
+	c.ExecuteTestSuite(&suite, config, view)
+	view.Conclusion(&suite)
+
+	if suite.Status != moduletest.Pass {
+		return 1
+	}
+	return 0
+}
+
+func (c *TestCommand) ExecuteTestSuite(suite *moduletest.Suite, config *configs.Config, view views.Test) {
+	var diags tfdiags.Diagnostics
+
+	opts, err := c.contextOpts()
+	diags = diags.Append(err)
+	if err != nil {
+		suite.Status = suite.Status.Merge(moduletest.Error)
+		c.View.Diagnostics(diags)
+		return
+	}
+
+	ctx, ctxDiags := terraform.NewContext(opts)
+	diags = diags.Append(ctxDiags)
+	if ctxDiags.HasErrors() {
+		suite.Status = suite.Status.Merge(moduletest.Error)
+		c.View.Diagnostics(diags)
+		return
+	}
+	c.View.Diagnostics(diags) // Print out any warnings from the setup.
+
+	var files []string
+	for name := range suite.Files {
+		files = append(files, name)
+	}
+	sort.Strings(files) // execute the files in alphabetical order
+
+	suite.Status = moduletest.Pass
+	for _, name := range files {
+		file := suite.Files[name]
+		c.ExecuteTestFile(ctx, file, config, view)
+
+		suite.Status = suite.Status.Merge(file.Status)
+	}
+}
+
+func (c *TestCommand) ExecuteTestFile(ctx *terraform.Context, file *moduletest.File, config *configs.Config, view views.Test) {
+	var diags tfdiags.Diagnostics
+
+	globalVariableValues, diags := c.CollectDefaultVariables(file.Config.Variables, config)
+	if diags.HasErrors() {
+		file.Status = file.Status.Merge(moduletest.Error)
+		view.File(file)
+		c.View.Diagnostics(diags)
+		return
+	}
+
+	state := states.NewState()
+	defer func() {
+
+		// Whatever happens, at the end of this test we don't want to leave
+		// active resources behind. So we'll do a destroy action against the
+		// state in a deferred function.
+
+		plan, planDiags := ctx.Plan(config, state, &terraform.PlanOpts{
+			Mode:         plans.DestroyMode,
+			SetVariables: globalVariableValues,
+		})
+		if planDiags.HasErrors() {
+			// This is bad, we need to tell the user that we couldn't clean up
+			// and they need to go and manually delete some resources.
+
+			c.Streams.Eprintf("Terraform encountered an error destroying resources created during the test.\n\n")
+			c.View.Diagnostics(planDiags)
+
+			if state.HasManagedResourceInstanceObjects() {
+				c.Streams.Eprintf("Terraform left the following resources in state, they need to be cleaned up manually:\n\n")
+				for _, resource := range state.AllResourceInstanceObjectAddrs() {
+					if resource.DeposedKey != states.NotDeposed {
+						c.Streams.Eprintf("  - %s (%s)\n", resource.Instance, resource.DeposedKey)
+						continue
+					}
+					c.Streams.Eprintf("  - %s\n", resource.Instance)
+				}
+			}
+
+			return
+		}
+		c.View.Diagnostics(planDiags) // Print out any warnings from the destroy plan.
+
+		finalState, applyDiags := ctx.Apply(plan, config)
+		if applyDiags.HasErrors() {
+			// This is bad, we need to tell the user that we couldn't clean up
+			// and they need to go and manually delete some resources.
+
+			c.Streams.Eprintf("Terraform encountered an error destroying resources created during the test.\n\n")
+		}
+		c.View.Diagnostics(applyDiags) // Print out any warnings from the destroy apply.
+
+		if finalState.HasManagedResourceInstanceObjects() {
+			// Then we need to print dialog telling the user they need to clean
+			// things up, and we should mark the overall test as errored.
+
+			c.Streams.Eprintf("Terraform left the following resources in state, they need to be cleaned up manually:\n\n")
+			for _, resource := range state.AllResourceInstanceObjectAddrs() {
+				if resource.DeposedKey != states.NotDeposed {
+					c.Streams.Eprintf("  - %s (%s)\n", resource.Instance, resource.DeposedKey)
+					continue
+				}
+				c.Streams.Eprintf("  - %s\n", resource.Instance)
+			}
+
+		}
+	}()
+
+	file.Status = file.Status.Merge(moduletest.Pass)
+	for _, run := range file.Runs {
+		if file.Status == moduletest.Error {
+			run.Status = moduletest.Skip
+			continue
+		}
+
+		state = c.ExecuteTestRun(ctx, run, state, config, globalVariableValues)
+		file.Status = file.Status.Merge(run.Status)
+	}
+
+	view.File(file)
+	c.View.Diagnostics(diags)
+
+	for _, run := range file.Runs {
+		view.Run(run)
+	}
+}
+
+func (c *TestCommand) ExecuteTestRun(ctx *terraform.Context, run *moduletest.Run, state *states.State, config *configs.Config, defaults terraform.InputValues) *states.State {
+
+	var targets []addrs.Targetable
+	for _, target := range run.Config.Options.Target {
+		addr, diags := addrs.ParseTarget(target)
+		run.Diagnostics = run.Diagnostics.Append(diags)
+		if diags.HasErrors() {
+			run.Status = moduletest.Error
+			return state
+		}
+
+		targets = append(targets, addr.Subject)
+	}
+
+	var replaces []addrs.AbsResourceInstance
+	for _, replace := range run.Config.Options.Replace {
+		addr, diags := addrs.ParseAbsResourceInstance(replace)
+		run.Diagnostics = run.Diagnostics.Append(diags)
+		if diags.HasErrors() {
+			run.Status = moduletest.Error
+			return state
+		}
+
+		if addr.Resource.Resource.Mode != addrs.ManagedResourceMode {
+			run.Diagnostics = run.Diagnostics.Append(hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "can only target managed resources for forced replacements",
+				Detail:   addr.String(),
+				Subject:  replace.SourceRange().Ptr(),
+			})
+			return state
+		}
+
+		replaces = append(replaces, addr)
+	}
+
+	variables, diags := c.OverrideDefaultVariables(run.Config.Variables, config, defaults)
+	run.Diagnostics = run.Diagnostics.Append(diags)
+	if diags.HasErrors() {
+		run.Status = moduletest.Error
+		return state
+	}
+
+	var references []*addrs.Reference
+	for _, assert := range run.Config.CheckRules {
+		for _, variable := range assert.Condition.Variables() {
+			reference, diags := addrs.ParseRef(variable)
+			run.Diagnostics = run.Diagnostics.Append(diags)
+			references = append(references, reference)
+		}
+	}
+	if run.Diagnostics.HasErrors() {
+		run.Status = moduletest.Error
+		return state
+	}
+
+	plan, diags := ctx.Plan(config, state, &terraform.PlanOpts{
+		Mode: func() plans.Mode {
+			switch run.Config.Options.Mode {
+			case configs.RefreshOnlyTestMode:
+				return plans.RefreshOnlyMode
+			default:
+				return plans.NormalMode
+			}
+		}(),
+		SetVariables:       variables,
+		Targets:            targets,
+		ForceReplace:       replaces,
+		SkipRefresh:        !run.Config.Options.Refresh,
+		ExternalReferences: references,
+	})
+	run.Diagnostics = run.Diagnostics.Append(diags)
+	if diags.HasErrors() {
+		run.Status = moduletest.Error
+		return state
+	}
+
+	if run.Config.Command == configs.ApplyTestCommand {
+		state, diags = ctx.Apply(plan, config)
+		run.Diagnostics = run.Diagnostics.Append(diags)
+		if diags.HasErrors() {
+			run.Status = moduletest.Error
+			return state
+		}
+
+		ctx.TestContext(config, state, plan, variables).EvaluateAgainstState(run)
+		return state
+	}
+
+	ctx.TestContext(config, plan.PlannedState, plan, variables).EvaluateAgainstPlan(run)
+	return state
+}
+
+func (c *TestCommand) CollectDefaultVariables(exprs map[string]hcl.Expression, config *configs.Config) (terraform.InputValues, tfdiags.Diagnostics) {
+	unparsed := make(map[string]backend.UnparsedVariableValue)
+	for key, value := range exprs {
+		unparsed[key] = unparsedVariableValueExpression{
+			expr:       value,
+			sourceType: terraform.ValueFromConfig,
+		}
+	}
+	return backend.ParseVariableValues(unparsed, config.Module.Variables)
+}
+
+func (c *TestCommand) OverrideDefaultVariables(exprs map[string]hcl.Expression, config *configs.Config, existing terraform.InputValues) (terraform.InputValues, tfdiags.Diagnostics) {
+	if len(exprs) == 0 {
+		return existing, nil
+	}
+
+	decls := make(map[string]*configs.Variable)
+	unparsed := make(map[string]backend.UnparsedVariableValue)
+	for name, variable := range exprs {
+
+		if config, ok := config.Module.Variables[name]; ok {
+			decls[name] = config
+		}
+
+		unparsed[name] = unparsedVariableValueExpression{
+			expr:       variable,
+			sourceType: terraform.ValueFromConfig,
+		}
+	}
+
+	overrides, diags := backend.ParseVariableValues(unparsed, decls)
+	values := make(terraform.InputValues)
+	for name, value := range existing {
+		if override, ok := overrides[name]; ok {
+			values[name] = override
+			continue
+		}
+		values[name] = value
+	}
+	return values, diags
+}

internal/command/test_test.go

@@ -0,0 +1,68 @@
+package command
+
+import (
+	"path"
+	"strings"
+	"testing"
+)
+
+func TestTest(t *testing.T) {
+	tcs := map[string]struct {
+		args     []string
+		expected string
+		code     int
+	}{
+		"simple_pass": {
+			expected: "1 passed, 0 failed.",
+			code:     0,
+		},
+		"simple_pass_nested": {
+			expected: "1 passed, 0 failed.",
+			code:     0,
+		},
+		"pass_with_locals": {
+			expected: "1 passed, 0 failed.",
+			code:     0,
+		},
+		"pass_with_variables": {
+			expected: "2 passed, 0 failed.",
+			code:     0,
+		},
+		"plan_then_apply": {
+			expected: "2 passed, 0 failed.",
+			code:     0,
+		},
+		"simple_fail": {
+			expected: "0 passed, 1 failed.",
+			code:     1,
+		},
+	}
+	for name, tc := range tcs {
+		t.Run(name, func(t *testing.T) {
+			td := t.TempDir()
+			testCopyDir(t, testFixturePath(path.Join("test", name)), td)
+			defer testChdir(t, td)()
+
+			p := planFixtureProvider()
+			view, done := testView(t)
+
+			c := &TestCommand{
+				Meta: Meta{
+					testingOverrides: metaOverridesForProvider(p),
+					View:             view,
+				},
+			}
+
+			code := c.Run(tc.args)
+			output := done(t)
+
+			if code != tc.code {
+				t.Errorf("expected status code %d but got %d", tc.code, code)
+			}
+
+			if !strings.Contains(output.Stdout(), tc.expected) {
+				t.Errorf("output didn't contain expected string:\n\n%s", output.All())
+			}
+		})
+	}
+}

internal/command/testdata/test/pass_with_locals/main.tf

@@ -0,0 +1,7 @@
+resource "test_instance" "foo" {
+  ami = "bar"
+}
+
+locals {
+  value = test_instance.foo.ami
+}

internal/command/testdata/test/pass_with_locals/main.tftest

@@ -0,0 +1,6 @@
+run "validate_test_instance" {
+  assert {
+    condition = local.value == "bar"
+    error_message = "invalid ami value"
+  }
+}

internal/command/testdata/test/pass_with_variables/main.tf

@@ -0,0 +1,7 @@
+variable "input" {
+  type = string
+}
+
+resource "test_instance" "foo" {
+  ami = var.input
+}

internal/command/testdata/test/pass_with_variables/main.tftest

@@ -0,0 +1,21 @@
+variables {
+  input = "bar"
+}
+
+run "validate_test_instance" {
+  assert {
+    condition = test_instance.foo.ami == "bar"
+    error_message = "invalid ami value"
+  }
+}
+
+run "validate_test_instance" {
+  variables {
+    input = "zap"
+  }
+
+  assert {
+    condition = test_instance.foo.ami == "zap"
+    error_message = "invalid ami value"
+  }
+}

internal/command/testdata/test/plan_then_apply/main.tf

@@ -0,0 +1,3 @@
+resource "test_instance" "foo" {
+  ami = "bar"
+}

internal/command/testdata/test/plan_then_apply/main.tftest

@@ -0,0 +1,16 @@
+run "validate_test_instance" {
+
+  command = plan
+
+  assert {
+    condition = test_instance.foo.ami == "bar"
+    error_message = "invalid ami value"
+  }
+}
+
+run "validate_test_instance" {
+  assert {
+    condition = test_instance.foo.ami == "bar"
+    error_message = "invalid ami value"
+  }
+}

internal/command/testdata/test/simple_fail/main.tf

@@ -0,0 +1,3 @@
+resource "test_instance" "foo" {
+  ami = "bar"
+}

internal/command/testdata/test/simple_fail/main.tftest

@@ -0,0 +1,6 @@
+run "validate_test_instance" {
+  assert {
+    condition = test_instance.foo.ami == "zap"
+    error_message = "invalid ami value"
+  }
+}

internal/command/testdata/test/simple_pass/main.tf

@@ -0,0 +1,3 @@
+resource "test_instance" "foo" {
+  ami = "bar"
+}

internal/command/testdata/test/simple_pass/main.tftest

@@ -0,0 +1,6 @@
+run "validate_test_instance" {
+  assert {
+    condition = test_instance.foo.ami == "bar"
+    error_message = "invalid ami value"
+  }
+}

internal/command/testdata/test/simple_pass_nested/main.tf

@@ -0,0 +1,3 @@
+resource "test_instance" "foo" {
+  ami = "bar"
+}

internal/command/testdata/test/simple_pass_nested/tests/main.tftest

@@ -0,0 +1,6 @@
+run "validate_test_instance" {
+  assert {
+    condition = test_instance.foo.ami == "bar"
+    error_message = "invalid ami value"
+  }
+}