[Testing framework] Implement JSON view functionality for test command (#33400)

internal/command/views/json/message_types.go

@@ -28,4 +28,11 @@ const (
 	MessageProvisionErrored  MessageType = "provision_errored"
 	MessageRefreshStart      MessageType = "refresh_start"
 	MessageRefreshComplete   MessageType = "refresh_complete"
+
+	// Test messages
+	MessageTestAbstract MessageType = "test_abstract"
+	MessageTestFile     MessageType = "test_file"
+	MessageTestRun      MessageType = "test_run"
+	MessageTestSummary  MessageType = "test_summary"
+	MessageTestCleanup  MessageType = "test_cleanup"
 )

internal/command/views/json/test.go

@@ -0,0 +1,43 @@
+package json
+
+import (
+	"strings"
+
+	"github.com/hashicorp/terraform/internal/moduletest"
+)
+
+type TestSuiteAbstract map[string][]string
+
+type TestStatus string
+
+type TestFileStatus struct {
+	Path   string     `json:"path"`
+	Status TestStatus `json:"status"`
+}
+
+type TestRunStatus struct {
+	Path   string     `json:"path"`
+	Run    string     `json:"run"`
+	Status TestStatus `json:"status"`
+}
+
+type TestSuiteSummary struct {
+	Status  TestStatus `json:"status"`
+	Passed  int        `json:"passed"`
+	Failed  int        `json:"failed"`
+	Errored int        `json:"errored"`
+	Skipped int        `json:"skipped"`
+}
+
+type TestFileCleanup struct {
+	FailedResources []TestFailedResource `json:"failed_resources"`
+}
+
+type TestFailedResource struct {
+	Instance   string `json:"instance"`
+	DeposedKey string `json:"deposed_key,omitempty"`
+}
+
+func ToTestStatus(status moduletest.Status) TestStatus {
+	return TestStatus(strings.ToLower(status.String()))
+}

internal/command/views/json_view.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 
 	"github.com/hashicorp/go-hclog"
+
 	"github.com/hashicorp/terraform/internal/command/views/json"
 	"github.com/hashicorp/terraform/internal/tfdiags"
 	tfversion "github.com/hashicorp/terraform/version"
@@ -69,23 +70,19 @@ func (v *JSONView) StateDump(state string) {
 	)
 }
 
-func (v *JSONView) Diagnostics(diags tfdiags.Diagnostics) {
+func (v *JSONView) Diagnostics(diags tfdiags.Diagnostics, metadata ...interface{}) {
 	sources := v.view.configSources()
 	for _, diag := range diags {
 		diagnostic := json.NewDiagnostic(diag, sources)
+
+		args := []interface{}{"type", json.MessageDiagnostic, "diagnostic", diagnostic}
+		args = append(args, metadata...)
+
 		switch diag.Severity() {
 		case tfdiags.Warning:
-			v.log.Warn(
-				fmt.Sprintf("Warning: %s", diag.Description().Summary),
-				"type", json.MessageDiagnostic,
-				"diagnostic", diagnostic,
-			)
+			v.log.Warn(fmt.Sprintf("Warning: %s", diag.Description().Summary), args...)
 		default:
-			v.log.Error(
-				fmt.Sprintf("Error: %s", diag.Description().Summary),
-				"type", json.MessageDiagnostic,
-				"diagnostic", diagnostic,
-			)
+			v.log.Error(fmt.Sprintf("Error: %s", diag.Description().Summary), args...)
 		}
 	}
 }

internal/command/views/json_view_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+
 	"github.com/hashicorp/terraform/internal/addrs"
 	viewsjson "github.com/hashicorp/terraform/internal/command/views/json"
 	"github.com/hashicorp/terraform/internal/plans"
@@ -104,6 +105,53 @@ func TestJSONView_Diagnostics(t *testing.T) {
 	testJSONViewOutputEquals(t, done(t).Stdout(), want)
 }
 
+func TestJSONView_DiagnosticsWithMetadata(t *testing.T) {
+	streams, done := terminal.StreamsForTesting(t)
+	jv := NewJSONView(NewView(streams))
+
+	var diags tfdiags.Diagnostics
+	diags = diags.Append(tfdiags.Sourceless(
+		tfdiags.Warning,
+		`Improper use of "less"`,
+		`You probably mean "10 buckets or fewer"`,
+	))
+	diags = diags.Append(tfdiags.Sourceless(
+		tfdiags.Error,
+		"Unusually stripey cat detected",
+		"Are you sure this random_pet isn't a cheetah?",
+	))
+
+	jv.Diagnostics(diags, "@meta", "extra_info")
+
+	want := []map[string]interface{}{
+		{
+			"@level":   "warn",
+			"@message": `Warning: Improper use of "less"`,
+			"@module":  "terraform.ui",
+			"type":     "diagnostic",
+			"diagnostic": map[string]interface{}{
+				"severity": "warning",
+				"summary":  `Improper use of "less"`,
+				"detail":   `You probably mean "10 buckets or fewer"`,
+			},
+			"@meta": "extra_info",
+		},
+		{
+			"@level":   "error",
+			"@message": "Error: Unusually stripey cat detected",
+			"@module":  "terraform.ui",
+			"type":     "diagnostic",
+			"diagnostic": map[string]interface{}{
+				"severity": "error",
+				"summary":  "Unusually stripey cat detected",
+				"detail":   "Are you sure this random_pet isn't a cheetah?",
+			},
+			"@meta": "extra_info",
+		},
+	}
+	testJSONViewOutputEquals(t, done(t).Stdout(), want)
+}
+
 func TestJSONView_PlannedChange(t *testing.T) {
 	streams, done := terminal.StreamsForTesting(t)
 	jv := NewJSONView(NewView(streams))

internal/command/views/test.go

@@ -1,12 +1,16 @@
 package views
 
 import (
+	"bytes"
 	"fmt"
 
 	"github.com/mitchellh/colorstring"
 
 	"github.com/hashicorp/terraform/internal/command/arguments"
+	"github.com/hashicorp/terraform/internal/command/views/json"
 	"github.com/hashicorp/terraform/internal/moduletest"
+	"github.com/hashicorp/terraform/internal/states"
+	"github.com/hashicorp/terraform/internal/tfdiags"
 )
 
 // Test renders outputs for test executions.
@@ -26,14 +30,22 @@ type Test interface {
 	File(file *moduletest.File)
 
 	// Run prints out the summary for a single test run block.
-	Run(run *moduletest.Run)
+	Run(run *moduletest.Run, file *moduletest.File)
+
+	// DestroySummary prints out the summary of the destroy step of each test
+	// file. If everything goes well, this should be empty.
+	DestroySummary(diags tfdiags.Diagnostics, file *moduletest.File, state *states.State)
+
+	// Diagnostics prints out the provided diagnostics.
+	Diagnostics(run *moduletest.Run, file *moduletest.File, diags tfdiags.Diagnostics)
 }
 
 func NewTest(vt arguments.ViewType, view *View) Test {
 	switch vt {
 	case arguments.ViewJSON:
-		// TODO(liamcervante): Add support for JSON outputs.
-		panic("not supported yet")
+		return &TestJSON{
+			view: NewJSONView(view),
+		}
 	case arguments.ViewHuman:
 		return &TestHuman{
 			view: view,
@@ -93,11 +105,169 @@ func (t *TestHuman) File(file *moduletest.File) {
 	t.view.streams.Printf("%s... %s\n", file.Name, colorizeTestStatus(file.Status, t.view.colorize))
 }
 
-func (t *TestHuman) Run(run *moduletest.Run) {
+func (t *TestHuman) Run(run *moduletest.Run, file *moduletest.File) {
 	t.view.streams.Printf("  run %q... %s\n", run.Name, colorizeTestStatus(run.Status, t.view.colorize))
 
 	// Finally we'll print out a summary of the diagnostics from the run.
-	t.view.Diagnostics(run.Diagnostics)
+	t.Diagnostics(run, file, run.Diagnostics)
+}
+
+func (t *TestHuman) DestroySummary(diags tfdiags.Diagnostics, file *moduletest.File, state *states.State) {
+	if diags.HasErrors() {
+		t.view.streams.Eprintf("Terraform encountered an error destroying resources created while executing %s.\n", file.Name)
+	}
+	t.Diagnostics(nil, file, diags)
+
+	if state.HasManagedResourceInstanceObjects() {
+		t.view.streams.Eprintf("\nTerraform left the following resources in state after executing %s, they need to be cleaned up manually:\n", file.Name)
+		for _, resource := range state.AllResourceInstanceObjectAddrs() {
+			if resource.DeposedKey != states.NotDeposed {
+				t.view.streams.Eprintf("  - %s (%s)\n", resource.Instance, resource.DeposedKey)
+				continue
+			}
+			t.view.streams.Eprintf("  - %s\n", resource.Instance)
+		}
+	}
+}
+
+func (t *TestHuman) Diagnostics(_ *moduletest.Run, _ *moduletest.File, diags tfdiags.Diagnostics) {
+	t.view.Diagnostics(diags)
+}
+
+type TestJSON struct {
+	view *JSONView
+}
+
+var _ Test = (*TestJSON)(nil)
+
+func (t TestJSON) Abstract(suite *moduletest.Suite) {
+	var fileCount, runCount int
+
+	abstract := json.TestSuiteAbstract{}
+	for name, file := range suite.Files {
+		fileCount++
+		var runs []string
+		for _, run := range file.Runs {
+			runCount++
+			runs = append(runs, run.Name)
+		}
+		abstract[name] = runs
+	}
+
+	files := "files"
+	runs := "run blocks"
+
+	if fileCount == 1 {
+		files = "file"
+	}
+
+	if runCount == 1 {
+		runs = "run block"
+	}
+
+	t.view.log.Info(
+		fmt.Sprintf("Found %d %s and %d %s", fileCount, files, runCount, runs),
+		"type", json.MessageTestAbstract,
+		json.MessageTestAbstract, abstract)
+}
+
+func (t TestJSON) Conclusion(suite *moduletest.Suite) {
+	summary := json.TestSuiteSummary{
+		Status: json.ToTestStatus(suite.Status),
+	}
+	for _, file := range suite.Files {
+		for _, run := range file.Runs {
+			switch run.Status {
+			case moduletest.Skip:
+				summary.Skipped++
+			case moduletest.Pass:
+				summary.Passed++
+			case moduletest.Error:
+				summary.Errored++
+			case moduletest.Fail:
+				summary.Failed++
+			}
+		}
+	}
+
+	var message bytes.Buffer
+	if suite.Status <= moduletest.Skip {
+		// Then no tests.
+		message.WriteString("Executed 0 tests")
+		if summary.Skipped > 0 {
+			message.WriteString(fmt.Sprintf(", %d skipped.", summary.Skipped))
+		} else {
+			message.WriteString(".")
+		}
+	} else {
+		if suite.Status == moduletest.Pass {
+			message.WriteString("Success!")
+		} else {
+			message.WriteString("Failure!")
+		}
+
+		message.WriteString(fmt.Sprintf(" %d passed, %d failed", summary.Passed, summary.Failed+summary.Errored))
+		if summary.Skipped > 0 {
+			message.WriteString(fmt.Sprintf(", %d skipped.", summary.Skipped))
+		} else {
+			message.WriteString(".")
+		}
+	}
+
+	t.view.log.Info(
+		message.String(),
+		"type", json.MessageTestSummary,
+		json.MessageTestSummary, summary)
+}
+
+func (t TestJSON) File(file *moduletest.File) {
+	t.view.log.Info(
+		fmt.Sprintf("%s... %s", file.Name, testStatus(file.Status)),
+		"type", json.MessageTestFile,
+		json.MessageTestFile, json.TestFileStatus{file.Name, json.ToTestStatus(file.Status)},
+		"@testfile", file.Name)
+}
+
+func (t TestJSON) Run(run *moduletest.Run, file *moduletest.File) {
+	t.view.log.Info(
+		fmt.Sprintf("  %q... %s", run.Name, testStatus(run.Status)),
+		"type", json.MessageTestRun,
+		json.MessageTestRun, json.TestRunStatus{file.Name, run.Name, json.ToTestStatus(run.Status)},
+		"@testfile", file.Name,
+		"@testrun", run.Name)
+
+	t.Diagnostics(run, file, run.Diagnostics)
+}
+
+func (t TestJSON) DestroySummary(diags tfdiags.Diagnostics, file *moduletest.File, state *states.State) {
+	if state.HasManagedResourceInstanceObjects() {
+		cleanup := json.TestFileCleanup{}
+		for _, resource := range state.AllResourceInstanceObjectAddrs() {
+			cleanup.FailedResources = append(cleanup.FailedResources, json.TestFailedResource{
+				Instance:   resource.Instance.String(),
+				DeposedKey: resource.DeposedKey.String(),
+			})
+		}
+
+		t.view.log.Error(
+			fmt.Sprintf("Terraform left some resources in state after executing %s, they need to be cleaned up manually.", file.Name),
+			"type", json.MessageTestCleanup,
+			json.MessageTestCleanup, cleanup,
+			"@testfile", file.Name)
+	}
+
+	t.Diagnostics(nil, file, diags)
+}
+
+func (t TestJSON) Diagnostics(run *moduletest.Run, file *moduletest.File, diags tfdiags.Diagnostics) {
+	var metadata []interface{}
+	if file != nil {
+		metadata = append(metadata, "@testfile", file.Name)
+	}
+	if run != nil {
+		metadata = append(metadata, "@testrun", run.Name)
+	}
+	t.view.Diagnostics(diags, metadata...)
 }
 
 func colorizeTestStatus(status moduletest.Status, color *colorstring.Colorize) string {
@@ -114,3 +284,18 @@ func colorizeTestStatus(status moduletest.Status, color *colorstring.Colorize) s
 		panic("unrecognized status: " + status.String())
 	}
 }
+
+func testStatus(status moduletest.Status) string {
+	switch status {
+	case moduletest.Error, moduletest.Fail:
+		return "fail"
+	case moduletest.Pass:
+		return "pass"
+	case moduletest.Skip:
+		return "skip"
+	case moduletest.Pending:
+		return "pending"
+	default:
+		panic("unrecognized status: " + status.String())
+	}
+}