testing framework: allow users to specify deeply nested testing directories (#33584)

internal/command/arguments/test.go

@@ -1,6 +1,9 @@
 package arguments
 
-import "github.com/hashicorp/terraform/internal/tfdiags"
+import (
+	"github.com/hashicorp/terraform/internal/configs"
+	"github.com/hashicorp/terraform/internal/tfdiags"
+)
 
 // Test represents the command-line arguments for the test command.
 type Test struct {
@@ -36,7 +39,7 @@ func ParseTest(args []string) (*Test, tfdiags.Diagnostics) {
 	var jsonOutput bool
 	cmdFlags := extendedFlagSet("test", nil, nil, test.Vars)
 	cmdFlags.Var((*flagStringSlice)(&test.Filter), "filter", "filter")
-	cmdFlags.StringVar(&test.TestDirectory, "test-directory", "tests", "test-directory")
+	cmdFlags.StringVar(&test.TestDirectory, "test-directory", configs.DefaultTestDirectory, "test-directory")
 	cmdFlags.BoolVar(&jsonOutput, "json", false, "json")
 	cmdFlags.BoolVar(&test.Verbose, "verbose", false, "verbose")
 

internal/command/test_test.go

@@ -37,6 +37,17 @@ func TestTest(t *testing.T) {
 			expected: "1 passed, 0 failed.",
 			code:     0,
 		},
+		"simple_pass_very_nested": {
+			args:     []string{"-test-directory", "tests/subdir"},
+			expected: "1 passed, 0 failed.",
+			code:     0,
+		},
+		"simple_pass_very_nested_alternate": {
+			override: "simple_pass_very_nested",
+			args:     []string{"-test-directory", "./tests/subdir"},
+			expected: "1 passed, 0 failed.",
+			code:     0,
+		},
 		"pass_with_locals": {
 			expected: "1 passed, 0 failed.",
 			code:     0,

internal/command/testdata/test/simple_pass_very_nested/main.tf

@@ -0,0 +1,3 @@
+resource "test_resource" "foo" {
+  value = "bar"
+}

internal/command/testdata/test/simple_pass_very_nested/tests/subdir/main.tftest.hcl

@@ -0,0 +1,6 @@
+run "validate_test_resource" {
+  assert {
+    condition = test_resource.foo.value == "bar"
+    error_message = "invalid value"
+  }
+}

internal/configs/parser_config_dir.go

@@ -6,12 +6,17 @@ package configs
 import (
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
 	"github.com/hashicorp/hcl/v2"
 )
 
+const (
+	DefaultTestDirectory = "tests"
+)
+
 // LoadConfigDir reads the .tf and .tf.json files in the given directory
 // as config files (using LoadConfigFile) and then combines these files into
 // a single Module.
@@ -129,6 +134,56 @@ func (p *Parser) loadFiles(paths []string, override bool) ([]*File, hcl.Diagnost
 func (p *Parser) dirFiles(dir string, testsDir string) (primary, override, tests []string, diags hcl.Diagnostics) {
 	includeTests := len(testsDir) > 0
 
+	if includeTests {
+		testPath := path.Join(dir, testsDir)
+
+		infos, err := p.fs.ReadDir(testPath)
+		if err != nil {
+			// Then we couldn't read from the testing directory for some reason.
+
+			if os.IsNotExist(err) {
+				// Then this means the testing directory did not exist.
+				// We won't actually stop loading the rest of the configuration
+				// for this, we will add a warning to explain to the user why
+				// test files weren't processed but leave it at that.
+				if testsDir != DefaultTestDirectory {
+					// We'll only add the warning if a directory other than the
+					// default has been requested. If the user is just loading
+					// the default directory then we have no expectation that
+					// it should actually exist.
+					diags = append(diags, &hcl.Diagnostic{
+						Severity: hcl.DiagWarning,
+						Summary:  "Test directory does not exist",
+						Detail:   fmt.Sprintf("Requested test directory %s does not exist.", testPath),
+					})
+				}
+			} else {
+				// Then there is some other reason we couldn't load. We will
+				// treat this as a full error.
+				diags = append(diags, &hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Failed to read test directory",
+					Detail:   fmt.Sprintf("Test directory %s could not be read: %v.", testPath, err),
+				})
+
+				// We'll also stop loading the rest of the config for this.
+				return
+			}
+
+		} else {
+			for _, testInfo := range infos {
+				if testInfo.IsDir() || IsIgnoredFile(testInfo.Name()) {
+					continue
+				}
+
+				if strings.HasSuffix(testInfo.Name(), ".tftest.hcl") || strings.HasSuffix(testInfo.Name(), ".tftest.json") {
+					tests = append(tests, filepath.Join(testPath, testInfo.Name()))
+				}
+			}
+		}
+
+	}
+
 	infos, err := p.fs.ReadDir(dir)
 	if err != nil {
 		diags = append(diags, &hcl.Diagnostic{
@@ -141,31 +196,7 @@ func (p *Parser) dirFiles(dir string, testsDir string) (primary, override, tests
 
 	for _, info := range infos {
 		if info.IsDir() {
-			if includeTests && info.Name() == testsDir {
-				testsDir := filepath.Join(dir, info.Name())
-				testInfos, err := p.fs.ReadDir(testsDir)
-				if err != nil {
-					diags = append(diags, &hcl.Diagnostic{
-						Severity: hcl.DiagError,
-						Summary:  "Failed to read module test directory",
-						Detail:   fmt.Sprintf("Module test directory %s does not exist or cannot be read.", testsDir),
-					})
-					return
-				}
-
-				for _, testInfo := range testInfos {
-					if testInfo.IsDir() || IsIgnoredFile(testInfo.Name()) {
-						continue
-					}
-
-					if strings.HasSuffix(testInfo.Name(), ".tftest.hcl") || strings.HasSuffix(testInfo.Name(), ".tftest.json") {
-						tests = append(tests, filepath.Join(testsDir, testInfo.Name()))
-					}
-				}
-			}
-
-			// We only care about the tests directory or terraform configuration
-			// files.
+			// We only care about terraform configuration files.
 			continue
 		}
 

internal/configs/parser_config_dir_test.go

@@ -118,15 +118,22 @@ func TestParserLoadConfigDirWithTests(t *testing.T) {
 		"testdata/valid-modules/with-tests",
 		"testdata/valid-modules/with-tests-expect-failures",
 		"testdata/valid-modules/with-tests-nested",
+		"testdata/valid-modules/with-tests-very-nested",
 		"testdata/valid-modules/with-tests-json",
 	}
 
 	for _, directory := range directories {
 		t.Run(directory, func(t *testing.T) {
+
+			testDirectory := DefaultTestDirectory
+			if directory == "testdata/valid-modules/with-tests-very-nested" {
+				testDirectory = "very/nested"
+			}
+
 			parser := NewParser(nil)
-			mod, diags := parser.LoadConfigDirWithTests(directory, "tests")
-			if diags.HasErrors() {
-				t.Errorf("unexpected error diagnostics")
+			mod, diags := parser.LoadConfigDirWithTests(directory, testDirectory)
+			if len(diags) > 0 { // We don't want any warnings or errors.
+				t.Errorf("unexpected diagnostics")
 				for _, diag := range diags {
 					t.Logf("- %s", diag)
 				}
@@ -139,6 +146,32 @@ func TestParserLoadConfigDirWithTests(t *testing.T) {
 	}
 }
 
+func TestParserLoadConfigDirWithTests_ReturnsWarnings(t *testing.T) {
+	parser := NewParser(nil)
+	mod, diags := parser.LoadConfigDirWithTests("testdata/valid-modules/with-tests", "not_real")
+	if len(diags) != 1 {
+		t.Errorf("expected exactly 1 diagnostic, but found %d", len(diags))
+	} else {
+		if diags[0].Severity != hcl.DiagWarning {
+			t.Errorf("expected warning severity but found %d", diags[0].Severity)
+		}
+
+		if diags[0].Summary != "Test directory does not exist" {
+			t.Errorf("expected summary to be \"Test directory does not exist\" but was \"%s\"", diags[0].Summary)
+		}
+
+		if diags[0].Detail != "Requested test directory testdata/valid-modules/with-tests/not_real does not exist." {
+			t.Errorf("expected detail to be \"Requested test directory testdata/valid-modules/with-tests/not_real does not exist.\" but was \"%s\"", diags[0].Detail)
+		}
+	}
+
+	// Despite the warning, should still have loaded the tests in the
+	// configuration directory.
+	if len(mod.Tests) != 2 {
+		t.Errorf("incorrect number of test files found: %d", len(mod.Tests))
+	}
+}
+
 // TestParseLoadConfigDirFailure is a simple test that just verifies that
 // a number of test configuration directories (in testdata/invalid-modules)
 // produce diagnostics when parsed.

internal/configs/testdata/valid-modules/with-tests-very-nested/main.tf

@@ -0,0 +1,11 @@
+
+variable "input" {
+  type = string
+}
+
+
+resource "foo_resource" "a" {
+  value = var.input
+}
+
+resource "bar_resource" "c" {}

internal/configs/testdata/valid-modules/with-tests-very-nested/very/nested/test_case_one.tftest.hcl

@@ -0,0 +1,31 @@
+variables {
+  input = "default"
+}
+
+# test_run_one runs a partial plan
+run "test_run_one" {
+  command = plan
+
+  plan_options {
+    target = [
+      foo_resource.a
+    ]
+  }
+
+  assert {
+    condition = foo_resource.a.value == "default"
+    error_message = "invalid value"
+  }
+}
+
+# test_run_two does a complete apply operation
+run "test_run_two" {
+  variables {
+    input = "custom"
+  }
+
+  assert {
+    condition = foo_resource.a.value == "custom"
+    error_message = "invalid value"
+  }
+}

internal/configs/testdata/valid-modules/with-tests-very-nested/very/nested/test_case_two.tftest.hcl

@@ -0,0 +1,46 @@
+# test_run_one does a complete apply
+run "test_run_one" {
+  variables {
+    input = "test_run_one"
+  }
+
+  assert {
+    condition = foo_resource.a.value == "test_run_one"
+    error_message = "invalid value"
+  }
+}
+
+# test_run_two does a refresh only apply
+run "test_run_two" {
+  plan_options {
+    mode = refresh-only
+  }
+
+  variables {
+    input = "test_run_two"
+  }
+
+  assert {
+    # value shouldn't change, as we're doing a refresh-only apply.
+    condition = foo_resource.a.value == "test_run_one"
+    error_message = "invalid value"
+  }
+}
+
+# test_run_three does an apply with a replace operation
+run "test_run_three" {
+  variables {
+    input = "test_run_three"
+  }
+
+  plan_options {
+    replace = [
+      bar_resource.c
+    ]
+  }
+
+  assert {
+    condition = foo_resource.a.value == "test_run_three"
+    error_message = "invalid value"
+  }
+}