Draft: TofuPipes experiment v1

Signed-off-by: AbstractionFactory <179820029+abstractionfactory@users.noreply.github.com>

internal/command/arguments/extended.go

@@ -228,8 +228,10 @@ func (o *Operation) Parse() tfdiags.Diagnostics {
 // desirable for the arguments package to handle the gathering of variables
 // directly, returning a map of variable values.
 type Vars struct {
-	vars     *flagNameValueSlice
+	vars        *flagNameValueSlice
-	varFiles *flagNameValueSlice
+	varFiles    *flagNameValueSlice
+	varCommands *flagNameValueSlice
+	varUpstream *flagNameValueSlice
 }
 
 func (v *Vars) All() []FlagNameValue {
@@ -279,10 +281,17 @@ func extendedFlagSet(name string, state *State, operation *Operation, vars *Vars
 	if vars != nil {
 		varsFlags := newFlagNameValueSlice("-var")
 		varFilesFlags := varsFlags.Alias("-var-file")
+		varCommandsFlags := varsFlags.Alias("-var-command")
+		varUpstreamFlags := varsFlags.Alias("-var-upstream")
+
 		vars.vars = &varsFlags
 		vars.varFiles = &varFilesFlags
+		vars.varCommands = &varCommandsFlags
+		vars.varUpstream = &varUpstreamFlags
 		f.Var(vars.vars, "var", "var")
 		f.Var(vars.varFiles, "var-file", "var-file")
+		f.Var(vars.varCommands, "var-command", "var-command")
+		f.Var(vars.varUpstream, "var-upstream", "var-upstream")
 	}
 
 	return f

internal/command/meta.go

@@ -611,8 +611,12 @@ func (m *Meta) varFlagSet(f *flag.FlagSet) {
 	}
 	varValues := m.variableArgs.Alias("-var")
 	varFiles := m.variableArgs.Alias("-var-file")
+	varCommands := m.variableArgs.Alias("-var-command")
+	varUpstream := m.variableArgs.Alias("var-upstream")
 	f.Var(varValues, "var", "variables")
 	f.Var(varFiles, "var-file", "variable file")
+	f.Var(varCommands, "var-command", "run this external command to obtain variable values")
+	f.Var(varUpstream, "var-upstream", "obtain values from this OpenTofu project's outputs")
 }
 
 // extendedFlagSet adds custom flags that are mostly used by commands

internal/command/meta_vars.go

@@ -6,9 +6,12 @@
 package command
 
 import (
+	"bytes"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 
 	"github.com/hashicorp/hcl/v2"
@@ -120,7 +123,22 @@ func (m *Meta) collectVariableValues() (map[string]backend.UnparsedVariableValue
 		case "-var-file":
 			moreDiags := m.addVarsFromFile(rawFlag.Value, tofu.ValueFromNamedFile, ret)
 			diags = diags.Append(moreDiags)
-
+		case "-var-upstream":
+			moreDiags := m.addVarsFromCommand(
+				[]string{os.Args[0], "-chdir=" + rawFlag.Value, "output", "-json"},
+				tofu.ValueFromUpstreamProject,
+				ret,
+			)
+			diags = diags.Append(moreDiags)
+		case "-var-command":
+			var args []string
+			if runtime.GOOS == "" {
+				args = []string{"cmd", "/C"}
+			} else {
+				args = []string{"/bin/sh", "-c"}
+			}
+			moreDiags := m.addVarsFromCommand(append(args, rawFlag.Value), tofu.ValueFromExternalCommand, ret)
+			diags = diags.Append(moreDiags)
 		default:
 			// Should never happen; always a bug in the code that built up
 			// the contents of m.variableArgs.
@@ -163,6 +181,28 @@ func (m *Meta) addVarsFromDir(currDir string, ret map[string]backend.UnparsedVar
 	return diags
 }
 
+func (m *Meta) addVarsFromCommand(args []string, sourceType tofu.ValueSourceType, to map[string]backend.UnparsedVariableValue) tfdiags.Diagnostics {
+	var diags tfdiags.Diagnostics
+	cmd := exec.CommandContext(m.CommandContext(), args[0], args[1:]...)
+	output := &bytes.Buffer{}
+	cmd.Stdout = output
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Error,
+			"Failed to execute external command",
+			err.Error(),
+		))
+		return diags
+	}
+	data := output.Bytes()
+	var json = false
+	if len(data) > 0 && data[0] == '{' {
+		json = true
+	}
+	return m.loadVariableSource("", json, sourceType, to, data)
+}
+
 func (m *Meta) addVarsFromFile(filename string, sourceType tofu.ValueSourceType, to map[string]backend.UnparsedVariableValue) tfdiags.Diagnostics {
 	var diags tfdiags.Diagnostics
 
@@ -184,17 +224,6 @@ func (m *Meta) addVarsFromFile(filename string, sourceType tofu.ValueSourceType,
 		return diags
 	}
 
-	loader, err := m.initConfigLoader()
-	if err != nil {
-		diags = diags.Append(err)
-		return diags
-	}
-
-	// Record the file source code for snippets in diagnostic messages.
-	loader.Parser().ForceFileSource(filename, src)
-
-	var f *hcl.File
-
 	extJSON := strings.HasSuffix(filename, ".json")
 	extTfvars := strings.HasSuffix(filename, DefaultVarsExtension)
 
@@ -202,7 +231,26 @@ func (m *Meta) addVarsFromFile(filename string, sourceType tofu.ValueSourceType,
 	// Ex: -var-file=<(./scripts/vars.sh)
 	detectJSON := !extJSON && !extTfvars && strings.HasPrefix(strings.TrimSpace(string(src)), "{")
 
-	if extJSON || detectJSON {
+	return m.loadVariableSource(filename, extJSON || detectJSON, sourceType, to, src)
+}
+
+func (m *Meta) loadVariableSource(filename string, json bool, sourceType tofu.ValueSourceType, to map[string]backend.UnparsedVariableValue, src []byte) tfdiags.Diagnostics {
+	var diags tfdiags.Diagnostics
+
+	loader, err := m.initConfigLoader()
+	if err != nil {
+		diags = diags.Append(err)
+		return diags
+	}
+
+	// Record the file source code for snippets in diagnostic messages.
+	if filename != "" {
+		loader.Parser().ForceFileSource(filename, src)
+	}
+
+	var f *hcl.File
+
+	if json {
 		var hclDiags hcl.Diagnostics
 		f, hclDiags = hcljson.Parse(src, filename)
 		diags = diags.Append(hclDiags)

internal/tofu/variables.go

@@ -107,6 +107,14 @@ const (
 	// ValueFromCaller indicates that the value was explicitly overridden by
 	// a caller to Context.SetVariable after the context was constructed.
 	ValueFromCaller ValueSourceType = 'S'
+
+	// ValueFromExternalCommand indicates that the value was obtained by executing
+	// an external command.
+	ValueFromExternalCommand ValueSourceType = 'X'
+
+	// ValueFromUpstreamProject indicates that the value was obtained by querying
+	// the state of an upstream project.
+	ValueFromUpstreamProject ValueSourceType = 'U'
 )
 
 func (v *InputValue) GoString() string {