Fixes #2337: External encryption method (#2367)

Signed-off-by: AbstractionFactory <179820029+abstractionfactory@users.noreply.github.com>

internal/encryption/default_registry.go

@@ -7,11 +7,12 @@ package encryption
 
 import (
 	"github.com/opentofu/opentofu/internal/encryption/keyprovider/aws_kms"
-	"github.com/opentofu/opentofu/internal/encryption/keyprovider/external"
+	externalKeyProvider "github.com/opentofu/opentofu/internal/encryption/keyprovider/external"
 	"github.com/opentofu/opentofu/internal/encryption/keyprovider/gcp_kms"
 	"github.com/opentofu/opentofu/internal/encryption/keyprovider/openbao"
 	"github.com/opentofu/opentofu/internal/encryption/keyprovider/pbkdf2"
 	"github.com/opentofu/opentofu/internal/encryption/method/aesgcm"
+	externalMethod "github.com/opentofu/opentofu/internal/encryption/method/external"
 	"github.com/opentofu/opentofu/internal/encryption/method/unencrypted"
 	"github.com/opentofu/opentofu/internal/encryption/registry/lockingencryptionregistry"
 )
@@ -31,12 +32,15 @@ func init() {
 	if err := DefaultRegistry.RegisterKeyProvider(openbao.New()); err != nil {
 		panic(err)
 	}
-	if err := DefaultRegistry.RegisterKeyProvider(external.New()); err != nil {
+	if err := DefaultRegistry.RegisterKeyProvider(externalKeyProvider.New()); err != nil {
 		panic(err)
 	}
 	if err := DefaultRegistry.RegisterMethod(aesgcm.New()); err != nil {
 		panic(err)
 	}
+	if err := DefaultRegistry.RegisterMethod(externalMethod.New()); err != nil {
+		panic(err)
+	}
 	if err := DefaultRegistry.RegisterMethod(unencrypted.New()); err != nil {
 		panic(err)
 	}

internal/encryption/keyprovider/external/protocol.go

@@ -9,6 +9,8 @@ import (
 	"github.com/opentofu/opentofu/internal/encryption/keyprovider"
 )
 
+// TODO #2386 / 1.11: consider if the external method changes and unify protocol with the external key provider.
+
 // HeaderMagic is the magic string that needs to be present in the header to identify
 // the external program as an external keyprovider for OpenTofu.
 const HeaderMagic = "OpenTofu-External-Key-Provider"

internal/encryption/keyprovider/external/testprovider/data/testprovider.py

@@ -17,6 +17,7 @@ if __name__ == "__main__":
 
     # Write the header:
     sys.stdout.write((json.dumps({"magic": "OpenTofu-External-Key-Provider", "version": 1}) + "\n"))
+    sys.stdout.flush()
 
     # Read the input:
     inputData = sys.stdin.read()

internal/encryption/keyprovider/external/testprovider/testprovider.go

@@ -25,6 +25,8 @@ var embedFS embed.FS
 // This binary will always return []byte{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16} as a hard-coded key.
 // You may pass --hello-world to change it to []byte("Hello world! 123")
 func Go(t *testing.T) []string {
+	t.Helper()
+
 	// goMod is embedded like this because the go:embed tag doesn't like having module files in embedded paths.
 	var goMod = []byte(`module testprovider
 
@@ -62,6 +64,8 @@ go 1.22`)
 // run the Python script, including the Python interpreter.
 // This script will always return []byte{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16} as a hard-coded key.
 func Python(t *testing.T) []string {
+	t.Helper()
+
 	tempDir := t.TempDir()
 	dir := path.Join(tempDir, "testprovider-py")
 	if err := os.MkdirAll(dir, 0700); err != nil { //nolint:mnd // This check is stupid
@@ -78,6 +82,8 @@ func Python(t *testing.T) []string {
 // POSIXShell returns a path to a POSIX shell script acting as a key provider.
 // This script will always return []byte{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16} as a hard-coded key.
 func POSIXShell(t *testing.T) []string {
+	t.Helper()
+
 	tempDir := t.TempDir()
 	dir := path.Join(tempDir, "testprovider-sh")
 	if err := os.MkdirAll(dir, 0700); err != nil { //nolint:mnd // This check is stupid

internal/encryption/method/compliancetest/compliance.go

@@ -202,6 +202,9 @@ type EncryptDecryptTestCase[TConfig method.Config, TMethod method.Method] struct
 	ValidEncryptOnlyConfig TConfig
 	// ValidFullConfig is a configuration that contains both an encryption and decryption key.
 	ValidFullConfig TConfig
+	// DecryptCannotBeVerified allows the decryption to succeed unencrypted data. This is needed for methods that
+	// cannot verify if data decrypted successfully (e.g. xor).
+	DecryptCannotBeVerified bool
 }
 
 func (m EncryptDecryptTestCase[TConfig, TMethod]) execute(t *testing.T) {
@@ -248,16 +251,18 @@ func (m EncryptDecryptTestCase[TConfig, TMethod]) execute(t *testing.T) {
 	}
 	typedDecryptError = nil
 
-	_, err = decryptMethod.Decrypt(plainData)
-	if err == nil {
-		compliancetest.Fail(t, "Decrypt() must return an error when decrypting unencrypted data, no error returned.")
-	} else {
-		compliancetest.Log(t, "Decrypt() correctly returned an error when decrypting unencrypted data.")
-	}
-	if !errors.As(err, &typedDecryptError) {
-		compliancetest.Fail(t, "Decrypt() returned a %T instead of a %T when decrypting unencrypted data. Please use the correct typed errors.", err, typedDecryptError)
-	} else {
-		compliancetest.Log(t, "Decrypt() returned the correct error type of %T when decrypting unencrypted data.", typedDecryptError)
+	if !m.DecryptCannotBeVerified {
+		_, err = decryptMethod.Decrypt(plainData)
+		if err == nil {
+			compliancetest.Fail(t, "Decrypt() must return an error when decrypting unencrypted data, no error returned.")
+		} else {
+			compliancetest.Log(t, "Decrypt() correctly returned an error when decrypting unencrypted data.")
+		}
+		if !errors.As(err, &typedDecryptError) {
+			compliancetest.Fail(t, "Decrypt() returned a %T instead of a %T when decrypting unencrypted data. Please use the correct typed errors.", err, typedDecryptError)
+		} else {
+			compliancetest.Log(t, "Decrypt() returned the correct error type of %T when decrypting unencrypted data.", typedDecryptError)
+		}
 	}
 
 	decryptedData, err := decryptMethod.Decrypt(encryptedData)

internal/encryption/method/errors.go

@@ -10,15 +10,20 @@ import "fmt"
 // ErrCryptoFailure indicates a generic cryptographic failure. This error should be embedded into
 // ErrEncryptionFailed, ErrDecryptionFailed, or ErrInvalidConfiguration.
 type ErrCryptoFailure struct {
-	Message string
-	Cause   error
+	Message          string
+	Cause            error
+	SupplementalData string
 }
 
 func (e ErrCryptoFailure) Error() string {
+	result := e.Message
 	if e.Cause != nil {
-		return fmt.Sprintf("%s: %v", e.Message, e.Cause)
+		result += " (" + e.Cause.Error() + ")"
 	}
-	return e.Message
+	if e.SupplementalData != "" {
+		result += "\n-----\n" + e.SupplementalData
+	}
+	return result
 }
 
 func (e ErrCryptoFailure) Unwrap() error {

internal/encryption/method/external/README.md

@@ -0,0 +1,25 @@
+# External encryption method
+
+
+> [!WARNING]
+> This file is not an end-user documentation, it is intended for developers. Please follow the user documentation on the OpenTofu website unless you want to work on the encryption code.
+
+This directory contains the `external` encryption method. You can configure it like this:
+
+```hcl
+terraform {
+  encryption {
+    method "external" "foo" {
+      keys = key_provider.some.provider
+      encrypt_command = ["/path/to/binary", "arg1", "arg2"]
+      decrypt_command = ["/path/to/binary", "arg1", "arg2"]
+    }
+  }
+}
+```
+
+The external method must implement the following protocol:
+
+1. On start, the method binary must emit the header line matching [the header schema](protocol/header.schema.json) on the standard output.
+2. OpenTofu supplies the input metadata matching [the input schema](protocol/input.schema.json) on the standard input.
+3. The method binary must emit the output matching [the output schema](protocol/output.schema.json) on the standard output.

internal/encryption/method/external/command.go

@@ -0,0 +1,188 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package external
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/opentofu/opentofu/internal/encryption/keyprovider"
+	"github.com/opentofu/opentofu/internal/encryption/method"
+)
+
+type command struct {
+	keys           *keyprovider.Output
+	encryptCommand []string
+	decryptCommand []string
+}
+
+func (c command) Encrypt(data []byte) ([]byte, error) {
+	var key []byte
+	if c.keys != nil {
+		key = c.keys.EncryptionKey
+	}
+	input := InputV1{
+		Key:     key,
+		Payload: data,
+	}
+	result, err := c.run(c.encryptCommand, input)
+	if err != nil {
+		return nil, &method.ErrEncryptionFailed{
+			Cause: err,
+		}
+	}
+	return result, nil
+}
+
+func (c command) Decrypt(data []byte) ([]byte, error) {
+	var key []byte
+	if c.keys != nil {
+		key = c.keys.DecryptionKey
+		if len(c.keys.EncryptionKey) > 0 && len(key) == 0 {
+			return nil, &method.ErrDecryptionKeyUnavailable{}
+		}
+	}
+	if len(data) == 0 {
+		return nil, &method.ErrDecryptionFailed{Cause: &method.ErrCryptoFailure{
+			Message: "Cannot decrypt empty data.",
+		}}
+	}
+	input := InputV1{
+		Key:     key,
+		Payload: data,
+	}
+	result, err := c.run(c.decryptCommand, input)
+	if err != nil {
+		return nil, &method.ErrDecryptionFailed{
+			Cause: err,
+		}
+	}
+	return result, nil
+}
+
+func (c command) run(command []string, input any) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+
+	inputData, err := json.Marshal(input)
+	if err != nil {
+		return nil, &method.ErrCryptoFailure{
+			Message: "failed to marshal input",
+			Cause:   err,
+		}
+	}
+
+	stderr := &bytes.Buffer{}
+
+	cmd := exec.CommandContext(ctx, command[0], command[1:]...) //nolint:gosec //Launching external commands here is the entire point.
+
+	handler := &ioHandler{
+		false,
+		bytes.NewBuffer(inputData),
+		[]byte{},
+		cancel,
+		nil,
+	}
+
+	cmd.Stdin = handler
+	cmd.Stdout = handler
+	cmd.Stderr = stderr
+	if err := cmd.Run(); err != nil {
+		if handler.err != nil {
+			return nil, &method.ErrCryptoFailure{
+				Message:          "external encryption method failure",
+				Cause:            handler.err,
+				SupplementalData: fmt.Sprintf("Stderr:\n-------\n%s\n", stderr.String()),
+			}
+		}
+
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			if exitErr.ExitCode() != 0 {
+				return nil, &method.ErrCryptoFailure{
+					Message:          "external encryption method exited with non-zero exit code",
+					Cause:            err,
+					SupplementalData: fmt.Sprintf("Stderr:\n-------\n%s\n", stderr.String()),
+				}
+			}
+		}
+		return nil, &method.ErrCryptoFailure{
+			Message:          "external encryption method exited with an error",
+			Cause:            err,
+			SupplementalData: fmt.Sprintf("Stderr:\n-------\n%s\n", stderr.String()),
+		}
+	}
+
+	var result *OutputV1
+	decoder := json.NewDecoder(bytes.NewReader(handler.output))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&result); err != nil {
+		return nil, &method.ErrCryptoFailure{
+			Message:          "external encryption method returned an invalid JSON",
+			Cause:            err,
+			SupplementalData: fmt.Sprintf("Stderr:\n-------\n%s\n", stderr.String()),
+		}
+	}
+
+	return result.Payload, nil
+}
+
+type ioHandler struct {
+	headerFinished bool
+	input          *bytes.Buffer
+	output         []byte
+	cancel         func()
+	err            error
+}
+
+func (i *ioHandler) Write(p []byte) (int, error) {
+	i.output = append(i.output, p...)
+	n := len(p)
+	if i.headerFinished {
+		// Header is finished, just collect the output.
+		return n, nil
+	}
+	// Check if the full header is present.
+	parts := strings.SplitN(string(i.output), "\n", 2) //nolint:mnd //This rule is dumb.
+	if len(parts) == 1 {
+		return n, nil
+	}
+	var header Header
+	// Note: this is intentionally not using strict decoding. Later protocol versions may introduce additional header
+	// fields.
+	if jsonErr := json.Unmarshal([]byte(parts[0]), &header); jsonErr != nil {
+		err := fmt.Errorf("failed to unmarshal header from external method (%w)", jsonErr)
+		i.err = err
+		i.cancel()
+		return n, err
+	}
+
+	if header.Magic != Magic {
+		err := fmt.Errorf("invalid magic received from external method: %s", header.Magic)
+		i.err = err
+		i.cancel()
+		return n, err
+	}
+	if header.Version != 1 {
+		err := fmt.Errorf("invalid version number received from external method: %d", header.Version)
+		i.err = err
+		i.cancel()
+		return n, err
+	}
+	i.headerFinished = true
+	i.output = []byte(parts[1])
+	return n, nil
+}
+
+func (i *ioHandler) Read(p []byte) (int, error) {
+	return i.input.Read(p)
+}

internal/encryption/method/external/compliance_test.go

@@ -0,0 +1,92 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package external
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/opentofu/opentofu/internal/encryption/keyprovider"
+	"github.com/opentofu/opentofu/internal/encryption/method/compliancetest"
+	"github.com/opentofu/opentofu/internal/encryption/method/external/testmethod"
+)
+
+func TestComplianceBinary(t *testing.T) {
+	runTest(t, testmethod.Go(t))
+}
+
+func TestCompliancePython(t *testing.T) {
+	runTest(t, testmethod.Python(t))
+}
+
+func runTest(t *testing.T, cmd []string) {
+	encryptCommand := append(cmd, "--encrypt") //nolint:gocritic //It's intentionally a different slice.
+	decryptCommand := append(cmd, "--decrypt") //nolint:gocritic //It's intentionally a different slice.
+
+	compliancetest.ComplianceTest(t, compliancetest.TestConfiguration[*descriptor, *Config, *command]{
+		Descriptor: New().(*descriptor), //nolint:errcheck //This is safe.
+		HCLParseTestCases: map[string]compliancetest.HCLParseTestCase[*descriptor, *Config, *command]{
+			"empty": {
+				HCL:        `method "external" "foo" {}`,
+				ValidHCL:   false,
+				ValidBuild: false,
+				Validate:   nil,
+			},
+			"empty-command": {
+				HCL: `method "external" "foo" {
+	encrypt_command = []
+	decrypt_command = []
+}`,
+				ValidHCL: true,
+			},
+			"command": {
+				HCL: fmt.Sprintf(`method "external" "foo" {
+	encrypt_command = ["%s"]
+	decrypt_command = ["%s"]
+}`, strings.Join(encryptCommand, `","`), strings.Join(decryptCommand, `","`)),
+				ValidHCL:   true,
+				ValidBuild: true,
+				Validate: func(config *Config, method *command) error {
+					if !slices.Equal(config.EncryptCommand, encryptCommand) {
+						return fmt.Errorf("incorrect encrypt command after HCL parsing")
+					}
+					if !slices.Equal(config.DecryptCommand, decryptCommand) {
+						return fmt.Errorf("incorrect decrypt command after HCL parsing")
+					}
+					return nil
+				},
+			},
+		},
+		ConfigStructTestCases: map[string]compliancetest.ConfigStructTestCase[*Config, *command]{
+			"empty": {
+				Config:     &Config{},
+				ValidBuild: false,
+				Validate:   nil,
+			},
+		},
+		EncryptDecryptTestCase: compliancetest.EncryptDecryptTestCase[*Config, *command]{
+			ValidEncryptOnlyConfig: &Config{
+				Keys: &keyprovider.Output{
+					EncryptionKey: []byte{20},
+					DecryptionKey: nil,
+				},
+				EncryptCommand: encryptCommand,
+				DecryptCommand: decryptCommand,
+			},
+			ValidFullConfig: &Config{
+				Keys: &keyprovider.Output{
+					EncryptionKey: []byte{20},
+					DecryptionKey: []byte{20},
+				},
+				EncryptCommand: encryptCommand,
+				DecryptCommand: decryptCommand,
+			},
+			DecryptCannotBeVerified: true,
+		},
+	})
+}

internal/encryption/method/external/config.go

@@ -0,0 +1,55 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package external
+
+import (
+	"github.com/opentofu/opentofu/internal/encryption/keyprovider"
+	"github.com/opentofu/opentofu/internal/encryption/method"
+)
+
+// Config is the configuration for the AES-GCM method.
+type Config struct {
+	Keys           *keyprovider.Output `hcl:"keys,optional" json:"keys,omitempty" yaml:"keys"`
+	EncryptCommand []string            `hcl:"encrypt_command" json:"encrypt_command" yaml:"encrypt_command"`
+	DecryptCommand []string            `hcl:"decrypt_command" json:"decrypt_command" yaml:"decrypt_command"`
+}
+
+// Build checks the validity of the configuration and returns a ready-to-use AES-GCM implementation.
+func (c *Config) Build() (method.Method, error) {
+	if len(c.EncryptCommand) < 1 {
+		return nil, &method.ErrInvalidConfiguration{
+			Cause: &method.ErrCryptoFailure{
+				Message: "the encrypt_command option is required",
+			},
+		}
+	}
+	if len(c.EncryptCommand[0]) == 0 {
+		return nil, &method.ErrInvalidConfiguration{
+			Cause: &method.ErrCryptoFailure{
+				Message: "the first entry of encrypt_command must not be empty",
+			},
+		}
+	}
+	if len(c.DecryptCommand) < 1 {
+		return nil, &method.ErrInvalidConfiguration{
+			Cause: &method.ErrCryptoFailure{
+				Message: "the decrypt_command option is required",
+			},
+		}
+	}
+	if len(c.DecryptCommand[0]) == 0 {
+		return nil, &method.ErrInvalidConfiguration{
+			Cause: &method.ErrCryptoFailure{
+				Message: "the first entry of decrypt_command must not be empty",
+			},
+		}
+	}
+	return &command{
+		keys:           c.Keys,
+		encryptCommand: c.EncryptCommand,
+		decryptCommand: c.DecryptCommand,
+	}, nil
+}

internal/encryption/method/external/descriptor.go

@@ -0,0 +1,38 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package external
+
+import (
+	"github.com/opentofu/opentofu/internal/encryption/method"
+)
+
+// Descriptor integrates the method.Descriptor and provides a TypedConfig for easier configuration.
+type Descriptor interface {
+	method.Descriptor
+
+	// TypedConfig returns a config typed for this method.
+	TypedConfig() *Config
+}
+
+// New creates a new descriptor for the AES-GCM encryption method, which requires a 32-byte key.
+func New() Descriptor {
+	return &descriptor{}
+}
+
+type descriptor struct {
+}
+
+func (f *descriptor) TypedConfig() *Config {
+	return &Config{}
+}
+
+func (f *descriptor) ID() method.ID {
+	return "external"
+}
+
+func (f *descriptor) ConfigStruct() method.Config {
+	return f.TypedConfig()
+}

internal/encryption/method/external/protocol.go

@@ -0,0 +1,36 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package external
+
+// TODO #2386 / 1.11: consider if the external method changes and unify protocol with the external key provider.
+
+// Magic is the magic string the external method needs to output in the Header.
+const Magic = "OpenTofu-External-Encryption-Method"
+
+// Header is the initial message the external method writes to stdout as a single-line JSON.
+type Header struct {
+	// Magic must always be "OpenTofu-External-Encryption-Method"
+	Magic string `json:"magic"`
+	// Version must always be 1.
+	Version int `json:"version"`
+}
+
+// InputV1 is an encryption/decryption request from OpenTofu to the external method. OpenTofu writes this message
+// to the standard input of the external method as a JSON message.
+type InputV1 struct {
+	// Key is the encryption or decryption key for this operation. On the wire, this is base64-encoded. If no key is
+	// present, this will be nil. The method should exit with a non-zero exit code.
+	Key []byte `json:"key,omitempty"`
+	// Payload is the payload to encrypt/decrypt.
+	Payload []byte `json:"payload"`
+}
+
+// OutputV1 is the returned encrypted/decrypted payload from the external method. The external method writes this
+// to the standard output as JSON.
+type OutputV1 struct {
+	// Payload is the payload that has been encrypted/decrypted by the external method.
+	Payload []byte `json:"payload"`
+}

internal/encryption/method/external/protocol/header.schema.json

@@ -0,0 +1,22 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/opentofu/opentofu/main/internal/encryption/keyprovider/externalcommand/protocol/header.schema.json",
+  "title": "OpenTofu External Encryption Method Header",
+  "description": "Header line output when an external method binary is launched. This must be written on a single line followed by a newline character. Note that the header may contain additional fields in later protocol versions.",
+  "type": "object",
+  "properties": {
+    "magic": {
+      "title": "Magic string",
+      "description": "Magic string identifying the external method as such.",
+      "type": "string",
+      "enum": ["OpenTofu-External-Encryption-Method"]
+    },
+    "version": {
+      "title": "Protocol version number",
+      "type": "integer",
+      "enum": [1]
+    }
+  },
+  "required": ["magic","version"],
+  "additionalProperties": true
+}

internal/encryption/method/external/protocol/input.schema.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/opentofu/opentofu/main/internal/encryption/keyprovider/externalcommand/protocol/input.schema.json",
+  "title": "OpenTofu External Encryption Method Input",
+  "description": "Input schema for the OpenTofu external encryption method protocol. The external encryption method must read the input from stdin and write the output to stdout. It may write to stderr to provide more error details.",
+  "type": "object",
+  "properties": {
+    "key": {
+      "title": "Key",
+      "description": "If present, this will contain the encryption or decryption key material. If no key is present (e.g. because no key provider is configured) this field will be missing.",
+      "type": "string",
+      "contentEncoding": "base64",
+      "contentMediaType": "application/octet-stream"
+    },
+    "payload": {
+      "title": "Payload",
+      "description": "The payload that should be encrypted/decrypted.",
+      "type": "string",
+      "contentEncoding": "base64",
+      "contentMediaType": "application/octet-stream"
+    }
+  },
+  "required": ["payload"],
+  "additionalProperties": false
+}

internal/encryption/method/external/protocol/output.schema.json

@@ -0,0 +1,18 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/opentofu/opentofu/main/internal/encryption/keyprovider/externalcommand/protocol/output.schema.json",
+  "title": "OpenTofu External Encryption Method Output",
+  "description": "Output schema for the OpenTofu external encryption method protocol. The external provider must read the input from stdin and write the output to stdout. It may write to stderr to provide more error details.",
+  "type": "object",
+  "properties": {
+    "payload": {
+      "title": "Payload",
+      "description": "The encrypted/decrypted data.",
+      "type": "string",
+      "contentEncoding": "base64",
+      "contentMediaType": "application/octet-stream"
+    }
+  },
+  "required": ["payload"],
+  "additionalProperties": false
+}

internal/encryption/method/external/testmethod/data/testmethod.go

@@ -0,0 +1,71 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"log"
+	"os"
+)
+
+type Header struct {
+	Magic   string `json:"magic"`
+	Version int    `json:"version"`
+}
+
+type Input struct {
+	Key     []byte `json:"key,omitempty"`
+	Payload []byte `json:"payload"`
+}
+
+type Output struct {
+	// Payload is the payload that has been encrypted/decrypted by the external method.
+	Payload []byte `json:"payload"`
+}
+
+// main implements a simple XOR-encryption. This is meant as an example and not suitable for any production use.
+func main() {
+	// Write logs to stderr
+	log.Default().SetOutput(os.Stderr)
+
+	// Write header
+	header := Header{
+		"OpenTofu-External-Encryption-Method",
+		1,
+	}
+	marshalledHeader, err := json.Marshal(header)
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+	_, _ = os.Stdout.Write(append(marshalledHeader, []byte("\n")...))
+
+	// Read input
+	input, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		log.Fatalf("Failed to read stdin: %v", err)
+	}
+	var inputData Input
+	if err = json.Unmarshal(input, &inputData); err != nil {
+		log.Fatalf("Failed to parse stdin: %v", err)
+	}
+
+	// Create output as an XOR of the key and input
+	outputPayload := make([]byte, len(inputData.Payload))
+	for i, b := range inputData.Payload {
+		outputPayload[i] = inputData.Key[i%len(inputData.Key)] ^ b
+	}
+
+	// Write output
+	output := Output{
+		Payload: outputPayload,
+	}
+	outputData, err := json.Marshal(output)
+	if err != nil {
+		log.Fatalf("Failed to stringify output: %v", err)
+	}
+	_, _ = os.Stdout.Write(outputData)
+}

internal/encryption/method/external/testmethod/data/testmethod.py

@@ -0,0 +1,38 @@
+#!/usr/bin/python
+# Copyright (c) The OpenTofu Authors
+# SPDX-License-Identifier: MPL-2.0
+# Copyright (c) 2023 HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+import base64
+import json
+import sys
+
+if __name__ == "__main__":
+    # Make sure that this program isn't running interactively:
+    if sys.stdout.isatty():
+        sys.stderr.write("This is an OpenTofu encryption method and is not meant to be run interactively. "
+                         "Please configure this program in your OpenTofu encryption block to use it.\n")
+        sys.exit(1)
+
+    # Write the header:
+    sys.stdout.write((json.dumps({"magic": "OpenTofu-External-Encryption-Method", "version": 1}) + "\n"))
+    sys.stdout.flush()
+
+    # Read the input:
+    inputData = sys.stdin.read()
+    data = json.loads(inputData)
+
+    key = base64.b64decode(data["key"])
+    payload = base64.b64decode(data["payload"])
+
+    # Encrypt the data:
+    outputPayload = bytearray()
+    for i in range(0, len(payload)):
+        b = payload[i]
+        outputPayload.append(key[i%len(key)] ^ b)
+
+    # Write the output:
+    sys.stdout.write(json.dumps({
+        "payload": base64.b64encode(outputPayload).decode('ascii'),
+    }))

internal/encryption/method/external/testmethod/testmethod.go

@@ -0,0 +1,105 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package testmethod
+
+import (
+	"context"
+	"embed"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+)
+
+//go:embed data/*
+var embedFS embed.FS
+
+// Go builds a key provider as a Go binary and returns its path.
+func Go(t *testing.T) []string {
+	t.Helper()
+
+	// goMod is embedded like this because the go:embed tag doesn't like having module files in embedded paths.
+	var goMod = []byte(`module testmethod
+
+go 1.22`)
+
+	tempDir := t.TempDir()
+	dir := path.Join(tempDir, "testmethod-go")
+	if err := os.MkdirAll(dir, 0700); err != nil { //nolint:mnd // This check is stupid
+		t.Errorf("Failed to create temporary directory (%v)", err)
+	}
+
+	if err := os.WriteFile(path.Join(dir, "go.mod"), goMod, 0600); err != nil { //nolint:mnd // This check is stupid
+		t.Errorf("%v", err)
+	}
+	if err := ejectFile("testmethod.go", path.Join(dir, "testmethod.go")); err != nil {
+		t.Errorf("%v", err)
+	}
+	targetBinary := path.Join(dir, "testmethod")
+	if runtime.GOOS == "windows" {
+		targetBinary += ".exe"
+	}
+	t.Logf("\033[32mCompiling test method binary...\033[0m")
+	cmd := exec.Command("go", "build", "-o", targetBinary)
+	cmd.Dir = dir
+	// TODO move this to a proper test logger once available.
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		t.Skipf("Failed to build test method binary (%v)", err)
+	}
+	return []string{targetBinary}
+}
+
+// Python returns the path to a Python script acting as an encryption method. The function returns all arguments
+// required to run the Python script, including the Python interpreter.
+func Python(t *testing.T) []string {
+	t.Helper()
+
+	tempDir := t.TempDir()
+	dir := path.Join(tempDir, "testmethod-py")
+	if err := os.MkdirAll(dir, 0700); err != nil { //nolint:mnd // This check is stupid
+		t.Errorf("Failed to create temporary directory (%v)", err)
+	}
+	target := path.Join(dir, "testmethod.py")
+	if err := ejectFile("testmethod.py", target); err != nil {
+		t.Errorf("%v", err)
+	}
+	python := findExecutable(t, []string{"python", "python3"}, []string{"--version"})
+	return []string{python, target}
+}
+
+func ejectFile(file string, target string) error {
+	contents, err := embedFS.ReadFile(path.Join("data", file))
+	if err != nil {
+		return fmt.Errorf("failed to read %s file from embedded dataset (%w)", file, err)
+	}
+	if err := os.WriteFile(target, contents, 0600); err != nil { //nolint:mnd // This check is stupid
+		return fmt.Errorf("failed to create %s file at %s (%w)", file, target, err)
+	}
+	return nil
+}
+
+func findExecutable(t *testing.T, options []string, testArguments []string) string {
+	for _, opt := range options {
+		var lastError error
+		func() {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+			defer cancel()
+			cmd := exec.CommandContext(ctx, opt, testArguments...)
+			lastError = cmd.Run()
+		}()
+		if lastError == nil {
+			return opt
+		}
+	}
+	t.Skipf("No viable alternative found between %s", strings.Join(options, ", "))
+	return ""
+}

website/docs/language/state/encryption.mdx

@@ -23,6 +23,12 @@ import ExternalOutput from '!!raw-loader!./examples/encryption/keyprovider-exter
 import ExternalGo from '!!raw-loader!./examples/encryption/keyprovider-external-provider.go'
 import ExternalPython from '!!raw-loader!./examples/encryption/keyprovider-external-provider.py'
 import ExternalSH from '!!raw-loader!./examples/encryption/keyprovider-external-provider.sh'
+import ExternalMethod from '!!raw-loader!./examples/encryption/external-method/method-external.tofu'
+import ExternalMethodHeader from '!!raw-loader!./examples/encryption/external-method/method-external-header.json'
+import ExternalMethodInput from '!!raw-loader!./examples/encryption/external-method/method-external-input.json'
+import ExternalMethodOutput from '!!raw-loader!./examples/encryption/external-method/method-external-output.json'
+import ExternalMethodGo from '!!raw-loader!./examples/encryption/external-method/method-external-method.go'
+import ExternalMethodPython from '!!raw-loader!./examples/encryption/external-method/method-external-method.py'
 import Sample from '!!raw-loader!./examples/encryption/sample.tf'
 import Fallback from '!!raw-loader!./examples/encryption/fallback.tf'
 import FallbackFromUnencrypted from '!!raw-loader!./examples/encryption/fallback_from_unencrypted.tf'
@@ -325,6 +331,70 @@ AES-GCM is a secure, industry-standard encryption algorithm, but suffers from "k
 
 :::
 
+### External (experimental)
+
+The external command method lets you run external commands in order to perform encryption and decryption. These programs must be specifically written to work with OpenTofu. This key provider has the following fields:
+
+| Option            | Description                                                                                          | Min. | Default |
+|-------------------|------------------------------------------------------------------------------------------------------|------|---------|
+| `encrypt_command` | External command to run for encryption in an array format, each parameter being an item in an array. | 1    |         |
+| `decrypt_command` | External command to run for decryption in an array format, each parameter being an item in an array. | 1    |         |
+| `keys`            | Reference to a key provider if the external command requires keys.                                   |      |         |
+
+For example, you can configure the external program as follows:
+
+<CodeBlock language="hcl">{ExternalMethod}</CodeBlock>
+
+#### Writing an external method
+
+An external method can be anything as long as it is runnable as an application. The protocol consists of 3 steps:
+
+1. The external program writes the header to the standard output.
+2. OpenTofu sends the key material and data to encrypt/decrypt to the external program over the standard input.
+3. The external program writes the encrypted/decrypted data to the standard output.
+
+<Tabs>
+    <TabItem value="step1" label="Step 1: Writing the header" default>
+        As a first step, the external program must output a header to the standard output so OpenTofu knows it is a valid external method. The header must always be a single line and contain the following:
+        <CodeBlock language={"json"}>{ExternalMethodHeader}</CodeBlock>
+        <Button
+            href="https://github.com/opentofu/opentofu/tree/main/internal/encryption/method/external/protocol/header.schema.json"
+            className="inline-flex"
+            target="_blank"
+        >
+            Open JSON schema file
+        </Button>
+    </TabItem>
+    <TabItem value="step2" label="Step 2: Reading the input">
+        Once the header is written, OpenTofu writes the key material and the data to process to the standard input of the external program. The key material may not be present if no key provider is configured. The input will always have the following format:
+        <CodeBlock language={"json"}>{ExternalMethodInput}</CodeBlock>
+        <Button
+            href="https://github.com/opentofu/opentofu/tree/main/internal/encryption/method/external/protocol/input.schema.json"
+            className="inline-flex"
+            target="_blank"
+        >
+            Open JSON schema file
+        </Button>
+    </TabItem>
+    <TabItem value="step3" label="Step 3: Writing the output">
+        With the input, the external program can now construct the output.
+        <CodeBlock language={"json"}>{ExternalMethodOutput}</CodeBlock>
+        <Button
+            href="https://github.com/opentofu/opentofu/tree/main/internal/encryption/method/external/protocol/output.schema.json"
+            className="inline-flex"
+            target="_blank"
+        >
+            Open JSON schema file
+        </Button>
+    </TabItem>
+    <TabItem value="example-go" label="Example: Go">
+        <CodeBlock language={"go"}>{ExternalMethodGo}</CodeBlock>
+    </TabItem>
+    <TabItem value="example-python" label="Example: Python">
+        <CodeBlock language={"python"}>{ExternalMethodPython}</CodeBlock>
+    </TabItem>
+</Tabs>
+
 ### Unencrypted
 
 The `unencrypted` method is used to provide an explicit migration path to and from encryption.  It takes no configuration and can be seen in use above in the [Initial Setup](#initial-setup) block.

website/docs/language/state/examples/encryption/external-method/method-external-header.json

@@ -0,0 +1 @@
+{"magic":"OpenTofu-External-Encryption-Method","version":1}

website/docs/language/state/examples/encryption/external-method/method-external-input.json

@@ -0,0 +1,4 @@
+{
+  "payload": "base64-encoded payload to encrypt or decrypt",
+  "key": "base64-encoded key for encryption or decryption (optional)"
+}

website/docs/language/state/examples/encryption/external-method/method-external-method.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"log"
+	"os"
+)
+
+// Header is the initial line that needs to be written as JSON when the program starts.
+type Header struct {
+	Magic   string `json:"magic"`
+	Version int    `json:"version"`
+}
+
+// Input is the input data received from OpenTofu in response to the header as JSON.
+type Input struct {
+	// Key is the encryption or decryption key, if present.
+	Key []byte `json:"key,omitempty"`
+	// Payload is the data to encrypt/decrypt.
+	Payload []byte `json:"payload"`
+}
+
+// Output is the data structure that should be written to the output.
+type Output struct {
+	// Payload is the payload that has been encrypted/decrypted by the external method.
+	Payload []byte `json:"payload"`
+}
+
+func main() {
+	// Write logs to stderr
+	log.Default().SetOutput(os.Stderr)
+
+	// Write header:
+	header := Header{
+		"OpenTofu-External-Encryption-Method",
+		1,
+	}
+	marshalledHeader, err := json.Marshal(header)
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+	_, err = os.Stdout.Write(append(marshalledHeader, []byte("\n")...))
+	if err != nil {
+		log.Fatalf("Failed to write output: %v", err)
+	}
+
+	// Read input:
+	input, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		log.Fatalf("Failed to read stdin: %v", err)
+	}
+	var inputData Input
+	if err = json.Unmarshal(input, &inputData); err != nil {
+		log.Fatalf("Failed to parse stdin: %v", err)
+	}
+
+	// Encrypt/decrypt the input and produce the output here.
+	var outputPayload []byte
+	if len(os.Args) != 2 {
+		log.Fatalf("Expected --encrypt or --decrypt")
+	}
+	switch os.Args[1] {
+	case "--encrypt":
+		// Encrypt the payload
+	case "--decrypt":
+		// Decrypt the payload
+	default:
+		log.Fatalf("Expected --encrypt or --decrypt")
+	}
+
+	// Write output
+	output := Output{
+		Payload: outputPayload,
+	}
+	outputData, err := json.Marshal(output)
+	if err != nil {
+		log.Fatalf("Failed to stringify output: %v", err)
+	}
+	_, err = os.Stdout.Write(outputData)
+	if err != nil {
+		log.Fatalf("Failed to write output: %v", err)
+	}
+}

website/docs/language/state/examples/encryption/external-method/method-external-method.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python
+import argparse
+import base64
+import json
+import sys
+
+if __name__ == "__main__":
+    # Make sure that this program isn't running interactively:
+    if sys.stdout.isatty():
+        sys.stderr.write("This is an OpenTofu encryption method and is not meant to be run interactively. "
+                         "Please configure this program in your OpenTofu encryption block to use it.\n")
+        sys.exit(1)
+    parser = argparse.ArgumentParser(prog='My External Encryption Method')
+    parser.add_argument('--encrypt', action='store_true')
+    parser.add_argument('--decrypt', action='store_true')
+    args = parser.parse_args()
+
+    # Write the header:
+    sys.stdout.write((json.dumps({"magic": "OpenTofu-External-Encryption-Method", "version": 1}) + "\n"))
+    sys.stdout.flush()
+
+    # Read the input:
+    inputData = sys.stdin.read()
+    data = json.loads(inputData)
+
+    key = base64.b64decode(data["key"])
+    payload = base64.b64decode(data["payload"])
+
+    # Produce the output payload here.
+    if args.encrypt:
+        outputPayload = b''
+    elif args.decrypt:
+        outputPayload = b''
+    else:
+        raise "Expected --encrypt or --decrypt."
+
+    # Write the output:
+    sys.stdout.write(json.dumps({
+        "payload": base64.b64encode(outputPayload).decode('ascii'),
+    }))

website/docs/language/state/examples/encryption/external-method/method-external-output.json

@@ -0,0 +1,3 @@
+{
+  "payload": "base64-encoded encrypted or decrypted payload"
+}

website/docs/language/state/examples/encryption/external-method/method-external.tofu

@@ -0,0 +1,10 @@
+terraform {
+  encryption {
+    key_provider "external" "foo" {
+      encrypt_command = ["./some_program", "--encrypt"]
+      decrypt_command = ["./some_program", "--decrypt"]
+      # Optional:
+      keys = key_provider.some.provider
+    }
+  }
+}

website/docs/language/state/examples/encryption/keyprovider-external-provider.py

@@ -9,6 +9,7 @@ if __name__ == "__main__":
     sys.stdout.write((json.dumps(
         {"magic": "OpenTofu-External-Key-Provider", "version": 1}) + "\n"
     ))
+    sys.stdout.flush()
 
     # Read the input:
     inputData = sys.stdin.read()