HTTP backend user-defined headers (#1487)

Signed-off-by: David Sims <simsdj82@gmail.com>

CHANGELOG.md

@@ -56,6 +56,7 @@ BUG FIXES:
 * Fix large number will be truncated in plan ([#1382](https://github.com/opentofu/opentofu/pull/1382))
 * S3 backend no longer requires to have permissions to use the default 'env:' workspace prefix ([#1445](https://github.com/opentofu/opentofu/pull/1445))
 * Fixed a crash when using a conditional with Twingate resource ([1446](https://github.com/opentofu/opentofu/pull/1446))
+* Added support for user-defined headers when configuring the HTTP backend ([1427](https://github.com/opentofu/opentofu/pull/1487))
 
 ## Previous Releases
 

internal/backend/remote-state/http/backend.go

@@ -14,6 +14,8 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"regexp"
+	"strings"
 	"time"
 
 	"github.com/hashicorp/go-retryablehttp"
@@ -119,6 +121,32 @@ func New(enc encryption.StateEncryption) backend.Backend {
 				DefaultFunc: schema.EnvDefaultFunc("TF_HTTP_CLIENT_PRIVATE_KEY_PEM", ""),
 				Description: "A PEM-encoded private key, required if client_certificate_pem is specified.",
 			},
+			"headers": &schema.Schema{
+				Type:     schema.TypeMap,
+				Elem:     &schema.Schema{Type: schema.TypeString},
+				Optional: true,
+				ValidateFunc: func(cv interface{}, ck string) ([]string, []error) {
+					nameRegex := regexp.MustCompile("[^a-zA-Z0-9-_]")
+					valueRegex := regexp.MustCompile("[^[:ascii:]]")
+
+					headers := cv.(map[string]interface{})
+					err := make([]error, 0, len(headers))
+					for name, value := range headers {
+						if len(name) == 0 || nameRegex.MatchString(name) {
+							err = append(err, fmt.Errorf(
+								"%s \"%s\" name must not be empty and only contain A-Za-z0-9-_ characters", ck, name))
+						}
+
+						v := value.(string)
+						if len(strings.TrimSpace(v)) == 0 || valueRegex.MatchString(v) {
+							err = append(err, fmt.Errorf(
+								"%s \"%s\" value must not be empty and only contain ascii characters", ck, name))
+						}
+					}
+					return nil, err
+				},
+				Description: "A map of headers, when set will be included with HTTP requests sent to the HTTP backend",
+			},
 		},
 	}
 
@@ -220,6 +248,28 @@ func (b *Backend) configure(ctx context.Context) error {
 
 	unlockMethod := data.Get("unlock_method").(string)
 
+	username := data.Get("username").(string)
+	password := data.Get("password").(string)
+
+	var headers map[string]string
+	if dv, ok := data.GetOk("headers"); ok {
+		dh := dv.(map[string]interface{})
+		headers = make(map[string]string, len(dh))
+
+		for k, v := range dh {
+			switch strings.ToLower(k) {
+			case "authorization":
+				if username != "" {
+					return fmt.Errorf("headers \"%s\" cannot be set when providing username", k)
+				}
+			case "content-type", "content-md5":
+				return fmt.Errorf("headers \"%s\" is reserved", k)
+			default:
+				headers[k] = v.(string)
+			}
+		}
+	}
+
 	rClient := retryablehttp.NewClient()
 	rClient.RetryMax = data.Get("retry_max").(int)
 	rClient.RetryWaitMin = time.Duration(data.Get("retry_wait_min").(int)) * time.Second
@@ -238,8 +288,9 @@ func (b *Backend) configure(ctx context.Context) error {
 		UnlockURL:    unlockURL,
 		UnlockMethod: unlockMethod,
 
-		Username: data.Get("username").(string),
-		Password: data.Get("password").(string),
+		Headers:  headers,
+		Username: username,
+		Password: password,
 
 		// accessible only for testing use
 		Client: rClient,

internal/backend/remote-state/http/backend_test.go

@@ -48,6 +48,10 @@ func TestHTTPClientFactory(t *testing.T) {
 		t.Fatal("Unexpected username or password")
 	}
 
+	if client.Headers != nil {
+		t.Fatal("Unexpected headers")
+	}
+
 	// custom
 	conf = map[string]cty.Value{
 		"address":        cty.StringVal("http://127.0.0.1:8888/foo"),
@@ -61,6 +65,9 @@ func TestHTTPClientFactory(t *testing.T) {
 		"retry_max":      cty.StringVal("999"),
 		"retry_wait_min": cty.StringVal("15"),
 		"retry_wait_max": cty.StringVal("150"),
+		"headers": cty.MapVal(map[string]cty.Value{
+			"user-defined": cty.StringVal("test"),
+		}),
 	}
 
 	b = backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), configs.SynthBody("synth", conf)).(*Backend)
@@ -93,6 +100,10 @@ func TestHTTPClientFactory(t *testing.T) {
 	if client.Client.RetryWaitMax != 150*time.Second {
 		t.Fatalf("Expected retry_wait_max \"%s\", got \"%s\"", 150*time.Second, client.Client.RetryWaitMax)
 	}
+
+	if len(client.Headers) != 1 || client.Headers["user-defined"] != "test" {
+		t.Fatalf("Expected headers \"user-defined\" to be \"test\", got \"%s\"", client.Headers)
+	}
 }
 
 func TestHTTPClientFactoryWithEnv(t *testing.T) {

internal/backend/remote-state/http/client.go

@@ -34,6 +34,7 @@ type httpClient struct {
 
 	// HTTP
 	Client   *retryablehttp.Client
+	Headers  map[string]string
 	Username string
 	Password string
 
@@ -53,7 +54,12 @@ func (c *httpClient) httpRequest(method string, url *url.URL, data *[]byte, what
 	if err != nil {
 		return nil, fmt.Errorf("Failed to make %s HTTP request: %w", what, err)
 	}
-	// Set up basic auth
+
+	// Add user-defined headers
+	for k, v := range c.Headers {
+		req.Header.Set(k, v)
+	}
+
 	if c.Username != "" {
 		req.SetBasicAuth(c.Username, c.Password)
 	}

internal/backend/remote-state/http/client_test.go

@@ -46,6 +46,34 @@ func TestHTTPClient(t *testing.T) {
 	}
 	remote.TestClient(t, p)
 
+	// Test headers
+	c := retryablehttp.NewClient()
+	c.RequestLogHook = func(_ retryablehttp.Logger, req *http.Request, _ int) {
+		// Test user defined header is part of the request
+		v := req.Header.Get("user-defined")
+		if v != "test" {
+			t.Fatalf("Expected header \"user-defined\" with value \"test\", got \"%s\"", v)
+		}
+
+		// Test the content-type header was not overridden
+		v = req.Header.Get("content-type")
+		if req.Method == "PUT" && v != "application/json" {
+			t.Fatalf("Expected header \"content-type\" with value \"application/json\", got \"%s\"", v)
+		}
+	}
+
+	p = &httpClient{
+		URL:          url,
+		UpdateMethod: "PUT",
+		Headers: map[string]string{
+			"user-defined": "test",
+			"content-type": "application/xml",
+		},
+		Client: c,
+	}
+
+	remote.TestClient(t, p)
+
 	// Test locking and alternative UpdateMethod
 	a := &httpClient{
 		URL:          url,

internal/command/apply_test.go

@@ -837,6 +837,7 @@ func TestApply_plan_remoteState(t *testing.T) {
 		"client_ca_certificate_pem": cty.NullVal(cty.String),
 		"client_certificate_pem":    cty.NullVal(cty.String),
 		"client_private_key_pem":    cty.NullVal(cty.String),
+		"headers":                   cty.NullVal(cty.String),
 	})
 	backendConfigRaw, err := plans.NewDynamicValue(backendConfig, backendConfig.Type())
 	if err != nil {