Improve environment variable support for the pg backend (#33045)

* Improve environment variable support for the pg backend

This patch does two things:
  - it adds environment variable support to the parameters that did
    not have it (and uses `PG_CONN_STR` instead of `PGDATABASE` which is
    actually more appropriate to match the behavior of other PostgreSQL
    utilities)
  - better documents how to give the connection parameters as environment
    variables for the ones that were already supported based on the
	recommendation of @bsouth00

I will prepare a backport of the documentation part of this once it is
merged.

Closes https://github.com/hashicorp/terraform/issues/33024

* Remove global variable in test of the PG backend

internal/backend/remote-state/pg/backend.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"os"
+	"strconv"
 
 	"github.com/hashicorp/terraform/internal/backend"
 	"github.com/hashicorp/terraform/internal/legacy/helper/schema"
@@ -15,41 +17,53 @@ const (
 	statesIndexName = "states_by_name"
 )
 
+func defaultBoolFunc(k string, dv bool) schema.SchemaDefaultFunc {
+	return func() (interface{}, error) {
+		if v := os.Getenv(k); v != "" {
+			return strconv.ParseBool(v)
+		}
+
+		return dv, nil
+	}
+}
+
 // New creates a new backend for Postgres remote state.
 func New() backend.Backend {
 	s := &schema.Backend{
 		Schema: map[string]*schema.Schema{
 			"conn_str": {
 				Type:        schema.TypeString,
-				Required:    true,
+				Optional:    true,
 				Description: "Postgres connection string; a `postgres://` URL",
-				DefaultFunc: schema.EnvDefaultFunc("PGDATABASE", nil),
+				DefaultFunc: schema.EnvDefaultFunc("PG_CONN_STR", nil),
 			},
 
 			"schema_name": {
 				Type:        schema.TypeString,
 				Optional:    true,
 				Description: "Name of the automatically managed Postgres schema to store state",
-				Default:     "terraform_remote_state",
+				DefaultFunc: schema.EnvDefaultFunc("PG_SCHEMA_NAME", "terraform_remote_state"),
 			},
 
 			"skip_schema_creation": {
 				Type:        schema.TypeBool,
 				Optional:    true,
 				Description: "If set to `true`, Terraform won't try to create the Postgres schema",
-				Default:     false,
+				DefaultFunc: defaultBoolFunc("PG_SKIP_SCHEMA_CREATION", false),
 			},
 
 			"skip_table_creation": {
 				Type:        schema.TypeBool,
 				Optional:    true,
 				Description: "If set to `true`, Terraform won't try to create the Postgres table",
+				DefaultFunc: defaultBoolFunc("PG_SKIP_TABLE_CREATION", false),
 			},
 
 			"skip_index_creation": {
 				Type:        schema.TypeBool,
 				Optional:    true,
 				Description: "If set to `true`, Terraform won't try to create the Postgres index",
+				DefaultFunc: defaultBoolFunc("PG_SKIP_INDEX_CREATION", false),
 			},
 		},
 	}

internal/backend/remote-state/pg/backend_test.go

@@ -6,6 +6,7 @@ package pg
 import (
 	"database/sql"
 	"fmt"
+	"net/url"
 	"os"
 	"strings"
 	"testing"
@@ -22,15 +23,22 @@ import (
 //
 // A running Postgres server identified by env variable
 // DATABASE_URL is required for acceptance tests.
-func testACC(t *testing.T) {
+func testACC(t *testing.T) string {
 	skip := os.Getenv("TF_ACC") == ""
 	if skip {
 		t.Log("pg backend tests require setting TF_ACC")
 		t.Skip()
 	}
-	if os.Getenv("DATABASE_URL") == "" {
-		os.Setenv("DATABASE_URL", "postgres://localhost/terraform_backend_pg_test?sslmode=disable")
+	databaseUrl, found := os.LookupEnv("DATABASE_URL")
+	if !found {
+		databaseUrl = "postgres://localhost/terraform_backend_pg_test?sslmode=disable"
+		os.Setenv("DATABASE_URL", databaseUrl)
 	}
+	u, err := url.Parse(databaseUrl)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return u.Path[1:]
 }
 
 func TestBackend_impl(t *testing.T) {
@@ -38,14 +46,15 @@ func TestBackend_impl(t *testing.T) {
 }
 
 func TestBackendConfig(t *testing.T) {
-	testACC(t)
+	databaseName := testACC(t)
 	connStr := getDatabaseUrl()
 
 	testCases := []struct {
-		Name        string
-		EnvVars     map[string]string
-		Config      map[string]interface{}
-		ExpectError string
+		Name                     string
+		EnvVars                  map[string]string
+		Config                   map[string]interface{}
+		ExpectConfigurationError string
+		ExpectConnectionError    string
 	}{
 		{
 			Name: "valid-config",
@@ -55,21 +64,71 @@ func TestBackendConfig(t *testing.T) {
 			},
 		},
 		{
-			Name: "missing-conn-str",
+			Name: "missing-conn_str-defaults-to-localhost",
+			EnvVars: map[string]string{
+				"PGSSLMODE":  "disable",
+				"PGDATABASE": databaseName,
+			},
 			Config: map[string]interface{}{
 				"schema_name": fmt.Sprintf("terraform_%s", t.Name()),
 			},
-			ExpectError: `The attribute "conn_str" is required, but no definition was found.`,
 		},
 		{
 			Name: "conn-str-env-var",
 			EnvVars: map[string]string{
-				"PGDATABASE": connStr,
+				"PG_CONN_STR": connStr,
 			},
 			Config: map[string]interface{}{
 				"schema_name": fmt.Sprintf("terraform_%s", t.Name()),
 			},
 		},
+		{
+			Name: "setting-credentials-using-env-vars",
+			EnvVars: map[string]string{
+				"PGUSER":     "baduser",
+				"PGPASSWORD": "badpassword",
+			},
+			Config: map[string]interface{}{
+				"conn_str":    connStr,
+				"schema_name": fmt.Sprintf("terraform_%s", t.Name()),
+			},
+			ExpectConnectionError: `role "baduser" does not exist`,
+		},
+		{
+			Name: "host-in-env-vars",
+			EnvVars: map[string]string{
+				"PGHOST": "hostthatdoesnotexist",
+			},
+			Config: map[string]interface{}{
+				"schema_name": fmt.Sprintf("terraform_%s", t.Name()),
+			},
+			ExpectConnectionError: `no such host`,
+		},
+		{
+			Name: "boolean-env-vars",
+			EnvVars: map[string]string{
+				"PGSSLMODE":               "disable",
+				"PG_SKIP_SCHEMA_CREATION": "f",
+				"PG_SKIP_TABLE_CREATION":  "f",
+				"PG_SKIP_INDEX_CREATION":  "f",
+				"PGDATABASE":              databaseName,
+			},
+			Config: map[string]interface{}{
+				"schema_name": fmt.Sprintf("terraform_%s", t.Name()),
+			},
+		},
+		{
+			Name: "wrong-boolean-env-vars",
+			EnvVars: map[string]string{
+				"PGSSLMODE":               "disable",
+				"PG_SKIP_SCHEMA_CREATION": "foo",
+				"PGDATABASE":              databaseName,
+			},
+			Config: map[string]interface{}{
+				"schema_name": fmt.Sprintf("terraform_%s", t.Name()),
+			},
+			ExpectConfigurationError: `error getting default for "skip_schema_creation"`,
+		},
 	}
 
 	for _, tc := range testCases {
@@ -97,12 +156,12 @@ func TestBackendConfig(t *testing.T) {
 			newObj, valDiags := b.PrepareConfig(obj)
 			diags = diags.Append(valDiags.InConfigBody(config, ""))
 
-			if tc.ExpectError != "" {
+			if tc.ExpectConfigurationError != "" {
 				if !diags.HasErrors() {
 					t.Fatal("error expected but got none")
 				}
-				if !strings.Contains(diags.ErrWithWarnings().Error(), tc.ExpectError) {
-					t.Fatalf("failed to find %q in %s", tc.ExpectError, diags.ErrWithWarnings())
+				if !strings.Contains(diags.ErrWithWarnings().Error(), tc.ExpectConfigurationError) {
+					t.Fatalf("failed to find %q in %s", tc.ExpectConfigurationError, diags.ErrWithWarnings())
 				}
 				return
 			} else if diags.HasErrors() {
@@ -112,7 +171,16 @@ func TestBackendConfig(t *testing.T) {
 			obj = newObj
 
 			confDiags := b.Configure(obj)
-			if len(confDiags) != 0 {
+			if tc.ExpectConnectionError != "" {
+				err := confDiags.InConfigBody(config, "").ErrWithWarnings()
+				if err == nil {
+					t.Fatal("error expected but got none")
+				}
+				if !strings.Contains(err.Error(), tc.ExpectConnectionError) {
+					t.Fatalf("failed to find %q in %s", tc.ExpectConnectionError, err)
+				}
+				return
+			} else if len(confDiags) != 0 {
 				confDiags = confDiags.InConfigBody(config, "")
 				t.Fatal(confDiags.ErrWithWarnings())
 			}

website/docs/language/settings/backends/pg.mdx

@@ -27,9 +27,17 @@ createdb terraform_backend
 
 This `createdb` command is found in [Postgres client applications](https://www.postgresql.org/docs/10/reference-client.html) which are installed along with the database server.
 
-We recommend using a
-[partial configuration](/terraform/language/settings/backends/configuration#partial-configuration)
-for the `conn_str` variable, because it typically contains access credentials that should not be committed to source control:
+
+### Using environment variables
+
+We recommend using environment variables to configure the `pg` backend in order
+not to have sensitive credentials written to  disk and committed to source
+control.
+
+The `pg` backend supports the standard [`libpq` environment variables](https://www.postgresql.org/docs/current/libpq-envars.html).
+
+The backend can be configured either by giving the whole configuration as an
+environment variable:
 
 ```hcl
 terraform {
@@ -37,16 +45,26 @@ terraform {
 }
 ```
 
-Then, set the credentials when initializing the configuration:
-
-```
-terraform init -backend-config="conn_str=postgres://user:pass@db.example.com/terraform_backend"
+```shellsession
+$ export PG_CONN_STR=postgres://user:pass@db.example.com/terraform_backend
+$ terraform init
 ```
 
-To use a Postgres server running on the same machine as Terraform, configure localhost with SSL disabled:
+or just the sensitive parameters:
 
+```hcl
+terraform {
+  backend "pg" {
+    conn_str = "postgres://db.example.com/terraform_backend"
+  }
+}
 ```
-terraform init -backend-config="conn_str=postgres://localhost/terraform_backend?sslmode=disable"
+
+```shellsession
+$ export PGUSER=user
+$ read -s PGPASSWORD
+$ export PGPASSWORD
+$ terraform init
 ```
 
 ## Data Source Configuration
@@ -68,11 +86,11 @@ data "terraform_remote_state" "network" {
 
 The following configuration options or environment variables are supported:
 
-- `conn_str` - (Required) Postgres connection string; a `postgres://` URL. `conn_str` can also be set using the `PGDATABASE` environment variable.
-- `schema_name` - Name of the automatically-managed Postgres schema, default `terraform_remote_state`.
-- `skip_schema_creation` - If set to `true`, the Postgres schema must already exist. Terraform won't try to create the schema, this is useful when it has already been created by a database administrator.
-- `skip_table_creation` - If set to `true`, the Postgres table must already exist. Terraform won't try to create the table, this is useful when it has already been created by a database administrator.
-- `skip_index_creation` - If set to `true`, the Postgres index must already exist. Terraform won't try to create the index, this is useful when it has already been created by a database administrator.
+- `conn_str` - Postgres connection string; a `postgres://` URL. The `PG_CONN_STR` and [standard `libpq`](https://www.postgresql.org/docs/current/libpq-envars.html) environment variables can also be used to indicate how to connect to the PostgreSQL database.
+- `schema_name` - Name of the automatically-managed Postgres schema, default to `terraform_remote_state`. Can also be set using the `PG_SCHEMA_NAME` environment variable.
+- `skip_schema_creation` - If set to `true`, the Postgres schema must already exist. Can also be set using the `PG_SKIP_SCHEMA_CREATION` environment variable. Terraform won't try to create the schema, this is useful when it has already been created by a database administrator.
+- `skip_table_creation` - If set to `true`, the Postgres table must already exist. Can also be set using the `PG_SKIP_TABLE_CREATION` environment variable. Terraform won't try to create the table, this is useful when it has already been created by a database administrator.
+- `skip_index_creation` - If set to `true`, the Postgres index must already exist. Can also be set using the `PG_SKIP_INDEX_CREATION` environment variable. Terraform won't try to create the index, this is useful when it has already been created by a database administrator.
 
 ## Technical Design