Make persist interval for remote state backend configurable (#1591)

Signed-off-by: Alex Ott <alexott@gmail.com>
Co-authored-by: Siddhartha Sonker <158144589+siddharthasonker95@users.noreply.github.com>

CHANGELOG.md

@@ -7,6 +7,7 @@ NEW FEATURES:
 ENHANCEMENTS:
 * Added `tofu test -json` types to website Machine-Readable UI documentation ([1408](https://github.com/opentofu/opentofu/issues/1408))
 * Made `tofu plan` with `generate-config-out` flag replace JSON strings with `jsonencode` functions calls. ([#1595](https://github.com/opentofu/opentofu/pull/1595))
+* Make state persistence interval configurable via `TF_STATE_PERSIST_INTERVAL` environment variable ([#1591](https://github.com/opentofu/opentofu/pull/1591))
 * Improved performance of writing state files and reduced their size using compact json encoding. ([#1647](https://github.com/opentofu/opentofu/pull/1647))
 
 BUG FIXES:

internal/backend/local/backend_apply.go

@@ -10,6 +10,8 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"os"
+	"strconv"
 	"time"
 
 	"github.com/opentofu/opentofu/internal/addrs"
@@ -27,6 +29,22 @@ import (
 // test hook called between plan+apply during opApply
 var testHookStopPlanApply func()
 
+const (
+	defaultPersistInterval                 = 20 // arbitrary interval that's hopefully a sweet spot
+	persistIntervalEnvironmentVariableName = "TF_STATE_PERSIST_INTERVAL"
+)
+
+func getEnvAsInt(envName string, defaultValue int) int {
+	if val, exists := os.LookupEnv(envName); exists {
+		parsedVal, err := strconv.Atoi(val)
+		if err == nil {
+			return parsedVal
+		}
+		panic(fmt.Sprintf("Can't parse value '%s' of environment variable '%s'", val, envName))
+	}
+	return defaultValue
+}
+
 func (b *Local) opApply(
 	stopCtx context.Context,
 	cancelCtx context.Context,
@@ -84,7 +102,12 @@ func (b *Local) opApply(
 	// stateHook uses schemas for when it periodically persists state to the
 	// persistent storage backend.
 	stateHook.Schemas = schemas
-	stateHook.PersistInterval = 20 * time.Second // arbitrary interval that's hopefully a sweet spot
+	persistInterval := getEnvAsInt(persistIntervalEnvironmentVariableName, defaultPersistInterval)
+	if persistInterval < defaultPersistInterval {
+		panic(fmt.Sprintf("Can't use value lower than %d for env variable %s, got %d",
+			defaultPersistInterval, persistIntervalEnvironmentVariableName, persistInterval))
+	}
+	stateHook.PersistInterval = time.Duration(persistInterval) * time.Second
 
 	var plan *plans.Plan
 	// If we weren't given a plan, then we refresh/plan

website/docs/cli/commands/apply.mdx

@@ -93,6 +93,10 @@ For configurations using
 `tofu apply` also accepts the legacy options
 [`-state`, `-state-out`, and `-backup`](../../language/settings/backends/local.mdx#command-line-arguments).
 
+### Environment variables
+
+You can further customize behavior of `apply` command by using [environment variables](../config/environment-variables.mdx).  For example, the [TF_STATE_PERSIST_INTERVAL](../config/environment-variables.mdx#tf_state_persist_interval) environment variable allows to specify the interval between state persistence.
+
 ## Passing a Different Configuration Directory
 
 If your workflow relies on overriding the root module directory, use

website/docs/cli/config/environment-variables.mdx

@@ -174,6 +174,14 @@ export TF_PROVIDER_DOWNLOAD_RETRY=3
 
 For more details on `.terraformignore`, please see [Excluding Files from Upload with .terraformignore](../../language/settings/backends/remote.mdx#excluding-files-from-upload-with-terraformignore).
 
+## TF_STATE_PERSIST_INTERVAL
+
+Set `TF_STATE_PERSIST_INTERVAL` to configure the interval (in seconds) between state persistence.  Increased interval could be useful when working with huge states (> 100k resources) where upload to a cloud service could take a significant amount of time.  Default persistence interval is 20 seconds (it also the lowest possible value for this parameter).  The following command sets persistence interval to 5 minutes (300 seconds):
+
+```shell
+export TF_STATE_PERSIST_INTERVAL=300
+```
+
 ## Cloud Backend CLI Integration
 
 The CLI integration with cloud backends lets you use them on the command line. The integration requires including a `cloud` block in your OpenTofu configuration. You can define its arguments directly in your configuration file or supply them through environment variables, which can be useful for non-interactive workflows like Continuous Integration (CI).