feat: add retry in provider install (#1255)

Signed-off-by: Yuvraj <evalsocket@gmail.com>
Signed-off-by: Christian Mesh <christianmesh1@gmail.com>
Co-authored-by: Christian Mesh <christianmesh1@gmail.com>
Co-authored-by: James Humphries <James@james-humphries.co.uk>

CHANGELOG.md

@@ -34,6 +34,7 @@ ENHANCEMENTS:
 * Allow for templatefile function recursion (up to 1024 call depth default). ([#1250](https://github.com/opentofu/opentofu/pull/1250))
 * Dump state file when `tofu test` fails to clean up resources. ([#1243](https://github.com/opentofu/opentofu/pull/1243))
 * Added aliases for `state list` (`state ls`), `state mv` (`state move`), and `state rm` (`state remove`) ([#1220](https://github.com/opentofu/opentofu/pull/1220))
+* Added mechanism to introduce automatic retries for provider installations, specifically targeting transient errors ([#1233](https://github.com/opentofu/opentofu/issues/1233))
 
 BUG FIXES:
 * Fix view hooks unit test flakiness by deterministically waiting for heartbeats to execute ([$1153](https://github.com/opentofu/opentofu/issues/1153))

internal/providercache/installer.go

@@ -297,8 +297,6 @@ NeedProvider:
 		}
 		available, warnings, err := i.source.AvailableVersions(ctx, provider)
 		if err != nil {
-			// TODO: Consider retrying a few times for certain types of
-			// source errors that seem likely to be transient.
 			errs[provider] = err
 			if cb := evts.QueryPackagesFailure; cb != nil {
 				cb(provider, err)

internal/providercache/package_install.go

@@ -8,15 +8,19 @@ package providercache
 import (
 	"context"
 	"fmt"
+	"log"
 	"net/http"
 	"os"
 	"path/filepath"
+	"strconv"
 
 	getter "github.com/hashicorp/go-getter"
+	"github.com/hashicorp/go-retryablehttp"
 
 	"github.com/opentofu/opentofu/internal/copy"
 	"github.com/opentofu/opentofu/internal/getproviders"
 	"github.com/opentofu/opentofu/internal/httpclient"
+	"github.com/opentofu/opentofu/internal/logging"
 )
 
 // We borrow the "unpack a zip file into a target directory" logic from
@@ -26,6 +30,39 @@ import (
 // specific protocol and set of expectations.)
 var unzip = getter.ZipDecompressor{}
 
+const (
+	// httpClientRetryCountEnvName is the environment variable name used to customize
+	// the HTTP retry count for module downloads.
+	httpClientRetryCountEnvName = "TF_PROVIDER_DOWNLOAD_RETRY"
+
+	defaultRetry = 2
+)
+
+func init() {
+	configureProviderDownloadRetry()
+}
+
+var (
+	maxRetryCount int
+)
+
+// will attempt for requests with retryable errors, like 502 status codes
+func configureProviderDownloadRetry() {
+	maxRetryCount = defaultRetry
+	if v := os.Getenv(httpClientRetryCountEnvName); v != "" {
+		retry, err := strconv.Atoi(v)
+		if err == nil && retry > 0 {
+			maxRetryCount = retry
+		}
+	}
+}
+
+func requestLogHook(logger retryablehttp.Logger, req *http.Request, i int) {
+	if i > 0 {
+		logger.Printf("[INFO] Previous request to the provider install failed, attempting retry.")
+	}
+}
+
 func installFromHTTPURL(ctx context.Context, meta getproviders.PackageMeta, targetDir string, allowedHashes []getproviders.Hash) (*getproviders.PackageAuthenticationResult, error) {
 	url := meta.Location.String()
 
@@ -37,19 +74,24 @@ func installFromHTTPURL(ctx context.Context, meta getproviders.PackageMeta, targ
 	// through X-Terraform-Get header, attempting partial fetches for
 	// files that already exist, etc.)
 
-	httpClient := httpclient.New()
-	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	retryableClient := retryablehttp.NewClient()
+	retryableClient.HTTPClient = httpclient.New()
+	retryableClient.RetryMax = maxRetryCount
+	retryableClient.RequestLogHook = requestLogHook
+	retryableClient.Logger = log.New(logging.LogOutput(), "", log.Flags())
+
+	req, err := retryablehttp.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
 		return nil, fmt.Errorf("invalid provider download request: %w", err)
 	}
-	resp, err := httpClient.Do(req)
+	resp, err := retryableClient.Do(req)
 	if err != nil {
 		if ctx.Err() == context.Canceled {
 			// "context canceled" is not a user-friendly error message,
 			// so we'll return a more appropriate one here.
 			return nil, fmt.Errorf("provider download was interrupted")
 		}
-		return nil, fmt.Errorf("%s: %w", getproviders.HostFromRequest(req), err)
+		return nil, fmt.Errorf("%s: %w", getproviders.HostFromRequest(req.Request), err)
 	}
 	defer resp.Body.Close()
 

website/docs/cli/config/environment-variables.mdx

@@ -162,6 +162,16 @@ If `TF_IGNORE` is set to "trace", OpenTofu will output debug messages to display
 export TF_IGNORE=trace
 ```
 
+## TF_PROVIDER_DOWNLOAD_RETRY
+
+Set `TF_PROVIDER_DOWNLOAD_RETRY` to configure the max number of request retries
+the remote provider client will attempt for client connection errors or
+500-range responses that are safe to retry.
+
+```shell
+export TF_PROVIDER_DOWNLOAD_RETRY=3
+```
+
 For more details on `.terraformignore`, please see [Excluding Files from Upload with .terraformignore](/docs/language/settings/backends/remote#excluding-files-from-upload-with-terraformignore).
 
 ## Cloud Backend CLI Integration