feat: add retry in provider install (#1255)

Signed-off-by: Yuvraj <evalsocket@gmail.com>
Signed-off-by: Christian Mesh <christianmesh1@gmail.com>
Co-authored-by: Christian Mesh <christianmesh1@gmail.com>
Co-authored-by: James Humphries <James@james-humphries.co.uk>
This commit is contained in:
Yuvraj 2024-04-01 22:11:18 +05:30 committed by GitHub
parent 8321f14786
commit aa8b4a7cca
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 57 additions and 6 deletions

View File

@ -34,6 +34,7 @@ ENHANCEMENTS:
* Allow for templatefile function recursion (up to 1024 call depth default). ([#1250](https://github.com/opentofu/opentofu/pull/1250))
* Dump state file when `tofu test` fails to clean up resources. ([#1243](https://github.com/opentofu/opentofu/pull/1243))
* Added aliases for `state list` (`state ls`), `state mv` (`state move`), and `state rm` (`state remove`) ([#1220](https://github.com/opentofu/opentofu/pull/1220))
* Added mechanism to introduce automatic retries for provider installations, specifically targeting transient errors ([#1233](https://github.com/opentofu/opentofu/issues/1233))
BUG FIXES:
* Fix view hooks unit test flakiness by deterministically waiting for heartbeats to execute ([$1153](https://github.com/opentofu/opentofu/issues/1153))

View File

@ -297,8 +297,6 @@ NeedProvider:
}
available, warnings, err := i.source.AvailableVersions(ctx, provider)
if err != nil {
// TODO: Consider retrying a few times for certain types of
// source errors that seem likely to be transient.
errs[provider] = err
if cb := evts.QueryPackagesFailure; cb != nil {
cb(provider, err)

View File

@ -8,15 +8,19 @@ package providercache
import (
"context"
"fmt"
"log"
"net/http"
"os"
"path/filepath"
"strconv"
getter "github.com/hashicorp/go-getter"
"github.com/hashicorp/go-retryablehttp"
"github.com/opentofu/opentofu/internal/copy"
"github.com/opentofu/opentofu/internal/getproviders"
"github.com/opentofu/opentofu/internal/httpclient"
"github.com/opentofu/opentofu/internal/logging"
)
// We borrow the "unpack a zip file into a target directory" logic from
@ -26,6 +30,39 @@ import (
// specific protocol and set of expectations.)
var unzip = getter.ZipDecompressor{}
const (
// httpClientRetryCountEnvName is the environment variable name used to customize
// the HTTP retry count for module downloads.
httpClientRetryCountEnvName = "TF_PROVIDER_DOWNLOAD_RETRY"
defaultRetry = 2
)
func init() {
configureProviderDownloadRetry()
}
var (
maxRetryCount int
)
// will attempt for requests with retryable errors, like 502 status codes
func configureProviderDownloadRetry() {
maxRetryCount = defaultRetry
if v := os.Getenv(httpClientRetryCountEnvName); v != "" {
retry, err := strconv.Atoi(v)
if err == nil && retry > 0 {
maxRetryCount = retry
}
}
}
func requestLogHook(logger retryablehttp.Logger, req *http.Request, i int) {
if i > 0 {
logger.Printf("[INFO] Previous request to the provider install failed, attempting retry.")
}
}
func installFromHTTPURL(ctx context.Context, meta getproviders.PackageMeta, targetDir string, allowedHashes []getproviders.Hash) (*getproviders.PackageAuthenticationResult, error) {
url := meta.Location.String()
@ -37,19 +74,24 @@ func installFromHTTPURL(ctx context.Context, meta getproviders.PackageMeta, targ
// through X-Terraform-Get header, attempting partial fetches for
// files that already exist, etc.)
httpClient := httpclient.New()
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
retryableClient := retryablehttp.NewClient()
retryableClient.HTTPClient = httpclient.New()
retryableClient.RetryMax = maxRetryCount
retryableClient.RequestLogHook = requestLogHook
retryableClient.Logger = log.New(logging.LogOutput(), "", log.Flags())
req, err := retryablehttp.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return nil, fmt.Errorf("invalid provider download request: %w", err)
}
resp, err := httpClient.Do(req)
resp, err := retryableClient.Do(req)
if err != nil {
if ctx.Err() == context.Canceled {
// "context canceled" is not a user-friendly error message,
// so we'll return a more appropriate one here.
return nil, fmt.Errorf("provider download was interrupted")
}
return nil, fmt.Errorf("%s: %w", getproviders.HostFromRequest(req), err)
return nil, fmt.Errorf("%s: %w", getproviders.HostFromRequest(req.Request), err)
}
defer resp.Body.Close()

View File

@ -162,6 +162,16 @@ If `TF_IGNORE` is set to "trace", OpenTofu will output debug messages to display
export TF_IGNORE=trace
```
## TF_PROVIDER_DOWNLOAD_RETRY
Set `TF_PROVIDER_DOWNLOAD_RETRY` to configure the max number of request retries
the remote provider client will attempt for client connection errors or
500-range responses that are safe to retry.
```shell
export TF_PROVIDER_DOWNLOAD_RETRY=3
```
For more details on `.terraformignore`, please see [Excluding Files from Upload with .terraformignore](/docs/language/settings/backends/remote#excluding-files-from-upload-with-terraformignore).
## Cloud Backend CLI Integration