mirror of
https://github.com/opentofu/opentofu.git
synced 2024-12-29 10:21:01 -06:00
383 lines
15 KiB
Go
383 lines
15 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: MPL-2.0
|
|
|
|
package command
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"github.com/apparentlymart/go-versions/versions"
|
|
"github.com/hashicorp/go-getter"
|
|
|
|
"github.com/opentofu/opentofu/internal/getproviders"
|
|
"github.com/opentofu/opentofu/internal/httpclient"
|
|
"github.com/opentofu/opentofu/internal/tfdiags"
|
|
)
|
|
|
|
// ProvidersMirrorCommand is a Command implementation that implements the
|
|
// "tofu providers mirror" command, which populates a directory with
|
|
// local copies of provider plugins needed by the current configuration so
|
|
// that the mirror can be used to work offline, or similar.
|
|
type ProvidersMirrorCommand struct {
|
|
Meta
|
|
}
|
|
|
|
func (c *ProvidersMirrorCommand) Synopsis() string {
|
|
return "Save local copies of all required provider plugins"
|
|
}
|
|
|
|
func (c *ProvidersMirrorCommand) Run(args []string) int {
|
|
args = c.Meta.process(args)
|
|
cmdFlags := c.Meta.defaultFlagSet("providers mirror")
|
|
var optPlatforms FlagStringSlice
|
|
cmdFlags.Var(&optPlatforms, "platform", "target platform")
|
|
cmdFlags.Usage = func() { c.Ui.Error(c.Help()) }
|
|
if err := cmdFlags.Parse(args); err != nil {
|
|
c.Ui.Error(fmt.Sprintf("Error parsing command-line flags: %s\n", err.Error()))
|
|
return 1
|
|
}
|
|
|
|
var diags tfdiags.Diagnostics
|
|
|
|
args = cmdFlags.Args()
|
|
if len(args) != 1 {
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"No output directory specified",
|
|
"The providers mirror command requires an output directory as a command-line argument.",
|
|
))
|
|
c.showDiagnostics(diags)
|
|
return 1
|
|
}
|
|
outputDir := args[0]
|
|
|
|
var platforms []getproviders.Platform
|
|
if len(optPlatforms) == 0 {
|
|
platforms = []getproviders.Platform{getproviders.CurrentPlatform}
|
|
} else {
|
|
platforms = make([]getproviders.Platform, 0, len(optPlatforms))
|
|
for _, platformStr := range optPlatforms {
|
|
platform, err := getproviders.ParsePlatform(platformStr)
|
|
if err != nil {
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Invalid target platform",
|
|
fmt.Sprintf("The string %q given in the -platform option is not a valid target platform: %s.", platformStr, err),
|
|
))
|
|
continue
|
|
}
|
|
platforms = append(platforms, platform)
|
|
}
|
|
}
|
|
|
|
// Installation steps can be cancelled by SIGINT and similar.
|
|
ctx, done := c.InterruptibleContext(c.CommandContext())
|
|
defer done()
|
|
|
|
config, confDiags := c.loadConfig(".")
|
|
diags = diags.Append(confDiags)
|
|
reqs, moreDiags := config.ProviderRequirements()
|
|
diags = diags.Append(moreDiags)
|
|
|
|
// Read lock file
|
|
lockedDeps, lockedDepsDiags := c.Meta.lockedDependencies()
|
|
diags = diags.Append(lockedDepsDiags)
|
|
|
|
// If we have any error diagnostics already then we won't proceed further.
|
|
if diags.HasErrors() {
|
|
c.showDiagnostics(diags)
|
|
return 1
|
|
}
|
|
|
|
// If lock file is present, validate it against configuration
|
|
if !lockedDeps.Empty() {
|
|
if errs := config.VerifyDependencySelections(lockedDeps); len(errs) > 0 {
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Inconsistent dependency lock file",
|
|
fmt.Sprintf("To update the locked dependency selections to match a changed configuration, run:\n tofu init -upgrade\n got:%v", errs),
|
|
))
|
|
}
|
|
}
|
|
|
|
// Unlike other commands, this command always consults the origin registry
|
|
// for every provider so that it can be used to update a local mirror
|
|
// directory without needing to first disable that local mirror
|
|
// in the CLI configuration.
|
|
source := getproviders.NewMemoizeSource(
|
|
getproviders.NewRegistrySource(c.Services),
|
|
)
|
|
|
|
// Providers from registries always use HTTP, so we don't need the full
|
|
// generality of go-getter but it's still handy to use the HTTP getter
|
|
// as an easy way to download over HTTP into a file on disk.
|
|
httpGetter := getter.HttpGetter{
|
|
Client: httpclient.New(),
|
|
Netrc: true,
|
|
XTerraformGetDisabled: true,
|
|
}
|
|
|
|
// The following logic is similar to that used by the provider installer
|
|
// in package providercache, but different in a few ways:
|
|
// - It produces the packed directory layout rather than the unpacked
|
|
// layout we require in provider cache directories.
|
|
// - It generates JSON index files that can be read by the
|
|
// getproviders.HTTPMirrorSource installation method if the result were
|
|
// copied into the docroot of an HTTP server.
|
|
// - It can mirror packages for potentially many different target platforms,
|
|
// so that we can construct a multi-platform mirror regardless of which
|
|
// platform we run this command on.
|
|
// - It ignores what's already present and just always downloads everything
|
|
// that the configuration requires. This is a command intended to be run
|
|
// infrequently to update a mirror, so it doesn't need to optimize away
|
|
// fetches of packages that might already be present.
|
|
|
|
for provider, constraints := range reqs {
|
|
if provider.IsBuiltIn() {
|
|
c.Ui.Output(fmt.Sprintf("- Skipping %s because it is built in to OpenTofu CLI", provider.ForDisplay()))
|
|
continue
|
|
}
|
|
constraintsStr := getproviders.VersionConstraintsString(constraints)
|
|
c.Ui.Output(fmt.Sprintf("- Mirroring %s...", provider.ForDisplay()))
|
|
// First we'll look for the latest version that matches the given
|
|
// constraint, which we'll then try to mirror for each target platform.
|
|
acceptable := versions.MeetingConstraints(constraints)
|
|
avail, _, err := source.AvailableVersions(ctx, provider)
|
|
candidates := avail.Filter(acceptable)
|
|
if err == nil && len(candidates) == 0 {
|
|
err = fmt.Errorf("no releases match the given constraints %s", constraintsStr)
|
|
}
|
|
if err != nil {
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Provider not available",
|
|
fmt.Sprintf("Failed to download %s from its origin registry: %s.", provider.String(), err),
|
|
))
|
|
continue
|
|
}
|
|
selected := candidates.Newest()
|
|
if !lockedDeps.Empty() {
|
|
selected = lockedDeps.Provider(provider).Version()
|
|
c.Ui.Output(fmt.Sprintf(" - Selected v%s to match dependency lock file", selected.String()))
|
|
} else if len(constraintsStr) > 0 {
|
|
c.Ui.Output(fmt.Sprintf(" - Selected v%s to meet constraints %s", selected.String(), constraintsStr))
|
|
} else {
|
|
c.Ui.Output(fmt.Sprintf(" - Selected v%s with no constraints", selected.String()))
|
|
}
|
|
for _, platform := range platforms {
|
|
c.Ui.Output(fmt.Sprintf(" - Downloading package for %s...", platform.String()))
|
|
meta, err := source.PackageMeta(ctx, provider, selected, platform)
|
|
if err != nil {
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Provider release not available",
|
|
fmt.Sprintf("Failed to download %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err),
|
|
))
|
|
continue
|
|
}
|
|
urlStr, ok := meta.Location.(getproviders.PackageHTTPURL)
|
|
if !ok {
|
|
// We don't expect to get non-HTTP locations here because we're
|
|
// using the registry source, so this seems like a bug in the
|
|
// registry source.
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Provider release not available",
|
|
fmt.Sprintf("Failed to download %s v%s for %s: OpenTofu's provider registry client returned unexpected location type %T. This is a bug in OpenTofu.", provider.String(), selected.String(), platform.String(), meta.Location),
|
|
))
|
|
continue
|
|
}
|
|
urlObj, err := url.Parse(string(urlStr))
|
|
if err != nil {
|
|
// We don't expect to get non-HTTP locations here because we're
|
|
// using the registry source, so this seems like a bug in the
|
|
// registry source.
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Invalid URL for provider release",
|
|
fmt.Sprintf("The origin registry for %s returned an invalid URL for v%s on %s: %s.", provider.String(), selected.String(), platform.String(), err),
|
|
))
|
|
continue
|
|
}
|
|
// targetPath is the path where we ultimately want to place the
|
|
// downloaded archive, but we'll place it initially at stagingPath
|
|
// so we can verify its checksums and signatures before making
|
|
// it discoverable to mirror clients. (stagingPath intentionally
|
|
// does not follow the filesystem mirror file naming convention.)
|
|
targetPath := meta.PackedFilePath(outputDir)
|
|
stagingPath := filepath.Join(filepath.Dir(targetPath), "."+filepath.Base(targetPath))
|
|
err = httpGetter.GetFile(stagingPath, urlObj)
|
|
if err != nil {
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Cannot download provider release",
|
|
fmt.Sprintf("Failed to download %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err),
|
|
))
|
|
continue
|
|
}
|
|
if meta.Authentication != nil {
|
|
result, err := meta.Authentication.AuthenticatePackage(getproviders.PackageLocalArchive(stagingPath))
|
|
if err != nil {
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Invalid provider package",
|
|
fmt.Sprintf("Failed to authenticate %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err),
|
|
))
|
|
continue
|
|
}
|
|
c.Ui.Output(fmt.Sprintf(" - Package authenticated: %s", result))
|
|
}
|
|
os.Remove(targetPath) // okay if it fails because we're going to try to rename over it next anyway
|
|
err = os.Rename(stagingPath, targetPath)
|
|
if err != nil {
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Cannot download provider release",
|
|
fmt.Sprintf("Failed to place %s package into mirror directory: %s.", provider.String(), err),
|
|
))
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
|
|
// Now we'll generate or update the JSON index files in the directory.
|
|
// We do this by scanning the directory to see what is present, rather than
|
|
// by relying on the selections we made above, because we want to still
|
|
// include in the indices any packages that were already present and
|
|
// not affected by the changes we just made.
|
|
available, err := getproviders.SearchLocalDirectory(outputDir)
|
|
if err != nil {
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Failed to update indexes",
|
|
fmt.Sprintf("Could not scan the output directory to get package metadata for the JSON indexes: %s.", err),
|
|
))
|
|
available = nil // the following loop will be a no-op
|
|
}
|
|
for provider, metas := range available {
|
|
if len(metas) == 0 {
|
|
continue // should never happen, but we'll be resilient
|
|
}
|
|
// The index files live in the same directory as the package files,
|
|
// so to figure that out without duplicating the path-building logic
|
|
// we'll ask the getproviders package to build an archive filename
|
|
// for a fictitious package and then use the directory portion of it.
|
|
indexDir := filepath.Dir(getproviders.PackedFilePathForPackage(
|
|
outputDir, provider, versions.Unspecified, getproviders.CurrentPlatform,
|
|
))
|
|
indexVersions := map[string]interface{}{}
|
|
indexArchives := map[getproviders.Version]map[string]interface{}{}
|
|
for _, meta := range metas {
|
|
archivePath, ok := meta.Location.(getproviders.PackageLocalArchive)
|
|
if !ok {
|
|
// only archive files are eligible to be included in JSON
|
|
// indices for a network mirror.
|
|
continue
|
|
}
|
|
archiveFilename := filepath.Base(string(archivePath))
|
|
version := meta.Version
|
|
platform := meta.TargetPlatform
|
|
hash, err := meta.Hash()
|
|
if err != nil {
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Failed to update indexes",
|
|
fmt.Sprintf("Failed to determine a hash value for %s v%s on %s: %s.", provider, version, platform, err),
|
|
))
|
|
continue
|
|
}
|
|
indexVersions[meta.Version.String()] = map[string]interface{}{}
|
|
if _, ok := indexArchives[version]; !ok {
|
|
indexArchives[version] = map[string]interface{}{}
|
|
}
|
|
indexArchives[version][platform.String()] = map[string]interface{}{
|
|
"url": archiveFilename, // a relative URL from the index file's URL
|
|
"hashes": []string{hash.String()}, // an array to allow for additional hash formats in future
|
|
}
|
|
}
|
|
mainIndex := map[string]interface{}{
|
|
"versions": indexVersions,
|
|
}
|
|
mainIndexJSON, err := json.MarshalIndent(mainIndex, "", " ")
|
|
if err != nil {
|
|
// Should never happen because the input here is entirely under
|
|
// our control.
|
|
panic(fmt.Sprintf("failed to encode main index: %s", err))
|
|
}
|
|
// TODO: Ideally we would do these updates as atomic swap operations by
|
|
// creating a new file and then renaming it over the old one, in case
|
|
// this directory is the docroot of a live mirror. An atomic swap
|
|
// requires platform-specific code though: os.Rename alone can't do it
|
|
// when running on Windows as of Go 1.13. We should revisit this once
|
|
// we're supporting network mirrors, to avoid having them briefly
|
|
// become corrupted during updates.
|
|
err = os.WriteFile(filepath.Join(indexDir, "index.json"), mainIndexJSON, 0644)
|
|
if err != nil {
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Failed to update indexes",
|
|
fmt.Sprintf("Failed to write an updated JSON index for %s: %s.", provider, err),
|
|
))
|
|
}
|
|
for version, archiveIndex := range indexArchives {
|
|
versionIndex := map[string]interface{}{
|
|
"archives": archiveIndex,
|
|
}
|
|
versionIndexJSON, err := json.MarshalIndent(versionIndex, "", " ")
|
|
if err != nil {
|
|
// Should never happen because the input here is entirely under
|
|
// our control.
|
|
panic(fmt.Sprintf("failed to encode version index: %s", err))
|
|
}
|
|
err = os.WriteFile(filepath.Join(indexDir, version.String()+".json"), versionIndexJSON, 0644)
|
|
if err != nil {
|
|
diags = diags.Append(tfdiags.Sourceless(
|
|
tfdiags.Error,
|
|
"Failed to update indexes",
|
|
fmt.Sprintf("Failed to write an updated JSON index for %s v%s: %s.", provider, version, err),
|
|
))
|
|
}
|
|
}
|
|
}
|
|
|
|
c.showDiagnostics(diags)
|
|
if diags.HasErrors() {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func (c *ProvidersMirrorCommand) Help() string {
|
|
return `
|
|
Usage: tofu [global options] providers mirror [options] <target-dir>
|
|
|
|
Populates a local directory with copies of the provider plugins needed for
|
|
the current configuration, so that the directory can be used either directly
|
|
as a filesystem mirror or as the basis for a network mirror and thus obtain
|
|
those providers without access to their origin registries in future.
|
|
|
|
The mirror directory will contain JSON index files that can be published
|
|
along with the mirrored packages on a static HTTP file server to produce
|
|
a network mirror. Those index files will be ignored if the directory is
|
|
used instead as a local filesystem mirror.
|
|
|
|
Options:
|
|
|
|
-platform=os_arch Choose which target platform to build a mirror for.
|
|
By default OpenTofu will obtain plugin packages
|
|
suitable for the platform where you run this command.
|
|
Use this flag multiple times to include packages for
|
|
multiple target systems.
|
|
|
|
Target names consist of an operating system and a CPU
|
|
architecture. For example, "linux_amd64" selects the
|
|
Linux operating system running on an AMD64 or x86_64
|
|
CPU. Each provider is available only for a limited
|
|
set of target platforms.
|
|
`
|
|
}
|