Ignore existing package hashes for providers lock command (#31389)

* Ignore existing package hashes for  command

* missing new line

* Fix incorrect logic when deciding change message

* fix imports
This commit is contained in:
Liam Cervante 2022-07-20 13:27:24 +01:00 committed by GitHub
parent afd273d636
commit 224728879d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 473 additions and 43 deletions

View File

@ -13,6 +13,14 @@ import (
"github.com/hashicorp/terraform/internal/tfdiags"
)
type providersLockChangeType string
const (
providersLockChangeTypeNoChange providersLockChangeType = "providersLockChangeTypeNoChange"
providersLockChangeTypeNewProvider providersLockChangeType = "providersLockChangeTypeNewProvider"
providersLockChangeTypeNewHashes providersLockChangeType = "providersLockChangeTypeNewHashes"
)
// ProvidersLockCommand is a Command implementation that implements the
// "terraform providers lock" command, which creates or updates the current
// configuration's dependency lock file using information from upstream
@ -225,7 +233,7 @@ func (c *ProvidersLockCommand) Run(args []string) int {
if keyID != "" {
keyID = c.Colorize().Color(fmt.Sprintf(", key ID [reset][bold]%s[reset]", keyID))
}
c.Ui.Output(fmt.Sprintf("- Obtained %s checksums for %s (%s%s)", provider.ForDisplay(), platform, auth, keyID))
c.Ui.Output(fmt.Sprintf("- Retrieved %s %s for %s (%s%s)", provider.ForDisplay(), version, platform, auth, keyID))
},
}
ctx := evts.OnContext(ctx)
@ -233,7 +241,7 @@ func (c *ProvidersLockCommand) Run(args []string) int {
dir := providercache.NewDirWithPlatform(tempDir, platform)
installer := providercache.NewInstaller(dir, source)
newLocks, err := installer.EnsureProviderVersions(ctx, oldLocks, reqs, providercache.InstallNewProvidersOnly)
newLocks, err := installer.EnsureProviderVersions(ctx, oldLocks, reqs, providercache.InstallNewProvidersForce)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
@ -252,6 +260,10 @@ func (c *ProvidersLockCommand) Run(args []string) int {
return 1
}
// Track whether we've made any changes to the lock file as part of this
// operation. We can customise the final message based on our actions.
madeAnyChange := false
// We now have a separate updated locks object for each platform. We need
// to merge those all together so that the final result has the union of
// all of the checksums we saw for each of the providers we've worked on.
@ -270,7 +282,7 @@ func (c *ProvidersLockCommand) Run(args []string) int {
constraints = oldLock.VersionConstraints()
hashes = append(hashes, oldLock.AllHashes()...)
}
for _, platformLocks := range updatedLocks {
for platform, platformLocks := range updatedLocks {
platformLock := platformLocks.Provider(provider)
if platformLock == nil {
continue // weird, but we'll tolerate it to avoid crashing
@ -282,6 +294,32 @@ func (c *ProvidersLockCommand) Run(args []string) int {
// platforms here, because the SetProvider method we call below
// handles that automatically.
hashes = append(hashes, platformLock.AllHashes()...)
// At this point, we've merged all the hashes for this (provider, platform)
// combo into the combined hashes for this provider. Let's take this
// opportunity to print out a summary for this particular combination.
switch providersLockCalculateChangeType(oldLock, platformLock) {
case providersLockChangeTypeNewProvider:
madeAnyChange = true
c.Ui.Output(
fmt.Sprintf(
"- Obtained %s checksums for %s; This was a new provider and the checksums for this platform are now tracked in the lock file",
provider.ForDisplay(),
platform))
case providersLockChangeTypeNewHashes:
madeAnyChange = true
c.Ui.Output(
fmt.Sprintf(
"- Obtained %s checksums for %s; Additional checksums for this platform are now tracked in the lock file",
provider.ForDisplay(),
platform))
case providersLockChangeTypeNoChange:
c.Ui.Output(
fmt.Sprintf(
"- Obtained %s checksums for %s; All checksums for this platform were already tracked in the lock file",
provider.ForDisplay(),
platform))
}
}
newLocks.SetProvider(provider, version, constraints, hashes)
}
@ -294,8 +332,12 @@ func (c *ProvidersLockCommand) Run(args []string) int {
return 1
}
if madeAnyChange {
c.Ui.Output(c.Colorize().Color("\n[bold][green]Success![reset] [bold]Terraform has updated the lock file.[reset]"))
c.Ui.Output("\nReview the changes in .terraform.lock.hcl and then commit to your\nversion control system to retain the new checksums.\n")
} else {
c.Ui.Output(c.Colorize().Color("\n[bold][green]Success![reset] [bold]Terraform has validated the lock file and found no need for changes.[reset]"))
}
return 0
}
@ -357,3 +399,28 @@ Options:
set of target platforms.
`
}
// providersLockCalculateChangeType works out whether there is any difference
// between oldLock and newLock and returns a variable the main function can use
// to decide on which message to print.
//
// One assumption made here that is not obvious without the context from the
// main function is that while platformLock contains the lock information for a
// single platform after the current run, oldLock contains the combined
// information of all platforms from when the versions were last checked. A
// simple equality check is not sufficient for deciding on change as we expect
// that oldLock will be a superset of platformLock if no new hashes have been
// found.
//
// We've separated this function out so we can write unit tests around the
// logic. This function assumes the platformLock is not nil, as the main
// function explicitly checks this before calling this function.
func providersLockCalculateChangeType(oldLock *depsfile.ProviderLock, platformLock *depsfile.ProviderLock) providersLockChangeType {
if oldLock == nil {
return providersLockChangeTypeNewProvider
}
if oldLock.ContainsAll(platformLock) {
return providersLockChangeTypeNoChange
}
return providersLockChangeTypeNewHashes
}

View File

@ -8,6 +8,9 @@ import (
"strings"
"testing"
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/depsfile"
"github.com/hashicorp/terraform/internal/getproviders"
"github.com/mitchellh/cli"
)
@ -33,8 +36,41 @@ func TestProvidersLock(t *testing.T) {
// This test depends on the -fs-mirror argument, so we always know what results to expect
t.Run("basic", func(t *testing.T) {
testDirectory := "providers-lock/basic"
expected := `# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/test" {
version = "1.0.0"
hashes = [
"h1:7MjN4eFisdTv4tlhXH5hL4QQd39Jy4baPhFxwAd/EFE=",
]
}
`
runProviderLockGenericTest(t, testDirectory, expected)
})
// This test depends on the -fs-mirror argument, so we always know what results to expect
t.Run("append", func(t *testing.T) {
testDirectory := "providers-lock/append"
expected := `# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/test" {
version = "1.0.0"
hashes = [
"h1:7MjN4eFisdTv4tlhXH5hL4QQd39Jy4baPhFxwAd/EFE=",
"h1:invalid",
]
}
`
runProviderLockGenericTest(t, testDirectory, expected)
})
}
func runProviderLockGenericTest(t *testing.T, testDirectory, expected string) {
td := t.TempDir()
testCopyDir(t, testFixturePath("providers-lock/basic"), td)
testCopyDir(t, testFixturePath(testDirectory), td)
defer testChdir(t, td)()
// Our fixture dir has a generic os_arch dir, which we need to customize
@ -67,20 +103,9 @@ func TestProvidersLock(t *testing.T) {
t.Fatal("error reading lockfile")
}
expected := `# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/test" {
version = "1.0.0"
hashes = [
"h1:7MjN4eFisdTv4tlhXH5hL4QQd39Jy4baPhFxwAd/EFE=",
]
}
`
if string(lockfile) != expected {
t.Fatalf("wrong lockfile content")
}
})
}
func TestProvidersLock_args(t *testing.T) {
@ -151,3 +176,81 @@ func TestProvidersLock_args(t *testing.T) {
}
})
}
func TestProvidersLockCalculateChangeType(t *testing.T) {
provider := addrs.NewDefaultProvider("provider")
v2 := getproviders.MustParseVersion("2.0.0")
v2EqConstraints := getproviders.MustParseVersionConstraints("2.0.0")
t.Run("oldLock == nil", func(t *testing.T) {
platformLock := depsfile.NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"K43RHM2klOoywtyW",
"swJPXfuCNhJsTM5c",
})
if ct := providersLockCalculateChangeType(nil, platformLock); ct != providersLockChangeTypeNewProvider {
t.Fatalf("output was %s but should be %s", ct, providersLockChangeTypeNewProvider)
}
})
t.Run("oldLock == platformLock", func(t *testing.T) {
platformLock := depsfile.NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"K43RHM2klOoywtyW",
"swJPXfuCNhJsTM5c",
})
oldLock := depsfile.NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"K43RHM2klOoywtyW",
"swJPXfuCNhJsTM5c",
})
if ct := providersLockCalculateChangeType(oldLock, platformLock); ct != providersLockChangeTypeNoChange {
t.Fatalf("output was %s but should be %s", ct, providersLockChangeTypeNoChange)
}
})
t.Run("oldLock > platformLock", func(t *testing.T) {
platformLock := depsfile.NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"K43RHM2klOoywtyW",
"swJPXfuCNhJsTM5c",
})
oldLock := depsfile.NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"1ZAChGWUMWn4zmIk",
"K43RHM2klOoywtyW",
"HWjRvIuWZ1LVatnc",
"swJPXfuCNhJsTM5c",
"KwhJK4p/U2dqbKhI",
})
if ct := providersLockCalculateChangeType(oldLock, platformLock); ct != providersLockChangeTypeNoChange {
t.Fatalf("output was %s but should be %s", ct, providersLockChangeTypeNoChange)
}
})
t.Run("oldLock < platformLock", func(t *testing.T) {
platformLock := depsfile.NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"1ZAChGWUMWn4zmIk",
"K43RHM2klOoywtyW",
"HWjRvIuWZ1LVatnc",
"swJPXfuCNhJsTM5c",
"KwhJK4p/U2dqbKhI",
})
oldLock := depsfile.NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"K43RHM2klOoywtyW",
"swJPXfuCNhJsTM5c",
})
if ct := providersLockCalculateChangeType(oldLock, platformLock); ct != providersLockChangeTypeNewHashes {
t.Fatalf("output was %s but should be %s", ct, providersLockChangeTypeNoChange)
}
})
}

View File

@ -0,0 +1,9 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/test" {
version = "1.0.0"
hashes = [
"h1:invalid",
]
}

View File

@ -0,0 +1,7 @@
terraform {
required_providers {
test = {
source = "hashicorp/test"
}
}
}

View File

@ -403,6 +403,30 @@ func (l *ProviderLock) AllHashes() []getproviders.Hash {
return l.hashes
}
// ContainsAll returns true if the hashes in this ProviderLock contains
// all the hashes in the target.
//
// This function assumes the hashes are in each ProviderLock are sorted.
// If the ProviderLock was created by the NewProviderLock constructor then
// the hashes are guaranteed to be sorted.
func (l *ProviderLock) ContainsAll(target *ProviderLock) bool {
if target == nil || len(target.hashes) == 0 {
return true
}
targetIndex := 0
for ix := 0; ix < len(l.hashes); ix++ {
if l.hashes[ix] == target.hashes[targetIndex] {
targetIndex++
if targetIndex >= len(target.hashes) {
return true
}
}
}
return false
}
// PreferredHashes returns a filtered version of the AllHashes return value
// which includes only the strongest of the availabile hash schemes, in
// case legacy hash schemes are deprecated over time but still supported for

View File

@ -216,3 +216,94 @@ func TestLocksProviderSetRemove(t *testing.T) {
}
}
}
func TestProviderLockContainsAll(t *testing.T) {
provider := addrs.NewDefaultProvider("provider")
v2 := getproviders.MustParseVersion("2.0.0")
v2EqConstraints := getproviders.MustParseVersionConstraints("2.0.0")
t.Run("non-symmetric", func(t *testing.T) {
target := NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"K43RHM2klOoywtyW",
"swJPXfuCNhJsTM5c",
})
original := NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"1ZAChGWUMWn4zmIk",
"K43RHM2klOoywtyW",
"HWjRvIuWZ1LVatnc",
"swJPXfuCNhJsTM5c",
"KwhJK4p/U2dqbKhI",
})
if !original.ContainsAll(target) {
t.Errorf("orginal should contain all hashes in target")
}
if target.ContainsAll(original) {
t.Errorf("target should not contain all hashes in orginal")
}
})
t.Run("symmetric", func(t *testing.T) {
target := NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"K43RHM2klOoywtyW",
"swJPXfuCNhJsTM5c",
})
original := NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"K43RHM2klOoywtyW",
"swJPXfuCNhJsTM5c",
})
if !original.ContainsAll(target) {
t.Errorf("orginal should contain all hashes in target")
}
if !target.ContainsAll(original) {
t.Errorf("target should not contain all hashes in orginal")
}
})
t.Run("edge case - null", func(t *testing.T) {
original := NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"K43RHM2klOoywtyW",
"swJPXfuCNhJsTM5c",
})
if !original.ContainsAll(nil) {
t.Fatalf("orginal should report true on nil")
}
})
t.Run("edge case - empty", func(t *testing.T) {
original := NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"K43RHM2klOoywtyW",
"swJPXfuCNhJsTM5c",
})
target := NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{})
if !original.ContainsAll(target) {
t.Fatalf("orginal should report true on empty")
}
})
t.Run("edge case - original empty", func(t *testing.T) {
original := NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{})
target := NewProviderLock(provider, v2, v2EqConstraints, []getproviders.Hash{
"9r3i9a9QmASqMnQM",
"K43RHM2klOoywtyW",
"swJPXfuCNhJsTM5c",
})
if original.ContainsAll(target) {
t.Fatalf("orginal should report false when empty")
}
})
}

View File

@ -464,7 +464,13 @@ NeedProvider:
installTo = i.targetDir
linkTo = nil // no linking needed
}
authResult, err := installTo.InstallPackage(ctx, meta, preferredHashes)
allowedHashes := preferredHashes
if mode.forceInstallChecksums() {
allowedHashes = []getproviders.Hash{}
}
authResult, err := installTo.InstallPackage(ctx, meta, allowedHashes)
if err != nil {
// TODO: Consider retrying for certain kinds of error that seem
// likely to be transient. For now, we just treat all errors equally.
@ -594,6 +600,11 @@ const (
// sets.
InstallNewProvidersOnly InstallMode = 'N'
// InstallNewProvidersForce is an InstallMode that follows the same
// logic as InstallNewProvidersOnly except it does not verify existing
// checksums but force installs new checksums for all given providers.
InstallNewProvidersForce InstallMode = 'F'
// InstallUpgrades is an InstallMode that causes the installer to check
// all requested providers to see if new versions are available that
// are also in the given version sets, even if a suitable version of
@ -605,6 +616,10 @@ func (m InstallMode) forceQueryAllProviders() bool {
return m == InstallUpgrades
}
func (m InstallMode) forceInstallChecksums() bool {
return m == InstallNewProvidersForce
}
// InstallerError is an error type that may be returned (but is not guaranteed)
// from Installer.EnsureProviderVersions to indicate potentially several
// separate failed installation outcomes for different providers included in

View File

@ -1364,6 +1364,120 @@ func TestEnsureProviderVersions(t *testing.T) {
}
},
},
"force mode ignores hashes": {
Source: getproviders.NewMockSource(
[]getproviders.PackageMeta{
{
Provider: beepProvider,
Version: getproviders.MustParseVersion("1.0.0"),
TargetPlatform: fakePlatform,
Location: beepProviderDir,
},
},
nil,
),
LockFile: `
provider "example.com/foo/beep" {
version = "1.0.0"
constraints = ">= 1.0.0"
hashes = [
"h1:does-not-match",
]
}
`,
Mode: InstallNewProvidersForce,
Reqs: getproviders.Requirements{
beepProvider: getproviders.MustParseVersionConstraints(">= 1.0.0"),
},
Check: func(t *testing.T, dir *Dir, locks *depsfile.Locks) {
if allCached := dir.AllAvailablePackages(); len(allCached) != 1 {
t.Errorf("wrong number of cache directory entries; want only one\n%s", spew.Sdump(allCached))
}
if allLocked := locks.AllProviders(); len(allLocked) != 1 {
t.Errorf("wrong number of provider lock entries; want only one\n%s", spew.Sdump(allLocked))
}
gotLock := locks.Provider(beepProvider)
wantLock := depsfile.NewProviderLock(
beepProvider,
getproviders.MustParseVersion("1.0.0"),
getproviders.MustParseVersionConstraints(">= 1.0.0"),
[]getproviders.Hash{beepProviderHash, "h1:does-not-match"},
)
if diff := cmp.Diff(wantLock, gotLock, depsfile.ProviderLockComparer); diff != "" {
t.Errorf("wrong lock entry\n%s", diff)
}
gotEntry := dir.ProviderLatestVersion(beepProvider)
wantEntry := &CachedProvider{
Provider: beepProvider,
Version: getproviders.MustParseVersion("1.0.0"),
PackageDir: filepath.Join(dir.BasePath(), "example.com/foo/beep/1.0.0/bleep_bloop"),
}
if diff := cmp.Diff(wantEntry, gotEntry); diff != "" {
t.Errorf("wrong cache entry\n%s", diff)
}
},
WantEvents: func(inst *Installer, dir *Dir) map[addrs.Provider][]*testInstallerEventLogItem {
return map[addrs.Provider][]*testInstallerEventLogItem{
noProvider: {
{
Event: "PendingProviders",
Args: map[addrs.Provider]getproviders.VersionConstraints{
beepProvider: getproviders.MustParseVersionConstraints(">= 1.0.0"),
},
},
{
Event: "ProvidersFetched",
Args: map[addrs.Provider]*getproviders.PackageAuthenticationResult{
beepProvider: nil,
},
},
},
beepProvider: {
{
Event: "QueryPackagesBegin",
Provider: beepProvider,
Args: struct {
Constraints string
Locked bool
}{">= 1.0.0", true},
},
{
Event: "QueryPackagesSuccess",
Provider: beepProvider,
Args: "1.0.0",
},
{
Event: "FetchPackageMeta",
Provider: beepProvider,
Args: "1.0.0",
},
{
Event: "FetchPackageBegin",
Provider: beepProvider,
Args: struct {
Version string
Location getproviders.PackageLocation
}{"1.0.0", beepProviderDir},
},
{
Event: "FetchPackageSuccess",
Provider: beepProvider,
Args: struct {
Version string
LocalDir string
AuthResult string
}{
"1.0.0",
filepath.Join(dir.BasePath(), "example.com/foo/beep/1.0.0/bleep_bloop"),
"unauthenticated",
},
},
},
}
},
},
}
ctx := context.Background()