internal/providercache: Linking from one cache to another

When a system-wide shared plugin cache is configured, we'll want to make
use of entries already in the shared cache when populating a local
(configuration-specific) cache.

This new method LinkFromOtherCache encapsulates the work of placing a link
from one cache to another. If possible it will create a symlink, therefore
retaining a key advantage of configuring a shared plugin cache, but
otherwise we'll do a deep copy of the package directory from one cache
to the other.

Our old provider installer would always skip trying to create symlinks on
Windows because Go standard library support for os.Symlink on Windows
was inconsistent in older versions. However, os.Symlink can now create
symlinks using a new API introduced in a Windows 10 update and cleanly
fail if symlink creation is impossible, so it's safe for us to just
try to create the symlink and react if that produces an error, just as we
used to do on non-Windows systems when possibly creating symlinks on
filesystems that cannot support them.

internal/providercache/dir_modify.go

@@ -0,0 +1,95 @@
+package providercache
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/terraform/internal/copydir"
+	"github.com/hashicorp/terraform/internal/getproviders"
+)
+
+// LinkFromOtherCache takes a CachedProvider value produced from another Dir
+// and links it into the cache represented by the receiver Dir.
+//
+// This is used to implement tiered caching, where new providers are first
+// populated into a system-wide shared cache and then linked from there into
+// a configuration-specific local cache.
+//
+// It's invalid to link a CachedProvider from a particular Dir into that same
+// Dir, because that would otherwise potentially replace a real package
+// directory with a circular link back to itself.
+func (d *Dir) LinkFromOtherCache(entry *CachedProvider) error {
+	newPath := getproviders.UnpackedDirectoryPathForPackage(
+		d.baseDir, entry.Provider, entry.Version, d.targetPlatform,
+	)
+	currentPath := entry.PackageDir
+
+	absNew, err := filepath.Abs(newPath)
+	if err != nil {
+		return fmt.Errorf("failed to make new path %s absolute: %s", newPath, err)
+	}
+	absCurrent, err := filepath.Abs(currentPath)
+	if err != nil {
+		return fmt.Errorf("failed to make existing cache path %s absolute: %s", currentPath, err)
+	}
+
+	// Before we do anything else, we'll do a quick check to make sure that
+	// these two paths are not pointing at the same physical directory on
+	// disk. This compares the files by their OS-level device and directory
+	// entry identifiers, not by their virtual filesystem paths.
+	if same, err := copydir.SameFile(absNew, absCurrent); same {
+		return fmt.Errorf("cannot link existing cache path %s to itself", newPath)
+	} else if err != nil {
+		return fmt.Errorf("failed to determine if %s and %s are the same: %s", currentPath, newPath, err)
+	}
+
+	// Invalidate our metaCache so that subsequent read calls will re-scan to
+	// incorporate any changes we make here.
+	d.metaCache = nil
+
+	// Delete anything that's already present at this path first.
+	err = os.RemoveAll(currentPath)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to remove existing %s before linking it to %s: %s", currentPath, newPath, err)
+	}
+
+	// We'll prefer to create a symlink if possible, but we'll fall back to
+	// a recursive copy if symlink creation fails. It could fail for a number
+	// of reasons, including being on Windows 8 without administrator
+	// privileges or being on a legacy filesystem like FAT that has no way
+	// to represent a symlink. (Generalized symlink support for Windows was
+	// introduced in a Windows 10 minor update.)
+	//
+	// We'd prefer to use a relative path for the symlink to reduce the risk
+	// of it being broken by moving things around later, but we'll fall back
+	// on the absolute path we already calculated if that isn't possible
+	// (e.g. because the two paths are on different "volumes" on an OS with
+	// that concept, like Windows with drive letters and UNC host/share names.)
+	linkTarget, err := filepath.Rel(newPath, absCurrent)
+	if err != nil {
+		linkTarget = absCurrent
+	}
+
+	parentDir := filepath.Dir(absNew)
+	err = os.MkdirAll(parentDir, 0755)
+	if err != nil && os.IsExist(err) {
+		return fmt.Errorf("failed to create parent directories leading to %s: %s", newPath, err)
+	}
+
+	err = os.Symlink(linkTarget, absNew)
+	if err == nil {
+		// Success, then!
+		return nil
+	}
+
+	// If we get down here then symlinking failed and we need a deep copy
+	// instead.
+	err = copydir.CopyDir(absNew, absCurrent)
+	if err != nil {
+		return fmt.Errorf("failed to either symlink or copy %s to %s: %s", absCurrent, absNew, err)
+	}
+
+	// If we got here then apparently our copy succeeded, so we're done.
+	return nil
+}

internal/providercache/dir_modify_test.go

@@ -0,0 +1,98 @@
+package providercache
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/apparentlymart/go-versions/versions"
+	"github.com/google/go-cmp/cmp"
+
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
+)
+
+func LinkFromOtherCache(t *testing.T) {
+	srcDirPath := "testdata/cachedir"
+	tmpDirPath, err := ioutil.TempDir("", "terraform-test-providercache")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDirPath)
+
+	windowsPlatform := getproviders.Platform{
+		OS:   "windows",
+		Arch: "amd64",
+	}
+	nullProvider := addrs.NewProvider(
+		addrs.DefaultRegistryHost, "hashicorp", "null",
+	)
+
+	srcDir := newDirWithPlatform(srcDirPath, windowsPlatform)
+	tmpDir := newDirWithPlatform(tmpDirPath, windowsPlatform)
+
+	// First we'll check our preconditions: srcDir should have only the
+	// null provider version 2.0.0 in it, because we're faking that we're on
+	// Windows, and tmpDir should have no providers in it at all.
+
+	gotSrcInitial := srcDir.AllAvailablePackages()
+	wantSrcInitial := map[addrs.Provider][]CachedProvider{
+		nullProvider: {
+			CachedProvider{
+				Provider: nullProvider,
+
+				// We want 2.0.0 rather than 2.1.0 because the 2.1.0 package is
+				// still packed and thus not considered to be a cache member.
+				Version: versions.MustParseVersion("2.0.0"),
+
+				PackageDir:     "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64",
+				ExecutableFile: "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64/terraform-provider-null.exe",
+			},
+		},
+	}
+	if diff := cmp.Diff(wantSrcInitial, gotSrcInitial); diff != "" {
+		t.Fatalf("incorrect initial source directory contents\n%s", diff)
+	}
+
+	gotTmpInitial := tmpDir.AllAvailablePackages()
+	wantTmpInitial := map[addrs.Provider][]CachedProvider{}
+	if diff := cmp.Diff(wantTmpInitial, gotTmpInitial); diff != "" {
+		t.Fatalf("incorrect initial temp directory contents\n%s", diff)
+	}
+
+	cacheEntry := srcDir.ProviderLatestVersion(nullProvider)
+	if cacheEntry == nil {
+		// This is weird because we just checked for the presence of this above
+		t.Fatalf("null provider has no latest version in source directory")
+	}
+
+	err = tmpDir.LinkFromOtherCache(cacheEntry)
+	if err != nil {
+		t.Fatalf("LinkFromOtherCache failed: %s", err)
+	}
+
+	// Now we should see the same version reflected in the temporary directory.
+	got := tmpDir.AllAvailablePackages()
+	want := map[addrs.Provider][]CachedProvider{
+		nullProvider: {
+			CachedProvider{
+				Provider: nullProvider,
+
+				// We want 2.0.0 rather than 2.1.0 because the 2.1.0 package is
+				// still packed and thus not considered to be a cache member.
+				Version: versions.MustParseVersion("2.0.0"),
+
+				// These are still pointed into the testdata directory because
+				// we created a symlink in our tmpDir. (This part of the test
+				// is expected to fail if the temporary directory is on a
+				// filesystem that cannot support symlinks, in which case
+				// we should see the temporary directory paths here instead.)
+				PackageDir:     "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64",
+				ExecutableFile: "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64/terraform-provider-null.exe",
+			},
+		},
+	}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("wrong cache contents after link\n%s", diff)
+	}
+}