CI: move grabpl build-plugins from grabpl to grafana (#53071)

* add grabpl command to build internal plugins

* grabpl build-plugins -> ./bin/build build-plugins

.drone.yml

@@ -334,7 +334,7 @@ steps:
   image: grafana/build-container:1.5.9
   name: build-frontend-packages
 - commands:
-  - ./bin/grabpl build-plugins --jobs 8 --edition oss
+  - ./bin/build  build-plugins --jobs 8 --edition oss
   depends_on:
   - gen-version
   - yarn-install
@@ -1123,7 +1123,7 @@ steps:
   image: grafana/build-container:1.5.9
   name: build-frontend-packages
 - commands:
-  - ./bin/grabpl build-plugins --jobs 8 --edition oss
+  - ./bin/build  build-plugins --jobs 8 --edition oss
   depends_on:
   - gen-version
   - yarn-install
@@ -1831,7 +1831,7 @@ steps:
   image: grafana/build-container:1.5.9
   name: build-frontend-packages
 - commands:
-  - ./bin/grabpl build-plugins --jobs 8 --edition oss
+  - ./bin/build  build-plugins --jobs 8 --edition oss
   depends_on:
   - gen-version
   - yarn-install
@@ -2468,7 +2468,7 @@ steps:
   image: grafana/build-container:1.5.9
   name: build-frontend-packages
 - commands:
-  - ./bin/grabpl build-plugins --jobs 8 --edition enterprise
+  - ./bin/build  build-plugins --jobs 8 --edition enterprise
   depends_on:
   - gen-version
   - yarn-install
@@ -3778,7 +3778,7 @@ steps:
   image: grafana/build-container:1.5.9
   name: build-frontend-packages
 - commands:
-  - ./bin/grabpl build-plugins --jobs 8 --edition oss
+  - ./bin/build  build-plugins --jobs 8 --edition oss
   depends_on:
   - gen-version
   - yarn-install
@@ -4361,7 +4361,7 @@ steps:
   image: grafana/build-container:1.5.9
   name: build-frontend-packages
 - commands:
-  - ./bin/grabpl build-plugins --jobs 8 --edition enterprise
+  - ./bin/build  build-plugins --jobs 8 --edition enterprise
   depends_on:
   - gen-version
   - yarn-install
@@ -5133,6 +5133,6 @@ kind: secret
 name: gcp_upload_artifacts_key
 ---
 kind: signature
-hmac: cf7b07bd7aa1c46665d485693553ff98e7dacf0e4853c5d640437d0218c91db7
+hmac: 7d799b8c9888eeb4b72f6aa1721fe335d4eb74c6fb2db9612970adbb9462d395
 
 ...

pkg/build/cmd/buildinternalplugins.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"context"
+	"log"
+	"path/filepath"
+
+	"github.com/grafana/grafana/pkg/build/config"
+	"github.com/grafana/grafana/pkg/build/errutil"
+	"github.com/grafana/grafana/pkg/build/plugins"
+	"github.com/grafana/grafana/pkg/build/syncutil"
+	"github.com/urfave/cli/v2"
+)
+
+func BuildInternalPlugins(c *cli.Context) error {
+	cfg := config.Config{
+		NumWorkers: c.Int("jobs"),
+	}
+
+	const grafanaDir = "."
+	metadata, err := config.GetMetadata(filepath.Join("dist", "version.json"))
+	if err != nil {
+		return err
+	}
+	verMode, err := config.GetVersion(metadata.ReleaseMode)
+	if err != nil {
+		return err
+	}
+
+	log.Println("Building internal Grafana plug-ins...")
+
+	ctx := context.Background()
+
+	p := syncutil.NewWorkerPool(cfg.NumWorkers)
+	defer p.Close()
+
+	var g *errutil.Group
+	g, ctx = errutil.GroupWithContext(ctx)
+	if err := plugins.Build(ctx, grafanaDir, p, g, verMode); err != nil {
+		return cli.Exit(err.Error(), 1)
+	}
+	if err := g.Wait(); err != nil {
+		return cli.Exit(err.Error(), 1)
+	}
+
+	if err := plugins.Download(ctx, grafanaDir, p); err != nil {
+		return cli.Exit(err.Error(), 1)
+	}
+
+	log.Println("Successfully built Grafana plug-ins!")
+
+	return nil
+}

pkg/build/cmd/flags.go

@@ -20,4 +20,16 @@ var (
 		Name:  "variants",
 		Usage: "Comma-separated list of variants to build",
 	}
+	noInstallDepsFlag = cli.BoolFlag{
+		Name:  "no-install-deps",
+		Usage: "Don't install dependencies",
+	}
+	signingAdminFlag = cli.BoolFlag{
+		Name:  "signing-admin",
+		Usage: "Use manifest signing admin API endpoint?",
+	}
+	signFlag = cli.BoolFlag{
+		Name:  "sign",
+		Usage: "Enable plug-in signing (you must set GRAFANA_API_KEY)",
+	}
 )

pkg/build/cmd/main.go

@@ -23,6 +23,18 @@ func main() {
 				&buildIDFlag,
 			},
 		},
+		{
+			Name:   "build-plugins",
+			Usage:  "Build internal plug-ins",
+			Action: ArgCountWrapper(1, BuildInternalPlugins),
+			Flags: []cli.Flag{
+				&jobsFlag,
+				&editionFlag,
+				&signingAdminFlag,
+				&signFlag,
+				&noInstallDepsFlag,
+			},
+		},
 	}
 
 	if err := app.Run(os.Args); err != nil {

pkg/build/config/version.go

@@ -5,8 +5,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
 )
 
 type Metadata struct {
@@ -83,5 +87,77 @@ func GetVersion(mode VersionMode) (*Version, error) {
 		return &v, nil
 	}
 
-	return nil, fmt.Errorf("mode not found in version list")
+	return nil, fmt.Errorf("mode '%s' not found in version list", mode)
+}
+
+func shortenBuildID(buildID string) string {
+	buildID = strings.ReplaceAll(buildID, "-", "")
+	if len(buildID) < 9 {
+		return buildID
+	}
+
+	return buildID[0:8]
+}
+
+// GetGrafanaVersion gets the Grafana version from the package.json
+func GetGrafanaVersion(buildID, grafanaDir string) (string, error) {
+	pkgJSONPath := filepath.Join(grafanaDir, "package.json")
+	//nolint:gosec
+	pkgJSONB, err := ioutil.ReadFile(pkgJSONPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to read %q: %w", pkgJSONPath, err)
+	}
+	pkgObj := map[string]interface{}{}
+	if err := json.Unmarshal(pkgJSONB, &pkgObj); err != nil {
+		return "", fmt.Errorf("failed decoding %q: %w", pkgJSONPath, err)
+	}
+
+	version := pkgObj["version"].(string)
+	if version == "" {
+		return "", fmt.Errorf("failed to read version from %q", pkgJSONPath)
+	}
+	if buildID != "" {
+		buildID = shortenBuildID(buildID)
+		verComponents := strings.Split(version, "-")
+		version = verComponents[0]
+		if len(verComponents) > 1 {
+			buildID = fmt.Sprintf("%s%s", buildID, verComponents[1])
+		}
+		version = fmt.Sprintf("%s-%s", version, buildID)
+	}
+
+	return version, nil
+}
+
+func CheckDroneTargetBranch() (VersionMode, error) {
+	reRlsBranch := regexp.MustCompile(`^v\d+\.\d+\.x$`)
+	target := os.Getenv("DRONE_TARGET_BRANCH")
+	if target == "" {
+		return "", fmt.Errorf("failed to get DRONE_TARGET_BRANCH environmental variable")
+	} else if target == string(MainMode) {
+		return MainMode, nil
+	}
+	if reRlsBranch.MatchString(target) {
+		return ReleaseBranchMode, nil
+	}
+	return "", fmt.Errorf("unrecognized target branch: %s", target)
+}
+
+func CheckSemverSuffix() (VersionMode, error) {
+	reBetaRls := regexp.MustCompile(`beta.*`)
+	reTestRls := regexp.MustCompile(`test.*`)
+	tagSuffix, ok := os.LookupEnv("DRONE_SEMVER_PRERELEASE")
+	if !ok || tagSuffix == "" {
+		fmt.Println("DRONE_SEMVER_PRERELEASE doesn't exist for a tag, this is a release event...")
+		return ReleaseMode, nil
+	}
+	switch {
+	case reBetaRls.MatchString(tagSuffix):
+		return BetaReleaseMode, nil
+	case reTestRls.MatchString(tagSuffix):
+		return TestReleaseMode, nil
+	default:
+		fmt.Printf("DRONE_SEMVER_PRERELEASE is custom string, release event with %s suffix\n", tagSuffix)
+		return ReleaseMode, nil
+	}
 }

pkg/build/grafana/variant.go

@@ -46,9 +46,9 @@ func BuildVariant(ctx context.Context, opts BuildVariantOpts) error {
 			stderr = bytes.NewBuffer(nil)
 		)
 
-		args.BuildOpts.Workdir = grafanaDir
-		args.BuildOpts.Stdout = stdout
-		args.BuildOpts.Stderr = stderr
+		args.Workdir = grafanaDir
+		args.Stdout = stdout
+		args.Stderr = stderr
 		args.Package = pkg
 
 		if err := BuildGrafanaBinary(ctx, binary, opts.Version, args, opts.Edition); err != nil {

pkg/build/plugins/build.go

@@ -0,0 +1,66 @@
+package plugins
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/grafana/grafana/pkg/build/config"
+	"github.com/grafana/grafana/pkg/build/errutil"
+	"github.com/grafana/grafana/pkg/build/syncutil"
+	"github.com/grafana/grafana/pkg/infra/fs"
+)
+
+type PluginSigningMode = int
+
+// BuildPlugins builds internal plugins.
+// The built plugins are placed in plugins-bundled/dist/.
+func Build(ctx context.Context, grafanaDir string, p syncutil.WorkerPool, g *errutil.Group, verMode *config.Version) error {
+	log.Printf("Building plugins in %q...", grafanaDir)
+
+	root := filepath.Join(grafanaDir, "plugins-bundled", "internal")
+	fis, err := ioutil.ReadDir(root)
+	if err != nil {
+		return err
+	}
+
+	for i := range fis {
+		fi := fis[i]
+		if !fi.IsDir() {
+			continue
+		}
+
+		dpath := filepath.Join(root, fi.Name())
+
+		p.Schedule(g.Wrap(func() error {
+			log.Printf("Building plugin %q...", dpath)
+
+			cmd := exec.Command("yarn", "build")
+			cmd.Dir = dpath
+			if output, err := cmd.CombinedOutput(); err != nil {
+				return fmt.Errorf("yarn build failed: %s", output)
+			}
+
+			dstPath := filepath.Join("plugins-bundled", "dist", fi.Name())
+			if err := fs.CopyRecursive(filepath.Join(dpath, "dist"), dstPath); err != nil {
+				return err
+			}
+			if !verMode.PluginSignature.Sign {
+				return nil
+			}
+
+			return BuildManifest(ctx, dstPath, verMode.PluginSignature.AdminSign)
+		}))
+	}
+
+	if err := g.Wait(); err != nil {
+		return err
+	}
+
+	log.Printf("Built all plug-ins successfully!")
+
+	return nil
+}

pkg/build/plugins/download.go

@@ -0,0 +1,118 @@
+package plugins
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/grafana/grafana/pkg/build/errutil"
+	"github.com/grafana/grafana/pkg/build/syncutil"
+)
+
+// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package.
+func logCloseError(closeFunc func() error) {
+	if err := closeFunc(); err != nil {
+		log.Println(err)
+	}
+}
+
+// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package.
+func logError(err error) {
+	if err != nil {
+		log.Println(err)
+	}
+}
+
+// pluginManifest has details of an external plugin package.
+type pluginManifest struct {
+	Name     string `json:"name"`
+	Version  string `json:"version"`
+	Checksum string `json:"checksum"`
+}
+
+// pluginsManifest represents a manifest of Grafana's external plugins.
+type pluginsManifest struct {
+	Plugins []pluginManifest `json:"plugins"`
+}
+
+// downloadPlugins downloads Grafana plugins that should be bundled into packages.
+//
+// The plugin archives are downloaded into <grafanaDir>/plugins-bundled.
+func Download(ctx context.Context, grafanaDir string, p syncutil.WorkerPool) error {
+	g, _ := errutil.GroupWithContext(ctx)
+
+	log.Println("Downloading external plugins...")
+
+	var m pluginsManifest
+	manifestPath := filepath.Join(grafanaDir, "plugins-bundled", "external.json")
+	//nolint:gosec
+	manifestB, err := ioutil.ReadFile(manifestPath)
+	if err != nil {
+		return fmt.Errorf("failed to open plugins manifest %q: %w", manifestPath, err)
+	}
+	if err := json.Unmarshal(manifestB, &m); err != nil {
+		return err
+	}
+
+	for i := range m.Plugins {
+		pm := m.Plugins[i]
+		p.Schedule(g.Wrap(func() error {
+			tgt := filepath.Join(grafanaDir, "plugins-bundled", fmt.Sprintf("%s-%s.zip", pm.Name, pm.Version))
+			out, err := os.Create(tgt)
+			if err != nil {
+				return err
+			}
+			defer logCloseError(out.Close)
+
+			u := fmt.Sprintf("http://storage.googleapis.com/plugins-ci/plugins/%s/%s-%s.zip", pm.Name, pm.Name,
+				pm.Version)
+			log.Printf("Downloading plugin %q to %q...", u, tgt)
+			// nolint:gosec
+			resp, err := http.Get(u)
+			if err != nil {
+				return fmt.Errorf("downloading %q failed: %w", u, err)
+			}
+			defer logError(resp.Body.Close())
+
+			if resp.StatusCode != http.StatusOK {
+				return fmt.Errorf("failed to download %q, status code %d", u, resp.StatusCode)
+			}
+
+			if _, err := io.Copy(out, resp.Body); err != nil {
+				return fmt.Errorf("downloading %q failed: %w", u, err)
+			}
+			if err := out.Close(); err != nil {
+				return fmt.Errorf("downloading %q failed: %w", u, err)
+			}
+
+			//nolint:gosec
+			fd, err := os.Open(tgt)
+			if err != nil {
+				return err
+			}
+			defer logCloseError(fd.Close)
+
+			h := sha256.New()
+			if _, err := io.Copy(h, fd); err != nil {
+				return err
+			}
+
+			chksum := hex.EncodeToString(h.Sum(nil))
+			if chksum != pm.Checksum {
+				return fmt.Errorf("plugin %q has bad checksum: %s (expected %s)", u, chksum, pm.Checksum)
+			}
+
+			return Unzip(tgt, filepath.Join(grafanaDir, "plugins-bundled"))
+		}))
+	}
+
+	return g.Wait()
+}

pkg/build/plugins/manifest.go

@@ -0,0 +1,196 @@
+package plugins
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+)
+
+type manifest struct {
+	Plugin  string            `json:"plugin"`
+	Version string            `json:"version"`
+	Files   map[string]string `json:"files"`
+}
+
+func getManifest(dpath string, chksums map[string]string) (manifest, error) {
+	m := manifest{}
+
+	type pluginInfo struct {
+		Version string `json:"version"`
+	}
+
+	type plugin struct {
+		ID   string     `json:"id"`
+		Info pluginInfo `json:"info"`
+	}
+
+	//nolint:gosec
+	f, err := os.Open(filepath.Join(dpath, "plugin.json"))
+	if err != nil {
+		return m, err
+	}
+	decoder := json.NewDecoder(f)
+	var p plugin
+	if err := decoder.Decode(&p); err != nil {
+		return m, err
+	}
+
+	if p.ID == "" {
+		return m, fmt.Errorf("plugin.json doesn't define id")
+	}
+	if p.Info.Version == "" {
+		return m, fmt.Errorf("plugin.json doesn't define info.version")
+	}
+
+	return manifest{
+		Plugin:  p.ID,
+		Version: p.Info.Version,
+		Files:   chksums,
+	}, nil
+}
+
+// BuildManifest requests a plugin's signed manifest file fromt he Grafana API.
+// If signingAdmin is true, the manifest signing admin endpoint (without plugin ID) will be used, and requires
+// an admin API key.
+func BuildManifest(ctx context.Context, dpath string, signingAdmin bool) error {
+	log.Printf("Building manifest for plug-in at %q", dpath)
+
+	apiKey := os.Getenv("GRAFANA_API_KEY")
+	if apiKey == "" {
+		return fmt.Errorf("GRAFANA_API_KEY must be set")
+	}
+
+	manifestPath := filepath.Join(dpath, "MANIFEST.txt")
+	chksums, err := getChksums(dpath, manifestPath)
+	if err != nil {
+		return err
+	}
+	m, err := getManifest(dpath, chksums)
+	if err != nil {
+		return err
+	}
+
+	b := bytes.NewBuffer(nil)
+	encoder := json.NewEncoder(b)
+	if err := encoder.Encode(&m); err != nil {
+		return err
+	}
+	jsonB := b.Bytes()
+	u := "https://grafana.com/api/plugins/ci/sign"
+	if !signingAdmin {
+		u = fmt.Sprintf("https://grafana.com/api/plugins/%s/ci/sign", m.Plugin)
+	}
+	log.Printf("Requesting signed manifest from Grafana API...")
+	req, err := http.NewRequestWithContext(ctx, "POST", u, bytes.NewReader(jsonB))
+	if err != nil {
+		return err
+	}
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey))
+	req.Header.Add("Content-Type", "application/json")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to get signed manifest from Grafana API: %w", err)
+	}
+	defer logError(resp.Body.Close())
+	if resp.StatusCode != 200 {
+		msg, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			log.Printf("Failed to read response body: %s", err)
+			msg = []byte("")
+		}
+		return fmt.Errorf("request for signed manifest failed with status code %d: %s", resp.StatusCode, string(msg))
+	}
+
+	log.Printf("Successfully signed manifest via Grafana API, writing to %q", manifestPath)
+	f, err := os.Create(manifestPath)
+	if err != nil {
+		return fmt.Errorf("failed to create %s: %w", manifestPath, err)
+	}
+	defer logCloseError(f.Close)
+	if _, err := io.Copy(f, resp.Body); err != nil {
+		return fmt.Errorf("failed to write %s: %w", manifestPath, err)
+	}
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("failed to write %s: %w", manifestPath, err)
+	}
+
+	return nil
+}
+
+func getChksums(dpath, manifestPath string) (map[string]string, error) {
+	manifestPath = filepath.Clean(manifestPath)
+
+	chksums := map[string]string{}
+	if err := filepath.Walk(dpath, func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if fi.IsDir() {
+			return nil
+		}
+
+		path = filepath.Clean(path)
+
+		// Handle symbolic links
+		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
+			finalPath, err := filepath.EvalSymlinks(path)
+			if err != nil {
+				return err
+			}
+
+			log.Printf("Handling symlink %q, pointing to %q", path, finalPath)
+
+			info, err := os.Stat(finalPath)
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+
+			if _, err := filepath.Rel(dpath, finalPath); err != nil {
+				return fmt.Errorf("symbolic link %q targets a file outside of the plugin directory: %q", path, finalPath)
+			}
+
+			if finalPath == manifestPath {
+				return nil
+			}
+		}
+
+		if path == manifestPath {
+			return nil
+		}
+
+		h := sha256.New()
+		//nolint:gosec
+		f, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer logCloseError(f.Close)
+		if _, err := io.Copy(h, f); err != nil {
+			return err
+		}
+
+		relPath, err := filepath.Rel(dpath, path)
+		if err != nil {
+			return err
+		}
+		chksums[relPath] = fmt.Sprintf("%x", h.Sum(nil))
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return chksums, nil
+}

pkg/build/plugins/zip.go

@@ -0,0 +1,64 @@
+package plugins
+
+import (
+	"archive/zip"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+)
+
+// Unzip unzips a plugin.
+func Unzip(fpath, tgtDir string) error {
+	log.Printf("Unzipping plugin %q into %q...", fpath, tgtDir)
+
+	r, err := zip.OpenReader(fpath)
+	if err != nil {
+		return err
+	}
+	defer logCloseError(r.Close)
+
+	// Closure to address file descriptors issue with all the deferred .Close() methods
+	extractAndWriteFile := func(f *zip.File) error {
+		log.Printf("Extracting zip member %q...", f.Name)
+
+		rc, err := f.Open()
+		if err != nil {
+			return err
+		}
+		defer logCloseError(rc.Close)
+
+		//nolint:gosec
+		dstPath := filepath.Join(tgtDir, f.Name)
+
+		if f.FileInfo().IsDir() {
+			return os.MkdirAll(dstPath, f.Mode())
+		}
+
+		if err := os.MkdirAll(filepath.Dir(dstPath), f.Mode()); err != nil {
+			return err
+		}
+
+		//nolint:gosec
+		fd, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+		if err != nil {
+			return err
+		}
+		defer logCloseError(fd.Close)
+
+		// nolint:gosec
+		if _, err := io.Copy(fd, rc); err != nil {
+			return err
+		}
+
+		return fd.Close()
+	}
+
+	for _, f := range r.File {
+		if err := extractAndWriteFile(f); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

scripts/drone/steps/lib.star

@@ -488,7 +488,7 @@ def build_plugins_step(edition, ver_mode):
         ],
         'commands': [
             # TODO: Use percentage for num jobs
-            './bin/grabpl build-plugins --jobs 8 --edition {}'.format(edition),
+            './bin/build  build-plugins --jobs 8 --edition {}'.format(edition),
         ],
     }