CI: move grabpl package command to pkg/build (#55651)

* add grabpl package

* update .drone.yml

* resolve lint errors

.drone.yml

@@ -406,7 +406,7 @@ steps:
   image: grafana/build-container:1.6.2
   name: build-plugins
 - commands:
-  - . scripts/build/gpg-test-vars.sh && ./bin/grabpl package --jobs 8 --edition oss
+  - . scripts/build/gpg-test-vars.sh && ./bin/build package --jobs 8 --edition oss
     --build-id ${DRONE_BUILD_NUMBER} --variants linux-amd64,linux-amd64-musl,darwin-amd64,windows-amd64
   depends_on:
   - build-plugins
@@ -1256,7 +1256,7 @@ steps:
   image: grafana/build-container:1.6.2
   name: build-plugins
 - commands:
-  - ./bin/grabpl package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign
+  - ./bin/build package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign
   depends_on:
   - build-plugins
   - build-backend
@@ -2005,7 +2005,7 @@ steps:
   image: grafana/build-container:1.6.2
   name: build-plugins
 - commands:
-  - ./bin/grabpl package --jobs 8 --edition oss  --sign ${DRONE_TAG}
+  - ./bin/build package --jobs 8 --edition oss  --sign ${DRONE_TAG}
   depends_on:
   - build-plugins
   - build-backend
@@ -2630,7 +2630,7 @@ steps:
   image: grafana/build-container:1.6.2
   name: build-backend-enterprise2
 - commands:
-  - ./bin/grabpl package --jobs 8 --edition enterprise  --sign ${DRONE_TAG}
+  - ./bin/build package --jobs 8 --edition enterprise  --sign ${DRONE_TAG}
   depends_on:
   - build-plugins
   - build-backend
@@ -2786,7 +2786,7 @@ steps:
   image: grafana/grafana-ci-deploy:1.3.3
   name: upload-packages
 - commands:
-  - ./bin/grabpl package --jobs 8 --edition enterprise2  --sign ${DRONE_TAG}
+  - ./bin/build package --jobs 8 --edition enterprise2  --sign ${DRONE_TAG}
   depends_on:
   - build-plugins
   - build-backend
@@ -4004,7 +4004,7 @@ steps:
   image: grafana/build-container:1.6.2
   name: build-plugins
 - commands:
-  - ./bin/grabpl package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign
+  - ./bin/build package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign
   depends_on:
   - build-plugins
   - build-backend
@@ -4574,7 +4574,7 @@ steps:
   image: grafana/build-container:1.6.2
   name: build-backend-enterprise2
 - commands:
-  - ./bin/grabpl package --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER}
+  - ./bin/build package --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER}
     --sign
   depends_on:
   - build-plugins
@@ -4737,7 +4737,7 @@ steps:
     repo:
     - grafana/grafana
 - commands:
-  - ./bin/grabpl package --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER}
+  - ./bin/build package --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER}
     --variants linux-amd64 --sign
   depends_on:
   - build-plugins
@@ -5349,6 +5349,6 @@ kind: secret
 name: packages_secret_access_key
 ---
 kind: signature
-hmac: d94e215384ace272b16ec265c6b2a588eea66b960f1793c5c65eece0fb5a3541
+hmac: 88244e8abc706fc4a5c08b37152198afdd406bcd93366c4c07195f5eab242f70
 
 ...

pkg/build/cmd/main.go

@@ -124,6 +124,19 @@ func main() {
 			Usage:  "Exports version in dist/grafana.version",
 			Action: ExportVersion,
 		},
+		{
+			Name:      "package",
+			Usage:     "Package one or more Grafana variants",
+			ArgsUsage: "[version]",
+			Action:    ArgCountWrapper(1, Package),
+			Flags: []cli.Flag{
+				&jobsFlag,
+				&variantsFlag,
+				&editionFlag,
+				&buildIDFlag,
+				&signFlag,
+			},
+		},
 		{
 			Name:   "store-storybook",
 			Usage:  "Integrity check for storybook build",

pkg/build/cmd/package.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+	"log"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/build/config"
+	"github.com/grafana/grafana/pkg/build/gpg"
+	"github.com/grafana/grafana/pkg/build/packaging"
+	"github.com/grafana/grafana/pkg/build/syncutil"
+	"github.com/urfave/cli/v2"
+)
+
+func Package(c *cli.Context) error {
+	metadata, err := GenerateMetadata(c)
+	if err != nil {
+		return err
+	}
+
+	edition := config.Edition(c.String("edition"))
+
+	releaseMode, err := metadata.GetReleaseMode()
+	if err != nil {
+		return cli.NewExitError(err.Error(), 1)
+	}
+
+	releaseModeConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode)
+	if err != nil {
+		return cli.NewExitError(err.Error(), 1)
+	}
+
+	cfg := config.Config{
+		NumWorkers: c.Int("jobs"),
+	}
+	if err := gpg.LoadGPGKeys(&cfg); err != nil {
+		return cli.Exit(err, 1)
+	}
+	defer gpg.RemoveGPGFiles(cfg)
+
+	ctx := context.Background()
+
+	variantStrs := strings.Split(c.String("variants"), ",")
+	variants := []config.Variant{}
+	for _, varStr := range variantStrs {
+		if varStr == "" {
+			continue
+		}
+
+		variants = append(variants, config.Variant(varStr))
+	}
+
+	if len(variants) == 0 {
+		variants = config.AllVariants
+	}
+
+	log.Printf("Packaging Grafana version %q, version mode %s, %s edition, variants %s", metadata.GrafanaVersion, releaseMode.Mode,
+		edition, strings.Join(variantStrs, ","))
+
+	if err := gpg.Import(cfg); err != nil {
+		return cli.Exit(err, 1)
+	}
+
+	p := syncutil.NewWorkerPool(cfg.NumWorkers)
+	defer p.Close()
+
+	if err := packaging.PackageGrafana(ctx, metadata.GrafanaVersion, ".", cfg, edition, variants, releaseModeConfig.PluginSignature.Sign, p); err != nil {
+		return cli.Exit(err, 1)
+	}
+
+	log.Println("Successfully packaged Grafana!")
+	return nil
+}

pkg/build/config/variant.go

@@ -15,6 +15,18 @@ const (
 	VariantWindowsAmd64   Variant = "windows-amd64"
 )
 
+var AllVariants = []Variant{
+	VariantArmV6,
+	VariantArmV7,
+	VariantArmV7Musl,
+	VariantArm64,
+	VariantArm64Musl,
+	VariantDarwinAmd64,
+	VariantWindowsAmd64,
+	VariantLinuxAmd64,
+	VariantLinuxAmd64Musl,
+}
+
 // Architecture is an allowed value in the GOARCH environment variable.
 type Architecture string
 

pkg/build/gpg/gpg.go

@@ -0,0 +1,84 @@
+package gpg
+
+import (
+	"encoding/base64"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/grafana/grafana/pkg/build/config"
+)
+
+func createTempFile(sfx string) (string, error) {
+	f, err := os.CreateTemp("", fmt.Sprintf("*-%s", sfx))
+	if err != nil {
+		return "", err
+	}
+	if err := f.Close(); err != nil {
+		return "", err
+	}
+
+	return f.Name(), nil
+}
+
+// LoadGPGKeys loads GPG key pair and password from the environment and writes them to corresponding files.
+//
+// The passed config's GPG fields also get updated. Make sure to call RemoveGPGFiles at application exit.
+func LoadGPGKeys(cfg *config.Config) error {
+	var err error
+	cfg.GPGPrivateKey, err = createTempFile("priv.key")
+	if err != nil {
+		return err
+	}
+	cfg.GPGPublicKey, err = createTempFile("pub.key")
+	if err != nil {
+		return err
+	}
+	cfg.GPGPassPath, err = createTempFile("")
+	if err != nil {
+		return err
+	}
+
+	gpgPrivKey := os.Getenv("GPG_PRIV_KEY")
+	if gpgPrivKey == "" {
+		return fmt.Errorf("$GPG_PRIV_KEY must be defined")
+	}
+	gpgPubKey := os.Getenv("GPG_PUB_KEY")
+	if gpgPubKey == "" {
+		return fmt.Errorf("$GPG_PUB_KEY must be defined")
+	}
+	gpgPass := os.Getenv("GPG_KEY_PASSWORD")
+	if gpgPass == "" {
+		return fmt.Errorf("$GPG_KEY_PASSWORD must be defined")
+	}
+
+	gpgPrivKeyB, err := base64.StdEncoding.DecodeString(gpgPrivKey)
+	if err != nil {
+		return fmt.Errorf("couldn't decode $GPG_PRIV_KEY: %w", err)
+	}
+	gpgPubKeyB, err := base64.StdEncoding.DecodeString(gpgPubKey)
+	if err != nil {
+		return fmt.Errorf("couldn't decode $GPG_PUB_KEY: %w", err)
+	}
+
+	if err := os.WriteFile(cfg.GPGPrivateKey, append(gpgPrivKeyB, '\n'), 0400); err != nil {
+		return fmt.Errorf("failed to write GPG private key file: %w", err)
+	}
+	if err := os.WriteFile(cfg.GPGPublicKey, append(gpgPubKeyB, '\n'), 0400); err != nil {
+		return fmt.Errorf("failed to write GPG public key file: %w", err)
+	}
+	if err := os.WriteFile(cfg.GPGPassPath, []byte(gpgPass+"\n"), 0400); err != nil {
+		return fmt.Errorf("failed to write GPG password file: %w", err)
+	}
+
+	return nil
+}
+
+// RemoveGPGFiles removes configured GPG files.
+func RemoveGPGFiles(cfg config.Config) {
+	for _, fpath := range []string{cfg.GPGPrivateKey, cfg.GPGPublicKey, cfg.GPGPassPath} {
+		if err := os.Remove(fpath); err != nil {
+			log.Printf("failed to remove %q", fpath)
+		}
+	}
+}

pkg/build/gpg/import.go

@@ -0,0 +1,73 @@
+package gpg
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/grafana/grafana/pkg/build/config"
+	"github.com/grafana/grafana/pkg/infra/fs"
+)
+
+// writeRpmMacros writes ~/.rpmmacros.
+func writeRpmMacros(homeDir, gpgPassPath string) error {
+	fpath := filepath.Join(homeDir, ".rpmmacros")
+	content := fmt.Sprintf(`%%_signature gpg
+%%_gpg_path %s/.gnupg
+%%_gpg_name Grafana
+%%_gpgbin /usr/bin/gpg
+%%__gpg_sign_cmd %%{__gpg} gpg --batch --yes --pinentry-mode loopback --no-armor --passphrase-file %s --no-secmem-warning -u "%%{_gpg_name}" -sbo %%{__signature_filename} %%{__plaintext_filename}
+`, homeDir, gpgPassPath)
+	//nolint:gosec
+	if err := os.WriteFile(fpath, []byte(content), 0600); err != nil {
+		return fmt.Errorf("failed to write %q: %w", fpath, err)
+	}
+
+	return nil
+}
+
+// Import imports the GPG package signing key.
+// ~/.rpmmacros also gets written.
+func Import(cfg config.Config) error {
+	exists, err := fs.Exists(cfg.GPGPrivateKey)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		return fmt.Errorf("GPG private key file doesn't exist: %q", cfg.GPGPrivateKey)
+	}
+
+	log.Printf("Importing GPG key %q...", cfg.GPGPrivateKey)
+	// nolint:gosec
+	cmd := exec.Command("gpg", "--batch", "--yes", "--no-tty", "--allow-secret-key-import", "--import",
+		cfg.GPGPrivateKey)
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to import private key: %s", output)
+	}
+
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return err
+	}
+
+	if err := writeRpmMacros(homeDir, cfg.GPGPassPath); err != nil {
+		return err
+	}
+
+	pubKeysPath := filepath.Join(homeDir, ".rpmdb", "pubkeys")
+	if err := os.MkdirAll(pubKeysPath, 0700); err != nil {
+		return fmt.Errorf("failed to make %s: %w", pubKeysPath, err)
+	}
+	gpgPub, err := os.ReadFile(cfg.GPGPublicKey)
+	if err != nil {
+		return err
+	}
+	//nolint:gosec
+	if err := os.WriteFile(filepath.Join(homeDir, ".rpmdb", "pubkeys", "grafana.key"), gpgPub, 0400); err != nil {
+		return fmt.Errorf("failed to write pub key to ~/.rpmdb: %w", err)
+	}
+
+	return nil
+}

pkg/build/packaging/deb.go

@@ -0,0 +1 @@
+package packaging

pkg/build/packaging/docs.go

@@ -0,0 +1,2 @@
+// Package packaging holds functions and types for creating the tar.gz, deb, and rpm packages of Grafana.
+package packaging

pkg/build/packaging/errors.go

@@ -0,0 +1 @@
+package packaging

pkg/build/packaging/rpm.go

@@ -0,0 +1 @@
+package packaging

pkg/build/validation/docs.go

@@ -0,0 +1,2 @@
+// Package validation provides functions and types for validating Grafana releases in both the pre-release and post-release phases.
+package validation

pkg/build/validation/validation.go

@@ -0,0 +1,27 @@
+package validation
+
+import (
+	"context"
+)
+
+type ArtifactType int
+
+const (
+	ArtifactTypeDockerHub ArtifactType = iota
+	ArtifactTypeGCSObject
+)
+
+type Artifact struct {
+	Type ArtifactType
+	URL  string
+}
+
+// ReleaseArtifacts generates a list of release artifacts
+func ReleaseArtifacts(version string) ([]Artifact, error) {
+	return nil, nil
+}
+
+// VerifyRelease tests that a that, given the information, a release will completed wholly and successfully.
+func VerifyRelease(ctx context.Context, version string) (bool, error) {
+	return false, nil
+}

scripts/drone/steps/lib.star

@@ -665,7 +665,7 @@ def package_step(edition, ver_mode, include_enterprise2=False, variants=None):
     # TODO: Use percentage for jobs
     if ver_mode == 'release':
         cmds = [
-            '{}./bin/grabpl package --jobs 8 --edition {} '.format(test_args, edition) + \
+            '{}./bin/build package --jobs 8 --edition {} '.format(test_args, edition) + \
             '{} ${{DRONE_TAG}}'.format(
                 sign_args
             ),
@@ -673,7 +673,7 @@ def package_step(edition, ver_mode, include_enterprise2=False, variants=None):
     else:
         build_no = '${DRONE_BUILD_NUMBER}'
         cmds = [
-            '{}./bin/grabpl package --jobs 8 --edition {} '.format(test_args, edition) + \
+            '{}./bin/build package --jobs 8 --edition {} '.format(test_args, edition) + \
             '--build-id {}{}{}'.format(build_no, variants_str, sign_args),
         ]