CI: Move upload-packages subcmd from grabpl (#58960)

* Move upload-packages subcmd from grabpl

# Conflicts:
#	.drone.yml

# Conflicts:
#	.drone.yml

* Fix lint - add ActivateServiceAccount

* Fix lint

.drone.yml

@@ -1578,7 +1578,7 @@ steps:
     repo:
     - grafana/grafana
 - commands:
-  - ./bin/grabpl upload-packages --edition oss
+  - ./bin/build upload-packages --edition oss
   depends_on:
   - end-to-end-tests-dashboards-suite
   - end-to-end-tests-panels-suite
@@ -2178,7 +2178,7 @@ steps:
   image: grafana/grafana-ci-deploy:1.3.3
   name: upload-cdn-assets
 - commands:
-  - ./bin/grabpl upload-packages --edition oss
+  - ./bin/build upload-packages --edition oss
   depends_on:
   - end-to-end-tests-dashboards-suite
   - end-to-end-tests-panels-suite
@@ -2830,7 +2830,7 @@ steps:
   image: grafana/grafana-ci-deploy:1.3.3
   name: upload-cdn-assets
 - commands:
-  - ./bin/grabpl upload-packages --edition enterprise
+  - ./bin/build upload-packages --edition enterprise
   depends_on:
   - package
   environment:
@@ -2882,7 +2882,7 @@ steps:
   image: grafana/build-container:1.6.4
   name: store-npm-packages
 - commands:
-  - ./bin/grabpl upload-packages --edition enterprise2
+  - ./bin/build upload-packages --edition enterprise2
   depends_on:
   - package-enterprise2
   environment:
@@ -4224,7 +4224,7 @@ steps:
     repo:
     - grafana/grafana
 - commands:
-  - ./bin/grabpl upload-packages --edition oss
+  - ./bin/build upload-packages --edition oss
   depends_on:
   - end-to-end-tests-dashboards-suite
   - end-to-end-tests-panels-suite
@@ -4834,7 +4834,7 @@ steps:
     repo:
     - grafana/grafana
 - commands:
-  - ./bin/grabpl upload-packages --edition enterprise
+  - ./bin/build upload-packages --edition enterprise
   depends_on:
   - package
   environment:
@@ -4879,7 +4879,7 @@ steps:
   image: grafana/grafana-ci-deploy:1.3.3
   name: upload-cdn-assets-enterprise2
 - commands:
-  - ./bin/grabpl upload-packages --edition enterprise2
+  - ./bin/build upload-packages --edition enterprise2
   depends_on:
   - package-enterprise2
   environment:
@@ -5570,6 +5570,6 @@ kind: secret
 name: packages_secret_access_key
 ---
 kind: signature
-hmac: 49c89d01105450cc0ae03afc0dc5aec89d61a758cb1e3a4da9fec1434a49da5c
+hmac: 2f3c13716c7cd1e42004a0f0d7057d5024d38bceedb22370b3554a79e786a46c
 
 ...

pkg/build/cmd/main.go

@@ -161,6 +161,15 @@ func main() {
 				},
 			},
 		},
+		{
+			Name:   "upload-packages",
+			Usage:  "Upload Grafana packages",
+			Action: UploadPackages,
+			Flags: []cli.Flag{
+				&jobsFlag,
+				&editionFlag,
+			},
+		},
 		{
 			Name:  "artifacts",
 			Usage: "Handle Grafana artifacts",

pkg/build/cmd/uploadpackages.go

@@ -0,0 +1,172 @@
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/build/config"
+	"github.com/grafana/grafana/pkg/build/gcloud"
+	"github.com/grafana/grafana/pkg/build/packaging"
+	"github.com/urfave/cli/v2"
+)
+
+const releaseFolder = "release"
+const mainFolder = "main"
+const releaseBranchFolder = "prerelease"
+
+type uploadConfig struct {
+	config.Config
+
+	edition     config.Edition
+	versionMode config.VersionMode
+	gcpKey      string
+	distDir     string
+}
+
+// UploadPackages implements the sub-command "upload-packages".
+func UploadPackages(c *cli.Context) error {
+	if c.NArg() > 0 {
+		if err := cli.ShowSubcommandHelp(c); err != nil {
+			return cli.NewExitError(err.Error(), 1)
+		}
+		return cli.NewExitError("", 1)
+	}
+
+	gcpKeyB64 := strings.TrimSpace(os.Getenv("GCP_KEY"))
+	if gcpKeyB64 == "" {
+		return cli.NewExitError("the environment variable GCP_KEY must be set", 1)
+	}
+	gcpKeyB, err := base64.StdEncoding.DecodeString(gcpKeyB64)
+	if err != nil {
+		return cli.NewExitError("failed to base64 decode $GCP_KEY", 1)
+	}
+	gcpKey := string(gcpKeyB)
+
+	distDir, err := filepath.Abs("dist")
+	if err != nil {
+		return cli.NewExitError(err.Error(), 1)
+	}
+
+	metadata, err := GenerateMetadata(c)
+	if err != nil {
+		return err
+	}
+
+	version := metadata.GrafanaVersion
+
+	releaseMode, err := metadata.GetReleaseMode()
+	if err != nil {
+		return cli.NewExitError(err.Error(), 1)
+	}
+
+	releaseModeConfig, err := config.GetBuildConfig(releaseMode.Mode)
+	if err != nil {
+		return cli.NewExitError(err.Error(), 1)
+	}
+
+	edition, ok := os.LookupEnv("EDITION")
+	if !ok {
+		return fmt.Errorf("EDITION envvar is missing, exitting")
+	}
+
+	// TODO: Verify config values
+	cfg := uploadConfig{
+		Config: config.Config{
+			Version: version,
+			Bucket:  releaseModeConfig.Buckets.Artifacts,
+		},
+		edition:     config.Edition(edition),
+		versionMode: releaseMode.Mode,
+		gcpKey:      gcpKey,
+		distDir:     distDir,
+	}
+
+	if cfg.edition == config.EditionEnterprise2 {
+		if releaseModeConfig.Buckets.ArtifactsEnterprise2 != "" {
+			cfg.Config.Bucket = releaseModeConfig.Buckets.ArtifactsEnterprise2
+		} else {
+			return fmt.Errorf("enterprise2 bucket var doesn't exist")
+		}
+	}
+
+	if err := uploadPackages(cfg); err != nil {
+		return cli.NewExitError(err.Error(), 1)
+	}
+
+	log.Println("Successfully uploaded packages!")
+	return nil
+}
+
+func uploadPackages(cfg uploadConfig) error {
+	log.Printf("Uploading Grafana packages, version %s, %s edition, %s mode...\n", cfg.Version, cfg.edition,
+		cfg.versionMode)
+
+	if err := gcloud.ActivateServiceAccount(); err != nil {
+		return fmt.Errorf("couldn't activate service account, err: %w", err)
+	}
+
+	edition := strings.ToLower(string(cfg.edition))
+
+	var sfx string
+	switch cfg.edition {
+	case config.EditionOSS:
+	case config.EditionEnterprise:
+		sfx = "-enterprise"
+	case config.EditionEnterprise2:
+		sfx = "-enterprise2"
+	default:
+		panic(fmt.Sprintf("unrecognized edition %q", cfg.edition))
+	}
+	matches, err := filepath.Glob(filepath.Join(cfg.distDir, fmt.Sprintf("grafana%s*", sfx)))
+	if err != nil {
+		return fmt.Errorf("failed to list packages: %w", err)
+	}
+	fpaths := []string{}
+	rePkg := packaging.PackageRegexp(cfg.edition)
+	for _, fpath := range matches {
+		fname := filepath.Base(fpath)
+		if strings.Contains(fname, "latest") || !rePkg.MatchString(fname) {
+			log.Printf("Ignoring file %q\n", fpath)
+			continue
+		}
+
+		fpaths = append(fpaths, fpath)
+	}
+
+	var versionFolder string
+	switch cfg.versionMode {
+	case config.TagMode:
+		versionFolder = releaseFolder
+	case config.MainMode, config.CustomMode:
+		versionFolder = mainFolder
+	case config.ReleaseBranchMode:
+		versionFolder = releaseBranchFolder
+	default:
+		panic(fmt.Sprintf("Unrecognized version mode: %s", cfg.versionMode))
+	}
+
+	var tag, gcsPath string
+	droneTag := strings.TrimSpace(os.Getenv("DRONE_TAG"))
+	if droneTag != "" {
+		tag = droneTag
+		gcsPath = fmt.Sprintf("gs://%s/%s/%s/%s", cfg.Bucket, tag, edition, versionFolder)
+	} else {
+		gcsPath = fmt.Sprintf("gs://%s/%s/%s/", cfg.Bucket, edition, versionFolder)
+	}
+	log.Printf("Uploading %d file(s) to GCS (%s)...\n", len(fpaths), gcsPath)
+
+	args := []string{"-m", "cp"}
+	args = append(args, fpaths...)
+	args = append(args, gcsPath)
+	cmd := exec.Command("gsutil", args...)
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to upload files to GCS: %s", output)
+	}
+
+	return nil
+}

scripts/drone/steps/lib.star

@@ -995,7 +995,7 @@ def upload_packages_step(edition, ver_mode, trigger=None):
             'GCP_KEY': from_secret('gcp_key'),
             'PRERELEASE_BUCKET': from_secret('prerelease_bucket'),
         },
-        'commands': ['./bin/grabpl upload-packages --edition {}'.format(edition),],
+        'commands': ['./bin/build upload-packages --edition {}'.format(edition),],
     }
     if trigger and ver_mode in ("release-branch", "main"):
         step = dict(step, when=trigger)