diff --git a/.drone.yml b/.drone.yml index 6f120401d38..35a0338455a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1578,7 +1578,7 @@ steps: repo: - grafana/grafana - commands: - - ./bin/grabpl upload-packages --edition oss + - ./bin/build upload-packages --edition oss depends_on: - end-to-end-tests-dashboards-suite - end-to-end-tests-panels-suite @@ -2178,7 +2178,7 @@ steps: image: grafana/grafana-ci-deploy:1.3.3 name: upload-cdn-assets - commands: - - ./bin/grabpl upload-packages --edition oss + - ./bin/build upload-packages --edition oss depends_on: - end-to-end-tests-dashboards-suite - end-to-end-tests-panels-suite @@ -2830,7 +2830,7 @@ steps: image: grafana/grafana-ci-deploy:1.3.3 name: upload-cdn-assets - commands: - - ./bin/grabpl upload-packages --edition enterprise + - ./bin/build upload-packages --edition enterprise depends_on: - package environment: @@ -2882,7 +2882,7 @@ steps: image: grafana/build-container:1.6.4 name: store-npm-packages - commands: - - ./bin/grabpl upload-packages --edition enterprise2 + - ./bin/build upload-packages --edition enterprise2 depends_on: - package-enterprise2 environment: @@ -4224,7 +4224,7 @@ steps: repo: - grafana/grafana - commands: - - ./bin/grabpl upload-packages --edition oss + - ./bin/build upload-packages --edition oss depends_on: - end-to-end-tests-dashboards-suite - end-to-end-tests-panels-suite @@ -4834,7 +4834,7 @@ steps: repo: - grafana/grafana - commands: - - ./bin/grabpl upload-packages --edition enterprise + - ./bin/build upload-packages --edition enterprise depends_on: - package environment: @@ -4879,7 +4879,7 @@ steps: image: grafana/grafana-ci-deploy:1.3.3 name: upload-cdn-assets-enterprise2 - commands: - - ./bin/grabpl upload-packages --edition enterprise2 + - ./bin/build upload-packages --edition enterprise2 depends_on: - package-enterprise2 environment: @@ -5570,6 +5570,6 @@ kind: secret name: packages_secret_access_key --- kind: signature -hmac: 49c89d01105450cc0ae03afc0dc5aec89d61a758cb1e3a4da9fec1434a49da5c +hmac: 2f3c13716c7cd1e42004a0f0d7057d5024d38bceedb22370b3554a79e786a46c ... diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index ba4f1f6eb0a..c523b2a755c 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -161,6 +161,15 @@ func main() { }, }, }, + { + Name: "upload-packages", + Usage: "Upload Grafana packages", + Action: UploadPackages, + Flags: []cli.Flag{ + &jobsFlag, + &editionFlag, + }, + }, { Name: "artifacts", Usage: "Handle Grafana artifacts", diff --git a/pkg/build/cmd/uploadpackages.go b/pkg/build/cmd/uploadpackages.go new file mode 100644 index 00000000000..01a43a43bdf --- /dev/null +++ b/pkg/build/cmd/uploadpackages.go @@ -0,0 +1,172 @@ +package main + +import ( + "encoding/base64" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/gcloud" + "github.com/grafana/grafana/pkg/build/packaging" + "github.com/urfave/cli/v2" +) + +const releaseFolder = "release" +const mainFolder = "main" +const releaseBranchFolder = "prerelease" + +type uploadConfig struct { + config.Config + + edition config.Edition + versionMode config.VersionMode + gcpKey string + distDir string +} + +// UploadPackages implements the sub-command "upload-packages". +func UploadPackages(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.NewExitError(err.Error(), 1) + } + return cli.NewExitError("", 1) + } + + gcpKeyB64 := strings.TrimSpace(os.Getenv("GCP_KEY")) + if gcpKeyB64 == "" { + return cli.NewExitError("the environment variable GCP_KEY must be set", 1) + } + gcpKeyB, err := base64.StdEncoding.DecodeString(gcpKeyB64) + if err != nil { + return cli.NewExitError("failed to base64 decode $GCP_KEY", 1) + } + gcpKey := string(gcpKeyB) + + distDir, err := filepath.Abs("dist") + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + + metadata, err := GenerateMetadata(c) + if err != nil { + return err + } + + version := metadata.GrafanaVersion + + releaseMode, err := metadata.GetReleaseMode() + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + + releaseModeConfig, err := config.GetBuildConfig(releaseMode.Mode) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + + edition, ok := os.LookupEnv("EDITION") + if !ok { + return fmt.Errorf("EDITION envvar is missing, exitting") + } + + // TODO: Verify config values + cfg := uploadConfig{ + Config: config.Config{ + Version: version, + Bucket: releaseModeConfig.Buckets.Artifacts, + }, + edition: config.Edition(edition), + versionMode: releaseMode.Mode, + gcpKey: gcpKey, + distDir: distDir, + } + + if cfg.edition == config.EditionEnterprise2 { + if releaseModeConfig.Buckets.ArtifactsEnterprise2 != "" { + cfg.Config.Bucket = releaseModeConfig.Buckets.ArtifactsEnterprise2 + } else { + return fmt.Errorf("enterprise2 bucket var doesn't exist") + } + } + + if err := uploadPackages(cfg); err != nil { + return cli.NewExitError(err.Error(), 1) + } + + log.Println("Successfully uploaded packages!") + return nil +} + +func uploadPackages(cfg uploadConfig) error { + log.Printf("Uploading Grafana packages, version %s, %s edition, %s mode...\n", cfg.Version, cfg.edition, + cfg.versionMode) + + if err := gcloud.ActivateServiceAccount(); err != nil { + return fmt.Errorf("couldn't activate service account, err: %w", err) + } + + edition := strings.ToLower(string(cfg.edition)) + + var sfx string + switch cfg.edition { + case config.EditionOSS: + case config.EditionEnterprise: + sfx = "-enterprise" + case config.EditionEnterprise2: + sfx = "-enterprise2" + default: + panic(fmt.Sprintf("unrecognized edition %q", cfg.edition)) + } + matches, err := filepath.Glob(filepath.Join(cfg.distDir, fmt.Sprintf("grafana%s*", sfx))) + if err != nil { + return fmt.Errorf("failed to list packages: %w", err) + } + fpaths := []string{} + rePkg := packaging.PackageRegexp(cfg.edition) + for _, fpath := range matches { + fname := filepath.Base(fpath) + if strings.Contains(fname, "latest") || !rePkg.MatchString(fname) { + log.Printf("Ignoring file %q\n", fpath) + continue + } + + fpaths = append(fpaths, fpath) + } + + var versionFolder string + switch cfg.versionMode { + case config.TagMode: + versionFolder = releaseFolder + case config.MainMode, config.CustomMode: + versionFolder = mainFolder + case config.ReleaseBranchMode: + versionFolder = releaseBranchFolder + default: + panic(fmt.Sprintf("Unrecognized version mode: %s", cfg.versionMode)) + } + + var tag, gcsPath string + droneTag := strings.TrimSpace(os.Getenv("DRONE_TAG")) + if droneTag != "" { + tag = droneTag + gcsPath = fmt.Sprintf("gs://%s/%s/%s/%s", cfg.Bucket, tag, edition, versionFolder) + } else { + gcsPath = fmt.Sprintf("gs://%s/%s/%s/", cfg.Bucket, edition, versionFolder) + } + log.Printf("Uploading %d file(s) to GCS (%s)...\n", len(fpaths), gcsPath) + + args := []string{"-m", "cp"} + args = append(args, fpaths...) + args = append(args, gcsPath) + cmd := exec.Command("gsutil", args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to upload files to GCS: %s", output) + } + + return nil +} diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index bf45fc1a63d..3ee32a19cb3 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -995,7 +995,7 @@ def upload_packages_step(edition, ver_mode, trigger=None): 'GCP_KEY': from_secret('gcp_key'), 'PRERELEASE_BUCKET': from_secret('prerelease_bucket'), }, - 'commands': ['./bin/grabpl upload-packages --edition {}'.format(edition),], + 'commands': ['./bin/build upload-packages --edition {}'.format(edition),], } if trigger and ver_mode in ("release-branch", "main"): step = dict(step, when=trigger)