DEB/RPM Packages: Remove old package publishing steps (#58071)

* Remove old publish package steps

* Don't ignore publish packaging failures

.drone.yml

@@ -1730,78 +1730,8 @@ type: docker
 clone:
   retries: 3
 depends_on:
-- main-test-frontend
-- main-test-backend
 - main-build-e2e-publish
 - main-integration-tests
-- main-windows
-image_pull_secrets:
-- dockerconfigjson
-kind: pipeline
-name: main-publish
-node:
-  type: no-parallel
-platform:
-  arch: amd64
-  os: linux
-services: []
-steps:
-- commands:
-  - mkdir -p bin
-  - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl
-  - chmod +x bin/grabpl
-  image: byrnedo/alpine-curl:0.1.8
-  name: grabpl
-- commands:
-  - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
-  depends_on: []
-  environment:
-    CGO_ENABLED: 0
-  image: golang:1.19.2
-  name: compile-build-cmd
-- commands:
-  - echo $DRONE_RUNNER_NAME
-  image: alpine:3.15.6
-  name: identify-runner
-- commands:
-  - ./bin/build publish packages --edition oss --gcp-key /tmp/gcpkey.json --build-id
-    ${DRONE_BUILD_NUMBER}
-  depends_on:
-  - compile-build-cmd
-  environment:
-    GCP_KEY:
-      from_secret: gcp_key
-    GPG_KEY_PASSWORD:
-      from_secret: gpg_key_password
-    GPG_PRIV_KEY:
-      from_secret: gpg_priv_key
-    GPG_PUB_KEY:
-      from_secret: gpg_pub_key
-    GRAFANA_COM_API_KEY:
-      from_secret: grafana_api_key
-  image: grafana/grafana-ci-deploy:1.3.3
-  name: publish-packages-oss
-trigger:
-  branch: main
-  event:
-  - push
-  paths:
-    exclude:
-    - '*.md'
-    - docs/**
-    - latest.json
-  repo:
-  - grafana/grafana
-type: docker
-volumes:
-- host:
-    path: /var/run/docker.sock
-  name: docker
----
-clone:
-  retries: 3
-depends_on:
-- main-publish
 image_pull_secrets:
 - dockerconfigjson
 kind: pipeline
@@ -1847,7 +1777,6 @@ depends_on:
 - main-build-e2e-publish
 - main-integration-tests
 - main-windows
-- main-publish
 kind: pipeline
 name: main-notify
 platform:
@@ -3659,37 +3588,8 @@ steps:
     CGO_ENABLED: 0
   image: golang:1.19.2
   name: compile-build-cmd
-- commands:
-  - ./bin/build publish packages --edition oss --gcp-key /tmp/gcpkey.json ${DRONE_TAG}
-  depends_on:
-  - compile-build-cmd
-  environment:
-    GCP_KEY:
-      from_secret: gcp_key
-    GPG_KEY_PASSWORD:
-      from_secret: gpg_key_password
-    GPG_PRIV_KEY:
-      from_secret: gpg_priv_key
-    GPG_PUB_KEY:
-      from_secret: gpg_pub_key
-    GRAFANA_COM_API_KEY:
-      from_secret: grafana_api_key
-  image: grafana/grafana-ci-deploy:1.3.3
-  name: publish-packages-oss
-- commands:
-  - ./bin/build publish grafana-com --edition oss ${DRONE_TAG}
-  depends_on:
-  - publish-packages-oss
-  environment:
-    GCP_KEY:
-      from_secret: gcp_key
-    GRAFANA_COM_API_KEY:
-      from_secret: grafana_api_key
-  image: grafana/grafana-ci-deploy:1.3.3
-  name: publish-grafanacom-oss
 - depends_on:
   - grabpl
-  failure: ignore
   image: us.gcr.io/kubernetes-dev/package-publish:latest
   name: publish-linux-packages-deb
   privileged: true
@@ -3711,7 +3611,6 @@ steps:
     target_bucket: grafana-packages
 - depends_on:
   - grabpl
-  failure: ignore
   image: us.gcr.io/kubernetes-dev/package-publish:latest
   name: publish-linux-packages-rpm
   privileged: true
@@ -3731,6 +3630,18 @@ steps:
     service_account_json:
       from_secret: packages_service_account
     target_bucket: grafana-packages
+- commands:
+  - ./bin/build publish grafana-com --edition oss ${DRONE_TAG}
+  depends_on:
+  - publish-linux-packages-deb
+  - publish-linux-packages-rpm
+  environment:
+    GCP_KEY:
+      from_secret: gcp_key
+    GRAFANA_COM_API_KEY:
+      from_secret: grafana_api_key
+  image: grafana/grafana-ci-deploy:1.3.3
+  name: publish-grafanacom-oss
 trigger:
   event:
   - promote
@@ -3772,37 +3683,8 @@ steps:
     CGO_ENABLED: 0
   image: golang:1.19.2
   name: compile-build-cmd
-- commands:
-  - ./bin/build publish packages --edition enterprise --gcp-key /tmp/gcpkey.json ${DRONE_TAG}
-  depends_on:
-  - compile-build-cmd
-  environment:
-    GCP_KEY:
-      from_secret: gcp_key
-    GPG_KEY_PASSWORD:
-      from_secret: gpg_key_password
-    GPG_PRIV_KEY:
-      from_secret: gpg_priv_key
-    GPG_PUB_KEY:
-      from_secret: gpg_pub_key
-    GRAFANA_COM_API_KEY:
-      from_secret: grafana_api_key
-  image: grafana/grafana-ci-deploy:1.3.3
-  name: publish-packages-enterprise
-- commands:
-  - ./bin/build publish grafana-com --edition enterprise ${DRONE_TAG}
-  depends_on:
-  - publish-packages-enterprise
-  environment:
-    GCP_KEY:
-      from_secret: gcp_key
-    GRAFANA_COM_API_KEY:
-      from_secret: grafana_api_key
-  image: grafana/grafana-ci-deploy:1.3.3
-  name: publish-grafanacom-enterprise
 - depends_on:
   - grabpl
-  failure: ignore
   image: us.gcr.io/kubernetes-dev/package-publish:latest
   name: publish-linux-packages-deb
   privileged: true
@@ -3824,7 +3706,6 @@ steps:
     target_bucket: grafana-packages
 - depends_on:
   - grabpl
-  failure: ignore
   image: us.gcr.io/kubernetes-dev/package-publish:latest
   name: publish-linux-packages-rpm
   privileged: true
@@ -3844,6 +3725,18 @@ steps:
     service_account_json:
       from_secret: packages_service_account
     target_bucket: grafana-packages
+- commands:
+  - ./bin/build publish grafana-com --edition enterprise ${DRONE_TAG}
+  depends_on:
+  - publish-linux-packages-deb
+  - publish-linux-packages-rpm
+  environment:
+    GCP_KEY:
+      from_secret: gcp_key
+    GRAFANA_COM_API_KEY:
+      from_secret: grafana_api_key
+  image: grafana/grafana-ci-deploy:1.3.3
+  name: publish-grafanacom-enterprise
 trigger:
   event:
   - promote
@@ -5423,6 +5316,6 @@ kind: secret
 name: packages_secret_access_key
 ---
 kind: signature
-hmac: bbc179e84a651150c80ff37442ae77605613dfdda47aa72604016b090715a12b
+hmac: b7a812befcaa64d70ce856f290029308d9c73f3b479bd24387e698081275e084
 
 ...

pkg/build/cmd/main.go

@@ -5,9 +5,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/grafana/grafana/pkg/build/config"
 	"github.com/grafana/grafana/pkg/build/docker"
-	"github.com/grafana/grafana/pkg/build/packaging"
 	"github.com/urfave/cli/v2"
 )
 
@@ -175,48 +173,6 @@ func main() {
 			Name:  "publish",
 			Usage: "Publish packages to Grafana com and repositories",
 			Subcommands: cli.Commands{
-				{
-					Name:      "packages",
-					Usage:     "publish Grafana packages",
-					ArgsUsage: "[version]",
-					Action:    PublishPackages,
-					Flags: []cli.Flag{
-						&jobsFlag,
-						&editionFlag,
-						&buildIDFlag,
-						&dryRunFlag,
-						&gcpKeyFlag,
-						&cli.StringFlag{
-							Name:  "packages-bucket",
-							Value: config.PublicBucket,
-							Usage: "Google Cloud Storage Debian database bucket",
-						},
-						&cli.StringFlag{
-							Name:  "deb-db-bucket",
-							Value: packaging.DefaultDebDBBucket,
-							Usage: "Google Cloud Storage Debian database bucket",
-						},
-						&cli.StringFlag{
-							Name:  "deb-repo-bucket",
-							Value: packaging.DefaultDebRepoBucket,
-							Usage: "Google Cloud Storage Debian repo bucket",
-						},
-						&cli.StringFlag{
-							Name:  "rpm-repo-bucket",
-							Value: packaging.DefaultRPMRepoBucket,
-							Usage: "Google Cloud Storage RPM repo bucket",
-						},
-						&cli.StringFlag{
-							Name:  "ttl",
-							Value: packaging.DefaultTTLSeconds,
-							Usage: "Cache time to live for uploaded packages",
-						},
-						&cli.BoolFlag{
-							Name:  "simulate-release",
-							Usage: "Only simulate creating release at grafana.com",
-						},
-					},
-				},
 				{
 					Name:   "grafana-com",
 					Usage:  "Publish packages to grafana.com",

pkg/build/cmd/publishpackages.go

@@ -1,117 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"strings"
-
-	"github.com/grafana/grafana/pkg/build/config"
-	"github.com/grafana/grafana/pkg/build/fsutil"
-	"github.com/grafana/grafana/pkg/build/gcloud"
-	"github.com/grafana/grafana/pkg/build/gpg"
-	"github.com/grafana/grafana/pkg/build/packaging"
-	"github.com/urfave/cli/v2"
-)
-
-// PublishPackages implements the sub-command "publish-packages".
-func PublishPackages(c *cli.Context) error {
-	if err := gcloud.ActivateServiceAccount(); err != nil {
-		return fmt.Errorf("couldn't activate service account, err: %w", err)
-	}
-
-	metadata, err := GenerateMetadata(c)
-	if err != nil {
-		return err
-	}
-
-	releaseMode, err := metadata.GetReleaseMode()
-	if err != nil {
-		return err
-	}
-
-	dryRun := c.Bool("dry-run")
-	simulateRelease := c.Bool("simulate-release")
-	// Test release mode and dryRun imply simulateRelease
-	if releaseMode.IsTest || dryRun {
-		simulateRelease = true
-	}
-
-	grafanaAPIKey := strings.TrimSpace(os.Getenv("GRAFANA_COM_API_KEY"))
-	if grafanaAPIKey == "" {
-		return cli.NewExitError("the environment variable GRAFANA_COM_API_KEY must be set", 1)
-	}
-
-	edition := config.Edition(c.String("edition"))
-
-	// TODO: Verify config values
-	cfg := packaging.PublishConfig{
-		Config: config.Config{
-			Version:       metadata.GrafanaVersion,
-			Bucket:        c.String("packages-bucket"),
-			DebDBBucket:   c.String("deb-db-bucket"),
-			DebRepoBucket: c.String("deb-repo-bucket"),
-			RPMRepoBucket: c.String("rpm-repo-bucket"),
-		},
-		Edition:         edition,
-		ReleaseMode:     releaseMode,
-		GrafanaAPIKey:   grafanaAPIKey,
-		DryRun:          dryRun,
-		TTL:             c.String("ttl"),
-		SimulateRelease: simulateRelease,
-	}
-	if err := gpg.LoadGPGKeys(&cfg.Config); err != nil {
-		return err
-	}
-	defer gpg.RemoveGPGFiles(cfg.Config)
-
-	// Only update package manager repos for releases.
-	// In test release mode, the operator should configure different GCS buckets for the package repos,
-	// so should be safe.
-	if cfg.ReleaseMode.Mode == config.TagMode {
-		workDir, err := fsutil.CreateTempDir("")
-		if err != nil {
-			return err
-		}
-		defer func() {
-			if err := os.RemoveAll(workDir); err != nil {
-				log.Printf("Failed to remove temporary directory %q: %s\n", workDir, err.Error())
-			}
-		}()
-		if err := updatePkgRepos(cfg, workDir); err != nil {
-			return err
-		}
-	}
-
-	log.Println("Successfully published packages!")
-	return nil
-}
-
-// updatePkgRepos updates package manager repositories.
-func updatePkgRepos(cfg packaging.PublishConfig, workDir string) error {
-	if err := gpg.Import(cfg.Config); err != nil {
-		return err
-	}
-
-	// If updating the Deb repo fails, still continue with the RPM repo, so we don't have to retry
-	// both by hand
-	debErr := packaging.UpdateDebRepo(cfg, workDir)
-	if debErr != nil {
-		log.Printf("Updating Deb repo failed: %s\n", debErr)
-	}
-	rpmErr := packaging.UpdateRPMRepo(cfg, workDir)
-	if rpmErr != nil {
-		log.Printf("Updating RPM repo failed: %s\n", rpmErr)
-	}
-
-	if debErr != nil {
-		return debErr
-	}
-	if rpmErr != nil {
-		return rpmErr
-	}
-
-	log.Println("Updated Deb and RPM repos successfully!")
-
-	return nil
-}

scripts/drone/events/main.star

@@ -37,11 +37,6 @@ load(
     'windows',
 )
 
-load(
-    'scripts/drone/pipelines/publish.star',
-    'publish',
-)
-
 load(
     'scripts/drone/pipelines/trigger_downstream.star',
     'enterprise_downstream_pipeline',
@@ -103,11 +98,10 @@ def main_pipelines(edition):
         name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger,
         template=drone_change_template, secret='drone-changes-webhook',
     ),
-    publish(trigger, ver_mode, edition),
     enterprise_downstream_pipeline(edition, ver_mode),
     notify_pipeline(
         name='main-notify', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']),
-        depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', 'main-publish'],
+        depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows'],
         template=failure_template, secret='slack_webhook'
     )]
 

scripts/drone/events/release.star

@@ -34,7 +34,6 @@ load(
     'benchmark_ldap_step',
     'store_storybook_step',
     'upload_packages_step',
-    'publish_packages_step',
     'publish_grafanacom_step',
     'upload_cdn_step',
     'verify_gen_cue_step',
@@ -387,19 +386,17 @@ def publish_packages_pipeline():
     oss_steps = [
         download_grabpl_step(),
         compile_build_cmd(),
-        publish_packages_step(edition='oss', ver_mode='release'),
-        publish_grafanacom_step(edition='oss', ver_mode='release'),
         publish_linux_packages_step(edition='oss', package_manager='deb'),
         publish_linux_packages_step(edition='oss', package_manager='rpm'),
+        publish_grafanacom_step(edition='oss', ver_mode='release'),
     ]
 
     enterprise_steps = [
         download_grabpl_step(),
         compile_build_cmd(),
-        publish_packages_step(edition='enterprise', ver_mode='release'),
-        publish_grafanacom_step(edition='enterprise', ver_mode='release'),
         publish_linux_packages_step(edition='enterprise', package_manager='deb'),
         publish_linux_packages_step(edition='enterprise', package_manager='rpm'),
+        publish_grafanacom_step(edition='enterprise', ver_mode='release'),
     ]
     deps = [
         'publish-artifacts-public',

scripts/drone/pipelines/publish.star

@@ -1,25 +0,0 @@
-load(
-    'scripts/drone/steps/lib.star',
-    'identify_runner_step',
-    'download_grabpl_step',
-    'publish_packages_step',
-    'compile_build_cmd',
-)
-
-load(
-    'scripts/drone/utils/utils.star',
-    'pipeline',
-)
-
-def publish(trigger, ver_mode, edition):
-    steps = [
-        download_grabpl_step(),
-        compile_build_cmd(),
-        identify_runner_step(),
-        publish_packages_step(edition=edition, ver_mode=ver_mode),
-    ]
-    return pipeline(
-                   name='main-publish', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
-                   steps=steps,
-                   depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', ],
-               )

scripts/drone/pipelines/trigger_downstream.star

@@ -22,7 +22,7 @@ trigger = {
 
 def enterprise_downstream_pipeline(edition, ver_mode):
     steps = [enterprise_downstream_step(edition, ver_mode)]
-    deps = ['main-publish']
+    deps = ['main-build-e2e-publish', 'main-integration-tests']
     return pipeline(
                 name='main-trigger-downstream', edition=edition, trigger=trigger, services=[], steps=steps, depends_on=deps
         )

scripts/drone/steps/lib.star

@@ -974,37 +974,6 @@ def upload_packages_step(edition, ver_mode, trigger=None):
     return step
 
 
-def publish_packages_step(edition, ver_mode):
-    if ver_mode == 'release':
-        cmd = './bin/build publish packages --edition {} --gcp-key /tmp/gcpkey.json ${{DRONE_TAG}}'.format(
-            edition,
-        )
-    elif ver_mode == 'main':
-        build_no = '${DRONE_BUILD_NUMBER}'
-        cmd = './bin/build publish packages --edition {} --gcp-key /tmp/gcpkey.json --build-id {}'.format(
-            edition, build_no,
-        )
-    else:
-        fail('Unexpected version mode {}'.format(ver_mode))
-
-    return {
-        'name': 'publish-packages-{}'.format(edition),
-        'image': publish_image,
-        'depends_on': [
-            'compile-build-cmd',
-        ],
-        'environment': {
-            'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'),
-            'GCP_KEY': from_secret('gcp_key'),
-            'GPG_PRIV_KEY': from_secret('gpg_priv_key'),
-            'GPG_PUB_KEY': from_secret('gpg_pub_key'),
-            'GPG_KEY_PASSWORD': from_secret('gpg_key_password'),
-        },
-        'commands': [
-            cmd,
-        ],
-    }
-
 def publish_grafanacom_step(edition, ver_mode):
     if ver_mode == 'release':
         cmd = './bin/build publish grafana-com --edition {} ${{DRONE_TAG}}'.format(
@@ -1022,7 +991,8 @@ def publish_grafanacom_step(edition, ver_mode):
         'name': 'publish-grafanacom-{}'.format(edition),
         'image': publish_image,
         'depends_on': [
-            'publish-packages-{}'.format(edition),
+            'publish-linux-packages-deb',
+            'publish-linux-packages-rpm',
         ],
         'environment': {
             'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'),
@@ -1042,7 +1012,6 @@ def publish_linux_packages_step(edition, package_manager='deb'):
             'grabpl'
         ],
         'privileged': True,
-        'failure': 'ignore', # While we're testing it
         'settings': {
             'access_key_id': from_secret('packages_access_key_id'),
             'secret_access_key': from_secret('packages_secret_access_key'),