diff --git a/.drone.yml b/.drone.yml index 6a49872fdd5..51a78d67250 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1730,78 +1730,8 @@ type: docker clone: retries: 3 depends_on: -- main-test-frontend -- main-test-backend - main-build-e2e-publish - main-integration-tests -- main-windows -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: main-publish -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.19.2 - name: compile-build-cmd -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 - name: identify-runner -- commands: - - ./bin/build publish packages --edition oss --gcp-key /tmp/gcpkey.json --build-id - ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-packages-oss -trigger: - branch: main - event: - - push - paths: - exclude: - - '*.md' - - docs/** - - latest.json - repo: - - grafana/grafana -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: -- main-publish image_pull_secrets: - dockerconfigjson kind: pipeline @@ -1847,7 +1777,6 @@ depends_on: - main-build-e2e-publish - main-integration-tests - main-windows -- main-publish kind: pipeline name: main-notify platform: @@ -3659,37 +3588,8 @@ steps: CGO_ENABLED: 0 image: golang:1.19.2 name: compile-build-cmd -- commands: - - ./bin/build publish packages --edition oss --gcp-key /tmp/gcpkey.json ${DRONE_TAG} - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-packages-oss -- commands: - - ./bin/build publish grafana-com --edition oss ${DRONE_TAG} - depends_on: - - publish-packages-oss - environment: - GCP_KEY: - from_secret: gcp_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-grafanacom-oss - depends_on: - grabpl - failure: ignore image: us.gcr.io/kubernetes-dev/package-publish:latest name: publish-linux-packages-deb privileged: true @@ -3711,7 +3611,6 @@ steps: target_bucket: grafana-packages - depends_on: - grabpl - failure: ignore image: us.gcr.io/kubernetes-dev/package-publish:latest name: publish-linux-packages-rpm privileged: true @@ -3731,6 +3630,18 @@ steps: service_account_json: from_secret: packages_service_account target_bucket: grafana-packages +- commands: + - ./bin/build publish grafana-com --edition oss ${DRONE_TAG} + depends_on: + - publish-linux-packages-deb + - publish-linux-packages-rpm + environment: + GCP_KEY: + from_secret: gcp_key + GRAFANA_COM_API_KEY: + from_secret: grafana_api_key + image: grafana/grafana-ci-deploy:1.3.3 + name: publish-grafanacom-oss trigger: event: - promote @@ -3772,37 +3683,8 @@ steps: CGO_ENABLED: 0 image: golang:1.19.2 name: compile-build-cmd -- commands: - - ./bin/build publish packages --edition enterprise --gcp-key /tmp/gcpkey.json ${DRONE_TAG} - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-packages-enterprise -- commands: - - ./bin/build publish grafana-com --edition enterprise ${DRONE_TAG} - depends_on: - - publish-packages-enterprise - environment: - GCP_KEY: - from_secret: gcp_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-grafanacom-enterprise - depends_on: - grabpl - failure: ignore image: us.gcr.io/kubernetes-dev/package-publish:latest name: publish-linux-packages-deb privileged: true @@ -3824,7 +3706,6 @@ steps: target_bucket: grafana-packages - depends_on: - grabpl - failure: ignore image: us.gcr.io/kubernetes-dev/package-publish:latest name: publish-linux-packages-rpm privileged: true @@ -3844,6 +3725,18 @@ steps: service_account_json: from_secret: packages_service_account target_bucket: grafana-packages +- commands: + - ./bin/build publish grafana-com --edition enterprise ${DRONE_TAG} + depends_on: + - publish-linux-packages-deb + - publish-linux-packages-rpm + environment: + GCP_KEY: + from_secret: gcp_key + GRAFANA_COM_API_KEY: + from_secret: grafana_api_key + image: grafana/grafana-ci-deploy:1.3.3 + name: publish-grafanacom-enterprise trigger: event: - promote @@ -5423,6 +5316,6 @@ kind: secret name: packages_secret_access_key --- kind: signature -hmac: bbc179e84a651150c80ff37442ae77605613dfdda47aa72604016b090715a12b +hmac: b7a812befcaa64d70ce856f290029308d9c73f3b479bd24387e698081275e084 ... diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index 322e367cd51..aa34b40cb25 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -5,9 +5,7 @@ import ( "os" "strings" - "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/docker" - "github.com/grafana/grafana/pkg/build/packaging" "github.com/urfave/cli/v2" ) @@ -175,48 +173,6 @@ func main() { Name: "publish", Usage: "Publish packages to Grafana com and repositories", Subcommands: cli.Commands{ - { - Name: "packages", - Usage: "publish Grafana packages", - ArgsUsage: "[version]", - Action: PublishPackages, - Flags: []cli.Flag{ - &jobsFlag, - &editionFlag, - &buildIDFlag, - &dryRunFlag, - &gcpKeyFlag, - &cli.StringFlag{ - Name: "packages-bucket", - Value: config.PublicBucket, - Usage: "Google Cloud Storage Debian database bucket", - }, - &cli.StringFlag{ - Name: "deb-db-bucket", - Value: packaging.DefaultDebDBBucket, - Usage: "Google Cloud Storage Debian database bucket", - }, - &cli.StringFlag{ - Name: "deb-repo-bucket", - Value: packaging.DefaultDebRepoBucket, - Usage: "Google Cloud Storage Debian repo bucket", - }, - &cli.StringFlag{ - Name: "rpm-repo-bucket", - Value: packaging.DefaultRPMRepoBucket, - Usage: "Google Cloud Storage RPM repo bucket", - }, - &cli.StringFlag{ - Name: "ttl", - Value: packaging.DefaultTTLSeconds, - Usage: "Cache time to live for uploaded packages", - }, - &cli.BoolFlag{ - Name: "simulate-release", - Usage: "Only simulate creating release at grafana.com", - }, - }, - }, { Name: "grafana-com", Usage: "Publish packages to grafana.com", diff --git a/pkg/build/cmd/publishpackages.go b/pkg/build/cmd/publishpackages.go deleted file mode 100644 index a44e30503aa..00000000000 --- a/pkg/build/cmd/publishpackages.go +++ /dev/null @@ -1,117 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "strings" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/fsutil" - "github.com/grafana/grafana/pkg/build/gcloud" - "github.com/grafana/grafana/pkg/build/gpg" - "github.com/grafana/grafana/pkg/build/packaging" - "github.com/urfave/cli/v2" -) - -// PublishPackages implements the sub-command "publish-packages". -func PublishPackages(c *cli.Context) error { - if err := gcloud.ActivateServiceAccount(); err != nil { - return fmt.Errorf("couldn't activate service account, err: %w", err) - } - - metadata, err := GenerateMetadata(c) - if err != nil { - return err - } - - releaseMode, err := metadata.GetReleaseMode() - if err != nil { - return err - } - - dryRun := c.Bool("dry-run") - simulateRelease := c.Bool("simulate-release") - // Test release mode and dryRun imply simulateRelease - if releaseMode.IsTest || dryRun { - simulateRelease = true - } - - grafanaAPIKey := strings.TrimSpace(os.Getenv("GRAFANA_COM_API_KEY")) - if grafanaAPIKey == "" { - return cli.NewExitError("the environment variable GRAFANA_COM_API_KEY must be set", 1) - } - - edition := config.Edition(c.String("edition")) - - // TODO: Verify config values - cfg := packaging.PublishConfig{ - Config: config.Config{ - Version: metadata.GrafanaVersion, - Bucket: c.String("packages-bucket"), - DebDBBucket: c.String("deb-db-bucket"), - DebRepoBucket: c.String("deb-repo-bucket"), - RPMRepoBucket: c.String("rpm-repo-bucket"), - }, - Edition: edition, - ReleaseMode: releaseMode, - GrafanaAPIKey: grafanaAPIKey, - DryRun: dryRun, - TTL: c.String("ttl"), - SimulateRelease: simulateRelease, - } - if err := gpg.LoadGPGKeys(&cfg.Config); err != nil { - return err - } - defer gpg.RemoveGPGFiles(cfg.Config) - - // Only update package manager repos for releases. - // In test release mode, the operator should configure different GCS buckets for the package repos, - // so should be safe. - if cfg.ReleaseMode.Mode == config.TagMode { - workDir, err := fsutil.CreateTempDir("") - if err != nil { - return err - } - defer func() { - if err := os.RemoveAll(workDir); err != nil { - log.Printf("Failed to remove temporary directory %q: %s\n", workDir, err.Error()) - } - }() - if err := updatePkgRepos(cfg, workDir); err != nil { - return err - } - } - - log.Println("Successfully published packages!") - return nil -} - -// updatePkgRepos updates package manager repositories. -func updatePkgRepos(cfg packaging.PublishConfig, workDir string) error { - if err := gpg.Import(cfg.Config); err != nil { - return err - } - - // If updating the Deb repo fails, still continue with the RPM repo, so we don't have to retry - // both by hand - debErr := packaging.UpdateDebRepo(cfg, workDir) - if debErr != nil { - log.Printf("Updating Deb repo failed: %s\n", debErr) - } - rpmErr := packaging.UpdateRPMRepo(cfg, workDir) - if rpmErr != nil { - log.Printf("Updating RPM repo failed: %s\n", rpmErr) - } - - if debErr != nil { - return debErr - } - if rpmErr != nil { - return rpmErr - } - - log.Println("Updated Deb and RPM repos successfully!") - - return nil -} diff --git a/scripts/drone/events/main.star b/scripts/drone/events/main.star index a718362e958..bfb0374ce6b 100644 --- a/scripts/drone/events/main.star +++ b/scripts/drone/events/main.star @@ -37,11 +37,6 @@ load( 'windows', ) -load( - 'scripts/drone/pipelines/publish.star', - 'publish', -) - load( 'scripts/drone/pipelines/trigger_downstream.star', 'enterprise_downstream_pipeline', @@ -103,11 +98,10 @@ def main_pipelines(edition): name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger, template=drone_change_template, secret='drone-changes-webhook', ), - publish(trigger, ver_mode, edition), enterprise_downstream_pipeline(edition, ver_mode), notify_pipeline( name='main-notify', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']), - depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', 'main-publish'], + depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows'], template=failure_template, secret='slack_webhook' )] diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index 7a6912cf1d3..7e7aa93f298 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -34,7 +34,6 @@ load( 'benchmark_ldap_step', 'store_storybook_step', 'upload_packages_step', - 'publish_packages_step', 'publish_grafanacom_step', 'upload_cdn_step', 'verify_gen_cue_step', @@ -387,19 +386,17 @@ def publish_packages_pipeline(): oss_steps = [ download_grabpl_step(), compile_build_cmd(), - publish_packages_step(edition='oss', ver_mode='release'), - publish_grafanacom_step(edition='oss', ver_mode='release'), publish_linux_packages_step(edition='oss', package_manager='deb'), publish_linux_packages_step(edition='oss', package_manager='rpm'), + publish_grafanacom_step(edition='oss', ver_mode='release'), ] enterprise_steps = [ download_grabpl_step(), compile_build_cmd(), - publish_packages_step(edition='enterprise', ver_mode='release'), - publish_grafanacom_step(edition='enterprise', ver_mode='release'), publish_linux_packages_step(edition='enterprise', package_manager='deb'), publish_linux_packages_step(edition='enterprise', package_manager='rpm'), + publish_grafanacom_step(edition='enterprise', ver_mode='release'), ] deps = [ 'publish-artifacts-public', diff --git a/scripts/drone/pipelines/publish.star b/scripts/drone/pipelines/publish.star deleted file mode 100644 index 9d80d4a55ae..00000000000 --- a/scripts/drone/pipelines/publish.star +++ /dev/null @@ -1,25 +0,0 @@ -load( - 'scripts/drone/steps/lib.star', - 'identify_runner_step', - 'download_grabpl_step', - 'publish_packages_step', - 'compile_build_cmd', -) - -load( - 'scripts/drone/utils/utils.star', - 'pipeline', -) - -def publish(trigger, ver_mode, edition): - steps = [ - download_grabpl_step(), - compile_build_cmd(), - identify_runner_step(), - publish_packages_step(edition=edition, ver_mode=ver_mode), - ] - return pipeline( - name='main-publish', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']), - steps=steps, - depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', ], - ) diff --git a/scripts/drone/pipelines/trigger_downstream.star b/scripts/drone/pipelines/trigger_downstream.star index 2c40f06d82a..36ab3f1ff0d 100644 --- a/scripts/drone/pipelines/trigger_downstream.star +++ b/scripts/drone/pipelines/trigger_downstream.star @@ -22,7 +22,7 @@ trigger = { def enterprise_downstream_pipeline(edition, ver_mode): steps = [enterprise_downstream_step(edition, ver_mode)] - deps = ['main-publish'] + deps = ['main-build-e2e-publish', 'main-integration-tests'] return pipeline( name='main-trigger-downstream', edition=edition, trigger=trigger, services=[], steps=steps, depends_on=deps ) diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index d9a21e4ab00..cbfc7490f73 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -974,37 +974,6 @@ def upload_packages_step(edition, ver_mode, trigger=None): return step -def publish_packages_step(edition, ver_mode): - if ver_mode == 'release': - cmd = './bin/build publish packages --edition {} --gcp-key /tmp/gcpkey.json ${{DRONE_TAG}}'.format( - edition, - ) - elif ver_mode == 'main': - build_no = '${DRONE_BUILD_NUMBER}' - cmd = './bin/build publish packages --edition {} --gcp-key /tmp/gcpkey.json --build-id {}'.format( - edition, build_no, - ) - else: - fail('Unexpected version mode {}'.format(ver_mode)) - - return { - 'name': 'publish-packages-{}'.format(edition), - 'image': publish_image, - 'depends_on': [ - 'compile-build-cmd', - ], - 'environment': { - 'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'), - 'GCP_KEY': from_secret('gcp_key'), - 'GPG_PRIV_KEY': from_secret('gpg_priv_key'), - 'GPG_PUB_KEY': from_secret('gpg_pub_key'), - 'GPG_KEY_PASSWORD': from_secret('gpg_key_password'), - }, - 'commands': [ - cmd, - ], - } - def publish_grafanacom_step(edition, ver_mode): if ver_mode == 'release': cmd = './bin/build publish grafana-com --edition {} ${{DRONE_TAG}}'.format( @@ -1022,7 +991,8 @@ def publish_grafanacom_step(edition, ver_mode): 'name': 'publish-grafanacom-{}'.format(edition), 'image': publish_image, 'depends_on': [ - 'publish-packages-{}'.format(edition), + 'publish-linux-packages-deb', + 'publish-linux-packages-rpm', ], 'environment': { 'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'), @@ -1042,7 +1012,6 @@ def publish_linux_packages_step(edition, package_manager='deb'): 'grabpl' ], 'privileged': True, - 'failure': 'ignore', # While we're testing it 'settings': { 'access_key_id': from_secret('packages_access_key_id'), 'secret_access_key': from_secret('packages_secret_access_key'),