diff --git a/.drone.yml b/.drone.yml index 664e0ed7ed1..fd6042fc3a2 100644 --- a/.drone.yml +++ b/.drone.yml @@ -2767,6 +2767,80 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + retries: 3 +depends_on: +- main-test-backend +- main-test-frontend +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-main-prerelease +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_publish_main.sh + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key_dev + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.8 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + PACKAGES_DESTINATION: gs://grafana-packages-testing + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-build + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + branch: main + event: + - push + paths: + exclude: + - '*.md' + - docs/** + - packages/**/*.md + - latest.json + repo: + - grafana/grafana +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- clone: retries: 3 depends_on: [] @@ -2805,9 +2879,6 @@ trigger: - refs/tags/*-cloud* include: - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: @@ -2863,9 +2934,6 @@ trigger: - refs/tags/*-cloud* include: - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: @@ -2943,69 +3011,6 @@ trigger: - refs/tags/*-cloud* include: - refs/tags/v* - repo: - exclude: - - grafana/grafana -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: -- main-test-backend -- main-test-frontend -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: rgm-main-prerelease -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - export GRAFANA_DIR=$$(pwd) - - cd /src && ./scripts/drone_publish_main.sh - environment: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: - from_secret: dagger_token - DESTINATION: - from_secret: destination - GCP_KEY_BASE64: - from_secret: gcp_key_base64 - GITHUB_TOKEN: - from_secret: github_token - GO_VERSION: 1.20.8 - GPG_PASSPHRASE: - from_secret: packages_gpg_passphrase - GPG_PRIVATE_KEY: - from_secret: packages_gpg_private_key - GPG_PUBLIC_KEY: - from_secret: packages_gpg_public_key - failure: ignore - image: grafana/grafana-build:main - name: rgm-build - pull: always - volumes: - - name: docker - path: /var/run/docker.sock -trigger: - branch: main - event: - - push - paths: - exclude: - - '*.md' - - docs/** - - packages/**/*.md - - latest.json - repo: - - grafana/grafana type: docker volumes: - host: @@ -3034,8 +3039,18 @@ steps: environment: _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination DESTINATION: from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key_dev GCP_KEY_BASE64: from_secret: gcp_key_base64 GITHUB_TOKEN: @@ -3047,6 +3062,11 @@ steps: from_secret: packages_gpg_private_key GPG_PUBLIC_KEY: from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + PACKAGES_DESTINATION: gs://grafana-packages-testing + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination image: grafana/grafana-build:main name: rgm-build pull: always @@ -3062,9 +3082,6 @@ trigger: - refs/tags/*-cloud* include: - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: @@ -3131,9 +3148,6 @@ trigger: - refs/tags/*-cloud* include: - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: @@ -3143,56 +3157,6 @@ volumes: clone: retries: 3 depends_on: -- release-test-backend -- release-test-frontend -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: rgm-version-branch-prerelease -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - export GRAFANA_DIR=$$(pwd) - - cd /src && ./scripts/drone_publish_tag_grafana.sh - environment: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: - from_secret: dagger_token - DESTINATION: - from_secret: destination - GCP_KEY_BASE64: - from_secret: gcp_key_base64 - GITHUB_TOKEN: - from_secret: github_token - GO_VERSION: 1.20.8 - GPG_PASSPHRASE: - from_secret: packages_gpg_passphrase - GPG_PRIVATE_KEY: - from_secret: packages_gpg_private_key - GPG_PUBLIC_KEY: - from_secret: packages_gpg_public_key - image: grafana/grafana-build:main - name: rgm-build - pull: always - volumes: - - name: docker - path: /var/run/docker.sock -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: - rgm-tag-prerelease - rgm-tag-prerelease-windows image_pull_secrets: @@ -3229,9 +3193,71 @@ trigger: - refs/tags/*-cloud* include: - refs/tags/v* - repo: - exclude: - - grafana/grafana +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- release-test-backend +- release-test-frontend +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-version-branch-prerelease +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_publish_tag_grafana.sh + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key_dev + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.8 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + PACKAGES_DESTINATION: gs://grafana-packages-testing + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-build + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + ref: + - refs/heads/v[0-9]* type: docker volumes: - host: @@ -3276,6 +3302,390 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + retries: 3 +depends_on: [] +environment: + EDITION: oss +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: nightly-test-frontend +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.18.3 + name: identify-runner +- commands: + - yarn install --immutable + depends_on: [] + image: node:18.12.0-alpine + name: yarn-install +- commands: + - apk add --update git bash + - yarn betterer ci + depends_on: + - yarn-install + image: node:18.12.0-alpine + name: betterer-frontend +- commands: + - yarn run ci:test-frontend + depends_on: + - yarn-install + environment: + TEST_MAX_WORKERS: 50% + image: node:18.12.0-alpine + name: test-frontend +trigger: + cron: + include: + - nightly + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: [] +environment: + EDITION: oss +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: nightly-test-backend +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.18.3 + name: identify-runner +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-cue + depends_on: [] + image: golang:1.20.8-alpine + name: verify-gen-cue +- commands: + - '# It is required that generated jsonnet is committed and in sync with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-jsonnet + depends_on: [] + image: golang:1.20.8-alpine + name: verify-gen-jsonnet +- commands: + - apk add --update make + - make gen-go + depends_on: + - verify-gen-cue + image: golang:1.20.8-alpine + name: wire-install +- commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang + - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... + depends_on: + - wire-install + image: golang:1.20.8-alpine + name: test-backend +- commands: + - apk add --update build-base + - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) + depends_on: + - wire-install + image: golang:1.20.8-alpine + name: test-backend-integration +trigger: + cron: + include: + - nightly + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- nightly-test-backend +- nightly-test-frontend +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-nightly-build +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_build_nightly_grafana.sh + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key_dev + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.8 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + PACKAGES_DESTINATION: gs://grafana-packages-testing + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-build + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - mkdir -p $${DESTINATION}/$${DRONE_BUILD_EVENT} + - printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - gcloud storage cp -r $${DRONE_WORKSPACE}/dist/* $${DESTINATION}/$${DRONE_BUILD_EVENT} + depends_on: + - rgm-build + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key_dev + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + PACKAGES_DESTINATION: gs://grafana-packages-testing + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: google/cloud-sdk:alpine + name: rgm-copy +trigger: + cron: + include: + - nightly + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- rgm-nightly-build +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-nightly-publish +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p $${DRONE_WORKSPACE}/dist + - printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - gcloud storage cp -r $${DESTINATION}/$${DRONE_BUILD_EVENT}/*_$${DRONE_BUILD_NUMBER}_* + $${DRONE_WORKSPACE}/dist + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key_dev + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + PACKAGES_DESTINATION: gs://grafana-packages-testing + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: google/cloud-sdk:alpine + name: rgm-copy +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_publish_nightly_grafana.sh + depends_on: + - rgm-copy + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key_dev + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.8 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + PACKAGES_DESTINATION: gs://grafana-packages-testing + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-publish + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +- depends_on: + - rgm-publish + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-deb + privileged: true + settings: + access_key_id: + from_secret: packages_access_key_id + gpg_passphrase: + from_secret: packages_gpg_passphrase + gpg_private_key: + from_secret: packages_gpg_private_key + gpg_public_key: + from_secret: packages_gpg_public_key + package_path: file:///drone/src/dist/*.deb + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account + target_bucket: grafana-packages-testing +- depends_on: + - rgm-publish + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-rpm + privileged: true + settings: + access_key_id: + from_secret: packages_access_key_id + gpg_passphrase: + from_secret: packages_gpg_passphrase + gpg_private_key: + from_secret: packages_gpg_private_key + gpg_public_key: + from_secret: packages_gpg_public_key + package_path: file:///drone/src/dist/*.rpm + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account + target_bucket: grafana-packages-testing +trigger: + cron: + include: + - nightly + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- clone: disable: true depends_on: [] @@ -4059,6 +4469,12 @@ get: kind: secret name: grafana_api_key --- +get: + name: api_key_dev + path: infra/data/ci/grafana-release-eng/grafanacom +kind: secret +name: grafana_api_key_dev +--- get: name: .dockerconfigjson path: secret/data/common/gcr @@ -4185,6 +4601,24 @@ get: kind: secret name: destination --- +get: + name: storybook_destination + path: infra/data/ci/grafana-release-eng/rgm +kind: secret +name: rgm_storybook_destination +--- +get: + name: cdn_destination + path: infra/data/ci/grafana-release-eng/rgm +kind: secret +name: rgm_cdn_destination +--- +get: + name: downloads_destination + path: infra/data/ci/grafana-release-eng/rgm +kind: secret +name: rgm_downloads_destination +--- get: name: dagger_token path: infra/data/ci/grafana-release-eng/rgm @@ -4216,6 +4650,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 852af171d897f0a2cc0b03375fa8dfeacc65c2df7113c5efd0e21b03195dd7af +hmac: 07868df8d2431c82e8c46fad453e7fbb0edbe0a4b0796158a80387b4595418d0 ... diff --git a/scripts/drone/rgm.star b/scripts/drone/rgm.star index 2789a24f7ae..9de9cd6e734 100644 --- a/scripts/drone/rgm.star +++ b/scripts/drone/rgm.star @@ -24,10 +24,15 @@ load( "scripts/drone/steps/lib_windows.star", "get_windows_steps", ) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "ignore_failure", "pipeline", + "with_deps", ) load( "scripts/drone/variables.star", @@ -36,29 +41,16 @@ load( load( "scripts/drone/vault.star", "from_secret", + "npm_token", + "rgm_cdn_destination", "rgm_dagger_token", "rgm_destination", + "rgm_downloads_destination", "rgm_gcp_key_base64", "rgm_github_token", + "rgm_storybook_destination", ) -def rgm_env_secrets(env): - """Adds the rgm secret ENV variables to the given env arg - - Args: - env: A map of environment varables. This function will adds the necessary secrets to it (and potentially overwrite them). - Returns: - Drone step. - """ - env["GCP_KEY_BASE64"] = from_secret(rgm_gcp_key_base64) - env["DESTINATION"] = from_secret(rgm_destination) - env["GITHUB_TOKEN"] = from_secret(rgm_github_token) - env["_EXPERIMENTAL_DAGGER_CLOUD_TOKEN"] = from_secret(rgm_dagger_token) - env["GPG_PRIVATE_KEY"] = from_secret("packages_gpg_private_key") - env["GPG_PUBLIC_KEY"] = from_secret("packages_gpg_public_key") - env["GPG_PASSPHRASE"] = from_secret("packages_gpg_passphrase") - return env - docs_paths = { "exclude": [ "*.md", @@ -69,11 +61,6 @@ docs_paths = { } tag_trigger = { - "repo": { - "exclude": [ - "grafana/grafana", - ], - }, "event": { "exclude": [ "promote", @@ -89,22 +76,61 @@ tag_trigger = { }, } +nightly_trigger = { + "event": { + "include": [ + "cron", + ], + }, + "cron": { + "include": [ + "nightly", + ], + }, +} + version_branch_trigger = {"ref": ["refs/heads/v[0-9]*"]} -def rgm_build(script = "drone_publish_main.sh", canFail = True): +def rgm_env_secrets(env): + """Adds the rgm secret ENV variables to the given env arg + + Args: + env: A map of environment varables. This function will adds the necessary secrets to it (and potentially overwrite them). + Returns: + Drone step. + """ + env["DESTINATION"] = from_secret(rgm_destination) + env["STORYBOOK_DESTINATION"] = from_secret(rgm_storybook_destination) + env["CDN_DESTINATION"] = from_secret(rgm_cdn_destination) + env["DOWNLOADS_DESTINATION"] = from_secret(rgm_downloads_destination) + env["PACKAGES_DESTINATION"] = "gs://grafana-packages-testing" + + env["GCP_KEY_BASE64"] = from_secret(rgm_gcp_key_base64) + env["GITHUB_TOKEN"] = from_secret(rgm_github_token) + env["_EXPERIMENTAL_DAGGER_CLOUD_TOKEN"] = from_secret(rgm_dagger_token) + env["GPG_PRIVATE_KEY"] = from_secret("packages_gpg_private_key") + env["GPG_PUBLIC_KEY"] = from_secret("packages_gpg_public_key") + env["GPG_PASSPHRASE"] = from_secret("packages_gpg_passphrase") + env["DOCKER_USERNAME"] = from_secret("docker_username") + env["DOCKER_PASSWORD"] = from_secret("docker_password") + env["NPM_TOKEN"] = from_secret(npm_token) + env["GCOM_API_KEY"] = from_secret("grafana_api_key_dev") + return env + +def rgm_run(name, script): """Returns a pipeline that does a full build & package of Grafana. Args: + name: The name of the pipeline step. script: The script in the container to run. - canFail: if true, then this pipeline can fail while the entire build will still succeed. Returns: Drone step. """ env = { "GO_VERSION": golang_version, } - rgm_build_step = { - "name": "rgm-build", + rgm_run_step = { + "name": name, "image": "grafana/grafana-build:main", "pull": "always", "commands": [ @@ -117,14 +143,72 @@ def rgm_build(script = "drone_publish_main.sh", canFail = True): "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], } - if canFail: - rgm_build_step["failure"] = "ignore" - return [ - rgm_build_step, + rgm_run_step, ] +def rgm_copy(src, dst): + """Copies file from/to GCS. + + Args: + src: source of the files. + dst: destination of the files. + + Returns: + Drone steps. + """ + commands = [ + "printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json", + "gcloud auth activate-service-account --key-file=/tmp/key.json", + "gcloud storage cp -r {} {}".format(src, dst), + ] + + if not dst.startswith("gs://"): + commands.insert(0, "mkdir -p {}".format(dst)) + + rgm_copy_step = { + "name": "rgm-copy", + "image": "google/cloud-sdk:alpine", + "commands": commands, + "environment": rgm_env_secrets({}), + } + + return [ + rgm_copy_step, + ] + +def rgm_publish_packages(bucket = "grafana-packages"): + """Publish deb and rpm packages. + + Args: + bucket: target bucket to publish the packages. + + Returns: + Drone steps. + """ + steps = [] + for package_manager in ["deb", "rpm"]: + steps.append({ + "name": "publish-{}".format(package_manager), + # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image + "image": images["package_publish"], + "privileged": True, + "settings": { + "access_key_id": from_secret("packages_access_key_id"), + "secret_access_key": from_secret("packages_secret_access_key"), + "service_account_json": from_secret("packages_service_account"), + "target_bucket": bucket, + "gpg_passphrase": from_secret("packages_gpg_passphrase"), + "gpg_public_key": from_secret("packages_gpg_public_key"), + "gpg_private_key": from_secret("packages_gpg_private_key"), + "package_path": "file:///drone/src/dist/*.{}".format(package_manager), + }, + }) + + return steps + def rgm_main(): + # Runs a package / build process (with some distros) when commits are merged to main trigger = { "event": [ "push", @@ -139,15 +223,16 @@ def rgm_main(): return pipeline( name = "rgm-main-prerelease", trigger = trigger, - steps = rgm_build(canFail = True), + steps = rgm_run("rgm-build", "drone_publish_main.sh"), depends_on = ["main-test-backend", "main-test-frontend"], ) def rgm_tag(): + # Runs a package / build process (with all distros) when a tag is made return pipeline( name = "rgm-tag-prerelease", trigger = tag_trigger, - steps = rgm_build(script = "drone_publish_tag_grafana.sh", canFail = False), + steps = rgm_run("rgm-build", "drone_publish_tag_grafana.sh"), depends_on = ["release-test-backend", "release-test-frontend"], ) @@ -166,22 +251,61 @@ def rgm_tag_windows(): ) def rgm_version_branch(): + # Runs a package / build proces (with all distros) when a commit lands on a version branch return pipeline( name = "rgm-version-branch-prerelease", trigger = version_branch_trigger, - steps = rgm_build(script = "drone_publish_tag_grafana.sh", canFail = False), + steps = rgm_run("rgm-build", "drone_publish_tag_grafana.sh"), depends_on = ["release-test-backend", "release-test-frontend"], ) -def rgm(): +def rgm_nightly_build(): + src = "$${DRONE_WORKSPACE}/dist/*" + dst = "$${DESTINATION}/$${DRONE_BUILD_EVENT}" + + copy_steps = with_deps(rgm_copy(src, dst), ["rgm-build"]) + + return pipeline( + name = "rgm-nightly-build", + trigger = nightly_trigger, + steps = rgm_run("rgm-build", "drone_build_nightly_grafana.sh") + copy_steps, + depends_on = ["nightly-test-backend", "nightly-test-frontend"], + ) + +def rgm_nightly_publish(): + """Nightly publish pipeline. + + Returns: + Drone pipeline. + """ + src = "$${DESTINATION}/$${DRONE_BUILD_EVENT}/*_$${DRONE_BUILD_NUMBER}_*" + dst = "$${DRONE_WORKSPACE}/dist" + + publish_steps = with_deps(rgm_run("rgm-publish", "drone_publish_nightly_grafana.sh"), ["rgm-copy"]) + package_steps = with_deps(rgm_publish_packages("grafana-packages-testing"), ["rgm-publish"]) + + return pipeline( + name = "rgm-nightly-publish", + trigger = nightly_trigger, + steps = rgm_copy(src, dst) + publish_steps + package_steps, + depends_on = ["rgm-nightly-build"], + ) + +def rgm_nightly_pipeline(): + return [ + test_frontend(nightly_trigger, "nightly"), + test_backend(nightly_trigger, "nightly"), + rgm_nightly_build(), + rgm_nightly_publish(), + ] + +def rgm_tag_pipeline(): return [ whats_new_checker_pipeline(tag_trigger), test_frontend(tag_trigger, "release"), test_backend(tag_trigger, "release"), - rgm_main(), # Runs a package / build process (with some distros) when commits are merged to main - rgm_tag(), # Runs a package / build process (with all distros) when a tag is made + rgm_tag(), rgm_tag_windows(), - rgm_version_branch(), # Runs a package / build proces (with all distros) when a commit lands on a version branch verify_release_pipeline( trigger = tag_trigger, name = "rgm-tag-verify-prerelease-assets", @@ -191,6 +315,11 @@ def rgm(): "rgm-tag-prerelease-windows", ], ), + ] + +def rgm_version_branch_pipeline(): + return [ + rgm_version_branch(), verify_release_pipeline( trigger = version_branch_trigger, name = "rgm-prerelease-verify-prerelease-assets", @@ -200,3 +329,16 @@ def rgm(): ], ), ] + +def rgm_main_pipeline(): + return [ + rgm_main(), + ] + +def rgm(): + return ( + rgm_main_pipeline() + + rgm_tag_pipeline() + + rgm_version_branch_pipeline() + + rgm_nightly_pipeline() + ) diff --git a/scripts/drone/vault.star b/scripts/drone/vault.star index 12e92ddd026..81c26ed4038 100644 --- a/scripts/drone/vault.star +++ b/scripts/drone/vault.star @@ -14,6 +14,9 @@ azure_tenant = "azure_tenant" rgm_gcp_key_base64 = "gcp_key_base64" rgm_destination = "destination" +rgm_storybook_destination = "rgm_storybook_destination" +rgm_cdn_destination = "rgm_cdn_destination" +rgm_downloads_destination = "rgm_downloads_destination" rgm_github_token = "github_token" rgm_dagger_token = "dagger_token" @@ -40,6 +43,7 @@ def secrets(): vault_secret(gcp_grafanauploads, "infra/data/ci/grafana-release-eng/grafanauploads", "credentials.json"), vault_secret(gcp_grafanauploads_base64, "infra/data/ci/grafana-release-eng/grafanauploads", "credentials_base64"), vault_secret("grafana_api_key", "infra/data/ci/grafana-release-eng/grafanacom", "api_key"), + vault_secret("grafana_api_key_dev", "infra/data/ci/grafana-release-eng/grafanacom", "api_key_dev"), vault_secret(pull_secret, "secret/data/common/gcr", ".dockerconfigjson"), vault_secret("github_token", "infra/data/ci/github/grafanabot", "pat"), vault_secret(drone_token, "infra/data/ci/drone", "machine-user-token"), @@ -122,6 +126,21 @@ def secrets(): "infra/data/ci/grafana-release-eng/rgm", "destination_prod", ), + vault_secret( + rgm_storybook_destination, + "infra/data/ci/grafana-release-eng/rgm", + "storybook_destination", + ), + vault_secret( + rgm_cdn_destination, + "infra/data/ci/grafana-release-eng/rgm", + "cdn_destination", + ), + vault_secret( + rgm_downloads_destination, + "infra/data/ci/grafana-release-eng/rgm", + "downloads_destination", + ), vault_secret( rgm_dagger_token, "infra/data/ci/grafana-release-eng/rgm",