CI: Consolidate package and docker steps (#95375)

* Consolidate package and docker steps

* load docker images (if there are any); fix tag-format casing

* I guess e2e tests use pkg/build...

* lint starlark

.drone.yml

@@ -569,13 +569,6 @@ steps:
   depends_on: []
   image: golang:1.23.1-alpine
   name: verify-gen-jsonnet
-- commands:
-  - apk add --update make
-  - make gen-go
-  depends_on:
-  - verify-gen-cue
-  image: golang:1.23.1-alpine
-  name: wire-install
 - commands:
   - yarn install --immutable || yarn install --immutable
   depends_on: []
@@ -606,9 +599,16 @@ steps:
     token:
       from_secret: drone_token
 - commands:
+  - docker run --privileged --rm tonistiigi/binfmt --install all
   - /src/grafana-build artifacts -a targz:grafana:linux/amd64 -a targz:grafana:linux/arm64
-    -a targz:grafana:linux/arm/v7 --go-version=1.23.1 --yarn-cache=$$YARN_CACHE_FOLDER
-    --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD > packages.txt
+    -a targz:grafana:linux/arm/v7 -a docker:grafana:linux/amd64 -a docker:grafana:linux/amd64:ubuntu
+    -a docker:grafana:linux/arm64 -a docker:grafana:linux/arm64:ubuntu -a docker:grafana:linux/arm/v7
+    -a docker:grafana:linux/arm/v7:ubuntu --go-version=1.23.1 --yarn-cache=$$YARN_CACHE_FOLDER
+    --build-id=$$DRONE_BUILD_NUMBER --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.20.3
+    --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{
+    .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' --verify='false' --grafana-dir=$$PWD
+    > packages.txt
+  - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i
   depends_on:
   - yarn-install
   environment:
@@ -620,6 +620,27 @@ steps:
   volumes:
   - name: docker
     path: /var/run/docker.sock
+- commands:
+  - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana
+  depends_on:
+  - rgm-package
+  environment:
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    DOCKER_USER:
+      from_secret: docker_username
+    GITHUB_APP_ID:
+      from_secret: delivery-bot-app-id
+    GITHUB_APP_INSTALLATION_ID:
+      from_secret: delivery-bot-app-installation-id
+    GITHUB_APP_PRIVATE_KEY:
+      from_secret: delivery-bot-app-private-key
+  failure: ignore
+  image: google/cloud-sdk:431.0.0
+  name: publish-images-grafana
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
 - commands:
   - yarn e2e:plugin:build
   depends_on:
@@ -855,47 +876,6 @@ steps:
   failure: always
   image: grafana/docker-puppeteer:1.1.0
   name: test-a11y-frontend
-- commands:
-  - docker run --privileged --rm tonistiigi/binfmt --install all
-  - /src/grafana-build artifacts -a docker:grafana:linux/amd64 -a docker:grafana:linux/amd64:ubuntu
-    -a docker:grafana:linux/arm64 -a docker:grafana:linux/arm64:ubuntu -a docker:grafana:linux/arm/v7
-    -a docker:grafana:linux/arm/v7:ubuntu --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER
-    --go-version=1.23.1 --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.20.3 --tag-format='{{
-    .version_base }}-{{ .buildID }}-{{ .arch }}' --grafana-dir=$$PWD --ubuntu-tag-format='{{
-    .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt
-  - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i
-  depends_on:
-  - yarn-install
-  environment:
-    _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
-      from_secret: dagger_token
-  image: grafana/grafana-build:main
-  name: rgm-build-docker
-  pull: always
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-- commands:
-  - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana
-  depends_on:
-  - rgm-build-docker
-  environment:
-    DOCKER_PASSWORD:
-      from_secret: docker_password
-    DOCKER_USER:
-      from_secret: docker_username
-    GITHUB_APP_ID:
-      from_secret: delivery-bot-app-id
-    GITHUB_APP_INSTALLATION_ID:
-      from_secret: delivery-bot-app-installation-id
-    GITHUB_APP_PRIVATE_KEY:
-      from_secret: delivery-bot-app-private-key
-  failure: ignore
-  image: google/cloud-sdk:431.0.0
-  name: publish-images-grafana
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
 trigger:
   event:
   - pull_request
@@ -1987,13 +1967,6 @@ steps:
   depends_on: []
   image: golang:1.23.1-alpine
   name: verify-gen-jsonnet
-- commands:
-  - apk add --update make
-  - make gen-go
-  depends_on:
-  - verify-gen-cue
-  image: golang:1.23.1-alpine
-  name: wire-install
 - commands:
   - yarn install --immutable || yarn install --immutable
   depends_on: []
@@ -2023,9 +1996,16 @@ steps:
   image: node:20.9.0-alpine
   name: build-frontend-packages
 - commands:
+  - docker run --privileged --rm tonistiigi/binfmt --install all
   - /src/grafana-build artifacts -a targz:grafana:linux/amd64 -a targz:grafana:linux/arm64
-    -a targz:grafana:linux/arm/v7 --go-version=1.23.1 --yarn-cache=$$YARN_CACHE_FOLDER
-    --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD > packages.txt
+    -a targz:grafana:linux/arm/v7 -a docker:grafana:linux/amd64 -a docker:grafana:linux/amd64:ubuntu
+    -a docker:grafana:linux/arm64 -a docker:grafana:linux/arm64:ubuntu -a docker:grafana:linux/arm/v7
+    -a docker:grafana:linux/arm/v7:ubuntu --go-version=1.23.1 --yarn-cache=$$YARN_CACHE_FOLDER
+    --build-id=$$DRONE_BUILD_NUMBER --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.20.3
+    --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{
+    .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' --verify='false' --grafana-dir=$$PWD
+    > packages.txt
+  - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i
   depends_on:
   - update-package-json-version
   environment:
@@ -2037,6 +2017,31 @@ steps:
   volumes:
   - name: docker
     path: /var/run/docker.sock
+- commands:
+  - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana
+  depends_on:
+  - rgm-package
+  environment:
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    DOCKER_USER:
+      from_secret: docker_username
+    GCP_KEY:
+      from_secret: gcp_grafanauploads
+    GITHUB_APP_ID:
+      from_secret: delivery-bot-app-id
+    GITHUB_APP_INSTALLATION_ID:
+      from_secret: delivery-bot-app-installation-id
+    GITHUB_APP_PRIVATE_KEY:
+      from_secret: delivery-bot-app-private-key
+  image: google/cloud-sdk:431.0.0
+  name: publish-images-grafana
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+  when:
+    repo:
+    - grafana/grafana
 - commands:
   - yarn e2e:plugin:build
   depends_on:
@@ -2308,55 +2313,10 @@ steps:
   when:
     repo:
     - grafana/grafana
-- commands:
-  - docker run --privileged --rm tonistiigi/binfmt --install all
-  - /src/grafana-build artifacts -a docker:grafana:linux/amd64 -a docker:grafana:linux/amd64:ubuntu
-    -a docker:grafana:linux/arm64 -a docker:grafana:linux/arm64:ubuntu -a docker:grafana:linux/arm/v7
-    -a docker:grafana:linux/arm/v7:ubuntu --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER
-    --go-version=1.23.1 --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.20.3 --tag-format='{{
-    .version_base }}-{{ .buildID }}-{{ .arch }}' --grafana-dir=$$PWD --ubuntu-tag-format='{{
-    .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt
-  - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i
-  depends_on:
-  - update-package-json-version
-  environment:
-    _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
-      from_secret: dagger_token
-  image: grafana/grafana-build:main
-  name: rgm-build-docker
-  pull: always
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-- commands:
-  - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana
-  depends_on:
-  - rgm-build-docker
-  environment:
-    DOCKER_PASSWORD:
-      from_secret: docker_password
-    DOCKER_USER:
-      from_secret: docker_username
-    GCP_KEY:
-      from_secret: gcp_grafanauploads
-    GITHUB_APP_ID:
-      from_secret: delivery-bot-app-id
-    GITHUB_APP_INSTALLATION_ID:
-      from_secret: delivery-bot-app-installation-id
-    GITHUB_APP_PRIVATE_KEY:
-      from_secret: delivery-bot-app-private-key
-  image: google/cloud-sdk:431.0.0
-  name: publish-images-grafana
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-  when:
-    repo:
-    - grafana/grafana
 - commands:
   - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana-oss
   depends_on:
-  - rgm-build-docker
+  - rgm-package
   environment:
     DOCKER_PASSWORD:
       from_secret: docker_password
@@ -2401,10 +2361,7 @@ steps:
 - commands:
   - ./bin/build upload-packages --edition oss
   depends_on:
-  - end-to-end-tests-dashboards-suite
-  - end-to-end-tests-panels-suite
-  - end-to-end-tests-smoke-tests-suite
-  - end-to-end-tests-various-suite
+  - rgm-package
   environment:
     GCP_KEY:
       from_secret: gcp_grafanauploads_base64
@@ -2418,7 +2375,7 @@ steps:
 - commands:
   - ./bin/build upload-cdn --edition oss
   depends_on:
-  - grafana-server
+  - rgm-package
   environment:
     GCP_KEY:
       from_secret: gcp_grafanauploads
@@ -5778,6 +5735,6 @@ kind: secret
 name: gcr_credentials
 ---
 kind: signature
-hmac: f49fe1151910b4a42835e0779c676649e2e7ec76ca50fdf522ffea72e60638cc
+hmac: 250d3c2ae9ddf7cbfd2a9fed01fe81312a882ba09b8f8725f4704629da00a774
 
 ...

scripts/drone/pipelines/build.star

@@ -27,13 +27,11 @@ load(
     "upload_packages_step",
     "verify_gen_cue_step",
     "verify_gen_jsonnet_step",
-    "wire_install_step",
     "yarn_install_step",
 )
 load(
     "scripts/drone/steps/rgm.star",
     "rgm_artifacts_step",
-    "rgm_build_docker_step",
 )
 load(
     "scripts/drone/utils/images.star",
@@ -44,6 +42,7 @@ load(
     "pipeline",
 )
 
+# This function isn't actually unused but I don't know why the linter thinks it is...
 # @unused
 def build_e2e(trigger, ver_mode):
     """Perform e2e building, testing, and publishing.
@@ -63,37 +62,58 @@ def build_e2e(trigger, ver_mode):
         compile_build_cmd(),
         verify_gen_cue_step(),
         verify_gen_jsonnet_step(),
-        wire_install_step(),
         yarn_install_step(),
     ]
 
     build_steps = []
 
+    create_packages = rgm_artifacts_step(
+        alpine = images["alpine"],
+        artifacts = [
+            "targz:grafana:linux/amd64",
+            "targz:grafana:linux/arm64",
+            "targz:grafana:linux/arm/v7",
+            "docker:grafana:linux/amd64",
+            "docker:grafana:linux/amd64:ubuntu",
+            "docker:grafana:linux/arm64",
+            "docker:grafana:linux/arm64:ubuntu",
+            "docker:grafana:linux/arm/v7",
+            "docker:grafana:linux/arm/v7:ubuntu",
+        ],
+        file = "packages.txt",
+        tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}",
+        ubuntu = images["ubuntu"],
+        ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}",
+    )
+
+    publish_docker = publish_images_step(
+        depends_on = [create_packages["name"]],
+        docker_repo = "grafana",
+        trigger = trigger_oss,
+        ver_mode = ver_mode,
+    )
+
     if ver_mode == "pr":
         build_steps.extend(
             [
                 build_frontend_package_step(),
                 enterprise_downstream_step(ver_mode = ver_mode),
-                rgm_artifacts_step(artifacts = ["targz:grafana:linux/amd64", "targz:grafana:linux/arm64", "targz:grafana:linux/arm/v7"], file = "packages.txt"),
             ],
         )
     else:
+        # The only other event or "ver_mode" where this is used is 'main'
+        update_package_json = update_package_json_version()
+        create_packages["depends_on"] = [update_package_json["name"]]
+
         build_steps.extend([
-            update_package_json_version(),
+            update_package_json,
             build_frontend_package_step(depends_on = ["update-package-json-version"]),
-            rgm_artifacts_step(
-                artifacts = [
-                    "targz:grafana:linux/amd64",
-                    "targz:grafana:linux/arm64",
-                    "targz:grafana:linux/arm/v7",
-                ],
-                depends_on = ["update-package-json-version"],
-                file = "packages.txt",
-            ),
         ])
 
     build_steps.extend(
         [
+            create_packages,
+            publish_docker,
             build_test_plugins_step(),
             grafana_server_step(),
             e2e_tests_step("dashboards-suite"),
@@ -123,45 +143,20 @@ def build_e2e(trigger, ver_mode):
             [
                 store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode),
                 frontend_metrics_step(trigger = trigger_oss),
-                rgm_build_docker_step(
-                    images["ubuntu"],
-                    images["alpine"],
-                    depends_on = ["update-package-json-version"],
-                    tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}",
-                    ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}",
-                ),
-                publish_images_step(
-                    docker_repo = "grafana",
-                    trigger = trigger_oss,
-                    ver_mode = ver_mode,
-                ),
                 publish_images_step(
+                    depends_on = [create_packages["name"]],
                     docker_repo = "grafana-oss",
                     trigger = trigger_oss,
                     ver_mode = ver_mode,
                 ),
                 release_canary_npm_packages_step(trigger = trigger_oss),
                 upload_packages_step(
+                    depends_on = [create_packages["name"]],
                     trigger = trigger_oss,
                     ver_mode = ver_mode,
                 ),
                 upload_cdn_step(
-                    trigger = trigger_oss,
-                    ver_mode = ver_mode,
-                ),
-            ],
-        )
-    elif ver_mode == "pr":
-        build_steps.extend(
-            [
-                rgm_build_docker_step(
-                    images["ubuntu"],
-                    images["alpine"],
-                    tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}",
-                    ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}",
-                ),
-                publish_images_step(
-                    docker_repo = "grafana",
+                    depends_on = [create_packages["name"]],
                     trigger = trigger_oss,
                     ver_mode = ver_mode,
                 ),

scripts/drone/steps/lib.star

@@ -405,13 +405,14 @@ def playwright_e2e_report_post_link():
         ],
     }
 
-def upload_cdn_step(ver_mode, trigger = None):
+def upload_cdn_step(ver_mode, trigger = None, depends_on = ["grafana-server"]):
     """Uploads CDN assets using the Grafana build tool.
 
     Args:
       ver_mode: only uses the step trigger when ver_mode == 'release-branch' or 'main'
       trigger: a Drone trigger for the step.
         Defaults to None.
+      depends_on: drone steps that this step depends on
 
     Returns:
       Drone step.
@@ -420,9 +421,7 @@ def upload_cdn_step(ver_mode, trigger = None):
     step = {
         "name": "upload-cdn-assets",
         "image": images["publish"],
-        "depends_on": [
-            "grafana-server",
-        ],
+        "depends_on": depends_on,
         "environment": {
             "GCP_KEY": from_secret(gcp_grafanauploads),
             "PRERELEASE_BUCKET": from_secret(prerelease_bucket),
@@ -929,7 +928,7 @@ def fetch_images_step():
         "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
     }
 
-def publish_images_step(ver_mode, docker_repo, trigger = None):
+def publish_images_step(ver_mode, docker_repo, trigger = None, depends_on = ["rgm-build-docker"]):
     """Generates a step for publishing public Docker images with grabpl.
 
     Args:
@@ -939,6 +938,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None):
         It is combined with the 'grafana/' library prefix.
       trigger: a Drone trigger for the pipeline.
         Defaults to None.
+      depends_on: drone steps that this step depends on
 
     Returns:
       Drone step.
@@ -959,7 +959,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None):
         docker_repo,
     )
 
-    deps = ["rgm-build-docker"]
+    deps = depends_on
     if ver_mode == "release":
         deps = ["fetch-images"]
         cmd += " --version-tag ${DRONE_TAG}"
@@ -1149,7 +1149,12 @@ def release_canary_npm_packages_step(trigger = None):
 
     return step
 
-def upload_packages_step(ver_mode, trigger = None):
+def upload_packages_step(ver_mode, trigger = None, depends_on = [
+    "end-to-end-tests-dashboards-suite",
+    "end-to-end-tests-panels-suite",
+    "end-to-end-tests-smoke-tests-suite",
+    "end-to-end-tests-various-suite",
+]):
     """Upload packages to object storage.
 
     Args:
@@ -1157,6 +1162,7 @@ def upload_packages_step(ver_mode, trigger = None):
         edition packages when executed.
       trigger: a Drone trigger for the step.
         Defaults to None.
+      depends_on: drone steps that this step depends on
 
     Returns:
       Drone step.
@@ -1164,7 +1170,7 @@ def upload_packages_step(ver_mode, trigger = None):
     step = {
         "name": "upload-packages",
         "image": images["publish"],
-        "depends_on": end_to_end_tests_deps(),
+        "depends_on": depends_on,
         "environment": {
             "GCP_KEY": from_secret(gcp_grafanauploads_base64),
             "PRERELEASE_BUCKET": from_secret("prerelease_bucket"),

scripts/drone/steps/rgm.star

@@ -3,6 +3,10 @@ Individual steps that use 'grafana-build' to replace existing individual steps.
 These aren't used in releases.
 """
 
+load(
+    "scripts/drone/utils/images.star",
+    "images",
+)
 load(
     "scripts/drone/variables.star",
     "golang_version",
@@ -22,7 +26,16 @@ def artifacts_cmd(artifacts = []):
     return cmd
 
 # rgm_artifacts_step will create artifacts using the '/src/build artifacts' command.
-def rgm_artifacts_step(name = "rgm-package", artifacts = ["targz:grafana:linux/amd64", "targz:grafana:linux/arm64"], file = "packages.txt", depends_on = ["yarn-install"]):
+def rgm_artifacts_step(
+        name = "rgm-package",
+        artifacts = ["targz:grafana:linux/amd64", "targz:grafana:linux/arm64"],
+        file = "packages.txt",
+        depends_on = ["yarn-install"],
+        tag_format = "{{ .version }}-{{ .arch }}",
+        ubuntu_tag_format = "{{ .version }}-ubuntu-{{ .arch }}",
+        verify = "false",
+        ubuntu = images["ubuntu"],
+        alpine = images["alpine"]):
     cmd = artifacts_cmd(artifacts = artifacts)
 
     return {
@@ -34,11 +47,18 @@ def rgm_artifacts_step(name = "rgm-package", artifacts = ["targz:grafana:linux/a
             "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN": from_secret(rgm_dagger_token),
         },
         "commands": [
+            "docker run --privileged --rm tonistiigi/binfmt --install all",
             cmd +
             "--go-version={} ".format(golang_version) +
             "--yarn-cache=$$YARN_CACHE_FOLDER " +
             "--build-id=$$DRONE_BUILD_NUMBER " +
+            "--ubuntu-base={} ".format(ubuntu) +
+            "--alpine-base={} ".format(alpine) +
+            "--tag-format='{}' ".format(tag_format) +
+            "--ubuntu-tag-format='{}' ".format(ubuntu_tag_format) +
+            "--verify='{}' ".format(verify) +
             "--grafana-dir=$$PWD > {}".format(file),
+            "find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i",
         ],
         "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
     }