CI: Consolidate package and docker steps (#95375)

* Consolidate package and docker steps

* load docker images (if there are any); fix tag-format casing

* I guess e2e tests use pkg/build...

* lint starlark

scripts/drone/pipelines/build.star

@@ -27,13 +27,11 @@ load(
     "upload_packages_step",
     "verify_gen_cue_step",
     "verify_gen_jsonnet_step",
-    "wire_install_step",
     "yarn_install_step",
 )
 load(
     "scripts/drone/steps/rgm.star",
     "rgm_artifacts_step",
-    "rgm_build_docker_step",
 )
 load(
     "scripts/drone/utils/images.star",
@@ -44,6 +42,7 @@ load(
     "pipeline",
 )
 
+# This function isn't actually unused but I don't know why the linter thinks it is...
 # @unused
 def build_e2e(trigger, ver_mode):
     """Perform e2e building, testing, and publishing.
@@ -63,37 +62,58 @@ def build_e2e(trigger, ver_mode):
         compile_build_cmd(),
         verify_gen_cue_step(),
         verify_gen_jsonnet_step(),
-        wire_install_step(),
         yarn_install_step(),
     ]
 
     build_steps = []
 
+    create_packages = rgm_artifacts_step(
+        alpine = images["alpine"],
+        artifacts = [
+            "targz:grafana:linux/amd64",
+            "targz:grafana:linux/arm64",
+            "targz:grafana:linux/arm/v7",
+            "docker:grafana:linux/amd64",
+            "docker:grafana:linux/amd64:ubuntu",
+            "docker:grafana:linux/arm64",
+            "docker:grafana:linux/arm64:ubuntu",
+            "docker:grafana:linux/arm/v7",
+            "docker:grafana:linux/arm/v7:ubuntu",
+        ],
+        file = "packages.txt",
+        tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}",
+        ubuntu = images["ubuntu"],
+        ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}",
+    )
+
+    publish_docker = publish_images_step(
+        depends_on = [create_packages["name"]],
+        docker_repo = "grafana",
+        trigger = trigger_oss,
+        ver_mode = ver_mode,
+    )
+
     if ver_mode == "pr":
         build_steps.extend(
             [
                 build_frontend_package_step(),
                 enterprise_downstream_step(ver_mode = ver_mode),
-                rgm_artifacts_step(artifacts = ["targz:grafana:linux/amd64", "targz:grafana:linux/arm64", "targz:grafana:linux/arm/v7"], file = "packages.txt"),
             ],
         )
     else:
+        # The only other event or "ver_mode" where this is used is 'main'
+        update_package_json = update_package_json_version()
+        create_packages["depends_on"] = [update_package_json["name"]]
+
         build_steps.extend([
-            update_package_json_version(),
+            update_package_json,
             build_frontend_package_step(depends_on = ["update-package-json-version"]),
-            rgm_artifacts_step(
-                artifacts = [
-                    "targz:grafana:linux/amd64",
-                    "targz:grafana:linux/arm64",
-                    "targz:grafana:linux/arm/v7",
-                ],
-                depends_on = ["update-package-json-version"],
-                file = "packages.txt",
-            ),
         ])
 
     build_steps.extend(
         [
+            create_packages,
+            publish_docker,
             build_test_plugins_step(),
             grafana_server_step(),
             e2e_tests_step("dashboards-suite"),
@@ -123,45 +143,20 @@ def build_e2e(trigger, ver_mode):
             [
                 store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode),
                 frontend_metrics_step(trigger = trigger_oss),
-                rgm_build_docker_step(
-                    images["ubuntu"],
-                    images["alpine"],
-                    depends_on = ["update-package-json-version"],
-                    tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}",
-                    ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}",
-                ),
-                publish_images_step(
-                    docker_repo = "grafana",
-                    trigger = trigger_oss,
-                    ver_mode = ver_mode,
-                ),
                 publish_images_step(
+                    depends_on = [create_packages["name"]],
                     docker_repo = "grafana-oss",
                     trigger = trigger_oss,
                     ver_mode = ver_mode,
                 ),
                 release_canary_npm_packages_step(trigger = trigger_oss),
                 upload_packages_step(
+                    depends_on = [create_packages["name"]],
                     trigger = trigger_oss,
                     ver_mode = ver_mode,
                 ),
                 upload_cdn_step(
-                    trigger = trigger_oss,
-                    ver_mode = ver_mode,
-                ),
-            ],
-        )
-    elif ver_mode == "pr":
-        build_steps.extend(
-            [
-                rgm_build_docker_step(
-                    images["ubuntu"],
-                    images["alpine"],
-                    tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}",
-                    ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}",
-                ),
-                publish_images_step(
-                    docker_repo = "grafana",
+                    depends_on = [create_packages["name"]],
                     trigger = trigger_oss,
                     ver_mode = ver_mode,
                 ),

scripts/drone/steps/lib.star

@@ -405,13 +405,14 @@ def playwright_e2e_report_post_link():
         ],
     }
 
-def upload_cdn_step(ver_mode, trigger = None):
+def upload_cdn_step(ver_mode, trigger = None, depends_on = ["grafana-server"]):
     """Uploads CDN assets using the Grafana build tool.
 
     Args:
       ver_mode: only uses the step trigger when ver_mode == 'release-branch' or 'main'
       trigger: a Drone trigger for the step.
         Defaults to None.
+      depends_on: drone steps that this step depends on
 
     Returns:
       Drone step.
@@ -420,9 +421,7 @@ def upload_cdn_step(ver_mode, trigger = None):
     step = {
         "name": "upload-cdn-assets",
         "image": images["publish"],
-        "depends_on": [
-            "grafana-server",
-        ],
+        "depends_on": depends_on,
         "environment": {
             "GCP_KEY": from_secret(gcp_grafanauploads),
             "PRERELEASE_BUCKET": from_secret(prerelease_bucket),
@@ -929,7 +928,7 @@ def fetch_images_step():
         "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
     }
 
-def publish_images_step(ver_mode, docker_repo, trigger = None):
+def publish_images_step(ver_mode, docker_repo, trigger = None, depends_on = ["rgm-build-docker"]):
     """Generates a step for publishing public Docker images with grabpl.
 
     Args:
@@ -939,6 +938,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None):
         It is combined with the 'grafana/' library prefix.
       trigger: a Drone trigger for the pipeline.
         Defaults to None.
+      depends_on: drone steps that this step depends on
 
     Returns:
       Drone step.
@@ -959,7 +959,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None):
         docker_repo,
     )
 
-    deps = ["rgm-build-docker"]
+    deps = depends_on
     if ver_mode == "release":
         deps = ["fetch-images"]
         cmd += " --version-tag ${DRONE_TAG}"
@@ -1149,7 +1149,12 @@ def release_canary_npm_packages_step(trigger = None):
 
     return step
 
-def upload_packages_step(ver_mode, trigger = None):
+def upload_packages_step(ver_mode, trigger = None, depends_on = [
+    "end-to-end-tests-dashboards-suite",
+    "end-to-end-tests-panels-suite",
+    "end-to-end-tests-smoke-tests-suite",
+    "end-to-end-tests-various-suite",
+]):
     """Upload packages to object storage.
 
     Args:
@@ -1157,6 +1162,7 @@ def upload_packages_step(ver_mode, trigger = None):
         edition packages when executed.
       trigger: a Drone trigger for the step.
         Defaults to None.
+      depends_on: drone steps that this step depends on
 
     Returns:
       Drone step.
@@ -1164,7 +1170,7 @@ def upload_packages_step(ver_mode, trigger = None):
     step = {
         "name": "upload-packages",
         "image": images["publish"],
-        "depends_on": end_to_end_tests_deps(),
+        "depends_on": depends_on,
         "environment": {
             "GCP_KEY": from_secret(gcp_grafanauploads_base64),
             "PRERELEASE_BUCKET": from_secret("prerelease_bucket"),

scripts/drone/steps/rgm.star

@@ -3,6 +3,10 @@ Individual steps that use 'grafana-build' to replace existing individual steps.
 These aren't used in releases.
 """
 
+load(
+    "scripts/drone/utils/images.star",
+    "images",
+)
 load(
     "scripts/drone/variables.star",
     "golang_version",
@@ -22,7 +26,16 @@ def artifacts_cmd(artifacts = []):
     return cmd
 
 # rgm_artifacts_step will create artifacts using the '/src/build artifacts' command.
-def rgm_artifacts_step(name = "rgm-package", artifacts = ["targz:grafana:linux/amd64", "targz:grafana:linux/arm64"], file = "packages.txt", depends_on = ["yarn-install"]):
+def rgm_artifacts_step(
+        name = "rgm-package",
+        artifacts = ["targz:grafana:linux/amd64", "targz:grafana:linux/arm64"],
+        file = "packages.txt",
+        depends_on = ["yarn-install"],
+        tag_format = "{{ .version }}-{{ .arch }}",
+        ubuntu_tag_format = "{{ .version }}-ubuntu-{{ .arch }}",
+        verify = "false",
+        ubuntu = images["ubuntu"],
+        alpine = images["alpine"]):
     cmd = artifacts_cmd(artifacts = artifacts)
 
     return {
@@ -34,11 +47,18 @@ def rgm_artifacts_step(name = "rgm-package", artifacts = ["targz:grafana:linux/a
             "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN": from_secret(rgm_dagger_token),
         },
         "commands": [
+            "docker run --privileged --rm tonistiigi/binfmt --install all",
             cmd +
             "--go-version={} ".format(golang_version) +
             "--yarn-cache=$$YARN_CACHE_FOLDER " +
             "--build-id=$$DRONE_BUILD_NUMBER " +
+            "--ubuntu-base={} ".format(ubuntu) +
+            "--alpine-base={} ".format(alpine) +
+            "--tag-format='{}' ".format(tag_format) +
+            "--ubuntu-tag-format='{}' ".format(ubuntu_tag_format) +
+            "--verify='{}' ".format(verify) +
             "--grafana-dir=$$PWD > {}".format(file),
+            "find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i",
         ],
         "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
     }