CI: Update RGM steps to use the artifacts command (#77470)

* update rgm steps to use artifacts subcmd

* format-drone

* make drone

.drone.yml

@@ -17,7 +17,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
@@ -67,7 +67,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
@@ -117,7 +117,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - yarn install --immutable
@@ -217,7 +217,7 @@ steps:
   name: clone-enterprise
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - yarn install --immutable
@@ -306,7 +306,7 @@ steps:
   name: clone-enterprise
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - '# It is required that code generated from Thema/CUE be committed and in sync
@@ -390,7 +390,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
@@ -485,7 +485,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - mkdir -p bin
@@ -556,9 +556,9 @@ steps:
     token:
       from_secret: drone_token
 - commands:
-  - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.10
-    --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD
-    > packages.txt
+  - /src/grafana-build artifacts -a targz:grafana:linux/amd64 -a targz:grafana:linux/arm64
+    --go-version=1.20.10 --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER
+    --grafana-dir=$$PWD > packages.txt
   depends_on:
   - yarn-install
   image: grafana/grafana-build:main
@@ -579,7 +579,7 @@ steps:
     GF_APP_MODE: development
     GF_SERVER_HTTP_PORT: "3001"
     GF_SERVER_ROUTER_LOGGING: "1"
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: grafana-server
 - commands:
   - ./bin/build e2e-tests --port 3001 --suite dashboards-suite
@@ -699,13 +699,15 @@ steps:
   name: test-a11y-frontend
 - commands:
   - docker run --privileged --rm tonistiigi/binfmt --install all
-  - /src/grafana-build docker $(cat packages.txt | grep tar.gz | grep -v docker |
-    grep -v sha256 | awk '{print "--package=" $0}') --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.3
-    --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{
-    .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt
+  - /src/grafana-build artifacts -a docker:grafana:linux/amd64 -a docker:grafana:linux/amd64:ubuntu
+    -a docker:grafana:linux/arm64 -a docker:grafana:linux/arm64:ubuntu --yarn-cache=$$YARN_CACHE_FOLDER
+    --build-id=$$DRONE_BUILD_NUMBER --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.4
+    --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --grafana-dir=$$PWD
+    --ubuntu-tag-format='{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' >
+    docker.txt
   - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i
   depends_on:
-  - rgm-package
+  - yarn-install
   image: grafana/grafana-build:main
   name: rgm-build-docker
   pull: always
@@ -844,7 +846,7 @@ steps:
   name: compile-build-cmd
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - '# It is required that code generated from Thema/CUE be committed and in sync
@@ -1030,7 +1032,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - yarn install --immutable
@@ -1385,7 +1387,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - yarn install --immutable
@@ -1460,7 +1462,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - yarn install --immutable
@@ -1517,7 +1519,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - yarn install --immutable
@@ -1584,7 +1586,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - '# It is required that code generated from Thema/CUE be committed and in sync
@@ -1663,7 +1665,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
@@ -1737,7 +1739,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - mkdir -p bin
@@ -1807,9 +1809,9 @@ steps:
   image: node:20.9.0-alpine
   name: build-frontend-packages
 - commands:
-  - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.10
-    --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD
-    > packages.txt
+  - /src/grafana-build artifacts -a targz:grafana:linux/amd64 -a targz:grafana:linux/arm64
+    --go-version=1.20.10 --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER
+    --grafana-dir=$$PWD > packages.txt
   depends_on:
   - update-package-json-version
   image: grafana/grafana-build:main
@@ -1830,7 +1832,7 @@ steps:
     GF_APP_MODE: development
     GF_SERVER_HTTP_PORT: "3001"
     GF_SERVER_ROUTER_LOGGING: "1"
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: grafana-server
 - commands:
   - ./bin/build e2e-tests --port 3001 --suite dashboards-suite
@@ -1986,13 +1988,15 @@ steps:
     - grafana/grafana
 - commands:
   - docker run --privileged --rm tonistiigi/binfmt --install all
-  - /src/grafana-build docker $(cat packages.txt | grep tar.gz | grep -v docker |
-    grep -v sha256 | awk '{print "--package=" $0}') --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.3
-    --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{
-    .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt
+  - /src/grafana-build artifacts -a docker:grafana:linux/amd64 -a docker:grafana:linux/amd64:ubuntu
+    -a docker:grafana:linux/arm64 -a docker:grafana:linux/arm64:ubuntu --yarn-cache=$$YARN_CACHE_FOLDER
+    --build-id=$$DRONE_BUILD_NUMBER --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.4
+    --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --grafana-dir=$$PWD
+    --ubuntu-tag-format='{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' >
+    docker.txt
   - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i
   depends_on:
-  - rgm-package
+  - yarn-install
   image: grafana/grafana-build:main
   name: rgm-build-docker
   pull: always
@@ -2193,7 +2197,7 @@ steps:
   name: compile-build-cmd
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - '# It is required that code generated from Thema/CUE be committed and in sync
@@ -2506,7 +2510,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - mkdir -p bin
@@ -2838,6 +2842,7 @@ steps:
   environment:
     _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
       from_secret: dagger_token
+    ALPINE_BASE: alpine:3.18.4
     CDN_DESTINATION:
       from_secret: rgm_cdn_destination
     DESTINATION:
@@ -2865,6 +2870,7 @@ steps:
       from_secret: npm_token
     STORYBOOK_DESTINATION:
       from_secret: rgm_storybook_destination
+    UBUNTU_BASE: ubuntu:22.04
   image: grafana/grafana-build:main
   name: rgm-build
   pull: always
@@ -2951,7 +2957,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - yarn install --immutable
@@ -3006,7 +3012,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - '# It is required that code generated from Thema/CUE be committed and in sync
@@ -3087,6 +3093,7 @@ steps:
   environment:
     _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
       from_secret: dagger_token
+    ALPINE_BASE: alpine:3.18.4
     CDN_DESTINATION:
       from_secret: rgm_cdn_destination
     DESTINATION:
@@ -3114,6 +3121,7 @@ steps:
       from_secret: npm_token
     STORYBOOK_DESTINATION:
       from_secret: rgm_storybook_destination
+    UBUNTU_BASE: ubuntu:22.04
   image: grafana/grafana-build:main
   name: rgm-build
   pull: always
@@ -3265,6 +3273,7 @@ steps:
   environment:
     _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
       from_secret: dagger_token
+    ALPINE_BASE: alpine:3.18.4
     CDN_DESTINATION:
       from_secret: rgm_cdn_destination
     DESTINATION:
@@ -3292,6 +3301,7 @@ steps:
       from_secret: npm_token
     STORYBOOK_DESTINATION:
       from_secret: rgm_storybook_destination
+    UBUNTU_BASE: ubuntu:22.04
   image: grafana/grafana-build:main
   name: rgm-build
   pull: always
@@ -3363,7 +3373,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - yarn install --immutable
@@ -3416,7 +3426,7 @@ services: []
 steps:
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - '# It is required that code generated from Thema/CUE be committed and in sync
@@ -3495,6 +3505,7 @@ steps:
   environment:
     _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
       from_secret: dagger_token
+    ALPINE_BASE: alpine:3.18.4
     CDN_DESTINATION:
       from_secret: rgm_cdn_destination
     DESTINATION:
@@ -3522,6 +3533,7 @@ steps:
       from_secret: npm_token
     STORYBOOK_DESTINATION:
       from_secret: rgm_storybook_destination
+    UBUNTU_BASE: ubuntu:22.04
   image: grafana/grafana-build:main
   name: rgm-build
   pull: always
@@ -3639,6 +3651,7 @@ steps:
   environment:
     _EXPERIMENTAL_DAGGER_CLOUD_TOKEN:
       from_secret: dagger_token
+    ALPINE_BASE: alpine:3.18.4
     CDN_DESTINATION:
       from_secret: rgm_cdn_destination
     DESTINATION:
@@ -3666,6 +3679,7 @@ steps:
       from_secret: npm_token
     STORYBOOK_DESTINATION:
       from_secret: rgm_storybook_destination
+    UBUNTU_BASE: ubuntu:22.04
   image: grafana/grafana-build:main
   name: rgm-publish
   pull: always
@@ -3846,7 +3860,7 @@ steps:
   name: grabpl
 - commands:
   - echo $DRONE_RUNNER_NAME
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: identify-runner
 - commands:
   - '# It is required that code generated from Thema/CUE be committed and in sync
@@ -4068,7 +4082,7 @@ steps:
 - commands:
   - if [ -z "${BUILD_CONTAINER_VERSION}" ]; then echo Missing BUILD_CONTAINER_VERSION;
     false; fi
-  image: alpine:3.18.3
+  image: alpine:3.18.4
   name: validate-version
 - commands:
   - printenv GCP_KEY > /tmp/key.json
@@ -4392,7 +4406,7 @@ steps:
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM node:20.9.0-alpine
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM google/cloud-sdk:431.0.0
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-deploy:1.3.3
-  - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine:3.18.3
+  - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine:3.18.4
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM ubuntu:22.04
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM byrnedo/alpine-curl:0.1.8
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM plugins/slack
@@ -4426,7 +4440,7 @@ steps:
   - trivy --exit-code 1 --severity HIGH,CRITICAL node:20.9.0-alpine
   - trivy --exit-code 1 --severity HIGH,CRITICAL google/cloud-sdk:431.0.0
   - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-deploy:1.3.3
-  - trivy --exit-code 1 --severity HIGH,CRITICAL alpine:3.18.3
+  - trivy --exit-code 1 --severity HIGH,CRITICAL alpine:3.18.4
   - trivy --exit-code 1 --severity HIGH,CRITICAL ubuntu:22.04
   - trivy --exit-code 1 --severity HIGH,CRITICAL byrnedo/alpine-curl:0.1.8
   - trivy --exit-code 1 --severity HIGH,CRITICAL plugins/slack
@@ -4670,6 +4684,6 @@ kind: secret
 name: gcr_credentials
 ---
 kind: signature
-hmac: 920b57d70a96a29872ce77dc83ca26e2d141ba37627eee58c9d3beebf9ec4055
+hmac: 88aa054b297d39ae9757ee3a13d16b51491931fabf7c78ba0d0c125a4088da33
 
 ...

scripts/drone/pipelines/build.star

@@ -28,8 +28,8 @@ load(
 )
 load(
     "scripts/drone/steps/rgm.star",
+    "rgm_artifacts_step",
     "rgm_build_docker_step",
-    "rgm_package_step",
 )
 load(
     "scripts/drone/utils/images.star",
@@ -70,14 +70,21 @@ def build_e2e(trigger, ver_mode):
             [
                 build_frontend_package_step(),
                 enterprise_downstream_step(ver_mode = ver_mode),
-                rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt"),
+                rgm_artifacts_step(artifacts = ["targz:grafana:linux/amd64", "targz:grafana:linux/arm64"], file = "packages.txt"),
             ],
         )
     else:
         build_steps.extend([
             update_package_json_version(),
             build_frontend_package_step(depends_on = ["update-package-json-version"]),
-            rgm_package_step(depends_on = ["update-package-json-version"], distros = "linux/amd64,linux/arm64", file = "packages.txt"),
+            rgm_artifacts_step(
+                artifacts = [
+                    "targz:grafana:linux/amd64",
+                    "targz:grafana:linux/arm64",
+                ],
+                depends_on = ["update-package-json-version"],
+                file = "packages.txt",
+            ),
         ])
 
     build_steps.extend(
@@ -104,7 +111,6 @@ def build_e2e(trigger, ver_mode):
                 store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode),
                 frontend_metrics_step(trigger = trigger_oss),
                 rgm_build_docker_step(
-                    "packages.txt",
                     images["ubuntu"],
                     images["alpine"],
                     tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}",
@@ -135,7 +141,6 @@ def build_e2e(trigger, ver_mode):
         build_steps.extend(
             [
                 rgm_build_docker_step(
-                    "packages.txt",
                     images["ubuntu"],
                     images["alpine"],
                     tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}",

scripts/drone/rgm.star

@@ -137,6 +137,8 @@ def rgm_run(name, script):
     """
     env = {
         "GO_VERSION": golang_version,
+        "ALPINE_BASE": images["alpine"],
+        "UBUNTU_BASE": images["ubuntu"],
     }
     rgm_run_step = {
         "name": name,

scripts/drone/steps/rgm.star

@@ -8,15 +8,25 @@ load(
     "golang_version",
 )
 
-# rgm_package_step will create a tar.gz for use in e2e tests or other PR testing related activities..
-def rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt", depends_on = ["yarn-install"]):
+def artifacts_cmd(artifacts = []):
+    cmd = "/src/grafana-build artifacts "
+
+    for artifact in artifacts:
+        cmd += "-a {} ".format(artifact)
+
+    return cmd
+
+# rgm_artifacts_step will create artifacts using the '/src/build artifacts' command.
+def rgm_artifacts_step(name = "rgm-package", artifacts = ["targz:grafana:linux/amd64", "targz:grafana:linux/arm64"], file = "packages.txt", depends_on = ["yarn-install"]):
+    cmd = artifacts_cmd(artifacts = artifacts)
+
     return {
-        "name": "rgm-package",
+        "name": name,
         "image": "grafana/grafana-build:main",
         "pull": "always",
         "depends_on": depends_on,
         "commands": [
-            "/src/grafana-build package --distro={} ".format(distros) +
+            cmd +
             "--go-version={} ".format(golang_version) +
             "--yarn-cache=$$YARN_CACHE_FOLDER " +
             "--build-id=$$DRONE_BUILD_NUMBER " +
@@ -28,31 +38,27 @@ def rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt",
 # rgm_build_backend will create compile the grafana backend for various platforms. It's preferred to use
 # 'rgm_package_step' if you creating a "usable" artifact. This should really only be used to verify that the code is
 # compilable.
-def rgm_build_backend_step(distros = "linux/amd64,linux/arm64"):
-    return {
-        "name": "rgm-package",
-        "image": "grafana/grafana-build:main",
-        "pull": "always",
-        "commands": [
-            "/src/grafana-build build " +
-            "--go-version={} ".format(golang_version) +
-            "--distro={} --grafana-dir=$$PWD".format(distros),
-        ],
-        "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
-    }
+def rgm_build_backend_step(artifacts = ["backend:grafana:linux/amd64", "backend:grafana:linux/arm64"]):
+    return rgm_artifacts_step(name = "rgm-build-backend", artifacts = artifacts, depends_on = [])
 
-def rgm_build_docker_step(packages, ubuntu, alpine, depends_on = ["rgm-package"], file = "docker.txt", tag_format = "{{ .version }}-{{ .arch }}", ubuntu_tag_format = "{{ .version }}-ubuntu-{{ .arch }}"):
+def rgm_build_docker_step(ubuntu, alpine, depends_on = ["yarn-install"], file = "docker.txt", tag_format = "{{ .version }}-{{ .arch }}", ubuntu_tag_format = "{{ .version }}-ubuntu-{{ .arch }}"):
     return {
         "name": "rgm-build-docker",
         "image": "grafana/grafana-build:main",
         "pull": "always",
         "commands": [
             "docker run --privileged --rm tonistiigi/binfmt --install all",
-            "/src/grafana-build docker " +
-            "$(cat {} | grep tar.gz | grep -v docker | grep -v sha256 | awk '{{print \"--package=\" $0}}') ".format(packages) +
+            "/src/grafana-build artifacts " +
+            "-a docker:grafana:linux/amd64 " +
+            "-a docker:grafana:linux/amd64:ubuntu " +
+            "-a docker:grafana:linux/arm64 " +
+            "-a docker:grafana:linux/arm64:ubuntu " +
+            "--yarn-cache=$$YARN_CACHE_FOLDER " +
+            "--build-id=$$DRONE_BUILD_NUMBER " +
             "--ubuntu-base={} ".format(ubuntu) +
             "--alpine-base={} ".format(alpine) +
             "--tag-format='{}' ".format(tag_format) +
+            "--grafana-dir=$$PWD " +
             "--ubuntu-tag-format='{}' > {}".format(ubuntu_tag_format, file),
             "find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i",
         ],

scripts/drone/utils/images.star

@@ -14,7 +14,7 @@ images = {
     "node": "node:{}-alpine".format(nodejs_version),
     "cloudsdk": "google/cloud-sdk:431.0.0",
     "publish": "grafana/grafana-ci-deploy:1.3.3",
-    "alpine": "alpine:3.18.3",
+    "alpine": "alpine:3.18.4",
     "ubuntu": "ubuntu:22.04",
     "curl": "byrnedo/alpine-curl:0.1.8",
     "plugins_slack": "plugins/slack",