Automation: Verify release artifacts on grafana.com (#89197)

* baldm0mma/verify_release/ create verify_release_for_download function

* baldm0mma/verify_release/ add name, image, env

* baldm0mma/verify_release/ add initial commands

* baldm0mma/verify_release/ add deps?

* baldm0mma/verify_release/ update location

* baldm0mma/verify_release/ add anno to lib-star

* bald0mma/verify_release/ update func name to verify_grafanacom_step

* baldm0mma/verify_release/ add verify shell script

* baldm0mma/verify_release/ add script content, first attempt

* baldm0mma/verify_release/ add node image to verify_grafanacom_step

* baldm0mma/verify_release/ add gcom secret note

* baldm0mma/verify_release/ add sudo to apt-get

* baldm0mma/verify_release/ add anno

* baldm0mma/verify_release/ add anno to secrets

* baldm0mma/verify_release/ update commands to reflect node env image

* baldm0mma/verify_release/ update annos

* baldm0mma/verify_release/ update tag variable

* baldm0mma/verify release/ add whitespace

* baldm0mma/verify_releases/ update with no bash loops

* baldm0mma/verify_release/ update exit logic

* baldm0mma/verify_release/ remove annos

* baldm0mma/verify_releasse/ resign and build yml

* baldm0mma/verify_release/ remove annos

* baldm0mma/verify_release/ update signature

* baldm0mma/verify_release/ download curl

* baldm0mma/verify_release/ remove temp key folder removal

* baldm0mma/verify_release/ account for artifact download time

* baldm0mma/verify_release/ add anno

* baldm0mma/verify_release/ update location

* baldm0mma/verify_release/ update script

* baldm0mma/verify_release/ make drone

* baldm0mma/verify_release/ update script for oss or ent

* baldm0mma/verify_release/ add promotion option

* baldm0mma/verify_release/ make drone

* Update scripts/drone/events/release.star

Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com>

* Update scripts/drone/steps/lib.star

Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com>

* Update scripts/drone/steps/lib.star

Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com>

* baldm0mma/verify_release/ update drone

* Update scripts/drone/events/release.star

Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com>

* baldm0mma/verify_release/ update drone

* Update scripts/drone/steps/lib.star

Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com>

* Apply suggestions from code review

Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com>

* baldm0mma/update drone

* baldm0mma/verify_release/ update path

* baldm0mma/verify_release/ make drone

* baldm0mma/update drone

* Apply suggestions from code review

Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com>

* baldm0mma/verify_release/ update for loop to account for failure

* baldm0mma/verify_release/ make drone

* baldm0mma/verify_release/ make format-drone

* baldm0mma/verify_release/ rem unused var

---------

Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com>

.drone.yml

@@ -2921,6 +2921,40 @@ volumes:
     path: /var/run/docker.sock
   name: docker
 ---
+clone:
+  retries: 3
+depends_on: []
+image_pull_secrets:
+- gcr
+- gar
+kind: pipeline
+name: verify-grafanacom-artifacts
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
+- commands:
+  - apk add curl bash
+  - "\n            for i in {1..5}; do\n                if ./scripts/drone/verify-grafanacom.sh;
+    then\n                    exit 0\n                elif [ $i -eq 5 ]; then\n                    exit
+    1\n                else\n                    sleep 60\n                fi\n            done\n
+    \           "
+  depends_on: []
+  image: node:20.9.0-alpine
+  name: verify-grafanacom
+trigger:
+  event:
+  - promote
+  target: verify-grafanacom-artifacts
+type: docker
+volumes:
+- host:
+    path: /var/run/docker.sock
+  name: docker
+---
 clone:
   retries: 3
 depends_on:
@@ -3001,6 +3035,16 @@ steps:
       from_secret: grafana_api_key
   image: grafana/grafana-ci-deploy:1.3.3
   name: publish-grafanacom
+- commands:
+  - apk add curl bash
+  - "\n            for i in {1..5}; do\n                if ./scripts/drone/verify-grafanacom.sh;
+    then\n                    exit 0\n                elif [ $i -eq 5 ]; then\n                    exit
+    1\n                else\n                    sleep 60\n                fi\n            done\n
+    \           "
+  depends_on:
+  - publish-grafanacom
+  image: node:20.9.0-alpine
+  name: verify-grafanacom
 trigger:
   event:
   - promote
@@ -4952,6 +4996,6 @@ kind: secret
 name: gcr_credentials
 ---
 kind: signature
-hmac: ba86e9c1fb16bb20bff8d56f158ea31f32c3e44f6d517a04ae774fc28f9101e7
+hmac: 06f574902baa67d8885abb48e48987f675d7637e30d4b783b3bb84e51b46cdaf
 
 ...

scripts/drone/events/release.star

@@ -21,6 +21,7 @@ load(
     "remote_alertmanager_integration_tests_steps",
     "verify_gen_cue_step",
     "verify_gen_jsonnet_step",
+    "verify_grafanacom_step",
     "wire_install_step",
     "yarn_install_step",
 )
@@ -203,6 +204,7 @@ def publish_packages_pipeline():
         publish_linux_packages_step(package_manager = "deb"),
         publish_linux_packages_step(package_manager = "rpm"),
         publish_grafanacom_step(ver_mode = "release"),
+        verify_grafanacom_step(),
     ]
 
     deps = [
@@ -211,6 +213,16 @@ def publish_packages_pipeline():
     ]
 
     return [
+        pipeline(
+            name = "verify-grafanacom-artifacts",
+            trigger = {
+                "event": ["promote"],
+                "target": "verify-grafanacom-artifacts",
+            },
+            steps = [
+                verify_grafanacom_step(depends_on = []),
+            ],
+        ),
         pipeline(
             name = "publish-packages",
             trigger = trigger,

scripts/drone/steps/lib.star

@@ -1152,6 +1152,34 @@ def publish_grafanacom_step(ver_mode):
         ],
     }
 
+def verify_grafanacom_step(depends_on = ["publish-grafanacom"]):
+    return {
+        "name": "verify-grafanacom",
+        "image": images["node"],
+        "commands": [
+            # Download and install `curl` and `bash` - both of which aren't available inside of the `node:{version}-alpine` docker image.
+            "apk add curl bash",
+
+            # There may be a slight lag between when artifacts are uploaded to Google Storage,
+            # and when they become available on the website. This `for` loop sould account for that discrepancy.
+            # We attempt the verification up to 5 times. If successful, exit the loop with a success (0) status.
+            # If any attempt fails, but it's not the final attempt, wait 60 seconds before the next attempt.
+            # If the 5th (final) attempt fails, exit with error (1) status.
+            """
+            for i in {1..5}; do
+                if ./scripts/drone/verify-grafanacom.sh; then
+                    exit 0
+                elif [ $i -eq 5 ]; then
+                    exit 1
+                else
+                    sleep 60
+                fi
+            done
+            """,
+        ],
+        "depends_on": depends_on,
+    }
+
 def publish_linux_packages_step(package_manager = "deb"):
     return {
         "name": "publish-linux-packages-{}".format(package_manager),

scripts/drone/verify-grafanacom.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+version=${1:-$TAG}
+
+# Construct the URL based on the provided version and edition
+if [ "$EDITION" = "enterprise" ]; then
+    url="https://grafana.com/api/downloads/grafana-enterprise/versions/$version"
+else
+    url="https://grafana.com/api/downloads/grafana/versions/$version"
+fi
+
+# Make a request to the GCOM API to retrieve the artifacts for the specified version. Exit if the request fails.
+if ! artifacts=$(curl "$url"); then
+  echo "Failed to retrieve artifact URLs from Grafana.com API. Please check the API key, authentication, edition, and version."
+  exit 1
+fi
+
+# Use Node.js to parse the JSON response and extract the download URLs
+url_string=$(node -e "
+  const artifacts = JSON.parse(JSON.stringify($artifacts));
+  const downloadUrls = artifacts.packages.map((package) => package.links.find((link) => link.rel === 'download').href);
+  console.log(downloadUrls.join(' '));
+")
+
+# Convert the url_string to a Bash array
+read -r -a urls <<< "$url_string"
+
+# If empty, no artifact URLs were found for the specified version. Exit with an error.
+if [ ${#urls[@]} -eq 0 ]; then
+  echo "No artifact URLs found for version $version. Please check the provided version."
+  exit 1
+fi
+
+# Iterate over the URLs and check the status code of each. If any URL does not return a 200 status code, add it to the failed_urls string.
+failed_urls=""
+for url in "${urls[@]}"; do
+  status_code=$(curl -L -s -o /dev/null -w "%{http_code}" "$url")
+  if [ "$status_code" -ne 200 ]; then
+    failed_urls+="$url\n"
+  fi
+done
+
+# If any URLs failed, print them and exit with an error.
+if [ -n "$failed_urls" ]; then
+  echo "The following URLs did not return a 200 status code:"
+  echo "$failed_urls"
+  exit 1
+else
+  echo "All URLs returned a 200 status code. Download links are valid for version $version."
+  exit 0
+fi