CircleCI: Cache packages in GCS instead of in workspace (#23130)

* CircleCI: Cache packages in GCS instead of workspace
* CircleCI: Cache yarn packages
* CircleCI: Consolidate on --frozen-lockfile

.circleci/config.yml

@@ -40,6 +40,19 @@ executors:
     docker:
       - image: grafana/wix-toolset-ci:v3
 
+commands:
+  install-google-cloud-sdk:
+    description: "Install Google Cloud SDK"
+    steps:
+      - run:
+          name: "Install Google Cloud SDK"
+          command: |
+            VERSION=286.0.0
+            CHECKSUM=f199e38c988d53a913fc780b6f57ebda72db5f3848073cea805b39fe3e8e34fd
+            curl -fLO https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${VERSION}-linux-x86_64.tar.gz
+            echo $CHECKSUM google-cloud-sdk-${VERSION}-linux-x86_64.tar.gz | sha256sum --check --status
+            tar xf google-cloud-sdk-${VERSION}-linux-x86_64.tar.gz
+
 jobs:
   install-grabpl:
     description: Install the Grafana Build Pipeline tool
@@ -48,7 +61,7 @@ jobs:
       - run:
           name: Install Grafana Build Pipeline
           command: |
-            curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.2.5/grabpl
+            curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.2.7/grabpl
             chmod +x grabpl
             mkdir bin
             mv grabpl bin/
@@ -129,6 +142,13 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
+      - restore_cache:
+          keys:
+            # Use own cache for front-end builds, since for some reason it can't be unpacked for parts of the
+            # pipeline
+            - v1-yarn-build-{{ checksum "yarn.lock" }}
+            # Used if checksum fails
+            - v1-yarn-build-
       - run:
           name: Increase node memory
           command: |
@@ -136,6 +156,13 @@ jobs:
       - run:
           name: CI job started
           command: ./scripts/ci-job-started.sh
+      - run:
+          name: Install dependencies
+          command: yarn install --frozen-lockfile --no-progress
+      - save_cache:
+          key: v1-yarn-build-{{ checksum "yarn.lock" }}
+          paths:
+            - node_modules
       - run:
           name: Build Grafana frontend
           command: /tmp/workspace/bin/grabpl build-frontend --edition << parameters.edition >>
@@ -212,20 +239,29 @@ jobs:
       - run:
           name: CI job started
           command: ./scripts/ci-job-started.sh
+      - install-google-cloud-sdk
       - run:
-          name: Copy artifacts from workspace
-          command: cp -r /tmp/workspace/oss/dist .
+          name: Download packages from GCS
+          command: |
+            # Work around an issue with the gsutil installation
+            export BOTO_CONFIG=/dev/null
+
+            echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json
+            ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json
+            mkdir dist
+            ./google-cloud-sdk/bin/gsutil -m rsync -d -r gs://grafana-build-pipeline/oss/packages/${CIRCLE_WORKFLOW_ID}/ dist/
       - run:
           name: Build Windows installer
           command: ./scripts/build/ci-msi-build/ci-msi-build-oss.sh
       - run:
-          name: Move artifacts
-          command: mkdir oss && mv dist oss/
-      - persist_to_workspace:
-          root: .
-          paths:
-            - oss/dist/grafana-*.msi
-            - oss/dist/grafana-*.msi.sha256
+          name: Upload installer to GCS
+          command: |
+            # Work around an issue with the gsutil installation
+            export BOTO_CONFIG=/dev/null
+
+            echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json
+            ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json
+            ./google-cloud-sdk/bin/gsutil -m cp dist/grafana-*.msi* gs://grafana-build-pipeline/oss/packages/${CIRCLE_WORKFLOW_ID}/
       - run:
           name: CI job failed
           command: ./scripts/ci-job-failed.sh
@@ -251,20 +287,29 @@ jobs:
       - run:
           name: CI job started
           command: "./scripts/ci-job-started.sh"
+      - install-google-cloud-sdk
       - run:
-          name: Copy artifacts from workspace
-          command: cp -r /tmp/workspace/enterprise/dist enterprise-dist
+          name: Download packages from GCS
+          command: |
+            # Work around an issue with the gsutil installation
+            export BOTO_CONFIG=/dev/null
+
+            echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json
+            ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json
+            mkdir enterprise-dist
+            ./google-cloud-sdk/bin/gsutil -m rsync -d -r gs://grafana-build-pipeline/enterprise/packages/${CIRCLE_WORKFLOW_ID}/ enterprise-dist/
       - run:
           name: Build Windows installer
           command: ./scripts/build/ci-msi-build/ci-msi-build-ee.sh
       - run:
-          name: Move artifacts
-          command: mkdir enterprise && mv enterprise-dist enterprise/dist
-      - persist_to_workspace:
-          root: .
-          paths:
-            - enterprise/dist/grafana-*.msi
-            - enterprise/dist/grafana-*.msi.sha256
+          name: Upload installer to GCS
+          command: |
+            # Work around an issue with the gsutil installation
+            export BOTO_CONFIG=/dev/null
+
+            echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json
+            ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json
+            ./google-cloud-sdk/bin/gsutil -m cp enterprise-dist/grafana-*.msi* gs://grafana-build-pipeline/enterprise/packages/${CIRCLE_WORKFLOW_ID}/
       - run:
           name: CI job failed
           command: "./scripts/ci-job-failed.sh"
@@ -317,6 +362,7 @@ jobs:
       - run:
           name: CI job started
           command: ./scripts/ci-job-started.sh
+      - install-google-cloud-sdk
       - run:
           name: Copy artifacts from workspace
           command: cp -r /tmp/workspace/oss/* .
@@ -343,14 +389,14 @@ jobs:
                 linux-x64,linux-x64-musl,osx64,win64
             fi
       - run:
-          name: Move artifacts
+          name: Upload packages to GCS
           command: |
-            mkdir -p oss
-            mv dist oss/
-      - persist_to_workspace:
-          root: .
-          paths:
-            - oss/dist/*
+            # Work around an issue with the gsutil installation
+            export BOTO_CONFIG=/dev/null
+
+            echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json
+            ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json
+            ./google-cloud-sdk/bin/gsutil -m rsync -r dist gs://grafana-build-pipeline/oss/packages/${CIRCLE_WORKFLOW_ID}
       - run:
           name: CI job failed
           command: ./scripts/ci-job-failed.sh
@@ -376,6 +422,7 @@ jobs:
       - run:
           name: CI job started
           command: ./scripts/ci-job-started.sh
+      - install-google-cloud-sdk
       - run:
           name: Copy artifacts from workspace
           command: cp -r /tmp/workspace/enterprise/* .
@@ -397,14 +444,14 @@ jobs:
                 linux-x64,linux-x64-musl,osx64,win64
             fi
       - run:
-          name: Move artifacts
+          name: Upload packages to GCS
           command: |
-            mkdir -p enterprise
-            mv dist enterprise/
-      - persist_to_workspace:
-          root: .
-          paths:
-            - enterprise/dist/*
+            # Work around an issue with the gsutil installation
+            export BOTO_CONFIG=/dev/null
+
+            echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json
+            ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json
+            ./google-cloud-sdk/bin/gsutil -m rsync -r dist gs://grafana-build-pipeline/enterprise/packages/${CIRCLE_WORKFLOW_ID}
       - run:
           name: CI job failed
           command: ./scripts/ci-job-failed.sh
@@ -434,10 +481,20 @@ jobs:
       - run:
           name: CI job started
           command: ./scripts/ci-job-started.sh
+      - install-google-cloud-sdk
+      - run:
+          name: Download packages from GCS
+          command: |
+            # Work around an issue with the gsutil installation
+            export BOTO_CONFIG=/dev/null
+
+            echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json
+            ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json
+            mkdir dist
+            ./google-cloud-sdk/bin/gsutil -m rsync -d -r gs://grafana-build-pipeline/<< parameters.edition >>/packages/${CIRCLE_WORKFLOW_ID}/ dist/
       - run:
           name: Publish packages
           command: |
-            cp -r /tmp/workspace/<< parameters.edition >>/dist .
             if [[ $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then
               # We're testing the release pipeline
               /tmp/workspace/bin/grabpl publish-packages --edition << parameters.edition >> \
@@ -518,13 +575,20 @@ jobs:
       - run:
           name: CI job started
           command: ./scripts/ci-job-started.sh
+      - install-google-cloud-sdk
+      - run:
+          name: Download packages from GCS
+          command: |
+            # Work around an issue with the gsutil installation
+            export BOTO_CONFIG=/dev/null
+
+            echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json
+            ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json
+            mkdir dist
+            ./google-cloud-sdk/bin/gsutil -m rsync -d -r gs://grafana-build-pipeline/<< parameters.edition >>/packages/${CIRCLE_WORKFLOW_ID}/ dist/
       - run: docker info
       # XXX: Is this necessary?
       - run: docker run --privileged linuxkit/binfmt:v0.6
-      - run:
-          name: Copy Grafana archives
-          command: |
-            cp -r /tmp/workspace/<< parameters.edition >>/dist .
       - run:
           name: Build Docker images
           command: |
@@ -569,14 +633,28 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
+      - install-google-cloud-sdk
+      - run:
+          name: Download packages from GCS
+          command: |
+            # Work around an issue with the gsutil installation
+            export BOTO_CONFIG=/dev/null
+
+            echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json
+            ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json
+            mkdir dist
+            ./google-cloud-sdk/bin/gsutil -m rsync -d -r gs://grafana-build-pipeline/oss/packages/${CIRCLE_WORKFLOW_ID}/ dist/
       - restore_cache:
-          key: dependency-cache-{{ checksum "yarn.lock" }}
+          keys:
+            - v2-yarn-{{ checksum "yarn.lock" }}
+            # Used if checksum fails
+            - v2-yarn-
       - run:
           name: yarn install
-          command: yarn install --pure-lockfile --no-progress
+          command: yarn install --frozen-lockfile --no-progress
           no_output_timeout: 5m
       - save_cache:
-          key: dependency-cache-{{ checksum "yarn.lock" }}
+          key: v2-yarn-{{ checksum "yarn.lock" }}
           paths:
             - node_modules
       - run:
@@ -718,13 +796,16 @@ jobs:
           name: CI job started
           command: "./scripts/ci-job-started.sh"
       - restore_cache:
-          key: dependency-cache-{{ checksum "yarn.lock" }}
+          keys:
+            - v2-yarn-{{ checksum "yarn.lock" }}
+            # Used if checksum fails
+            - v2-yarn-
       - run:
           name: yarn install
           command: "yarn install --frozen-lockfile --no-progress"
           no_output_timeout: 15m
       - save_cache:
-          key: dependency-cache-{{ checksum "yarn.lock" }}
+          key: v2-yarn-{{ checksum "yarn.lock" }}
           paths:
             - node_modules
       - run:

e2e/start-server

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -eo pipefail
 
 . e2e/variables
 

e2e/variables

@@ -2,6 +2,6 @@
 
 RUNDIR=e2e/tmp
 PIDFILE=$RUNDIR/pid
-PACKAGE_FILE=/tmp/workspace/oss/dist/grafana-*linux-amd64.tar.gz
+PACKAGE_FILE=dist/grafana-*linux-amd64.tar.gz
 PROV_DIR=$RUNDIR/conf/provisioning
 PORT=3001