CI: Add new RRC pipeline (#90731)

* CI: Add new RRC pipeline

.drone.star

@@ -17,6 +17,7 @@ load(
     "publish_npm_pipelines",
     "publish_packages_pipeline",
 )
+load("scripts/drone/events/rrc-patch.star", "rrc_patch_pipelines")
 load(
     "scripts/drone/pipelines/ci_images.star",
     "publish_ci_windows_test_image_pipeline",
@@ -39,6 +40,7 @@ def main(_ctx):
     return (
         pr_pipelines() +
         main_pipelines() +
+        rrc_patch_pipelines() +
         publish_image_pipelines_public() +
         publish_artifacts_pipelines("public") +
         publish_npm_pipelines() +

.drone.yml

@@ -2814,6 +2814,549 @@ image_pull_secrets:
 - gcr
 - gar
 kind: pipeline
+name: rrc-test-frontend
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
+- commands:
+  - echo $DRONE_RUNNER_NAME
+  image: alpine:3.19.1
+  name: identify-runner
+- commands:
+  - yarn install --immutable || yarn install --immutable
+  depends_on: []
+  image: node:20.9.0-alpine
+  name: yarn-install
+- commands:
+  - apk add --update git bash
+  - yarn betterer ci
+  depends_on:
+  - yarn-install
+  image: node:20.9.0-alpine
+  name: betterer-frontend
+- commands:
+  - yarn run ci:test-frontend
+  depends_on:
+  - yarn-install
+  environment:
+    TEST_MAX_WORKERS: 50%
+  image: node:20.9.0-alpine
+  name: test-frontend
+trigger:
+  branch:
+  - instant
+  - fast
+  - steady
+  - slow
+  ref:
+    include:
+    - refs/tags/rrc*
+type: docker
+volumes:
+- host:
+    path: /var/run/docker.sock
+  name: docker
+---
+clone:
+  retries: 3
+depends_on: []
+environment:
+  EDITION: oss
+image_pull_secrets:
+- gcr
+- gar
+kind: pipeline
+name: rrc-lint-frontend
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
+- commands:
+  - echo $DRONE_RUNNER_NAME
+  image: alpine:3.19.1
+  name: identify-runner
+- commands:
+  - yarn install --immutable || yarn install --immutable
+  depends_on: []
+  image: node:20.9.0-alpine
+  name: yarn-install
+- commands:
+  - yarn run prettier:check
+  - yarn run lint
+  - yarn run typecheck
+  depends_on:
+  - yarn-install
+  environment:
+    TEST_MAX_WORKERS: 50%
+  image: node:20.9.0-alpine
+  name: lint-frontend
+- commands:
+  - |-
+    make i18n-extract || (echo "
+    Extraction failed. Make sure that you have no dynamic translation phrases, such as 't(\`preferences.theme.\$${themeID}\`, themeName)' and that no translation key is used twice. Search the output for '[warning]' to find the offending file." && false)
+  - "\n            file_diff=$(git diff --dirstat public/locales)\n            if
+    [ -n \"$file_diff\" ]; then\n                echo $file_diff\n                echo
+    \"\nTranslation extraction has not been committed. Please run 'make i18n-extract',
+    commit the changes and push again.\"\n                exit 1\n            fi\n
+    \           "
+  depends_on:
+  - yarn-install
+  image: node:20-bookworm
+  name: verify-i18n
+trigger:
+  branch:
+  - instant
+  - fast
+  - steady
+  - slow
+  ref:
+    include:
+    - refs/tags/rrc*
+type: docker
+volumes:
+- host:
+    path: /var/run/docker.sock
+  name: docker
+---
+clone:
+  retries: 3
+depends_on: []
+environment:
+  EDITION: oss
+image_pull_secrets:
+- gcr
+- gar
+kind: pipeline
+name: rrc-test-backend
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
+- commands:
+  - echo $DRONE_RUNNER_NAME
+  image: alpine:3.19.1
+  name: identify-runner
+- commands:
+  - '# It is required that code generated from Thema/CUE be committed and in sync
+    with its inputs.'
+  - '# The following command will fail if running code generators produces any diff
+    in output.'
+  - apk add --update make
+  - CODEGEN_VERIFY=1 make gen-cue
+  depends_on: []
+  image: golang:1.22.4-alpine
+  name: verify-gen-cue
+- commands:
+  - '# It is required that generated jsonnet is committed and in sync with its inputs.'
+  - '# The following command will fail if running code generators produces any diff
+    in output.'
+  - apk add --update make
+  - CODEGEN_VERIFY=1 make gen-jsonnet
+  depends_on: []
+  image: golang:1.22.4-alpine
+  name: verify-gen-jsonnet
+- commands:
+  - apk add --update make
+  - make gen-go
+  depends_on:
+  - verify-gen-cue
+  image: golang:1.22.4-alpine
+  name: wire-install
+- commands:
+  - apk add --update build-base shared-mime-info shared-mime-info-lang
+  - go list -f '{{.Dir}}/...' -m | xargs go test -tags requires_buildifer -short -covermode=atomic
+    -timeout=5m
+  depends_on:
+  - wire-install
+  image: golang:1.22.4-alpine
+  name: test-backend
+- commands:
+  - apk add --update build-base
+  - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
+  depends_on:
+  - wire-install
+  image: golang:1.22.4-alpine
+  name: test-backend-integration
+trigger:
+  branch:
+  - instant
+  - fast
+  - steady
+  - slow
+  ref:
+    include:
+    - refs/tags/rrc*
+type: docker
+volumes:
+- host:
+    path: /var/run/docker.sock
+  name: docker
+---
+clone:
+  retries: 3
+depends_on: []
+environment:
+  EDITION: oss
+image_pull_secrets:
+- gcr
+- gar
+kind: pipeline
+name: rrc-lint-backend
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
+- commands:
+  - echo $DRONE_RUNNER_NAME
+  image: alpine:3.19.1
+  name: identify-runner
+- commands:
+  - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
+  depends_on: []
+  environment:
+    CGO_ENABLED: 0
+  image: golang:1.22.4-alpine
+  name: compile-build-cmd
+- commands:
+  - apk add --update make
+  - make gen-go
+  depends_on: []
+  image: golang:1.22.4-alpine
+  name: wire-install
+- commands:
+  - go run scripts/modowners/modowners.go check go.mod
+  image: golang:1.22.4-alpine
+  name: validate-modfile
+- commands:
+  - apk add --update make
+  - make swagger-validate
+  image: golang:1.22.4-alpine
+  name: validate-openapi-spec
+trigger:
+  branch:
+  - instant
+  - fast
+  - steady
+  - slow
+  ref:
+    include:
+    - refs/tags/rrc*
+type: docker
+volumes:
+- host:
+    path: /var/run/docker.sock
+  name: docker
+---
+clone:
+  retries: 3
+depends_on: []
+environment:
+  EDITION: oss
+image_pull_secrets:
+- gcr
+- gar
+kind: pipeline
+name: rrc-integration-tests
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services:
+- environment:
+    PGDATA: /var/lib/postgresql/data/pgdata
+    POSTGRES_DB: grafanatest
+    POSTGRES_PASSWORD: grafanatest
+    POSTGRES_USER: grafanatest
+  image: postgres:12.3-alpine
+  name: postgres
+  volumes:
+  - name: postgres
+    path: /var/lib/postgresql/data/pgdata
+- commands:
+  - docker-entrypoint.sh mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
+  environment:
+    MYSQL_DATABASE: grafana_tests
+    MYSQL_PASSWORD: password
+    MYSQL_ROOT_PASSWORD: rootpass
+    MYSQL_USER: grafana
+  image: mysql:5.7.39
+  name: mysql57
+  volumes:
+  - name: mysql57
+    path: /var/lib/mysql
+- commands:
+  - docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password
+  environment:
+    MYSQL_DATABASE: grafana_tests
+    MYSQL_PASSWORD: password
+    MYSQL_ROOT_PASSWORD: rootpass
+    MYSQL_USER: grafana
+  image: mysql:8.0.32
+  name: mysql80
+  volumes:
+  - name: mysql80
+    path: /var/lib/mysql
+- commands:
+  - /bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled
+  environment: {}
+  image: grafana/mimir-alpine:r295-a23e559
+  name: mimir_backend
+- environment: {}
+  image: redis:6.2.11-alpine
+  name: redis
+- environment: {}
+  image: memcached:1.6.9-alpine
+  name: memcached
+steps:
+- commands:
+  - mkdir -p bin
+  - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.50/grabpl
+  - chmod +x bin/grabpl
+  image: byrnedo/alpine-curl:0.1.8
+  name: grabpl
+- commands:
+  - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
+  depends_on: []
+  environment:
+    CGO_ENABLED: 0
+  image: golang:1.22.4-alpine
+  name: compile-build-cmd
+- commands:
+  - echo $DRONE_RUNNER_NAME
+  image: alpine:3.19.1
+  name: identify-runner
+- commands:
+  - '# It is required that code generated from Thema/CUE be committed and in sync
+    with its inputs.'
+  - '# The following command will fail if running code generators produces any diff
+    in output.'
+  - apk add --update make
+  - CODEGEN_VERIFY=1 make gen-cue
+  depends_on: []
+  image: golang:1.22.4-alpine
+  name: verify-gen-cue
+- commands:
+  - '# It is required that generated jsonnet is committed and in sync with its inputs.'
+  - '# The following command will fail if running code generators produces any diff
+    in output.'
+  - apk add --update make
+  - CODEGEN_VERIFY=1 make gen-jsonnet
+  depends_on: []
+  image: golang:1.22.4-alpine
+  name: verify-gen-jsonnet
+- commands:
+  - apk add --update make
+  - make gen-go
+  depends_on:
+  - verify-gen-cue
+  image: golang:1.22.4-alpine
+  name: wire-install
+- commands:
+  - dockerize -wait tcp://postgres:5432 -timeout 120s
+  image: jwilder/dockerize:0.6.1
+  name: wait-for-postgres
+- commands:
+  - apk add --update build-base
+  - apk add --update postgresql-client
+  - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
+  - go clean -testcache
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
+  depends_on:
+  - wire-install
+  - wait-for-postgres
+  environment:
+    GRAFANA_TEST_DB: postgres
+    PGPASSWORD: grafanatest
+    POSTGRES_HOST: postgres
+  image: golang:1.22.4-alpine
+  name: postgres-integration-tests
+- commands:
+  - dockerize -wait tcp://mysql57:3306 -timeout 120s
+  image: jwilder/dockerize:0.6.1
+  name: wait-for-mysql-5.7
+- commands:
+  - apk add --update build-base
+  - apk add --update mysql-client
+  - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root
+    -prootpass
+  - go clean -testcache
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
+  depends_on:
+  - wire-install
+  - wait-for-mysql-5.7
+  environment:
+    GRAFANA_TEST_DB: mysql
+    MYSQL_HOST: mysql57
+  image: golang:1.22.4-alpine
+  name: mysql-5.7-integration-tests
+- commands:
+  - dockerize -wait tcp://mysql80:3306 -timeout 120s
+  image: jwilder/dockerize:0.6.1
+  name: wait-for-mysql-8.0
+- commands:
+  - apk add --update build-base
+  - apk add --update mysql-client
+  - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root
+    -prootpass
+  - go clean -testcache
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
+  depends_on:
+  - wire-install
+  - wait-for-mysql-8.0
+  environment:
+    GRAFANA_TEST_DB: mysql
+    MYSQL_HOST: mysql80
+  image: golang:1.22.4-alpine
+  name: mysql-8.0-integration-tests
+- commands:
+  - dockerize -wait tcp://redis:6379 -timeout 120s
+  image: jwilder/dockerize:0.6.1
+  name: wait-for-redis
+- commands:
+  - apk add --update build-base
+  - go clean -testcache
+  - go list -f '{{.Dir}}/...' -m | xargs go test -run IntegrationRedis -covermode=atomic
+    -timeout=2m
+  depends_on:
+  - wire-install
+  - wait-for-redis
+  environment:
+    REDIS_URL: redis://redis:6379/0
+  image: golang:1.22.4-alpine
+  name: redis-integration-tests
+- commands:
+  - dockerize -wait tcp://memcached:11211 -timeout 120s
+  image: jwilder/dockerize:0.6.1
+  name: wait-for-memcached
+- commands:
+  - apk add --update build-base
+  - go clean -testcache
+  - go list -f '{{.Dir}}/...' -m | xargs go test -run IntegrationMemcached -covermode=atomic
+    -timeout=2m
+  depends_on:
+  - wire-install
+  - wait-for-memcached
+  environment:
+    MEMCACHED_HOSTS: memcached:11211
+  image: golang:1.22.4-alpine
+  name: memcached-integration-tests
+- commands:
+  - dockerize -wait tcp://mimir_backend:8080 -timeout 120s
+  image: jwilder/dockerize:0.6.1
+  name: wait-for-remote-alertmanager
+- commands:
+  - apk add --update build-base
+  - go clean -testcache
+  - go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/...
+  depends_on:
+  - wire-install
+  - wait-for-remote-alertmanager
+  environment:
+    AM_TENANT_ID: test
+    AM_URL: http://mimir_backend:8080
+  failure: ignore
+  image: golang:1.22.4-alpine
+  name: remote-alertmanager-integration-tests
+trigger:
+  branch:
+  - instant
+  - fast
+  - steady
+  - slow
+  ref:
+    include:
+    - refs/tags/rrc*
+type: docker
+volumes:
+- host:
+    path: /var/run/docker.sock
+  name: docker
+- name: postgres
+  temp:
+    medium: memory
+- name: mysql57
+  temp:
+    medium: memory
+- name: mysql80
+  temp:
+    medium: memory
+---
+clone:
+  retries: 3
+depends_on:
+- rrc-integration-tests
+environment:
+  EDITION: oss
+image_pull_secrets:
+- gcr
+- gar
+kind: pipeline
+name: rrc-trigger-downstream
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
+- image: grafana/drone-downstream
+  name: trigger-enterprise-downstream
+  settings:
+    params:
+    - SOURCE_BUILD_NUMBER=${DRONE_COMMIT}
+    - SOURCE_COMMIT=${DRONE_COMMIT}
+    - SOURCE_TAG=${DRONE_TAG}
+    repositories:
+    - grafana/grafana-enterprise@${DRONE_SOURCE_BRANCH}
+    server: https://drone.grafana.net
+    token:
+      from_secret: drone_token
+trigger:
+  branch:
+  - instant
+  - fast
+  - steady
+  - slow
+  ref:
+    include:
+    - refs/tags/rrc*
+type: docker
+volumes:
+- host:
+    path: /var/run/docker.sock
+  name: docker
+---
+clone:
+  retries: 3
+depends_on: []
+environment:
+  EDITION: oss
+image_pull_secrets:
+- gcr
+- gar
+kind: pipeline
 name: publish-docker-public
 node:
   type: no-parallel
@@ -5508,6 +6051,6 @@ kind: secret
 name: gcr_credentials
 ---
 kind: signature
-hmac: f82713cb6d029eded29632a21a394d16f0778462a6cd7de647422f34cd1fae39
+hmac: 9abbec7da57a95a1be56c254db4c1e473bcdd5c9564cca6909d99c70d15dbc1d
 
 ...

scripts/drone/events/rrc-patch.star

@@ -0,0 +1,73 @@
+"""
+This module returns all the pipelines used in the event of pushes to an RRC branch.
+"""
+
+load(
+    "scripts/drone/pipelines/integration_tests.star",
+    "integration_tests",
+)
+load(
+    "scripts/drone/pipelines/lint_backend.star",
+    "lint_backend_pipeline",
+)
+load(
+    "scripts/drone/pipelines/lint_frontend.star",
+    "lint_frontend_pipeline",
+)
+load(
+    "scripts/drone/pipelines/test_backend.star",
+    "test_backend",
+)
+load(
+    "scripts/drone/pipelines/test_frontend.star",
+    "test_frontend",
+)
+load(
+    "scripts/drone/steps/lib.star",
+    "enterprise_downstream_step",
+)
+load(
+    "scripts/drone/utils/utils.star",
+    "pipeline",
+)
+
+ver_mode = "rrc"
+trigger = {
+    "ref": {
+        "include": [
+            "refs/tags/rrc*",
+        ],
+    },
+    "branch": [
+        "instant",
+        "fast",
+        "steady",
+        "slow",
+    ],
+}
+
+def rrc_patch_pipelines():
+    pipelines = [
+        test_frontend(trigger, ver_mode),
+        lint_frontend_pipeline(trigger, ver_mode),
+        test_backend(trigger, ver_mode),
+        lint_backend_pipeline(trigger, ver_mode),
+        integration_tests(trigger, prefix = ver_mode, ver_mode = ver_mode),
+        rrc_enterprise_downstream_pipeline(trigger = trigger),
+    ]
+
+    return pipelines
+
+def rrc_enterprise_downstream_pipeline(trigger):
+    # Triggers a downstream pipeline in the grafana-enterprise repository for the rrc branch
+    environment = {"EDITION": "oss"}
+    steps = [
+        enterprise_downstream_step(ver_mode = ver_mode),
+    ]
+    return pipeline(
+        name = "rrc-trigger-downstream",
+        trigger = trigger,
+        steps = steps,
+        depends_on = ["rrc-integration-tests"],
+        environment = environment,
+    )

scripts/drone/steps/lib.star

@@ -172,7 +172,7 @@ def enterprise_downstream_step(ver_mode):
       Drone step.
     """
     repo = "grafana/grafana-enterprise@"
-    if ver_mode == "pr":
+    if ver_mode == "pr" or ver_mode == "rrc":
         repo += "${DRONE_SOURCE_BRANCH}"
     else:
         repo += "main"
@@ -197,6 +197,9 @@ def enterprise_downstream_step(ver_mode):
         step.update({"failure": "ignore"})
         step["settings"]["params"].append("OSS_PULL_REQUEST=${DRONE_PULL_REQUEST}")
 
+    if ver_mode == "rrc":
+        step["settings"]["params"].append("SOURCE_TAG=${DRONE_TAG}")
+
     return step
 
 def validate_modfile_step():