From 66c728d26bbe34b28bf62703c448ef539e4f9299 Mon Sep 17 00:00:00 2001 From: Kevin Minehart <5140827+kminehart@users.noreply.github.com> Date: Wed, 23 Oct 2024 14:26:52 -0500 Subject: [PATCH] CI: use linux to build msi installers (#95215) * Build the MSI installers using Linux and wine --- .drone.star | 18 +- .drone.yml | 432 +++++++++++------------- scripts/drone/events/main.star | 8 +- scripts/drone/pipelines/ci_images.star | 66 ---- scripts/drone/pipelines/windows.star | 85 ----- scripts/drone/rgm.star | 34 +- scripts/drone/steps/lib_windows.star | 187 ---------- scripts/drone/utils/images.star | 1 + scripts/drone/utils/windows_images.star | 17 - scripts/drone/variables.star | 2 +- scripts/drone/windows.star | 149 ++++++++ 11 files changed, 361 insertions(+), 638 deletions(-) delete mode 100644 scripts/drone/pipelines/ci_images.star delete mode 100644 scripts/drone/pipelines/windows.star delete mode 100644 scripts/drone/steps/lib_windows.star delete mode 100644 scripts/drone/utils/windows_images.star create mode 100644 scripts/drone/windows.star diff --git a/.drone.star b/.drone.star index 30819f32036..14575d5693d 100644 --- a/.drone.star +++ b/.drone.star @@ -18,23 +18,19 @@ load( "publish_packages_pipeline", ) load("scripts/drone/events/rrc-patch.star", "rrc_patch_pipelines") -load( - "scripts/drone/pipelines/ci_images.star", - "publish_ci_windows_test_image_pipeline", -) load( "scripts/drone/pipelines/publish_images.star", "publish_image_pipelines_public", ) -load( - "scripts/drone/pipelines/windows.star", - "windows_test_backend", -) load( "scripts/drone/rgm.star", "rgm", ) load("scripts/drone/vault.star", "secrets") +load( + "scripts/drone/windows.star", + "windows_manual_pipeline", +) def main(_ctx): return ( @@ -46,12 +42,8 @@ def main(_ctx): publish_npm_pipelines() + publish_packages_pipeline() + rgm() + - [windows_test_backend({ - "event": ["promote"], - "target": ["test-windows"], - }, "oss", "testing")] + integration_test_pipelines() + - publish_ci_windows_test_image_pipeline() + cronjobs() + + [windows_manual_pipeline()] + secrets() ) diff --git a/.drone.yml b/.drone.yml index 14b2beaaf57..7954c89dbd1 100644 --- a/.drone.yml +++ b/.drone.yml @@ -539,7 +539,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -995,7 +995,7 @@ steps: name: clone-enterprise - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1957,7 +1957,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2510,7 +2510,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2692,53 +2692,6 @@ volumes: clone: retries: 3 depends_on: -- main-test-frontend -- main-test-backend -- main-build-e2e-publish -- main-integration-tests -environment: - EDITION: oss -image_pull_secrets: -- gcr -- gar -kind: pipeline -name: main-windows -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner -- commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/windows/grabpl.exe - -OutFile grabpl.exe - image: grafana/ci-wix:0.1.1 - name: windows-init -trigger: - branch: main - event: - - push - paths: - exclude: - - '*.md' - - docs/** - - latest.json - repo: - - grafana/grafana -type: docker -volumes: -- host: - path: //./pipe/docker_engine/ - name: docker ---- -clone: - retries: 3 -depends_on: - main-build-e2e-publish - main-integration-tests environment: @@ -2790,7 +2743,6 @@ depends_on: - main-test-backend - main-build-e2e-publish - main-integration-tests -- main-windows kind: pipeline name: main-notify platform: @@ -3142,7 +3094,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3387,7 +3339,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3519,7 +3471,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -4265,51 +4217,6 @@ volumes: clone: retries: 3 depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- gcr -- gar -kind: pipeline -name: release-whatsnew-checker -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.23.1-alpine - name: compile-build-cmd -- commands: - - ./bin/build whatsnew-checker - depends_on: - - compile-build-cmd - image: golang:1.23.1-alpine - name: whats-new-checker -trigger: - event: - exclude: - - promote - ref: - exclude: - - refs/tags/*-cloud* - include: - - refs/tags/v* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] image_pull_secrets: - gcr - gar @@ -4380,53 +4287,34 @@ volumes: --- clone: retries: 3 -depends_on: -- rgm-tag-prerelease +depends_on: [] +environment: + EDITION: oss image_pull_secrets: - gcr - gar kind: pipeline -name: rgm-tag-prerelease-windows +name: release-whatsnew-checker +node: + type: no-parallel platform: arch: amd64 - os: windows - version: "1809" + os: linux services: [] steps: - commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner -- commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/windows/grabpl.exe - -OutFile grabpl.exe - image: grafana/ci-wix:0.1.1 - name: windows-init -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - - .\grabpl.exe windows-installer --target gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/grafana-${DRONE_TAG:1}.windows-amd64.zip - --edition oss ${DRONE_TAG} - - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp $$fname gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ - - gsutil cp "$$fname.sha256" gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ - depends_on: - - windows-init + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer + CGO_ENABLED: 0 + image: golang:1.23.1-alpine + name: compile-build-cmd +- commands: + - ./bin/build whatsnew-checker + depends_on: + - compile-build-cmd + image: golang:1.23.1-alpine + name: whats-new-checker trigger: event: exclude: @@ -4439,14 +4327,96 @@ trigger: type: docker volumes: - host: - path: //./pipe/docker_engine/ + path: /var/run/docker.sock name: docker --- clone: retries: 3 depends_on: - rgm-tag-prerelease -- rgm-tag-prerelease-windows +image_pull_secrets: +- gcr +- gar +kind: pipeline +name: prerelease-windows-msi +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - curl -L0 https://nssm.cc/release/nssm-2.24.zip -o nssm-2.24.zip + image: byrnedo/alpine-curl:0.1.8 + name: downlad-nssm +- commands: + - mkdir wix3 && cd wix3 + - curl -L0 https://github.com/wixtoolset/wix3/releases/download/wix3141rtm/wix314-binaries.zip + -o wix3.zip + - unzip wix3.zip + image: byrnedo/alpine-curl:0.1.8 + name: download-wix3 +- commands: + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - bash -c 'gcloud storage cp gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release/grafana-$${DRONE_TAG:1}.windows-amd64.zip + grafana.zip' + environment: + GCP_KEY: + from_secret: gcp_key_base64 + image: google/cloud-sdk:431.0.0 + name: download-zip +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - export WINEPATH=$(winepath ./wix3) + - ./bin/grabpl windows-installer --target grafana.zip --edition oss + depends_on: + - downlad-nssm + - download-wix3 + - grabpl + - download-zip + entrypoint: + - /bin/bash + image: scottyhardy/docker-wine:stable-9.0 + name: build-msi +- commands: + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - bash -c 'gcloud storage cp *.msi gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release' + - bash -c 'gcloud storage cp *.msi.sha256 gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release' + depends_on: + - build-msi + environment: + GCP_KEY: + from_secret: gcp_key_base64 + image: google/cloud-sdk:431.0.0 + name: upload-msi-installer +trigger: + event: + exclude: + - promote + ref: + exclude: + - refs/tags/*-cloud* + include: + - refs/tags/v* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- prerelease-windows-msi +- rgm-tag-prerelease image_pull_secrets: - gcr - gar @@ -5082,59 +5052,6 @@ volumes: path: /var/run/docker.sock name: docker --- -clone: - disable: true -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- gcr -- gar -kind: pipeline -name: testing-test-backend-windows -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - git clone "https://$$env:GITHUB_TOKEN@github.com/$$env:DRONE_REPO.git" . - - git checkout -f $$env:DRONE_COMMIT - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/ci-wix:0.1.1 - name: clone -- commands: [] - depends_on: - - clone - image: golang:1.23.1-windowsservercore-1809 - name: windows-init -- commands: - - go install github.com/google/wire/cmd/wire@v0.5.0 - - wire gen -tags oss ./pkg/server - depends_on: - - windows-init - image: golang:1.23.1-windowsservercore-1809 - name: wire-install -- commands: - - go test -short -covermode=atomic -timeout=5m ./pkg/... - depends_on: - - wire-install - image: golang:1.23.1-windowsservercore-1809 - name: test-backend -trigger: - event: - - promote - target: - - test-windows -type: docker -volumes: -- host: - path: //./pipe/docker_engine/ - name: docker ---- clone: retries: 3 depends_on: [] @@ -5199,7 +5116,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -5364,55 +5281,6 @@ volumes: temp: medium: memory --- -clone: - disable: true -depends_on: [] -image_pull_secrets: -- gcr -- gar -kind: pipeline -name: publish-ci-windows-test-image -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-ci-sandbox.git" - . - - git checkout -f $$env:DRONE_COMMIT - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/ci-wix:0.1.1 - name: clone -- commands: - - cd scripts\build\ci-windows-test - - docker login -u $$env:DOCKER_USERNAME -p $$env:DOCKER_PASSWORD - - docker build -t grafana/grafana-ci-windows-test:$$env:TAG . - - docker push grafana/grafana-ci-windows-test:$$env:TAG - environment: - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USERNAME: - from_secret: docker_username - image: docker:windowsservercore-1809 - name: build-and-publish - volumes: - - name: docker - path: //./pipe/docker_engine/ -trigger: - event: - - promote - target: - - ci-windows-test-image -type: docker -volumes: -- host: - path: //./pipe/docker_engine/ - name: docker ---- clone: retries: 3 kind: pipeline @@ -5723,6 +5591,7 @@ steps: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM jwilder/dockerize:0.6.1 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM koalaman/shellcheck:stable - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM rockylinux:9 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM scottyhardy/docker-wine:stable-9.0 depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -5760,6 +5629,7 @@ steps: - trivy --exit-code 1 --severity HIGH,CRITICAL jwilder/dockerize:0.6.1 - trivy --exit-code 1 --severity HIGH,CRITICAL koalaman/shellcheck:stable - trivy --exit-code 1 --severity HIGH,CRITICAL rockylinux:9 + - trivy --exit-code 1 --severity HIGH,CRITICAL scottyhardy/docker-wine:stable-9.0 depends_on: - authenticate-gcr environment: @@ -5792,6 +5662,82 @@ volumes: - name: config temp: {} --- +clone: + retries: 3 +depends_on: [] +image_pull_secrets: +- gcr +- gar +kind: pipeline +name: windows-pipeline-manual +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - curl -L0 https://nssm.cc/release/nssm-2.24.zip -o nssm-2.24.zip + image: byrnedo/alpine-curl:0.1.8 + name: downlad-nssm +- commands: + - mkdir wix3 && cd wix3 + - curl -L0 https://github.com/wixtoolset/wix3/releases/download/wix3141rtm/wix314-binaries.zip + -o wix3.zip + - unzip wix3.zip + image: byrnedo/alpine-curl:0.1.8 + name: download-wix3 +- commands: + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - bash -c 'gcloud storage cp gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release/grafana-$${DRONE_TAG:1}.windows-amd64.zip + grafana.zip' + environment: + GCP_KEY: + from_secret: gcp_key_base64 + image: google/cloud-sdk:431.0.0 + name: download-zip +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - export WINEPATH=$(winepath ./wix3) + - ./bin/grabpl windows-installer --target grafana.zip --edition oss + depends_on: + - downlad-nssm + - download-wix3 + - grabpl + - download-zip + entrypoint: + - /bin/bash + image: scottyhardy/docker-wine:stable-9.0 + name: build-msi +- commands: + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - bash -c 'gcloud storage cp *.msi gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release' + - bash -c 'gcloud storage cp *.msi.sha256 gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release' + depends_on: + - build-msi + environment: + GCP_KEY: + from_secret: gcp_key_base64 + image: google/cloud-sdk:431.0.0 + name: upload-msi-installer +trigger: + event: + - promote + target: build-msi +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- get: name: credentials.json path: infra/data/ci/grafana-release-eng/grafanauploads @@ -5991,6 +5937,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 41df5b1fdbd1b3c9aa915919ae5be16d2a188cbaf6b243c14fd66f94db0db8d8 +hmac: a8084d907b286e710ed331c4c09a23f16c1afed5c428a9a1b99da3ad05ab3a3f ... diff --git a/scripts/drone/events/main.star b/scripts/drone/events/main.star index 6c04ab2345c..fabbbff907d 100644 --- a/scripts/drone/events/main.star +++ b/scripts/drone/events/main.star @@ -39,10 +39,6 @@ load( "scripts/drone/pipelines/verify_storybook.star", "verify_storybook", ) -load( - "scripts/drone/pipelines/windows.star", - "windows", -) load( "scripts/drone/utils/utils.star", "failure_template", @@ -68,6 +64,8 @@ trigger = { } def main_pipelines(): + # This is how we should define any new pipelines. At some point we should update existing ones. + # Let's make an effort to reduce the amount of string constants in "depends_on" lists. pipelines = [ docs_pipelines(ver_mode, trigger_docs_main()), test_frontend(trigger, ver_mode), @@ -77,7 +75,6 @@ def main_pipelines(): verify_storybook(trigger, ver_mode), build_e2e(trigger, ver_mode), integration_tests(trigger, prefix = ver_mode, ver_mode = ver_mode), - windows(trigger, ver_mode = ver_mode), enterprise_downstream_pipeline(), notify_pipeline( name = "main-notify", @@ -88,7 +85,6 @@ def main_pipelines(): "main-test-backend", "main-build-e2e-publish", "main-integration-tests", - "main-windows", ], template = failure_template, secret = "slack_webhook", diff --git a/scripts/drone/pipelines/ci_images.star b/scripts/drone/pipelines/ci_images.star deleted file mode 100644 index 1304cdedebd..00000000000 --- a/scripts/drone/pipelines/ci_images.star +++ /dev/null @@ -1,66 +0,0 @@ -""" -This module contains steps and pipelines relating to creating CI Docker images. -""" - -load( - "scripts/drone/utils/utils.star", - "pipeline", -) -load( - "scripts/drone/utils/windows_images.star", - "windows_images", -) -load( - "scripts/drone/vault.star", - "from_secret", -) - -def publish_ci_windows_test_image_pipeline(): - trigger = { - "event": ["promote"], - "target": ["ci-windows-test-image"], - } - pl = pipeline( - name = "publish-ci-windows-test-image", - trigger = trigger, - platform = "windows", - steps = [ - { - "name": "clone", - "image": windows_images["wix"], - "environment": { - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": [ - 'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-ci-sandbox.git" .', - "git checkout -f $$env:DRONE_COMMIT", - ], - }, - { - "name": "build-and-publish", - "image": windows_images["windows_server_core"], - "environment": { - "DOCKER_USERNAME": from_secret("docker_username"), - "DOCKER_PASSWORD": from_secret("docker_password"), - }, - "commands": [ - "cd scripts\\build\\ci-windows-test", - "docker login -u $$env:DOCKER_USERNAME -p $$env:DOCKER_PASSWORD", - "docker build -t grafana/grafana-ci-windows-test:$$env:TAG .", - "docker push grafana/grafana-ci-windows-test:$$env:TAG", - ], - "volumes": [ - { - "name": "docker", - "path": "//./pipe/docker_engine/", - }, - ], - }, - ], - ) - - pl["clone"] = { - "disable": True, - } - - return [pl] diff --git a/scripts/drone/pipelines/windows.star b/scripts/drone/pipelines/windows.star deleted file mode 100644 index 59c12a19536..00000000000 --- a/scripts/drone/pipelines/windows.star +++ /dev/null @@ -1,85 +0,0 @@ -""" -This module returns the pipeline used for building Grafana on Windows. -""" - -load( - "scripts/drone/steps/lib_windows.star", - "clone_step_windows", - "get_windows_steps", - "test_backend_step_windows", - "wire_install_step_windows", -) -load( - "scripts/drone/utils/utils.star", - "pipeline", -) -load( - "scripts/drone/utils/windows_images.star", - "windows_images", -) - -def windows_test_backend(trigger, edition, ver_mode): - """ Generates a pipeline that runs backend tests on Windows - - Args: - trigger: a Drone trigger for the pipeline - edition: controls whether enterprise code is included or not - ver_mode: controls whether a pre-release or actual release pipeline is generated. - Returns: - A single pipeline running backend tests for Windows - """ - environment = {"EDITION": edition} - steps = [ - clone_step_windows(), - ] - - steps.extend([{ - "name": "windows-init", - "image": windows_images["go"], - "depends_on": ["clone"], - "commands": [], - }]) - - steps.extend([ - wire_install_step_windows(edition), - test_backend_step_windows(), - ]) - pl = pipeline( - name = "{}-test-backend-windows".format(ver_mode), - trigger = trigger, - steps = steps, - depends_on = [], - platform = "windows", - environment = environment, - ) - pl["clone"] = { - "disable": True, - } - return pl - -def windows(trigger, ver_mode): - """Generates the pipeline used for building Grafana on Windows. - - Args: - trigger: a Drone trigger for the pipeline. - ver_mode: controls whether a pre-release or actual release pipeline is generated. - Also indirectly controls which version of enterprise code is used. - - Returns: - Drone pipeline. - """ - environment = {"EDITION": "oss"} - - return pipeline( - name = "main-windows", - trigger = dict(trigger, repo = ["grafana/grafana"]), - steps = get_windows_steps(ver_mode), - depends_on = [ - "main-test-frontend", - "main-test-backend", - "main-build-e2e-publish", - "main-integration-tests", - ], - platform = "windows", - environment = environment, - ) diff --git a/scripts/drone/rgm.star b/scripts/drone/rgm.star index 7efbb35d88a..85e956d57a8 100644 --- a/scripts/drone/rgm.star +++ b/scripts/drone/rgm.star @@ -20,10 +20,6 @@ load( "scripts/drone/pipelines/whats_new_checker.star", "whats_new_checker_pipeline", ) -load( - "scripts/drone/steps/lib_windows.star", - "get_windows_steps", -) load( "scripts/drone/utils/images.star", "images", @@ -49,6 +45,10 @@ load( "rgm_github_token", "rgm_storybook_destination", ) +load( + "scripts/drone/windows.star", + "windows_pipeline_release", +) docs_paths = { "exclude": [ @@ -228,18 +228,6 @@ def rgm_tag(): steps = rgm_run("rgm-build", "drone_build_tag_grafana.sh"), ) -def rgm_tag_windows(): - return pipeline( - name = "rgm-tag-prerelease-windows", - trigger = tag_trigger, - steps = get_windows_steps( - ver_mode = "release", - bucket = "grafana-prerelease", - ), - depends_on = ["rgm-tag-prerelease"], - platform = "windows", - ) - def rgm_version_branch(): # Runs a package / build proces (with all distros) when a commit lands on a version branch return pipeline( @@ -299,17 +287,23 @@ def rgm_nightly_pipeline(): ] def rgm_tag_pipeline(): + build = rgm_tag() + + # the Windows step requires an uploaded .zip file to base its compilation on + windows = windows_pipeline_release(trigger = tag_trigger, depends_on = [ + build["name"], + ]) return [ + build, whats_new_checker_pipeline(tag_trigger), - rgm_tag(), - rgm_tag_windows(), + windows, verify_release_pipeline( trigger = tag_trigger, name = "rgm-tag-verify-prerelease-assets", bucket = "grafana-prerelease", depends_on = [ - "rgm-tag-prerelease", - "rgm-tag-prerelease-windows", + windows["name"], + build["name"], ], ), ] diff --git a/scripts/drone/steps/lib_windows.star b/scripts/drone/steps/lib_windows.star deleted file mode 100644 index 13e6b0d11af..00000000000 --- a/scripts/drone/steps/lib_windows.star +++ /dev/null @@ -1,187 +0,0 @@ -""" -This module is a library of Drone steps that exclusively run on windows machines. -""" - -load( - "scripts/drone/utils/windows_images.star", - "windows_images", -) -load( - "scripts/drone/variables.star", - "grabpl_version", -) -load( - "scripts/drone/vault.star", - "from_secret", - "gcp_grafanauploads_base64", - "prerelease_bucket", -) - -def identify_runner_step_windows(): - return { - "name": "identify-runner", - "image": windows_images["1809"], - "commands": [ - "echo $env:DRONE_RUNNER_NAME", - ], - } - -def get_windows_steps(ver_mode, bucket = "%PRERELEASE_BUCKET%"): - """Generate the list of Windows steps. - - Args: - ver_mode: used to differentiate steps for different version modes. - bucket: used to override prerelease bucket. - - Returns: - List of Drone steps. - """ - steps = [ - identify_runner_step_windows(), - ] - - init_cmds = [ - '$$ProgressPreference = "SilentlyContinue"', - "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( - grabpl_version, - ), - ] - - steps.extend( - [ - { - "name": "windows-init", - "image": windows_images["wix"], - "commands": init_cmds, - }, - ], - ) - - if ver_mode in ( - "release", - "release-branch", - ): - gcp_bucket = "{}/artifacts/downloads".format(bucket) - if ver_mode == "release": - ver_part = "${DRONE_TAG}" - dir = "release" - else: - dir = "main" - gcp_bucket = "grafana-downloads" - build_no = "DRONE_BUILD_NUMBER" - ver_part = "--build-id $$env:{}".format(build_no) - installer_commands = [ - "$$gcpKey = $$env:GCP_KEY", - "[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json", - # gcloud fails to read the file unless converted with dos2unix - "dos2unix gcpkey.json", - "gcloud auth activate-service-account --key-file=gcpkey.json", - "rm gcpkey.json", - "cp C:\\App\\nssm-2.24.zip .", - ] - - if ver_mode in ("release",): - version = "${DRONE_TAG:1}" - installer_commands.extend( - [ - ".\\grabpl.exe windows-installer --target {} --edition oss {}".format( - "gs://{}/{}/oss/{}/grafana-{}.windows-amd64.zip".format(gcp_bucket, ver_part, ver_mode, version), - ver_part, - ), - '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', - ], - ) - if ver_mode == "main": - installer_commands.extend( - [ - "gsutil cp $$fname gs://{}/oss/{}/".format(gcp_bucket, dir), - 'gsutil cp "$$fname.sha256" gs://{}/oss/{}/'.format( - gcp_bucket, - dir, - ), - ], - ) - else: - installer_commands.extend( - [ - "gsutil cp $$fname gs://{}/{}/oss/{}/".format( - gcp_bucket, - ver_part, - dir, - ), - 'gsutil cp "$$fname.sha256" gs://{}/{}/oss/{}/'.format( - gcp_bucket, - ver_part, - dir, - ), - ], - ) - steps.append( - { - "name": "build-windows-installer", - "image": windows_images["wix"], - "depends_on": [ - "windows-init", - ], - "environment": { - "GCP_KEY": from_secret(gcp_grafanauploads_base64), - "PRERELEASE_BUCKET": from_secret(prerelease_bucket), - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": installer_commands, - }, - ) - - return steps - -def download_grabpl_step_windows(): - return { - "name": "grabpl", - "image": windows_images["wix"], - "commands": [ - '$$ProgressPreference = "SilentlyContinue"', - "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( - grabpl_version, - ), - ], - } - -def test_backend_step_windows(): - # TODO: This is mostly a duplicate of "test_backend_step" in lib.star; but this file can't import that one, - # otherwise it creates an import cycle. - return { - "name": "test-backend", - "image": windows_images["go"], - "depends_on": [ - "wire-install", - ], - "commands": [ - "go test -short -covermode=atomic -timeout=5m ./pkg/...", - ], - } - -def clone_step_windows(): - return { - "name": "clone", - "image": windows_images["wix"], - "environment": { - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": [ - 'git clone "https://$$env:GITHUB_TOKEN@github.com/$$env:DRONE_REPO.git" .', - "git checkout -f $$env:DRONE_COMMIT", - ], - } - -def wire_install_step_windows(edition): - return { - "name": "wire-install", - "image": windows_images["go"], - "commands": [ - "go install github.com/google/wire/cmd/wire@v0.5.0", - "wire gen -tags {} ./pkg/server".format(edition), - ], - "depends_on": [ - "windows-init", - ], - } diff --git a/scripts/drone/utils/images.star b/scripts/drone/utils/images.star index c087127e664..74d4a51ec94 100644 --- a/scripts/drone/utils/images.star +++ b/scripts/drone/utils/images.star @@ -36,4 +36,5 @@ images = { "dockerize": "jwilder/dockerize:0.6.1", "shellcheck": "koalaman/shellcheck:stable", "rocky": "rockylinux:9", + "wine": "scottyhardy/docker-wine:stable-9.0", } diff --git a/scripts/drone/utils/windows_images.star b/scripts/drone/utils/windows_images.star deleted file mode 100644 index a25c637a11e..00000000000 --- a/scripts/drone/utils/windows_images.star +++ /dev/null @@ -1,17 +0,0 @@ -""" -This module contains all the windows docker images that are used to build test and publish Grafana. -All the windows images needed to be in a different file than the other images, since they cannot be scanned -by trivy. Related issue: https://github.com/aquasecurity/trivy/issues/1392 -""" - -load( - "scripts/drone/variables.star", - "golang_version", -) - -windows_images = { - "1809": "mcr.microsoft.com/windows:1809", - "wix": "grafana/ci-wix:0.1.1", - "windows_server_core": "docker:windowsservercore-1809", - "go": "golang:{}-windowsservercore-1809".format(golang_version), -} diff --git a/scripts/drone/variables.star b/scripts/drone/variables.star index 8bf6f765588..1ab4fc7590d 100644 --- a/scripts/drone/variables.star +++ b/scripts/drone/variables.star @@ -2,7 +2,7 @@ global variables """ -grabpl_version = "v3.0.56" +grabpl_version = "v3.1.1" golang_version = "1.23.1" # nodejs_version should match what's in ".nvmrc", but without the v prefix. diff --git a/scripts/drone/windows.star b/scripts/drone/windows.star new file mode 100644 index 00000000000..2c0943b9dc2 --- /dev/null +++ b/scripts/drone/windows.star @@ -0,0 +1,149 @@ +""" +This module is a library of Drone steps that exclusively run on windows machines. +""" + +load( + "scripts/drone/steps/lib.star", + "download_grabpl_step", +) +load( + "scripts/drone/utils/images.star", + "images", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", +) +load( + "scripts/drone/vault.star", + "from_secret", + "rgm_gcp_key_base64", +) + +def download_nssm_step(): + return { + "name": "downlad-nssm", + "image": images["curl"], + "commands": [ + # We don't need to extract nssm-2.24 because the wix / build process extracts it. It just needs to be in + # PWD and be named `nssm-2.24`. + "curl -L0 https://nssm.cc/release/nssm-2.24.zip -o nssm-2.24.zip", + ], + } + +def download_wix_step(): + return { + "name": "download-wix3", + "image": images["curl"], + "commands": [ + "mkdir wix3 && cd wix3", + "curl -L0 https://github.com/wixtoolset/wix3/releases/download/wix3141rtm/wix314-binaries.zip -o wix3.zip", + "unzip wix3.zip", + ], + } + +def download_zip_step(target = ""): + path = "{}/grafana-$${{DRONE_TAG:1}}.windows-amd64.zip".format(target) + return { + "name": "download-zip", + "image": images["cloudsdk"], + "commands": [ + "printenv GCP_KEY | base64 -d > /tmp/key.json", + "gcloud auth activate-service-account --key-file=/tmp/key.json", + "bash -c 'gcloud storage cp {} grafana.zip'".format(path), + ], + "environment": { + "GCP_KEY": from_secret(rgm_gcp_key_base64), + }, + } + +def windows_msi_pipeline(target = "", name = "", trigger = {}, depends_on = [], environment = {}): + """windows_msi_pipeline is a pipeline which creates an MSI from a .zip file. + + Args: + target: GCS path (with gs:// scheme) to the oflder containing the zip file + name: Name of the pipeline, should be unique. + trigger: The conditions which trigger the pipeline + depends_on: dependencies (strings) + environment: map of environment variables + Returns: + Drone step. + """ + nssm = download_nssm_step() + wix = download_wix_step() + grabpl = download_grabpl_step() + zip = download_zip_step(target = target) + build = build_msi_step( + depends_on = [ + nssm["name"], + wix["name"], + grabpl["name"], + zip["name"], + ], + ) + upload = upload_msi_step( + depends_on = [ + build["name"], + ], + target = target, + ) + + return pipeline( + name = name, + steps = [ + nssm, + wix, + zip, + grabpl, + build, + upload, + ], + trigger = trigger, + depends_on = depends_on, + environment = environment, + ) + +def windows_pipeline_release(name = "prerelease-windows-msi", depends_on = [], trigger = {}, environment = {}): + target = "gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release" + return windows_msi_pipeline(name = name, target = target, depends_on = depends_on, trigger = trigger, environment = environment) + +def windows_pipeline_main(depends_on = [], trigger = {}, environment = {}): + target = "gs://grafana-downloads/oss/main" + return windows_msi_pipeline(name = "main-windows-msi", target = target, depends_on = depends_on, trigger = trigger, environment = environment) + +def upload_msi_step(depends_on = [], target = ""): + return { + "name": "upload-msi-installer", + "image": images["cloudsdk"], + "commands": [ + "printenv GCP_KEY | base64 -d > /tmp/key.json", + "gcloud auth activate-service-account --key-file=/tmp/key.json", + "bash -c 'gcloud storage cp *.msi {}'".format(target), + "bash -c 'gcloud storage cp *.msi.sha256 {}'".format(target), + ], + "depends_on": depends_on, + "environment": { + "GCP_KEY": from_secret(rgm_gcp_key_base64), + }, + } + +def build_msi_step(depends_on = []): + return { + "name": "build-msi", + "image": images["wine"], + "entrypoint": ["/bin/bash"], + "commands": [ + "export WINEPATH=$(winepath ./wix3)", + "./bin/grabpl windows-installer --target grafana.zip --edition oss", + ], + "depends_on": depends_on, + } + +def windows_manual_pipeline(): + return windows_pipeline_release( + name = "windows-pipeline-manual", + trigger = { + "event": ["promote"], + "target": "build-msi", + }, + )