IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

CI: Migrate to Drone for master builds (#27087)

Browse Source 
* CI: Migrate to Drone for master builds
This commit is contained in:
Arve Knudsen 2020-09-24 15:20:48 +02:00 committed by GitHub
parent b98c85d81c
commit 755f3d0de7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 418 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
.circleci/config.yml
View File

@ -10,10 +10,9 @@ aliases:
- &filter-only-master
branches:
only: master
- &filter-master-or-release
- &filter-release
branches:
only:
- master
- chore/test-release-pipeline
tags:
only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
@ -86,10 +85,6 @@ jobs:
# We're testing the release pipeline
/tmp/grabpl build-backend --github-token "${GITHUB_GRAFANABOT_TOKEN}" --edition << parameters.edition >> \
--variants << parameters.variant >> v7.0.0-test
else
# A master or PR build
/tmp/grabpl build-backend --github-token "${GITHUB_GRAFANABOT_TOKEN}" --edition << parameters.edition >> \
--variants << parameters.variant >> --build-id $CIRCLE_WORKFLOW_ID
fi
- run:
name: Move artifacts
@ -154,9 +149,6 @@ jobs:
elif [[ $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then
# We're testing the release pipeline
/tmp/grabpl build-frontend --github-token "${GITHUB_GRAFANABOT_TOKEN}" --edition << parameters.edition >> 7.0.0-test
else
# A master or PR build
/tmp/grabpl build-frontend --github-token "${GITHUB_GRAFANABOT_TOKEN}" --edition << parameters.edition >> --build-id $CIRCLE_WORKFLOW_ID
fi
- run:
name: Move artifacts
@ -410,17 +402,6 @@ jobs:
elif [[ $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then
# We're testing the release pipeline
/tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition oss --sign v7.0.0-test
elif [[ $CIRCLE_BRANCH == "master" ]]; then
# A master build
/tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition oss --sign --build-id $CIRCLE_WORKFLOW_ID
elif [[ -n "$CIRCLE_PR_NUMBER" ]]; then
# A forked PR build, don't sign as it requires an API secret
/tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition oss --build-id $CIRCLE_WORKFLOW_ID --variants \
linux-x64,linux-x64-musl,osx64,win64
else
# A non-forked PR build
/tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition oss --sign --build-id $CIRCLE_WORKFLOW_ID --variants \
linux-x64,linux-x64-musl,osx64,win64
fi
- run:
name: Move artifacts
@ -472,17 +453,6 @@ jobs:
elif [[ $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then
# We're testing the release pipeline
/tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition enterprise --sign v7.0.0-test
elif [[ $CIRCLE_BRANCH == "master" ]]; then
# A master build
/tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition enterprise --sign --build-id $CIRCLE_WORKFLOW_ID
elif [[ -n "$CIRCLE_PR_NUMBER" ]]; then
# A forked PR build, don't sign as it requires an API secret
/tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition enterprise --build-id $CIRCLE_WORKFLOW_ID --variants \
linux-x64,linux-x64-musl,osx64,win64
else
# A PR build
/tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition enterprise --sign --build-id $CIRCLE_WORKFLOW_ID --variants \
linux-x64,linux-x64-musl,osx64,win64
fi
- run:
name: Move artifacts
@ -566,10 +536,6 @@ jobs:
if [[ $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then
# We're testing the release pipeline
echo Testing release
elif [[ $CIRCLE_BRANCH == "master" ]]; then
echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json
gcloud auth activate-service-account --key-file=/tmp/gcpkey.json
gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/canary
elif [[ -n $CIRCLE_TAG ]]; then
echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json
gcloud auth activate-service-account --key-file=/tmp/gcpkey.json
@ -633,7 +599,7 @@ jobs:
name: Build Docker images
command: |
source google-cloud-sdk/path.bash.inc
if [[ -n $CIRCLE_TAG || $CIRCLE_BRANCH == "chore/test-release-pipeline" || $CIRCLE_BRANCH == "master" ]]; then
if [[ -n $CIRCLE_TAG || $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then
# It's a full build
/tmp/grabpl build-docker --jobs 4 --edition << parameters.edition >> \
--ubuntu=<< parameters.ubuntu >>
@ -645,7 +611,7 @@ jobs:
- run:
name: Exit if PR
command: |
if [[ -z $CIRCLE_TAG && $CIRCLE_BRANCH != "chore/test-release-pipeline" && $CIRCLE_BRANCH != "master" ]]; then
if [[ -z $CIRCLE_TAG && $CIRCLE_BRANCH != "chore/test-release-pipeline" ]]; then
echo "Nothing to do for PRs, so marking this step successful"
circleci step halt
fi
@ -944,19 +910,6 @@ jobs:
command: "./scripts/ci-job-succeeded.sh"
when: on_success
deploy-to-kubernetes:
description: "Deploy Grafana master Docker image to Kubernetes"
executor: base
steps:
- attach_workspace:
at: /tmp/workspace
- install-grabpl
- run:
name: Deploy to Kubernetes
command: |
cp -r /tmp/workspace/enterprise/dist .
/tmp/grabpl deploy-to-k8s
release-packages:
executor: node
steps:
@ -1040,193 +993,193 @@ workflows:
build-pipeline:
jobs:
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
edition: oss
variant: armv6
name: build-oss-backend-armv6
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
edition: oss
variant: armv7
name: build-oss-backend-armv7
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
edition: oss
variant: armv7-musl
name: build-oss-backend-armv7-musl
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
edition: oss
variant: arm64
name: build-oss-backend-arm64
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
edition: oss
variant: arm64-musl
name: build-oss-backend-arm64-musl
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
edition: oss
variant: osx64
name: build-oss-backend-osx64
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
edition: oss
variant: win64
name: build-oss-backend-win64
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
edition: oss
variant: linux-x64
name: build-oss-backend-linux-x64
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
edition: oss
variant: linux-x64-musl
name: build-oss-backend-linux-x64-musl
requires:
- lint-go
- build-frontend:
filters: *filter-master-or-release
filters: *filter-release
name: build-oss-frontend
edition: oss
- build-plugins:
filters: *filter-master-or-release
filters: *filter-release
name: build-oss-plugins
edition: oss
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
name: build-enterprise-backend-armv6
edition: enterprise
variant: armv6
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
name: build-enterprise-backend-armv7
edition: enterprise
variant: armv7
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
name: build-enterprise-backend-armv7-musl
edition: enterprise
variant: armv7-musl
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
name: build-enterprise-backend-arm64
edition: enterprise
variant: arm64
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
name: build-enterprise-backend-arm64-musl
edition: enterprise
variant: arm64-musl
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
name: build-enterprise-backend-osx64
edition: enterprise
variant: osx64
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
name: build-enterprise-backend-win64
edition: enterprise
variant: win64
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
name: build-enterprise-backend-linux-x64
edition: enterprise
variant: linux-x64
requires:
- lint-go
- build-backend:
filters: *filter-master-or-release
filters: *filter-release
name: build-enterprise-backend-linux-x64-musl
edition: enterprise
variant: linux-x64-musl
requires:
- lint-go
- build-frontend:
filters: *filter-master-or-release
filters: *filter-release
name: build-enterprise-frontend
edition: enterprise
- build-plugins:
filters: *filter-master-or-release
filters: *filter-release
name: build-enterprise-plugins
edition: enterprise
requires:
- lint-go
- build-release-publisher:
filters: *filter-master-or-release
filters: *filter-release
- codespell:
filters: *filter-master-or-release
filters: *filter-release
- lint-go:
filters: *filter-master-or-release
filters: *filter-release
- shellcheck:
filters: *filter-master-or-release
filters: *filter-release
- test-backend:
filters: *filter-master-or-release
filters: *filter-release
name: test-oss-backend
edition: oss
requires:
- lint-go
- test-backend:
filters: *filter-master-or-release
filters: *filter-release
name: test-enterprise-backend
edition: enterprise
requires:
- lint-go
- test-frontend:
filters: *filter-master-or-release
filters: *filter-release
name: test-oss-frontend
edition: oss
- test-frontend:
filters: *filter-master-or-release
filters: *filter-release
name: test-enterprise-frontend
edition: enterprise
- mysql-integration-test:
filters: *filter-master-or-release
filters: *filter-release
requires:
- lint-go
- test-oss-backend
- test-oss-frontend
- postgres-integration-test:
filters: *filter-master-or-release
filters: *filter-release
requires:
- lint-go
- test-oss-backend
- test-oss-frontend
- package-oss:
filters: *filter-master-or-release
filters: *filter-release
requires:
- build-oss-backend-armv6
- build-oss-backend-armv7
@ -1244,7 +1197,7 @@ workflows:
- shellcheck
- build-oss-plugins
- package-enterprise:
filters: *filter-master-or-release
filters: *filter-release
requires:
- build-enterprise-backend-armv6
- build-enterprise-backend-armv7
@ -1262,17 +1215,13 @@ workflows:
- shellcheck
- build-enterprise-plugins
- build-oss-windows-installer:
filters: *filter-master-or-release
filters: *filter-release
requires:
- package-oss
- build-enterprise-windows-installer:
filters: *filter-master-or-release
filters: *filter-release
requires:
- package-enterprise
- release-next-packages:
filters: *filter-only-master
requires:
- end-to-end-tests
- release-packages:
filters: *filter-only-release
requires:
@ -1280,7 +1229,7 @@ workflows:
- mysql-integration-test
- postgres-integration-test
- publish-packages:
filters: *filter-master-or-release
filters: *filter-release
name: publish-oss-packages
edition: oss
requires:
@ -1291,7 +1240,7 @@ workflows:
- postgres-integration-test
- build-release-publisher
- publish-packages:
filters: *filter-master-or-release
filters: *filter-release
name: publish-enterprise-packages
edition: enterprise
requires:
@ -1302,12 +1251,12 @@ workflows:
- postgres-integration-test
- build-release-publisher
- publish-storybook:
filters: *filter-master-or-release
filters: *filter-release
requires:
- test-oss-backend
- test-oss-frontend
- build-docker-images:
filters: *filter-master-or-release
filters: *filter-release
name: build-oss-docker-images
edition: oss
ubuntu: false
@ -1318,7 +1267,7 @@ workflows:
- package-oss
- build-oss-windows-installer
- build-docker-images:
filters: *filter-master-or-release
filters: *filter-release
name: build-oss-ubuntu-docker-images
edition: oss
ubuntu: true
@ -1329,7 +1278,7 @@ workflows:
- package-oss
- build-oss-windows-installer
- build-docker-images:
filters: *filter-master-or-release
filters: *filter-release
name: build-enterprise-docker-images
edition: enterprise
ubuntu: false
@ -1340,7 +1289,7 @@ workflows:
- package-enterprise
- build-enterprise-windows-installer
- build-docker-images:
filters: *filter-master-or-release
filters: *filter-release
name: build-enterprise-ubuntu-docker-images
edition: enterprise
ubuntu: true
@ -1351,13 +1300,9 @@ workflows:
- package-enterprise
- build-enterprise-windows-installer
- end-to-end-tests:
filters: *filter-master-or-release
filters: *filter-release
requires:
- package-oss
- deploy-to-kubernetes:
filters: *filter-only-master
requires:
- build-enterprise-docker-images
nightly:
triggers:

122
.drone.yml
View File

@ -16,7 +16,7 @@ steps:
- name: initialize
image: grafana/build-container:1.2.27
commands:
- curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v$${GRABPL_VERSION}/grabpl
- curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.5.10/grabpl
- chmod +x grabpl
- mkdir -p bin
- mv grabpl bin
@ -26,7 +26,6 @@ steps:
- yarn install --frozen-lockfile --no-progress
environment:
DOCKERIZE_VERSION: 0.6.1
GRABPL_VERSION: 0.5.9
- name: lint-backend
image: grafana/build-container:1.2.27
@ -84,7 +83,7 @@ steps:
- name: build-backend
image: grafana/build-container:1.2.27
commands:
- ./bin/grabpl build-backend --jobs 8 --edition oss --build-id $DRONE_BUILD_NUMBER --variants linux-x64,linux-x64-musl,osx64,win64 --no-pull-enterprise
- ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --variants linux-x64,linux-x64-musl,osx64,win64 --no-pull-enterprise
depends_on:
- initialize
- lint-backend
@ -93,7 +92,7 @@ steps:
- name: build-frontend
image: grafana/build-container:1.2.27
commands:
- ./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition oss --build-id $DRONE_BUILD_NUMBER --no-pull-enterprise
- ./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition oss --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise
depends_on:
- initialize
- test-frontend
@ -109,7 +108,7 @@ steps:
- name: package
image: grafana/build-container:1.2.27
commands:
- . scripts/build/gpg-test-vars.sh && ./bin/grabpl package --jobs 8 --edition oss --build-id $DRONE_BUILD_NUMBER --no-pull-enterprise --variants linux-x64,linux-x64-musl,osx64,win64
- . scripts/build/gpg-test-vars.sh && ./bin/grabpl package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise --variants linux-x64,linux-x64-musl,osx64,win64
depends_on:
- build-backend
- build-frontend
@ -161,7 +160,7 @@ steps:
- package
- name: build-docker-images
image: grafana/drone-grafana-docker:0.2.0
image: grafana/drone-grafana-docker:0.3.1
settings:
archs: amd64
dry_run: true
@ -225,7 +224,33 @@ trigger:
---
kind: pipeline
type: docker
name: test-master
name: windows-pr
platform:
os: windows
arch: amd64
version: 1809
steps:
- name: identify-runner
image: mcr.microsoft.com/windows:1809
commands:
- echo $env:DRONE_RUNNER_NAME
- name: initialize
image: grafana/ci-wix:0.1.1
commands:
- $$ProgressPreference = "SilentlyContinue"
- Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.5.10/windows/grabpl.exe -OutFile grabpl.exe
trigger:
event:
- pull_request
---
kind: pipeline
type: docker
name: build-master
platform:
os: linux
@ -240,7 +265,7 @@ steps:
- name: initialize
image: grafana/build-container:1.2.27
commands:
- curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v$${GRABPL_VERSION}/grabpl
- curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.5.10/grabpl
- chmod +x grabpl
- mkdir -p bin
- mv grabpl bin
@ -250,7 +275,18 @@ steps:
- yarn install --frozen-lockfile --no-progress
environment:
DOCKERIZE_VERSION: 0.6.1
GRABPL_VERSION: 0.5.9
- name: trigger-enterprise-downstream
image: grafana/drone-downstream
settings:
params:
- SOURCE_BUILD_NUMBER=${DRONE_BUILD_NUMBER}
- SOURCE_COMMIT=${DRONE_COMMIT}
repositories:
- grafana/grafana-enterprise
server: https://drone.grafana.net
token:
from_secret: drone_token
- name: lint-backend
image: grafana/build-container:1.2.27
@ -318,7 +354,7 @@ steps:
- name: build-backend
image: grafana/build-container:1.2.27
commands:
- ./bin/grabpl build-backend --jobs 8 --edition oss --build-id $DRONE_BUILD_NUMBER --no-pull-enterprise
- ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise
depends_on:
- initialize
- lint-backend
@ -327,7 +363,7 @@ steps:
- name: build-frontend
image: grafana/build-container:1.2.27
commands:
- ./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition oss --build-id $DRONE_BUILD_NUMBER --no-pull-enterprise
- ./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition oss --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise
depends_on:
- initialize
- test-frontend
@ -335,7 +371,10 @@ steps:
- name: build-plugins
image: grafana/build-container:1.2.27
commands:
- ./bin/grabpl build-plugins --jobs 8 --edition oss --no-install-deps
- ./bin/grabpl build-plugins --jobs 8 --edition oss --no-install-deps --sign --signing-admin
environment:
GRAFANA_API_KEY:
from_secret: grafana_api_key
depends_on:
- initialize
- lint-backend
@ -343,7 +382,16 @@ steps:
- name: package
image: grafana/build-container:1.2.27
commands:
- . scripts/build/gpg-test-vars.sh && ./bin/grabpl package --jobs 8 --edition oss --build-id $DRONE_BUILD_NUMBER --no-pull-enterprise
- ./bin/grabpl package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise --sign
environment:
GPG_KEY_PASSWORD:
from_secret: gpg_key_password
GPG_PRIV_KEY:
from_secret: gpg_priv_key
GPG_PUB_KEY:
from_secret: gpg_pub_key
GRAFANA_API_KEY:
from_secret: grafana_api_key
depends_on:
- build-backend
- build-frontend
@ -383,7 +431,7 @@ steps:
commands:
- printenv GCP_KEY | base64 -d > /tmp/gcpkey.json
- gcloud auth activate-service-account --key-file=/tmp/gcpkey.json
- echo gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/canary
- gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/canary
environment:
GCP_KEY:
from_secret: gcp_key
@ -407,19 +455,25 @@ steps:
- package
- name: build-docker-images
image: grafana/drone-grafana-docker:0.2.0
image: grafana/drone-grafana-docker:0.3.1
settings:
dry_run: true
edition: oss
password:
from_secret: docker_password
username:
from_secret: docker_user
depends_on:
- copy-packages-for-docker
- name: build-docker-images-ubuntu
image: grafana/drone-grafana-docker:0.2.0
image: grafana/drone-grafana-docker:0.3.1
settings:
dry_run: true
edition: oss
password:
from_secret: docker_password
ubuntu: true
username:
from_secret: docker_user
depends_on:
- copy-packages-for-docker
@ -461,7 +515,7 @@ steps:
commands:
- npx lerna bootstrap
- echo "//registry.npmjs.org/:_authToken=$${NPM_TOKEN}" >> ~/.npmrc
- echo ./scripts/circle-release-next-packages.sh
- ./scripts/circle-release-next-packages.sh
environment:
NPM_TOKEN:
from_secret: npm_token
@ -471,7 +525,12 @@ steps:
- name: publish-packages
image: grafana/grafana-ci-deploy:1.2.6
commands:
- echo ./bin/grabpl publish-packages --edition oss
- ./bin/grabpl publish-packages --edition oss
environment:
GCP_GRAFANA_UPLOAD_KEY:
from_secret: gcp_key
GRAFANA_COM_API_KEY:
from_secret: grafana_api_key
depends_on:
- package
- end-to-end-tests
@ -503,7 +562,7 @@ trigger:
---
kind: pipeline
type: docker
name: windows-installer-master
name: windows-master
platform:
os: windows
@ -514,7 +573,13 @@ steps:
- name: identify-runner
image: mcr.microsoft.com/windows:1809
commands:
- echo $Env:DRONE_RUNNER_NAME
- echo $env:DRONE_RUNNER_NAME
- name: initialize
image: grafana/ci-wix:0.1.1
commands:
- $$ProgressPreference = "SilentlyContinue"
- Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.5.10/windows/grabpl.exe -OutFile grabpl.exe
- name: build-windows-installer
image: grafana/ci-wix:0.1.1
@ -524,17 +589,16 @@ steps:
- dos2unix gcpkey.json
- gcloud auth activate-service-account --key-file=gcpkey.json
- rm gcpkey.json
- $$ProgressPreference = "SilentlyContinue"
- Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.5.9/windows/grabpl.exe -OutFile grabpl.exe
- Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/oss/master/grafana-7.2.0-9fffe273pre.windows-amd64.zip -OutFile grafana.zip
- cp C:\App\nssm-2.24.zip .
- ./grabpl.exe windows-installer --edition oss grafana.zip
- .\grabpl.exe windows-installer --edition oss --build-id $$env:DRONE_BUILD_NUMBER
- $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]
- echo "gsutil cp $$fname gs://grafana-downloads/oss/master/"
- echo "gsutil cp $$fname.sha256 gs://grafana-downloads/oss/master/"
- gsutil cp $$fname gs://grafana-downloads/oss/master/
- gsutil cp $$fname.sha256 gs://grafana-downloads/oss/master/
environment:
GCP_KEY:
from_secret: gcp_key
depends_on:
- initialize
trigger:
branch:
@ -543,6 +607,6 @@ trigger:
- push
depends_on:
- test-master
- build-master
...

372
scripts/lib.star
View File

@ -1,9 +1,12 @@
build_image = 'grafana/build-container:1.2.27'
publish_image = 'grafana/grafana-ci-deploy:1.2.6'
grafana_docker_image = 'grafana/drone-grafana-docker:0.2.0'
grafana_docker_image = 'grafana/drone-grafana-docker:0.3.1'
alpine_image = 'alpine:3.12'
windows_image = 'mcr.microsoft.com/windows:1809'
grabpl_version = '0.5.9'
grabpl_version = '0.5.10'
git_image = 'alpine/git:v2.26.2'
dockerize_version = '0.6.1'
wix_image = 'grafana/ci-wix:0.1.1'
def pr_pipelines(edition):
services = [
@ -47,14 +50,51 @@ def pr_pipelines(edition):
postgres_integration_tests_step(),
mysql_integration_tests_step(),
]
windows_steps = get_windows_steps(edition=edition, version_mode='pr')
trigger = {
'event': ['pull_request',],
}
return [
pipeline(
name='test-pr', edition=edition, trigger={
'event': ['pull_request',],
}, services=services, steps=steps
name='test-pr', edition=edition, trigger=trigger, services=services, steps=steps
),
pipeline(
name='windows-pr', edition=edition, trigger=trigger, steps=windows_steps, platform='windows',
),
]
def master_steps(edition, is_downstream=False):
publish = edition != 'enterprise' or is_downstream
steps = [
enterprise_downstream_step(edition),
lint_backend_step(edition),
codespell_step(),
shellcheck_step(),
test_backend_step(),
test_frontend_step(),
frontend_metrics_step(edition=edition),
build_backend_step(edition=edition, is_downstream=is_downstream),
build_frontend_step(edition=edition, is_downstream=is_downstream),
build_plugins_step(edition=edition, sign=True),
package_step(edition=edition, sign=True, is_downstream=is_downstream),
e2e_tests_server_step(),
e2e_tests_step(),
build_storybook_step(edition=edition),
publish_storybook_step(edition=edition),
build_docs_website_step(),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition, publish=publish),
build_docker_images_step(edition=edition, ubuntu=True, publish=publish),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
release_next_npm_packages_step(edition),
publish_packages_step(edition, is_downstream),
deploy_to_kubernetes_step(edition, is_downstream),
]
windows_steps = get_windows_steps(edition=edition, version_mode='master', is_downstream=is_downstream)
return steps, windows_steps
def master_pipelines(edition):
services = [
{
@ -77,49 +117,38 @@ def master_pipelines(edition):
},
},
]
steps = [
lint_backend_step(edition),
codespell_step(),
shellcheck_step(),
test_backend_step(),
test_frontend_step(),
frontend_metrics_step(),
build_backend_step(edition=edition),
build_frontend_step(edition=edition),
build_plugins_step(edition=edition),
package_step(edition=edition),
e2e_tests_server_step(),
e2e_tests_step(),
build_storybook_step(edition=edition),
publish_storybook_step(edition=edition),
build_docs_website_step(),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition),
build_docker_images_step(edition=edition, ubuntu=True),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
release_next_npm_packages_step(edition),
publish_packages_step(edition),
deploy_to_kubernetes_step(edition),
]
windows_steps = [
windows_installer_step(edition),
]
trigger = {
'event': ['push',],
'branch': 'master',
}
return [
steps, windows_steps = master_steps(edition=edition)
pipelines = [
pipeline(
name='test-master', edition=edition, trigger=trigger, services=services, steps=steps
name='build-master', edition=edition, trigger=trigger, services=services, steps=steps
),
pipeline(
name='windows-installer-master', edition=edition, trigger=trigger,
steps=windows_steps, platform='windows', depends_on=['test-master'],
name='windows-master', edition=edition, trigger=trigger, steps=windows_steps, platform='windows',
depends_on=['build-master'],
),
]
if edition == 'enterprise':
# Add downstream enterprise pipelines triggerable from OSS builds
trigger = {
'event': ['custom',],
}
steps, windows_steps = master_steps(edition=edition, is_downstream=True)
pipelines.append(pipeline(
name='build-master-downstream', edition=edition, trigger=trigger, services=services, steps=steps,
is_downstream=True,
))
pipelines.append(pipeline(
name='windows-master-downstream', edition=edition, trigger=trigger, steps=windows_steps,
platform='windows', depends_on=['build-master-downstream'], is_downstream=True,
))
def pipeline(name, edition, trigger, steps, services=[], platform='linux', depends_on=[]):
return pipelines
def pipeline(name, edition, trigger, steps, services=[], platform='linux', depends_on=[], is_downstream=False):
if platform != 'windows':
platform_conf = {
'os': 'linux',
@ -139,7 +168,7 @@ def pipeline(name, edition, trigger, steps, services=[], platform='linux', depen
'name': name,
'trigger': trigger,
'services': services,
'steps': init_steps(edition, platform) + steps,
'steps': init_steps(edition, platform, is_downstream=is_downstream) + steps,
'depends_on': depends_on,
}
@ -151,14 +180,14 @@ def pipeline(name, edition, trigger, steps, services=[], platform='linux', depen
return pipeline
def init_steps(edition, platform):
def init_steps(edition, platform, is_downstream=False):
if platform == 'windows':
return [
{
'name': 'identify-runner',
'image': windows_image,
'commands': [
'echo $Env:DRONE_RUNNER_NAME',
'echo $env:DRONE_RUNNER_NAME',
],
},
]
@ -178,11 +207,15 @@ def init_steps(edition, platform):
'yarn install --frozen-lockfile --no-progress',
]
if edition == 'enterprise':
if is_downstream:
source_commit = ' $${SOURCE_COMMIT}'
else:
source_commit = ''
return [
identify_runner_step,
{
'name': 'clone',
'image': 'alpine/git:v2.26.2',
'image': git_image,
'environment': {
'GITHUB_TOKEN': {
'from_secret': 'github_token',
@ -198,18 +231,19 @@ def init_steps(edition, platform):
'name': 'initialize',
'image': build_image,
'environment': {
'GRABPL_VERSION': grabpl_version,
'DOCKERIZE_VERSION': '0.6.1',
'DOCKERIZE_VERSION': dockerize_version,
},
'depends_on': [
'clone',
],
'commands': [
'curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v$${GRABPL_VERSION}/grabpl',
'curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/grabpl'.format(
grabpl_version
),
'chmod +x grabpl',
'mv grabpl /tmp',
'mv grafana-enterprise /tmp/',
'/tmp/grabpl init-enterprise /tmp/grafana-enterprise',
'/tmp/grabpl init-enterprise /tmp/grafana-enterprise{}'.format(source_commit),
'mkdir bin',
'mv /tmp/grabpl bin/'
] + common_cmds,
@ -222,11 +256,12 @@ def init_steps(edition, platform):
'name': 'initialize',
'image': build_image,
'environment': {
'GRABPL_VERSION': grabpl_version,
'DOCKERIZE_VERSION': '0.6.1',
'DOCKERIZE_VERSION': dockerize_version,
},
'commands': [
'curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v$${GRABPL_VERSION}/grabpl',
'curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/grabpl'.format(
grabpl_version
),
'chmod +x grabpl',
'mkdir -p bin',
'mv grabpl bin',
@ -234,6 +269,28 @@ def init_steps(edition, platform):
},
]
def enterprise_downstream_step(edition):
if edition == 'enterprise':
return None
return {
'name': 'trigger-enterprise-downstream',
'image': 'grafana/drone-downstream',
'settings': {
'server': 'https://drone.grafana.net',
'token': {
'from_secret': 'drone_token',
},
'repositories': [
'grafana/grafana-enterprise',
],
'params': [
'SOURCE_BUILD_NUMBER=${DRONE_BUILD_NUMBER}',
'SOURCE_COMMIT=${DRONE_COMMIT}',
],
},
}
def lint_backend_step(edition):
return {
'name': 'lint-backend',
@ -285,11 +342,15 @@ def publish_storybook_step(edition):
'commands': [
'printenv GCP_KEY | base64 -d > /tmp/gcpkey.json',
'gcloud auth activate-service-account --key-file=/tmp/gcpkey.json',
'echo gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/canary',
'gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/canary',
],
}
def build_backend_step(edition, variants=None):
def build_backend_step(edition, variants=None, is_downstream=False):
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
if variants:
variants_str = ' --variants {}'.format(','.join(variants))
else:
@ -304,13 +365,17 @@ def build_backend_step(edition, variants=None):
],
'commands': [
# TODO: Convert number of jobs to percentage
'./bin/grabpl build-backend --jobs 8 --edition {} --build-id $DRONE_BUILD_NUMBER{} --no-pull-enterprise'.format(
edition, variants_str
'./bin/grabpl build-backend --jobs 8 --edition {} --build-id {}{} --no-pull-enterprise'.format(
edition, build_no, variants_str,
),
],
}
def build_frontend_step(edition):
def build_frontend_step(edition, is_downstream=False):
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
return {
'name': 'build-frontend',
'image': build_image,
@ -321,11 +386,21 @@ def build_frontend_step(edition):
'commands': [
# TODO: Use percentage for num jobs
'./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition {} '.format(edition) +
'--build-id $DRONE_BUILD_NUMBER --no-pull-enterprise',
'--build-id {} --no-pull-enterprise'.format(build_no),
],
}
def build_plugins_step(edition):
def build_plugins_step(edition, sign=False):
if sign:
env = {
'GRAFANA_API_KEY': {
'from_secret': 'grafana_api_key',
},
}
sign_args = ' --sign --signing-admin'
else:
env = None
sign_args = ''
return {
'name': 'build-plugins',
'image': build_image,
@ -333,9 +408,10 @@ def build_plugins_step(edition):
'initialize',
'lint-backend',
],
'environment': env,
'commands': [
# TODO: Use percentage for num jobs
'./bin/grabpl build-plugins --jobs 8 --edition {} --no-install-deps'.format(edition),
'./bin/grabpl build-plugins --jobs 8 --edition {} --no-install-deps{}'.format(edition, sign_args),
],
}
@ -370,7 +446,10 @@ def test_frontend_step():
],
}
def frontend_metrics_step():
def frontend_metrics_step(edition):
if edition == 'enterprise':
return None
return {
'name': 'frontend-metrics',
'image': build_image,
@ -425,11 +504,37 @@ def shellcheck_step():
],
}
def package_step(edition, variants=None):
def package_step(edition, variants=None, sign=False, is_downstream=False):
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
if variants:
variants_str = ' --variants {}'.format(','.join(variants))
else:
variants_str = ''
if sign:
sign_args = ' --sign'
env = {
'GRAFANA_API_KEY': {
'from_secret': 'grafana_api_key',
},
'GPG_PRIV_KEY': {
'from_secret': 'gpg_priv_key',
},
'GPG_PUB_KEY': {
'from_secret': 'gpg_pub_key',
},
'GPG_KEY_PASSWORD': {
'from_secret': 'gpg_key_password',
},
}
test_args = ''
else:
sign_args = ''
env = None
test_args = '. scripts/build/gpg-test-vars.sh && '
return {
'name': 'package',
'image': build_image,
@ -442,10 +547,11 @@ def package_step(edition, variants=None):
'codespell',
'shellcheck',
],
'environment': env,
'commands': [
# TODO: Use percentage for jobs
'. scripts/build/gpg-test-vars.sh && ./bin/grabpl package --jobs 8 --edition {} '.format(edition) +
'--build-id $DRONE_BUILD_NUMBER --no-pull-enterprise' + variants_str,
'{}./bin/grabpl package --jobs 8 --edition {} '.format(test_args, edition) +
'--build-id {} --no-pull-enterprise{}{}'.format(build_no, variants_str, sign_args),
],
}
@ -507,15 +613,22 @@ def copy_packages_for_docker_step():
],
}
def build_docker_images_step(edition, archs=None, ubuntu=False):
def build_docker_images_step(edition, archs=None, ubuntu=False, publish=False):
sfx = ''
if ubuntu:
sfx = '-ubuntu'
settings = {
'dry_run': True,
'dry_run': not publish,
'edition': edition,
'ubuntu': ubuntu,
}
if publish:
settings['username'] = {
'from_secret': 'docker_user',
}
settings['password'] = {
'from_secret': 'docker_password',
}
if archs:
settings['archs'] = ','.join(archs)
return {
@ -593,13 +706,12 @@ def release_next_npm_packages_step(edition):
'commands': [
'npx lerna bootstrap',
'echo "//registry.npmjs.org/:_authToken=$${NPM_TOKEN}" >> ~/.npmrc',
# TODO: Enable
'echo ./scripts/circle-release-next-packages.sh',
'./scripts/circle-release-next-packages.sh',
],
}
def deploy_to_kubernetes_step(edition):
if edition != 'enterprise':
def deploy_to_kubernetes_step(edition, is_downstream):
if edition != 'enterprise' or not is_downstream:
return None
return {
@ -608,13 +720,20 @@ def deploy_to_kubernetes_step(edition):
'depends_on': [
'build-docker-images',
],
'environment': {
'CIRCLE_TOKEN': {
'from_secret': 'deployment_tools_circle_token',
},
},
'commands': [
# TODO: Enable
'echo ./bin/grabpl deploy-to-k8s',
'./bin/grabpl deploy-to-k8s',
],
}
def publish_packages_step(edition):
def publish_packages_step(edition, is_downstream):
if edition == 'enterprise' and not is_downstream:
return None
return {
'name': 'publish-packages',
'image': publish_image,
@ -624,39 +743,104 @@ def publish_packages_step(edition):
'mysql-integration-tests',
'postgres-integration-tests',
],
'commands': [
# TODO: Enable
'echo ./bin/grabpl publish-packages --edition {}'.format(edition),
],
}
def windows_installer_step(edition):
sfx = ''
if edition == 'enterprise':
sfx = '-enterprise'
return {
'name': 'build-windows-installer',
'image': 'grafana/ci-wix:0.1.1',
'environment': {
'GCP_KEY': {
'GCP_GRAFANA_UPLOAD_KEY': {
'from_secret': 'gcp_key',
},
'GRAFANA_COM_API_KEY': {
'from_secret': 'grafana_api_key',
},
},
'commands': [
'./bin/grabpl publish-packages --edition {}'.format(edition),
],
}
def get_windows_steps(edition, version_mode, is_downstream=False):
if not is_downstream:
source_commit = ''
else:
source_commit = ' $$env:SOURCE_COMMIT'
sfx = ''
if edition == 'enterprise':
sfx = '-enterprise'
if not is_downstream:
build_no = 'DRONE_BUILD_NUMBER'
else:
build_no = 'SOURCE_BUILD_NUMBER'
steps = [
{
'name': 'initialize',
'image': wix_image,
'commands': [
'$$ProgressPreference = "SilentlyContinue"',
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/windows/grabpl.exe -OutFile grabpl.exe'.format(grabpl_version),
],
},
]
if version_mode == 'master':
installer_commands = [
'$$gcpKey = $$env:GCP_KEY',
'[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json',
# gcloud fails to read the file unless converted with dos2unix
'dos2unix gcpkey.json',
'gcloud auth activate-service-account --key-file=gcpkey.json',
'rm gcpkey.json',
'$$ProgressPreference = "SilentlyContinue"',
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/windows/grabpl.exe -OutFile grabpl.exe'.format(grabpl_version),
# TODO: Infer correct Grafana version
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/{}/master/grafana{}-7.2.0-9fffe273pre.windows-amd64.zip -OutFile grafana.zip'.format(edition, sfx),
'cp C:\\App\\nssm-2.24.zip .',
'./grabpl.exe windows-installer --edition {} grafana.zip'.format(edition),
'$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]',
'echo "gsutil cp $$fname gs://grafana-downloads/{}/master/"'.format(edition),
'echo "gsutil cp $$fname.sha256 gs://grafana-downloads/{}/master/"'.format(edition),
],
}
'.\\grabpl.exe windows-installer --edition {} --build-id $$env:{}'.format(edition, build_no),
]
if edition != 'enterprise' or is_downstream:
installer_commands.extend([
'$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]',
'gsutil cp $$fname gs://grafana-downloads/{}/{}/'.format(edition, version_mode),
'gsutil cp $$fname.sha256 gs://grafana-downloads/{}/{}/'.format(edition, version_mode),
])
steps.append({
'name': 'build-windows-installer',
'image': wix_image,
'environment': {
'GCP_KEY': {
'from_secret': 'gcp_key',
},
},
'commands': installer_commands,
'depends_on': [
'initialize',
],
})
if edition == 'enterprise':
# For enterprise, we have to clone both OSS and enterprise and merge the latter into the former
clone_commands = [
'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"',
]
if not is_downstream:
clone_commands.extend([
'cd grafana-enterprise',
'git checkout $$env:DRONE_COMMIT',
])
steps.insert(0, {
'name': 'clone',
'image': wix_image,
'environment': {
'GITHUB_TOKEN': {
'from_secret': 'github_token',
},
},
'commands': clone_commands,
})
steps[1]['depends_on'] = [
'clone',
]
steps[1]['commands'].extend([
# Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into
'cp -r grafana-enterprise C:\\App\\grafana-enterprise',
'rm -r -force grafana-enterprise',
'cp grabpl.exe C:\\App\\grabpl.exe',
'rm -force grabpl.exe',
'C:\\App\\grabpl.exe init-enterprise C:\\App\\grafana-enterprise{}'.format(source_commit),
'cp C:\\App\\grabpl.exe grabpl.exe',
])
return steps