IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

CI: Refactor/Reorder *.star files (#52384)

Browse Source 
* Refactor test pipelines

* Move pr, main and release starlark files under scripts/drone/event

* Move cron.star to scripts/drone/events

* Further unused code removal

* Introducing build.star

* Extract verify_drone.star

* Extract windows.star

* Extract publish.star

* Remove unused code

* Fix formatting in main.star
This commit is contained in:
Dimitris Sotirakis 2022-07-20 16:43:19 +03:00 committed by GitHub
parent c8be92e827
commit 3b6cef9dfa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 642 additions and 474 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.drone.star
View File

@ -4,12 +4,12 @@
# 3. Run `make drone`
# More information about this process here: https://github.com/grafana/deployment_tools/blob/master/docs/infrastructure/drone/signing.md
load('scripts/drone/pipelines/pr.star', 'pr_pipelines')
load('scripts/drone/pipelines/main.star', 'main_pipelines')
load('scripts/drone/events/pr.star', 'pr_pipelines')
load('scripts/drone/events/main.star', 'main_pipelines')
load('scripts/drone/pipelines/docs.star', 'docs_pipelines')
load('scripts/drone/pipelines/release.star', 'release_pipelines', 'publish_image_pipelines', 'publish_artifacts_pipelines', 'publish_npm_pipelines', 'publish_packages_pipeline', 'artifacts_page_pipeline')
load('scripts/drone/events/release.star', 'release_pipelines', 'publish_image_pipelines', 'publish_artifacts_pipelines', 'publish_npm_pipelines', 'publish_packages_pipeline', 'artifacts_page_pipeline')
load('scripts/drone/version.star', 'version_branch_pipelines')
load('scripts/drone/pipelines/cron.star', 'cronjobs')
load('scripts/drone/events/cron.star', 'cronjobs')
load('scripts/drone/vault.star', 'secrets')
def main(ctx):

68
.drone.yml
View File

@ -413,6 +413,13 @@ steps:
NODE_OPTIONS: --max_old_space_size=4096
image: grafana/build-container:1.5.8
name: build-storybook
- commands:
- ls dist/*.tar.gz*
- cp dist/*.tar.gz* packaging/docker/
depends_on:
- package
image: grafana/build-container:1.5.8
name: copy-packages-for-docker
- commands:
- yarn wait-on http://$HOST:$PORT
- pa11y-ci --config .pa11yci-pr.conf.js
@ -426,13 +433,6 @@ steps:
failure: always
image: grafana/docker-puppeteer:1.0.0
name: test-a11y-frontend
- commands:
- ls dist/*.tar.gz*
- cp dist/*.tar.gz* packaging/docker/
depends_on:
- package
image: grafana/build-container:1.5.8
name: copy-packages-for-docker
- commands:
- ./bin/grabpl build-docker --edition oss -archs amd64
depends_on:
@ -850,12 +850,6 @@ steps:
- verify-gen-cue
image: grafana/build-container:1.5.8
name: wire-install
- commands:
- ./bin/grabpl verify-drone
depends_on:
- grabpl
image: byrnedo/alpine-curl:0.1.8
name: lint-drone
- commands:
- |-
echo -e "unknwon
@ -894,6 +888,12 @@ steps:
- wire-install
image: grafana/build-container:1.5.8
name: test-backend-integration
- commands:
- ./bin/grabpl verify-drone
depends_on:
- grabpl
image: byrnedo/alpine-curl:0.1.8
name: lint-drone
trigger:
branch: main
event:
@ -1133,6 +1133,26 @@ steps:
NODE_OPTIONS: --max_old_space_size=4096
image: grafana/build-container:1.5.8
name: build-storybook
- commands:
- ls dist/*.tar.gz*
- cp dist/*.tar.gz* packaging/docker/
depends_on:
- package
image: grafana/build-container:1.5.8
name: copy-packages-for-docker
- commands:
- yarn wait-on http://$HOST:$PORT
- pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json
depends_on:
- grafana-server
environment:
GRAFANA_MISC_STATS_API_KEY:
from_secret: grafana_misc_stats_api_key
HOST: grafana-server
PORT: 3001
failure: ignore
image: grafana/docker-puppeteer:1.0.0
name: test-a11y-frontend
- commands:
- ./bin/grabpl store-storybook --deployment canary --src-bucket grafana-storybook
depends_on:
@ -1151,19 +1171,6 @@ steps:
when:
repo:
- grafana/grafana
- commands:
- yarn wait-on http://$HOST:$PORT
- pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json
depends_on:
- grafana-server
environment:
GRAFANA_MISC_STATS_API_KEY:
from_secret: grafana_misc_stats_api_key
HOST: grafana-server
PORT: 3001
failure: ignore
image: grafana/docker-puppeteer:1.0.0
name: test-a11y-frontend
- commands:
- ./scripts/ci-frontend-metrics.sh | ./bin/grabpl publish-metrics $${GRAFANA_MISC_STATS_API_KEY}
depends_on:
@ -1177,13 +1184,6 @@ steps:
when:
repo:
- grafana/grafana
- commands:
- ls dist/*.tar.gz*
- cp dist/*.tar.gz* packaging/docker/
depends_on:
- package
image: grafana/build-container:1.5.8
name: copy-packages-for-docker
- commands:
- ./bin/grabpl build-docker --edition oss
depends_on:
@ -4874,6 +4874,6 @@ kind: secret
name: gcp_upload_artifacts_key
---
kind: signature
hmac: d7d85528bb22e0a126fb5fb04ec3bdde3001ab0d5a9a88a34db3cd9985eeb9fa
hmac: fdbc2724cf738ea9a2cd2be46f70d13928c96cc29179a79b7108650af40740ff
...

0
scripts/drone/pipelines/cron.star → scripts/drone/events/cron.star
View File

88
scripts/drone/events/main.star Normal file
View File

@ -0,0 +1,88 @@
load(
'scripts/drone/utils/utils.star',
'pipeline',
'notify_pipeline',
'failure_template',
'drone_change_template',
)
load(
'scripts/drone/pipelines/docs.star',
'docs_pipelines',
)
load(
'scripts/drone/pipelines/test_frontend.star',
'test_frontend',
)
load(
'scripts/drone/pipelines/test_backend.star',
'test_backend',
)
load(
'scripts/drone/pipelines/integration_tests.star',
'integration_tests',
)
load(
'scripts/drone/pipelines/build.star',
'build_e2e',
)
load(
'scripts/drone/pipelines/windows.star',
'windows',
)
load(
'scripts/drone/pipelines/publish.star',
'publish',
)
load('scripts/drone/vault.star', 'from_secret')
ver_mode = 'main'
trigger = {
'event': ['push',],
'branch': 'main',
}
def main_pipelines(edition):
drone_change_trigger = {
'event': ['push',],
'branch': 'main',
'repo': [
'grafana/grafana',
],
'paths': {
'include': [
'.drone.yml',
],
'exclude': [
'exclude',
],
},
}
pipelines = [
docs_pipelines(edition, ver_mode, trigger),
test_frontend(trigger, ver_mode),
test_backend(trigger, ver_mode),
build_e2e(trigger, ver_mode, edition),
integration_tests(trigger, ver_mode, edition),
windows(trigger, edition, ver_mode),
notify_pipeline(
name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger,
template=drone_change_template, secret='drone-changes-webhook',
),
publish(trigger, ver_mode, edition),
notify_pipeline(
name='main-notify', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']),
depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', 'main-publish'],
template=failure_template, secret='slack_webhook'
)]
return pipelines

81
scripts/drone/events/pr.star Normal file
View File

@ -0,0 +1,81 @@
load(
'scripts/drone/utils/utils.star',
'pipeline',
)
load(
'scripts/drone/pipelines/test_frontend.star',
'test_frontend',
)
load(
'scripts/drone/pipelines/test_backend.star',
'test_backend',
)
load(
'scripts/drone/pipelines/integration_tests.star',
'integration_tests',
)
load(
'scripts/drone/pipelines/build.star',
'build_e2e',
)
load(
'scripts/drone/pipelines/verify_drone.star',
'verify_drone',
)
load(
'scripts/drone/pipelines/docs.star',
'docs_pipelines',
'trigger_docs',
)
ver_mode = 'pr'
trigger = {
'event': [
'pull_request',
],
'paths': {
'exclude': [
'*.md',
'docs/**',
'latest.json',
],
},
}
def pr_pipelines(edition):
return [
verify_drone(get_pr_trigger(include_paths=['scripts/drone/**', '.drone.yml', '.drone.star']), ver_mode),
test_frontend(get_pr_trigger(exclude_paths=['pkg/**', 'packaging/**', 'go.sum', 'go.mod']), ver_mode),
test_backend(get_pr_trigger(include_paths=['pkg/**', 'packaging/**', '.drone.yml', 'conf/**', 'go.sum', 'go.mod', 'public/app/plugins/**/plugin.json']), ver_mode),
build_e2e(trigger, ver_mode, edition),
integration_tests(trigger, ver_mode, edition),
docs_pipelines(edition, ver_mode, trigger_docs())
]
def get_pr_trigger(include_paths=None, exclude_paths=None):
paths_ex = ['docs/**', '*.md']
paths_in = []
if include_paths:
for path in include_paths:
paths_in.extend([path])
if exclude_paths:
for path in exclude_paths:
paths_ex.extend([path])
return {
'event': [
'pull_request',
],
'paths': {
'exclude': paths_ex,
'include': paths_in,
},
}

0
scripts/drone/pipelines/release.star → scripts/drone/events/release.star
View File

107
scripts/drone/pipelines/build.star Normal file
View File

@ -0,0 +1,107 @@
load(
'scripts/drone/steps/lib.star',
'download_grabpl_step',
'build_image',
'identify_runner_step',
'gen_version_step',
'wire_install_step',
'yarn_install_step',
'build_backend_step',
'build_frontend_step',
'build_frontend_package_step',
'build_plugins_step',
'package_step',
'grafana_server_step',
'e2e_tests_step',
'e2e_tests_artifacts',
'build_storybook_step',
'copy_packages_for_docker_step',
'build_docker_images_step',
'publish_images_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
'redis_integration_tests_step',
'memcached_integration_tests_step',
'get_windows_steps',
'benchmark_ldap_step',
'enterprise_downstream_step',
'frontend_metrics_step',
'store_storybook_step',
'release_canary_npm_packages_step',
'upload_packages_step',
'store_packages_step',
'upload_cdn_step',
'verify_gen_cue_step',
'test_a11y_frontend_step',
'trigger_oss',
'betterer_frontend_step',
'trigger_test_release'
)
load(
'scripts/drone/utils/utils.star',
'pipeline',
)
def build_e2e(trigger, ver_mode, edition):
variants = ['linux-amd64', 'linux-amd64-musl', 'darwin-amd64', 'windows-amd64',]
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
verify_gen_cue_step(edition="oss"),
wire_install_step(),
yarn_install_step(),
]
build_steps = []
if ver_mode == 'main':
build_steps.extend([trigger_test_release()])
build_steps.extend([
enterprise_downstream_step(edition=edition, ver_mode=ver_mode),
build_backend_step(edition=edition, ver_mode=ver_mode),
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_frontend_package_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition, ver_mode=ver_mode),
])
if ver_mode == 'main':
build_steps.extend([package_step(edition=edition, ver_mode=ver_mode)])
elif ver_mode == 'pr':
build_steps.extend([package_step(edition=edition, ver_mode=ver_mode, variants=variants)])
build_steps.extend([
grafana_server_step(edition=edition),
e2e_tests_step('dashboards-suite', edition=edition),
e2e_tests_step('smoke-tests-suite', edition=edition),
e2e_tests_step('panels-suite', edition=edition),
e2e_tests_step('various-suite', edition=edition),
e2e_tests_artifacts(edition=edition),
build_storybook_step(edition=edition, ver_mode=ver_mode),
copy_packages_for_docker_step(),
test_a11y_frontend_step(ver_mode=ver_mode, edition=edition),
])
if ver_mode == 'main':
build_steps.extend([
store_storybook_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
frontend_metrics_step(edition=edition, trigger=trigger_oss)
])
if ver_mode == 'main':
build_steps.extend([
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=False),
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=False),
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana', trigger=trigger_oss),
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana-oss', trigger=trigger_oss),
release_canary_npm_packages_step(edition, trigger=trigger_oss),
upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)
])
elif ver_mode == 'pr':
build_steps.extend([build_docker_images_step(edition=edition, ver_mode=ver_mode, archs=['amd64', ])])
publish_suffix = ''
if ver_mode == 'main':
publish_suffix = '-publish'
return pipeline(
name='{}-build-e2e{}'.format(ver_mode, publish_suffix), edition="oss", trigger=trigger, services=[], steps=init_steps + build_steps,
)

40
scripts/drone/pipelines/integration_tests.star Normal file
View File

@ -0,0 +1,40 @@
load(
'scripts/drone/steps/lib.star',
'identify_runner_step',
'download_grabpl_step',
'verify_gen_cue_step',
'wire_install_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
)
load(
'scripts/drone/services/services.star',
'integration_test_services',
'integration_test_services_volumes',
'ldap_service',
)
load(
'scripts/drone/utils/utils.star',
'pipeline',
)
def integration_tests(trigger, ver_mode, edition):
services = integration_test_services(edition)
volumes = integration_test_services_volumes()
init_steps = [
download_grabpl_step(),
identify_runner_step(),
verify_gen_cue_step(edition="oss"),
wire_install_step(),
]
test_steps = [
postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
]
return pipeline(
name='{}-integration-tests'.format(ver_mode), edition="oss", trigger=trigger, services=services, steps=init_steps + test_steps,
volumes=volumes
)

243
scripts/drone/pipelines/main.star
View File

@ -1,243 +0,0 @@
load(
'scripts/drone/steps/lib.star',
'download_grabpl_step',
'build_image',
'identify_runner_step',
'gen_version_step',
'wire_install_step',
'yarn_install_step',
'lint_drone_step',
'lint_backend_step',
'lint_frontend_step',
'codespell_step',
'shellcheck_step',
'test_backend_step',
'test_backend_integration_step',
'test_frontend_step',
'build_backend_step',
'build_frontend_step',
'build_frontend_package_step',
'build_plugins_step',
'package_step',
'grafana_server_step',
'e2e_tests_step',
'e2e_tests_artifacts',
'build_storybook_step',
'copy_packages_for_docker_step',
'build_docker_images_step',
'publish_images_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
'redis_integration_tests_step',
'memcached_integration_tests_step',
'get_windows_steps',
'benchmark_ldap_step',
'enterprise_downstream_step',
'frontend_metrics_step',
'store_storybook_step',
'release_canary_npm_packages_step',
'upload_packages_step',
'store_packages_step',
'upload_cdn_step',
'verify_gen_cue_step',
'test_a11y_frontend_step',
'trigger_oss',
'betterer_frontend_step'
)
load(
'scripts/drone/services/services.star',
'integration_test_services',
'integration_test_services_volumes',
'ldap_service',
)
load(
'scripts/drone/utils/utils.star',
'pipeline',
'notify_pipeline',
'failure_template',
'drone_change_template',
)
load(
'scripts/drone/pipelines/docs.star',
'docs_pipelines',
)
load('scripts/drone/vault.star', 'from_secret')
ver_mode = 'main'
trigger = {
'event': ['push',],
'branch': 'main',
}
def main_test_frontend():
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
yarn_install_step(),
]
test_steps = [
lint_frontend_step(),
betterer_frontend_step(),
test_frontend_step(),
]
return pipeline(
name='main-test-frontend', edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
)
def main_test_backend():
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
verify_gen_cue_step(edition="oss"),
wire_install_step(),
]
test_steps = [
lint_drone_step(),
codespell_step(),
shellcheck_step(),
lint_backend_step(edition="oss"),
test_backend_step(edition="oss"),
test_backend_integration_step(edition="oss"),
]
return pipeline(
name='main-test-backend', edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
)
def get_steps(edition):
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
verify_gen_cue_step(edition="oss"),
wire_install_step(),
yarn_install_step(),
]
build_steps = [
trigger_test_release(),
enterprise_downstream_step(edition=edition, ver_mode=ver_mode),
build_backend_step(edition=edition, ver_mode=ver_mode),
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_frontend_package_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition, ver_mode=ver_mode),
]
integration_test_steps = [
postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
]
# Insert remaining steps
build_steps.extend([
package_step(edition=edition, ver_mode=ver_mode),
grafana_server_step(edition=edition),
e2e_tests_step('dashboards-suite', edition=edition),
e2e_tests_step('smoke-tests-suite', edition=edition),
e2e_tests_step('panels-suite', edition=edition),
e2e_tests_step('various-suite', edition=edition),
e2e_tests_artifacts(edition=edition),
build_storybook_step(edition=edition, ver_mode=ver_mode),
store_storybook_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
test_a11y_frontend_step(ver_mode=ver_mode, edition=edition),
frontend_metrics_step(edition=edition, trigger=trigger_oss),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=False),
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=False),
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana', trigger=trigger_oss),
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana-oss', trigger=trigger_oss)
])
build_steps.extend([
release_canary_npm_packages_step(edition, trigger=trigger_oss),
upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)
])
windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode)
store_steps = [store_packages_step(edition=edition, ver_mode=ver_mode),]
return init_steps, build_steps, integration_test_steps, windows_steps, store_steps
def trigger_test_release():
return {
'name': 'trigger-test-release',
'image': build_image,
'environment': {
'GITHUB_TOKEN': from_secret('github_token'),
'DOWNSTREAM_REPO': from_secret('downstream'),
'TEST_TAG': 'v0.0.0-test',
},
'commands': [
'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" --depth=1',
'cd grafana-enterprise',
'git fetch origin "refs/tags/*:refs/tags/*"',
'git tag -d $${TEST_TAG} && git push --delete origin $${TEST_TAG} && git tag $${TEST_TAG} && git push origin $${TEST_TAG}',
'cd -',
'git fetch origin "refs/tags/*:refs/tags/*"',
'git remote add downstream https://$${GITHUB_TOKEN}@github.com/grafana/$${DOWNSTREAM_REPO}.git',
'git tag -d $${TEST_TAG} && git push --delete downstream --quiet $${TEST_TAG} && git tag $${TEST_TAG} && git push downstream $${TEST_TAG} --quiet',
],
'failure': 'ignore',
'when': {
'paths': {
'include': [
'.drone.yml',
]
},
'repo': [
'grafana/grafana',
]
}
}
def main_pipelines(edition):
services = integration_test_services(edition)
volumes = integration_test_services_volumes()
drone_change_trigger = {
'event': ['push',],
'branch': 'main',
'repo': [
'grafana/grafana',
],
'paths': {
'include': [
'.drone.yml',
],
'exclude': [
'exclude',
],
},
}
init_steps, build_steps, integration_test_steps, windows_steps, store_steps = get_steps(edition=edition)
pipelines = [docs_pipelines(edition, ver_mode, trigger), main_test_frontend(), main_test_backend(), pipeline(
name='main-build-e2e-publish', edition=edition, trigger=trigger, services=[],
steps=init_steps + build_steps,
), pipeline(
name='main-integration-tests', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step(), identify_runner_step(), verify_gen_cue_step(edition="oss"), wire_install_step(), ] + integration_test_steps,
volumes=volumes,
), pipeline(
name='main-windows', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
steps=[identify_runner_step('windows')] + windows_steps,
depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows',
), notify_pipeline(
name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger,
template=drone_change_template, secret='drone-changes-webhook',
), pipeline(
name='main-publish', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
steps=[download_grabpl_step(), gen_version_step(ver_mode), identify_runner_step(),] + store_steps,
depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', ],
), notify_pipeline(
name='main-notify', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']),
depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', 'main-publish'],
template=failure_template, secret='slack_webhook'
)]
return pipelines

192
scripts/drone/pipelines/pr.star
View File

@ -1,192 +0,0 @@
load(
'scripts/drone/steps/lib.star',
'download_grabpl_step',
'gen_version_step',
'yarn_install_step',
'wire_install_step',
'identify_runner_step',
'lint_drone_step',
'lint_backend_step',
'lint_frontend_step',
'codespell_step',
'shellcheck_step',
'build_backend_step',
'build_frontend_step',
'build_frontend_package_step',
'build_plugins_step',
'test_backend_step',
'test_backend_integration_step',
'test_frontend_step',
'package_step',
'grafana_server_step',
'e2e_tests_step',
'e2e_tests_artifacts',
'build_storybook_step',
'copy_packages_for_docker_step',
'build_docker_images_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
'redis_integration_tests_step',
'memcached_integration_tests_step',
'benchmark_ldap_step',
'verify_gen_cue_step',
'test_a11y_frontend_step',
'enterprise_downstream_step',
'betterer_frontend_step',
)
load(
'scripts/drone/services/services.star',
'integration_test_services',
'integration_test_services_volumes',
'ldap_service',
)
load(
'scripts/drone/utils/utils.star',
'notify_pipeline',
'pipeline',
'failure_template',
'drone_change_template',
)
load(
'scripts/drone/pipelines/docs.star',
'docs_pipelines',
'trigger_docs',
)
ver_mode = 'pr'
trigger = {
'event': [
'pull_request',
],
'paths': {
'exclude': [
'*.md',
'docs/**',
'latest.json',
],
},
}
def pr_verify_drone():
steps = [
identify_runner_step(),
download_grabpl_step(),
lint_drone_step(),
]
return pipeline(
name='pr-verify-drone', edition="oss", trigger=get_pr_trigger(include_paths=['scripts/drone/**', '.drone.yml', '.drone.star']), services=[], steps=steps,
)
def pr_test_frontend():
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
yarn_install_step(),
]
test_steps = [
lint_frontend_step(),
betterer_frontend_step(),
test_frontend_step(),
]
return pipeline(
name='pr-test-frontend', edition="oss", trigger=get_pr_trigger(exclude_paths=['pkg/**', 'packaging/**', 'go.sum', 'go.mod']), services=[], steps=init_steps + test_steps,
)
def pr_test_backend():
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
verify_gen_cue_step(edition="oss"),
wire_install_step(),
]
test_steps = [
codespell_step(),
shellcheck_step(),
lint_backend_step(edition="oss"),
test_backend_step(edition="oss"),
test_backend_integration_step(edition="oss"),
]
return pipeline(
name='pr-test-backend', edition="oss", trigger=get_pr_trigger(include_paths=['pkg/**', 'packaging/**', '.drone.yml', 'conf/**', 'go.sum', 'go.mod', 'public/app/plugins/**/plugin.json']), services=[], steps=init_steps + test_steps,
)
def pr_pipelines(edition):
services = integration_test_services(edition)
volumes = integration_test_services_volumes()
variants = ['linux-amd64', 'linux-amd64-musl', 'darwin-amd64', 'windows-amd64',]
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
verify_gen_cue_step(edition="oss"),
wire_install_step(),
yarn_install_step(),
]
build_steps = [
enterprise_downstream_step(edition=edition, ver_mode=ver_mode),
build_backend_step(edition=edition, ver_mode=ver_mode),
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_frontend_package_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition, ver_mode=ver_mode),
]
integration_test_steps = [
postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
]
# Insert remaining build_steps
build_steps.extend([
package_step(edition=edition, ver_mode=ver_mode, variants=variants),
grafana_server_step(edition=edition),
e2e_tests_step('dashboards-suite', edition=edition),
e2e_tests_step('smoke-tests-suite', edition=edition),
e2e_tests_step('panels-suite', edition=edition),
e2e_tests_step('various-suite', edition=edition),
e2e_tests_artifacts(edition=edition),
build_storybook_step(edition=edition, ver_mode=ver_mode),
test_a11y_frontend_step(ver_mode=ver_mode, edition=edition),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition, ver_mode=ver_mode, archs=['amd64', ]),
])
return [
pr_verify_drone(),
pr_test_frontend(),
pr_test_backend(),
pipeline(
name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=init_steps + build_steps,
), pipeline(
name='pr-integration-tests', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step(), identify_runner_step(), verify_gen_cue_step(edition="oss"), wire_install_step(), ] + integration_test_steps,
volumes=volumes,
), docs_pipelines(edition, ver_mode, trigger_docs())
]
def get_pr_trigger(include_paths=None, exclude_paths=None):
paths_ex = ['docs/**', '*.md']
paths_in = []
if include_paths:
for path in include_paths:
paths_in.extend([path])
if exclude_paths:
for path in exclude_paths:
paths_ex.extend([path])
return {
'event': [
'pull_request',
],
'paths': {
'exclude': paths_ex,
'include': paths_in,
},
}

25
scripts/drone/pipelines/publish.star Normal file
View File

@ -0,0 +1,25 @@
load(
'scripts/drone/steps/lib.star',
'identify_runner_step',
'gen_version_step',
'download_grabpl_step',
'store_packages_step'
)
load(
'scripts/drone/utils/utils.star',
'pipeline',
)
def publish(trigger, ver_mode, edition):
steps = [
download_grabpl_step(),
gen_version_step(ver_mode),
identify_runner_step(),
store_packages_step(edition=edition, ver_mode=ver_mode),
]
return pipeline(
name='main-publish', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
steps=steps,
depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', ],
)

41
scripts/drone/pipelines/test_backend.star Normal file
View File

@ -0,0 +1,41 @@
load(
'scripts/drone/steps/lib.star',
'identify_runner_step',
'download_grabpl_step',
'gen_version_step',
'wire_install_step',
'codespell_step',
'shellcheck_step',
'lint_backend_step',
'lint_drone_step',
'test_backend_step',
'test_backend_integration_step',
'verify_gen_cue_step',
)
load(
'scripts/drone/utils/utils.star',
'pipeline',
)
def test_backend(trigger, ver_mode):
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
verify_gen_cue_step(edition="oss"),
wire_install_step(),
]
test_steps = [
codespell_step(),
shellcheck_step(),
lint_backend_step(edition="oss"),
test_backend_step(edition="oss"),
test_backend_integration_step(edition="oss"),
]
if ver_mode == 'main':
test_steps.extend([lint_drone_step()])
return pipeline(
name='{}-test-backend'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
)

31
scripts/drone/pipelines/test_frontend.star Normal file
View File

@ -0,0 +1,31 @@
load(
'scripts/drone/steps/lib.star',
'identify_runner_step',
'download_grabpl_step',
'gen_version_step',
'yarn_install_step',
'lint_frontend_step',
'betterer_frontend_step',
'test_frontend_step',
)
load(
'scripts/drone/utils/utils.star',
'pipeline',
)
def test_frontend(trigger, ver_mode):
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
yarn_install_step(),
]
test_steps = [
lint_frontend_step(),
betterer_frontend_step(),
test_frontend_step(),
]
return pipeline(
name='{}-test-frontend'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
)

21
scripts/drone/pipelines/verify_drone.star Normal file
View File

@ -0,0 +1,21 @@
load(
'scripts/drone/steps/lib.star',
'identify_runner_step',
'download_grabpl_step',
'lint_drone_step',
)
load(
'scripts/drone/utils/utils.star',
'pipeline',
)
def verify_drone(trigger, ver_mode):
steps = [
identify_runner_step(),
download_grabpl_step(),
lint_drone_step(),
]
return pipeline(
name='{}-verify-drone'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=steps,
)

137
scripts/drone/pipelines/windows.star Normal file
View File

@ -0,0 +1,137 @@
load(
'scripts/drone/steps/lib.star',
'grabpl_version',
'wix_image',
'identify_runner_step',
)
load(
'scripts/drone/utils/utils.star',
'pipeline',
)
load('scripts/drone/vault.star', 'from_secret', 'prerelease_bucket', 'github_token')
def windows(trigger, edition, ver_mode):
init_cmds = []
sfx = ''
if edition in ('enterprise', 'enterprise2'):
sfx = '-{}'.format(edition)
else:
init_cmds.extend([
'$$ProgressPreference = "SilentlyContinue"',
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
grabpl_version),
])
steps = [
{
'name': 'windows-init',
'image': wix_image,
'commands': init_cmds,
},
]
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in (
'release', 'release-branch',
):
bucket_part = ''
bucket = '%PRERELEASE_BUCKET%/artifacts/downloads'
if ver_mode == 'release':
ver_part = '${DRONE_TAG}'
dir = 'release'
else:
dir = 'main'
bucket = 'grafana-downloads'
bucket_part = ' --packages-bucket {}'.format(bucket)
build_no = 'DRONE_BUILD_NUMBER'
ver_part = '--build-id $$env:{}'.format(build_no)
installer_commands = [
'$$gcpKey = $$env:GCP_KEY',
'[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json',
# gcloud fails to read the file unless converted with dos2unix
'dos2unix gcpkey.json',
'gcloud auth activate-service-account --key-file=gcpkey.json',
'rm gcpkey.json',
'cp C:\\App\\nssm-2.24.zip .',
]
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in (
'release',
):
installer_commands.extend([
'.\\grabpl.exe gen-version {}'.format(ver_part),
'.\\grabpl.exe windows-installer --edition {}{} {}'.format(edition, bucket_part, ver_part),
'$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]',
])
if ver_mode == 'main':
installer_commands.extend([
'gsutil cp $$fname gs://{}/{}/{}/'.format(bucket, edition, dir),
'gsutil cp "$$fname.sha256" gs://{}/{}/{}/'.format(bucket, edition, dir),
])
else:
installer_commands.extend([
'gsutil cp $$fname gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir),
'gsutil cp "$$fname.sha256" gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir),
])
steps.append({
'name': 'build-windows-installer',
'image': wix_image,
'depends_on': [
'windows-init',
],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket),
'GITHUB_TOKEN': from_secret('github_token')
},
'commands': installer_commands,
})
if edition in ('enterprise', 'enterprise2'):
if ver_mode == 'release':
committish = '${DRONE_TAG}'
elif ver_mode == 'release-branch':
committish = '$$env:DRONE_BRANCH'
else:
committish = '$$env:DRONE_COMMIT'
# For enterprise, we have to clone both OSS and enterprise and merge the latter into the former
download_grabpl_step_cmds = [
'$$ProgressPreference = "SilentlyContinue"',
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
grabpl_version),
]
clone_cmds = [
'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"',
]
clone_cmds.extend([
'cd grafana-enterprise',
'git checkout {}'.format(committish),
])
steps.insert(0, {
'name': 'clone',
'image': wix_image,
'environment': {
'GITHUB_TOKEN': from_secret(github_token),
},
'commands': download_grabpl_step_cmds + clone_cmds,
})
steps[1]['depends_on'] = [
'clone',
]
steps[1]['commands'].extend([
# Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into
'cp -r grafana-enterprise C:\\App\\grafana-enterprise',
'rm -r -force grafana-enterprise',
'cp grabpl.exe C:\\App\\grabpl.exe',
'rm -force grabpl.exe',
'C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise',
'cp C:\\App\\grabpl.exe grabpl.exe',
])
if 'environment' in steps[1]:
steps[1]['environment'] + {'GITHUB_TOKEN': from_secret(github_token)}
else:
steps[1]['environment'] = {'GITHUB_TOKEN': from_secret(github_token)}
return pipeline(
name='main-windows', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
steps=[identify_runner_step('windows')] + steps,
depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows',
)

32
scripts/drone/steps/lib.star
View File

@ -1173,6 +1173,38 @@ def verify_gen_cue_step(edition):
],
}
def trigger_test_release():
return {
'name': 'trigger-test-release',
'image': build_image,
'environment': {
'GITHUB_TOKEN': from_secret('github_token'),
'DOWNSTREAM_REPO': from_secret('downstream'),
'TEST_TAG': 'v0.0.0-test',
},
'commands': [
'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" --depth=1',
'cd grafana-enterprise',
'git fetch origin "refs/tags/*:refs/tags/*"',
'git tag -d $${TEST_TAG} && git push --delete origin $${TEST_TAG} && git tag $${TEST_TAG} && git push origin $${TEST_TAG}',
'cd -',
'git fetch origin "refs/tags/*:refs/tags/*"',
'git remote add downstream https://$${GITHUB_TOKEN}@github.com/grafana/$${DOWNSTREAM_REPO}.git',
'git tag -d $${TEST_TAG} && git push --delete downstream --quiet $${TEST_TAG} && git tag $${TEST_TAG} && git push downstream $${TEST_TAG} --quiet',
],
'failure': 'ignore',
'when': {
'paths': {
'include': [
'.drone.yml',
]
},
'repo': [
'grafana/grafana',
]
}
}
def artifacts_page_step():
return {
'name': 'artifacts-page',

2
scripts/drone/version.star
View File

@ -1,5 +1,5 @@
load(
'scripts/drone/pipelines/release.star',
'scripts/drone/events/release.star',
'release_pipelines',
)