IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

CI: Remove initialize step (#47622)

Browse Source 
* Remove initialize step from pr.star

* Remove initialize step from main.star

* Remove initialize step from release.star

* Reorder dependencies

* Re-add identify runner step

* Add enterprise steps

* Remove is_downstream variable from starlark logic (#47715)

* Make init steps depend on init-enterprise for enterprise pipelines

* Exclude shellcheck from enterprise pipelines

* Add init steps to enterprise integration tests pipeline

* Add dependency for windows enterprise step

* Update grabpl version
This commit is contained in:
Dimitris Sotirakis 2022-04-14 14:56:14 +03:00 committed by GitHub
parent e50bd5cac8
commit 90892050d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 680 additions and 544 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

699
.drone.yml
View File

File diff suppressed because it is too large Load Diff

17
scripts/drone/pipelines/docs.star
View File

@ -1,7 +1,9 @@
load(
'scripts/drone/steps/lib.star',
'build_image',
'initialize_step',
'yarn_install_step',
'identify_runner_step',
'gen_version_step',
'download_grabpl_step',
'lint_frontend_step',
'codespell_step',
@ -25,16 +27,17 @@ load(
def docs_pipelines(edition, ver_mode, trigger):
steps = [download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode)
# Insert remaining steps
steps.extend([
steps = [
download_grabpl_step(),
identify_runner_step(),
gen_version_step(ver_mode),
yarn_install_step(),
codespell_step(),
lint_docs(),
build_frontend_package_step(edition=edition, ver_mode=ver_mode),
build_frontend_docs_step(edition=edition),
build_docs_website_step(),
])
]
return pipeline(
name='{}-docs'.format(ver_mode), edition=edition, trigger=trigger, services=[], steps=steps,
@ -45,7 +48,7 @@ def lint_docs():
'name': 'lint-docs',
'image': build_image,
'depends_on': [
'initialize',
'yarn-install',
],
'environment': {
'NODE_OPTIONS': '--max_old_space_size=8192',

52
scripts/drone/pipelines/main.star
View File

@ -2,7 +2,10 @@ load(
'scripts/drone/steps/lib.star',
'download_grabpl_step',
'build_image',
'initialize_step',
'identify_runner_step',
'gen_version_step',
'wire_install_step',
'yarn_install_step',
'lint_drone_step',
'lint_backend_step',
'lint_frontend_step',
@ -67,10 +70,16 @@ load('scripts/drone/vault.star', 'from_secret')
ver_mode = 'main'
def get_steps(edition, is_downstream=False):
def get_steps(edition):
services = integration_test_services(edition)
publish = edition != 'enterprise' or is_downstream
include_enterprise2 = edition == 'enterprise'
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
wire_install_step(),
yarn_install_step(),
]
test_steps = [
lint_drone_step(),
codespell_step(),
@ -84,9 +93,9 @@ def get_steps(edition, is_downstream=False):
build_steps = [
trigger_test_release(),
enterprise_downstream_step(edition=edition),
build_backend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_frontend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_frontend_package_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_backend_step(edition=edition, ver_mode=ver_mode),
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_frontend_package_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition, sign=True),
validate_scuemata_step(),
ensure_cuetsified_step(),
@ -106,12 +115,12 @@ def get_steps(edition, is_downstream=False):
test_backend_integration_step(edition=edition2),
])
build_steps.extend([
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-amd64'], is_downstream=is_downstream),
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-amd64']),
])
# Insert remaining steps
build_steps.extend([
package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2, is_downstream=is_downstream),
package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2),
grafana_server_step(edition=edition),
e2e_tests_step('dashboards-suite', edition=edition),
e2e_tests_step('smoke-tests-suite', edition=edition),
@ -134,27 +143,27 @@ def get_steps(edition, is_downstream=False):
build_steps.extend([
release_canary_npm_packages_step(edition, trigger=trigger_oss),
upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream, trigger=trigger_oss),
upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)
])
if include_enterprise2:
edition2 = 'enterprise2'
build_steps.extend([
package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-amd64'], is_downstream=is_downstream),
upload_packages_step(edition=edition2, ver_mode=ver_mode, is_downstream=is_downstream),
package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-amd64']),
upload_packages_step(edition=edition2, ver_mode=ver_mode),
upload_cdn_step(edition=edition2, ver_mode=ver_mode)
])
windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream)
if edition == 'enterprise' and not is_downstream:
windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode)
if edition == 'enterprise':
store_steps = []
else:
store_steps = [
store_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
store_packages_step(edition=edition, ver_mode=ver_mode),
]
return test_steps, build_steps, integration_test_steps, windows_steps, store_steps
return init_steps, test_steps, build_steps, integration_test_steps, windows_steps, store_steps
def trigger_test_release():
return {
@ -210,7 +219,7 @@ def main_pipelines(edition):
],
},
}
test_steps, build_steps, integration_test_steps, windows_steps, store_steps = get_steps(edition=edition)
init_steps, test_steps, build_steps, integration_test_steps, windows_steps, store_steps = get_steps(edition=edition)
if edition == 'enterprise':
services.append(ldap_service())
@ -218,27 +227,26 @@ def main_pipelines(edition):
pipelines = [docs_pipelines(edition, ver_mode, trigger), pipeline(
name='main-test', edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + test_steps,
steps=init_steps + test_steps,
volumes=[],
), pipeline(
name='main-build-e2e-publish', edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + build_steps,
steps=init_steps + build_steps,
volumes=volumes,
), pipeline(
name='main-integration-tests', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + integration_test_steps,
steps=[download_grabpl_step(), identify_runner_step(),] + integration_test_steps,
volumes=volumes,
), pipeline(
name='windows-main', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps,
steps=[identify_runner_step('windows')] + windows_steps,
depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows',
), notify_pipeline(
name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger,
template=drone_change_template, secret='drone-changes-webhook',
), pipeline(
name='publish-main', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode,
install_deps=False) + store_steps,
steps=[download_grabpl_step(), identify_runner_step(),] + store_steps,
depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main', ],
), notify_pipeline(
name='notify-main', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']),

20
scripts/drone/pipelines/pr.star
View File

@ -1,7 +1,10 @@
load(
'scripts/drone/steps/lib.star',
'download_grabpl_step',
'initialize_step',
'gen_version_step',
'yarn_install_step',
'wire_install_step',
'identify_runner_step',
'lint_drone_step',
'lint_backend_step',
'lint_frontend_step',
@ -59,6 +62,13 @@ def pr_pipelines(edition):
volumes = integration_test_services_volumes()
variants = ['linux-amd64', 'linux-amd64-musl', 'darwin-amd64', 'windows-amd64', 'armv6',]
include_enterprise2 = edition == 'enterprise'
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
wire_install_step(),
yarn_install_step(),
]
test_steps = [
lint_drone_step(),
codespell_step(),
@ -132,14 +142,12 @@ def pr_pipelines(edition):
return [
pipeline(
name='pr-test', edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode)
+ test_steps,
name='pr-test', edition=edition, trigger=trigger, services=[], steps=init_steps + test_steps,
), pipeline(
name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode)
+ build_steps,
name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=init_steps + build_steps,
), pipeline(
name='pr-integration-tests', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + integration_test_steps,
steps=[download_grabpl_step(), identify_runner_step(),] + integration_test_steps,
volumes=volumes,
), docs_pipelines(edition, ver_mode, trigger_docs())
]

73
scripts/drone/pipelines/release.star
View File

@ -1,10 +1,15 @@
load(
'scripts/drone/steps/lib.star',
'disable_tests',
'clone_enterprise_step',
'download_grabpl_step',
'initialize_step',
'gen_version_step',
'yarn_install_step',
'wire_install_step',
'init_enterprise_step',
'lint_drone_step',
'build_image',
'identify_runner_step',
'publish_image',
'lint_backend_step',
'lint_frontend_step',
@ -77,7 +82,7 @@ def retrieve_npm_packages_step():
'name': 'retrieve-npm-packages',
'image': publish_image,
'depends_on': [
'initialize',
'grabpl',
],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
@ -150,16 +155,26 @@ def get_steps(edition, ver_mode):
should_upload = should_publish or ver_mode in ('release-branch',)
include_enterprise2 = edition == 'enterprise'
edition2 = 'enterprise2'
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
wire_install_step(),
yarn_install_step(),
]
test_steps = [
test_steps = []
if edition != 'enterprise':
test_steps.extend([shellcheck_step()])
test_steps.extend([
codespell_step(),
shellcheck_step(),
lint_backend_step(edition=edition),
lint_frontend_step(),
test_backend_step(edition=edition),
test_backend_integration_step(edition=edition),
test_frontend_step(),
]
])
build_steps = [
build_backend_step(edition=edition, ver_mode=ver_mode),
@ -233,16 +248,16 @@ def get_steps(edition, ver_mode):
if step:
publish_steps.append(step)
return test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps
return init_steps, test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps
def get_oss_pipelines(trigger, ver_mode):
edition = 'oss'
services = integration_test_services(edition=edition)
volumes = integration_test_services_volumes()
test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
init_steps, test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
windows_pipeline = pipeline(
name='oss-windows-{}'.format(ver_mode), edition=edition, trigger=trigger,
steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_package_steps,
steps=[identify_runner_step('windows')] + windows_package_steps,
platform='windows', depends_on=[
'oss-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode),
],
@ -250,8 +265,7 @@ def get_oss_pipelines(trigger, ver_mode):
pipelines = [
pipeline(
name='oss-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
build_steps + package_steps + publish_steps,
steps=init_steps + build_steps + package_steps + publish_steps,
volumes=volumes,
),
]
@ -259,14 +273,12 @@ def get_oss_pipelines(trigger, ver_mode):
pipelines.extend([
pipeline(
name='oss-test-{}'.format(ver_mode), edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
test_steps,
steps=init_steps + test_steps,
volumes=[],
),
pipeline(
name='oss-integration-tests-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
integration_test_steps,
steps=[download_grabpl_step(), identify_runner_step(),] + integration_test_steps,
volumes=volumes,
)
])
@ -286,10 +298,28 @@ def get_enterprise_pipelines(trigger, ver_mode):
edition = 'enterprise'
services = integration_test_services(edition=edition)
volumes = integration_test_services_volumes()
test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
deps_on_clone_enterprise_step = {
'depends_on': [
'init-enterprise',
]
}
_, test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
init_steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(ver_mode),
init_enterprise_step(ver_mode)
]
for step in [wire_install_step(), yarn_install_step(), gen_version_step(ver_mode)]:
step.update(deps_on_clone_enterprise_step)
init_steps.extend([step])
for step in integration_test_steps:
step.update(deps_on_clone_enterprise_step)
windows_pipeline = pipeline(
name='enterprise-windows-{}'.format(ver_mode), edition=edition, trigger=trigger,
steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_package_steps,
steps=[identify_runner_step('windows')] + windows_package_steps,
platform='windows', depends_on=[
'enterprise-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode),
],
@ -297,8 +327,7 @@ def get_enterprise_pipelines(trigger, ver_mode):
pipelines = [
pipeline(
name='enterprise-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
build_steps + package_steps + publish_steps,
steps=init_steps + build_steps + package_steps + publish_steps,
volumes=volumes,
),
]
@ -306,14 +335,12 @@ def get_enterprise_pipelines(trigger, ver_mode):
pipelines.extend([
pipeline(
name='enterprise-test-{}'.format(ver_mode), edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
test_steps,
steps=init_steps + test_steps,
volumes=[],
),
pipeline(
name='enterprise-integration-tests-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
integration_test_steps,
steps=[download_grabpl_step(), identify_runner_step(), clone_enterprise_step(ver_mode), init_enterprise_step(ver_mode),] + integration_test_steps,
volumes=volumes,
),
])
@ -401,7 +428,7 @@ def publish_npm_pipelines(mode):
]
return [pipeline(
name='publish-npm-packages-{}'.format(mode), trigger=trigger, steps = initialize_step(edition='oss', platform='linux', ver_mode='release') + steps, edition="all"
name='publish-npm-packages-{}'.format(mode), trigger=trigger, steps = steps, edition="all"
)]
def release_pipelines(ver_mode='release', trigger=None, environment=None):

363
scripts/drone/steps/lib.star
View File

@ -1,6 +1,6 @@
load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket')
grabpl_version = 'v2.9.34'
grabpl_version = 'v2.9.35'
build_image = 'grafana/build-container:1.5.3'
publish_image = 'grafana/grafana-ci-deploy:1.3.1'
deploy_docker_image = 'us.gcr.io/kubernetes-dev/drone/plugins/deploy-image'
@ -18,6 +18,7 @@ trigger_oss = {
}
}
def slack_step(channel, template, secret):
return {
'name': 'slack',
@ -30,106 +31,49 @@ def slack_step(channel, template, secret):
}
def initialize_step(edition, platform, ver_mode, is_downstream=False, install_deps=True):
if platform == 'windows':
return [
{
'name': 'identify-runner',
'image': windows_image,
'commands': [
'echo $env:DRONE_RUNNER_NAME',
],
},
]
common_cmds = [
# Generate Go code, will install Wire
# TODO: Install Wire in Docker image instead
'make gen-go',
]
def gen_version_step(ver_mode):
if ver_mode == 'release':
args = '${DRONE_TAG}'
common_cmds.append('./bin/grabpl verify-version ${DRONE_TAG}')
else:
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
build_no = '${DRONE_BUILD_NUMBER}'
args = '--build-id {}'.format(build_no)
identify_runner = identify_runner_step(platform)
if install_deps:
common_cmds.extend([
return {
'name': 'gen-version',
'image': build_image,
'depends_on': [
'grabpl',
],
'commands': [
'./bin/grabpl gen-version {}'.format(args),
],
}
def yarn_install_step():
return {
'name': 'yarn-install',
'image': build_image,
'commands': [
'yarn install --immutable',
])
if edition in ('enterprise', 'enterprise2'):
source_commit = ''
if ver_mode == 'release':
committish = '${DRONE_TAG}'
source_commit = ' ${DRONE_TAG}'
environment = {
'GITHUB_TOKEN': from_secret(github_token),
}
token = "--github-token $${GITHUB_TOKEN}"
elif ver_mode == 'release-branch':
committish = '${DRONE_BRANCH}'
environment = {}
token = ""
else:
environment = {}
if is_downstream:
source_commit = ' $${SOURCE_COMMIT}'
committish = '${DRONE_COMMIT}'
token = ""
steps = [
identify_runner,
clone_enterprise(committish),
{
'name': 'initialize',
'image': build_image,
'depends_on': [
'clone-enterprise',
],
'environment': environment,
'commands': [
'mv bin/grabpl /tmp/',
'rmdir bin',
'mv grafana-enterprise /tmp/',
'/tmp/grabpl init-enterprise {} /tmp/grafana-enterprise{}'.format(token, source_commit),
'mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json',
'mkdir bin',
'mv /tmp/grabpl bin/'
] + common_cmds,
},
]
return steps
steps = [
identify_runner,
{
'name': 'initialize',
'image': build_image,
'commands': common_cmds,
},
]
return steps
],
'depends_on': [
'grabpl',
],
}
def identify_runner_step(platform):
if platform == 'linux':
return {
'name': 'identify-runner',
'image': alpine_image,
'commands': [
'echo $DRONE_RUNNER_NAME',
],
}
else:
def wire_install_step():
return {
'name': 'wire-install',
'image': build_image,
'commands': [
'make gen-go',
],
}
def identify_runner_step(platform='linux'):
if platform == 'windows':
return {
'name': 'identify-runner',
'image': windows_image,
@ -137,9 +81,23 @@ def identify_runner_step(platform):
'echo $env:DRONE_RUNNER_NAME',
],
}
else:
return {
'name': 'identify-runner',
'image': alpine_image,
'commands': [
'echo $DRONE_RUNNER_NAME',
],
}
def clone_enterprise(committish):
def clone_enterprise_step(ver_mode):
if ver_mode == 'release':
committish = '${DRONE_TAG}'
elif ver_mode == 'release-branch':
committish = '${DRONE_BRANCH}'
else:
committish = '${DRONE_COMMIT}'
return {
'name': 'clone-enterprise',
'image': build_image,
@ -153,8 +111,51 @@ def clone_enterprise(committish):
],
}
def init_enterprise_step(ver_mode):
source_commit = ''
if ver_mode == 'release':
source_commit = ' ${DRONE_TAG}'
environment = {
'GITHUB_TOKEN': from_secret(github_token),
}
token = "--github-token $${GITHUB_TOKEN}"
elif ver_mode == 'release-branch':
environment = {}
token = ""
else:
environment = {}
token = ""
return {
'name': 'init-enterprise',
'image': build_image,
'depends_on': [
'clone-enterprise',
],
'environment': environment,
'commands': [
'mv bin/grabpl /tmp/',
'rmdir bin',
'mv grafana-enterprise /tmp/',
'/tmp/grabpl init-enterprise {} /tmp/grafana-enterprise{}'.format(token, source_commit),
'mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json',
'mkdir bin',
'mv /tmp/grabpl bin/'
],
}
def download_grabpl_step(platform="linux"):
if platform == 'windows':
return {
'name': 'grabpl',
'image': wix_image,
'commands': [
'$$ProgressPreference = "SilentlyContinue"',
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
grabpl_version),
]
}
def download_grabpl_step():
return {
'name': 'grabpl',
'image': curl_image,
@ -211,7 +212,7 @@ def lint_backend_step(edition):
'CGO_ENABLED': '1',
},
'depends_on': [
'initialize',
'wire-install',
],
'commands': [
# Don't use Make since it will re-download the linters
@ -224,9 +225,6 @@ def benchmark_ldap_step():
return {
'name': 'benchmark-ldap',
'image': build_image,
'depends_on': [
'initialize',
],
'environment': {
'LDAP_HOSTNAME': 'ldap',
},
@ -263,7 +261,6 @@ def store_storybook_step(edition, ver_mode, trigger=None):
if edition in ('enterprise', 'enterprise2'):
return None
commands = []
if ver_mode == 'release':
commands.extend([
@ -273,12 +270,12 @@ def store_storybook_step(edition, ver_mode, trigger=None):
else:
# main pipelines should deploy storybook to grafana-storybook/canary public bucket
commands = ['./bin/grabpl store-storybook --deployment canary --src-bucket grafana-storybook',]
commands = ['./bin/grabpl store-storybook --deployment canary --src-bucket grafana-storybook', ]
step = {
'name': 'store-storybook',
'image': publish_image,
'depends_on': ['build-storybook',] + end_to_end_tests_deps(edition),
'depends_on': ['build-storybook', ] + end_to_end_tests_deps(edition),
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
@ -289,6 +286,7 @@ def store_storybook_step(edition, ver_mode, trigger=None):
step.update(trigger)
return step
def e2e_tests_artifacts(edition):
return {
'name': 'e2e-tests-artifacts-upload' + enterprise2_suffix(edition),
@ -363,7 +361,7 @@ def upload_cdn_step(edition, ver_mode, trigger=None):
return step
def build_backend_step(edition, ver_mode, variants=None, is_downstream=False):
def build_backend_step(edition, ver_mode, variants=None):
variants_str = ''
if variants:
variants_str = ' --variants {}'.format(','.join(variants))
@ -379,10 +377,7 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False):
),
]
else:
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
build_no = '${DRONE_BUILD_NUMBER}'
env = {}
cmds = [
'./bin/grabpl build-backend --jobs 8 --edition {} --build-id {}{}'.format(
@ -393,19 +388,17 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False):
return {
'name': 'build-backend' + enterprise2_suffix(edition),
'image': build_image,
'depends_on': [
'initialize',
],
'environment': env,
'depends_on': [
'gen-version',
'wire-install',
],
'commands': cmds,
}
def build_frontend_step(edition, ver_mode, is_downstream=False):
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
def build_frontend_step(edition, ver_mode):
build_no = '${DRONE_BUILD_NUMBER}'
# TODO: Use percentage for num jobs
if ver_mode == 'release':
@ -422,42 +415,42 @@ def build_frontend_step(edition, ver_mode, is_downstream=False):
return {
'name': 'build-frontend',
'image': build_image,
'depends_on': [
'initialize',
],
'environment': {
'NODE_OPTIONS': '--max_old_space_size=8192',
},
'depends_on': [
'gen-version',
'yarn-install',
],
'commands': cmds,
}
def build_frontend_package_step(edition, ver_mode, is_downstream=False):
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
def build_frontend_package_step(edition, ver_mode):
build_no = '${DRONE_BUILD_NUMBER}'
# TODO: Use percentage for num jobs
if ver_mode == 'release':
cmds = [
'./bin/grabpl build-frontend-packages --jobs 8 --github-token $${GITHUB_TOKEN} ' + \
'--edition {} ${{DRONE_TAG}}'.format(edition),
]
]
else:
cmds = [
'./bin/grabpl build-frontend-packages --jobs 8 --edition {} '.format(edition) + \
'--build-id {}'.format(build_no),
]
]
return {
'name': 'build-frontend-packages',
'image': build_image,
'depends_on': [
'initialize',
],
'environment': {
'NODE_OPTIONS': '--max_old_space_size=8192',
},
'depends_on': [
'gen-version',
'yarn-install',
],
'commands': cmds,
}
@ -487,10 +480,11 @@ def build_plugins_step(edition, sign=False):
return {
'name': 'build-plugins',
'image': build_image,
'depends_on': [
'initialize',
],
'environment': env,
'depends_on': [
'gen-version',
'yarn-install',
],
'commands': [
# TODO: Use percentage for num jobs
'./bin/grabpl build-plugins --jobs 8 --edition {}{}'.format(edition, sign_args),
@ -503,7 +497,7 @@ def test_backend_step(edition):
'name': 'test-backend' + enterprise2_suffix(edition),
'image': build_image,
'depends_on': [
'initialize',
'wire-install',
],
'commands': [
'./bin/grabpl test-backend --edition {}'.format(edition),
@ -516,7 +510,7 @@ def test_backend_integration_step(edition):
'name': 'test-backend-integration' + enterprise2_suffix(edition),
'image': build_image,
'depends_on': [
'initialize',
'wire-install',
],
'commands': [
'./bin/grabpl integration-tests --edition {}'.format(edition),
@ -528,12 +522,12 @@ def test_frontend_step():
return {
'name': 'test-frontend',
'image': build_image,
'depends_on': [
'initialize',
],
'environment': {
'TEST_MAX_WORKERS': '50%',
},
'depends_on': [
'yarn-install',
],
'commands': [
'yarn run ci:test-frontend',
],
@ -544,16 +538,16 @@ def lint_frontend_step():
return {
'name': 'lint-frontend',
'image': build_image,
'depends_on': [
'initialize',
],
'environment': {
'TEST_MAX_WORKERS': '50%',
},
'depends_on': [
'yarn-install',
],
'commands': [
'yarn run prettier:check',
'yarn run lint',
'yarn run i18n:compile', # TODO: right place for this?
'yarn run i18n:compile', # TODO: right place for this?
'yarn run typecheck',
],
}
@ -617,9 +611,6 @@ def codespell_step():
return {
'name': 'codespell',
'image': build_image,
'depends_on': [
'initialize',
],
'commands': [
# Important: all words have to be in lowercase, and separated by "\n".
'echo -e "unknwon\nreferer\nerrorstring\neror\niam\nwan" > words_to_ignore.txt',
@ -634,7 +625,7 @@ def shellcheck_step():
'name': 'shellcheck',
'image': build_image,
'depends_on': [
'initialize',
'grabpl',
],
'commands': [
'./bin/grabpl shellcheck',
@ -642,7 +633,7 @@ def shellcheck_step():
}
def package_step(edition, ver_mode, include_enterprise2=False, variants=None, is_downstream=False):
def package_step(edition, ver_mode, include_enterprise2=False, variants=None):
deps = [
'build-plugins',
'build-backend',
@ -683,10 +674,7 @@ def package_step(edition, ver_mode, include_enterprise2=False, variants=None, is
),
]
else:
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
build_no = '${DRONE_BUILD_NUMBER}'
cmds = [
'{}./bin/grabpl package --jobs 8 --edition {} '.format(test_args, edition) + \
'--build-id {}{}{}'.format(build_no, variants_str, sign_args),
@ -731,6 +719,7 @@ def grafana_server_step(edition, port=3001):
],
}
def e2e_tests_step(suite, edition, port=3001, tries=None):
cmd = './bin/grabpl e2e-tests --port {} --suite {}'.format(port, suite)
if tries:
@ -810,13 +799,15 @@ def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publis
},
}
def publish_images_step(edition, ver_mode, mode, docker_repo, trigger=None):
if mode == 'security':
mode = '--{} '.format(mode)
else:
mode = ''
cmd = './bin/grabpl artifacts docker publish {}--dockerhub-repo {} --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7'.format(mode, docker_repo)
cmd = './bin/grabpl artifacts docker publish {}--dockerhub-repo {} --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7'.format(
mode, docker_repo)
if ver_mode == 'release':
deps = ['fetch-images-{}'.format(edition)]
@ -847,10 +838,7 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger=None):
def postgres_integration_tests_step(edition, ver_mode):
deps = []
if edition in ('enterprise', 'enterprise2') and ver_mode in ('release-branch', 'release'):
deps.extend(['initialize'])
else:
deps.extend(['grabpl'])
deps.extend(['grabpl'])
return {
'name': 'postgres-integration-tests',
'image': build_image,
@ -875,10 +863,7 @@ def postgres_integration_tests_step(edition, ver_mode):
def mysql_integration_tests_step(edition, ver_mode):
deps = []
if edition in ('enterprise', 'enterprise2') and ver_mode in ('release-branch', 'release'):
deps.extend(['initialize'])
else:
deps.extend(['grabpl'])
deps.extend(['grabpl'])
return {
'name': 'mysql-integration-tests',
'image': build_image,
@ -901,10 +886,7 @@ def mysql_integration_tests_step(edition, ver_mode):
def redis_integration_tests_step(edition, ver_mode):
deps = []
if edition in ('enterprise', 'enterprise2') and ver_mode in ('release-branch', 'release'):
deps.extend(['initialize'])
else:
deps.extend(['grabpl'])
deps.extend(['grabpl'])
return {
'name': 'redis-integration-tests',
'image': build_image,
@ -921,10 +903,7 @@ def redis_integration_tests_step(edition, ver_mode):
def memcached_integration_tests_step(edition, ver_mode):
deps = []
if edition in ('enterprise', 'enterprise2') and ver_mode in ('release-branch', 'release'):
deps.extend(['initialize'])
else:
deps.extend(['grabpl'])
deps.extend(['grabpl'])
return {
'name': 'memcached-integration-tests',
'image': build_image,
@ -965,15 +944,16 @@ def enterprise2_suffix(edition):
return ''
def upload_packages_step(edition, ver_mode, is_downstream=False, trigger=None):
if ver_mode == 'main' and edition in ('enterprise', 'enterprise2') and not is_downstream:
def upload_packages_step(edition, ver_mode, trigger=None):
if ver_mode == 'main' and edition in ('enterprise', 'enterprise2'):
return None
if ver_mode == 'release':
packages_bucket = '$${{PRERELEASE_BUCKET}}/artifacts/downloads{}'.format(enterprise2_suffix(edition))
cmd = './bin/grabpl upload-packages --edition {} --packages-bucket {}'.format(edition, packages_bucket)
elif edition == 'enterprise2':
cmd = './bin/grabpl upload-packages --edition {} --packages-bucket grafana-downloads-enterprise2'.format(edition)
cmd = './bin/grabpl upload-packages --edition {} --packages-bucket grafana-downloads-enterprise2'.format(
edition)
else:
cmd = './bin/grabpl upload-packages --edition {} --packages-bucket grafana-downloads'.format(edition)
@ -981,7 +961,7 @@ def upload_packages_step(edition, ver_mode, is_downstream=False, trigger=None):
if edition in 'enterprise2' or not end_to_end_tests_deps(edition):
deps.extend([
'package' + enterprise2_suffix(edition),
])
])
else:
deps.extend(end_to_end_tests_deps(edition))
@ -1000,16 +980,13 @@ def upload_packages_step(edition, ver_mode, is_downstream=False, trigger=None):
return step
def store_packages_step(edition, ver_mode, is_downstream=False):
def store_packages_step(edition, ver_mode):
if ver_mode == 'release':
cmd = './bin/grabpl store-packages --edition {} --packages-bucket grafana-downloads --gcp-key /tmp/gcpkey.json ${{DRONE_TAG}}'.format(
edition,
)
elif ver_mode == 'main':
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
build_no = '${DRONE_BUILD_NUMBER}'
cmd = './bin/grabpl store-packages --edition {} --gcp-key /tmp/gcpkey.json --build-id {}'.format(
edition, build_no,
)
@ -1035,12 +1012,7 @@ def store_packages_step(edition, ver_mode, is_downstream=False):
}
def get_windows_steps(edition, ver_mode, is_downstream=False):
if not is_downstream:
source_commit = ''
else:
source_commit = ' $$env:SOURCE_COMMIT'
def get_windows_steps(edition, ver_mode):
init_cmds = []
sfx = ''
if edition in ('enterprise', 'enterprise2'):
@ -1053,12 +1025,12 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
])
steps = [
{
'name': 'initialize',
'name': 'windows-init',
'image': wix_image,
'commands': init_cmds,
},
]
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2') or is_downstream)) or ver_mode in (
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in (
'release', 'release-branch',
):
bucket_part = ''
@ -1070,10 +1042,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
dir = 'main'
bucket = 'grafana-downloads'
bucket_part = ' --packages-bucket {}'.format(bucket)
if not is_downstream:
build_no = 'DRONE_BUILD_NUMBER'
else:
build_no = 'SOURCE_BUILD_NUMBER'
build_no = 'DRONE_BUILD_NUMBER'
ver_part = '--build-id $$env:{}'.format(build_no)
installer_commands = [
'$$gcpKey = $$env:GCP_KEY',
@ -1084,7 +1053,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
'rm gcpkey.json',
'cp C:\\App\\nssm-2.24.zip .',
]
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2') or is_downstream)) or ver_mode in (
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in (
'release',
):
installer_commands.extend([
@ -1105,15 +1074,15 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
steps.append({
'name': 'build-windows-installer',
'image': wix_image,
'depends_on': [
'windows-init',
],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket),
'GITHUB_TOKEN': from_secret('github_token')
},
'commands': installer_commands,
'depends_on': [
'initialize',
],
})
if edition in ('enterprise', 'enterprise2'):
@ -1132,11 +1101,10 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
clone_cmds = [
'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"',
]
if not is_downstream:
clone_cmds.extend([
'cd grafana-enterprise',
'git checkout {}'.format(committish),
])
clone_cmds.extend([
'cd grafana-enterprise',
'git checkout {}'.format(committish),
])
steps.insert(0, {
'name': 'clone',
'image': wix_image,
@ -1154,7 +1122,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
'rm -r -force grafana-enterprise',
'cp grabpl.exe C:\\App\\grabpl.exe',
'rm -force grabpl.exe',
'C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise{}'.format(source_commit),
'C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise',
'cp C:\\App\\grabpl.exe grabpl.exe',
])
if 'environment' in steps[1]:
@ -1194,6 +1162,7 @@ def ensure_cuetsified_step():
],
}
def end_to_end_tests_deps(edition):
if disable_tests:
return []