IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

[v9.5.x] CI: Removes enterprise specific pipelines and steps (#71780)

Browse Source 
[WIP] CI: Removes enterprise specific pipelines and steps (#70815)

* Removes enterprise specific pipelines and steps (#123)

* Comment out enterprise related pipelines and steps

* Suppress unused variable warning

* Removes all edition arguments

* Remove leftover comments

* Remove redundant oss on pipelines and steps names

* Remove leftover unused variable

* Remove leftovers

* Remove pipeline dependencies

* Rename pipelines

* Fix starlark

---------

Co-authored-by: dsotirakis <dimitrios.sotirakis@grafana.com>
(cherry picked from commit 642a81ba75e79138246797302aba5c35575f030d)

* Add editions for static assets

(cherry picked from commit b13939b9af)

Co-authored-by: Dimitris Sotirakis <dimitrios.sotirakis@grafana.com>
This commit is contained in:
Guilherme Caulada 2023-07-18 12:24:27 -03:00 committed by GitHub
parent b93e956c7a
commit c47c8ddf14
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 181 additions and 3570 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.drone.star
View File

@ -11,8 +11,6 @@ load("scripts/drone/events/pr.star", "pr_pipelines")
load("scripts/drone/events/main.star", "main_pipelines")
load(
"scripts/drone/events/release.star",
"enterprise2_pipelines",
"enterprise_pipelines",
"integration_test_pipelines",
"oss_pipelines",
"publish_artifacts_pipelines",
@ -33,8 +31,6 @@ load(
"publish_ci_build_container_image_pipeline",
"publish_ci_windows_test_image_pipeline",
)
load("scripts/drone/pipelines/github.star", "publish_github_pipeline")
load("scripts/drone/pipelines/aws_marketplace.star", "publish_aws_marketplace_pipeline")
load(
"scripts/drone/pipelines/windows.star",
"windows_test_backend",
@ -48,13 +44,7 @@ def main(_ctx):
pr_pipelines() +
main_pipelines() +
oss_pipelines() +
enterprise_pipelines() +
enterprise2_pipelines() +
publish_image_pipelines_public() +
publish_github_pipeline("public") +
publish_github_pipeline("security") +
publish_aws_marketplace_pipeline("public") +
publish_artifacts_pipelines("security") +
publish_artifacts_pipelines("public") +
publish_npm_pipelines() +
publish_packages_pipeline() +

2568
.drone.yml
View File

File diff suppressed because it is too large Load Diff

2
scripts/drone/events/main.star
View File

@ -88,7 +88,7 @@ def main_pipelines():
lint_backend_pipeline(trigger, ver_mode),
build_e2e(trigger, ver_mode),
integration_tests(trigger, prefix = ver_mode, ver_mode = ver_mode),
windows(trigger, edition = "oss", ver_mode = ver_mode),
windows(trigger, ver_mode = ver_mode),
notify_pipeline(
name = "notify-drone-changes",
slack_channel = "slack-webhooks-test",

410
scripts/drone/events/release.star
View File

@ -10,25 +10,19 @@ load(
"build_frontend_step",
"build_plugins_step",
"build_storybook_step",
"clone_enterprise_step",
"compile_build_cmd",
"copy_packages_for_docker_step",
"download_grabpl_step",
"e2e_tests_artifacts",
"e2e_tests_step",
"fetch_images_step",
"get_windows_steps",
"grafana_server_step",
"identify_runner_step",
"init_enterprise_step",
"memcached_integration_tests_step",
"mysql_integration_tests_step",
"package_step",
"postgres_integration_tests_step",
"publish_grafanacom_step",
"publish_images_step",
"publish_linux_packages_step",
"redis_integration_tests_step",
"store_storybook_step",
"trigger_oss",
"upload_cdn_step",
@ -46,17 +40,14 @@ load(
load(
"scripts/drone/utils/utils.star",
"pipeline",
"with_deps",
)
load(
"scripts/drone/pipelines/test_frontend.star",
"test_frontend",
"test_frontend_enterprise",
)
load(
"scripts/drone/pipelines/test_backend.star",
"test_backend",
"test_backend_enterprise",
)
load("scripts/drone/vault.star", "from_secret", "prerelease_bucket")
load(
@ -159,19 +150,18 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
]
build_steps = [
build_backend_step(edition = "oss", ver_mode = ver_mode),
build_frontend_step(edition = "oss", ver_mode = ver_mode),
build_frontend_package_step(edition = "oss", ver_mode = ver_mode),
build_plugins_step(edition = "oss", ver_mode = ver_mode),
package_step(edition = "oss", ver_mode = ver_mode),
build_backend_step(ver_mode = ver_mode),
build_frontend_step(ver_mode = ver_mode),
build_frontend_package_step(ver_mode = ver_mode),
build_plugins_step(ver_mode = ver_mode),
package_step(ver_mode = ver_mode),
copy_packages_for_docker_step(),
build_docker_images_step(edition = "oss", publish = True),
build_docker_images_step(publish = True),
build_docker_images_step(
edition = "oss",
publish = True,
ubuntu = True,
),
grafana_server_step(edition = "oss"),
grafana_server_step(),
e2e_tests_step("dashboards-suite", tries = 3),
e2e_tests_step("smoke-tests-suite", tries = 3),
e2e_tests_step("panels-suite", tries = 3),
@ -188,9 +178,8 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
):
publish_steps.extend(
[
upload_cdn_step(edition = "oss", ver_mode = ver_mode, trigger = trigger_oss),
upload_cdn_step(ver_mode = ver_mode, trigger = trigger_oss),
upload_packages_step(
edition = "oss",
ver_mode = ver_mode,
trigger = trigger_oss,
),
@ -219,23 +208,13 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
integration_test_steps = []
volumes = []
windows_pipeline = pipeline(
name = "{}-oss-windows".format(ver_mode),
edition = "oss",
trigger = trigger,
steps = get_windows_steps(edition = "oss", ver_mode = ver_mode),
platform = "windows",
depends_on = [
"{}-oss-build-e2e-publish".format(ver_mode),
"{}-oss-test-frontend".format(ver_mode),
],
environment = environment,
)
windows_pipeline_dependencies = [
"{}-build-e2e-publish".format(ver_mode),
"{}-test-frontend".format(ver_mode),
]
pipelines.extend([
pipeline(
name = "{}-oss-build-e2e-publish".format(ver_mode),
edition = "oss",
name = "{}-build-e2e-publish".format(ver_mode),
trigger = trigger,
services = [],
steps = init_steps + build_steps + publish_steps,
@ -248,8 +227,7 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
if ver_mode not in ("release"):
pipelines.append(pipeline(
name = "{}-oss-integration-tests".format(ver_mode),
edition = "oss",
name = "{}-integration-tests".format(ver_mode),
trigger = trigger,
services = services,
steps = [
@ -264,291 +242,19 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
volumes = volumes,
))
pipelines.append(windows_pipeline)
return pipelines
def enterprise_pipelines(ver_mode = ver_mode, trigger = release_trigger):
"""Generates all pipelines used for Grafana Enterprise.
Args:
ver_mode: controls which steps are included in the pipeline.
Defaults to 'release'.
trigger: controls which events can trigger the pipeline execution.
Defaults to tag events for tags with a 'v' prefix.
Returns:
List of Drone pipelines.
"""
if ver_mode == "release":
source = "${DRONE_TAG}"
elif ver_mode == "release-branch":
source = "${DRONE_BRANCH}"
else:
source = "${DRONE_COMMIT}"
environment = {"EDITION": "enterprise"}
services = integration_test_services(edition = "enterprise")
volumes = integration_test_services_volumes()
init_steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(source = source),
init_enterprise_step(ver_mode),
compile_build_cmd("enterprise"),
] + with_deps(
[
wire_install_step(),
yarn_install_step(),
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
],
[
"init-enterprise",
],
)
build_steps = [
build_backend_step(edition = "enterprise", ver_mode = ver_mode),
build_frontend_step(edition = "enterprise", ver_mode = ver_mode),
build_frontend_package_step(edition = "enterprise", ver_mode = ver_mode),
build_plugins_step(edition = "enterprise", ver_mode = ver_mode),
package_step(
edition = "enterprise",
ver_mode = ver_mode,
),
copy_packages_for_docker_step(),
build_docker_images_step(edition = "enterprise", publish = True),
build_docker_images_step(
edition = "enterprise",
publish = True,
ubuntu = True,
),
grafana_server_step(edition = "enterprise"),
e2e_tests_step("dashboards-suite", tries = 3),
e2e_tests_step("smoke-tests-suite", tries = 3),
e2e_tests_step("panels-suite", tries = 3),
e2e_tests_step("various-suite", tries = 3),
e2e_tests_artifacts(),
]
publish_steps = []
if ver_mode in (
"release",
"release-branch",
):
upload_packages_enterprise = upload_packages_step(
edition = "enterprise",
ver_mode = ver_mode,
trigger = trigger_oss,
)
upload_packages_enterprise["depends_on"] = ["package"]
publish_steps.extend(
[
upload_cdn_step(
edition = "enterprise",
ver_mode = ver_mode,
trigger = trigger_oss,
),
upload_packages_enterprise,
],
)
integration_test_steps = [
postgres_integration_tests_step(),
mysql_integration_tests_step(),
redis_integration_tests_step(),
memcached_integration_tests_step(),
]
# We don't need to run integration tests at release time since they have
# been run multiple times before:
if ver_mode in ("release"):
integration_test_steps = []
volumes = []
windows_pipeline = pipeline(
name = "{}-enterprise-windows".format(ver_mode),
edition = "enterprise",
name = "{}-windows".format(ver_mode),
trigger = trigger,
steps = get_windows_steps(edition = "enterprise", ver_mode = ver_mode),
steps = get_windows_steps(ver_mode = ver_mode),
platform = "windows",
depends_on = [
"{}-enterprise-build-e2e-publish".format(ver_mode),
"{}-enterprise-test-frontend".format(ver_mode),
"{}-enterprise-test-backend".format(ver_mode),
],
depends_on = windows_pipeline_dependencies,
environment = environment,
)
pipelines = [
pipeline(
name = "{}-enterprise-build-e2e-publish".format(ver_mode),
edition = "enterprise",
trigger = trigger,
services = [],
steps = init_steps + build_steps + publish_steps,
environment = environment,
volumes = volumes,
),
test_frontend_enterprise(trigger, ver_mode, source = source),
test_backend_enterprise(trigger, ver_mode, source = source),
]
if ver_mode not in ("release"):
pipelines.append(pipeline(
name = "{}-enterprise-integration-tests".format(ver_mode),
edition = "enterprise",
trigger = trigger,
services = services,
steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(source = source),
init_enterprise_step(ver_mode),
] +
with_deps(
[
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
],
[
"init-enterprise",
],
) +
[
wire_install_step(),
] +
integration_test_steps,
environment = environment,
volumes = volumes,
))
pipelines.append(windows_pipeline)
return pipelines
def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_trigger):
"""Generate the next generation of pipelines for Grafana Enterprise.
Args:
prefix: a prefix for the pipeline name used to differentiate multiple instances of
the same pipeline.
Defaults to ''.
ver_mode: controls which steps are included in the pipeline.
Defaults to 'release'.
trigger: controls which events can trigger the pipeline execution.
Defaults to tag events for tags with a 'v' prefix.
Returns:
List of Drone pipelines.
"""
if ver_mode == "release":
source = "${DRONE_TAG}"
elif ver_mode == "release-branch":
source = "${DRONE_BRANCH}"
else:
source = "${DRONE_COMMIT}"
environment = {
"EDITION": "enterprise2",
}
volumes = integration_test_services_volumes()
init_steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(source = source),
init_enterprise_step(ver_mode),
compile_build_cmd("enterprise"),
] + with_deps(
[
wire_install_step(),
yarn_install_step(),
verify_gen_cue_step(),
],
[
"init-enterprise",
],
)
build_steps = [
build_frontend_step(edition = "enterprise", ver_mode = ver_mode),
build_frontend_package_step(edition = "enterprise", ver_mode = ver_mode),
build_plugins_step(edition = "enterprise", ver_mode = ver_mode),
build_backend_step(
edition = "enterprise2",
ver_mode = ver_mode,
variants = ["linux-amd64"],
),
]
fetch_images = fetch_images_step("enterprise2")
fetch_images.update(
{"depends_on": ["build-docker-images", "build-docker-images-ubuntu"]},
)
upload_cdn = upload_cdn_step(edition = "enterprise2", ver_mode = ver_mode)
upload_cdn["environment"].update(
{"ENTERPRISE2_CDN_PATH": from_secret("enterprise2-cdn-path")},
)
build_steps.extend(
[
package_step(
edition = "enterprise2",
ver_mode = ver_mode,
),
upload_cdn,
copy_packages_for_docker_step(edition = "enterprise2"),
build_docker_images_step(
edition = "enterprise2",
publish = True,
),
build_docker_images_step(
edition = "enterprise2",
publish = True,
ubuntu = True,
),
fetch_images,
publish_images_step(
"enterprise2",
"release",
docker_repo = "${{DOCKER_ENTERPRISE2_REPO}}",
),
],
)
publish_steps = []
if ver_mode in (
"release",
"release-branch",
):
step = upload_packages_step(edition = "enterprise2", ver_mode = ver_mode)
step["depends_on"] = ["package-enterprise2"]
publish_steps.append(step)
pipelines = [
pipeline(
name = "{}{}-enterprise2-build-e2e-publish".format(prefix, ver_mode),
edition = "enterprise",
trigger = trigger,
services = [],
steps = init_steps + build_steps + publish_steps,
volumes = volumes,
environment = environment,
),
]
return pipelines
def publish_artifacts_step():
return {
"name": "publish-artifacts",
@ -573,7 +279,7 @@ def publish_static_assets_step():
"STATIC_ASSET_EDITIONS": from_secret("static_asset_editions"),
},
"commands": [
"./bin/build artifacts static-assets --tag ${DRONE_TAG}",
"./bin/build artifacts static-assets --tag ${DRONE_TAG} --static-asset-editions=grafana-oss",
],
"depends_on": ["compile-build-cmd"],
}
@ -618,13 +324,12 @@ def publish_artifacts_pipelines(mode):
name = "publish-artifacts-{}".format(mode),
trigger = trigger,
steps = steps,
edition = "all",
environment = {"EDITION": "all"},
environment = {"EDITION": "oss"},
),
]
def publish_packages_pipeline():
"""Generates pipelines used for publishing packages for both OSS and enterprise.
"""Generates pipelines used for publishing packages for OSS.
Returns:
List of Drone pipelines. One for each of OSS and enterprise packages.
@ -636,40 +341,24 @@ def publish_packages_pipeline():
}
oss_steps = [
compile_build_cmd(),
publish_linux_packages_step(edition = "oss", package_manager = "deb"),
publish_linux_packages_step(edition = "oss", package_manager = "rpm"),
publish_grafanacom_step(edition = "oss", ver_mode = "release"),
publish_linux_packages_step(package_manager = "deb"),
publish_linux_packages_step(package_manager = "rpm"),
publish_grafanacom_step(ver_mode = "release"),
]
enterprise_steps = [
compile_build_cmd(),
publish_linux_packages_step(edition = "enterprise", package_manager = "deb"),
publish_linux_packages_step(edition = "enterprise", package_manager = "rpm"),
publish_grafanacom_step(edition = "enterprise", ver_mode = "release"),
]
deps = [
"publish-artifacts-public",
"publish-docker-oss-public",
"publish-docker-enterprise-public",
"publish-docker-public",
]
return [
pipeline(
name = "publish-packages-oss",
name = "publish-packages",
trigger = trigger,
steps = oss_steps,
edition = "all",
depends_on = deps,
environment = {"EDITION": "oss"},
),
pipeline(
name = "publish-packages-enterprise",
trigger = trigger,
steps = enterprise_steps,
edition = "all",
depends_on = deps,
environment = {"EDITION": "enterprise"},
),
]
def publish_npm_pipelines():
@ -689,8 +378,7 @@ def publish_npm_pipelines():
name = "publish-npm-packages-public",
trigger = trigger,
steps = steps,
edition = "all",
environment = {"EDITION": "all"},
environment = {"EDITION": "oss"},
),
]
@ -703,7 +391,7 @@ def integration_test_pipelines():
particular build.
Returns:
List of Drone pipelines (one for enterprise and one for oss integration tests)
List of Drone pipelines
"""
trigger = {
"event": ["promote"],
@ -715,15 +403,9 @@ def integration_test_pipelines():
postgres_integration_tests_step(),
mysql_integration_tests_step(),
]
enterprise_integration_test_steps = oss_integration_test_steps + [
redis_integration_tests_step(),
memcached_integration_tests_step(),
]
source = "${DRONE_TAG}"
pipelines.append(pipeline(
name = "integration-tests-oss",
edition = "oss",
name = "integration-tests",
trigger = trigger,
services = integration_test_services(edition = "oss"),
steps = [
@ -738,34 +420,6 @@ def integration_test_pipelines():
volumes = volumes,
))
pipelines.append(pipeline(
name = "integration-tests-enterprise",
edition = "enterprise",
trigger = trigger,
services = integration_test_services(edition = "enterprise"),
steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(source = source),
init_enterprise_step(ver_mode),
] +
with_deps(
[
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
],
[
"init-enterprise",
],
) +
[
wire_install_step(),
] +
enterprise_integration_test_steps,
environment = {"EDITION": "enterprise"},
volumes = volumes,
))
return pipelines
def verify_release_pipeline(
@ -775,11 +429,8 @@ def verify_release_pipeline(
version = "${DRONE_TAG}",
trigger = release_trigger,
depends_on = [
"release-oss-build-e2e-publish",
"release-enterprise-build-e2e-publish",
"release-enterprise2-build-e2e-publish",
"release-oss-windows",
"release-enterprise-windows",
"release-build-e2e-publish",
"release-windows",
]):
"""
Runs a script that 'gsutil stat's every artifact that should have been produced by the pre-release process.
@ -805,7 +456,6 @@ def verify_release_pipeline(
return pipeline(
depends_on = depends_on,
name = name,
edition = "all",
trigger = trigger,
steps = [step],
)

45
scripts/drone/pipelines/aws_marketplace.star
View File

@ -1,45 +0,0 @@
"""
This module contains steps and pipelines publishing to AWS Marketplace.
"""
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
)
load("scripts/drone/vault.star", "from_secret")
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/utils/images.star",
"images",
)
def publish_aws_marketplace_step():
return {
"name": "publish-aws-marketplace",
"image": images["publish_image"],
"commands": ["./bin/build publish aws --image grafana/grafana-enterprise --repo grafana-labs/grafanaenterprise --product 422b46fb-bea6-4f27-8bcc-832117bd627e"],
"depends_on": ["compile-build-cmd"],
"environment": {
"AWS_REGION": from_secret("aws_region"),
"AWS_ACCESS_KEY_ID": from_secret("aws_access_key_id"),
"AWS_SECRET_ACCESS_KEY": from_secret("aws_secret_access_key"),
},
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
}
def publish_aws_marketplace_pipeline(mode):
trigger = {
"event": ["promote"],
"target": [mode],
}
return [pipeline(
name = "publish-aws-marketplace-{}".format(mode),
trigger = trigger,
steps = [compile_build_cmd(), publish_aws_marketplace_step()],
edition = "",
depends_on = ["publish-docker-enterprise-public"],
environment = {"EDITION": "enterprise2"},
)]

25
scripts/drone/pipelines/build.star
View File

@ -49,8 +49,7 @@ def build_e2e(trigger, ver_mode):
Drone pipeline.
"""
edition = "oss"
environment = {"EDITION": edition}
environment = {"EDITION": "oss"}
init_steps = [
identify_runner_step(),
download_grabpl_step(),
@ -73,12 +72,12 @@ def build_e2e(trigger, ver_mode):
build_steps.extend(
[
build_backend_step(edition = edition, ver_mode = ver_mode),
build_frontend_step(edition = edition, ver_mode = ver_mode),
build_frontend_package_step(edition = edition, ver_mode = ver_mode),
build_plugins_step(edition = edition, ver_mode = ver_mode),
package_step(edition = edition, ver_mode = ver_mode),
grafana_server_step(edition = edition),
build_backend_step(ver_mode = ver_mode),
build_frontend_step(ver_mode = ver_mode),
build_frontend_package_step(ver_mode = ver_mode),
build_plugins_step(ver_mode = ver_mode),
package_step(ver_mode = ver_mode),
grafana_server_step(),
e2e_tests_step("dashboards-suite"),
e2e_tests_step("smoke-tests-suite"),
e2e_tests_step("panels-suite"),
@ -101,34 +100,28 @@ def build_e2e(trigger, ver_mode):
store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode),
frontend_metrics_step(trigger = trigger_oss),
build_docker_images_step(
edition = edition,
publish = False,
),
build_docker_images_step(
edition = edition,
publish = False,
ubuntu = True,
),
publish_images_step(
docker_repo = "grafana",
edition = edition,
trigger = trigger_oss,
ver_mode = ver_mode,
),
publish_images_step(
docker_repo = "grafana-oss",
edition = edition,
trigger = trigger_oss,
ver_mode = ver_mode,
),
release_canary_npm_packages_step(trigger = trigger_oss),
upload_packages_step(
edition = edition,
trigger = trigger_oss,
ver_mode = ver_mode,
),
upload_cdn_step(
edition = edition,
trigger = trigger_oss,
ver_mode = ver_mode,
),
@ -141,18 +134,15 @@ def build_e2e(trigger, ver_mode):
archs = [
"amd64",
],
edition = edition,
),
build_docker_images_step(
archs = [
"amd64",
],
edition = edition,
ubuntu = True,
),
publish_images_step(
docker_repo = "grafana",
edition = edition,
trigger = trigger_oss,
ver_mode = ver_mode,
),
@ -165,7 +155,6 @@ def build_e2e(trigger, ver_mode):
return pipeline(
name = "{}-build-e2e{}".format(ver_mode, publish_suffix),
edition = "oss",
environment = environment,
services = [],
steps = init_steps + build_steps,

2
scripts/drone/pipelines/ci_images.star
View File

@ -28,7 +28,6 @@ def publish_ci_windows_test_image_pipeline():
pl = pipeline(
name = "publish-ci-windows-test-image",
trigger = trigger,
edition = "",
platform = "windows",
steps = [
{
@ -79,7 +78,6 @@ def publish_ci_build_container_image_pipeline():
pl = pipeline(
name = "publish-ci-build-container-image",
trigger = trigger,
edition = "",
steps = [
{
"name": "validate-version",

1
scripts/drone/pipelines/docs.star
View File

@ -39,7 +39,6 @@ def docs_pipelines(ver_mode, trigger):
return pipeline(
name = "{}-docs".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = steps,

43
scripts/drone/pipelines/github.star
View File

@ -1,43 +0,0 @@
"""
This module contains steps and pipelines relating to GitHub.
"""
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"fetch_images_step",
)
load("scripts/drone/vault.star", "from_secret")
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/utils/images.star",
"images",
)
def publish_github_step():
return {
"name": "publish-github",
"image": images["publish_image"],
"commands": ["./bin/build publish github --repo $${GH_REGISTRY} --create"],
"depends_on": ["fetch-images-enterprise2"],
"environment": {
"GH_TOKEN": from_secret("github_token"),
"GH_REGISTRY": from_secret("gh_registry"),
},
}
def publish_github_pipeline(mode):
trigger = {
"event": ["promote"],
"target": [mode],
}
return [pipeline(
name = "publish-github-{}".format(mode),
trigger = trigger,
steps = [compile_build_cmd(), fetch_images_step("enterprise2"), publish_github_step()],
edition = "",
environment = {"EDITION": "enterprise2"},
)]

6
scripts/drone/pipelines/integration_tests.star
View File

@ -49,11 +49,6 @@ def integration_tests(trigger, prefix, ver_mode = "pr"):
# In pull requests, attempt to clone grafana enterprise.
init_steps.append(enterprise_setup_step())
# Ensure that verif_gen_cue happens after we clone enterprise
# At the time of writing this, very_gen_cue is depended on by the wire step which is what everything else depends on.
verify_step["depends_on"].append("clone-enterprise")
verify_jsonnet_step["depends_on"].append("clone-enterprise")
init_steps += [
download_grabpl_step(),
compile_build_cmd(),
@ -70,7 +65,6 @@ def integration_tests(trigger, prefix, ver_mode = "pr"):
return pipeline(
name = "{}-integration-tests".format(prefix),
edition = "oss",
trigger = trigger,
environment = environment,
services = services,

2
scripts/drone/pipelines/lint_backend.star
View File

@ -39,7 +39,6 @@ def lint_backend_pipeline(trigger, ver_mode):
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
init_steps.append(enterprise_setup_step())
wire_step["depends_on"].append("clone-enterprise")
init_steps.append(wire_step)
@ -52,7 +51,6 @@ def lint_backend_pipeline(trigger, ver_mode):
return pipeline(
name = "{}-lint-backend".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = init_steps + test_steps,

4
scripts/drone/pipelines/lint_frontend.star
View File

@ -32,9 +32,6 @@ def lint_frontend_pipeline(trigger, ver_mode):
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
init_steps = [enterprise_setup_step()]
# Ensure the lint step happens after the clone-enterprise step
lint_step["depends_on"].append("clone-enterprise")
init_steps += [
identify_runner_step(),
@ -46,7 +43,6 @@ def lint_frontend_pipeline(trigger, ver_mode):
return pipeline(
name = "{}-lint-frontend".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = init_steps + test_steps,

30
scripts/drone/pipelines/publish_images.star
View File

@ -15,13 +15,10 @@ load(
"pipeline",
)
def publish_image_steps(edition, docker_repo):
def publish_image_steps(docker_repo):
"""Generates the steps used for publising Docker images using grabpl.
Args:
edition: controls which version of an image is fetched in the case of a release.
It also controls which publishing implementation is used.
If edition == 'oss', it additionally publishes the grafana/grafana-oss repository.
docker_repo: the Docker image name.
It is combined with the 'grafana/' library prefix.
@ -32,15 +29,11 @@ def publish_image_steps(edition, docker_repo):
identify_runner_step(),
download_grabpl_step(),
compile_build_cmd(),
fetch_images_step(edition),
publish_images_step(edition, "release", docker_repo),
fetch_images_step(),
publish_images_step("release", docker_repo),
publish_images_step("release", "grafana-oss"),
]
if edition == "oss":
steps.append(
publish_images_step(edition, "release", "grafana-oss"),
)
return steps
def publish_image_pipelines_public():
@ -56,20 +49,9 @@ def publish_image_pipelines_public():
}
return [
pipeline(
name = "publish-docker-oss-{}".format(mode),
name = "publish-docker-{}".format(mode),
trigger = trigger,
steps = publish_image_steps(edition = "oss", docker_repo = "grafana"),
edition = "",
steps = publish_image_steps(docker_repo = "grafana"),
environment = {"EDITION": "oss"},
),
pipeline(
name = "publish-docker-enterprise-{}".format(mode),
trigger = trigger,
steps = publish_image_steps(
edition = "enterprise",
docker_repo = "grafana-enterprise",
),
edition = "",
environment = {"EDITION": "enterprise"},
),
]

1
scripts/drone/pipelines/shellcheck.star
View File

@ -46,7 +46,6 @@ def shellcheck_pipeline():
]
return pipeline(
name = "pr-shellcheck",
edition = "oss",
trigger = trigger,
services = [],
steps = steps,

68
scripts/drone/pipelines/test_backend.star
View File

@ -4,12 +4,9 @@ This module returns the pipeline used for testing backend code.
load(
"scripts/drone/steps/lib.star",
"clone_enterprise_step",
"compile_build_cmd",
"download_grabpl_step",
"enterprise_setup_step",
"identify_runner_step",
"init_enterprise_step",
"test_backend_integration_step",
"test_backend_step",
"verify_gen_cue_step",
@ -19,7 +16,6 @@ load(
load(
"scripts/drone/utils/utils.star",
"pipeline",
"with_deps",
)
def test_backend(trigger, ver_mode):
@ -43,14 +39,9 @@ def test_backend(trigger, ver_mode):
# In pull requests, attempt to clone grafana enterprise.
steps.append(enterprise_setup_step())
# Ensure that verif_gen_cue happens after we clone enterprise
# At the time of writing this, very_gen_cue is depended on by the wire step which is what everything else depends on.
verify_step["depends_on"].append("clone-enterprise")
verify_jsonnet_step["depends_on"].append("clone-enterprise")
steps += [
identify_runner_step(),
compile_build_cmd(edition = "oss"),
compile_build_cmd(),
verify_step,
verify_jsonnet_step,
wire_install_step(),
@ -58,63 +49,8 @@ def test_backend(trigger, ver_mode):
test_backend_integration_step(),
]
pipeline_name = "{}-test-backend".format(ver_mode)
if ver_mode in ("release-branch", "release"):
pipeline_name = "{}-{}-test-backend".format(ver_mode, "oss")
return pipeline(
name = pipeline_name,
edition = "oss",
trigger = trigger,
steps = steps,
environment = environment,
)
def test_backend_enterprise(trigger, ver_mode, source, edition = "enterprise"):
"""Generates the pipeline used for testing backend enterprise code.
Args:
trigger: a Drone trigger for the pipeline.
ver_mode: affects the pipeline name.
source: controls what revision of enterprise code to test with. The source of the PR, usually.
edition: affects the clone step in the pipeline and also affects the pipeline name.
Returns:
Drone pipeline.
"""
environment = {"EDITION": edition}
steps = (
[
clone_enterprise_step(source),
download_grabpl_step(),
init_enterprise_step(ver_mode),
identify_runner_step(),
compile_build_cmd(edition),
] +
with_deps(
[
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
],
[
"init-enterprise",
],
) +
[
wire_install_step(),
test_backend_step(),
test_backend_integration_step(),
]
)
pipeline_name = "{}-test-backend".format(ver_mode)
if ver_mode in ("release-branch", "release"):
pipeline_name = "{}-{}-test-backend".format(ver_mode, edition)
return pipeline(
name = pipeline_name,
edition = edition,
name = "{}-test-backend".format(ver_mode),
trigger = trigger,
steps = steps,
environment = environment,

58
scripts/drone/pipelines/test_frontend.star
View File

@ -5,18 +5,15 @@ This module returns the pipeline used for testing backend code.
load(
"scripts/drone/steps/lib.star",
"betterer_frontend_step",
"clone_enterprise_step",
"download_grabpl_step",
"enterprise_setup_step",
"identify_runner_step",
"init_enterprise_step",
"test_frontend_step",
"yarn_install_step",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
"with_deps",
)
def test_frontend(trigger, ver_mode):
@ -35,68 +32,19 @@ def test_frontend(trigger, ver_mode):
identify_runner_step(),
download_grabpl_step(),
yarn_install_step(),
betterer_frontend_step(edition = "oss"),
betterer_frontend_step(),
]
pipeline_name = "{}-test-frontend".format(ver_mode)
test_step = test_frontend_step(edition = "oss")
test_step = test_frontend_step()
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
steps.append(enterprise_setup_step())
# Also, make the test step depend on 'clone-enterprise
test_step["depends_on"].append("clone-enterprise")
steps.append(test_step)
pipeline_name = "{}-test-frontend".format(ver_mode)
if ver_mode in ("release-branch", "release"):
pipeline_name = "{}-oss-test-frontend".format(ver_mode)
return pipeline(
name = pipeline_name,
edition = "oss",
trigger = trigger,
steps = steps,
environment = environment,
)
def test_frontend_enterprise(trigger, ver_mode, source, edition = "enterprise"):
"""Generates the pipeline used for testing frontend enterprise code.
Args:
trigger: a Drone trigger for the pipeline.
ver_mode: affects the pipeline name.
source: controls what revision of Grafana code to test with.
edition: affects the clone step in the pipeline and also affects the pipeline name.
Returns:
Drone pipeline.
"""
environment = {"EDITION": edition}
steps = (
[
download_grabpl_step(),
clone_enterprise_step(source),
init_enterprise_step(ver_mode),
identify_runner_step(),
] +
with_deps([yarn_install_step()], ["init-enterprise"]) +
[
betterer_frontend_step(edition),
test_frontend_step(edition),
]
)
pipeline_name = "{}-test-frontend".format(ver_mode)
if ver_mode in ("release-branch", "release"):
pipeline_name = "{}-{}-test-frontend".format(ver_mode, edition)
return pipeline(
name = pipeline_name,
edition = edition,
name = "{}-test-frontend".format(ver_mode),
trigger = trigger,
steps = steps,
environment = environment,

1
scripts/drone/pipelines/trigger_downstream.star
View File

@ -36,7 +36,6 @@ def enterprise_downstream_pipeline():
]
return pipeline(
name = "main-trigger-downstream",
edition = "oss",
trigger = trigger,
services = [],
steps = steps,

1
scripts/drone/pipelines/verify_drone.star
View File

@ -22,7 +22,6 @@ def verify_drone(trigger, ver_mode):
]
return pipeline(
name = "{}-verify-drone".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = steps,

1
scripts/drone/pipelines/verify_starlark.star
View File

@ -22,7 +22,6 @@ def verify_starlark(trigger, ver_mode):
]
return pipeline(
name = "{}-verify-starlark".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = steps,

1
scripts/drone/pipelines/whats_new_checker.star
View File

@ -35,7 +35,6 @@ def whats_new_checker_pipeline(trigger):
]
return pipeline(
name = "release-whatsnew-checker",
edition = "oss",
trigger = trigger,
services = [],
steps = steps,

27
scripts/drone/pipelines/windows.star
View File

@ -10,7 +10,6 @@ load(
"scripts/drone/steps/lib.star",
"get_windows_steps",
"windows_clone_step",
"windows_init_enterprise_steps",
"windows_test_backend_step",
"windows_wire_install_step",
)
@ -34,23 +33,19 @@ def windows_test_backend(trigger, edition, ver_mode):
windows_clone_step(),
]
if edition == "enterprise":
steps.extend(windows_init_enterprise_steps(ver_mode))
else:
steps.extend([{
"name": "windows-init",
"image": windows_images["windows_go_image"],
"depends_on": ["clone"],
"commands": [],
}])
steps.extend([{
"name": "windows-init",
"image": windows_images["windows_go_image"],
"depends_on": ["clone"],
"commands": [],
}])
steps.extend([
windows_wire_install_step(edition),
windows_test_backend_step(),
])
pl = pipeline(
name = "{}-{}-test-backend-windows".format(ver_mode, edition),
edition = edition,
name = "{}-test-backend-windows".format(ver_mode),
trigger = trigger,
steps = steps,
depends_on = [],
@ -62,25 +57,23 @@ def windows_test_backend(trigger, edition, ver_mode):
}
return pl
def windows(trigger, edition, ver_mode):
def windows(trigger, ver_mode):
"""Generates the pipeline used for building Grafana on Windows.
Args:
trigger: a Drone trigger for the pipeline.
edition: controls whether enterprise code is included in the pipeline steps.
ver_mode: controls whether a pre-release or actual release pipeline is generated.
Also indirectly controls which version of enterprise code is used.
Returns:
Drone pipeline.
"""
environment = {"EDITION": edition}
environment = {"EDITION": "oss"}
return pipeline(
name = "main-windows",
edition = edition,
trigger = dict(trigger, repo = ["grafana/grafana"]),
steps = get_windows_steps(edition, ver_mode),
steps = get_windows_steps(ver_mode),
depends_on = [
"main-test-frontend",
"main-test-backend",

2
scripts/drone/rgm.star
View File

@ -70,7 +70,6 @@ def rgm_main():
return pipeline(
name = "rgm-main-prerelease",
edition = "all",
trigger = trigger,
steps = rgm_build(),
depends_on = ["main-test-backend", "main-test-frontend"],
@ -95,7 +94,6 @@ tag_trigger = {
def rgm_tag():
return pipeline(
name = "rgm-tag-prerelease",
edition = "all",
trigger = tag_trigger,
steps = rgm_build(script = "drone_publish_tag.sh"),
depends_on = [],

414
scripts/drone/steps/lib.star
View File

@ -99,29 +99,6 @@ def enterprise_setup_step(source = "${DRONE_SOURCE_BRANCH}", canFail = True):
return step
def clone_enterprise_step(source = "${DRONE_COMMIT}"):
"""Clone the enterprise source into the ./grafana-enterprise directory.
Args:
source: controls which revision of grafana-enterprise is checked out, if it exists. The name 'source' derives from the 'source branch' of a pull request.
Returns:
Drone step.
"""
step = {
"name": "clone-enterprise",
"image": images["build_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
"commands": [
'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git"',
"cd grafana-enterprise",
"git checkout {}".format(source),
],
}
return step
def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFail = False, location = "grafana-enterprise"):
"""Clone the enterprise source into the ./grafana-enterprise directory.
@ -153,109 +130,6 @@ def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFai
return step
def init_enterprise_step(ver_mode):
"""Adds the enterprise deployment configuration into the source directory.
Args:
ver_mode: controls what revision of the OSS source to use.
If ver_mode is 'release', the step uses the tagged revision.
Otherwise, the DRONE_SOURCE_BRANCH is used.
Returns:
Drone step.
"""
source_commit = ""
if ver_mode == "release":
source_commit = " ${DRONE_TAG}"
environment = {
"GITHUB_TOKEN": from_secret("github_token"),
}
token = "--github-token $${GITHUB_TOKEN}"
elif ver_mode == "release-branch":
environment = {
"GITHUB_TOKEN": from_secret("github_token"),
}
token = "--github-token $${GITHUB_TOKEN}"
else:
environment = {}
token = ""
return {
"name": "init-enterprise",
"image": images["build_image"],
"depends_on": [
"clone-enterprise",
"grabpl",
],
"environment": environment,
"commands": [
"mv bin/grabpl /tmp/",
"rmdir bin",
"mv grafana-enterprise /tmp/",
"/tmp/grabpl init-enterprise {} /tmp/grafana-enterprise{}".format(
token,
source_commit,
).rstrip(),
"mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json",
"mkdir bin",
"mv /tmp/grabpl bin/",
],
}
def windows_init_enterprise_steps(ver_mode):
"""Performs init-enterprise steps in a Windows environment
Args:
ver_mode: in what mode should this be run
Returns:
A list of steps setting up an enterprise folder
"""
if ver_mode == "release":
source = "${DRONE_TAG}"
elif ver_mode == "release-branch":
source = "$$env:DRONE_BRANCH"
else:
source = "main"
clone_cmds = [
'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"',
"cd grafana-enterprise",
"git checkout {}".format(source),
]
init_cmds = [
# Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into
"cp -r grafana-enterprise C:\\App\\grafana-enterprise",
"rm -r -force grafana-enterprise",
"cp grabpl.exe C:\\App\\grabpl.exe",
"rm -force grabpl.exe",
"C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise {}".format(source),
"cp C:\\App\\grabpl.exe grabpl.exe",
]
steps = []
steps.extend(
[
download_grabpl_step(platform = "windows"),
{
"name": "clone",
"image": windows_images["wix_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
"commands": clone_cmds,
},
{
"name": "windows-init",
"image": windows_images["wix_image"],
"commands": init_cmds,
"depends_on": ["clone"],
"environment": {"GITHUB_TOKEN": from_secret("github_token")},
},
],
)
return steps
def download_grabpl_step(platform = "linux"):
if platform == "windows":
return {
@ -486,11 +360,10 @@ def e2e_tests_artifacts():
],
}
def upload_cdn_step(edition, ver_mode, trigger = None):
def upload_cdn_step(ver_mode, trigger = None):
"""Uploads CDN assets using the Grafana build tool.
Args:
edition: controls the output directory for the CDN assets.
ver_mode: only uses the step trigger when ver_mode == 'release-branch' or 'main'
trigger: a Drone trigger for the step.
Defaults to None.
@ -498,41 +371,29 @@ def upload_cdn_step(edition, ver_mode, trigger = None):
Returns:
Drone step.
"""
deps = []
if edition in "enterprise2":
deps.extend(
[
"package" + enterprise2_suffix(edition),
],
)
else:
deps.extend(
[
"grafana-server",
],
)
step = {
"name": "upload-cdn-assets" + enterprise2_suffix(edition),
"name": "upload-cdn-assets",
"image": images["publish_image"],
"depends_on": deps,
"depends_on": [
"grafana-server",
],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
"PRERELEASE_BUCKET": from_secret(prerelease_bucket),
},
"commands": [
"./bin/build upload-cdn --edition {}".format(edition),
"./bin/build upload-cdn --edition oss",
],
}
if trigger and ver_mode in ("release-branch", "main"):
step = dict(step, when = trigger)
return step
def build_backend_step(edition, ver_mode, variants = None):
def build_backend_step(ver_mode, variants = None):
"""Build the backend code using the Grafana build tool.
Args:
edition: controls which edition of the backend is built.
ver_mode: if ver_mode != 'release', pass the DRONE_BUILD_NUMBER environment
variable as the value for the --build-id option.
TODO: is this option actually used by the build-backend subcommand?
@ -550,22 +411,19 @@ def build_backend_step(edition, ver_mode, variants = None):
# TODO: Convert number of jobs to percentage
if ver_mode == "release":
cmds = [
"./bin/build build-backend --jobs 8 --edition {} ${{DRONE_TAG}}".format(
edition,
),
"./bin/build build-backend --jobs 8 --edition oss ${DRONE_TAG}",
]
else:
build_no = "${DRONE_BUILD_NUMBER}"
cmds = [
"./bin/build build-backend --jobs 8 --edition {} --build-id {}{}".format(
edition,
"./bin/build build-backend --jobs 8 --edition oss --build-id {}{}".format(
build_no,
variants_str,
),
]
return {
"name": "build-backend" + enterprise2_suffix(edition),
"name": "build-backend",
"image": images["build_image"],
"depends_on": [
"wire-install",
@ -574,11 +432,10 @@ def build_backend_step(edition, ver_mode, variants = None):
"commands": cmds,
}
def build_frontend_step(edition, ver_mode):
def build_frontend_step(ver_mode):
"""Build the frontend code using the Grafana build tool.
Args:
edition: controls which edition of the frontend is built.
ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment
variable as a build identifier.
@ -591,11 +448,11 @@ def build_frontend_step(edition, ver_mode):
if ver_mode == "release":
cmds = [
"./bin/build build-frontend --jobs 8 " +
"--edition {} ${{DRONE_TAG}}".format(edition),
"--edition oss ${DRONE_TAG}",
]
else:
cmds = [
"./bin/build build-frontend --jobs 8 --edition {} ".format(edition) +
"./bin/build build-frontend --jobs 8 --edition oss" +
"--build-id {}".format(build_no),
]
@ -612,11 +469,10 @@ def build_frontend_step(edition, ver_mode):
"commands": cmds,
}
def build_frontend_package_step(edition, ver_mode):
def build_frontend_package_step(ver_mode):
"""Build the frontend packages using the Grafana build tool.
Args:
edition: controls which edition of the frontend is built.
ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment
variable as a build identifier.
@ -629,11 +485,11 @@ def build_frontend_package_step(edition, ver_mode):
if ver_mode == "release":
cmds = [
"./bin/build build-frontend-packages --jobs 8 " +
"--edition {} ${{DRONE_TAG}}".format(edition),
"--edition oss ${DRONE_TAG}",
]
else:
cmds = [
"./bin/build build-frontend-packages --jobs 8 --edition {} ".format(edition) +
"./bin/build build-frontend-packages --jobs 8 --edition oss" +
"--build-id {}".format(build_no),
]
@ -650,7 +506,7 @@ def build_frontend_package_step(edition, ver_mode):
"commands": cmds,
}
def build_plugins_step(edition, ver_mode):
def build_plugins_step(ver_mode):
if ver_mode != "pr":
env = {
"GRAFANA_API_KEY": from_secret("grafana_api_key"),
@ -667,7 +523,7 @@ def build_plugins_step(edition, ver_mode):
],
"commands": [
# TODO: Use percentage for num jobs
"./bin/build build-plugins --jobs 8 --edition {}".format(edition),
"./bin/build build-plugins --jobs 8 --edition oss",
],
}
@ -699,50 +555,40 @@ def test_backend_integration_step():
],
}
def betterer_frontend_step(edition = "oss"):
def betterer_frontend_step():
"""Run betterer on frontend code.
Args:
edition: controls whether enterprise code is also included in the source.
Defaults to 'oss'.
Returns:
Drone step.
"""
deps = []
if edition == "enterprise":
deps.extend(["init-enterprise"])
deps.extend(["yarn-install"])
return {
"name": "betterer-frontend",
"image": images["build_image"],
"depends_on": deps,
"depends_on": [
"yarn-install",
],
"commands": [
"yarn betterer ci",
],
}
def test_frontend_step(edition = "oss"):
def test_frontend_step():
"""Runs tests on frontend code.
Args:
edition: controls whether enterprise code is also included in the source.
Defaults to 'oss'.
Returns:
Drone step.
"""
deps = []
if edition == "enterprise":
deps.extend(["init-enterprise"])
deps.extend(["yarn-install"])
return {
"name": "test-frontend",
"image": images["build_image"],
"environment": {
"TEST_MAX_WORKERS": "50%",
},
"depends_on": deps,
"depends_on": [
"yarn-install",
],
"commands": [
"yarn run ci:test-frontend",
],
@ -852,11 +698,10 @@ def codespell_step():
],
}
def package_step(edition, ver_mode):
def package_step(ver_mode):
"""Packages Grafana with the Grafana build tool.
Args:
edition: controls which edition of Grafana is packaged.
ver_mode: controls whether the packages are signed for a release.
If ver_mode != 'release', use the DRONE_BUILD_NUMBER environment
variable as a build identifier.
@ -866,7 +711,7 @@ def package_step(edition, ver_mode):
"""
deps = [
"build-plugins",
"build-backend" + enterprise2_suffix(edition),
"build-backend",
"build-frontend",
"build-frontend-packages",
]
@ -890,29 +735,28 @@ def package_step(edition, ver_mode):
# TODO: Use percentage for jobs
if ver_mode == "release":
cmds = [
"{}./bin/build package --jobs 8 --edition {} ".format(test_args, edition) +
"{} ${{DRONE_TAG}}".format(sign_args),
"{}./bin/build package --jobs 8 --edition oss ".format(test_args) +
"{} $${{DRONE_TAG}}".format(sign_args),
]
else:
build_no = "${DRONE_BUILD_NUMBER}"
cmds = [
"{}./bin/build package --jobs 8 --edition {} ".format(test_args, edition) +
"{}./bin/build package --jobs 8 --edition oss ".format(test_args) +
"--build-id {}{}".format(build_no, sign_args),
]
return {
"name": "package" + enterprise2_suffix(edition),
"name": "package",
"image": images["build_image"],
"depends_on": deps,
"environment": env,
"commands": cmds,
}
def grafana_server_step(edition, port = 3001):
def grafana_server_step(port = 3001):
"""Runs the grafana-server binary as a service.
Args:
edition: controls which edition of grafana-server to run.
port: port to listen on.
Defaults to 3001.
@ -920,8 +764,6 @@ def grafana_server_step(edition, port = 3001):
Drone step.
"""
environment = {"PORT": port, "ARCH": "linux-amd64"}
if edition == "enterprise":
environment["RUNDIR"] = "scripts/grafana-server/tmp-grafana-enterprise"
return {
"name": "grafana-server",
@ -1019,12 +861,12 @@ def build_docs_website_step():
],
}
def copy_packages_for_docker_step(edition = None):
def copy_packages_for_docker_step():
return {
"name": "copy-packages-for-docker",
"image": images["build_image"],
"depends_on": [
"package" + enterprise2_suffix(edition),
"package",
],
"commands": [
"ls dist/*.tar.gz*",
@ -1032,11 +874,10 @@ def copy_packages_for_docker_step(edition = None):
],
}
def build_docker_images_step(edition, archs = None, ubuntu = False, publish = False):
def build_docker_images_step(archs = None, ubuntu = False, publish = False):
"""Build Docker images using the Grafana build tool.
Args:
edition: controls which repository the image is published to.
archs: a list of architectures to build the image for.
Defaults to None.
ubuntu: controls whether the final image is built from an Ubuntu base image.
@ -1047,7 +888,7 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa
Returns:
Drone step.
"""
cmd = "./bin/build build-docker --edition {}".format(edition)
cmd = "./bin/build build-docker --edition oss"
if publish:
cmd += " --shouldSave"
@ -1063,11 +904,6 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa
"GCP_KEY": from_secret("gcp_key"),
}
if edition == "enterprise2":
environment.update(
{"DOCKER_ENTERPRISE2_REPO": from_secret("docker_enterprise2_repo")},
)
return {
"name": "build-docker-images" + ubuntu_sfx,
"image": images["cloudsdk_image"],
@ -1080,27 +916,24 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa
"environment": environment,
}
def fetch_images_step(edition):
def fetch_images_step():
return {
"name": "fetch-images-{}".format(edition),
"name": "fetch-images",
"image": images["cloudsdk_image"],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
"DOCKER_USER": from_secret("docker_username"),
"DOCKER_PASSWORD": from_secret("docker_password"),
"DOCKER_ENTERPRISE2_REPO": from_secret("docker_enterprise2_repo"),
},
"commands": ["./bin/build artifacts docker fetch --edition {}".format(edition)],
"commands": ["./bin/build artifacts docker fetch --edition oss"],
"depends_on": ["compile-build-cmd"],
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
}
def publish_images_step(edition, ver_mode, docker_repo, trigger = None):
def publish_images_step(ver_mode, docker_repo, trigger = None):
"""Generates a step for publishing public Docker images with grabpl.
Args:
edition: controls which version of an image is fetched in the case of a release.
It also controls which publishing implementation is used.
ver_mode: controls whether the image needs to be built or retrieved from a previous build.
If ver_mode == 'release', the previously built image is fetched instead of being built again.
docker_repo: the Docker image name.
@ -1129,22 +962,9 @@ def publish_images_step(edition, ver_mode, docker_repo, trigger = None):
deps = ["build-docker-images", "build-docker-images-ubuntu"]
if ver_mode == "release":
deps = ["fetch-images-{}".format(edition)]
deps = ["fetch-images"]
cmd += " --version-tag ${DRONE_TAG}"
if edition == "enterprise2":
name = edition
docker_repo = "$${DOCKER_ENTERPRISE2_REPO}"
environment.update(
{
"GCP_KEY": from_secret("gcp_key_hg"),
"DOCKER_ENTERPRISE2_REPO": from_secret("docker_enterprise2_repo"),
},
)
cmd = "./bin/build artifacts docker publish-enterprise2 --dockerhub-repo {}".format(
docker_repo,
)
if ver_mode == "pr":
environment = {
"DOCKER_USER": from_secret("docker_username_pr"),
@ -1268,16 +1088,10 @@ def release_canary_npm_packages_step(trigger = None):
step = dict(step, when = trigger)
return step
def enterprise2_suffix(edition):
if edition == "enterprise2":
return "-{}".format(edition)
return ""
def upload_packages_step(edition, ver_mode, trigger = None):
def upload_packages_step(ver_mode, trigger = None):
"""Upload packages to object storage.
Args:
edition: controls which edition of Grafana packages to upload.
ver_mode: when ver_mode == 'main', inhibit upload of enterprise
edition packages when executed.
trigger: a Drone trigger for the step.
@ -1287,7 +1101,7 @@ def upload_packages_step(edition, ver_mode, trigger = None):
Drone step.
"""
step = {
"name": "upload-packages" + enterprise2_suffix(edition),
"name": "upload-packages",
"image": images["publish_image"],
"depends_on": end_to_end_tests_deps(),
"environment": {
@ -1295,18 +1109,17 @@ def upload_packages_step(edition, ver_mode, trigger = None):
"PRERELEASE_BUCKET": from_secret("prerelease_bucket"),
},
"commands": [
"./bin/build upload-packages --edition {}".format(edition),
"./bin/build upload-packages --edition oss",
],
}
if trigger and ver_mode in ("release-branch", "main"):
step = dict(step, when = trigger)
return step
def publish_grafanacom_step(edition, ver_mode):
def publish_grafanacom_step(ver_mode):
"""Publishes Grafana packages to grafana.com.
Args:
edition: controls which edition of Grafana to publish to.
ver_mode: if ver_mode == 'main', pass the DRONE_BUILD_NUMBER environment
variable as the value for the --build-id option.
TODO: is this actually used by the grafanacom subcommand? I think it might
@ -1316,20 +1129,17 @@ def publish_grafanacom_step(edition, ver_mode):
Drone step.
"""
if ver_mode == "release":
cmd = "./bin/build publish grafana-com --edition {} ${{DRONE_TAG}}".format(
edition,
)
cmd = "./bin/build publish grafana-com --edition oss ${DRONE_TAG}"
elif ver_mode == "main":
build_no = "${DRONE_BUILD_NUMBER}"
cmd = "./bin/build publish grafana-com --edition {} --build-id {}".format(
edition,
cmd = "./bin/build publish grafana-com --edition oss --build-id {}".format(
build_no,
)
else:
fail("Unexpected version mode {}".format(ver_mode))
return {
"name": "publish-grafanacom-{}".format(edition),
"name": "publish-grafanacom",
"image": images["publish_image"],
"depends_on": [
"publish-linux-packages-deb",
@ -1344,7 +1154,7 @@ def publish_grafanacom_step(edition, ver_mode):
],
}
def publish_linux_packages_step(edition, package_manager = "deb"):
def publish_linux_packages_step(package_manager = "deb"):
return {
"name": "publish-linux-packages-{}".format(package_manager),
# See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image
@ -1360,8 +1170,7 @@ def publish_linux_packages_step(edition, package_manager = "deb"):
"gpg_passphrase": from_secret("packages_gpg_passphrase"),
"gpg_public_key": from_secret("packages_gpg_public_key"),
"gpg_private_key": from_secret("packages_gpg_private_key"),
"package_path": "gs://grafana-prerelease/artifacts/downloads/*${{DRONE_TAG}}/{}/**.{}".format(
edition,
"package_path": "gs://grafana-prerelease/artifacts/downloads/*$${{DRONE_TAG}}/oss/**.{}".format(
package_manager,
),
},
@ -1380,11 +1189,10 @@ def windows_clone_step():
],
}
def get_windows_steps(edition, ver_mode):
def get_windows_steps(ver_mode):
"""Generate the list of Windows steps.
Args:
edition: used to differentiate steps for different Grafana editions.
ver_mode: used to differentiate steps for different version modes.
Returns:
@ -1394,80 +1202,24 @@ def get_windows_steps(edition, ver_mode):
identify_runner_step("windows"),
]
if edition in ("enterprise", "enterprise2"):
if ver_mode == "release":
source = "${DRONE_TAG}"
elif ver_mode == "release-branch":
source = "$$env:DRONE_BRANCH"
else:
source = "$$env:DRONE_COMMIT"
init_cmds = [
'$$ProgressPreference = "SilentlyContinue"',
"Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format(
grabpl_version,
),
]
# For enterprise, we have to clone both OSS and enterprise and merge the latter into the former
download_grabpl_cmds = [
'$$ProgressPreference = "SilentlyContinue"',
"Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format(
grabpl_version,
),
]
steps.extend(
[
{
"name": "windows-init",
"image": windows_images["wix_image"],
"commands": init_cmds,
},
],
)
clone_cmds = [
'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"',
"cd grafana-enterprise",
"git checkout {}".format(source),
]
init_cmds = [
# Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into
"cp -r grafana-enterprise C:\\App\\grafana-enterprise",
"rm -r -force grafana-enterprise",
"cp grabpl.exe C:\\App\\grabpl.exe",
"rm -force grabpl.exe",
"C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise {}".format(source),
"cp C:\\App\\grabpl.exe grabpl.exe",
]
steps.extend(
[
{
"name": "clone",
"image": windows_images["wix_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
"commands": download_grabpl_cmds + clone_cmds,
},
{
"name": "windows-init",
"image": windows_images["wix_image"],
"commands": init_cmds,
"depends_on": ["clone"],
"environment": {"GITHUB_TOKEN": from_secret("github_token")},
},
],
)
else:
init_cmds = [
'$$ProgressPreference = "SilentlyContinue"',
"Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format(
grabpl_version,
),
]
steps.extend(
[
{
"name": "windows-init",
"image": windows_images["wix_image"],
"commands": init_cmds,
},
],
)
# TODO: Run windows backend tests
if (
ver_mode == "main" and (edition not in ("enterprise", "enterprise2"))
) or ver_mode in (
if ver_mode in (
"release",
"release-branch",
):
@ -1489,13 +1241,11 @@ def get_windows_steps(edition, ver_mode):
"rm gcpkey.json",
"cp C:\\App\\nssm-2.24.zip .",
]
if (
ver_mode == "main" and (edition not in ("enterprise", "enterprise2"))
) or ver_mode in ("release",):
if ver_mode in ("release",):
installer_commands.extend(
[
".\\grabpl.exe windows-installer --edition {} {}".format(
edition,
".\\grabpl.exe windows-installer --edition oss {}".format(
ver_part,
),
'$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]',
@ -1504,10 +1254,9 @@ def get_windows_steps(edition, ver_mode):
if ver_mode == "main":
installer_commands.extend(
[
"gsutil cp $$fname gs://{}/{}/{}/".format(bucket, edition, dir),
'gsutil cp "$$fname.sha256" gs://{}/{}/{}/'.format(
"gsutil cp $$fname gs://{}/oss/{}/".format(bucket, dir),
'gsutil cp "$$fname.sha256" gs://{}/oss/{}/'.format(
bucket,
edition,
dir,
),
],
@ -1515,16 +1264,14 @@ def get_windows_steps(edition, ver_mode):
else:
installer_commands.extend(
[
"gsutil cp $$fname gs://{}/{}/{}/{}/".format(
"gsutil cp $$fname gs://{}/{}/oss/{}/".format(
bucket,
ver_part,
edition,
dir,
),
'gsutil cp "$$fname.sha256" gs://{}/{}/{}/{}/'.format(
'gsutil cp "$$fname.sha256" gs://{}/{}/oss/{}/'.format(
bucket,
ver_part,
edition,
dir,
),
],
@ -1613,12 +1360,9 @@ def end_to_end_tests_deps():
"end-to-end-tests-various-suite",
]
def compile_build_cmd(edition = "oss"):
def compile_build_cmd():
dependencies = []
if edition in ("enterprise", "enterprise2"):
dependencies = [
"init-enterprise",
]
return {
"name": "compile-build-cmd",
"image": images["go_image"],

8
scripts/drone/utils/utils.star
View File

@ -13,7 +13,6 @@ drone_change_template = "`.drone.yml` and `starlark` files have been changed on
def pipeline(
name,
edition,
trigger,
steps,
services = [],
@ -30,7 +29,6 @@ def pipeline(
Args:
name: controls the pipeline name.
edition: used to differentiate the pipeline for enterprise builds.
trigger: a Drone trigger for the pipeline.
steps: the Drone steps for the pipeline.
services: auxilliary services used during the pipeline.
@ -98,12 +96,6 @@ def pipeline(
pipeline["volumes"].extend(volumes)
pipeline.update(platform_conf)
if edition in ("enterprise", "enterprise2"):
# We have a custom clone step for enterprise
pipeline["clone"] = {
"disable": True,
}
return pipeline
def notify_pipeline(

15
scripts/drone/vault.star
View File

@ -90,21 +90,6 @@ def secrets():
"infra/data/ci/packages-publish/bucket-credentials",
"Secret",
),
vault_secret(
"aws_region",
"secret/data/common/aws-marketplace",
"aws_region",
),
vault_secret(
"aws_access_key_id",
"secret/data/common/aws-marketplace",
"aws_access_key_id",
),
vault_secret(
"aws_secret_access_key",
"secret/data/common/aws-marketplace",
"aws_secret_access_key",
),
vault_secret(
"static_asset_editions",
"infra/data/ci/grafana-release-eng/artifact-publishing",

6
scripts/drone/version.star
View File

@ -4,8 +4,6 @@ This module returns the pipeline used for version branches.
load(
"scripts/drone/events/release.star",
"enterprise2_pipelines",
"enterprise_pipelines",
"oss_pipelines",
)
@ -14,7 +12,5 @@ trigger = {"ref": ["refs/heads/v[0-9]*"]}
def version_branch_pipelines():
return (
oss_pipelines(ver_mode = ver_mode, trigger = trigger) +
enterprise_pipelines(ver_mode = ver_mode, trigger = trigger) +
enterprise2_pipelines(ver_mode = ver_mode, trigger = trigger)
oss_pipelines(ver_mode = ver_mode, trigger = trigger)
)