IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

CI: Add nightly prerelease builds (#74119)

Browse Source 
* Add nightly prerelease builds

* Fix duplicated pipeline names

* Fix misnamed dependencies

* Fix misnamed dependencies

* Fix string formatting

* Add option to specify bucket to RGM pipeline

* Fix trigger?

* Comment out cron triggers

* Fix windows bucket for nightly

* Fix versioning for windows and verify pipelines

* Use grafana/grafana-build:dev-209553c

* Fix version on windows steps

* Fix version on windows steps

* Fix windows .zip path

* Fix windows .zip path

* Remove windows builds from nightly for now

* Remove verify release pipeline from nightly

* Add docstring to rgm_release

* Revert changes to get_windows_steps

* Simplify changes to rgm.star

* Use grafana/grafana-build:dev-f5a15d4

* Add rgm copy step

* Use grafana/grafana-build:dev-d88be0f

* Fix destination variable

* Escape copy destination environment variable

* Add -r flag to rgm copy command

* Add dependency to rgm-copy step

* Add dist volume

* Use absolute path for dist volume

* Move dist folder to drone workspace

* Delegate drone workspace path to grafana-build

* Use grafana/grafana-build:dev-66149b8

* Lower folder depth

* Use grafana/grafana-build:dev-7355791

* Add rgm-nightly-publish pipeline

* Merge imports on rgm.star

* Fix rgm_copy to allow copying to local destination

* Use grafana/grafana-build:dev-36ec1e2

* Use grafana/grafana-build:dev-634d8dc

* Use grafana/grafana-build:dev-7a93728

* Use grafana/grafana-build:dev-5e36725

* Use grafana/grafana-build:dev-f5ebe1f

* Fix copy source for nightly builds

* Fix drone build number on rgm-copy step

* Use grafana/grafana-build:dev-637583f

* Use grafana/grafana-build:dev-f2cc524

* Allow tag trigger on grafana/grafana for testing

* Use grafana/grafana-build:dev-c71d4b7

* Use grafana/grafana-build:dev-63beac8

* Use grafana/grafana-build:dev-224a0dd

* Add environment variables for package publishing

* Revert unintentional change to dataquery.cue

* Add package publish step to nightly pipeline

* Use GCS path for package publish

* Pre-evaluate drone workspace on packages path

* Use hardcoded drone workspace path

* Remove unused env from publish packages step

* Use grafana/grafana-build:dev-657ea6a

* Use grafana/grafana-build:dev-1a9beec

* Use grafana/grafana-build:dev-f0053c8

* Use grafana/grafana-build:main

* Use grafana/grafana-build:dev-ae5182f

* Use grafana/grafana-build:dev-ec3ec36

* Use grafana/grafana-build:dev-5e160d8

* Use grafana/grafana-build:dev-142d2dc

* Use grafana/grafana-build:dev-db6bff1

* Use grafana/grafana-build:main

* Change nightly trigger to cron
This commit is contained in:
Guilherme Caulada
2023-10-04 13:55:43 -03:00
committed by GitHub
parent 1b4c2fc948
commit 55781b486c
3 changed files with 761 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/drone/rgm.star
+179 -37
View File
@@ -24,10 +24,15 @@ load(
"scripts/drone/steps/lib_windows.star",
"get_windows_steps",
)
load(
"scripts/drone/utils/images.star",
"images",
)
load(
"scripts/drone/utils/utils.star",
"ignore_failure",
"pipeline",
"with_deps",
)
load(
"scripts/drone/variables.star",
@@ -36,29 +41,16 @@ load(
load(
"scripts/drone/vault.star",
"from_secret",
"npm_token",
"rgm_cdn_destination",
"rgm_dagger_token",
"rgm_destination",
"rgm_downloads_destination",
"rgm_gcp_key_base64",
"rgm_github_token",
"rgm_storybook_destination",
)
def rgm_env_secrets(env):
"""Adds the rgm secret ENV variables to the given env arg
Args:
env: A map of environment varables. This function will adds the necessary secrets to it (and potentially overwrite them).
Returns:
Drone step.
"""
env["GCP_KEY_BASE64"] = from_secret(rgm_gcp_key_base64)
env["DESTINATION"] = from_secret(rgm_destination)
env["GITHUB_TOKEN"] = from_secret(rgm_github_token)
env["_EXPERIMENTAL_DAGGER_CLOUD_TOKEN"] = from_secret(rgm_dagger_token)
env["GPG_PRIVATE_KEY"] = from_secret("packages_gpg_private_key")
env["GPG_PUBLIC_KEY"] = from_secret("packages_gpg_public_key")
env["GPG_PASSPHRASE"] = from_secret("packages_gpg_passphrase")
return env
docs_paths = {
"exclude": [
"*.md",
@@ -69,11 +61,6 @@ docs_paths = {
}
tag_trigger = {
"repo": {
"exclude": [
"grafana/grafana",
],
},
"event": {
"exclude": [
"promote",
@@ -89,22 +76,61 @@ tag_trigger = {
},
}
nightly_trigger = {
"event": {
"include": [
"cron",
],
},
"cron": {
"include": [
"nightly",
],
},
}
version_branch_trigger = {"ref": ["refs/heads/v[0-9]*"]}
def rgm_build(script = "drone_publish_main.sh", canFail = True):
def rgm_env_secrets(env):
"""Adds the rgm secret ENV variables to the given env arg
Args:
env: A map of environment varables. This function will adds the necessary secrets to it (and potentially overwrite them).
Returns:
Drone step.
"""
env["DESTINATION"] = from_secret(rgm_destination)
env["STORYBOOK_DESTINATION"] = from_secret(rgm_storybook_destination)
env["CDN_DESTINATION"] = from_secret(rgm_cdn_destination)
env["DOWNLOADS_DESTINATION"] = from_secret(rgm_downloads_destination)
env["PACKAGES_DESTINATION"] = "gs://grafana-packages-testing"
env["GCP_KEY_BASE64"] = from_secret(rgm_gcp_key_base64)
env["GITHUB_TOKEN"] = from_secret(rgm_github_token)
env["_EXPERIMENTAL_DAGGER_CLOUD_TOKEN"] = from_secret(rgm_dagger_token)
env["GPG_PRIVATE_KEY"] = from_secret("packages_gpg_private_key")
env["GPG_PUBLIC_KEY"] = from_secret("packages_gpg_public_key")
env["GPG_PASSPHRASE"] = from_secret("packages_gpg_passphrase")
env["DOCKER_USERNAME"] = from_secret("docker_username")
env["DOCKER_PASSWORD"] = from_secret("docker_password")
env["NPM_TOKEN"] = from_secret(npm_token)
env["GCOM_API_KEY"] = from_secret("grafana_api_key_dev")
return env
def rgm_run(name, script):
"""Returns a pipeline that does a full build & package of Grafana.
Args:
name: The name of the pipeline step.
script: The script in the container to run.
canFail: if true, then this pipeline can fail while the entire build will still succeed.
Returns:
Drone step.
"""
env = {
"GO_VERSION": golang_version,
}
rgm_build_step = {
"name": "rgm-build",
rgm_run_step = {
"name": name,
"image": "grafana/grafana-build:main",
"pull": "always",
"commands": [
@@ -117,14 +143,72 @@ def rgm_build(script = "drone_publish_main.sh", canFail = True):
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
}
if canFail:
rgm_build_step["failure"] = "ignore"
return [
rgm_build_step,
rgm_run_step,
]
def rgm_copy(src, dst):
"""Copies file from/to GCS.
Args:
src: source of the files.
dst: destination of the files.
Returns:
Drone steps.
"""
commands = [
"printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json",
"gcloud auth activate-service-account --key-file=/tmp/key.json",
"gcloud storage cp -r {} {}".format(src, dst),
]
if not dst.startswith("gs://"):
commands.insert(0, "mkdir -p {}".format(dst))
rgm_copy_step = {
"name": "rgm-copy",
"image": "google/cloud-sdk:alpine",
"commands": commands,
"environment": rgm_env_secrets({}),
}
return [
rgm_copy_step,
]
def rgm_publish_packages(bucket = "grafana-packages"):
"""Publish deb and rpm packages.
Args:
bucket: target bucket to publish the packages.
Returns:
Drone steps.
"""
steps = []
for package_manager in ["deb", "rpm"]:
steps.append({
"name": "publish-{}".format(package_manager),
# See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image
"image": images["package_publish"],
"privileged": True,
"settings": {
"access_key_id": from_secret("packages_access_key_id"),
"secret_access_key": from_secret("packages_secret_access_key"),
"service_account_json": from_secret("packages_service_account"),
"target_bucket": bucket,
"gpg_passphrase": from_secret("packages_gpg_passphrase"),
"gpg_public_key": from_secret("packages_gpg_public_key"),
"gpg_private_key": from_secret("packages_gpg_private_key"),
"package_path": "file:///drone/src/dist/*.{}".format(package_manager),
},
})
return steps
def rgm_main():
# Runs a package / build process (with some distros) when commits are merged to main
trigger = {
"event": [
"push",
@@ -139,15 +223,16 @@ def rgm_main():
return pipeline(
name = "rgm-main-prerelease",
trigger = trigger,
steps = rgm_build(canFail = True),
steps = rgm_run("rgm-build", "drone_publish_main.sh"),
depends_on = ["main-test-backend", "main-test-frontend"],
)
def rgm_tag():
# Runs a package / build process (with all distros) when a tag is made
return pipeline(
name = "rgm-tag-prerelease",
trigger = tag_trigger,
steps = rgm_build(script = "drone_publish_tag_grafana.sh", canFail = False),
steps = rgm_run("rgm-build", "drone_publish_tag_grafana.sh"),
depends_on = ["release-test-backend", "release-test-frontend"],
)
@@ -166,22 +251,61 @@ def rgm_tag_windows():
)
def rgm_version_branch():
# Runs a package / build proces (with all distros) when a commit lands on a version branch
return pipeline(
name = "rgm-version-branch-prerelease",
trigger = version_branch_trigger,
steps = rgm_build(script = "drone_publish_tag_grafana.sh", canFail = False),
steps = rgm_run("rgm-build", "drone_publish_tag_grafana.sh"),
depends_on = ["release-test-backend", "release-test-frontend"],
)
def rgm():
def rgm_nightly_build():
src = "$${DRONE_WORKSPACE}/dist/*"
dst = "$${DESTINATION}/$${DRONE_BUILD_EVENT}"
copy_steps = with_deps(rgm_copy(src, dst), ["rgm-build"])
return pipeline(
name = "rgm-nightly-build",
trigger = nightly_trigger,
steps = rgm_run("rgm-build", "drone_build_nightly_grafana.sh") + copy_steps,
depends_on = ["nightly-test-backend", "nightly-test-frontend"],
)
def rgm_nightly_publish():
"""Nightly publish pipeline.
Returns:
Drone pipeline.
"""
src = "$${DESTINATION}/$${DRONE_BUILD_EVENT}/*_$${DRONE_BUILD_NUMBER}_*"
dst = "$${DRONE_WORKSPACE}/dist"
publish_steps = with_deps(rgm_run("rgm-publish", "drone_publish_nightly_grafana.sh"), ["rgm-copy"])
package_steps = with_deps(rgm_publish_packages("grafana-packages-testing"), ["rgm-publish"])
return pipeline(
name = "rgm-nightly-publish",
trigger = nightly_trigger,
steps = rgm_copy(src, dst) + publish_steps + package_steps,
depends_on = ["rgm-nightly-build"],
)
def rgm_nightly_pipeline():
return [
test_frontend(nightly_trigger, "nightly"),
test_backend(nightly_trigger, "nightly"),
rgm_nightly_build(),
rgm_nightly_publish(),
]
def rgm_tag_pipeline():
return [
whats_new_checker_pipeline(tag_trigger),
test_frontend(tag_trigger, "release"),
test_backend(tag_trigger, "release"),
rgm_main(), # Runs a package / build process (with some distros) when commits are merged to main
rgm_tag(), # Runs a package / build process (with all distros) when a tag is made
rgm_tag(),
rgm_tag_windows(),
rgm_version_branch(), # Runs a package / build proces (with all distros) when a commit lands on a version branch
verify_release_pipeline(
trigger = tag_trigger,
name = "rgm-tag-verify-prerelease-assets",
@@ -191,6 +315,11 @@ def rgm():
"rgm-tag-prerelease-windows",
],
),
]
def rgm_version_branch_pipeline():
return [
rgm_version_branch(),
verify_release_pipeline(
trigger = version_branch_trigger,
name = "rgm-prerelease-verify-prerelease-assets",
@@ -200,3 +329,16 @@ def rgm():
],
),
]
def rgm_main_pipeline():
return [
rgm_main(),
]
def rgm():
return (
rgm_main_pipeline() +
rgm_tag_pipeline() +
rgm_version_branch_pipeline() +
rgm_nightly_pipeline()
)