CI: Add github app token generation in pipelines that use GITHUB_TOKEN (#96646)

* Add github app token generation in pipelines that use GITHUB_TOKEN

* ci?

* clone gh repo using x-access-token user

* address linting issues

* use mounted volume for exporting token

* remove unused github_token env var swagger gen step

* replace pat on release_pr pipepline

* cleanup GH PAT references

* linting

* Update scripts/drone/steps/lib.star

* make drone

---------

Co-authored-by: Matheus Macabu <macabu.matheus@gmail.com>
This commit is contained in:
Kevin Minehart
2024-11-21 09:08:02 -06:00
committed by GitHub
parent 3984756ccc
commit 2400483d6c
15 changed files with 487 additions and 110 deletions

View File

@@ -7,6 +7,12 @@ load(
"integration_test_services",
"integration_test_services_volumes",
)
load(
"scripts/drone/steps/github.star",
"github_app_generate_token_step",
"github_app_pipeline_volumes",
"github_app_step_volumes",
)
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
@@ -69,10 +75,10 @@ def release_pr_step(depends_on = []):
"image": images["curl"],
"depends_on": depends_on,
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
"GH_CLI_URL": "https://github.com/cli/cli/releases/download/v2.50.0/gh_2.50.0_linux_amd64.tar.gz",
},
"commands": [
"export GITHUB_TOKEN=$(cat /github-app/token)",
"apk add perl",
"v_target=`echo $${{TAG}} | perl -pe 's/{}/v\\1.\\2.x/'`".format(semver_regex),
# Install gh CLI
@@ -86,6 +92,7 @@ def release_pr_step(depends_on = []):
"-f latest=$${LATEST} " +
"--repo=grafana/grafana release-pr.yml",
],
"volumes": github_app_step_volumes(),
}
def release_npm_packages_step():
@@ -149,7 +156,8 @@ def publish_artifacts_pipelines(mode):
compile_build_cmd(),
publish_artifacts_step(),
publish_storybook_step(),
release_pr_step(depends_on = ["publish-artifacts"]),
github_app_generate_token_step(),
release_pr_step(depends_on = ["publish-artifacts", github_app_generate_token_step()["name"]]),
]
return [
@@ -162,6 +170,7 @@ def publish_artifacts_pipelines(mode):
steps = [
release_pr_step(),
],
volumes = github_app_pipeline_volumes(),
),
pipeline(
name = "publish-artifacts-{}".format(mode),

View File

@@ -7,6 +7,11 @@ load(
"integration_test_services",
"integration_test_services_volumes",
)
load(
"scripts/drone/steps/github.star",
"github_app_generate_token_step",
"github_app_pipeline_volumes",
)
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
@@ -32,10 +37,13 @@ def integration_benchmarks(prefix):
environment = {"EDITION": "oss"}
services = integration_test_services()
volumes = integration_test_services_volumes()
volumes = integration_test_services_volumes() + github_app_pipeline_volumes()
# In pull requests, attempt to clone grafana enterprise.
init_steps = [enterprise_setup_step(isPromote = True)]
init_steps = [
github_app_generate_token_step(),
enterprise_setup_step(isPromote = True),
]
verify_step = verify_gen_cue_step()
verify_jsonnet_step = verify_gen_jsonnet_step()

View File

@@ -1,5 +1,10 @@
"""This module contains the comprehensive build pipeline."""
load(
"scripts/drone/steps/github.star",
"github_app_generate_token_step",
"github_app_pipeline_volumes",
)
load(
"scripts/drone/steps/lib.star",
"build_frontend_package_step",
@@ -57,6 +62,7 @@ def build_e2e(trigger, ver_mode):
environment = {"EDITION": "oss"}
init_steps = [
github_app_generate_token_step(),
identify_runner_step(),
download_grabpl_step(),
compile_build_cmd(),
@@ -173,4 +179,5 @@ def build_e2e(trigger, ver_mode):
services = [],
steps = init_steps + build_steps,
trigger = trigger,
volumes = github_app_pipeline_volumes(),
)

View File

@@ -7,6 +7,11 @@ load(
"integration_test_services",
"integration_test_services_volumes",
)
load(
"scripts/drone/steps/github.star",
"github_app_generate_token_step",
"github_app_pipeline_volumes",
)
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
@@ -50,8 +55,11 @@ def integration_tests(trigger, prefix, ver_mode = "pr"):
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
init_steps.append(github_app_generate_token_step())
init_steps.append(enterprise_setup_step())
volumes += github_app_pipeline_volumes()
init_steps += [
download_grabpl_step(),
compile_build_cmd(),

View File

@@ -2,6 +2,11 @@
This module returns the pipeline used for linting backend code.
"""
load(
"scripts/drone/steps/github.star",
"github_app_generate_token_step",
"github_app_pipeline_volumes",
)
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
@@ -37,10 +42,15 @@ def lint_backend_pipeline(trigger, ver_mode):
compile_build_cmd(),
]
volumes = []
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
init_steps.append(github_app_generate_token_step())
init_steps.append(enterprise_setup_step())
volumes += github_app_pipeline_volumes()
init_steps.append(wire_step)
test_steps = [
@@ -57,4 +67,5 @@ def lint_backend_pipeline(trigger, ver_mode):
services = [],
steps = init_steps + test_steps,
environment = environment,
volumes = volumes,
)

View File

@@ -2,6 +2,11 @@
This module returns the pipeline used for linting frontend code.
"""
load(
"scripts/drone/steps/github.star",
"github_app_generate_token_step",
"github_app_pipeline_volumes",
)
load(
"scripts/drone/steps/lib.star",
"enterprise_setup_step",
@@ -31,9 +36,16 @@ def lint_frontend_pipeline(trigger, ver_mode):
lint_step = lint_frontend_step()
i18n_step = verify_i18n_step()
volumes = []
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
init_steps = [enterprise_setup_step()]
init_steps = [
github_app_generate_token_step(),
enterprise_setup_step(),
]
volumes += github_app_pipeline_volumes()
init_steps += [
identify_runner_step(),
@@ -50,4 +62,5 @@ def lint_frontend_pipeline(trigger, ver_mode):
services = [],
steps = init_steps + test_steps,
environment = environment,
volumes = volumes,
)

View File

@@ -2,6 +2,11 @@
This module returns all pipelines used in OpenAPI specification generation of Grafana HTTP APIs
"""
load(
"scripts/drone/steps/github.star",
"github_app_generate_token_step",
"github_app_pipeline_volumes",
)
load(
"scripts/drone/steps/lib.star",
"enterprise_setup_step",
@@ -14,10 +19,6 @@ load(
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/vault.star",
"from_secret",
)
def swagger_gen_step(ver_mode):
if ver_mode != "pr":
@@ -26,9 +27,6 @@ def swagger_gen_step(ver_mode):
return {
"name": "swagger-gen",
"image": images["go"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
"commands": [
"apk add --update git make",
"make swagger-clean && make openapi3-gen",
@@ -42,6 +40,7 @@ def swagger_gen_step(ver_mode):
def swagger_gen(ver_mode, source = "${DRONE_SOURCE_BRANCH}"):
test_steps = [
github_app_generate_token_step(),
enterprise_setup_step(source = source, canFail = True),
swagger_gen_step(ver_mode = ver_mode),
]
@@ -53,6 +52,7 @@ def swagger_gen(ver_mode, source = "${DRONE_SOURCE_BRANCH}"):
},
services = [],
steps = test_steps,
volumes = github_app_pipeline_volumes(),
)
return p

View File

@@ -2,6 +2,11 @@
This module returns the pipeline used for testing backend code.
"""
load(
"scripts/drone/steps/github.star",
"github_app_generate_token_step",
"github_app_pipeline_volumes",
)
load(
"scripts/drone/steps/lib.star",
"enterprise_setup_step",
@@ -34,10 +39,15 @@ def test_backend(trigger, ver_mode):
verify_step = verify_gen_cue_step()
verify_jsonnet_step = verify_gen_jsonnet_step()
volumes = []
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
steps.append(github_app_generate_token_step())
steps.append(enterprise_setup_step())
volumes += github_app_pipeline_volumes()
steps += [
identify_runner_step(),
verify_step,
@@ -52,4 +62,5 @@ def test_backend(trigger, ver_mode):
trigger = trigger,
steps = steps,
environment = environment,
volumes = volumes,
)

View File

@@ -2,6 +2,11 @@
This module returns the pipeline used for testing backend code.
"""
load(
"scripts/drone/steps/github.star",
"github_app_generate_token_step",
"github_app_pipeline_volumes",
)
load(
"scripts/drone/steps/lib.star",
"betterer_frontend_step",
@@ -35,10 +40,15 @@ def test_frontend(trigger, ver_mode):
test_step = test_frontend_step()
volumes = []
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
steps.append(github_app_generate_token_step())
steps.append(enterprise_setup_step())
volumes += github_app_pipeline_volumes()
steps.append(test_step)
return pipeline(
@@ -46,4 +56,5 @@ def test_frontend(trigger, ver_mode):
trigger = trigger,
steps = steps,
environment = environment,
volumes = volumes,
)

View File

@@ -20,6 +20,11 @@ load(
"scripts/drone/pipelines/whats_new_checker.star",
"whats_new_checker_pipeline",
)
load(
"scripts/drone/steps/github.star",
"github_app_generate_token_step",
"github_app_step_volumes",
)
load(
"scripts/drone/utils/images.star",
"images",
@@ -42,7 +47,6 @@ load(
"rgm_destination",
"rgm_downloads_destination",
"rgm_gcp_key_base64",
"rgm_github_token",
"rgm_storybook_destination",
)
@@ -111,7 +115,6 @@ def rgm_env_secrets(env):
env["DOWNLOADS_DESTINATION"] = from_secret(rgm_downloads_destination)
env["GCP_KEY_BASE64"] = from_secret(rgm_gcp_key_base64)
env["GITHUB_TOKEN"] = from_secret(rgm_github_token)
env["_EXPERIMENTAL_DAGGER_CLOUD_TOKEN"] = from_secret(rgm_dagger_token)
env["GPG_PRIVATE_KEY"] = from_secret("packages_gpg_private_key")
env["GPG_PUBLIC_KEY"] = from_secret("packages_gpg_public_key")
@@ -142,12 +145,13 @@ def rgm_run(name, script):
"pull": "always",
"commands": [
"export GRAFANA_DIR=$$(pwd)",
"export GITHUB_TOKEN=$(cat /github-app/token)",
"cd /src && ./scripts/{}".format(script),
],
"environment": rgm_env_secrets(env),
# The docker socket is a requirement for running dagger programs
# In the future we should find a way to use dagger without mounting the docker socket.
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}] + github_app_step_volumes(),
}
return [
@@ -345,6 +349,7 @@ def rgm_promotion_pipeline():
"image": "grafana/grafana-build:main",
"pull": "always",
"commands": [
"export GITHUB_TOKEN=$(cat /github-app/token)",
"dagger run --silent /src/grafana-build artifacts " +
"-a $${ARTIFACTS} " +
"--grafana-ref=$${GRAFANA_REF} " +
@@ -356,12 +361,16 @@ def rgm_promotion_pipeline():
"environment": rgm_env_secrets(env),
# The docker socket is a requirement for running dagger programs
# In the future we should find a way to use dagger without mounting the docker socket.
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}] + github_app_step_volumes(),
}
generate_token_step = github_app_generate_token_step()
publish_step = rgm_copy("dist/*", "$${UPLOAD_TO}")
build_step["depends_on"] = [
generate_token_step["name"],
]
steps = [
generate_token_step,
build_step,
publish_step,
]

View File

@@ -0,0 +1,40 @@
"""
This module is used to interface with the GitHub App to extract temporary installation tokens.
"""
load(
"scripts/drone/utils/images.star",
"images",
)
load(
"scripts/drone/vault.star",
"from_secret",
"github_app_app_id",
"github_app_app_installation_id",
"github_app_private_key",
)
def github_app_step_volumes():
return [
{"name": "github-app", "path": "/github-app"},
]
def github_app_pipeline_volumes():
return [
{"name": "github-app", "temp": {}},
]
def github_app_generate_token_step():
return {
"name": "github-app-generate-token",
"image": images["github_app_secret_writer"],
"environment": {
"GITHUB_APP_ID": from_secret(github_app_app_id),
"GITHUB_APP_INSTALLATION_ID": from_secret(github_app_app_installation_id),
"GITHUB_APP_PRIVATE_KEY": from_secret(github_app_private_key),
},
"commands": [
"echo $(/usr/bin/github-app-external-token) > /github-app/token",
],
"volumes": github_app_step_volumes(),
}

View File

@@ -2,6 +2,11 @@
This module is a library of Drone steps and other pipeline components.
"""
load(
"scripts/drone/steps/github.star",
"github_app_generate_token_step",
"github_app_step_volumes",
)
load(
"scripts/drone/steps/rgm.star",
"rgm_build_backend_step",
@@ -99,23 +104,25 @@ def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFai
check = []
else:
check = [
'is_fork=$(curl --retry 5 "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork)',
'is_fork=$(curl --retry 5 "https://$${GITHUB_TOKEN}@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork)',
'if [ "$is_fork" != false ]; then return 1; fi', # Only clone if we're confident that 'fork' is 'false'. Fail if it's also empty.
]
step = {
"name": "clone-enterprise",
"image": images["git"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
"commands": [
"apk add --update curl jq bash",
"GITHUB_TOKEN=$(cat /github-app/token)",
] + check + [
'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" ' + location,
'git clone "https://x-access-token:$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" ' + location,
"cd {}".format(location),
'if git checkout {0}; then echo "checked out {0}"; elif git checkout {1}; then echo "git checkout {1}"; else git checkout main; fi'.format(source, target),
],
"depends_on": [
github_app_generate_token_step()["name"],
],
"volumes": github_app_step_volumes(),
}
if canFail:
@@ -316,6 +323,7 @@ def e2e_tests_artifacts():
"end-to-end-tests-panels-suite",
"end-to-end-tests-smoke-tests-suite",
"end-to-end-tests-various-suite",
github_app_generate_token_step()["name"],
],
"failure": "ignore",
"when": {
@@ -327,9 +335,9 @@ def e2e_tests_artifacts():
"environment": {
"GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY": from_secret(gcp_upload_artifacts_key),
"E2E_TEST_ARTIFACTS_BUCKET": "releng-pipeline-artifacts-dev",
"GITHUB_TOKEN": from_secret("github_token"),
},
"commands": [
"export GITHUB_TOKEN=$(cat /github-app/token)",
# if no videos found do nothing
"if [ -z `find ./e2e -type f -name *spec.ts.mp4` ]; then echo 'missing videos'; false; fi",
"apt-get update",
@@ -344,6 +352,7 @@ def e2e_tests_artifacts():
'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} -H "Authorization: token $${GITHUB_TOKEN}" -d ' +
'"{\\"state\\":\\"success\\",\\"target_url\\":\\"$${E2E_ARTIFACTS_VIDEO_ZIP}\\", \\"description\\": \\"Click on the details to download e2e recording videos\\", \\"context\\": \\"e2e_artifacts\\"}"',
],
"volumes": github_app_step_volumes(),
}
def playwright_e2e_report_upload():
@@ -380,6 +389,7 @@ def playwright_e2e_report_post_link():
"image": images["curl"],
"depends_on": [
"playwright-e2e-report-upload",
github_app_generate_token_step()["name"],
],
"failure": "ignore",
"when": {
@@ -388,10 +398,8 @@ def playwright_e2e_report_post_link():
"failure",
],
},
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
"commands": [
"GITHUB_TOKEN=$(cat /github-app/token)",
# if the trace doesn't folder exists, it means that there are no failed tests.
"if [ ! -d ./playwright-report/trace ]; then echo 'all tests passed'; exit 0; fi",
# if it exists, we will post a comment on the PR with the link to the report
@@ -403,6 +411,7 @@ def playwright_e2e_report_post_link():
'-H "X-GitHub-Api-Version: 2022-11-28" -d ' +
'"{\\"body\\":\\"❌ Failed to run Playwright plugin e2e tests. <br /> <br /> Click [here]($${E2E_PLAYWRIGHT_REPORT_URL}) to browse the Playwright report and trace viewer. <br /> For information on how to run Playwright tests locally, refer to the [Developer guide](https://github.com/grafana/grafana/blob/main/contribute/developer-guide.md#to-run-the-playwright-tests). \\"}"',
],
"volumes": github_app_step_volumes(),
}
def upload_cdn_step(ver_mode, trigger = None, depends_on = ["grafana-server"]):
@@ -852,7 +861,6 @@ def cloud_plugins_e2e_tests_step(suite, cloud, trigger = None):
environment = {
"CYPRESS_CI": "true",
"HOST": "grafana-server",
"GITHUB_TOKEN": from_secret("github_token"),
"AZURE_SP_APP_ID": from_secret("azure_sp_app_id"),
"AZURE_SP_PASSWORD": from_secret("azure_sp_app_pw"),
"AZURE_TENANT": from_secret("azure_tenant"),
@@ -873,9 +881,15 @@ def cloud_plugins_e2e_tests_step(suite, cloud, trigger = None):
"image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e-13.10.0:1.0.0",
"depends_on": [
"grafana-server",
github_app_generate_token_step()["name"],
],
"environment": environment,
"commands": ["cd /", "./cpp-e2e/scripts/ci-run.sh {} {}".format(cloud, branch)],
"commands": [
"GITHUB_TOKEN=$(cat /github-app/token)",
"cd /",
"./cpp-e2e/scripts/ci-run.sh {} {}".format(cloud, branch),
],
"volumes": github_app_step_volumes(),
}
step = dict(step, when = when)
return step

View File

@@ -37,4 +37,5 @@ images = {
"shellcheck": "koalaman/shellcheck:stable",
"rocky": "rockylinux:9",
"wine": "scottyhardy/docker-wine:stable-9.0",
"github_app_secret_writer": "us-docker.pkg.dev/grafanalabs-global/docker-deployment-tools-prod/github-app-secret-writer:2024-11-05-v11688112090.1-83920c59",
}

View File

@@ -9,16 +9,20 @@ gcp_upload_artifacts_key = "gcp_upload_artifacts_key"
gcp_grafanauploads = "gcp_grafanauploads"
gcp_grafanauploads_base64 = "gcp_grafanauploads_base64"
gcp_download_build_container_assets_key = "gcp_download_build_container_assets_key"
azure_sp_app_id = "azure_sp_app_id"
azure_sp_app_pw = "azure_sp_app_pw"
azure_tenant = "azure_tenant"
github_app_app_id = "github-app-app-id"
github_app_app_installation_id = "github-app-installation-id"
github_app_private_key = "github-app-private-key"
rgm_gcp_key_base64 = "gcp_key_base64"
rgm_destination = "destination"
rgm_storybook_destination = "rgm_storybook_destination"
rgm_cdn_destination = "rgm_cdn_destination"
rgm_downloads_destination = "rgm_downloads_destination"
rgm_github_token = "github_token"
rgm_dagger_token = "dagger_token"
docker_username = "docker_username"
@@ -41,12 +45,14 @@ def vault_secret(name, path, key):
def secrets():
return [
vault_secret(github_app_app_id, "ci/data/repo/grafana/grafana/github-app", "app-id"),
vault_secret(github_app_app_installation_id, "ci/data/repo/grafana/grafana/github-app", "app-installation-id"),
vault_secret(github_app_private_key, "ci/data/repo/grafana/grafana/github-app", "private-key"),
vault_secret(gcp_grafanauploads, "infra/data/ci/grafana-release-eng/grafanauploads", "credentials.json"),
vault_secret(gcp_grafanauploads_base64, "infra/data/ci/grafana-release-eng/grafanauploads", "credentials_base64"),
vault_secret("grafana_api_key", "infra/data/ci/grafana-release-eng/grafanacom", "api_key"),
vault_secret(gcr_pull_secret, "secret/data/common/gcr", ".dockerconfigjson"),
vault_secret(gar_pull_secret, "secret/data/common/gar", ".dockerconfigjson"),
vault_secret("github_token", "ci/data/repo/grafana/grafana/grafanabot", "pat"),
vault_secret(drone_token, "infra/data/ci/drone", "machine-user-token"),
vault_secret(prerelease_bucket, "infra/data/ci/grafana/prerelease", "bucket"),
vault_secret(docker_username, "infra/data/ci/grafanaci-docker-hub", "username"),