IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Security Scans: Add trivy scans to every docker image used for building/testing/publishing (#69911)

Browse Source 
* Created images.star

* Fix typo

* Add cronjobs for build-images
This commit is contained in:
Dimitris Sotirakis 2023-06-12 16:41:18 +03:00 committed by GitHub
parent d363741d39
commit 2cda971796
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 290 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
.drone.yml
View File

@ -6925,7 +6925,7 @@ steps:
- commands:
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest
image: aquasec/trivy:0.21.0
name: scan-unkown-low-medium-vulnerabilities
name: scan-unknown-low-medium-vulnerabilities
- commands:
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest
image: aquasec/trivy:0.21.0
@ -6955,7 +6955,7 @@ steps:
- commands:
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main
image: aquasec/trivy:0.21.0
name: scan-unkown-low-medium-vulnerabilities
name: scan-unknown-low-medium-vulnerabilities
- commands:
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main
image: aquasec/trivy:0.21.0
@ -6985,7 +6985,7 @@ steps:
- commands:
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest-ubuntu
image: aquasec/trivy:0.21.0
name: scan-unkown-low-medium-vulnerabilities
name: scan-unknown-low-medium-vulnerabilities
- commands:
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest-ubuntu
image: aquasec/trivy:0.21.0
@ -7016,7 +7016,7 @@ steps:
- commands:
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main-ubuntu
image: aquasec/trivy:0.21.0
name: scan-unkown-low-medium-vulnerabilities
name: scan-unknown-low-medium-vulnerabilities
- commands:
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main-ubuntu
image: aquasec/trivy:0.21.0
@ -7039,6 +7039,80 @@ type: docker
clone:
retries: 3
kind: pipeline
name: scan-build-test-and-publish-docker-images
platform:
arch: amd64
os: linux
steps:
- commands:
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM google/cloud-sdk:431.0.0
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/build-container:1.7.4
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-deploy:1.3.3
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine:3.17.1
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM byrnedo/alpine-curl:0.1.8
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mcr.microsoft.com/windows:1809
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/ci-wix:0.1.1
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.20.4
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-windows-test:0.1.0
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM plugins/slack
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM postgres:12.3-alpine
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:5.7.39
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:8.0.32
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM redis:6.2.11-alpine
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM memcached:1.6.9-alpine
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM docker:windowsservercore-1809
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM us.gcr.io/kubernetes-dev/package-publish:latest
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM osixia/openldap:1.4.0
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/drone-downstream
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docker-puppeteer:1.1.0
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docs-base:latest
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest
image: aquasec/trivy:0.21.0
name: scan-unknown-low-medium-vulnerabilities
- commands:
- trivy --exit-code 1 --severity HIGH,CRITICAL google/cloud-sdk:431.0.0
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/build-container:1.7.4
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-deploy:1.3.3
- trivy --exit-code 1 --severity HIGH,CRITICAL alpine:3.17.1
- trivy --exit-code 1 --severity HIGH,CRITICAL byrnedo/alpine-curl:0.1.8
- trivy --exit-code 1 --severity HIGH,CRITICAL mcr.microsoft.com/windows:1809
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/ci-wix:0.1.1
- trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.20.4
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-windows-test:0.1.0
- trivy --exit-code 1 --severity HIGH,CRITICAL plugins/slack
- trivy --exit-code 1 --severity HIGH,CRITICAL postgres:12.3-alpine
- trivy --exit-code 1 --severity HIGH,CRITICAL mysql:5.7.39
- trivy --exit-code 1 --severity HIGH,CRITICAL mysql:8.0.32
- trivy --exit-code 1 --severity HIGH,CRITICAL redis:6.2.11-alpine
- trivy --exit-code 1 --severity HIGH,CRITICAL memcached:1.6.9-alpine
- trivy --exit-code 1 --severity HIGH,CRITICAL docker:windowsservercore-1809
- trivy --exit-code 1 --severity HIGH,CRITICAL us.gcr.io/kubernetes-dev/package-publish:latest
- trivy --exit-code 1 --severity HIGH,CRITICAL osixia/openldap:1.4.0
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/drone-downstream
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docker-puppeteer:1.1.0
- trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docs-base:latest
- trivy --exit-code 1 --severity HIGH,CRITICAL cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
- trivy --exit-code 1 --severity HIGH,CRITICAL us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest
image: aquasec/trivy:0.21.0
name: scan-high-critical-vulnerabilities
- image: plugins/slack
name: slack-notify-failure
settings:
channel: grafana-backend-ops
template: 'Nightly docker image scan job for build-images failed: {{build.link}}'
webhook:
from_secret: slack_webhook_backend
when:
status: failure
trigger:
cron: nightly
event: cron
type: docker
---
clone:
retries: 3
kind: pipeline
name: grafana-com-nightly
platform:
arch: amd64
@ -7242,6 +7316,6 @@ kind: secret
name: delivery-bot-app-private-key
---
kind: signature
hmac: caf02d8ce78dd5c2fd6d841d0661702e74571c07939c6af602ac11d1790b31b8
hmac: 44f95e0e3d9eb3fc8891e94f7205c18e85adacab25b69906d3c5212875baa383
...

73
scripts/drone/events/cron.star
View File

@ -6,7 +6,10 @@ load("scripts/drone/vault.star", "from_secret")
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"publish_image",
)
load(
"scripts/drone/utils/images.star",
"images",
)
aquasec_trivy_image = "aquasec/trivy:0.21.0"
@ -17,6 +20,7 @@ def cronjobs():
scan_docker_image_pipeline("main"),
scan_docker_image_pipeline("latest-ubuntu"),
scan_docker_image_pipeline("main-ubuntu"),
scan_build_test_publish_docker_image_pipeline(),
grafana_com_nightly_pipeline(),
]
@ -54,34 +58,77 @@ def scan_docker_image_pipeline(tag):
cronName = "nightly",
name = "scan-" + docker_image + "-image",
steps = [
scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image),
scan_docker_image_unknown_low_medium_vulnerabilities_step(docker_image),
scan_docker_image_high_critical_vulnerabilities_step(docker_image),
slack_job_failed_step("grafana-backend-ops", docker_image),
],
)
def scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image):
return {
"name": "scan-unkown-low-medium-vulnerabilities",
"image": aquasec_trivy_image,
"commands": [
"trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image,
def scan_build_test_publish_docker_image_pipeline():
"""Generates a cronjob pipeline for nightly scans of grafana Docker images.
Returns:
Drone cronjob pipeline.
"""
return cron_job_pipeline(
cronName = "nightly",
name = "scan-build-test-and-publish-docker-images",
steps = [
scan_docker_image_unknown_low_medium_vulnerabilities_step("all"),
scan_docker_image_high_critical_vulnerabilities_step("all"),
slack_job_failed_step("grafana-backend-ops", "build-images"),
],
)
def scan_docker_image_unknown_low_medium_vulnerabilities_step(docker_image):
"""Generates a step for scans of Grafana Docker images.
Args:
docker_image: determines which image is scanned.
Returns:
Drone cronjob step .
"""
cmds = []
if docker_image == "all":
for key in images:
cmds = cmds + ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + images[key]]
else:
cmds = ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image]
return {
"name": "scan-unknown-low-medium-vulnerabilities",
"image": aquasec_trivy_image,
"commands": cmds,
}
def scan_docker_image_high_critical_vulnerabilities_step(docker_image):
"""Generates a step for scans of Grafana Docker images.
Args:
docker_image: determines which image is scanned.
Returns:
Drone cronjob step .
"""
cmds = []
if docker_image == "all":
for key in images:
cmds = cmds + ["trivy --exit-code 1 --severity HIGH,CRITICAL " + images[key]]
else:
cmds = ["trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image]
return {
"name": "scan-high-critical-vulnerabilities",
"image": aquasec_trivy_image,
"commands": [
"trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image,
],
"commands": cmds,
}
def slack_job_failed_step(channel, image):
return {
"name": "slack-notify-failure",
"image": "plugins/slack",
"image": images["plugins_slack_image"],
"settings": {
"webhook": from_secret("slack_webhook_backend"),
"channel": channel,
@ -95,7 +142,7 @@ def slack_job_failed_step(channel, image):
def post_to_grafana_com_step():
return {
"name": "post-to-grafana-com",
"image": publish_image,
"image": images["publish_image"],
"environment": {
"GRAFANA_COM_API_KEY": from_secret("grafana_api_key"),
"GCP_KEY": from_secret("gcp_key"),

21
scripts/drone/events/release.star
View File

@ -9,11 +9,9 @@ load(
"build_docker_images_step",
"build_frontend_package_step",
"build_frontend_step",
"build_image",
"build_plugins_step",
"build_storybook_step",
"clone_enterprise_step",
"cloudsdk_image",
"compile_build_cmd",
"copy_packages_for_docker_step",
"download_grabpl_step",
@ -29,7 +27,6 @@ load(
"package_step",
"postgres_integration_tests_step",
"publish_grafanacom_step",
"publish_image",
"publish_images_step",
"publish_linux_packages_step",
"redis_integration_tests_step",
@ -63,6 +60,10 @@ load(
"test_backend_enterprise",
)
load("scripts/drone/vault.star", "from_secret", "prerelease_bucket")
load(
"scripts/drone/utils/images.star",
"images",
)
ver_mode = "release"
release_trigger = {
@ -84,7 +85,7 @@ release_trigger = {
def store_npm_packages_step():
return {
"name": "store-npm-packages",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"compile-build-cmd",
"build-frontend-packages",
@ -99,7 +100,7 @@ def store_npm_packages_step():
def retrieve_npm_packages_step():
return {
"name": "retrieve-npm-packages",
"image": publish_image,
"image": images["publish_image"],
"depends_on": [
"compile-build-cmd",
"yarn-install",
@ -115,7 +116,7 @@ def retrieve_npm_packages_step():
def release_npm_packages_step():
return {
"name": "release-npm-packages",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"compile-build-cmd",
"retrieve-npm-packages",
@ -554,7 +555,7 @@ def publish_artifacts_step(mode):
security = "--security "
return {
"name": "publish-artifacts",
"image": publish_image,
"image": images["publish_image"],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
"PRERELEASE_BUCKET": from_secret("prerelease_bucket"),
@ -572,7 +573,7 @@ def publish_artifacts_step(mode):
def publish_static_assets_step():
return {
"name": "publish-static-assets",
"image": publish_image,
"image": images["publish_image"],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
"PRERELEASE_BUCKET": from_secret("prerelease_bucket"),
@ -587,7 +588,7 @@ def publish_static_assets_step():
def publish_storybook_step():
return {
"name": "publish-storybook",
"image": publish_image,
"image": images["publish_image"],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
"PRERELEASE_BUCKET": from_secret("prerelease_bucket"),
@ -817,7 +818,7 @@ def verify_release_pipeline(
step = {
"name": "gsutil-stat",
"depends_on": ["clone"],
"image": cloudsdk_image,
"image": images["cloudsdk_image"],
"environment": {
"BUCKET": bucket,
"GCP_KEY": gcp_key,

7
scripts/drone/pipelines/aws_marketplace.star
View File

@ -5,18 +5,21 @@ This module contains steps and pipelines publishing to AWS Marketplace.
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"publish_image",
)
load("scripts/drone/vault.star", "from_secret")
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/utils/images.star",
"images",
)
def publish_aws_marketplace_step():
return {
"name": "publish-aws-marketplace",
"image": publish_image,
"image": images["publish_image"],
"commands": ["./bin/build publish aws --image grafana/grafana-enterprise --repo grafana-labs/grafanaenterprise --product 422b46fb-bea6-4f27-8bcc-832117bd627e"],
"depends_on": ["compile-build-cmd"],
"environment": {

12
scripts/drone/pipelines/ci_images.star
View File

@ -2,10 +2,6 @@
This module contains steps and pipelines relating to creating CI Docker images.
"""
load(
"scripts/drone/steps/lib.star",
"wix_image",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
@ -14,6 +10,10 @@ load(
"scripts/drone/vault.star",
"from_secret",
)
load(
"scripts/drone/utils/images.star",
"images",
)
def publish_ci_windows_test_image_pipeline():
trigger = {
@ -28,7 +28,7 @@ def publish_ci_windows_test_image_pipeline():
steps = [
{
"name": "clone",
"image": wix_image,
"image": images["wix_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
@ -39,7 +39,7 @@ def publish_ci_windows_test_image_pipeline():
},
{
"name": "build-and-publish",
"image": "docker:windowsservercore-1809",
"image": images["windows_server_core_image"],
"environment": {
"DOCKER_USERNAME": from_secret("docker_username"),
"DOCKER_PASSWORD": from_secret("docker_password"),

7
scripts/drone/pipelines/docs.star
View File

@ -5,11 +5,14 @@ This module returns all the pipelines used in the event of documentation changes
load(
"scripts/drone/steps/lib.star",
"build_docs_website_step",
"build_image",
"codespell_step",
"identify_runner_step",
"yarn_install_step",
)
load(
"scripts/drone/utils/images.star",
"images",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
@ -46,7 +49,7 @@ def docs_pipelines(ver_mode, trigger):
def lint_docs():
return {
"name": "lint-docs",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"yarn-install",
],

7
scripts/drone/pipelines/github.star
View File

@ -6,18 +6,21 @@ load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"fetch_images_step",
"publish_image",
)
load("scripts/drone/vault.star", "from_secret")
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/utils/images.star",
"images",
)
def publish_github_step():
return {
"name": "publish-github",
"image": publish_image,
"image": images["publish_image"],
"commands": ["./bin/build publish github --repo $${GH_REGISTRY} --create"],
"depends_on": ["fetch-images-enterprise2"],
"environment": {

8
scripts/drone/pipelines/shellcheck.star
View File

@ -2,11 +2,15 @@
This module returns a Drone step and pipeline for linting with shellcheck.
"""
load("scripts/drone/steps/lib.star", "build_image", "compile_build_cmd")
load("scripts/drone/steps/lib.star", "compile_build_cmd")
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/utils/images.star",
"images",
)
trigger = {
"event": [
@ -25,7 +29,7 @@ trigger = {
def shellcheck_step():
return {
"name": "shellcheck",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"compile-build-cmd",
],

7
scripts/drone/pipelines/windows.star
View File

@ -10,11 +10,14 @@ load(
"scripts/drone/steps/lib.star",
"get_windows_steps",
"windows_clone_step",
"windows_go_image",
"windows_init_enterprise_steps",
"windows_test_backend_step",
"windows_wire_install_step",
)
load(
"scripts/drone/utils/images.star",
"images",
)
def windows_test_backend(trigger, edition, ver_mode):
""" Generates a pipeline that runs backend tests on Windows
@ -36,7 +39,7 @@ def windows_test_backend(trigger, edition, ver_mode):
else:
steps.extend([{
"name": "windows-init",
"image": windows_go_image,
"image": images["windows_go_image"],
"depends_on": ["clone"],
"commands": [],
}])

17
scripts/drone/services/services.star
View File

@ -2,6 +2,11 @@
This module has functions for Drone services to be used in pipelines.
"""
load(
"scripts/drone/utils/images.star",
"images",
)
def integration_test_services_volumes():
return [
{"name": "postgres", "temp": {"medium": "memory"}},
@ -13,7 +18,7 @@ def integration_test_services():
services = [
{
"name": "postgres",
"image": "postgres:12.3-alpine",
"image": images["postgres_alpine_image"],
"environment": {
"POSTGRES_USER": "grafanatest",
"POSTGRES_PASSWORD": "grafanatest",
@ -26,7 +31,7 @@ def integration_test_services():
},
{
"name": "mysql57",
"image": "mysql:5.7.39",
"image": images["mysql5_image"],
"environment": {
"MYSQL_ROOT_PASSWORD": "rootpass",
"MYSQL_DATABASE": "grafana_tests",
@ -37,7 +42,7 @@ def integration_test_services():
},
{
"name": "mysql80",
"image": "mysql:8.0.32",
"image": images["mysql8_image"],
"environment": {
"MYSQL_ROOT_PASSWORD": "rootpass",
"MYSQL_DATABASE": "grafana_tests",
@ -49,12 +54,12 @@ def integration_test_services():
},
{
"name": "redis",
"image": "redis:6.2.11-alpine",
"image": images["redis_alpine_image"],
"environment": {},
},
{
"name": "memcached",
"image": "memcached:1.6.9-alpine",
"image": images["memcached_alpine_image"],
"environment": {},
},
]
@ -64,7 +69,7 @@ def integration_test_services():
def ldap_service():
return {
"name": "ldap",
"image": "osixia/openldap:1.4.0",
"image": images["openldap_image"],
"environment": {
"LDAP_ADMIN_PASSWORD": "grafana",
"LDAP_DOMAIN": "grafana.org",

142
scripts/drone/steps/lib.star
View File

@ -7,18 +7,12 @@ load(
"from_secret",
"prerelease_bucket",
)
load(
"scripts/drone/utils/images.star",
"images",
)
grabpl_version = "v3.0.38"
cloudsdk_image = "google/cloud-sdk:431.0.0"
build_image = "grafana/build-container:1.7.4"
publish_image = "grafana/grafana-ci-deploy:1.3.3"
deploy_docker_image = "us.gcr.io/kubernetes-dev/drone/plugins/deploy-image"
alpine_image = "alpine:3.17.1"
curl_image = "byrnedo/alpine-curl:0.1.8"
windows_image = "mcr.microsoft.com/windows:1809"
wix_image = "grafana/ci-wix:0.1.1"
go_image = "golang:1.20.4"
windows_go_image = "grafana/grafana-ci-windows-test:0.1.0"
trigger_oss = {
"repo": [
@ -29,7 +23,7 @@ trigger_oss = {
def slack_step(channel, template, secret):
return {
"name": "slack",
"image": "plugins/slack",
"image": images["plugins_slack_image"],
"settings": {
"webhook": from_secret(secret),
"channel": channel,
@ -40,7 +34,7 @@ def slack_step(channel, template, secret):
def yarn_install_step():
return {
"name": "yarn-install",
"image": build_image,
"image": images["build_image"],
"commands": [
"yarn install --immutable",
],
@ -50,7 +44,7 @@ def yarn_install_step():
def wire_install_step():
return {
"name": "wire-install",
"image": build_image,
"image": images["build_image"],
"commands": [
"make gen-go",
],
@ -62,7 +56,7 @@ def wire_install_step():
def windows_wire_install_step(edition):
return {
"name": "wire-install",
"image": windows_go_image,
"image": images["windows_go_image"],
"commands": [
"go install github.com/google/wire/cmd/wire@v0.5.0",
"wire gen -tags {} ./pkg/server".format(edition),
@ -76,7 +70,7 @@ def identify_runner_step(platform = "linux"):
if platform == "linux":
return {
"name": "identify-runner",
"image": alpine_image,
"image": images["alpine_image"],
"commands": [
"echo $DRONE_RUNNER_NAME",
],
@ -84,7 +78,7 @@ def identify_runner_step(platform = "linux"):
else:
return {
"name": "identify-runner",
"image": windows_image,
"image": images["windows_image"],
"commands": [
"echo $env:DRONE_RUNNER_NAME",
],
@ -111,7 +105,7 @@ def clone_enterprise_step(source = "${DRONE_COMMIT}"):
"""
step = {
"name": "clone-enterprise",
"image": build_image,
"image": images["build_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
@ -137,7 +131,7 @@ def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFai
"""
step = {
"name": "clone-enterprise",
"image": build_image,
"image": images["build_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
@ -183,7 +177,7 @@ def init_enterprise_step(ver_mode):
token = ""
return {
"name": "init-enterprise",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"clone-enterprise",
"grabpl",
@ -241,7 +235,7 @@ def windows_init_enterprise_steps(ver_mode):
download_grabpl_step(platform = "windows"),
{
"name": "clone",
"image": wix_image,
"image": images["wix_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
@ -249,7 +243,7 @@ def windows_init_enterprise_steps(ver_mode):
},
{
"name": "windows-init",
"image": wix_image,
"image": images["wix_image"],
"commands": init_cmds,
"depends_on": ["clone"],
"environment": {"GITHUB_TOKEN": from_secret("github_token")},
@ -262,7 +256,7 @@ def download_grabpl_step(platform = "linux"):
if platform == "windows":
return {
"name": "grabpl",
"image": wix_image,
"image": images["wix_image"],
"commands": [
'$$ProgressPreference = "SilentlyContinue"',
"Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format(
@ -273,7 +267,7 @@ def download_grabpl_step(platform = "linux"):
return {
"name": "grabpl",
"image": curl_image,
"image": images["curl_image"],
"commands": [
"mkdir -p bin",
"curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/grabpl".format(
@ -286,7 +280,7 @@ def download_grabpl_step(platform = "linux"):
def lint_drone_step():
return {
"name": "lint-drone",
"image": curl_image,
"image": images["curl_image"],
"commands": [
"./bin/build verify-drone",
],
@ -298,7 +292,7 @@ def lint_drone_step():
def lint_starlark_step():
return {
"name": "lint-starlark",
"image": build_image,
"image": images["build_image"],
"commands": [
"./bin/build verify-starlark .",
],
@ -325,7 +319,7 @@ def enterprise_downstream_step(ver_mode):
step = {
"name": "trigger-enterprise-downstream",
"image": "grafana/drone-downstream",
"image": images["drone_downstream_image"],
"settings": {
"server": "https://drone.grafana.net",
"token": from_secret("drone_token"),
@ -349,7 +343,7 @@ def lint_backend_step():
return {
"name": "lint-backend",
# TODO: build_image or go_image?
"image": go_image,
"image": images["go_image"],
"environment": {
# We need CGO because of go-sqlite3
"CGO_ENABLED": "1",
@ -367,7 +361,7 @@ def lint_backend_step():
def validate_modfile_step():
return {
"name": "validate-modfile",
"image": go_image,
"image": images["go_image"],
"failure": "ignore",
"commands": [
"go run scripts/modowners/modowners.go check go.mod",
@ -377,7 +371,7 @@ def validate_modfile_step():
def benchmark_ldap_step():
return {
"name": "benchmark-ldap",
"image": build_image,
"image": images["build_image"],
"environment": {
"LDAP_HOSTNAME": "ldap",
},
@ -390,7 +384,7 @@ def benchmark_ldap_step():
def build_storybook_step(ver_mode):
return {
"name": "build-storybook",
"image": build_image,
"image": images["build_image"],
"depends_on": [
# Best to ensure that this step doesn't mess with what's getting built and packaged
"build-frontend",
@ -434,7 +428,7 @@ def store_storybook_step(ver_mode, trigger = None):
step = {
"name": "store-storybook",
"image": publish_image,
"image": images["publish_image"],
"depends_on": [
"build-storybook",
] +
@ -464,7 +458,7 @@ def store_storybook_step(ver_mode, trigger = None):
def e2e_tests_artifacts():
return {
"name": "e2e-tests-artifacts-upload",
"image": cloudsdk_image,
"image": images["cloudsdk_image"],
"depends_on": [
"end-to-end-tests-dashboards-suite",
"end-to-end-tests-panels-suite",
@ -526,7 +520,7 @@ def upload_cdn_step(edition, ver_mode, trigger = None):
step = {
"name": "upload-cdn-assets" + enterprise2_suffix(edition),
"image": publish_image,
"image": images["publish_image"],
"depends_on": deps,
"environment": {
"GCP_KEY": from_secret("gcp_key"),
@ -578,7 +572,7 @@ def build_backend_step(edition, ver_mode, variants = None):
return {
"name": "build-backend" + enterprise2_suffix(edition),
"image": build_image,
"image": images["build_image"],
"depends_on": [
"wire-install",
"compile-build-cmd",
@ -613,7 +607,7 @@ def build_frontend_step(edition, ver_mode):
return {
"name": "build-frontend",
"image": build_image,
"image": images["build_image"],
"environment": {
"NODE_OPTIONS": "--max_old_space_size=8192",
},
@ -653,7 +647,7 @@ def build_frontend_package_step(edition, ver_mode):
return {
"name": "build-frontend-packages",
"image": build_image,
"image": images["build_image"],
"environment": {
"NODE_OPTIONS": "--max_old_space_size=8192",
},
@ -673,7 +667,7 @@ def build_plugins_step(edition, ver_mode):
env = None
return {
"name": "build-plugins",
"image": build_image,
"image": images["build_image"],
"environment": env,
"depends_on": [
"compile-build-cmd",
@ -685,7 +679,7 @@ def build_plugins_step(edition, ver_mode):
],
}
def test_backend_step(image = build_image):
def test_backend_step(image = images["build_image"]):
return {
"name": "test-backend",
"image": image,
@ -698,13 +692,13 @@ def test_backend_step(image = build_image):
}
def windows_test_backend_step():
step = test_backend_step(image = windows_go_image)
step = test_backend_step(image = images["windows_go_image"])
return step
def test_backend_integration_step():
return {
"name": "test-backend-integration",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"wire-install",
],
@ -729,7 +723,7 @@ def betterer_frontend_step(edition = "oss"):
deps.extend(["yarn-install"])
return {
"name": "betterer-frontend",
"image": build_image,
"image": images["build_image"],
"depends_on": deps,
"commands": [
"yarn betterer ci",
@ -752,7 +746,7 @@ def test_frontend_step(edition = "oss"):
deps.extend(["yarn-install"])
return {
"name": "test-frontend",
"image": build_image,
"image": images["build_image"],
"environment": {
"TEST_MAX_WORKERS": "50%",
},
@ -765,7 +759,7 @@ def test_frontend_step(edition = "oss"):
def lint_frontend_step():
return {
"name": "lint-frontend",
"image": build_image,
"image": images["build_image"],
"environment": {
"TEST_MAX_WORKERS": "50%",
},
@ -784,7 +778,7 @@ def verify_i18n_step():
uncommited_error_message = "\nTranslation extraction has not been committed. Please run 'yarn i18n:extract', commit the changes and push again."
return {
"name": "verify-i18n",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"yarn-install",
],
@ -836,7 +830,7 @@ def test_a11y_frontend_step(ver_mode, port = 3001):
return {
"name": "test-a11y-frontend",
# TODO which image should be used?
"image": "grafana/docker-puppeteer:1.1.0",
"image": images["docker_puppeteer_image"],
"depends_on": [
"grafana-server",
],
@ -861,7 +855,7 @@ def frontend_metrics_step(trigger = None):
"""
step = {
"name": "publish-frontend-metrics",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"test-a11y-frontend",
],
@ -880,7 +874,7 @@ def frontend_metrics_step(trigger = None):
def codespell_step():
return {
"name": "codespell",
"image": build_image,
"image": images["build_image"],
"commands": [
# Important: all words have to be in lowercase, and separated by "\n".
'echo -e "unknwon\nreferer\nerrorstring\neror\niam\nwan" > words_to_ignore.txt',
@ -939,7 +933,7 @@ def package_step(edition, ver_mode):
return {
"name": "package" + enterprise2_suffix(edition),
"image": build_image,
"image": images["build_image"],
"depends_on": deps,
"environment": env,
"commands": cmds,
@ -962,7 +956,7 @@ def grafana_server_step(edition, port = 3001):
return {
"name": "grafana-server",
"image": build_image,
"image": images["build_image"],
"detach": True,
"depends_on": [
"build-plugins",
@ -982,7 +976,7 @@ def e2e_tests_step(suite, port = 3001, tries = None):
cmd += " --tries {}".format(tries)
return {
"name": "end-to-end-tests-{}".format(suite),
"image": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97",
"image": images["cypress_image"],
"depends_on": [
"grafana-server",
],
@ -1034,7 +1028,7 @@ def cloud_plugins_e2e_tests_step(suite, cloud, trigger = None):
branch = "${DRONE_SOURCE_BRANCH}".replace("/", "-")
step = {
"name": "end-to-end-tests-{}-{}".format(suite, cloud),
"image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest",
"image": images["cloud_datasources_e2e_image"],
"depends_on": [
"grafana-server",
],
@ -1048,7 +1042,7 @@ def build_docs_website_step():
return {
"name": "build-docs-website",
# Use latest revision here, since we want to catch if it breaks
"image": "grafana/docs-base:latest",
"image": images["docs_image"],
"commands": [
"mkdir -p /hugo/content/docs/grafana/latest",
"cp -r docs/sources/* /hugo/content/docs/grafana/latest/",
@ -1059,7 +1053,7 @@ def build_docs_website_step():
def copy_packages_for_docker_step(edition = None):
return {
"name": "copy-packages-for-docker",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"package" + enterprise2_suffix(edition),
],
@ -1107,7 +1101,7 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa
return {
"name": "build-docker-images" + ubuntu_sfx,
"image": cloudsdk_image,
"image": images["cloudsdk_image"],
"depends_on": [
"copy-packages-for-docker",
"compile-build-cmd",
@ -1120,7 +1114,7 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa
def fetch_images_step(edition):
return {
"name": "fetch-images-{}".format(edition),
"image": cloudsdk_image,
"image": images["cloudsdk_image"],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
"DOCKER_USER": from_secret("docker_username"),
@ -1199,7 +1193,7 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None):
step = {
"name": "publish-images-{}".format(name),
"image": cloudsdk_image,
"image": images["cloudsdk_image"],
"environment": environment,
"commands": [cmd],
"depends_on": deps,
@ -1224,7 +1218,7 @@ def postgres_integration_tests_step():
]
return {
"name": "postgres-integration-tests",
"image": build_image,
"image": images["build_image"],
"depends_on": ["wire-install"],
"environment": {
"PGPASSWORD": "grafanatest",
@ -1245,7 +1239,7 @@ def mysql_integration_tests_step(hostname, version):
]
return {
"name": "mysql-{}-integration-tests".format(version),
"image": build_image,
"image": images["build_image"],
"depends_on": ["wire-install"],
"environment": {
"GRAFANA_TEST_DB": "mysql",
@ -1257,7 +1251,7 @@ def mysql_integration_tests_step(hostname, version):
def redis_integration_tests_step():
return {
"name": "redis-integration-tests",
"image": build_image,
"image": images["build_image"],
"depends_on": ["wire-install"],
"environment": {
"REDIS_URL": "redis://redis:6379/0",
@ -1272,7 +1266,7 @@ def redis_integration_tests_step():
def memcached_integration_tests_step():
return {
"name": "memcached-integration-tests",
"image": build_image,
"image": images["build_image"],
"depends_on": ["wire-install"],
"environment": {
"MEMCACHED_HOSTS": "memcached:11211",
@ -1296,7 +1290,7 @@ def release_canary_npm_packages_step(trigger = None):
"""
step = {
"name": "release-canary-npm-packages",
"image": build_image,
"image": images["build_image"],
"depends_on": end_to_end_tests_deps(),
"environment": {
"NPM_TOKEN": from_secret("npm_token"),
@ -1339,7 +1333,7 @@ def upload_packages_step(edition, ver_mode, trigger = None):
"""
step = {
"name": "upload-packages" + enterprise2_suffix(edition),
"image": publish_image,
"image": images["publish_image"],
"depends_on": end_to_end_tests_deps(),
"environment": {
"GCP_KEY": from_secret("gcp_key"),
@ -1381,7 +1375,7 @@ def publish_grafanacom_step(edition, ver_mode):
return {
"name": "publish-grafanacom-{}".format(edition),
"image": publish_image,
"image": images["publish_image"],
"depends_on": [
"publish-linux-packages-deb",
"publish-linux-packages-rpm",
@ -1399,7 +1393,7 @@ def publish_linux_packages_step(edition, package_manager = "deb"):
return {
"name": "publish-linux-packages-{}".format(package_manager),
# See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image
"image": "us.gcr.io/kubernetes-dev/package-publish:latest",
"image": images["package_publish_image"],
"depends_on": ["compile-build-cmd"],
"privileged": True,
"settings": {
@ -1421,7 +1415,7 @@ def publish_linux_packages_step(edition, package_manager = "deb"):
def windows_clone_step():
return {
"name": "clone",
"image": wix_image,
"image": images["wix_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
@ -1481,7 +1475,7 @@ def get_windows_steps(edition, ver_mode):
[
{
"name": "clone",
"image": wix_image,
"image": images["wix_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
@ -1489,7 +1483,7 @@ def get_windows_steps(edition, ver_mode):
},
{
"name": "windows-init",
"image": wix_image,
"image": images["wix_image"],
"commands": init_cmds,
"depends_on": ["clone"],
"environment": {"GITHUB_TOKEN": from_secret("github_token")},
@ -1508,7 +1502,7 @@ def get_windows_steps(edition, ver_mode):
[
{
"name": "windows-init",
"image": wix_image,
"image": images["wix_image"],
"commands": init_cmds,
},
],
@ -1583,7 +1577,7 @@ def get_windows_steps(edition, ver_mode):
steps.append(
{
"name": "build-windows-installer",
"image": wix_image,
"image": images["wix_image"],
"depends_on": [
"windows-init",
],
@ -1601,7 +1595,7 @@ def get_windows_steps(edition, ver_mode):
def verify_gen_cue_step():
return {
"name": "verify-gen-cue",
"image": build_image,
"image": images["build_image"],
"depends_on": [],
"commands": [
"# It is required that code generated from Thema/CUE be committed and in sync with its inputs.",
@ -1613,7 +1607,7 @@ def verify_gen_cue_step():
def verify_gen_jsonnet_step():
return {
"name": "verify-gen-jsonnet",
"image": build_image,
"image": images["build_image"],
"depends_on": [],
"commands": [
"# It is required that generated jsonnet is committed and in sync with its inputs.",
@ -1625,7 +1619,7 @@ def verify_gen_jsonnet_step():
def trigger_test_release():
return {
"name": "trigger-test-release",
"image": build_image,
"image": images["build_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token_pr"),
"TEST_TAG": "v0.0.0-test",
@ -1659,7 +1653,7 @@ def trigger_test_release():
def artifacts_page_step():
return {
"name": "artifacts-page",
"image": build_image,
"image": images["build_image"],
"depends_on": [
"compile-build-cmd",
],
@ -1687,7 +1681,7 @@ def compile_build_cmd(edition = "oss"):
]
return {
"name": "compile-build-cmd",
"image": go_image,
"image": images["go_image"],
"commands": [
"go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd",
],

29
scripts/drone/utils/images.star Normal file
View File

@ -0,0 +1,29 @@
"""
This module contains all the docker images that are used to build test and publish Grafana.
"""
images = {
"cloudsdk_image": "google/cloud-sdk:431.0.0",
"build_image": "grafana/build-container:1.7.4",
"publish_image": "grafana/grafana-ci-deploy:1.3.3",
"alpine_image": "alpine:3.17.1",
"curl_image": "byrnedo/alpine-curl:0.1.8",
"windows_image": "mcr.microsoft.com/windows:1809",
"wix_image": "grafana/ci-wix:0.1.1",
"go_image": "golang:1.20.4",
"windows_go_image": "grafana/grafana-ci-windows-test:0.1.0",
"plugins_slack_image": "plugins/slack",
"postgres_alpine_image": "postgres:12.3-alpine",
"mysql5_image": "mysql:5.7.39",
"mysql8_image": "mysql:8.0.32",
"redis_alpine_image": "redis:6.2.11-alpine",
"memcached_alpine_image": "memcached:1.6.9-alpine",
"windows_server_core_image": "docker:windowsservercore-1809",
"package_publish_image": "us.gcr.io/kubernetes-dev/package-publish:latest",
"openldap_image": "osixia/openldap:1.4.0",
"drone_downstream_image": "grafana/drone-downstream",
"docker_puppeteer_image": "grafana/docker-puppeteer:1.1.0",
"docs_image": "grafana/docs-base:latest",
"cypress_image": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97",
"cloud_datasources_e2e_image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest",
}