CI: Cleanup - Remove security related steps (#70788)

* Remove security related steps

* More cleanup
This commit is contained in:
Dimitris Sotirakis 2023-06-28 11:12:47 +03:00 committed by GitHub
parent dde4a03544
commit 7f55ba9c6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 24 additions and 295 deletions

View File

@ -11,7 +11,6 @@ load("scripts/drone/events/pr.star", "pr_pipelines")
load("scripts/drone/events/main.star", "main_pipelines")
load(
"scripts/drone/events/release.star",
"artifacts_page_pipeline",
"enterprise2_pipelines",
"enterprise_pipelines",
"integration_test_pipelines",
@ -28,7 +27,6 @@ load(
load(
"scripts/drone/pipelines/publish_images.star",
"publish_image_pipelines_public",
"publish_image_pipelines_security",
)
load(
"scripts/drone/pipelines/ci_images.star",
@ -52,7 +50,6 @@ def main(_ctx):
enterprise_pipelines() +
enterprise2_pipelines() +
publish_image_pipelines_public() +
publish_image_pipelines_security() +
publish_github_pipeline("public") +
publish_github_pipeline("security") +
publish_aws_marketplace_pipeline("public") +
@ -66,7 +63,6 @@ def main(_ctx):
"event": ["promote"],
"target": ["test-windows"],
}, "oss", "testing")] +
artifacts_page_pipeline() +
version_branch_pipelines() +
integration_test_pipelines() +
publish_ci_windows_test_image_pipeline() +

View File

@ -3888,91 +3888,6 @@ volumes:
clone:
retries: 3
depends_on: []
environment:
EDITION: enterprise
image_pull_secrets:
- dockerconfigjson
kind: pipeline
name: publish-docker-enterprise-security
node:
type: no-parallel
platform:
arch: amd64
os: linux
services: []
steps:
- commands:
- echo $DRONE_RUNNER_NAME
image: alpine:3.17.1
name: identify-runner
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
- commands:
- go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
depends_on: []
environment:
CGO_ENABLED: 0
image: golang:1.20.4
name: compile-build-cmd
- commands:
- ./bin/build artifacts docker fetch --edition enterprise
depends_on:
- compile-build-cmd
environment:
DOCKER_ENTERPRISE2_REPO:
from_secret: docker_enterprise2_repo
DOCKER_PASSWORD:
from_secret: docker_password
DOCKER_USER:
from_secret: docker_username
GCP_KEY:
from_secret: gcp_key
image: google/cloud-sdk:431.0.0
name: fetch-images-enterprise
volumes:
- name: docker
path: /var/run/docker.sock
- commands:
- ./bin/grabpl artifacts docker publish --security --dockerhub-repo grafana/grafana-enterprise
--version-tag ${DRONE_TAG}
depends_on:
- fetch-images-enterprise
environment:
DOCKER_PASSWORD:
from_secret: docker_password
DOCKER_USER:
from_secret: docker_username
GCP_KEY:
from_secret: gcp_key
GITHUB_APP_ID:
from_secret: delivery-bot-app-id
GITHUB_APP_INSTALLATION_ID:
from_secret: delivery-bot-app-installation-id
GITHUB_APP_PRIVATE_KEY:
from_secret: delivery-bot-app-private-key
image: google/cloud-sdk:431.0.0
name: publish-images-grafana-enterprise
volumes:
- name: docker
path: /var/run/docker.sock
trigger:
event:
- promote
target:
- security
type: docker
volumes:
- host:
path: /var/run/docker.sock
name: docker
---
clone:
retries: 3
depends_on: []
environment:
EDITION: enterprise2
image_pull_secrets:
@ -4172,18 +4087,14 @@ steps:
image: golang:1.20.4
name: compile-build-cmd
- commands:
- ./bin/build artifacts packages --security --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET}
- ./bin/build artifacts packages --tag $${{DRONE_TAG}} --src-bucket $${{PRERELEASE_BUCKET}}
depends_on:
- compile-build-cmd
environment:
ENTERPRISE2_SECURITY_PREFIX:
from_secret: enterprise2_security_prefix
GCP_KEY:
from_secret: gcp_key
PRERELEASE_BUCKET:
from_secret: prerelease_bucket
SECURITY_DEST_BUCKET:
from_secret: security_dest_bucket
image: grafana/grafana-ci-deploy:1.3.3
name: publish-artifacts
- commands:
@ -4199,6 +4110,17 @@ steps:
from_secret: static_asset_editions
image: grafana/grafana-ci-deploy:1.3.3
name: publish-static-assets
- commands:
- ./bin/build artifacts storybook --tag ${DRONE_TAG}
depends_on:
- compile-build-cmd
environment:
GCP_KEY:
from_secret: gcp_key
PRERELEASE_BUCKET:
from_secret: prerelease_bucket
image: grafana/grafana-ci-deploy:1.3.3
name: publish-storybook
trigger:
event:
- promote
@ -4234,18 +4156,14 @@ steps:
image: golang:1.20.4
name: compile-build-cmd
- commands:
- ./bin/build artifacts packages --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET}
- ./bin/build artifacts packages --tag $${{DRONE_TAG}} --src-bucket $${{PRERELEASE_BUCKET}}
depends_on:
- compile-build-cmd
environment:
ENTERPRISE2_SECURITY_PREFIX:
from_secret: enterprise2_security_prefix
GCP_KEY:
from_secret: gcp_key
PRERELEASE_BUCKET:
from_secret: prerelease_bucket
SECURITY_DEST_BUCKET:
from_secret: security_dest_bucket
image: grafana/grafana-ci-deploy:1.3.3
name: publish-artifacts
- commands:
@ -4780,81 +4698,6 @@ volumes:
path: //./pipe/docker_engine/
name: docker
---
clone:
disable: true
depends_on: []
environment:
EDITION: enterprise
image_pull_secrets:
- dockerconfigjson
kind: pipeline
name: publish-artifacts-page
node:
type: no-parallel
platform:
arch: amd64
os: linux
services: []
steps:
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
- commands:
- git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git"
- cd grafana-enterprise
- git checkout ${DRONE_TAG}
environment:
GITHUB_TOKEN:
from_secret: github_token
image: grafana/build-container:1.7.4
name: clone-enterprise
- commands:
- mv bin/grabpl /tmp/
- rmdir bin
- mv grafana-enterprise /tmp/
- /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise
${DRONE_TAG}
- mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json
- mkdir bin
- mv /tmp/grabpl bin/
depends_on:
- clone-enterprise
- grabpl
environment:
GITHUB_TOKEN:
from_secret: github_token
image: grafana/build-container:1.7.4
name: init-enterprise
- commands:
- go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
depends_on:
- init-enterprise
environment:
CGO_ENABLED: 0
image: golang:1.20.4
name: compile-build-cmd
- commands:
- ./bin/build artifacts-page
depends_on:
- compile-build-cmd
environment:
GCP_KEY:
from_secret: gcp_key
image: grafana/build-container:1.7.4
name: artifacts-page
trigger:
event:
- promote
target: security
type: docker
volumes:
- host:
path: /var/run/docker.sock
name: docker
---
clone:
retries: 3
depends_on: []
@ -7291,36 +7134,18 @@ get:
kind: secret
name: aws_secret_access_key
---
get:
name: bucket
path: infra/data/ci/grafana-release-eng/security-bucket
kind: secret
name: security_dest_bucket
---
get:
name: static_asset_editions
path: infra/data/ci/grafana-release-eng/artifact-publishing
kind: secret
name: static_asset_editions
---
get:
name: security_prefix
path: infra/data/ci/grafana-release-eng/enterprise2
kind: secret
name: enterprise2_security_prefix
---
get:
name: cdn_path
path: infra/data/ci/grafana-release-eng/enterprise2
kind: secret
name: enterprise2-cdn-path
---
get:
name: security_prefix
path: infra/data/ci/grafana-release-eng/enterprise2
kind: secret
name: enterprise2_security_prefix
---
get:
name: gcp_service_account_base64
path: infra/data/ci/grafana-release-eng/rgm
@ -7364,6 +7189,6 @@ kind: secret
name: delivery-bot-app-private-key
---
kind: signature
hmac: 2cfe53f430c7cfec1b422a329b009b7c81cd52e897abb7a940115e524796178a
hmac: 5b576f21a7afb08759ad4ac7b74a54afcfbf9b5eac0e5894857ce4cdd0feed50
...

View File

@ -4,7 +4,6 @@ This module returns all the pipelines used in the event of a release along with
load(
"scripts/drone/steps/lib.star",
"artifacts_page_step",
"build_backend_step",
"build_docker_images_step",
"build_frontend_package_step",
@ -525,7 +524,6 @@ def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_tr
publish_images_step(
"enterprise2",
"release",
mode = "enterprise2",
docker_repo = "${{DOCKER_ENTERPRISE2_REPO}}",
),
],
@ -556,23 +554,16 @@ def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_tr
return pipelines
def publish_artifacts_step(mode):
security = ""
if mode == "security":
security = "--security "
def publish_artifacts_step():
return {
"name": "publish-artifacts",
"image": images["publish_image"],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
"PRERELEASE_BUCKET": from_secret("prerelease_bucket"),
"ENTERPRISE2_SECURITY_PREFIX": from_secret("enterprise2_security_prefix"),
"SECURITY_DEST_BUCKET": from_secret("security_dest_bucket"),
},
"commands": [
"./bin/build artifacts packages {}--tag $${{DRONE_TAG}} --src-bucket $${{PRERELEASE_BUCKET}}".format(
security,
),
"./bin/build artifacts packages --tag $${{DRONE_TAG}} --src-bucket $${{PRERELEASE_BUCKET}}",
],
"depends_on": ["compile-build-cmd"],
}
@ -622,11 +613,10 @@ def publish_artifacts_pipelines(mode):
}
steps = [
compile_build_cmd(),
publish_artifacts_step(mode),
publish_artifacts_step(),
publish_static_assets_step(),
publish_storybook_step(),
]
if mode != "security":
steps.extend([publish_storybook_step()])
return [
pipeline(
@ -709,27 +699,6 @@ def publish_npm_pipelines():
),
]
def artifacts_page_pipeline():
trigger = {
"event": ["promote"],
"target": "security",
}
return [
pipeline(
name = "publish-artifacts-page",
trigger = trigger,
steps = [
download_grabpl_step(),
clone_enterprise_step(source = "${DRONE_TAG}"),
init_enterprise_step("release"),
compile_build_cmd("enterprise"),
artifacts_page_step(),
],
edition = "enterprise",
environment = {"EDITION": "enterprise"},
),
]
def integration_test_pipelines():
"""
Trigger integration tests on release builds

View File

@ -112,14 +112,12 @@ def build_e2e(trigger, ver_mode):
publish_images_step(
docker_repo = "grafana",
edition = edition,
mode = "",
trigger = trigger_oss,
ver_mode = ver_mode,
),
publish_images_step(
docker_repo = "grafana-oss",
edition = edition,
mode = "",
trigger = trigger_oss,
ver_mode = ver_mode,
),
@ -155,7 +153,6 @@ def build_e2e(trigger, ver_mode):
publish_images_step(
docker_repo = "grafana",
edition = edition,
mode = "",
trigger = trigger_oss,
ver_mode = ver_mode,
),

View File

@ -15,14 +15,13 @@ load(
"pipeline",
)
def publish_image_steps(edition, mode, docker_repo):
def publish_image_steps(edition, docker_repo):
"""Generates the steps used for publising Docker images using grabpl.
Args:
edition: controls which version of an image is fetched in the case of a release.
It also controls which publishing implementation is used.
If edition == 'oss', it additionally publishes the grafana/grafana-oss repository.
mode: uses to control the publishing of security images when mode == 'security'.
docker_repo: the Docker image name.
It is combined with the 'grafana/' library prefix.
@ -34,12 +33,12 @@ def publish_image_steps(edition, mode, docker_repo):
download_grabpl_step(),
compile_build_cmd(),
fetch_images_step(edition),
publish_images_step(edition, "release", mode, docker_repo),
publish_images_step(edition, "release", docker_repo),
]
if edition == "oss":
steps.append(
publish_images_step(edition, "release", mode, "grafana-oss"),
publish_images_step(edition, "release", "grafana-oss"),
)
return steps
@ -59,7 +58,7 @@ def publish_image_pipelines_public():
pipeline(
name = "publish-docker-oss-{}".format(mode),
trigger = trigger,
steps = publish_image_steps(edition = "oss", mode = mode, docker_repo = "grafana"),
steps = publish_image_steps(edition = "oss", docker_repo = "grafana"),
edition = "",
environment = {"EDITION": "oss"},
),
@ -68,27 +67,6 @@ def publish_image_pipelines_public():
trigger = trigger,
steps = publish_image_steps(
edition = "enterprise",
mode = mode,
docker_repo = "grafana-enterprise",
),
edition = "",
environment = {"EDITION": "enterprise"},
),
]
def publish_image_pipelines_security():
mode = "security"
trigger = {
"event": ["promote"],
"target": [mode],
}
return [
pipeline(
name = "publish-docker-enterprise-{}".format(mode),
trigger = trigger,
steps = publish_image_steps(
edition = "enterprise",
mode = mode,
docker_repo = "grafana-enterprise",
),
edition = "",

View File

@ -1130,7 +1130,7 @@ def fetch_images_step(edition):
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
}
def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None):
def publish_images_step(edition, ver_mode, docker_repo, trigger = None):
"""Generates a step for publishing public Docker images with grabpl.
Args:
@ -1138,7 +1138,6 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None):
It also controls which publishing implementation is used.
ver_mode: controls whether the image needs to be built or retrieved from a previous build.
If ver_mode == 'release', the previously built image is fetched instead of being built again.
mode: uses to control the publishing of security images when mode == 'security'.
docker_repo: the Docker image name.
It is combined with the 'grafana/' library prefix.
trigger: a Drone trigger for the pipeline.
@ -1149,10 +1148,6 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None):
"""
name = docker_repo
docker_repo = "grafana/{}".format(docker_repo)
if mode == "security":
mode = "--{} ".format(mode)
else:
mode = ""
environment = {
"GCP_KEY": from_secret("gcp_key"),
@ -1163,8 +1158,7 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None):
"GITHUB_APP_PRIVATE_KEY": from_secret("delivery-bot-app-private-key"),
}
cmd = "./bin/grabpl artifacts docker publish {}--dockerhub-repo {}".format(
mode,
cmd = "./bin/grabpl artifacts docker publish --dockerhub-repo {}".format(
docker_repo,
)
@ -1654,21 +1648,6 @@ def trigger_test_release():
},
}
def artifacts_page_step():
return {
"name": "artifacts-page",
"image": images["build_image"],
"depends_on": [
"compile-build-cmd",
],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
},
"commands": [
"./bin/build artifacts-page",
],
}
def end_to_end_tests_deps():
return [
"end-to-end-tests-dashboards-suite",

View File

@ -99,31 +99,16 @@ def secrets():
"secret/data/common/aws-marketplace",
"aws_secret_access_key",
),
vault_secret(
"security_dest_bucket",
"infra/data/ci/grafana-release-eng/security-bucket",
"bucket",
),
vault_secret(
"static_asset_editions",
"infra/data/ci/grafana-release-eng/artifact-publishing",
"static_asset_editions",
),
vault_secret(
"enterprise2_security_prefix",
"infra/data/ci/grafana-release-eng/enterprise2",
"security_prefix",
),
vault_secret(
"enterprise2-cdn-path",
"infra/data/ci/grafana-release-eng/enterprise2",
"cdn_path",
),
vault_secret(
"enterprise2_security_prefix",
"infra/data/ci/grafana-release-eng/enterprise2",
"security_prefix",
),
vault_secret(
rgm_gcp_key_base64,
"infra/data/ci/grafana-release-eng/rgm",