From 99900cbdd830651004eb40673165cf18ce3f77a7 Mon Sep 17 00:00:00 2001 From: Dimitris Sotirakis Date: Mon, 15 Nov 2021 19:14:33 +0200 Subject: [PATCH] CI: Make `grabpl` and `lint-drone` standalone steps (#41644) * Extract grabpl download as separate step * Change image to alpine * Change image to curlimage * Change image to alpine-curl * Extract lint-drone step * Add lint-drone to release and main pipelines * Rename lint_drone to lint_drone_step * Sign drone --- .drone.yml | 208 ++++++++++++++++++++------- scripts/drone/pipelines/main.star | 2 + scripts/drone/pipelines/pr.star | 2 + scripts/drone/pipelines/release.star | 2 + scripts/drone/steps/lib.star | 40 ++++-- scripts/drone/utils/utils.star | 5 +- 6 files changed, 199 insertions(+), 60 deletions(-) diff --git a/.drone.yml b/.drone.yml index 95d7a4434ef..28684f4687e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -22,20 +22,28 @@ services: image: mysql:5.6.48 name: mysql steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.14.2 name: identify-runner - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl - - chmod +x bin/grabpl - - ./bin/grabpl verify-drone - make gen-go - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} - yarn install --immutable image: grafana/build-container:1.4.5 name: initialize +- commands: + - ./bin/grabpl verify-drone + depends_on: + - grabpl + image: byrnedo/alpine-curl:0.1.8 + name: lint-drone - commands: - |- echo -e "unknwon @@ -281,20 +289,28 @@ services: image: mysql:5.6.48 name: mysql steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.14.2 name: identify-runner - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl - - chmod +x bin/grabpl - - ./bin/grabpl verify-drone - make gen-go - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} - yarn install --immutable image: grafana/build-container:1.4.5 name: initialize +- commands: + - ./bin/grabpl verify-drone + depends_on: + - grabpl + image: byrnedo/alpine-curl:0.1.8 + name: lint-drone - image: grafana/drone-downstream name: trigger-enterprise-downstream settings: @@ -607,6 +623,12 @@ platform: version: "1809" services: [] steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $env:DRONE_RUNNER_NAME image: mcr.microsoft.com/windows:1809 @@ -654,15 +676,17 @@ platform: os: linux services: [] steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.14.2 name: identify-runner - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl - - chmod +x bin/grabpl - - ./bin/grabpl verify-drone - make gen-go image: grafana/build-container:1.4.5 name: initialize @@ -742,21 +766,29 @@ services: image: mysql:5.6.48 name: mysql steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.14.2 name: identify-runner - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl - - chmod +x bin/grabpl - - ./bin/grabpl verify-drone - make gen-go - ./bin/grabpl verify-version ${DRONE_TAG} - ./bin/grabpl gen-version ${DRONE_TAG} - yarn install --immutable image: grafana/build-container:1.4.5 name: initialize +- commands: + - ./bin/grabpl verify-drone + depends_on: + - grabpl + image: byrnedo/alpine-curl:0.1.8 + name: lint-drone - commands: - |- echo -e "unknwon @@ -1033,6 +1065,12 @@ platform: version: "1809" services: [] steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $env:DRONE_RUNNER_NAME image: mcr.microsoft.com/windows:1809 @@ -1100,14 +1138,17 @@ services: image: memcached:1.6.9-alpine name: memcached steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.14.2 name: identify-runner - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl - - chmod +x bin/grabpl - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - cd grafana-enterprise - git checkout ${DRONE_TAG} @@ -1124,15 +1165,20 @@ steps: - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - mkdir bin - mv /tmp/grabpl bin/ - - ./bin/grabpl verify-drone - make gen-go - ./bin/grabpl verify-version ${DRONE_TAG} - ./bin/grabpl gen-version ${DRONE_TAG} - yarn install --immutable depends_on: - - clone + - grabpl image: grafana/build-container:1.4.5 name: initialize +- commands: + - ./bin/grabpl verify-drone + depends_on: + - grabpl + image: byrnedo/alpine-curl:0.1.8 + name: lint-drone - commands: - |- echo -e "unknwon @@ -1499,6 +1545,12 @@ platform: version: "1809" services: [] steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $env:DRONE_RUNNER_NAME image: mcr.microsoft.com/windows:1809 @@ -1564,15 +1616,17 @@ platform: os: linux services: [] steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.14.2 name: identify-runner - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl - - chmod +x bin/grabpl - - ./bin/grabpl verify-drone - make gen-go - ./bin/grabpl verify-version ${DRONE_TAG} image: grafana/build-container:1.4.5 @@ -1671,21 +1725,29 @@ services: image: mysql:5.6.48 name: mysql steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.14.2 name: identify-runner - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl - - chmod +x bin/grabpl - - ./bin/grabpl verify-drone - make gen-go - ./bin/grabpl verify-version v7.3.0-test - ./bin/grabpl gen-version v7.3.0-test - yarn install --immutable image: grafana/build-container:1.4.5 name: initialize +- commands: + - ./bin/grabpl verify-drone + depends_on: + - grabpl + image: byrnedo/alpine-curl:0.1.8 + name: lint-drone - commands: - |- echo -e "unknwon @@ -1950,6 +2012,12 @@ platform: version: "1809" services: [] steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $env:DRONE_RUNNER_NAME image: mcr.microsoft.com/windows:1809 @@ -2018,14 +2086,17 @@ services: image: memcached:1.6.9-alpine name: memcached steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.14.2 name: identify-runner - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl - - chmod +x bin/grabpl - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - cd grafana-enterprise - git checkout main @@ -2042,15 +2113,20 @@ steps: - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - mkdir bin - mv /tmp/grabpl bin/ - - ./bin/grabpl verify-drone - make gen-go - ./bin/grabpl verify-version v7.3.0-test - ./bin/grabpl gen-version v7.3.0-test - yarn install --immutable depends_on: - - clone + - grabpl image: grafana/build-container:1.4.5 name: initialize +- commands: + - ./bin/grabpl verify-drone + depends_on: + - grabpl + image: byrnedo/alpine-curl:0.1.8 + name: lint-drone - commands: - |- echo -e "unknwon @@ -2409,6 +2485,12 @@ platform: version: "1809" services: [] steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $env:DRONE_RUNNER_NAME image: mcr.microsoft.com/windows:1809 @@ -2475,15 +2557,17 @@ platform: os: linux services: [] steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.14.2 name: identify-runner - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl - - chmod +x bin/grabpl - - ./bin/grabpl verify-drone - make gen-go - ./bin/grabpl verify-version v7.3.0-test image: grafana/build-container:1.4.5 @@ -2587,20 +2671,28 @@ services: image: mysql:5.6.48 name: mysql steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.14.2 name: identify-runner - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl - - chmod +x bin/grabpl - - ./bin/grabpl verify-drone - make gen-go - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} - yarn install --immutable image: grafana/build-container:1.4.5 name: initialize +- commands: + - ./bin/grabpl verify-drone + depends_on: + - grabpl + image: byrnedo/alpine-curl:0.1.8 + name: lint-drone - commands: - |- echo -e "unknwon @@ -2843,6 +2935,12 @@ platform: version: "1809" services: [] steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $env:DRONE_RUNNER_NAME image: mcr.microsoft.com/windows:1809 @@ -2906,14 +3004,17 @@ services: image: memcached:1.6.9-alpine name: memcached steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.14.2 name: identify-runner - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl - - chmod +x bin/grabpl - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - cd grafana-enterprise - git checkout ${DRONE_BRANCH} @@ -2930,14 +3031,19 @@ steps: - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - mkdir bin - mv /tmp/grabpl bin/ - - ./bin/grabpl verify-drone - make gen-go - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} - yarn install --immutable depends_on: - - clone + - grabpl image: grafana/build-container:1.4.5 name: initialize +- commands: + - ./bin/grabpl verify-drone + depends_on: + - grabpl + image: byrnedo/alpine-curl:0.1.8 + name: lint-drone - commands: - |- echo -e "unknwon @@ -3301,6 +3407,12 @@ platform: version: "1809" services: [] steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.5.5/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl - commands: - echo $env:DRONE_RUNNER_NAME image: mcr.microsoft.com/windows:1809 @@ -3430,6 +3542,6 @@ kind: secret name: drone_token --- kind: signature -hmac: d2db1d71a52922f66df9d327cb8991944d6fd050b4ad4d36e5f7cfb9f79f9297 +hmac: 73b6a37371bd68484809eb311063fc37319a1cff2eb3aabbdad230ba88e13ff4 ... diff --git a/scripts/drone/pipelines/main.star b/scripts/drone/pipelines/main.star index 70cc840b94f..02e93681c00 100644 --- a/scripts/drone/pipelines/main.star +++ b/scripts/drone/pipelines/main.star @@ -1,5 +1,6 @@ load( 'scripts/drone/steps/lib.star', + 'lint_drone_step', 'lint_backend_step', 'lint_frontend_step', 'codespell_step', @@ -53,6 +54,7 @@ def get_steps(edition, is_downstream=False): publish = edition != 'enterprise' or is_downstream include_enterprise2 = edition == 'enterprise' steps = [ + lint_drone_step(), enterprise_downstream_step(edition=edition), codespell_step(), shellcheck_step(), diff --git a/scripts/drone/pipelines/pr.star b/scripts/drone/pipelines/pr.star index fd838951a2b..c55a7536e06 100644 --- a/scripts/drone/pipelines/pr.star +++ b/scripts/drone/pipelines/pr.star @@ -1,5 +1,6 @@ load( 'scripts/drone/steps/lib.star', + 'lint_drone_step', 'lint_backend_step', 'lint_frontend_step', 'codespell_step', @@ -46,6 +47,7 @@ def pr_pipelines(edition): variants = ['linux-x64', 'linux-x64-musl', 'osx64', 'win64', 'armv6',] include_enterprise2 = edition == 'enterprise' steps = [ + lint_drone_step(), codespell_step(), shellcheck_step(), lint_backend_step(edition=edition), diff --git a/scripts/drone/pipelines/release.star b/scripts/drone/pipelines/release.star index 72739465133..d4f221184d6 100644 --- a/scripts/drone/pipelines/release.star +++ b/scripts/drone/pipelines/release.star @@ -1,5 +1,6 @@ load( 'scripts/drone/steps/lib.star', + 'lint_drone_step', 'test_release_ver', 'build_image', 'publish_image', @@ -82,6 +83,7 @@ def get_steps(edition, ver_mode): tries = 5 steps = [ + lint_drone_step(), codespell_step(), shellcheck_step(), lint_backend_step(edition=edition), diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index deda4f4c3ea..0f52bf333d9 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -6,6 +6,7 @@ publish_image = 'grafana/grafana-ci-deploy:1.3.1' grafana_docker_image = 'grafana/drone-grafana-docker:0.3.2' deploy_docker_image = 'us.gcr.io/kubernetes-dev/drone/plugins/deploy-image' alpine_image = 'alpine:3.14.2' +curl_image = 'byrnedo/alpine-curl:0.1.8' windows_image = 'mcr.microsoft.com/windows:1809' wix_image = 'grafana/ci-wix:0.1.1' test_release_ver = 'v7.3.0-test' @@ -33,15 +34,7 @@ def initialize_step(edition, platform, ver_mode, is_downstream=False, install_de }, ] - download_grabpl_cmds = [ - 'mkdir -p bin', - 'curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/grabpl'.format( - grabpl_version - ), - 'chmod +x bin/grabpl', - ] common_cmds = [ - './bin/grabpl verify-drone', # Generate Go code, will install Wire # TODO: Install Wire in Docker image instead 'make gen-go', @@ -94,7 +87,7 @@ def initialize_step(edition, platform, ver_mode, is_downstream=False, install_de 'environment': { 'GITHUB_TOKEN': from_secret(github_token), }, - 'commands': download_grabpl_cmds + [ + 'commands': [ 'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git"', 'cd grafana-enterprise', 'git checkout {}'.format(committish), @@ -104,7 +97,7 @@ def initialize_step(edition, platform, ver_mode, is_downstream=False, install_de 'name': 'initialize', 'image': build_image, 'depends_on': [ - 'clone', + 'grabpl', ], 'commands': [ 'mv bin/grabpl /tmp/', @@ -125,12 +118,37 @@ def initialize_step(edition, platform, ver_mode, is_downstream=False, install_de { 'name': 'initialize', 'image': build_image, - 'commands': download_grabpl_cmds + common_cmds, + 'commands': common_cmds, }, ] return steps +def download_grabpl(): + return { + 'name': 'grabpl', + 'image': curl_image, + 'commands': [ + 'mkdir -p bin', + 'curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/grabpl'.format( + grabpl_version + ), + 'chmod +x bin/grabpl', + ] + } + +def lint_drone_step(): + return { + 'name': 'lint-drone', + 'image': curl_image, + 'commands': [ + './bin/grabpl verify-drone', + ], + 'depends_on': [ + 'grabpl', + ], + } + def enterprise_downstream_step(edition): if edition in ('enterprise', 'enterprise2'): return None diff --git a/scripts/drone/utils/utils.star b/scripts/drone/utils/utils.star index 7cc8ef01a48..4e9de0ed5ed 100644 --- a/scripts/drone/utils/utils.star +++ b/scripts/drone/utils/utils.star @@ -1,6 +1,7 @@ load( 'scripts/drone/steps/lib.star', 'initialize_step', + 'download_grabpl', 'slack_step', ) @@ -31,13 +32,15 @@ def pipeline( } } + grabpl_step = [download_grabpl()] + pipeline = { 'kind': 'pipeline', 'type': 'docker', 'name': name, 'trigger': trigger, 'services': services, - 'steps': initialize_step( + 'steps': grabpl_step + initialize_step( edition, platform, is_downstream=is_downstream, install_deps=install_deps, ver_mode=ver_mode, ) + steps, 'depends_on': depends_on,