From 4386604751daecb5adc2fe6a1a6b282423d07810 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Mon, 23 Sep 2019 12:00:18 +0200 Subject: [PATCH] Build: Split up task in the CI pipeline to ease running outside circleci (#18861) * build: make sign rpm packages not depend on checking out private key * build: move commands from circleci config into verify signed packages script * build: split update and publish of deb and rpm into two scripts * use files argument for sign and verify packages * validate files argument for sign and verify packages * update test publish of deb/rpm readme --- .circleci/config.yml | 57 ++++++++++++++----- scripts/build/load-signing-key.sh | 7 --- scripts/build/prepare_signing_key.sh | 6 ++ scripts/build/sign_packages.sh | 18 +++++- scripts/build/update_repo/README.md | 17 +++--- scripts/build/update_repo/load-signing-key.sh | 6 ++ scripts/build/update_repo/publish-deb.sh | 30 ++++++++++ scripts/build/update_repo/publish-rpm.sh | 41 +++++++++++++ .../update_repo/test-publish-deb-repo.sh | 3 + .../update_repo/test-publish-rpm-repo.sh | 3 + .../build/update_repo/test-update-deb-repo.sh | 2 +- scripts/build/update_repo/update-deb.sh | 10 ---- scripts/build/update_repo/update-rpm.sh | 4 -- scripts/build/verify_signed_packages.sh | 8 +++ 14 files changed, 164 insertions(+), 48 deletions(-) delete mode 100755 scripts/build/load-signing-key.sh create mode 100755 scripts/build/prepare_signing_key.sh create mode 100755 scripts/build/update_repo/load-signing-key.sh create mode 100755 scripts/build/update_repo/publish-deb.sh create mode 100755 scripts/build/update_repo/publish-rpm.sh create mode 100755 scripts/build/update_repo/test-publish-deb-repo.sh create mode 100755 scripts/build/update_repo/test-publish-rpm-repo.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 36808e052b9..19fa401ba23 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -214,15 +214,15 @@ jobs: - run: name: build and package grafana command: './scripts/build/build-all.sh' + - run: + name: Prepare GPG private key + command: './scripts/build/prepare_signing_key.sh' - run: name: sign packages - command: './scripts/build/sign_packages.sh' + command: './scripts/build/sign_packages.sh dist/*.rpm' - run: name: verify signed packages - command: | - mkdir -p ~/.rpmdb/pubkeys - curl -s https://packages.grafana.com/gpg.key > ~/.rpmdb/pubkeys/grafana.key - ./scripts/build/verify_signed_packages.sh dist/*.rpm + command: './scripts/build/verify_signed_packages.sh dist/*.rpm' - run: name: sha-sum packages command: 'go run build.go sha-dist' @@ -249,9 +249,12 @@ jobs: - run: name: build and package grafana command: './scripts/build/build.sh' + - run: + name: Prepare GPG private key + command: './scripts/build/prepare_signing_key.sh' - run: name: sign packages - command: './scripts/build/sign_packages.sh' + command: './scripts/build/sign_packages.sh dist/*.rpm' - run: name: sha-sum packages command: 'go run build.go sha-dist' @@ -360,9 +363,12 @@ jobs: - run: name: package grafana command: './scripts/build/build.sh --fast --package-only' + - run: + name: Prepare GPG private key + command: './scripts/build/prepare_signing_key.sh' - run: name: sign packages - command: './scripts/build/sign_packages.sh' + command: './scripts/build/sign_packages.sh dist/*.rpm' - run: name: sha-sum packages command: 'go run build.go sha-dist' @@ -435,9 +441,12 @@ jobs: - run: name: build and package enterprise command: './scripts/build/build.sh -enterprise' + - run: + name: Prepare GPG private key + command: './scripts/build/prepare_signing_key.sh' - run: name: sign packages - command: './scripts/build/sign_packages.sh' + command: './scripts/build/sign_packages.sh dist/*.rpm' - run: name: sha-sum packages command: 'go run build.go sha-dist' @@ -476,15 +485,15 @@ jobs: - run: name: build and package grafana command: './scripts/build/build-all.sh -enterprise' + - run: + name: Prepare GPG private key + command: './scripts/build/prepare_signing_key.sh' - run: name: sign packages - command: './scripts/build/sign_packages.sh' + command: './scripts/build/sign_packages.sh dist/*.rpm' - run: name: verify signed packages - command: | - mkdir -p ~/.rpmdb/pubkeys - curl -s https://packages.grafana.com/gpg.key > ~/.rpmdb/pubkeys/grafana.key - ./scripts/build/verify_signed_packages.sh dist/*.rpm + command: './scripts/build/verify_signed_packages.sh dist/*.rpm' - run: name: sha-sum packages command: 'go run build.go sha-dist' @@ -537,15 +546,24 @@ jobs: - run: name: Deploy to Grafana.com command: './scripts/build/publish.sh --enterprise' + - run: + name: Prepare GPG private key + command: './scripts/build/prepare_signing_key.sh' - run: name: Load GPG private key - command: './scripts/build/load-signing-key.sh' + command: './scripts/build/update_repo/load-signing-key.sh' - run: name: Update Debian repository command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"' + - run: + name: Publish Debian repository + command: './scripts/build/update_repo/publish-deb.sh "enterprise"' - run: name: Update RPM repository command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"' + - run: + name: Publish RPM repository + command: './scripts/build/update_repo/publish-rpm.sh "enterprise" "$CIRCLE_TAG"' deploy-master: @@ -591,15 +609,24 @@ jobs: - run: name: Deploy to Grafana.com command: './scripts/build/publish.sh' + - run: + name: Prepare GPG private key + command: './scripts/build/prepare_signing_key.sh' - run: name: Load GPG private key - command: './scripts/build/load-signing-key.sh' + command: './scripts/build/update_repo/load-signing-key.sh' - run: name: Update Debian repository command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"' + - run: + name: Publish Debian repository + command: './scripts/build/update_repo/publish-deb.sh "oss"' - run: name: Update RPM repository command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"' + - run: + name: Publish RPM repository + command: './scripts/build/update_repo/publish-rpm.sh "oss" "$CIRCLE_TAG"' build-oss-msi: docker: diff --git a/scripts/build/load-signing-key.sh b/scripts/build/load-signing-key.sh deleted file mode 100755 index aa70d289443..00000000000 --- a/scripts/build/load-signing-key.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -e - -git clone git@github.com:torkelo/private.git ~/private-repo -gpg --batch --allow-secret-key-import --import ~/private-repo/signing/private.key -pkill gpg-agent \ No newline at end of file diff --git a/scripts/build/prepare_signing_key.sh b/scripts/build/prepare_signing_key.sh new file mode 100755 index 00000000000..5b16620f753 --- /dev/null +++ b/scripts/build/prepare_signing_key.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e + +git clone git@github.com:torkelo/private.git ~/private-repo +cp ~/private-repo/signing/private.key /private.key diff --git a/scripts/build/sign_packages.sh b/scripts/build/sign_packages.sh index 89581e0f274..86eca561092 100755 --- a/scripts/build/sign_packages.sh +++ b/scripts/build/sign_packages.sh @@ -1,12 +1,24 @@ #!/bin/bash -git clone git@github.com:torkelo/private.git ~/private-repo +set -e -gpg --allow-secret-key-import --import ~/private-repo/signing/private.key +_files=$* + +if [ -z "$_files" ]; then + echo "_files (arg 1) has to be set" + exit 1 +fi + +if [ -z "$GPG_KEY_PASSWORD" ]; then + echo "GPG_KEY_PASSWORD has to be set" + exit 1 +fi + +gpg --allow-secret-key-import --import /private.key cp ./scripts/build/rpmmacros ~/.rpmmacros -for package in dist/*.rpm; do +for package in $_files; do [ -e "$package" ] || continue ./scripts/build/sign_expect "$GPG_KEY_PASSWORD" "$package" done diff --git a/scripts/build/update_repo/README.md b/scripts/build/update_repo/README.md index 22f2ea1241d..8936454db37 100644 --- a/scripts/build/update_repo/README.md +++ b/scripts/build/update_repo/README.md @@ -7,22 +7,23 @@ It's possible to test the repo updates for rpm and deb by running the test scripts within a docker container like this. Tests are being executed by using two buckets on gcp setup for testing. ```bash -docker run -ti --rm -u 0:0 grafana/grafana-ci-deploy:1.2.0 bash # 1.2.0 is the newest image at the time of writing +docker run -ti --rm -u 0:0 grafana/grafana-ci-deploy:1.2.2 bash # 1.2.2 is the newest image at the time of writing # in the container: -mkdir -p /go/src/github.com/grafana/dist -cd /go/src/github.com/grafana +mkdir -p /dist #outside of container: cd /.. -docker cp grafana :/go/src/github.com/grafana/. +docker cp grafana :/ docker cp :/private.key #in container: -gpg --batch --allow-secret-key-import --import /private.key +./scripts/build/update_repo/load-signing-key.sh cd dist && wget https://dl.grafana.com/oss/release/grafana_5.4.3_amd64.deb && wget https://dl.grafana.com/oss/release/grafana-5.4.3-1.x86_64.rpm && cd .. -#run these scripts: -./script/build/update_repo/test-update-deb-repo.sh -./script/build/update_repo/test-update-rpm-repo.sh +#run these scripts to update local deb and rpm repos and publish them: +./scripts/build/update_repo/test-update-deb-repo.sh +./scripts/build/update_repo/test-publish-deb-repo.sh +./scripts/build/update_repo/test-update-rpm-repo.sh +./scripts/build/update_repo/test-publish-rpm-repo.sh ``` diff --git a/scripts/build/update_repo/load-signing-key.sh b/scripts/build/update_repo/load-signing-key.sh new file mode 100755 index 00000000000..2ac456429da --- /dev/null +++ b/scripts/build/update_repo/load-signing-key.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e + +gpg --batch --allow-secret-key-import --import /private.key +pkill gpg-agent \ No newline at end of file diff --git a/scripts/build/update_repo/publish-deb.sh b/scripts/build/update_repo/publish-deb.sh new file mode 100755 index 00000000000..c8c295fe2e9 --- /dev/null +++ b/scripts/build/update_repo/publish-deb.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +RELEASE_TYPE="${1:-}" +GCP_DB_BUCKET="${2:-grafana-aptly-db}" +GCP_REPO_BUCKET="${3:-grafana-repo}" + +if [ -z "$RELEASE_TYPE" ]; then + echo "RELEASE_TYPE (arg 1) has to be set" + exit 1 +fi + +if [[ "$RELEASE_TYPE" != "oss" && "$RELEASE_TYPE" != "enterprise" ]]; then + echo "RELEASE_TYPE (arg 1) must be either oss or enterprise." + exit 1 +fi + +set -e + +# Update the repo and db on gcp + +gsutil -m rsync -r -d /deb-repo/db "gs://$GCP_DB_BUCKET/$RELEASE_TYPE" + +# Uploads the binaries before the metadata (to prevent 404's for debs) +gsutil -m rsync -r /deb-repo/repo/grafana/pool "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb/pool" + +gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb" + +# usage: +# +# deb https://packages.grafana.com/oss/deb stable main diff --git a/scripts/build/update_repo/publish-rpm.sh b/scripts/build/update_repo/publish-rpm.sh new file mode 100755 index 00000000000..d1d5ff86079 --- /dev/null +++ b/scripts/build/update_repo/publish-rpm.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +RELEASE_TYPE="${1:-}" +RELEASE_TAG="${2:-}" +GCP_REPO_BUCKET="${3:-grafana-repo}" + +REPO="rpm" + +if [ -z "$RELEASE_TYPE" ]; then + echo "RELEASE_TYPE (arg 1) has to be set" + exit 1 +fi + +if [[ "$RELEASE_TYPE" != "oss" && "$RELEASE_TYPE" != "enterprise" ]]; then + echo "RELEASE_TYPE (arg 1) must be either oss or enterprise." + exit 1 +fi + +if echo "$RELEASE_TAG" | grep -q "beta"; then + REPO="rpm-beta" +fi + +set -e + +# Setup environment +BUCKET="gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/$REPO" + +# Update the repo and db on gcp +gsutil -m cp /rpm-repo/*.rpm "$BUCKET" # sync binaries first to avoid cache misses +gsutil -m rsync -r -d /rpm-repo "$BUCKET" + +# usage: +# [grafana] +# name=grafana +# baseurl=https://packages.grafana.com/oss/rpm +# repo_gpgcheck=1 +# enabled=1 +# gpgcheck=1 +# gpgkey=https://packages.grafana.com/gpg.key +# sslverify=1 +# sslcacert=/etc/pki/tls/certs/ca-bundle.crt diff --git a/scripts/build/update_repo/test-publish-deb-repo.sh b/scripts/build/update_repo/test-publish-deb-repo.sh new file mode 100755 index 00000000000..2db342027e5 --- /dev/null +++ b/scripts/build/update_repo/test-publish-deb-repo.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +./scripts/build/update_repo/publish-deb.sh "oss" "grafana-testing-aptly-db" "grafana-testing-repo" diff --git a/scripts/build/update_repo/test-publish-rpm-repo.sh b/scripts/build/update_repo/test-publish-rpm-repo.sh new file mode 100755 index 00000000000..0ee037da642 --- /dev/null +++ b/scripts/build/update_repo/test-publish-rpm-repo.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +./scripts/build/update_repo/publish-rpm.sh "oss" "v5.4.3" "grafana-testing-repo" diff --git a/scripts/build/update_repo/test-update-deb-repo.sh b/scripts/build/update_repo/test-update-deb-repo.sh index f27e9bec265..a8ffc430f02 100755 --- a/scripts/build/update_repo/test-update-deb-repo.sh +++ b/scripts/build/update_repo/test-update-deb-repo.sh @@ -2,4 +2,4 @@ GPG_PASS=${1:-} -./scripts/build/update_repo/update-deb.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-aptly-db" "grafana-testing-repo" +./scripts/build/update_repo/update-deb.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-aptly-db" diff --git a/scripts/build/update_repo/update-deb.sh b/scripts/build/update_repo/update-deb.sh index a51648c096c..3fb59e382ba 100755 --- a/scripts/build/update_repo/update-deb.sh +++ b/scripts/build/update_repo/update-deb.sh @@ -5,7 +5,6 @@ GPG_PASS="${2:-}" RELEASE_TAG="${3:-}" DIST_PATH="${4:-}" GCP_DB_BUCKET="${5:-grafana-aptly-db}" -GCP_REPO_BUCKET="${6:-grafana-repo}" REPO="grafana" @@ -54,15 +53,6 @@ rm /tmp/sign-this /tmp/sign-this.asc aptly publish update stable filesystem:repo:grafana aptly publish update beta filesystem:repo:grafana -# Update the repo and db on gcp - -gsutil -m rsync -r -d /deb-repo/db "gs://$GCP_DB_BUCKET/$RELEASE_TYPE" - -# Uploads the binaries before the metadata (to prevent 404's for debs) -gsutil -m rsync -r /deb-repo/repo/grafana/pool "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb/pool" - -gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb" - # usage: # # deb https://packages.grafana.com/oss/deb stable main diff --git a/scripts/build/update_repo/update-rpm.sh b/scripts/build/update_repo/update-rpm.sh index 304a2487b54..12eb2e08b8f 100755 --- a/scripts/build/update_repo/update-rpm.sh +++ b/scripts/build/update_repo/update-rpm.sh @@ -46,10 +46,6 @@ rm /rpm-repo/repodata/repomd.xml.asc || true pkill gpg-agent || true ./scripts/build/update_repo/sign-rpm-repo.sh "$GPG_PASS" -# Update the repo and db on gcp -gsutil -m cp /rpm-repo/*.rpm "$BUCKET" # sync binaries first to avoid cache misses -gsutil -m rsync -r -d /rpm-repo "$BUCKET" - # usage: # [grafana] # name=grafana diff --git a/scripts/build/verify_signed_packages.sh b/scripts/build/verify_signed_packages.sh index dcc260d81dd..332221218e2 100755 --- a/scripts/build/verify_signed_packages.sh +++ b/scripts/build/verify_signed_packages.sh @@ -2,6 +2,14 @@ _files=$* +if [ -z "$_files" ]; then + echo "_files (arg 1) has to be set" + exit 1 +fi + +mkdir -p ~/.rpmdb/pubkeys +curl -s https://packages.grafana.com/gpg.key > ~/.rpmdb/pubkeys/grafana.key + ALL_SIGNED=0 for file in $_files; do