IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2024-11-24 09:50:29 -06:00
Code Issues Packages Projects Releases Wiki Activity

Build: Split up task in the CI pipeline to ease running outside circleci (#18861)

Browse Source 
* build: make sign rpm packages not depend on checking out private key

* build: move commands from circleci config into verify signed packages script

* build: split update and publish of deb and rpm into two scripts

* use files argument for sign and verify packages

* validate files argument for sign and verify packages

* update test publish of deb/rpm readme
This commit is contained in:
Marcus Efraimsson 2019-09-23 12:00:18 +02:00 committed by GitHub
parent 8f9c487c44
commit 4386604751
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 164 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
.circleci/config.yml
View File

@ -214,15 +214,15 @@ jobs:
- run:
name: build and package grafana
command: './scripts/build/build-all.sh'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: sign packages
command: './scripts/build/sign_packages.sh'
command: './scripts/build/sign_packages.sh dist/*.rpm'
- run:
name: verify signed packages
command: |
mkdir -p ~/.rpmdb/pubkeys
curl -s https://packages.grafana.com/gpg.key > ~/.rpmdb/pubkeys/grafana.key
./scripts/build/verify_signed_packages.sh dist/*.rpm
command: './scripts/build/verify_signed_packages.sh dist/*.rpm'
- run:
name: sha-sum packages
command: 'go run build.go sha-dist'
@ -249,9 +249,12 @@ jobs:
- run:
name: build and package grafana
command: './scripts/build/build.sh'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: sign packages
command: './scripts/build/sign_packages.sh'
command: './scripts/build/sign_packages.sh dist/*.rpm'
- run:
name: sha-sum packages
command: 'go run build.go sha-dist'
@ -360,9 +363,12 @@ jobs:
- run:
name: package grafana
command: './scripts/build/build.sh --fast --package-only'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: sign packages
command: './scripts/build/sign_packages.sh'
command: './scripts/build/sign_packages.sh dist/*.rpm'
- run:
name: sha-sum packages
command: 'go run build.go sha-dist'
@ -435,9 +441,12 @@ jobs:
- run:
name: build and package enterprise
command: './scripts/build/build.sh -enterprise'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: sign packages
command: './scripts/build/sign_packages.sh'
command: './scripts/build/sign_packages.sh dist/*.rpm'
- run:
name: sha-sum packages
command: 'go run build.go sha-dist'
@ -476,15 +485,15 @@ jobs:
- run:
name: build and package grafana
command: './scripts/build/build-all.sh -enterprise'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: sign packages
command: './scripts/build/sign_packages.sh'
command: './scripts/build/sign_packages.sh dist/*.rpm'
- run:
name: verify signed packages
command: |
mkdir -p ~/.rpmdb/pubkeys
curl -s https://packages.grafana.com/gpg.key > ~/.rpmdb/pubkeys/grafana.key
./scripts/build/verify_signed_packages.sh dist/*.rpm
command: './scripts/build/verify_signed_packages.sh dist/*.rpm'
- run:
name: sha-sum packages
command: 'go run build.go sha-dist'
@ -537,15 +546,24 @@ jobs:
- run:
name: Deploy to Grafana.com
command: './scripts/build/publish.sh --enterprise'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: Load GPG private key
command: './scripts/build/load-signing-key.sh'
command: './scripts/build/update_repo/load-signing-key.sh'
- run:
name: Update Debian repository
command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'
- run:
name: Publish Debian repository
command: './scripts/build/update_repo/publish-deb.sh "enterprise"'
- run:
name: Update RPM repository
command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'
- run:
name: Publish RPM repository
command: './scripts/build/update_repo/publish-rpm.sh "enterprise" "$CIRCLE_TAG"'
deploy-master:
@ -591,15 +609,24 @@ jobs:
- run:
name: Deploy to Grafana.com
command: './scripts/build/publish.sh'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: Load GPG private key
command: './scripts/build/load-signing-key.sh'
command: './scripts/build/update_repo/load-signing-key.sh'
- run:
name: Update Debian repository
command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
- run:
name: Publish Debian repository
command: './scripts/build/update_repo/publish-deb.sh "oss"'
- run:
name: Update RPM repository
command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
- run:
name: Publish RPM repository
command: './scripts/build/update_repo/publish-rpm.sh "oss" "$CIRCLE_TAG"'
build-oss-msi:
docker:

7
scripts/build/load-signing-key.sh
View File

@ -1,7 +0,0 @@
#!/usr/bin/env bash
set -e
git clone git@github.com:torkelo/private.git ~/private-repo
gpg --batch --allow-secret-key-import --import ~/private-repo/signing/private.key
pkill gpg-agent

6
scripts/build/prepare_signing_key.sh Executable file
View File

@ -0,0 +1,6 @@
#!/bin/bash
set -e
git clone git@github.com:torkelo/private.git ~/private-repo
cp ~/private-repo/signing/private.key /private.key

18
scripts/build/sign_packages.sh
View File

@ -1,12 +1,24 @@
#!/bin/bash
git clone git@github.com:torkelo/private.git ~/private-repo
set -e
gpg --allow-secret-key-import --import ~/private-repo/signing/private.key
_files=$*
if [ -z "$_files" ]; then
echo "_files (arg 1) has to be set"
exit 1
fi
if [ -z "$GPG_KEY_PASSWORD" ]; then
echo "GPG_KEY_PASSWORD has to be set"
exit 1
fi
gpg --allow-secret-key-import --import /private.key
cp ./scripts/build/rpmmacros ~/.rpmmacros
for package in dist/*.rpm; do
for package in $_files; do
[ -e "$package" ] || continue
./scripts/build/sign_expect "$GPG_KEY_PASSWORD" "$package"
done

17
scripts/build/update_repo/README.md
View File

@ -7,22 +7,23 @@
It's possible to test the repo updates for rpm and deb by running the test scripts within a docker container like this. Tests are being executed by using two buckets on gcp setup for testing.
```bash
docker run -ti --rm -u 0:0 grafana/grafana-ci-deploy:1.2.0 bash # 1.2.0 is the newest image at the time of writing
docker run -ti --rm -u 0:0 grafana/grafana-ci-deploy:1.2.2 bash # 1.2.2 is the newest image at the time of writing
# in the container:
mkdir -p /go/src/github.com/grafana/dist
cd /go/src/github.com/grafana
mkdir -p /dist
#outside of container:
cd <grafana project dir>/..
docker cp grafana <container_name>:/go/src/github.com/grafana/.
docker cp grafana <container_name>:/
docker cp <gpg.key used for signing> <container_name>:/private.key
#in container:
gpg --batch --allow-secret-key-import --import /private.key
./scripts/build/update_repo/load-signing-key.sh
cd dist && wget https://dl.grafana.com/oss/release/grafana_5.4.3_amd64.deb && wget https://dl.grafana.com/oss/release/grafana-5.4.3-1.x86_64.rpm && cd ..
#run these scripts:
./script/build/update_repo/test-update-deb-repo.sh <gpg key password>
./script/build/update_repo/test-update-rpm-repo.sh <gpg key password>
#run these scripts to update local deb and rpm repos and publish them:
./scripts/build/update_repo/test-update-deb-repo.sh <gpg key password>
./scripts/build/update_repo/test-publish-deb-repo.sh
./scripts/build/update_repo/test-update-rpm-repo.sh <gpg key password>
./scripts/build/update_repo/test-publish-rpm-repo.sh
```

6
scripts/build/update_repo/load-signing-key.sh Executable file
View File

@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -e
gpg --batch --allow-secret-key-import --import /private.key
pkill gpg-agent

30
scripts/build/update_repo/publish-deb.sh Executable file
View File

@ -0,0 +1,30 @@
#!/usr/bin/env bash
RELEASE_TYPE="${1:-}"
GCP_DB_BUCKET="${2:-grafana-aptly-db}"
GCP_REPO_BUCKET="${3:-grafana-repo}"
if [ -z "$RELEASE_TYPE" ]; then
echo "RELEASE_TYPE (arg 1) has to be set"
exit 1
fi
if [[ "$RELEASE_TYPE" != "oss" && "$RELEASE_TYPE" != "enterprise" ]]; then
echo "RELEASE_TYPE (arg 1) must be either oss or enterprise."
exit 1
fi
set -e
# Update the repo and db on gcp
gsutil -m rsync -r -d /deb-repo/db "gs://$GCP_DB_BUCKET/$RELEASE_TYPE"
# Uploads the binaries before the metadata (to prevent 404's for debs)
gsutil -m rsync -r /deb-repo/repo/grafana/pool "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb/pool"
gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb"
# usage:
#
# deb https://packages.grafana.com/oss/deb stable main

41
scripts/build/update_repo/publish-rpm.sh Executable file
View File

@ -0,0 +1,41 @@
#!/usr/bin/env bash
RELEASE_TYPE="${1:-}"
RELEASE_TAG="${2:-}"
GCP_REPO_BUCKET="${3:-grafana-repo}"
REPO="rpm"
if [ -z "$RELEASE_TYPE" ]; then
echo "RELEASE_TYPE (arg 1) has to be set"
exit 1
fi
if [[ "$RELEASE_TYPE" != "oss" && "$RELEASE_TYPE" != "enterprise" ]]; then
echo "RELEASE_TYPE (arg 1) must be either oss or enterprise."
exit 1
fi
if echo "$RELEASE_TAG" | grep -q "beta"; then
REPO="rpm-beta"
fi
set -e
# Setup environment
BUCKET="gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/$REPO"
# Update the repo and db on gcp
gsutil -m cp /rpm-repo/*.rpm "$BUCKET" # sync binaries first to avoid cache misses
gsutil -m rsync -r -d /rpm-repo "$BUCKET"
# usage:
# [grafana]
# name=grafana
# baseurl=https://packages.grafana.com/oss/rpm
# repo_gpgcheck=1
# enabled=1
# gpgcheck=1
# gpgkey=https://packages.grafana.com/gpg.key
# sslverify=1
# sslcacert=/etc/pki/tls/certs/ca-bundle.crt

3
scripts/build/update_repo/test-publish-deb-repo.sh Executable file
View File

@ -0,0 +1,3 @@
#!/usr/bin/env bash
./scripts/build/update_repo/publish-deb.sh "oss" "grafana-testing-aptly-db" "grafana-testing-repo"

3
scripts/build/update_repo/test-publish-rpm-repo.sh Executable file
View File

@ -0,0 +1,3 @@
#!/usr/bin/env bash
./scripts/build/update_repo/publish-rpm.sh "oss" "v5.4.3" "grafana-testing-repo"

2
scripts/build/update_repo/test-update-deb-repo.sh
View File

@ -2,4 +2,4 @@
GPG_PASS=${1:-}
./scripts/build/update_repo/update-deb.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-aptly-db" "grafana-testing-repo"
./scripts/build/update_repo/update-deb.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-aptly-db"

10
scripts/build/update_repo/update-deb.sh
View File

@ -5,7 +5,6 @@ GPG_PASS="${2:-}"
RELEASE_TAG="${3:-}"
DIST_PATH="${4:-}"
GCP_DB_BUCKET="${5:-grafana-aptly-db}"
GCP_REPO_BUCKET="${6:-grafana-repo}"
REPO="grafana"
@ -54,15 +53,6 @@ rm /tmp/sign-this /tmp/sign-this.asc
aptly publish update stable filesystem:repo:grafana
aptly publish update beta filesystem:repo:grafana
# Update the repo and db on gcp
gsutil -m rsync -r -d /deb-repo/db "gs://$GCP_DB_BUCKET/$RELEASE_TYPE"
# Uploads the binaries before the metadata (to prevent 404's for debs)
gsutil -m rsync -r /deb-repo/repo/grafana/pool "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb/pool"
gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb"
# usage:
#
# deb https://packages.grafana.com/oss/deb stable main

4
scripts/build/update_repo/update-rpm.sh
View File

@ -46,10 +46,6 @@ rm /rpm-repo/repodata/repomd.xml.asc || true
pkill gpg-agent || true
./scripts/build/update_repo/sign-rpm-repo.sh "$GPG_PASS"
# Update the repo and db on gcp
gsutil -m cp /rpm-repo/*.rpm "$BUCKET" # sync binaries first to avoid cache misses
gsutil -m rsync -r -d /rpm-repo "$BUCKET"
# usage:
# [grafana]
# name=grafana

8
scripts/build/verify_signed_packages.sh
View File

@ -2,6 +2,14 @@
_files=$*
if [ -z "$_files" ]; then
echo "_files (arg 1) has to be set"
exit 1
fi
mkdir -p ~/.rpmdb/pubkeys
curl -s https://packages.grafana.com/gpg.key > ~/.rpmdb/pubkeys/grafana.key
ALL_SIGNED=0
for file in $_files; do