Merge pull request #14924 from xlson/repo-update-improvements

Repo update stability improvements
2025-02-25 18:55:37 -06:00 · 2019-01-17 16:56:10 +01:00
parent dfe1b20f3d aafa66f644
commit dce180e8ac
8 changed files with 110 additions and 41 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -323,7 +323,7 @@ jobs:

  deploy-enterprise-master:
    docker:
-      - image: grafana/grafana-ci-deploy:1.1.0
+      - image: grafana/grafana-ci-deploy:1.2.0
    steps:
      - attach_workspace:
          at: .
@@ -346,7 +346,7 @@ jobs:

  deploy-enterprise-release:
    docker:
-    - image: grafana/grafana-ci-deploy:1.1.0
+    - image: grafana/grafana-ci-deploy:1.2.0
    steps:
      - attach_workspace:
         at: .
@@ -370,15 +370,15 @@ jobs:
          command: './scripts/build/load-signing-key.sh'
      - run:
          name: Update Debian repository
-          command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
+          command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'
      - run:
          name: Update RPM repository
-          command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
+          command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'


  deploy-master:
    docker:
-      - image: grafana/grafana-ci-deploy:1.1.0
+      - image: grafana/grafana-ci-deploy:1.2.0
    steps:
      - attach_workspace:
          at: .
@@ -408,7 +408,7 @@ jobs:

  deploy-release:
    docker:
-      - image: grafana/grafana-ci-deploy:1.1.0
+      - image: grafana/grafana-ci-deploy:1.2.0
    steps:
      - checkout
      - attach_workspace:
@@ -433,10 +433,10 @@ jobs:
          command: './scripts/build/load-signing-key.sh'
      - run:
          name: Update Debian repository
-          command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
+          command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
      - run:
          name: Update RPM repository
-          command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
+          command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'

 workflows:
  version: 2
--- a/scripts/build/build-all.sh
+++ b/scripts/build/build-all.sh
@@ -30,10 +30,12 @@ fi

 echo "Build arguments: $OPT"

+# build only amd64 for enterprise
+if echo "$EXTRA_OPTS" | grep -vq enterprise ; then
  go run build.go -goarch armv7 -cc ${CCARMV7} ${OPT} build
  go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build
  go run build.go -goos darwin -cc ${CCOSX64} ${OPT} build
-
+fi
 go run build.go -goos windows -cc ${CCWIN64} ${OPT} build

 # Do not remove CC from the linux build, its there for compatibility with Centos6
@@ -62,6 +64,9 @@ echo "Packaging"
 go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only
 #removing amd64 phantomjs bin for armv7/arm64 packages
 rm tools/phantomjs/phantomjs
+
+# build only amd64 for enterprise
+if echo "$EXTRA_OPTS" | grep -vq enterprise ; then
  go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only
  go run build.go -goos linux -pkg-arch arm64 ${OPT} package-only

@@ -71,10 +76,11 @@ else
    echo 'PhantomJS binaries for darwin missing!'
  fi
  go run build.go -goos darwin -pkg-arch amd64 ${OPT} package-only
+fi

 if [ -d '/tmp/phantomjs/windows' ]; then
  cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe
-  rm tools/phantomjs/phantomjs
+  rm tools/phantomjs/phantomjs || true
 else
    echo 'PhantomJS binaries for Windows missing!'
 fi
--- a/scripts/build/update_repo/README.md
+++ b/scripts/build/update_repo/README.md
@@ -0,0 +1,28 @@
+# Repository updates deb/rpm
+
+
+
+## Testing
+
+It's possible to test the repo updates for rpm and deb by running the test scripts within a docker container like this. Tests are being executed by using two buckets on gcp setup for testing.
+
+```bash
+docker run -ti --rm -u 0:0 grafana/grafana-ci-deploy:1.2.0 bash # 1.2.0 is the newest image at the time of writing
+# in the container:
+mkdir -p /go/src/github.com/grafana/dist
+cd /go/src/github.com/grafana
+
+#outside of container:
+cd <grafana project dir>/..
+docker cp grafana <container_name>:/go/src/github.com/grafana/.
+docker cp <gpg.key used for signing> <container_name>:/private.key
+
+#in container:
+gpg --batch --allow-secret-key-import --import /private.key
+cd dist && wget https://dl.grafana.com/oss/release/grafana_5.4.3_amd64.deb && wget https://dl.grafana.com/oss/release/grafana-5.4.3-1.x86_64.rpm && cd ..
+
+#run these scripts:
+./script/build/update_repo/test-update-deb-repo.sh <gpg key password>
+./script/build/update_repo/test-update-rpm-repo.sh <gpg key password>
+
+```
--- a/scripts/build/update_repo/init-deb-repo.sh
+++ b/scripts/build/update_repo/init-deb-repo.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# Run this if you need to recreate the debian repository for some reason
+
+# Setup environment
+cp scripts/build/update_repo/aptly.conf /etc/aptly.conf
+mkdir -p /deb-repo/db   \
+         /deb-repo/repo \
+         /deb-repo/tmp
+
+aptly repo create -distribution=stable -component=main grafana
+aptly repo create -distribution=beta -component=main beta
--- a/scripts/build/update_repo/test-update-deb-repo.sh
+++ b/scripts/build/update_repo/test-update-deb-repo.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+GPG_PASS=${1:-}
+
+./scripts/build/update_repo/update-deb.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-aptly-db" "grafana-testing-repo"
--- a/scripts/build/update_repo/test-update-rpm-repo.sh
+++ b/scripts/build/update_repo/test-update-rpm-repo.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+GPG_PASS=${1:-}
+
+./scripts/build/update_repo/update-rpm.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-repo"
--- a/scripts/build/update_repo/update-deb.sh
+++ b/scripts/build/update_repo/update-deb.sh
@@ -3,10 +3,14 @@
 RELEASE_TYPE="${1:-}"
 GPG_PASS="${2:-}"
 RELEASE_TAG="${3:-}"
+DIST_PATH="${4:-}"
+GCP_DB_BUCKET="${5:-grafana-aptly-db}"
+GCP_REPO_BUCKET="${6:-grafana-repo}"
+
 REPO="grafana"

-if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" ]; then
-    echo "Both RELEASE_TYPE (arg 1) and GPG_PASS (arg 2) has to be set"
+if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" -o -z "$DIST_PATH" ]; then
+    echo "Both RELEASE_TYPE (arg 1), GPG_PASS (arg 2) and DIST_PATH (arg 4) has to be set"
    exit 1
 fi

@@ -22,36 +26,42 @@ fi
 set -e

 # Setup environment
-cp scripts/build/update_repo/aptly.conf /etc/aptly.conf
+cp scripts/build/update_repo/aptly.conf ~/.aptly.conf
+
 mkdir -p /deb-repo/db   \
         /deb-repo/repo \
         /deb-repo/tmp

 # Download the database
-gsutil -m rsync -r "gs://grafana-aptly-db/$RELEASE_TYPE" /deb-repo/db
+gsutil -m rsync -r -d "gs://$GCP_DB_BUCKET/$RELEASE_TYPE" /deb-repo/db

 # Add the new release to the repo
-aptly publish drop grafana filesystem:repo:grafana || true
-aptly publish drop beta filesystem:repo:grafana || true
-cp ./dist/*.deb /deb-repo/tmp
+cp $DIST_PATH/*.deb /deb-repo/tmp
 rm /deb-repo/tmp/grafana_latest*.deb || true
-aptly repo add "$REPO" ./dist
+aptly repo add "$REPO" /deb-repo/tmp #adds too many packages in enterprise

 # Setup signing and sign the repo

 echo "allow-loopback-pinentry" > ~/.gnupg/gpg-agent.conf
 echo "pinentry-mode loopback" > ~/.gnupg/gpg.conf

+pkill gpg-agent || true
 touch /tmp/sign-this
+rm /tmp/sign-this.asc || true
 ./scripts/build/update_repo/unlock-gpg-key.sh "$GPG_PASS"
 rm /tmp/sign-this /tmp/sign-this.asc

-aptly publish repo grafana filesystem:repo:grafana
-aptly publish repo beta filesystem:repo:grafana
+aptly publish update stable filesystem:repo:grafana
+aptly publish update beta filesystem:repo:grafana

 # Update the repo and db on gcp
-gsutil -m rsync -r -d /deb-repo/db "gs://grafana-aptly-db/$RELEASE_TYPE"
-gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://grafana-repo/$RELEASE_TYPE/deb"
+
+gsutil -m rsync -r -d /deb-repo/db "gs://$GCP_DB_BUCKET/$RELEASE_TYPE"
+
+# Uploads the binaries before the metadata (to prevent 404's for debs)
+gsutil -m rsync -r /deb-repo/repo/grafana/pool "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb/pool"
+
+gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb"

 # usage:
 # 
--- a/scripts/build/update_repo/update-rpm.sh
+++ b/scripts/build/update_repo/update-rpm.sh
@@ -2,12 +2,14 @@

 RELEASE_TYPE="${1:-}"
 GPG_PASS="${2:-}"
-
 RELEASE_TAG="${3:-}"
+DIST_PATH="${4:-}"
+GCP_REPO_BUCKET="${5:-grafana-repo}"
+
 REPO="rpm"

-if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" ]; then
-    echo "Both RELEASE_TYPE (arg 1) and GPG_PASS (arg 2) has to be set"
+if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" -o -z "$DIST_PATH" ]; then
+    echo "Both RELEASE_TYPE (arg 1), GPG_PASS (arg 2) and DIST_PATH (arg 4) has to be set"
    exit 1
 fi

@@ -23,17 +25,17 @@ fi
 set -e

 # Setup environment
-BUCKET="gs://grafana-repo/$RELEASE_TYPE/$REPO"
+BUCKET="gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/$REPO"
+
 mkdir -p /rpm-repo

 # Download the database
 gsutil -m rsync -r "$BUCKET" /rpm-repo

 # Add the new release to the repo
-cp ./dist/*.rpm /rpm-repo
+cp $DIST_PATH/*.rpm /rpm-repo # adds to many files for enterprise
 rm /rpm-repo/grafana-latest-1*.rpm || true
-cd /rpm-repo
-createrepo .
+createrepo /rpm-repo

 # Setup signing and sign the repo

@@ -45,6 +47,7 @@ pkill gpg-agent || true
 ./scripts/build/update_repo/sign-rpm-repo.sh "$GPG_PASS"

 # Update the repo and db on gcp
+gsutil -m cp /rpm-repo/*.rpm "$BUCKET" # sync binaries first to avoid cache misses
 gsutil -m rsync -r -d /rpm-repo "$BUCKET"

 # usage: