From c9d2b354f73835088b12038475fa474813e6e668 Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Fri, 19 Jul 2024 14:08:25 -0500 Subject: [PATCH] Further adventures in nightly release automation Pushing this up in order to end-to-end test automated nightly releases. Please enter the commit message for your changes. Lines starting --- .gitlab-ci.yml | 23 +++++++----- INSTALL.md | 36 +++++++++++++++++- .../distribute-nightly-packages.sh | 28 +++++--------- .../distribute-stable-packages.sh | 37 ++++++++++--------- .../rpm-repo-building/Dockerfile | 10 +++++ .../rpm-repo-building/repobuild.sh | 33 +++++++++++++++++ 6 files changed, 121 insertions(+), 46 deletions(-) create mode 100644 scripts/cicd/build-orchestration/rpm-repo-building/Dockerfile create mode 100644 scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 91017ff7..85316d75 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -106,7 +106,7 @@ package_amd64_deb: - build-amd64-deb script: - earthly bootstrap - - earthly +package-linux-amd64-deb --IS_NIGHTLY=$IS_NIGHTLY + - earthly +package-linux-amd64-deb - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -119,7 +119,7 @@ package_arm64_deb: - build-arm64-deb script: - earthly bootstrap - - earthly +package-linux-arm64-deb --IS_NIGHTLY=$IS_NIGHTLY + - earthly +package-linux-arm64-deb - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -132,7 +132,7 @@ package_amd64_rpm: - build-amd64-rpm script: - earthly bootstrap - - earthly +package-linux-amd64-rpm --IS_NIGHTLY=$IS_NIGHTLY + - earthly +package-linux-amd64-rpm - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -168,8 +168,11 @@ build_repositories: tags: - build-orchestration variables: - SECURE_FILES_DOWNLOAD_PATH: '$HOME' + SECURE_FILES_DOWNLOAD_PATH: '/home/gitlab-runner/' script: + - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash + - cp scripts/cicd/build-orchestration/rpm-repo-building/Dockerfile ~/rpm-build-container + - cp scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh ~/rpm-build-container - cp scripts/cicd/build-orchestration/generate-stable-release.sh ~ - bash scripts/cicd/build-orchestration/distribute-stable-packages.sh rules: @@ -280,8 +283,9 @@ dryrun_build_repositories: tags: - build-orchestration variables: - SECURE_FILES_DOWNLOAD_PATH: './' + SECURE_FILES_DOWNLOAD_PATH: '/home/gitlab-runner/' script: + - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash - cp scripts/cicd/build-orchestration/generate-release.sh ~ - bash scripts/cicd/build-orchestration/distribute-packages.sh rules: @@ -333,7 +337,7 @@ nightly_package_amd64_deb: - build-amd64-deb script: - earthly bootstrap - - earthly +package-linux-amd64-deb #--IS_NIGHTLY="$IS_NIGHTLY" + - earthly +package-linux-amd64-deb --IS_NIGHTLY="$IS_NIGHTLY" - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh rules: - if: '$IS_NIGHTLY == true' @@ -346,7 +350,7 @@ nightly_package_arm64_deb: - build-arm64-deb script: - earthly bootstrap - - earthly +package-linux-arm64-deb #--IS_NIGHTLY="$IS_NIGHTLY" + - earthly +package-linux-arm64-deb --IS_NIGHTLY="$IS_NIGHTLY" - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh rules: - if: '$IS_NIGHTLY == true' @@ -359,7 +363,7 @@ nightly_package_amd64_rpm: - build-amd64-rpm script: - earthly bootstrap - - earthly +package-linux-amd64-rpm #--IS_NIGHTLY="$IS_NIGHTLY" + - earthly +package-linux-amd64-rpm --IS_NIGHTLY="$IS_NIGHTLY" - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh rules: - if: '$IS_NIGHTLY == true' @@ -369,8 +373,9 @@ nightly_build_repositories: tags: - build-orchestration variables: - SECURE_FILES_DOWNLOAD_PATH: '$HOME' + SECURE_FILES_DOWNLOAD_PATH: '/home/gitlab-runner/' script: + - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash - cp scripts/cicd/build-orchestration/generate-nightly-release.sh ~ - bash scripts/cicd/build-orchestration/distribute-nightly-packages.sh rules: diff --git a/INSTALL.md b/INSTALL.md index a1b38b29..c5eda9da 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -39,6 +39,7 @@ dpkg --print-architecture ```shell echo "deb [arch=arm64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null ``` + - For *NIGHTLY* (bleeding edge) releases - **AMD64** based systems run this command: @@ -69,6 +70,18 @@ sudo apt update sudo apt install veilid-server veilid-cli ``` +***Step 6***: Start veilid-server.service + +Go to [Start headless node](#start-headless-node) + +***Step 7***: View Node Activity + +Invoke the Veilid CLI utility. + +```shell +veilid-cli +``` + ### RPM-based Follow the steps here to add the repo to @@ -77,15 +90,36 @@ and install Veilid. **Step 1**: Add Veilid to your list of available software. +- For *STABLE* releases + ```shell -sudo yum-config-manager --add-repo https://packages.veilid.net/rpm/veilid-rpm-repo.repo +sudo dnf config-manager --add-repo https://packages.veilid.net/rpm/stable/x86_64/veilid-stable-x86_64-rpm.repo ``` + +- For *NIGHTLY* (bleeding edge) releases + +```shell +sudo dnf config-manager --add-repo https://packages.veilid.net/rpm/nightly/x86_64/veilid-nightly-x86_64-rpm.repo +``` + **Step 2**: Install Veilid. ```shell sudo dnf install veilid-server veilid-cli ``` +***Step 3***: Start veilid-server.service + +Go to [Start headless node](#start-headless-node) + +***Step 4***: View Node Activity + +Invoke the Veilid CLI utility. + +```shell +veilid-cli +``` + ### macOS Veilid is available [via Homebrew](https://formulae.brew.sh/formula/veilid). diff --git a/scripts/cicd/build-orchestration/distribute-nightly-packages.sh b/scripts/cicd/build-orchestration/distribute-nightly-packages.sh index 4ffc5d65..68a72f69 100644 --- a/scripts/cicd/build-orchestration/distribute-nightly-packages.sh +++ b/scripts/cicd/build-orchestration/distribute-nightly-packages.sh @@ -4,14 +4,9 @@ echo "Setting up the workspace" # Rsync active repo to local workspace rsync --archive gitlab-runner@10.116.0.3:/srv/ $HOME/srv/ -# Ensure repo directory structure exists -mkdir -p $HOME/srv/{gpg,rpm/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64},apt/{dists/{stable/main/{binary-amd64,binary-arm64},nightly/main/{binary-amd64,binary-arm64}},pool/{stable/main,nightly/main}}} # Delete previous versions of packages rm -rf $HOME/srv/apt/pool/nightly/main/*.deb -rm -rf $HOME/srv/rpm/{nightly/x86_64/*,nightly/x86_64/*} -# Ensure RPM workspace setup -mkdir -p $HOME/rpm-build-container/mount/repo/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64} -rm -rf $HOME/rpm-builder/mount/repo/{nightly/x86_64/*,nightly/x86_64/*} +rm -rf $HOME/srv/rpm/nightly/x86_64/* # Setup crypto export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)" @@ -44,11 +39,11 @@ tar -xf amd64-rpms.tar echo "Copying rpms to container workspace" cp *x86_64.rpm $HOME/rpm-build-container/mount/repo/nightly/x86_64 echo "Copying signing material to container workspace" -cp -R $GNUPGHOME $HOME/rpm-build-container/mount/keystore +cp -R $GNUPGHOME/* $HOME/rpm-build-container/mount/keystore echo "Executing container actions" -docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8 +docker run --rm -d -it -e IS_NIGHTLY=$IS_NIGHTLY --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v12 sleep 2 -cp -R $HOME/rpm-build-container/mount/repo/nightly/* $HOME/srv/rpm/nightly +cp -R $HOME/rpm-build-container/mount/repo/nightly/x86_64/* $HOME/srv/rpm/nightly/x86_64 cd $HOME/srv/rpm/nightly/x86_64 echo "Signing the rpm repository" gpg --default-key admin@veilid.org --detach-sign --armor $HOME/srv/rpm/nightly/x86_64/repodata/repomd.xml @@ -58,22 +53,19 @@ name=Veilid Nightly x86_64 RPM Repo baseurl=https://packages.veilid.net/rpm/nightly/x86_64 enabled=1 gpgcheck=1 -gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/nightly/x86_64/veilid-rpm-repo.repo +gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/nightly/x86_64/veilid-nightly-x86_64-rpm.repo # Tar the repo data and transfer to the repo server echo "Moving the repo scaffold to the repo server" cd $HOME -rsync --archive $HOME/srv/ gitlab-runner@10.116.0.3:/srv/ -# tar -cf $HOME/repo.tar srv -# scp -i $HOME/.ssh/id_ed25519 $HOME/repo.tar gitlab-runner@10.116.0.3:~ +rsync --archive $HOME/srv/* gitlab-runner@10.116.0.3:/srv # Cleanup echo "Cleaning up the workspace" rm -rf $GNUPGHOME -# rm $HOME/repo.tar +rm $HOME/*.tar rm $HOME/*.deb rm $HOME/*.rpm -rm -rf $HOME/rpm-build-container/mount/keystore -# rm rpm-build-container/mount/repo/*.rpm -# rm -rf rpm-build-container/mount/repo/repodata/* -echo "Process complete" \ No newline at end of file +rm -rf $HOME/rpm-build-container/mount/keystore/* +rm -rf $HOME/rpm-build-container/mount/repo/nightly/x86_64/* +echo "Nightly packages distribution process complete" \ No newline at end of file diff --git a/scripts/cicd/build-orchestration/distribute-stable-packages.sh b/scripts/cicd/build-orchestration/distribute-stable-packages.sh index 488f08f2..ea5fddc8 100644 --- a/scripts/cicd/build-orchestration/distribute-stable-packages.sh +++ b/scripts/cicd/build-orchestration/distribute-stable-packages.sh @@ -3,15 +3,10 @@ # Clean and reset the workspaces echo "Setting up the workspace" # Rsync active repo to local workspace -rsync --archive gitlab-runner@10.116.0.3:/srv $HOME -# Ensure repo directory structure exists -mkdir -p $HOME/srv/{gpg,rpm/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64},apt/{dists/{stable/main/{binary-amd64,binary-arm64},nightly/main/{binary-amd64,binary-arm64}},pool/{stable/main,nightly/main}}} +rsync --archive gitlab-runner@10.116.0.3:/srv/ $HOME/srv/ # Delete previous versions of packages rm -rf $HOME/srv/apt/pool/stable/main/*.deb -rm -rf $HOME/srv/rpm/{stable/x86_64/*,stable/x86_64/*} -# Ensure RPM workspace setup -mkdir -p $HOME/rpm-build-container/mount/repo/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64} -rm -rf $HOME/rpm-builder/mount/repo/{stable/x86_64/*,stable/x86_64/*} +rm -rf $HOME/srv/rpm/stable/x86_64/* # Setup crypto export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)" @@ -32,7 +27,7 @@ cat dists/stable/main/binary-amd64/Packages | gzip -9 > dists/stable/main/binary cat dists/stable/main/binary-arm64/Packages | gzip -9 > dists/stable/main/binary-arm64/Packages.gz echo "Creating Release file" cd $HOME/srv/apt/dists/stable -~/generate-release.sh > Release +$HOME/generate-release.sh > Release echo "Signing Release file and creating InRelease" cat $HOME/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs > $HOME/srv/apt/dists/stable/Release.gpg cat $HOME/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs --clearsign > $HOME/srv/apt/dists/stable/InRelease @@ -46,34 +41,40 @@ cp *x86_64.rpm $HOME/rpm-build-container/mount/repo/stable/x86_64 echo "Copying signing material to container workspace" cp -R $GNUPGHOME $HOME/rpm-build-container/mount/keystore echo "Executing container actions" -docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8 +docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v12 sleep 2 -cp -R $HOME/rpm-build-container/mount/repo/stable ~/srv/rpm/stable +cp -R $HOME/rpm-build-container/mount/repo/stable/x86_64/* $HOME/srv/rpm/stable/x86_64/ cd $HOME/srv/rpm/stable/x86_64 echo "Signing the rpm repository" gpg --default-key admin@veilid.org --detach-sign --armor $HOME/srv/rpm/stable/x86_64/repodata/repomd.xml +# Generate .repo file for stable x86_64 releases echo "[veilid-stable-x86_64-rpm-repo] name=Veilid Stable x86_64 RPM Repo baseurl=https://packages.veilid.net/rpm/stable/x86_64 enabled=1 gpgcheck=1 -gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/stable/x86_64/veilid-rpm-repo.repo +gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/stable/x86_64/veilid-stable-x86_64-rpm.repo + +# Generate .repo file for stable arm64 releases -- to be added +# echo "[veilid-stable-arm64-rpm-repo] +# name=Veilid Stable x86_64 RPM Repo +# baseurl=https://packages.veilid.net/rpm/stable/arm64 +# enabled=1 +# gpgcheck=1 +# gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/stable/x86_64/veilid-stable-arm64-rpm.repo # Tar the repo data and transfer to the repo server echo "Moving the repo scaffold to the repo server" cd $HOME rsync --archive $HOME/srv/* gitlab-runner@10.116.0.3:/srv -# tar -cf $HOME/repo.tar srv -# scp -i $HOME/.ssh/id_ed25519 $HOME/repo.tar gitlab-runner@10.116.0.3:~ # Cleanup echo "Cleaning up the workspace" rm -rf $GNUPGHOME -# rm $HOME/repo.tar +rm $HOME/*.tar rm $HOME/*.deb rm $HOME/*.rpm -rm -rf $HOME/rpm-build-container/mount/keystore -# rm rpm-build-container/mount/repo/*.rpm -# rm -rf rpm-build-container/mount/repo/repodata/* -echo "Process complete" \ No newline at end of file +rm -rf $HOME/rpm-build-container/mount/keystore/* +rm -rf $HOME/rpm-build-container/mount/repo/nightly/x86_64/* +echo "Stable packages distribution process complete" \ No newline at end of file diff --git a/scripts/cicd/build-orchestration/rpm-repo-building/Dockerfile b/scripts/cicd/build-orchestration/rpm-repo-building/Dockerfile new file mode 100644 index 00000000..72ff0b88 --- /dev/null +++ b/scripts/cicd/build-orchestration/rpm-repo-building/Dockerfile @@ -0,0 +1,10 @@ +FROM rockylinux:8 +LABEL author=admin@veilid.org +ENV IS_NIGHTLY false + +COPY repobuild.sh repobuild.sh +RUN dnf install -y createrepo rpm-build rpm-sign yum-utils +RUN chmod +x repobuild.sh + + +ENTRYPOINT ["./repobuild.sh"] \ No newline at end of file diff --git a/scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh b/scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh new file mode 100644 index 00000000..112115ab --- /dev/null +++ b/scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +echo "==========Log start $(date +%F_%T)==========" &>> /mount/logfile + +echo "setting GNUPGHOME $(date +%F_%T)" &>> /mount/logfile +export GNUPGHOME=/mount/keystore + +echo "Adding key to rpm utility $(date +%F_%T)" &>> /mount/logfile +echo "%_signature gpg +%_gpg_name 516C76D1E372C5C96EE54E22AE0E059BC64CD052" > /root/.rpmmacros + +if [ "$IS_NIGHTLY" = "true" ] +then + echo "Taking nightly actions branch $(date +%F_%T)" &>> /mount/logfile + cd /mount/repo/nightly/x86_64 +elif [ "$IS_NIGHTLY" = "false" ] +then + echo "Taking stable branch actions $(date +%F_%T)" &>> /mount/logfile + cd /mount/repo/stable/x86_64 +else + echo $IS_NIGHTLY "is not a valid state to determine if the build is STABLE or NIGHTLY (RPM RepoBuild)" &>> /mount/logfile +fi + +echo "Signing RPMs $(date +%F_%T)" &>> /mount/logfile +rpm --addsign *.rpm &>> /mount/logfile + +echo "Creating repo metadata $(date +%F_%T)" &>> /mount/logfile +createrepo . &>> /mount/logfile + +echo "Setting file ownership $(date +%F_%T)" &>> /mount/logfile +chown -R 1000:1000 /mount + +echo "==========RPM Packaging Process complete $(date +%F_%T)==========" &>> /mount/logfile \ No newline at end of file