Adding automatic nightly release builds

- Restructured linux repo directory structure to include Stable and Nightly releases - Reworked Earthfile to pass IS_NIGHTLY boolean to package build scripts for use in file names - Added nightly pipeline to CICD config yaml. Pipeline is triggered by IS_NIGHTLY variable - Two distribution scripts now exist, one for Stable and one for Nightly - Reworked RPM build spec files to create appropriate file names - Reworked debian release generation scripts - RPM repo directory structure is prepped for arm64 builds - Switched from scp to rsync for moving the completed repo from ochestration to repo server - Created schedule in Gitlab UI to trigger nightly release at 2300 Central Time
2024-11-21 16:37:39 -06:00 · 2024-07-14 17:22:50 -05:00 · 2024-07-14 17:22:50 -05:00 · fc7debba3f
commit fc7debba3f
parent 8c297acdee
14 changed files with 323 additions and 92 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -106,7 +106,7 @@ package_amd64_deb:
    - build-amd64-deb
  script:
    - earthly bootstrap
-    - earthly +package-linux-amd64-deb
+    - earthly +package-linux-amd64-deb --IS_NIGHTLY=$IS_NIGHTLY
    - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
  rules:
    - if: '$CI_COMMIT_TAG =~ /v\d.+/'
@ -119,7 +119,7 @@ package_arm64_deb:
    - build-arm64-deb
  script:
    - earthly bootstrap
-    - earthly +package-linux-arm64-deb
+    - earthly +package-linux-arm64-deb --IS_NIGHTLY=$IS_NIGHTLY
    - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
  rules:
    - if: '$CI_COMMIT_TAG =~ /v\d.+/'
@ -132,7 +132,7 @@ package_amd64_rpm:
    - build-amd64-rpm
  script:
    - earthly bootstrap
-    - earthly +package-linux-amd64-rpm
+    - earthly +package-linux-amd64-rpm --IS_NIGHTLY=$IS_NIGHTLY
    - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
  rules:
    - if: '$CI_COMMIT_TAG =~ /v\d.+/'
@ -168,9 +168,9 @@ build_repositories:
  tags:
    - build-orchestration
  variables:
-    SECURE_FILES_DOWNLOAD_PATH: './'
+    SECURE_FILES_DOWNLOAD_PATH: '$HOME'
  script:
-    - cp scripts/cicd/build-orchestration/generate-release.sh ~
+    - cp scripts/cicd/build-orchestration/generate-stable-release.sh ~
    - bash scripts/cicd/build-orchestration/distribute-packages.sh
  rules:
    - if: '$CI_COMMIT_TAG =~ /v\d.+/'
@ -311,4 +311,80 @@ dryrun_delete_build_machines:
  rules:
    - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/

-# Just a note so I can try dry run
+
+# Nightly build pipeline
+
+nightly_create_build_machines:
+  stage: build_packages
+  tags:
+    - build-orchestration
+  script:
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm
+  rules:
+    - if: '$IS_NIGHTLY == true'
+  
+nightly_package_amd64_deb:
+  stage: build_packages
+  needs:
+    - nightly_create_build_machines
+  tags:
+    - build-amd64-deb
+  script:
+    - earthly bootstrap
+    - earthly +package-linux-amd64-deb --IS_NIGHTLY=$IS_NIGHTLY
+    - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
+  rules:
+    - if: '$IS_NIGHTLY == true'
+
+nightly_package_arm64_deb:
+  stage: build_packages
+  needs:
+    - nightly_create_build_machines
+  tags:
+    - build-arm64-deb
+  script:
+    - earthly bootstrap
+    - earthly +package-linux-arm64-deb --IS_NIGHTLY=$IS_NIGHTLY
+    - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
+  rules:
+    - if: '$IS_NIGHTLY == true'
+
+nightly_package_amd64_rpm:
+  stage: build_packages
+  needs:
+    - nightly_create_build_machines
+  tags:
+    - build-amd64-rpm
+  script:
+    - earthly bootstrap
+    - earthly +package-linux-amd64-rpm --IS_NIGHTLY=$IS_NIGHTLY
+    - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
+  rules:
+    - if: '$IS_NIGHTLY == true'
+
+nightly_build_repositories:
+  stage: distribute
+  tags:
+    - build-orchestration
+  variables:
+    SECURE_FILES_DOWNLOAD_PATH: '$HOME'
+  script:
+    - cp scripts/cicd/build-orchestration/generate-nightly-release.sh ~
+    - bash scripts/cicd/build-orchestration/distribute-packages.sh
+  rules:
+    - if: '$IS_NIGHTLY == true'
+
+nightly_delete_build_machines:
+  stage: distribute
+  needs:
+    - deploy_repos
+  tags:
+    - build-orchestration
+  script:
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
+  rules:
+    - if: '$IS_NIGHTLY == true'
--- a/16
+++ b/16
@ -212,10 +212,10 @@ package-linux-amd64-deb:
    #################################
    COPY --dir package /veilid
    # veilid-server
-    RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh amd64 x86_64-unknown-linux-gnu
+    RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh amd64 x86_64-unknown-linux-gnu $IS_NIGHTLY
    SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/
    # veilid-cli
-    RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh amd64 x86_64-unknown-linux-gnu
+    RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh amd64 x86_64-unknown-linux-gnu $IS_NIGHTLY
    # save artifacts
    SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/

@ -231,10 +231,10 @@ package-linux-amd64-rpm:
    COPY +build-linux-amd64/x86_64-unknown-linux-gnu /veilid/target/x86_64-unknown-linux-gnu
    RUN mkdir -p /rpm-work-dir/veilid-server
    # veilid-server
-    RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh x86_64 x86_64-unknown-linux-gnu
+    RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh x86_64 x86_64-unknown-linux-gnu $IS_NIGHTLY
    #SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/x86_64/*.rpm AS LOCAL ./target/packages/
    # veilid-cli
-    RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh x86_64 x86_64-unknown-linux-gnu
+    RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh x86_64 x86_64-unknown-linux-gnu $IS_NIGHTLY
    # save artifacts
    SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/x86_64/*.rpm AS LOCAL ./target/packages/
    
@ -245,10 +245,10 @@ package-linux-arm64-deb:
    #################################
    COPY --dir package /veilid
    # veilid-server
-    RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh arm64 aarch64-unknown-linux-gnu
+    RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh arm64 aarch64-unknown-linux-gnu $IS_NIGHTLY
    SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/
    # veilid-cli
-    RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh arm64 aarch64-unknown-linux-gnu
+    RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh arm64 aarch64-unknown-linux-gnu $IS_NIGHTLY
    # save artifacts
    SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/

@ -264,10 +264,10 @@ package-linux-arm64-rpm:
    COPY +build-linux-arm64/aarch64-unknown-linux-gnu /veilid/target/aarch64-unknown-linux-gnu
    RUN mkdir -p /rpm-work-dir/veilid-server
    # veilid-server
-    RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh aarch64 aarch64-unknown-linux-gnu
+    RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh aarch64 aarch64-unknown-linux-gnu --IS_NIGHTLY=$IS_NIGHTLY
    #SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/
    # veilid-cli
-    RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh aarch64 aarch64-unknown-linux-gnu
+    RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh aarch64 aarch64-unknown-linux-gnu --IS_NIGHTLY=$IS_NIGHTLY
    # save artifacts
    SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/

--- a/package/debian/earthly_make_veilid_cli_deb.sh
+++ b/package/debian/earthly_make_veilid_cli_deb.sh
@ -3,6 +3,8 @@ set -e

 ARCH=$1
 CARGO_ARCH=$2
+IS_NIGHTLY=$3
+BUILD_DATE=$(date '+%Y%m%d')
 CARGO_VERSION="$(/veilid/package/cargo_version.sh /veilid/veilid-cli/Cargo.toml)"
 rm -rf /dpkg
 mkdir -p /dpkg/out
@ -16,4 +18,13 @@ mkdir -p /dpkg/veilid-cli/usr/bin
 cp -f /veilid/target/$CARGO_ARCH/release/veilid-cli /dpkg/veilid-cli/usr/bin
 # pack it up
 dpkg-deb -b /dpkg/veilid-cli/
-mv /dpkg/veilid-cli.deb /dpkg/out/veilid-cli-$CARGO_VERSION\_$ARCH.deb
+# Appropriatly name the package for STABLE or NIGHTLY release
+if [ "$3" = true ]
+then
+    mv /dpkg/veilid-cli.deb /dpkg/out/veilid-cli-$BUILD_DATE\_$ARCH.deb
+elif [ "$3" = false ]
+then
+    mv /dpkg/veilid-cli.deb /dpkg/out/veilid-cli-$CARGO_VERSION\_$ARCH.deb
+else
+    echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY"
+fi
--- a/package/debian/earthly_make_veilid_server_deb.sh
+++ b/package/debian/earthly_make_veilid_server_deb.sh
@ -3,6 +3,8 @@ set -e

 ARCH=$1
 CARGO_ARCH=$2
+IS_NIGHTLY=$3
+BUILD_DATE=$(date '+%Y%m%d')
 CARGO_VERSION="$(/veilid/package/cargo_version.sh /veilid/veilid-server/Cargo.toml)"
 rm -rf /dpkg
 mkdir -p /dpkg/out
@ -22,4 +24,13 @@ mkdir -p /dpkg/veilid-server/usr/bin
 cp -f /veilid/target/$CARGO_ARCH/release/veilid-server /dpkg/veilid-server/usr/bin
 # pack it up
 dpkg-deb -b /dpkg/veilid-server/
-mv /dpkg/veilid-server.deb /dpkg/out/veilid-server-$CARGO_VERSION\_$ARCH.deb
+# Appropriatly name the package for STABLE or NIGHTLY release
+if [ "$3" = true ]
+then
+    mv /dpkg/veilid-server.deb /dpkg/out/veilid-server-$BUILD_DATE\_$ARCH.deb
+elif [ "$3" = false ]
+then
+    mv /dpkg/veilid-server.deb /dpkg/out/veilid-server-$CARGO_VERSION\_$ARCH.deb
+else
+    echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY"
+fi
--- a/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh
+++ b/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh
@ -1,10 +1,20 @@
 #!/bin/bash
 ARCH=$1
 CARGO_ARCH=$2
+IS_NIGHTLY=$3
+BUILD_DATE=$(date '+%Y%m%d')
 CARGO_VERSION="$(/veilid/package/cargo_version.sh /veilid/veilid-cli/Cargo.toml)"
    
 # veilid-cli spec file
 cp -rf /veilid/package/rpm/veilid-cli/veilid-cli.spec /root/rpmbuild/SPECS/
+if [ "$3" = true ]
+then
+    /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-cli.spec RELEASE_VERSION $BUILD_DATE
+elif [ "$3" = false ]
+    /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-cli.spec RELEASE_VERSION $CARGO_VERSION
+else
+    echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY"
+fi
 /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-cli.spec CARGO_VERSION $CARGO_VERSION 
 /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-cli.spec ARCH $ARCH
 /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-cli.spec CARGO_ARCH $CARGO_ARCH
--- a/package/rpm/veilid-cli/veilid-cli.spec
+++ b/package/rpm/veilid-cli/veilid-cli.spec
@ -1,6 +1,6 @@
 Summary: Veilid Server Command Line Interface
 Name: veilid-cli
-Version: $CARGO_VERSION
+Version: $RELEASE_VERSION
 Release: 1
 URL: https://veilid.com
 Group: System
--- a/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh
+++ b/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh
@ -1,11 +1,21 @@
 #!/bin/bash
 ARCH=$1
 CARGO_ARCH=$2
+IS_NIGHTLY=$3
+BUILD_DATE=$(date '+%Y%m%d')
 CARGO_VERSION="$(/veilid/package/cargo_version.sh /veilid/veilid-server/Cargo.toml)"
    
 # veilid-server spec file
 cp -rf /veilid/package/rpm/veilid-server/veilid-server.spec /root/rpmbuild/SPECS/
-/veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-server.spec CARGO_VERSION $CARGO_VERSION 
+# Select CARGO_VERSION for STABLE releases or BUILD_DATE for NIGHTLY releases
+if [ "$3" = true ]
+then
+    /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-server.spec RELEASE_VERSION $BUILD_DATE
+elif [ "$3" = false ]
+    /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-server.spec RELEASE_VERSION $CARGO_VERSION
+else
+    echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY"
+fi
 /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-server.spec ARCH $ARCH
 /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-server.spec CARGO_ARCH $CARGO_ARCH

--- a/package/rpm/veilid-server/veilid-server.spec
+++ b/package/rpm/veilid-server/veilid-server.spec
@ -1,6 +1,6 @@
 Summary: Install a server grade, headless Veilid node
 Name: veilid-server
-Version: $CARGO_VERSION
+Version: $RELEASE_VERSION
 Release: 1
 URL: https://veilid.com
 Group: System
--- a/scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
+++ b/scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
@ -7,5 +7,5 @@ cd workspace
 cp ~/builds/t338Uo9fn/0/veilid/veilid/target/packages/*.deb .
 tar -cf amd64-debs.tar *.deb
 scp *.tar gitlab-runner@10.116.0.5:~
-cd ~
+cd ../
 rm -rf workspace
--- a/scripts/cicd/build-orchestration/distribute-nightly-packages.sh
+++ b/scripts/cicd/build-orchestration/distribute-nightly-packages.sh
@ -0,0 +1,79 @@
+#!/bin/bash
+
+# Clean and reset the workspaces
+echo "Setting up the workspace"
+# Rsync active repo to local workspace
+rsync --archive gitlab-runner@10.116.0.3:/srv $HOME
+# Ensure repo directory structure exists
+mkdir -p $HOME/srv/{gpg,rpm/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64},apt/{dists/{stable/main/{binary-amd64,binary-arm64},nightly/main/{binary-amd64,binary-arm64}},pool/{stable/main,nightly/main}}}
+# Delete previous versions of packages
+rm -rf $HOME/srv/apt/pool/nightly/main/*.deb
+rm -rf $HOME/srv/rpm/{nightly/x86_64/*,nightly/x86_64/*}
+# Ensure RPM workspace setup
+mkdir -p $HOME/rpm-build-container/mount/repo/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64}
+rm -rf $HOME/rpm-builder/mount/repo/{nightly/x86_64/*,nightly/x86_64/*}
+
+# Setup crypto
+export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)"
+cat $HOME/package-signing-key.private | gpg --import
+gpg --armor --export admin@veilid.org > $HOME/srv/gpg/veilid-packages-key.public
+
+# Copy .deb files into the workspace and generate repo files
+echo "Starting deb process"
+cd ~
+tar -xf amd64-debs.tar
+tar -xf arm64-debs.tar
+cp *.deb /home/gitlab-runner/srv/apt/pool/nightly/main
+cd /home/gitlab-runner/srv/apt
+echo "Creating Packages file"
+dpkg-scanpackages --arch amd64 pool/nightly > dists/nightly/main/binary-amd64/Packages
+dpkg-scanpackages --arch arm64 pool/nightly > dists/nightly/main/binary-arm64/Packages
+cat dists/nightly/main/binary-amd64/Packages | gzip -9 > dists/nightly/main/binary-amd64/Packages.gz
+cat dists/nightly/main/binary-arm64/Packages | gzip -9 > dists/nightly/main/binary-arm64/Packages.gz
+echo "Creating Release file"
+cd /home/gitlab-runner/srv/apt/dists/nightly
+~/generate-nightly-release.sh > Release
+echo "Signing Release file and creating InRelease"
+cat /home/gitlab-runner/srv/apt/dists/nightly/Release | gpg --default-key admin@veilid.org -abs > /home/gitlab-runner/srv/apt/dists/nightly/Release.gpg
+cat /home/gitlab-runner/srv/apt/dists/nightly/Release | gpg --default-key admin@veilid.org -abs --clearsign > /home/gitlab-runner/srv/apt/dists/nightly/InRelease
+
+# Copy .rpm files into the workspace and generate repo files
+echo "Starting rpm process"
+cd $HOME
+tar -xf amd64-rpms.tar
+echo "Copying rpms to container workspace"
+cp *x86_64.rpm $HOME/rpm-build-container/mount/repo/nightly/x86_64
+echo "Copying signing material to container workspace"
+cp -R $GNUPGHOME $HOME/rpm-build-container/mount/keystore
+echo "Executing container actions"
+docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8
+sleep 2
+cp -R $HOME/rpm-build-container/mount/repo/nightly $HOME/srv/rpm/nightly
+cd $HOME/srv/rpm/nightly/x86_64
+echo "Signing the rpm repository"
+gpg --default-key admin@veilid.org --detach-sign --armor $HOME/srv/rpm/nightly/x86_64/repodata/repomd.xml
+
+echo "[veilid-nightly-x86_64-rpm-repo]
+name=Veilid Nightly x86_64 RPM Repo
+baseurl=https://packages.veilid.net/rpm/nightly/x86_64
+enabled=1
+gpgcheck=1
+gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/nightly/x86_64/veilid-rpm-repo.repo
+
+# Tar the repo data and transfer to the repo server
+echo "Moving the repo scaffold to the repo server"
+cd $HOME
+rsync --archive $HOME/srv gitlab-runner@10.116.0.3:/srv
+# tar -cf $HOME/repo.tar srv
+# scp -i $HOME/.ssh/id_ed25519 $HOME/repo.tar gitlab-runner@10.116.0.3:~
+
+# Cleanup
+echo "Cleaning up the workspace"
+rm -rf $GNUPGHOME
+# rm $HOME/repo.tar
+rm $HOME/*.deb
+rm $HOME/*.rpm
+rm -rf $HOME/rpm-build-container/mount/keystore
+# rm rpm-build-container/mount/repo/*.rpm
+# rm -rf rpm-build-container/mount/repo/repodata/*
+echo "Process complete"
--- a/scripts/cicd/build-orchestration/distribute-packages.sh
+++ b/scripts/cicd/build-orchestration/distribute-packages.sh
@ -1,70 +0,0 @@
-#!/bin/bash
-
-# Clean and reset the workspace
-echo "Setting up the workspace"
-rm -rf /home/gitlab-runner/srv
-mkdir -p /home/gitlab-runner/srv/{gpg,rpm,apt/{dists/stable/main/{binary-amd64,binary-arm64},pool/main}}
-
-# Setup crypto
-export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)"
-cat ~/package-signing-key.private | gpg --import
-gpg --armor --export admin@veilid.org > ~/srv/gpg/veilid-packages-key.public
-
-# Copy .deb files into the workspace and generate repo files
-echo "Starting deb process"
-cd ~
-tar -xf amd64-debs.tar
-tar -xf arm64-debs.tar
-cp *.deb ~/srv/apt/pool/main
-cd ~/srv/apt
-echo "Creating Packages file"
-dpkg-scanpackages --arch amd64 pool/ > dists/stable/main/binary-amd64/Packages
-dpkg-scanpackages --arch arm64 pool/ > dists/stable/main/binary-arm64/Packages
-cat dists/stable/main/binary-amd64/Packages | gzip -9 > dists/stable/main/binary-amd64/Packages.gz
-cat dists/stable/main/binary-arm64/Packages | gzip -9 > dists/stable/main/binary-arm64/Packages.gz
-echo "Creating Release file"
-cd ~/srv/apt/dists/stable
-~/generate-release.sh > Release
-echo "Signing Release file and creating InRelease"
-cat ~/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs > ~/srv/apt/dists/stable/Release.gpg
-cat ~/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs --clearsign > ~/srv/apt/dists/stable/InRelease
-
-# Copy .rpm files into the workspace and generate repo files
-echo "Starting rpm process"
-cd ~
-tar -xf amd64-rpms.tar
-echo "Copying rpms to container workspace"
-cp *.rpm /home/gitlab-runner/rpm-build-container/mount/repo
-echo "Copying signing material to container workspace"
-cp -R $GNUPGHOME /home/gitlab-runner/rpm-build-container/mount/keystore
-echo "Executing container actions"
-docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=/home/gitlab-runner/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8
-sleep 2
-cp -R /home/gitlab-runner/rpm-build-container/mount/repo/* ~/srv/rpm
-cd ~/srv/rpm
-echo "Signing the rpm repository"
-gpg --default-key admin@veilid.org --detach-sign --armor ~/srv/rpm/repodata/repomd.xml
-
-echo "[veilid-rpm-repo]
-name=Veilid RPM Repo
-baseurl=https://packages.veilid.net/rpm
-enabled=1
-gpgcheck=1
-gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > /home/gitlab-runner/srv/rpm/veilid-rpm-repo.repo
-
-# Tar the repo data and transfer to the repo server
-echo "Moving the repo scaffold to the repo server"
-cd ~
-tar -cf /home/gitlab-runner/repo.tar srv
-scp -i /home/gitlab-runner/.ssh/id_ed25519 /home/gitlab-runner/repo.tar gitlab-runner@10.116.0.3:~
-
-# Cleanup
-echo "Cleaning up the workspace"
-rm -rf $GNUPGHOME
-rm /home/gitlab-runner/repo.tar
-rm /home/gitlab-runner/*.deb
-rm /home/gitlab-runner/*.rpm
-rm -rf /home/gitlab-runner/rpm-build-container/mount/keystore
-rm rpm-build-container/mount/repo/*.rpm
-rm -rf rpm-build-container/mount/repo/repodata/*
-echo "Process complete"
--- a/scripts/cicd/build-orchestration/distribute-stable-packages.sh
+++ b/scripts/cicd/build-orchestration/distribute-stable-packages.sh
@ -0,0 +1,79 @@
+#!/bin/bash
+
+# Clean and reset the workspaces
+echo "Setting up the workspace"
+# Rsync active repo to local workspace
+rsync --archive gitlab-runner@10.116.0.3:/srv $HOME
+# Ensure repo directory structure exists
+mkdir -p $HOME/srv/{gpg,rpm/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64},apt/{dists/{stable/main/{binary-amd64,binary-arm64},nightly/main/{binary-amd64,binary-arm64}},pool/{stable/main,nightly/main}}}
+# Delete previous versions of packages
+rm -rf $HOME/srv/apt/pool/stable/main/*.deb
+rm -rf $HOME/srv/rpm/{stable/x86_64/*,stable/x86_64/*}
+# Ensure RPM workspace setup
+mkdir -p $HOME/rpm-build-container/mount/repo/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64}
+rm -rf $HOME/rpm-builder/mount/repo/{stable/x86_64/*,stable/x86_64/*}
+
+# Setup crypto
+export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)"
+cat $HOME/package-signing-key.private | gpg --import
+gpg --armor --export admin@veilid.org > $HOME/srv/gpg/veilid-packages-key.public
+
+# Copy .deb files into the workspace and generate repo files
+echo "Starting deb process"
+cd $HOME
+tar -xf amd64-debs.tar
+tar -xf arm64-debs.tar
+cp *.deb $HOME/srv/apt/pool/stable/main
+cd $HOME/srv/apt
+echo "Creating Packages file"
+dpkg-scanpackages --arch amd64 pool/stable > dists/stable/main/binary-amd64/Packages
+dpkg-scanpackages --arch arm64 pool/stable > dists/stable/main/binary-arm64/Packages
+cat dists/stable/main/binary-amd64/Packages | gzip -9 > dists/stable/main/binary-amd64/Packages.gz
+cat dists/stable/main/binary-arm64/Packages | gzip -9 > dists/stable/main/binary-arm64/Packages.gz
+echo "Creating Release file"
+cd $HOME/srv/apt/dists/stable
+~/generate-release.sh > Release
+echo "Signing Release file and creating InRelease"
+cat $HOME/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs > $HOME/srv/apt/dists/stable/Release.gpg
+cat $HOME/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs --clearsign > $HOME/srv/apt/dists/stable/InRelease
+
+# Copy .rpm files into the workspace and generate repo files
+echo "Starting rpm process"
+cd $HOME
+tar -xf amd64-rpms.tar
+echo "Copying rpms to container workspace"
+cp *x86_64.rpm $HOME/rpm-build-container/mount/repo/stable/x86_64
+echo "Copying signing material to container workspace"
+cp -R $GNUPGHOME $HOME/rpm-build-container/mount/keystore
+echo "Executing container actions"
+docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8
+sleep 2
+cp -R $HOME/rpm-build-container/mount/repo/stable ~/srv/rpm/stable
+cd $HOME/srv/rpm/stable/x86_64
+echo "Signing the rpm repository"
+gpg --default-key admin@veilid.org --detach-sign --armor $HOME/srv/rpm/stable/x86_64/repodata/repomd.xml
+
+echo "[veilid-stable-x86_64-rpm-repo]
+name=Veilid Stable x86_64 RPM Repo
+baseurl=https://packages.veilid.net/rpm/stable/x86_64
+enabled=1
+gpgcheck=1
+gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/stable/x86_64/veilid-rpm-repo.repo
+
+# Tar the repo data and transfer to the repo server
+echo "Moving the repo scaffold to the repo server"
+cd $HOME
+rsync --archive $HOME/srv gitlab-runner@10.116.0.3:/srv
+# tar -cf $HOME/repo.tar srv
+# scp -i $HOME/.ssh/id_ed25519 $HOME/repo.tar gitlab-runner@10.116.0.3:~
+
+# Cleanup
+echo "Cleaning up the workspace"
+rm -rf $GNUPGHOME
+# rm $HOME/repo.tar
+rm $HOME/*.deb
+rm $HOME/*.rpm
+rm -rf $HOME/rpm-build-container/mount/keystore
+# rm rpm-build-container/mount/repo/*.rpm
+# rm -rf rpm-build-container/mount/repo/repodata/*
+echo "Process complete"
--- a/scripts/cicd/build-orchestration/generate-nightly-release.sh
+++ b/scripts/cicd/build-orchestration/generate-nightly-release.sh
@ -0,0 +1,27 @@
+#!/bin/sh
+set -e
+
+do_hash() {
+    HASH_NAME=$1
+    HASH_CMD=$2
+    echo "${HASH_NAME}:"
+    for f in $(find -type f); do
+        f=$(echo $f | cut -c3-) # remove ./ prefix
+        if [ "$f" = "Release" ]; then
+            continue
+        fi
+        echo " $(${HASH_CMD} ${f}  | cut -d" " -f1) $(wc -c $f)"
+    done
+}
+
+cat << EOF
+Origin: packages.veilid.net
+Suite: nightly
+Architectures: amd64 arm64
+Components: main
+Description: Official repository for nightly released Veilid binaries.
+Date: $(date -Ru)
+EOF
+do_hash "MD5Sum" "md5sum"
+do_hash "SHA1" "sha1sum"
+do_hash "SHA256" "sha256sum"
--- a/scripts/cicd/build-orchestration/generate-stable-release.sh
+++ b/scripts/cicd/build-orchestration/generate-stable-release.sh
@ -16,9 +16,7 @@ do_hash() {

 cat << EOF
 Origin: packages.veilid.net
-Label: packages.veilid.net
 Suite: stable
-Codename: bullseye
 Architectures: amd64 arm64
 Components: main
 Description: Official repository for Veilid binaries.