From 30413e7f7093a881ea1ab210c8e2888da301616f Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Wed, 17 Jul 2024 11:17:54 -0500 Subject: [PATCH 1/3] Saving my work so I can switch back to main for a few minutes of troubleshooting --- Earthfile | 22 +++++++++++-------- INSTALL.md | 18 +++++++++++++-- package/debian/earthly_make_veilid_cli_deb.sh | 12 ++++++++-- .../debian/earthly_make_veilid_server_deb.sh | 10 +++++++++ .../distribute-nightly-packages.sh | 16 +++++++------- 5 files changed, 57 insertions(+), 21 deletions(-) diff --git a/Earthfile b/Earthfile index d3985593..02eafc22 100644 --- a/Earthfile +++ b/Earthfile @@ -206,21 +206,23 @@ unit-tests-linux: # Package package-linux-amd64-deb: + ARG IS_NIGHTLY="false" FROM +build-linux-amd64 ################################# ### DEBIAN DPKG .DEB FILES ################################# COPY --dir package /veilid # veilid-server - RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh amd64 x86_64-unknown-linux-gnu true + RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh amd64 x86_64-unknown-linux-gnu "$IS_NIGHTLY" SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/ # veilid-cli - RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh amd64 x86_64-unknown-linux-gnu true + RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh amd64 x86_64-unknown-linux-gnu "$IS_NIGHTLY" # save artifacts SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/ package-linux-amd64-rpm: - FROM --platform amd64 rockylinux:8 + ARG IS_NIGHTLY="false" + FROM --platform amd64 rockylinux:9 RUN yum install -y createrepo rpm-build rpm-sign yum-utils rpmdevtools RUN rpmdev-setuptree ################################# @@ -231,28 +233,30 @@ package-linux-amd64-rpm: COPY +build-linux-amd64/x86_64-unknown-linux-gnu /veilid/target/x86_64-unknown-linux-gnu RUN mkdir -p /rpm-work-dir/veilid-server # veilid-server - RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh x86_64 x86_64-unknown-linux-gnu true + RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh x86_64 x86_64-unknown-linux-gnu "$IS_NIGHTLY" #SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/x86_64/*.rpm AS LOCAL ./target/packages/ # veilid-cli - RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh x86_64 x86_64-unknown-linux-gnu true + RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh x86_64 x86_64-unknown-linux-gnu "$IS_NIGHTLY" # save artifacts SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/x86_64/*.rpm AS LOCAL ./target/packages/ package-linux-arm64-deb: + ARG IS_NIGHTLY="false" FROM +build-linux-arm64 ################################# ### DEBIAN DPKG .DEB FILES ################################# COPY --dir package /veilid # veilid-server - RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh arm64 aarch64-unknown-linux-gnu true + RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh arm64 aarch64-unknown-linux-gnu "$IS_NIGHTLY" SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/ # veilid-cli - RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh arm64 aarch64-unknown-linux-gnu true + RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh arm64 aarch64-unknown-linux-gnu "$IS_NIGHTLY" # save artifacts SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/ package-linux-arm64-rpm: + ARG IS_NIGHTLY="false" FROM --platform arm64 rockylinux:8 RUN yum install -y createrepo rpm-build rpm-sign yum-utils rpmdevtools RUN rpmdev-setuptree @@ -264,10 +268,10 @@ package-linux-arm64-rpm: COPY +build-linux-arm64/aarch64-unknown-linux-gnu /veilid/target/aarch64-unknown-linux-gnu RUN mkdir -p /rpm-work-dir/veilid-server # veilid-server - RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh aarch64 aarch64-unknown-linux-gnu --IS_NIGHTLY=$IS_NIGHTLY + RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh aarch64 aarch64-unknown-linux-gnu "$IS_NIGHTLY" #SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/ # veilid-cli - RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh aarch64 aarch64-unknown-linux-gnu --IS_NIGHTLY=$IS_NIGHTLY + RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh aarch64 aarch64-unknown-linux-gnu "$IS_NIGHTLY" # save artifacts SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/ diff --git a/INSTALL.md b/INSTALL.md index 56f9f983..a1b38b29 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -27,17 +27,31 @@ dpkg --print-architecture **Step 3**: Add Veilid to your list of available software.
*Explanation*: Use the result of your command in **Step 2** and run **one** of the following: -- For **AMD64** based systems run this command: +- For *STABLE* releases + - **AMD64** based systems run this command: ```shell echo "deb [arch=amd64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null ``` -- For **ARM64** based systems run this command: + - **ARM64** based systems run this command: ```shell echo "deb [arch=arm64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null ``` +- For *NIGHTLY* (bleeding edge) releases + - **AMD64** based systems run this command: + + ```shell + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt nightly main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null + ``` + + - **ARM64** based systems run this command: + + ```shell + echo "deb [arch=arm64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt nightly main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null + ``` + *Explanation*: Each of the above commands will create a new file called `veilid.list` in the `/etc/apt/sources.list.d/`. This file contains instructions that tell the operating system where to download Veilid. diff --git a/package/debian/earthly_make_veilid_cli_deb.sh b/package/debian/earthly_make_veilid_cli_deb.sh index f971f5fa..86c04b06 100755 --- a/package/debian/earthly_make_veilid_cli_deb.sh +++ b/package/debian/earthly_make_veilid_cli_deb.sh @@ -10,8 +10,16 @@ rm -rf /dpkg mkdir -p /dpkg/out # veilid-cli dpkg control -cp -rf /veilid/package/debian/veilid-cli /dpkg -/veilid/package/replace_variable.sh /dpkg/veilid-cli/DEBIAN/control CARGO_VERSION $CARGO_VERSION +cp -rf /veilid/package/debian/veilid-cli /dpkg# Appropriatly name the package for STABLE or NIGHTLY release +if [ "$3" = "true" ] +then + /veilid/package/replace_variable.sh /dpkg/veilid-cli/DEBIAN/control CARGO_VERSION $BUILD_DATE +elif [ "$3" = "false" ] +then + /veilid/package/replace_variable.sh /dpkg/veilid-cli/DEBIAN/control CARGO_VERSION $CARGO_VERSION +else + echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY" +fi /veilid/package/replace_variable.sh /dpkg/veilid-cli/DEBIAN/control ARCH $ARCH # veilid-cli executable mkdir -p /dpkg/veilid-cli/usr/bin diff --git a/package/debian/earthly_make_veilid_server_deb.sh b/package/debian/earthly_make_veilid_server_deb.sh index dfca29dd..bdf7916b 100755 --- a/package/debian/earthly_make_veilid_server_deb.sh +++ b/package/debian/earthly_make_veilid_server_deb.sh @@ -11,6 +11,16 @@ mkdir -p /dpkg/out # veilid-server dpkg control cp -rf /veilid/package/debian/veilid-server /dpkg +# Appropriatly name the package for STABLE or NIGHTLY release +# if [ "$3" = "true" ] +# then +# /veilid/package/replace_variable.sh /dpkg/veilid-server/DEBIAN/control CARGO_VERSION $BUILD_DATE +# elif [ "$3" = "false" ] +# then +# /veilid/package/replace_variable.sh /dpkg/veilid-server/DEBIAN/control CARGO_VERSION $CARGO_VERSION +# else +# echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY" +# fi /veilid/package/replace_variable.sh /dpkg/veilid-server/DEBIAN/control CARGO_VERSION $CARGO_VERSION /veilid/package/replace_variable.sh /dpkg/veilid-server/DEBIAN/control ARCH $ARCH # veilid-server configuration diff --git a/scripts/cicd/build-orchestration/distribute-nightly-packages.sh b/scripts/cicd/build-orchestration/distribute-nightly-packages.sh index 344b828b..4ffc5d65 100644 --- a/scripts/cicd/build-orchestration/distribute-nightly-packages.sh +++ b/scripts/cicd/build-orchestration/distribute-nightly-packages.sh @@ -3,7 +3,7 @@ # Clean and reset the workspaces echo "Setting up the workspace" # Rsync active repo to local workspace -rsync --archive gitlab-runner@10.116.0.3:/srv $HOME +rsync --archive gitlab-runner@10.116.0.3:/srv/ $HOME/srv/ # Ensure repo directory structure exists mkdir -p $HOME/srv/{gpg,rpm/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64},apt/{dists/{stable/main/{binary-amd64,binary-arm64},nightly/main/{binary-amd64,binary-arm64}},pool/{stable/main,nightly/main}}} # Delete previous versions of packages @@ -20,7 +20,7 @@ gpg --armor --export admin@veilid.org > $HOME/srv/gpg/veilid-packages-key.public # Copy .deb files into the workspace and generate repo files echo "Starting deb process" -cd ~ +cd $HOME tar -xf amd64-debs.tar tar -xf arm64-debs.tar cp *.deb /home/gitlab-runner/srv/apt/pool/nightly/main @@ -31,11 +31,11 @@ dpkg-scanpackages --arch arm64 pool/nightly > dists/nightly/main/binary-arm64/Pa cat dists/nightly/main/binary-amd64/Packages | gzip -9 > dists/nightly/main/binary-amd64/Packages.gz cat dists/nightly/main/binary-arm64/Packages | gzip -9 > dists/nightly/main/binary-arm64/Packages.gz echo "Creating Release file" -cd /home/gitlab-runner/srv/apt/dists/nightly -~/generate-nightly-release.sh > Release +cd $HOME/srv/apt/dists/nightly +bash $HOME/generate-nightly-release.sh > Release echo "Signing Release file and creating InRelease" -cat /home/gitlab-runner/srv/apt/dists/nightly/Release | gpg --default-key admin@veilid.org -abs > /home/gitlab-runner/srv/apt/dists/nightly/Release.gpg -cat /home/gitlab-runner/srv/apt/dists/nightly/Release | gpg --default-key admin@veilid.org -abs --clearsign > /home/gitlab-runner/srv/apt/dists/nightly/InRelease +cat $HOME/srv/apt/dists/nightly/Release | gpg --default-key admin@veilid.org -abs > /home/gitlab-runner/srv/apt/dists/nightly/Release.gpg +cat $HOME/srv/apt/dists/nightly/Release | gpg --default-key admin@veilid.org -abs --clearsign > /home/gitlab-runner/srv/apt/dists/nightly/InRelease # Copy .rpm files into the workspace and generate repo files echo "Starting rpm process" @@ -48,7 +48,7 @@ cp -R $GNUPGHOME $HOME/rpm-build-container/mount/keystore echo "Executing container actions" docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8 sleep 2 -cp -R $HOME/rpm-build-container/mount/repo/nightly $HOME/srv/rpm/nightly +cp -R $HOME/rpm-build-container/mount/repo/nightly/* $HOME/srv/rpm/nightly cd $HOME/srv/rpm/nightly/x86_64 echo "Signing the rpm repository" gpg --default-key admin@veilid.org --detach-sign --armor $HOME/srv/rpm/nightly/x86_64/repodata/repomd.xml @@ -63,7 +63,7 @@ gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/r # Tar the repo data and transfer to the repo server echo "Moving the repo scaffold to the repo server" cd $HOME -rsync --archive $HOME/srv gitlab-runner@10.116.0.3:/srv +rsync --archive $HOME/srv/ gitlab-runner@10.116.0.3:/srv/ # tar -cf $HOME/repo.tar srv # scp -i $HOME/.ssh/id_ed25519 $HOME/repo.tar gitlab-runner@10.116.0.3:~ From 6c0b645ba67d720a381b90aa3edb7d50cd78f18e Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Wed, 17 Jul 2024 15:18:59 -0500 Subject: [PATCH 2/3] Commiting and pushing so I can test on a RPM build machine --- package/debian/earthly_make_veilid_cli_deb.sh | 6 +++-- .../debian/earthly_make_veilid_server_deb.sh | 24 +++++++++---------- package/debian/veilid-cli/DEBIAN/control | 2 +- package/debian/veilid-server/DEBIAN/control | 4 +++- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/package/debian/earthly_make_veilid_cli_deb.sh b/package/debian/earthly_make_veilid_cli_deb.sh index 86c04b06..c0feb9a1 100755 --- a/package/debian/earthly_make_veilid_cli_deb.sh +++ b/package/debian/earthly_make_veilid_cli_deb.sh @@ -10,7 +10,8 @@ rm -rf /dpkg mkdir -p /dpkg/out # veilid-cli dpkg control -cp -rf /veilid/package/debian/veilid-cli /dpkg# Appropriatly name the package for STABLE or NIGHTLY release +cp -rf /veilid/package/debian/veilid-cli /dpkg +# Appropriatly set vars for STABLE or NIGHTLY release if [ "$3" = "true" ] then /veilid/package/replace_variable.sh /dpkg/veilid-cli/DEBIAN/control CARGO_VERSION $BUILD_DATE @@ -35,4 +36,5 @@ then mv /dpkg/veilid-cli.deb /dpkg/out/veilid-cli-$CARGO_VERSION\_$ARCH.deb else echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY" -fi \ No newline at end of file +fi +echo "make veilid-cli deb process complete" \ No newline at end of file diff --git a/package/debian/earthly_make_veilid_server_deb.sh b/package/debian/earthly_make_veilid_server_deb.sh index bdf7916b..55d4e2a4 100755 --- a/package/debian/earthly_make_veilid_server_deb.sh +++ b/package/debian/earthly_make_veilid_server_deb.sh @@ -11,17 +11,16 @@ mkdir -p /dpkg/out # veilid-server dpkg control cp -rf /veilid/package/debian/veilid-server /dpkg -# Appropriatly name the package for STABLE or NIGHTLY release -# if [ "$3" = "true" ] -# then -# /veilid/package/replace_variable.sh /dpkg/veilid-server/DEBIAN/control CARGO_VERSION $BUILD_DATE -# elif [ "$3" = "false" ] -# then -# /veilid/package/replace_variable.sh /dpkg/veilid-server/DEBIAN/control CARGO_VERSION $CARGO_VERSION -# else -# echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY" -# fi -/veilid/package/replace_variable.sh /dpkg/veilid-server/DEBIAN/control CARGO_VERSION $CARGO_VERSION +# Appropriatly set vars for for STABLE or NIGHTLY release +if [ "$3" = "true" ] +then + /veilid/package/replace_variable.sh /dpkg/veilid-server/DEBIAN/control CARGO_VERSION $BUILD_DATE +elif [ "$3" = "false" ] +then + /veilid/package/replace_variable.sh /dpkg/veilid-server/DEBIAN/control CARGO_VERSION $CARGO_VERSION +else + echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY" +fi /veilid/package/replace_variable.sh /dpkg/veilid-server/DEBIAN/control ARCH $ARCH # veilid-server configuration mkdir -p /dpkg/veilid-server/etc/veilid-server @@ -43,4 +42,5 @@ then mv /dpkg/veilid-server.deb /dpkg/out/veilid-server-$CARGO_VERSION\_$ARCH.deb else echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY" -fi \ No newline at end of file +fi +echo "make veilid-server deb process complete" \ No newline at end of file diff --git a/package/debian/veilid-cli/DEBIAN/control b/package/debian/veilid-cli/DEBIAN/control index 3efe932f..150d20a1 100644 --- a/package/debian/veilid-cli/DEBIAN/control +++ b/package/debian/veilid-cli/DEBIAN/control @@ -6,4 +6,4 @@ Architecture: $ARCH Depends: libc6 (>= 2.23) Maintainer: jsmith@veilid.org Description: Veilid Server Command Line Interface - The Veilid peer-to-peer network server command line interface + This package installs a a CLI interface for locally installed Veilid peer-to-peer network headless nodes. diff --git a/package/debian/veilid-server/DEBIAN/control b/package/debian/veilid-server/DEBIAN/control index 53d2d3de..8ad25070 100644 --- a/package/debian/veilid-server/DEBIAN/control +++ b/package/debian/veilid-server/DEBIAN/control @@ -6,4 +6,6 @@ Architecture: $ARCH Depends: libc6 (>= 2.23) Maintainer: jsmith@veilid.org Description: Veilid Server - The Veilid peer-to-peer network server + This package installs a Veilid peer-to-peer network headless node which can be deployed for + supporting the overall Veilid network or for local development/experimental purposes. See + https://veilid.com for more information. From c9d2b354f73835088b12038475fa474813e6e668 Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Fri, 19 Jul 2024 14:08:25 -0500 Subject: [PATCH 3/3] Further adventures in nightly release automation Pushing this up in order to end-to-end test automated nightly releases. Please enter the commit message for your changes. Lines starting --- .gitlab-ci.yml | 23 +++++++----- INSTALL.md | 36 +++++++++++++++++- .../distribute-nightly-packages.sh | 28 +++++--------- .../distribute-stable-packages.sh | 37 ++++++++++--------- .../rpm-repo-building/Dockerfile | 10 +++++ .../rpm-repo-building/repobuild.sh | 33 +++++++++++++++++ 6 files changed, 121 insertions(+), 46 deletions(-) create mode 100644 scripts/cicd/build-orchestration/rpm-repo-building/Dockerfile create mode 100644 scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 91017ff7..85316d75 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -106,7 +106,7 @@ package_amd64_deb: - build-amd64-deb script: - earthly bootstrap - - earthly +package-linux-amd64-deb --IS_NIGHTLY=$IS_NIGHTLY + - earthly +package-linux-amd64-deb - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -119,7 +119,7 @@ package_arm64_deb: - build-arm64-deb script: - earthly bootstrap - - earthly +package-linux-arm64-deb --IS_NIGHTLY=$IS_NIGHTLY + - earthly +package-linux-arm64-deb - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -132,7 +132,7 @@ package_amd64_rpm: - build-amd64-rpm script: - earthly bootstrap - - earthly +package-linux-amd64-rpm --IS_NIGHTLY=$IS_NIGHTLY + - earthly +package-linux-amd64-rpm - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -168,8 +168,11 @@ build_repositories: tags: - build-orchestration variables: - SECURE_FILES_DOWNLOAD_PATH: '$HOME' + SECURE_FILES_DOWNLOAD_PATH: '/home/gitlab-runner/' script: + - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash + - cp scripts/cicd/build-orchestration/rpm-repo-building/Dockerfile ~/rpm-build-container + - cp scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh ~/rpm-build-container - cp scripts/cicd/build-orchestration/generate-stable-release.sh ~ - bash scripts/cicd/build-orchestration/distribute-stable-packages.sh rules: @@ -280,8 +283,9 @@ dryrun_build_repositories: tags: - build-orchestration variables: - SECURE_FILES_DOWNLOAD_PATH: './' + SECURE_FILES_DOWNLOAD_PATH: '/home/gitlab-runner/' script: + - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash - cp scripts/cicd/build-orchestration/generate-release.sh ~ - bash scripts/cicd/build-orchestration/distribute-packages.sh rules: @@ -333,7 +337,7 @@ nightly_package_amd64_deb: - build-amd64-deb script: - earthly bootstrap - - earthly +package-linux-amd64-deb #--IS_NIGHTLY="$IS_NIGHTLY" + - earthly +package-linux-amd64-deb --IS_NIGHTLY="$IS_NIGHTLY" - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh rules: - if: '$IS_NIGHTLY == true' @@ -346,7 +350,7 @@ nightly_package_arm64_deb: - build-arm64-deb script: - earthly bootstrap - - earthly +package-linux-arm64-deb #--IS_NIGHTLY="$IS_NIGHTLY" + - earthly +package-linux-arm64-deb --IS_NIGHTLY="$IS_NIGHTLY" - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh rules: - if: '$IS_NIGHTLY == true' @@ -359,7 +363,7 @@ nightly_package_amd64_rpm: - build-amd64-rpm script: - earthly bootstrap - - earthly +package-linux-amd64-rpm #--IS_NIGHTLY="$IS_NIGHTLY" + - earthly +package-linux-amd64-rpm --IS_NIGHTLY="$IS_NIGHTLY" - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh rules: - if: '$IS_NIGHTLY == true' @@ -369,8 +373,9 @@ nightly_build_repositories: tags: - build-orchestration variables: - SECURE_FILES_DOWNLOAD_PATH: '$HOME' + SECURE_FILES_DOWNLOAD_PATH: '/home/gitlab-runner/' script: + - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash - cp scripts/cicd/build-orchestration/generate-nightly-release.sh ~ - bash scripts/cicd/build-orchestration/distribute-nightly-packages.sh rules: diff --git a/INSTALL.md b/INSTALL.md index a1b38b29..c5eda9da 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -39,6 +39,7 @@ dpkg --print-architecture ```shell echo "deb [arch=arm64 signed-by=/usr/share/keyrings/veilid-packages-keyring.gpg] https://packages.veilid.net/apt stable main" | sudo tee /etc/apt/sources.list.d/veilid.list 1>/dev/null ``` + - For *NIGHTLY* (bleeding edge) releases - **AMD64** based systems run this command: @@ -69,6 +70,18 @@ sudo apt update sudo apt install veilid-server veilid-cli ``` +***Step 6***: Start veilid-server.service + +Go to [Start headless node](#start-headless-node) + +***Step 7***: View Node Activity + +Invoke the Veilid CLI utility. + +```shell +veilid-cli +``` + ### RPM-based Follow the steps here to add the repo to @@ -77,15 +90,36 @@ and install Veilid. **Step 1**: Add Veilid to your list of available software. +- For *STABLE* releases + ```shell -sudo yum-config-manager --add-repo https://packages.veilid.net/rpm/veilid-rpm-repo.repo +sudo dnf config-manager --add-repo https://packages.veilid.net/rpm/stable/x86_64/veilid-stable-x86_64-rpm.repo ``` + +- For *NIGHTLY* (bleeding edge) releases + +```shell +sudo dnf config-manager --add-repo https://packages.veilid.net/rpm/nightly/x86_64/veilid-nightly-x86_64-rpm.repo +``` + **Step 2**: Install Veilid. ```shell sudo dnf install veilid-server veilid-cli ``` +***Step 3***: Start veilid-server.service + +Go to [Start headless node](#start-headless-node) + +***Step 4***: View Node Activity + +Invoke the Veilid CLI utility. + +```shell +veilid-cli +``` + ### macOS Veilid is available [via Homebrew](https://formulae.brew.sh/formula/veilid). diff --git a/scripts/cicd/build-orchestration/distribute-nightly-packages.sh b/scripts/cicd/build-orchestration/distribute-nightly-packages.sh index 4ffc5d65..68a72f69 100644 --- a/scripts/cicd/build-orchestration/distribute-nightly-packages.sh +++ b/scripts/cicd/build-orchestration/distribute-nightly-packages.sh @@ -4,14 +4,9 @@ echo "Setting up the workspace" # Rsync active repo to local workspace rsync --archive gitlab-runner@10.116.0.3:/srv/ $HOME/srv/ -# Ensure repo directory structure exists -mkdir -p $HOME/srv/{gpg,rpm/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64},apt/{dists/{stable/main/{binary-amd64,binary-arm64},nightly/main/{binary-amd64,binary-arm64}},pool/{stable/main,nightly/main}}} # Delete previous versions of packages rm -rf $HOME/srv/apt/pool/nightly/main/*.deb -rm -rf $HOME/srv/rpm/{nightly/x86_64/*,nightly/x86_64/*} -# Ensure RPM workspace setup -mkdir -p $HOME/rpm-build-container/mount/repo/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64} -rm -rf $HOME/rpm-builder/mount/repo/{nightly/x86_64/*,nightly/x86_64/*} +rm -rf $HOME/srv/rpm/nightly/x86_64/* # Setup crypto export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)" @@ -44,11 +39,11 @@ tar -xf amd64-rpms.tar echo "Copying rpms to container workspace" cp *x86_64.rpm $HOME/rpm-build-container/mount/repo/nightly/x86_64 echo "Copying signing material to container workspace" -cp -R $GNUPGHOME $HOME/rpm-build-container/mount/keystore +cp -R $GNUPGHOME/* $HOME/rpm-build-container/mount/keystore echo "Executing container actions" -docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8 +docker run --rm -d -it -e IS_NIGHTLY=$IS_NIGHTLY --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v12 sleep 2 -cp -R $HOME/rpm-build-container/mount/repo/nightly/* $HOME/srv/rpm/nightly +cp -R $HOME/rpm-build-container/mount/repo/nightly/x86_64/* $HOME/srv/rpm/nightly/x86_64 cd $HOME/srv/rpm/nightly/x86_64 echo "Signing the rpm repository" gpg --default-key admin@veilid.org --detach-sign --armor $HOME/srv/rpm/nightly/x86_64/repodata/repomd.xml @@ -58,22 +53,19 @@ name=Veilid Nightly x86_64 RPM Repo baseurl=https://packages.veilid.net/rpm/nightly/x86_64 enabled=1 gpgcheck=1 -gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/nightly/x86_64/veilid-rpm-repo.repo +gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/nightly/x86_64/veilid-nightly-x86_64-rpm.repo # Tar the repo data and transfer to the repo server echo "Moving the repo scaffold to the repo server" cd $HOME -rsync --archive $HOME/srv/ gitlab-runner@10.116.0.3:/srv/ -# tar -cf $HOME/repo.tar srv -# scp -i $HOME/.ssh/id_ed25519 $HOME/repo.tar gitlab-runner@10.116.0.3:~ +rsync --archive $HOME/srv/* gitlab-runner@10.116.0.3:/srv # Cleanup echo "Cleaning up the workspace" rm -rf $GNUPGHOME -# rm $HOME/repo.tar +rm $HOME/*.tar rm $HOME/*.deb rm $HOME/*.rpm -rm -rf $HOME/rpm-build-container/mount/keystore -# rm rpm-build-container/mount/repo/*.rpm -# rm -rf rpm-build-container/mount/repo/repodata/* -echo "Process complete" \ No newline at end of file +rm -rf $HOME/rpm-build-container/mount/keystore/* +rm -rf $HOME/rpm-build-container/mount/repo/nightly/x86_64/* +echo "Nightly packages distribution process complete" \ No newline at end of file diff --git a/scripts/cicd/build-orchestration/distribute-stable-packages.sh b/scripts/cicd/build-orchestration/distribute-stable-packages.sh index 488f08f2..ea5fddc8 100644 --- a/scripts/cicd/build-orchestration/distribute-stable-packages.sh +++ b/scripts/cicd/build-orchestration/distribute-stable-packages.sh @@ -3,15 +3,10 @@ # Clean and reset the workspaces echo "Setting up the workspace" # Rsync active repo to local workspace -rsync --archive gitlab-runner@10.116.0.3:/srv $HOME -# Ensure repo directory structure exists -mkdir -p $HOME/srv/{gpg,rpm/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64},apt/{dists/{stable/main/{binary-amd64,binary-arm64},nightly/main/{binary-amd64,binary-arm64}},pool/{stable/main,nightly/main}}} +rsync --archive gitlab-runner@10.116.0.3:/srv/ $HOME/srv/ # Delete previous versions of packages rm -rf $HOME/srv/apt/pool/stable/main/*.deb -rm -rf $HOME/srv/rpm/{stable/x86_64/*,stable/x86_64/*} -# Ensure RPM workspace setup -mkdir -p $HOME/rpm-build-container/mount/repo/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64} -rm -rf $HOME/rpm-builder/mount/repo/{stable/x86_64/*,stable/x86_64/*} +rm -rf $HOME/srv/rpm/stable/x86_64/* # Setup crypto export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)" @@ -32,7 +27,7 @@ cat dists/stable/main/binary-amd64/Packages | gzip -9 > dists/stable/main/binary cat dists/stable/main/binary-arm64/Packages | gzip -9 > dists/stable/main/binary-arm64/Packages.gz echo "Creating Release file" cd $HOME/srv/apt/dists/stable -~/generate-release.sh > Release +$HOME/generate-release.sh > Release echo "Signing Release file and creating InRelease" cat $HOME/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs > $HOME/srv/apt/dists/stable/Release.gpg cat $HOME/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs --clearsign > $HOME/srv/apt/dists/stable/InRelease @@ -46,34 +41,40 @@ cp *x86_64.rpm $HOME/rpm-build-container/mount/repo/stable/x86_64 echo "Copying signing material to container workspace" cp -R $GNUPGHOME $HOME/rpm-build-container/mount/keystore echo "Executing container actions" -docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8 +docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v12 sleep 2 -cp -R $HOME/rpm-build-container/mount/repo/stable ~/srv/rpm/stable +cp -R $HOME/rpm-build-container/mount/repo/stable/x86_64/* $HOME/srv/rpm/stable/x86_64/ cd $HOME/srv/rpm/stable/x86_64 echo "Signing the rpm repository" gpg --default-key admin@veilid.org --detach-sign --armor $HOME/srv/rpm/stable/x86_64/repodata/repomd.xml +# Generate .repo file for stable x86_64 releases echo "[veilid-stable-x86_64-rpm-repo] name=Veilid Stable x86_64 RPM Repo baseurl=https://packages.veilid.net/rpm/stable/x86_64 enabled=1 gpgcheck=1 -gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/stable/x86_64/veilid-rpm-repo.repo +gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/stable/x86_64/veilid-stable-x86_64-rpm.repo + +# Generate .repo file for stable arm64 releases -- to be added +# echo "[veilid-stable-arm64-rpm-repo] +# name=Veilid Stable x86_64 RPM Repo +# baseurl=https://packages.veilid.net/rpm/stable/arm64 +# enabled=1 +# gpgcheck=1 +# gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/stable/x86_64/veilid-stable-arm64-rpm.repo # Tar the repo data and transfer to the repo server echo "Moving the repo scaffold to the repo server" cd $HOME rsync --archive $HOME/srv/* gitlab-runner@10.116.0.3:/srv -# tar -cf $HOME/repo.tar srv -# scp -i $HOME/.ssh/id_ed25519 $HOME/repo.tar gitlab-runner@10.116.0.3:~ # Cleanup echo "Cleaning up the workspace" rm -rf $GNUPGHOME -# rm $HOME/repo.tar +rm $HOME/*.tar rm $HOME/*.deb rm $HOME/*.rpm -rm -rf $HOME/rpm-build-container/mount/keystore -# rm rpm-build-container/mount/repo/*.rpm -# rm -rf rpm-build-container/mount/repo/repodata/* -echo "Process complete" \ No newline at end of file +rm -rf $HOME/rpm-build-container/mount/keystore/* +rm -rf $HOME/rpm-build-container/mount/repo/nightly/x86_64/* +echo "Stable packages distribution process complete" \ No newline at end of file diff --git a/scripts/cicd/build-orchestration/rpm-repo-building/Dockerfile b/scripts/cicd/build-orchestration/rpm-repo-building/Dockerfile new file mode 100644 index 00000000..72ff0b88 --- /dev/null +++ b/scripts/cicd/build-orchestration/rpm-repo-building/Dockerfile @@ -0,0 +1,10 @@ +FROM rockylinux:8 +LABEL author=admin@veilid.org +ENV IS_NIGHTLY false + +COPY repobuild.sh repobuild.sh +RUN dnf install -y createrepo rpm-build rpm-sign yum-utils +RUN chmod +x repobuild.sh + + +ENTRYPOINT ["./repobuild.sh"] \ No newline at end of file diff --git a/scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh b/scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh new file mode 100644 index 00000000..112115ab --- /dev/null +++ b/scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +echo "==========Log start $(date +%F_%T)==========" &>> /mount/logfile + +echo "setting GNUPGHOME $(date +%F_%T)" &>> /mount/logfile +export GNUPGHOME=/mount/keystore + +echo "Adding key to rpm utility $(date +%F_%T)" &>> /mount/logfile +echo "%_signature gpg +%_gpg_name 516C76D1E372C5C96EE54E22AE0E059BC64CD052" > /root/.rpmmacros + +if [ "$IS_NIGHTLY" = "true" ] +then + echo "Taking nightly actions branch $(date +%F_%T)" &>> /mount/logfile + cd /mount/repo/nightly/x86_64 +elif [ "$IS_NIGHTLY" = "false" ] +then + echo "Taking stable branch actions $(date +%F_%T)" &>> /mount/logfile + cd /mount/repo/stable/x86_64 +else + echo $IS_NIGHTLY "is not a valid state to determine if the build is STABLE or NIGHTLY (RPM RepoBuild)" &>> /mount/logfile +fi + +echo "Signing RPMs $(date +%F_%T)" &>> /mount/logfile +rpm --addsign *.rpm &>> /mount/logfile + +echo "Creating repo metadata $(date +%F_%T)" &>> /mount/logfile +createrepo . &>> /mount/logfile + +echo "Setting file ownership $(date +%F_%T)" &>> /mount/logfile +chown -R 1000:1000 /mount + +echo "==========RPM Packaging Process complete $(date +%F_%T)==========" &>> /mount/logfile \ No newline at end of file