From 038f4d21218d0d6270fd5de0f383ef944e685e85 Mon Sep 17 00:00:00 2001
From: TC Johnson <tc@veilid.org>
Date: Mon, 29 Apr 2024 14:40:07 -0500
Subject: [PATCH 01/12] Added CICD dry run
This is the first attempt at setting up a dry run pipeline
to test changes to the CICD config without actually publishing
the compiled binaries and packages built by the process.
The dry run should be triggered by any changes to .gitlab-ci.yml
or changes to any of the scripts under scripts/cicd/.
---
.gitlab-ci.yml | 142 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 142 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c16d688b..d48765ff 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,6 +55,8 @@ test_build:
resource_group: test
# when: manual
+# Actual release -- triggered by pushing a new version tag
+
release_job:
stage: release
image: registry.gitlab.com/gitlab-org/release-cli:latest
@@ -177,3 +179,143 @@ delete_build_machines:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
rules:
- if: '$CI_COMMIT_TAG =~ /v\d.+/'
+
+# Dryrun release -- triggered by changes in .gitlab-ci.yml or CICD scripts
+
+dryrun_create_build_machines:
+ stage: build_packages
+ tags:
+ - build-orchestration
+ script:
+ - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb
+ - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb
+ - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "push"
+ changes:
+ - .gitlab-ci.yml
+ - scripts/cicd/**/*
+
+dryrun_package_amd64_deb:
+ stage: build_packages
+ needs:
+ - dryrun_create_build_machines
+ tags:
+ - build-amd64-deb
+ script:
+ - earthly bootstrap
+ - earthly +package-linux-amd64-deb
+ - bash scripts/cicd/build-machine/scp-to-orchestrator.sh
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "push"
+ changes:
+ - .gitlab-ci.yml
+ - scripts/cicd/**/*
+
+dryrun_package_arm64_deb:
+ stage: build_packages
+ needs:
+ - dryrun_create_build_machines
+ tags:
+ - build-arm64-deb
+ script:
+ - earthly bootstrap
+ - earthly +package-linux-arm64-deb
+ - bash scripts/cicd/build-machine/scp-to-orchestrator.sh
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "push"
+ changes:
+ - .gitlab-ci.yml
+ - scripts/cicd/**/*
+
+dryrun_package_amd64_rpm:
+ stage: build_packages
+ needs:
+ - dryrun_create_build_machines
+ tags:
+ - build-amd64-rpm
+ script:
+ - earthly bootstrap
+ - earthly +package-linux-amd64-rpm
+ - bash scripts/cicd/build-machine/scp-to-orchestrator.sh
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "push"
+ changes:
+ - .gitlab-ci.yml
+ - scripts/cicd/**/*
+
+dryrun_publish_crates:
+ stage: build_packages
+ needs:
+ - dryrun_create_build_machines
+ tags:
+ - build-amd64-deb
+ script:
+ - vlt login
+ - vlt run --command="cargo publish -p veilid-tools --dry-run"
+ - vlt run --command="cargo publish -p veilid-core --dry-run"
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "push"
+ changes:
+ - .gitlab-ci.yml
+ - scripts/cicd/**/*
+
+dryrun_publish_python:
+ stage: build_packages
+ needs:
+ - dryrun_create_build_machines
+ tags:
+ - build-amd64-deb
+ script:
+ - vlt login
+ - cd veilid-python && /home/gitlab-runner/.local/bin/poetry build
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "push"
+ changes:
+ - .gitlab-ci.yml
+ - scripts/cicd/**/*
+
+dryrun_build_repositories:
+ stage: distribute
+ tags:
+ - build-orchestration
+ variables:
+ SECURE_FILES_DOWNLOAD_PATH: './'
+ script:
+ - cp scripts/cicd/build-orchestration/generate-release.sh ~
+ - bash scripts/cicd/build-orchestration/distribute-packages.sh
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "push"
+ changes:
+ - .gitlab-ci.yml
+ - scripts/cicd/**/*
+
+dryrun_deploy_repos:
+ stage: distribute
+ needs:
+ - dryrun_build_repositories
+ tags:
+ - repo-server
+ script:
+ - ls -al repo.tar
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "push"
+ changes:
+ - .gitlab-ci.yml
+ - scripts/cicd/**/*
+
+dryrun_delete_build_machines:
+ stage: distribute
+ needs:
+ - dryrun_deploy_repos
+ tags:
+ - build-orchestration
+ script:
+ - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb
+ - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb
+ - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "push"
+ changes:
+ - .gitlab-ci.yml
+ - scripts/cicd/**/*
\ No newline at end of file
From c33f0f19daea7732705a99529e40538ae1c5c97a Mon Sep 17 00:00:00 2001
From: TC Johnson <tc@veilid.org>
Date: Mon, 29 Apr 2024 15:24:04 -0500
Subject: [PATCH 02/12] Added Earthfile condition and RPM spec fix
Dry run will now trigger for Earthfile changes. I also reordered the
changelog in veilid-server.spec to descending to correct that error.
Commented out the crates.io publishing dry run. This branch does not
have access to the protected variables.
---
.gitlab-ci.yml | 43 ++++++++++++--------
package/rpm/veilid-server/veilid-server.spec | 5 ++-
2 files changed, 29 insertions(+), 19 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d48765ff..bc1ddf12 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -180,7 +180,7 @@ delete_build_machines:
rules:
- if: '$CI_COMMIT_TAG =~ /v\d.+/'
-# Dryrun release -- triggered by changes in .gitlab-ci.yml or CICD scripts
+# Dryrun release -- triggered by changes in .gitlab-ci.yml, CICD scripts, or Earthfile
dryrun_create_build_machines:
stage: build_packages
@@ -195,6 +195,7 @@ dryrun_create_build_machines:
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
+ - Earthfile
dryrun_package_amd64_deb:
stage: build_packages
@@ -211,6 +212,7 @@ dryrun_package_amd64_deb:
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
+ - Earthfile
dryrun_package_arm64_deb:
stage: build_packages
@@ -227,6 +229,7 @@ dryrun_package_arm64_deb:
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
+ - Earthfile
dryrun_package_amd64_rpm:
stage: build_packages
@@ -243,22 +246,24 @@ dryrun_package_amd64_rpm:
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
+ - Earthfile
-dryrun_publish_crates:
- stage: build_packages
- needs:
- - dryrun_create_build_machines
- tags:
- - build-amd64-deb
- script:
- - vlt login
- - vlt run --command="cargo publish -p veilid-tools --dry-run"
- - vlt run --command="cargo publish -p veilid-core --dry-run"
- rules:
- - if: $CI_PIPELINE_SOURCE == "push"
- changes:
- - .gitlab-ci.yml
- - scripts/cicd/**/*
+# dryrun_publish_crates:
+# stage: build_packages
+# needs:
+# - dryrun_create_build_machines
+# tags:
+# - build-amd64-deb
+# script:
+# - vlt login
+# - vlt run --command="cargo publish -p veilid-tools --dry-run"
+# - vlt run --command="cargo publish -p veilid-core --dry-run"
+# rules:
+# - if: $CI_PIPELINE_SOURCE == "push"
+# changes:
+# - .gitlab-ci.yml
+# - scripts/cicd/**/*
+# - Earthfile
dryrun_publish_python:
stage: build_packages
@@ -274,6 +279,7 @@ dryrun_publish_python:
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
+ - Earthfile
dryrun_build_repositories:
stage: distribute
@@ -289,6 +295,7 @@ dryrun_build_repositories:
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
+ - Earthfile
dryrun_deploy_repos:
stage: distribute
@@ -303,6 +310,7 @@ dryrun_deploy_repos:
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
+ - Earthfile
dryrun_delete_build_machines:
stage: distribute
@@ -318,4 +326,5 @@ dryrun_delete_build_machines:
- if: $CI_PIPELINE_SOURCE == "push"
changes:
- .gitlab-ci.yml
- - scripts/cicd/**/*
\ No newline at end of file
+ - scripts/cicd/**/*
+ - Earthfile
\ No newline at end of file
diff --git a/package/rpm/veilid-server/veilid-server.spec b/package/rpm/veilid-server/veilid-server.spec
index 9d3d50d1..f8a881c2 100644
--- a/package/rpm/veilid-server/veilid-server.spec
+++ b/package/rpm/veilid-server/veilid-server.spec
@@ -57,7 +57,8 @@ else
fi
%changelog
-* Sun Jul 2 2023 TC <tc@veilid.org>
-- experimental RPM building
* Sun Apr 28 2024 Christien Rioux <chris@veilid.org>
- add ipc directory to installation
+* Sun Jul 2 2023 TC <tc@veilid.org>
+- experimental RPM building
+
From fa1ca77d35e5b10f65a81b3d109d76a05f46a4bc Mon Sep 17 00:00:00 2001
From: TC Johnson <tc@veilid.org>
Date: Mon, 29 Apr 2024 16:03:38 -0500
Subject: [PATCH 03/12] Further dry run trigger refinements
The dry run should now be configured if changes to .gitlab-ci.yml,
Earthfile, contents of scripts/cicd, or contents of package arrive
as the result of a push OR merge but NOT when a tag is present. I
think that will prevent the dry run executing alongside the actual
release process when a new release tag is pushed but that also
contains changes to the above mentioned files. <fingers crossed emoji>
---
.gitlab-ci.yml | 32 ++++++++++++++++++++------------
1 file changed, 20 insertions(+), 12 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bc1ddf12..cdc4e271 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -191,11 +191,12 @@ dryrun_create_build_machines:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm
rules:
- - if: $CI_PIPELINE_SOURCE == "push"
+ - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
- Earthfile
+ - package/**/*
dryrun_package_amd64_deb:
stage: build_packages
@@ -208,11 +209,12 @@ dryrun_package_amd64_deb:
- earthly +package-linux-amd64-deb
- bash scripts/cicd/build-machine/scp-to-orchestrator.sh
rules:
- - if: $CI_PIPELINE_SOURCE == "push"
+ - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
- Earthfile
+ - package/**/*
dryrun_package_arm64_deb:
stage: build_packages
@@ -225,12 +227,13 @@ dryrun_package_arm64_deb:
- earthly +package-linux-arm64-deb
- bash scripts/cicd/build-machine/scp-to-orchestrator.sh
rules:
- - if: $CI_PIPELINE_SOURCE == "push"
+ - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
- Earthfile
-
+ - package/**/*
+
dryrun_package_amd64_rpm:
stage: build_packages
needs:
@@ -242,11 +245,12 @@ dryrun_package_amd64_rpm:
- earthly +package-linux-amd64-rpm
- bash scripts/cicd/build-machine/scp-to-orchestrator.sh
rules:
- - if: $CI_PIPELINE_SOURCE == "push"
+ - ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
- Earthfile
+ - package/**/*
# dryrun_publish_crates:
# stage: build_packages
@@ -259,11 +263,12 @@ dryrun_package_amd64_rpm:
# - vlt run --command="cargo publish -p veilid-tools --dry-run"
# - vlt run --command="cargo publish -p veilid-core --dry-run"
# rules:
-# - if: $CI_PIPELINE_SOURCE == "push"
+# - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
# changes:
# - .gitlab-ci.yml
# - scripts/cicd/**/*
# - Earthfile
+# - package/**/*
dryrun_publish_python:
stage: build_packages
@@ -272,14 +277,14 @@ dryrun_publish_python:
tags:
- build-amd64-deb
script:
- - vlt login
- cd veilid-python && /home/gitlab-runner/.local/bin/poetry build
rules:
- - if: $CI_PIPELINE_SOURCE == "push"
+ - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
- Earthfile
+ - package/**/*
dryrun_build_repositories:
stage: distribute
@@ -291,11 +296,12 @@ dryrun_build_repositories:
- cp scripts/cicd/build-orchestration/generate-release.sh ~
- bash scripts/cicd/build-orchestration/distribute-packages.sh
rules:
- - if: $CI_PIPELINE_SOURCE == "push"
+ - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
- Earthfile
+ - package/**/*
dryrun_deploy_repos:
stage: distribute
@@ -306,11 +312,12 @@ dryrun_deploy_repos:
script:
- ls -al repo.tar
rules:
- - if: $CI_PIPELINE_SOURCE == "push"
+ - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
- Earthfile
+ - package/**/*
dryrun_delete_build_machines:
stage: distribute
@@ -323,8 +330,9 @@ dryrun_delete_build_machines:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
rules:
- - if: $CI_PIPELINE_SOURCE == "push"
+ - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
- - Earthfile
\ No newline at end of file
+ - Earthfile
+ - package/**/*
\ No newline at end of file
From 0a1a37f048cd0ed41935dceb1d340c7f5ef58af2 Mon Sep 17 00:00:00 2001
From: TC Johnson <tc@veilid.org>
Date: Mon, 29 Apr 2024 16:12:37 -0500
Subject: [PATCH 04/12] Replaced missing 'if:' on line 249.
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cdc4e271..ef1838bf 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -245,7 +245,7 @@ dryrun_package_amd64_rpm:
- earthly +package-linux-amd64-rpm
- bash scripts/cicd/build-machine/scp-to-orchestrator.sh
rules:
- - ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
+ - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
changes:
- .gitlab-ci.yml
- scripts/cicd/**/*
From bd4600576f2a9e7899f6db40229e002a61f463a5 Mon Sep 17 00:00:00 2001
From: TC Johnson <tc@veilid.org>
Date: Mon, 29 Apr 2024 16:22:58 -0500
Subject: [PATCH 05/12] Switched to regex trigger
The previous trigger condition wasn't liked by Gitlab's
yaml parser. I'm switching to trying a commit message
based regex trigger. This message does not contain the
trigger and so should not fire the dry run.
---
.gitlab-ci.yml | 56 ++++++++------------------------------------------
1 file changed, 8 insertions(+), 48 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ef1838bf..41f95a56 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -191,12 +191,7 @@ dryrun_create_build_machines:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm
rules:
- - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
- changes:
- - .gitlab-ci.yml
- - scripts/cicd/**/*
- - Earthfile
- - package/**/*
+ - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_package_amd64_deb:
stage: build_packages
@@ -209,12 +204,7 @@ dryrun_package_amd64_deb:
- earthly +package-linux-amd64-deb
- bash scripts/cicd/build-machine/scp-to-orchestrator.sh
rules:
- - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
- changes:
- - .gitlab-ci.yml
- - scripts/cicd/**/*
- - Earthfile
- - package/**/*
+ - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_package_arm64_deb:
stage: build_packages
@@ -227,12 +217,7 @@ dryrun_package_arm64_deb:
- earthly +package-linux-arm64-deb
- bash scripts/cicd/build-machine/scp-to-orchestrator.sh
rules:
- - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
- changes:
- - .gitlab-ci.yml
- - scripts/cicd/**/*
- - Earthfile
- - package/**/*
+ - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_package_amd64_rpm:
stage: build_packages
@@ -245,12 +230,7 @@ dryrun_package_amd64_rpm:
- earthly +package-linux-amd64-rpm
- bash scripts/cicd/build-machine/scp-to-orchestrator.sh
rules:
- - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
- changes:
- - .gitlab-ci.yml
- - scripts/cicd/**/*
- - Earthfile
- - package/**/*
+ - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
# dryrun_publish_crates:
# stage: build_packages
@@ -279,12 +259,7 @@ dryrun_publish_python:
script:
- cd veilid-python && /home/gitlab-runner/.local/bin/poetry build
rules:
- - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
- changes:
- - .gitlab-ci.yml
- - scripts/cicd/**/*
- - Earthfile
- - package/**/*
+ - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_build_repositories:
stage: distribute
@@ -296,12 +271,7 @@ dryrun_build_repositories:
- cp scripts/cicd/build-orchestration/generate-release.sh ~
- bash scripts/cicd/build-orchestration/distribute-packages.sh
rules:
- - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
- changes:
- - .gitlab-ci.yml
- - scripts/cicd/**/*
- - Earthfile
- - package/**/*
+ - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_deploy_repos:
stage: distribute
@@ -312,12 +282,7 @@ dryrun_deploy_repos:
script:
- ls -al repo.tar
rules:
- - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
- changes:
- - .gitlab-ci.yml
- - scripts/cicd/**/*
- - Earthfile
- - package/**/*
+ - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_delete_build_machines:
stage: distribute
@@ -330,9 +295,4 @@ dryrun_delete_build_machines:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
rules:
- - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
- changes:
- - .gitlab-ci.yml
- - scripts/cicd/**/*
- - Earthfile
- - package/**/*
\ No newline at end of file
+ - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
\ No newline at end of file
From 37dbe70e0570f6352c643c038890697a0374ea78 Mon Sep 17 00:00:00 2001
From: TC Johnson <tc@veilid.org>
Date: Mon, 29 Apr 2024 16:26:13 -0500
Subject: [PATCH 06/12] Testing dry run trigger
The trigger is [ci dryrun]
---
.gitlab-ci.yml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 41f95a56..2ef0b8c6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -295,4 +295,6 @@ dryrun_delete_build_machines:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
rules:
- - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
\ No newline at end of file
+ - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
+
+# Added to test commit message dry run trigger
\ No newline at end of file
From 4d9787fb86f4b96a5947c49db59c5bd3ac945cae Mon Sep 17 00:00:00 2001
From: TC Johnson <tc@veilid.org>
Date: Mon, 29 Apr 2024 17:07:40 -0500
Subject: [PATCH 07/12] Fixes scp from build machines to orchestrator
Each machine needed a script this specified that machine's arch
and OS type. Also added a rule on the cache and test jobs to not
run if the dry run trigger is present. [ci dryrun]
---
.gitlab-ci.yml | 20 ++++++++++++-------
...r.sh => scp-amd64-debs-to-orchestrator.sh} | 0
.../scp-amd64-rpms-to-orchestrator.sh | 9 +++++++++
.../scp-arm64-debs-to-orchestrator.sh | 9 +++++++++
4 files changed, 31 insertions(+), 7 deletions(-)
rename scripts/cicd/build-machine/{scp-to-orchestrator.sh => scp-amd64-debs-to-orchestrator.sh} (100%)
create mode 100644 scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
create mode 100644 scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2ef0b8c6..3dc2cd4a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -40,6 +40,10 @@ build_cache:
- else
- echo "No need to rebuild"
- fi
+ rules:
+ - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
+ - when: never
+
# Runs a basic unit test build, this task will use the `build-cache:latest` as set up in the projects Container Registry
test_build:
@@ -53,7 +57,9 @@ test_build:
- *earthly_setup
- earthly --use-inline-cache +unit-tests-linux --BASE=container $project_args
resource_group: test
- # when: manual
+ rules:
+ - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
+ - when: never
# Actual release -- triggered by pushing a new version tag
@@ -88,7 +94,7 @@ package_amd64_deb:
script:
- earthly bootstrap
- earthly +package-linux-amd64-deb
- - bash scripts/cicd/build-machine/scp-to-orchestrator.sh
+ - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
rules:
- if: '$CI_COMMIT_TAG =~ /v\d.+/'
@@ -101,7 +107,7 @@ package_arm64_deb:
script:
- earthly bootstrap
- earthly +package-linux-arm64-deb
- - bash scripts/cicd/build-machine/scp-to-orchestrator.sh
+ - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
rules:
- if: '$CI_COMMIT_TAG =~ /v\d.+/'
@@ -114,7 +120,7 @@ package_amd64_rpm:
script:
- earthly bootstrap
- earthly +package-linux-amd64-rpm
- - bash scripts/cicd/build-machine/scp-to-orchestrator.sh
+ - bash scripts/cicd/build-machine/scp-amd64-rmps-to-orchestrator.sh
rules:
- if: '$CI_COMMIT_TAG =~ /v\d.+/'
@@ -202,7 +208,7 @@ dryrun_package_amd64_deb:
script:
- earthly bootstrap
- earthly +package-linux-amd64-deb
- - bash scripts/cicd/build-machine/scp-to-orchestrator.sh
+ - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
@@ -215,7 +221,7 @@ dryrun_package_arm64_deb:
script:
- earthly bootstrap
- earthly +package-linux-arm64-deb
- - bash scripts/cicd/build-machine/scp-to-orchestrator.sh
+ - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
@@ -228,7 +234,7 @@ dryrun_package_amd64_rpm:
script:
- earthly bootstrap
- earthly +package-linux-amd64-rpm
- - bash scripts/cicd/build-machine/scp-to-orchestrator.sh
+ - bash scripts/cicd/build-machine/scp-amd64-rmps-to-orchestrator.sh
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
diff --git a/scripts/cicd/build-machine/scp-to-orchestrator.sh b/scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
similarity index 100%
rename from scripts/cicd/build-machine/scp-to-orchestrator.sh
rename to scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
diff --git a/scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh b/scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
new file mode 100644
index 00000000..1f616f8d
--- /dev/null
+++ b/scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+mkdir workspace
+cd workspace
+cp ~/builds/C6yRimG-M/0/veilid/veilid/target/packages/*.rpm .
+tar -cf amd64-rpms.tar *.rpm
+scp *.tar gitlab-runner@10.116.0.5:~
+cd ~
+rm -rf workspace
\ No newline at end of file
diff --git a/scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh b/scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
new file mode 100644
index 00000000..7216ec59
--- /dev/null
+++ b/scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+mkdir workspace
+cd workspace
+cp ~/builds/7TYBLKUtG/0/veilid/veilid/target/packages/*.deb .
+tar -cf arm64-debs.tar *.deb
+scp *.tar gitlab-runner@10.116.0.5:~
+cd ~
+rm -rf workspace
\ No newline at end of file
From 31347f3ce077661363cee03f177a238a25af90f2 Mon Sep 17 00:00:00 2001
From: TC <tc@geekministry.org>
Date: Mon, 29 Apr 2024 22:15:19 +0000
Subject: [PATCH 08/12] [ci dryrun]
---
.gitlab-ci.yml | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3dc2cd4a..117306cb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -49,17 +49,16 @@ build_cache:
test_build:
extends: .base
stage: test
- only:
- - main
- - pushes
- - merge_requests
script:
- *earthly_setup
- earthly --use-inline-cache +unit-tests-linux --BASE=container $project_args
resource_group: test
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
- - when: never
+ when: never
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+ - if: $CI_PIPELINE_SOURCE == "push"
+ - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
# Actual release -- triggered by pushing a new version tag
From 058df249339c6da5a9467f010bfb75971d5fc09e Mon Sep 17 00:00:00 2001
From: TC <tc@geekministry.org>
Date: Mon, 29 Apr 2024 22:16:38 +0000
Subject: [PATCH 09/12] [ci dryrun]
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 117306cb..5da597ff 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -42,7 +42,7 @@ build_cache:
- fi
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
- - when: never
+ when: never
# Runs a basic unit test build, this task will use the `build-cache:latest` as set up in the projects Container Registry
From 59d261b899a2e869047430b7b7f008dbbcf965f9 Mon Sep 17 00:00:00 2001
From: TC <tc@geekministry.org>
Date: Mon, 29 Apr 2024 22:34:52 +0000
Subject: [PATCH 10/12] [ci dryrun]
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5da597ff..ff8fc6bf 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -119,7 +119,7 @@ package_amd64_rpm:
script:
- earthly bootstrap
- earthly +package-linux-amd64-rpm
- - bash scripts/cicd/build-machine/scp-amd64-rmps-to-orchestrator.sh
+ - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
rules:
- if: '$CI_COMMIT_TAG =~ /v\d.+/'
@@ -233,7 +233,7 @@ dryrun_package_amd64_rpm:
script:
- earthly bootstrap
- earthly +package-linux-amd64-rpm
- - bash scripts/cicd/build-machine/scp-amd64-rmps-to-orchestrator.sh
+ - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
From c3035a040611e48f0744da924854563b57aa1cac Mon Sep 17 00:00:00 2001
From: TC <tc@geekministry.org>
Date: Tue, 30 Apr 2024 00:02:59 +0000
Subject: [PATCH 11/12] Uncommented crates.io section.
---
.gitlab-ci.yml | 36 +++++++++++++++++-------------------
1 file changed, 17 insertions(+), 19 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ff8fc6bf..60919f5b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -237,23 +237,23 @@ dryrun_package_amd64_rpm:
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
-# dryrun_publish_crates:
-# stage: build_packages
-# needs:
-# - dryrun_create_build_machines
-# tags:
-# - build-amd64-deb
-# script:
-# - vlt login
-# - vlt run --command="cargo publish -p veilid-tools --dry-run"
-# - vlt run --command="cargo publish -p veilid-core --dry-run"
-# rules:
-# - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
-# changes:
-# - .gitlab-ci.yml
-# - scripts/cicd/**/*
-# - Earthfile
-# - package/**/*
+ dryrun_publish_crates:
+ stage: build_packages
+ needs:
+ - dryrun_create_build_machines
+ tags:
+ - build-amd64-deb
+ script:
+ - vlt login
+ - vlt run --command="cargo publish -p veilid-tools --dry-run"
+ - vlt run --command="cargo publish -p veilid-core --dry-run"
+ rules:
+ - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
+ changes:
+ - .gitlab-ci.yml
+ - scripts/cicd/**/*
+ - Earthfile
+ - package/**/*
dryrun_publish_python:
stage: build_packages
@@ -301,5 +301,3 @@ dryrun_delete_build_machines:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
-
-# Added to test commit message dry run trigger
\ No newline at end of file
From 6a7afe9017ccf39d7844edc54cda00f368f32673 Mon Sep 17 00:00:00 2001
From: TC <tc@geekministry.org>
Date: Tue, 30 Apr 2024 00:04:30 +0000
Subject: [PATCH 12/12] Uncommented crates.io section.
---
.gitlab-ci.yml | 29 ++++++++++++-----------------
1 file changed, 12 insertions(+), 17 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 60919f5b..bd016d37 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -237,23 +237,18 @@ dryrun_package_amd64_rpm:
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
- dryrun_publish_crates:
- stage: build_packages
- needs:
- - dryrun_create_build_machines
- tags:
- - build-amd64-deb
- script:
- - vlt login
- - vlt run --command="cargo publish -p veilid-tools --dry-run"
- - vlt run --command="cargo publish -p veilid-core --dry-run"
- rules:
- - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL
- changes:
- - .gitlab-ci.yml
- - scripts/cicd/**/*
- - Earthfile
- - package/**/*
+dryrun_publish_crates:
+ stage: build_packages
+ needs:
+ - dryrun_create_build_machines
+ tags:
+ - build-amd64-deb
+ script:
+ - vlt login
+ - vlt run --command="cargo publish -p veilid-tools --dry-run"
+ - vlt run --command="cargo publish -p veilid-core --dry-run"
+ rules:
+ - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_publish_python:
stage: build_packages