CI: Additional changes for +security versions (#94854)

* Build: Fix docker manifest create not using correct IMAGE_TAG

* Support publishing security versions of NPM packages

---------

Co-authored-by: Andreas Christou <andreas.christou@grafana.com>
Co-authored-by: Kevin Minehart <kmineh0151@gmail.com>
Co-authored-by: Diego Augusto Molina <diegoaugustomolina@gmail.com>
This commit is contained in:
Josh Hunt 2024-10-17 20:13:42 +01:00 committed by GitHub
parent 125b7c2fd8
commit 8f7352e862
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 96 additions and 54 deletions

View File

@ -539,7 +539,7 @@ steps:
name: identify-runner
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@ -995,7 +995,7 @@ steps:
name: clone-enterprise
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@ -1957,7 +1957,7 @@ steps:
name: identify-runner
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@ -2510,7 +2510,7 @@ services:
steps:
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@ -2715,7 +2715,7 @@ steps:
name: identify-runner
- commands:
- $$ProgressPreference = "SilentlyContinue"
- Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/windows/grabpl.exe
- Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/windows/grabpl.exe
-OutFile grabpl.exe
image: grafana/ci-wix:0.1.1
name: windows-init
@ -3142,7 +3142,7 @@ services:
steps:
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@ -3387,7 +3387,7 @@ steps:
name: identify-runner
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@ -3433,9 +3433,9 @@ steps:
$$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
# Create the grafana manifests
$$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7
$$debug docker manifest create grafana/grafana:$${IMAGE_TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7
$$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
$$debug docker manifest create grafana/grafana:$${IMAGE_TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
# Push the grafana manifests
$$debug docker manifest push grafana/grafana:$${IMAGE_TAG}
@ -3519,7 +3519,7 @@ steps:
name: identify-runner
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@ -3565,9 +3565,9 @@ steps:
$$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
# Create the grafana manifests
$$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7
$$debug docker manifest create grafana/grafana:$${IMAGE_TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7
$$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
$$debug docker manifest create grafana/grafana:$${IMAGE_TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
# Push the grafana manifests
$$debug docker manifest push grafana/grafana:$${IMAGE_TAG}
@ -3829,6 +3829,7 @@ platform:
services: []
steps:
- commands:
- export version=$(echo ${TAG} | sed -e "s/+security-/-/g")
- 'echo "Step 1: Updating package lists..."'
- apt-get update >/dev/null 2>&1
- 'echo "Step 2: Installing prerequisites..."'
@ -3844,7 +3845,7 @@ steps:
- 'echo "Step 5: Installing Grafana..."'
- for i in $(seq 1 60); do
- ' if apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get
install -yq grafana=${TAG} >/dev/null 2>&1; then'
install -yq grafana=$version >/dev/null 2>&1; then'
- ' echo "Command succeeded on attempt $i"'
- ' break'
- ' else'
@ -3858,10 +3859,10 @@ steps:
- ' fi'
- done
- 'echo "Step 6: Verifying Grafana installation..."'
- 'if dpkg -s grafana | grep -q "Version: ${TAG}"; then'
- ' echo "Successfully verified Grafana version ${TAG}"'
- 'if dpkg -s grafana | grep -q "Version: $version"; then'
- ' echo "Successfully verified Grafana version $version"'
- else
- ' echo "Failed to verify Grafana version ${TAG}"'
- ' echo "Failed to verify Grafana version $version"'
- ' exit 1'
- fi
- echo "Verification complete."
@ -3889,11 +3890,12 @@ steps:
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
' > /etc/yum.repos.d/grafana.repo
- 'echo "Step 5: Checking RPM repository..."'
- dnf list available grafana-${TAG}
- export version=$(echo "${TAG}" | sed -e "s/+security-/^security_/g")
- dnf list available grafana-$version
- if [ $? -eq 0 ]; then
- ' echo "Grafana package found in repository. Installing from repo..."'
- for i in $(seq 1 60); do
- ' if dnf install -y --nogpgcheck grafana-${TAG} >/dev/null 2>&1; then'
- ' if dnf install -y --nogpgcheck grafana-$version >/dev/null 2>&1; then'
- ' echo "Command succeeded on attempt $i"'
- ' break'
- ' else'
@ -3910,16 +3912,16 @@ steps:
- ' rpm --import https://rpm.grafana.com/gpg.key'
- ' rpm -qa gpg-pubkey* | xargs rpm -qi | grep -i grafana'
- else
- ' echo "Grafana package version ${TAG} not found in repository."'
- ' echo "Grafana package version $version not found in repository."'
- ' dnf repolist'
- ' dnf list available grafana*'
- ' exit 1'
- fi
- 'echo "Step 6: Verifying Grafana installation..."'
- if rpm -q grafana | grep -q "${TAG}"; then
- ' echo "Successfully verified Grafana version ${TAG}"'
- if rpm -q grafana | grep -q "$verison"; then
- ' echo "Successfully verified Grafana version $version"'
- else
- ' echo "Failed to verify Grafana version ${TAG}"'
- ' echo "Failed to verify Grafana version $version"'
- ' exit 1'
- fi
- echo "Verification complete."
@ -4006,6 +4008,7 @@ steps:
from_secret: packages_service_account
target_bucket: grafana-packages
- commands:
- export version=$(echo ${TAG} | sed -e "s/+security-/-/g")
- 'echo "Step 1: Updating package lists..."'
- apt-get update >/dev/null 2>&1
- 'echo "Step 2: Installing prerequisites..."'
@ -4021,7 +4024,7 @@ steps:
- 'echo "Step 5: Installing Grafana..."'
- for i in $(seq 1 60); do
- ' if apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get
install -yq grafana=${TAG} >/dev/null 2>&1; then'
install -yq grafana=$version >/dev/null 2>&1; then'
- ' echo "Command succeeded on attempt $i"'
- ' break'
- ' else'
@ -4035,10 +4038,10 @@ steps:
- ' fi'
- done
- 'echo "Step 6: Verifying Grafana installation..."'
- 'if dpkg -s grafana | grep -q "Version: ${TAG}"; then'
- ' echo "Successfully verified Grafana version ${TAG}"'
- 'if dpkg -s grafana | grep -q "Version: $version"; then'
- ' echo "Successfully verified Grafana version $version"'
- else
- ' echo "Failed to verify Grafana version ${TAG}"'
- ' echo "Failed to verify Grafana version $version"'
- ' exit 1'
- fi
- echo "Verification complete."
@ -4067,11 +4070,12 @@ steps:
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
' > /etc/yum.repos.d/grafana.repo
- 'echo "Step 5: Checking RPM repository..."'
- dnf list available grafana-${TAG}
- export version=$(echo "${TAG}" | sed -e "s/+security-/^security_/g")
- dnf list available grafana-$version
- if [ $? -eq 0 ]; then
- ' echo "Grafana package found in repository. Installing from repo..."'
- for i in $(seq 1 60); do
- ' if dnf install -y --nogpgcheck grafana-${TAG} >/dev/null 2>&1; then'
- ' if dnf install -y --nogpgcheck grafana-$version >/dev/null 2>&1; then'
- ' echo "Command succeeded on attempt $i"'
- ' break'
- ' else'
@ -4088,16 +4092,16 @@ steps:
- ' rpm --import https://rpm.grafana.com/gpg.key'
- ' rpm -qa gpg-pubkey* | xargs rpm -qi | grep -i grafana'
- else
- ' echo "Grafana package version ${TAG} not found in repository."'
- ' echo "Grafana package version $version not found in repository."'
- ' dnf repolist'
- ' dnf list available grafana*'
- ' exit 1'
- fi
- 'echo "Step 6: Verifying Grafana installation..."'
- if rpm -q grafana | grep -q "${TAG}"; then
- ' echo "Successfully verified Grafana version ${TAG}"'
- if rpm -q grafana | grep -q "$verison"; then
- ' echo "Successfully verified Grafana version $version"'
- else
- ' echo "Failed to verify Grafana version ${TAG}"'
- ' echo "Failed to verify Grafana version $version"'
- ' exit 1'
- fi
- echo "Verification complete."
@ -4395,7 +4399,7 @@ steps:
name: identify-runner
- commands:
- $$ProgressPreference = "SilentlyContinue"
- Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/windows/grabpl.exe
- Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/windows/grabpl.exe
-OutFile grabpl.exe
image: grafana/ci-wix:0.1.1
name: windows-init
@ -5195,7 +5199,7 @@ services:
steps:
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@ -5987,6 +5991,6 @@ kind: secret
name: gcr_credentials
---
kind: signature
hmac: dc30a3a00ee542fb289da36ef6db4274684db4533c472f7f903468919d1046ac
hmac: 41df5b1fdbd1b3c9aa915919ae5be16d2a188cbaf6b243c14fd66f94db0db8d8
...

View File

@ -145,6 +145,9 @@ func Builds(baseURL *url.URL, grafana, version string, packages []packaging.Buil
if arch == "aarch64" {
arch = "arm64"
}
if arch == "x86_64" {
arch = "amd64"
}
}
if v.Distro == "deb" {

View File

@ -2,6 +2,7 @@ package main
import (
"fmt"
"log"
"os"
"strings"
@ -23,6 +24,11 @@ func NpmRetrieveAction(c *cli.Context) error {
return fmt.Errorf("no tag version specified, exitting")
}
if strings.Contains(tag, "security") {
log.Printf("skipping npm publish because version '%s' has 'security'", tag)
return nil
}
prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET"))
if prereleaseBucket == "" {
return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1)
@ -48,6 +54,11 @@ func NpmStoreAction(c *cli.Context) error {
return fmt.Errorf("no tag version specified, exiting")
}
if strings.Contains(tag, "security") {
log.Printf("skipping npm publish because version '%s' has 'security'", tag)
return nil
}
prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET"))
if prereleaseBucket == "" {
return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1)
@ -73,6 +84,11 @@ func NpmReleaseAction(c *cli.Context) error {
return fmt.Errorf("no tag version specified, exitting")
}
if strings.Contains(tag, "security") {
log.Printf("skipping npm publish because version '%s' has 'security'", tag)
return nil
}
err := npm.PublishNpmPackages(c.Context, tag)
if err != nil {
return err

View File

@ -113,6 +113,16 @@ var ARMArtifacts = []BuildArtifact{
Arch: "armv7",
Ext: "tar.gz",
},
{
Distro: "linux",
Arch: "arm64",
Ext: "tar.gz",
},
{
Distro: "linux",
Arch: "amd64",
Ext: "tar.gz",
},
}
func join(a []BuildArtifact, b ...[]BuildArtifact) []BuildArtifact {

View File

@ -13,15 +13,17 @@ import (
)
var (
reGrafanaTag = regexp.MustCompile(`^v(\d+\.\d+\.\d+$)`)
reGrafanaTagPreview = regexp.MustCompile(`^v(\d+\.\d+\.\d+-preview)`)
reGrafanaTagCustom = regexp.MustCompile(`^v(\d+\.\d+\.\d+-\w+)`)
reGrafanaTag = regexp.MustCompile(`^v(\d+\.\d+\.\d+$)`)
reGrafanaTagPreview = regexp.MustCompile(`^v(\d+\.\d+\.\d+-preview)`)
reGrafanaTagCustom = regexp.MustCompile(`^v(\d+\.\d+\.\d+-\w+)`)
reGrafanaTagSecurity = regexp.MustCompile(`^v(\d+\.\d+\.\d+\+\w+\-\d+)`)
)
const (
Latest = "latest"
Next = "next"
Test = "test"
Latest = "latest"
Next = "next"
Test = "test"
Security = "security"
)
type Version struct {
@ -152,6 +154,11 @@ func GetVersion(tag string) (*Version, error) {
Version: reGrafanaTagCustom.FindStringSubmatch(tag)[1],
Channel: Test,
}
case reGrafanaTagSecurity.MatchString(tag):
version = Version{
Version: reGrafanaTagSecurity.FindStringSubmatch(tag)[1],
Channel: Security,
}
default:
return nil, fmt.Errorf("%s not a supported Grafana version, exitting", tag)
}

View File

@ -45,12 +45,12 @@ def publish_image_public_step():
$$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
# Create the grafana manifests
$$debug docker manifest create grafana/grafana:${TAG} \
$$debug docker manifest create grafana/grafana:$${IMAGE_TAG} \
grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 \
grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 \
grafana/grafana-image-tags:$${IMAGE_TAG}-armv7
$$debug docker manifest create grafana/grafana:${TAG}-ubuntu \
$$debug docker manifest create grafana/grafana:$${IMAGE_TAG}-ubuntu \
grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 \
grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 \
grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7

View File

@ -1283,13 +1283,14 @@ def retry_command(command, attempts = 60, delay = 30):
]
def verify_linux_DEB_packages_step(depends_on = []):
install_command = "apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get install -yq grafana=${TAG} >/dev/null 2>&1"
install_command = "apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get install -yq grafana=$version >/dev/null 2>&1"
return {
"name": "verify-linux-DEB-packages",
"image": images["ubuntu"],
"environment": {},
"commands": [
'export version=$(echo ${TAG} | sed -e "s/+security-/-/g")',
'echo "Step 1: Updating package lists..."',
"apt-get update >/dev/null 2>&1",
'echo "Step 2: Installing prerequisites..."',
@ -1303,10 +1304,10 @@ def verify_linux_DEB_packages_step(depends_on = []):
# The packages take a bit of time to propogate within the repo. This retry will check their availability within 10 minutes.
] + retry_command(install_command) + [
'echo "Step 6: Verifying Grafana installation..."',
'if dpkg -s grafana | grep -q "Version: ${TAG}"; then',
' echo "Successfully verified Grafana version ${TAG}"',
'if dpkg -s grafana | grep -q "Version: $version"; then',
' echo "Successfully verified Grafana version $version"',
"else",
' echo "Failed to verify Grafana version ${TAG}"',
' echo "Failed to verify Grafana version $version"',
" exit 1",
"fi",
'echo "Verification complete."',
@ -1327,7 +1328,7 @@ def verify_linux_RPM_packages_step(depends_on = []):
"sslcacert=/etc/pki/tls/certs/ca-bundle.crt\n"
)
install_command = "dnf install -y --nogpgcheck grafana-${TAG} >/dev/null 2>&1"
install_command = "dnf install -y --nogpgcheck grafana-$version >/dev/null 2>&1"
return {
"name": "verify-linux-RPM-packages",
@ -1343,7 +1344,8 @@ def verify_linux_RPM_packages_step(depends_on = []):
'echo "Step 4: Configuring Grafana repository..."',
"echo -e '" + repo_config + "' > /etc/yum.repos.d/grafana.repo",
'echo "Step 5: Checking RPM repository..."',
"dnf list available grafana-${TAG}",
'export version=$(echo "${TAG}" | sed -e "s/+security-/^security_/g")',
"dnf list available grafana-$version",
"if [ $? -eq 0 ]; then",
' echo "Grafana package found in repository. Installing from repo..."',
] + retry_command(install_command) + [
@ -1351,16 +1353,16 @@ def verify_linux_RPM_packages_step(depends_on = []):
" rpm --import https://rpm.grafana.com/gpg.key",
" rpm -qa gpg-pubkey* | xargs rpm -qi | grep -i grafana",
"else",
' echo "Grafana package version ${TAG} not found in repository."',
' echo "Grafana package version $version not found in repository."',
" dnf repolist",
" dnf list available grafana*",
" exit 1",
"fi",
'echo "Step 6: Verifying Grafana installation..."',
'if rpm -q grafana | grep -q "${TAG}"; then',
' echo "Successfully verified Grafana version ${TAG}"',
'if rpm -q grafana | grep -q "$verison"; then',
' echo "Successfully verified Grafana version $version"',
"else",
' echo "Failed to verify Grafana version ${TAG}"',
' echo "Failed to verify Grafana version $version"',
" exit 1",
"fi",
'echo "Verification complete."',

View File

@ -2,7 +2,7 @@
global variables
"""
grabpl_version = "v3.0.53"
grabpl_version = "v3.0.56"
golang_version = "1.23.1"
# nodejs_version should match what's in ".nvmrc", but without the v prefix.