CI: Add artifacts npm commands from grabpl (#61908)

This commit is contained in:
Horst Gutmann 2023-01-23 11:32:51 +01:00 committed by GitHub
parent 0be920e61c
commit e32cd6d4ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 645 additions and 13 deletions

View File

@ -2214,8 +2214,9 @@ steps:
event:
- tag
- commands:
- ./bin/grabpl artifacts npm store --tag ${DRONE_TAG}
- ./bin/build artifacts npm store --tag ${DRONE_TAG}
depends_on:
- compile-build-cmd
- build-frontend-packages
environment:
GCP_KEY:
@ -4352,19 +4353,21 @@ platform:
services: []
steps:
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.20/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
- go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
depends_on: []
environment:
CGO_ENABLED: 0
image: golang:1.19.4
name: compile-build-cmd
- commands:
- yarn install --immutable
depends_on: []
image: grafana/build-container:1.6.6
name: yarn-install
- commands:
- ./bin/grabpl artifacts npm retrieve --tag ${DRONE_TAG}
- ./bin/build artifacts npm retrieve --tag ${DRONE_TAG}
depends_on:
- compile-build-cmd
- yarn-install
environment:
GCP_KEY:
@ -4375,8 +4378,9 @@ steps:
image: grafana/grafana-ci-deploy:1.3.3
name: retrieve-npm-packages
- commands:
- ./bin/grabpl artifacts npm release --tag ${DRONE_TAG}
- ./bin/build artifacts npm release --tag ${DRONE_TAG}
depends_on:
- compile-build-cmd
- retrieve-npm-packages
environment:
NPM_TOKEN:
@ -6439,6 +6443,6 @@ kind: secret
name: aws_secret_access_key
---
kind: signature
hmac: 7f74588652e48e6c51cd54fb9973289c144f09d2f232fe30fe66b5d51b9b317e
hmac: 59d802aec2892f9bdd89caaa68d0a563cf14cedefe9e76646f3b734e69afa40d
...

View File

@ -210,6 +210,46 @@ func main() {
},
},
},
{
Name: "npm",
Usage: "Handle Grafana npm packages",
Subcommands: cli.Commands{
{
Name: "release",
Usage: "Release npm packages",
ArgsUsage: "[version]",
Action: NpmReleaseAction,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "tag",
Usage: "Grafana version tag",
},
},
},
{
Name: "store",
Usage: "Store npm packages tarball",
Action: NpmStoreAction,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "tag",
Usage: "Grafana version tag",
},
},
},
{
Name: "retrieve",
Usage: "Retrieve npm packages tarball",
Action: NpmRetrieveAction,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "tag",
Usage: "Grafana version tag",
},
},
},
},
},
},
},
{

88
pkg/build/cmd/npm.go Normal file
View File

@ -0,0 +1,88 @@
package main
import (
"fmt"
"os"
"os/exec"
"strings"
"github.com/grafana/grafana/pkg/build/npm"
"github.com/urfave/cli/v2"
)
func NpmRetrieveAction(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
tag := c.String("tag")
if tag == "" {
return fmt.Errorf("no tag version specified, exitting")
}
prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET"))
if prereleaseBucket == "" {
return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1)
}
err := npm.FetchNpmPackages(c.Context, tag, prereleaseBucket)
if err != nil {
return err
}
return nil
}
func NpmStoreAction(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
tag := c.String("tag")
if tag == "" {
return fmt.Errorf("no tag version specified, exiting")
}
prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET"))
if prereleaseBucket == "" {
return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1)
}
err := npm.StoreNpmPackages(c.Context, tag, prereleaseBucket)
if err != nil {
return err
}
return nil
}
func NpmReleaseAction(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
tag := c.String("tag")
if tag == "" {
return fmt.Errorf("no tag version specified, exitting")
}
cmd := exec.Command("git", "checkout", ".")
if err := cmd.Run(); err != nil {
fmt.Println("command failed to run, err: ", err)
return err
}
err := npm.PublishNpmPackages(c.Context, tag)
if err != nil {
return err
}
return nil
}

View File

@ -1,6 +1,7 @@
package lerna
import (
"context"
"encoding/json"
"fmt"
"os"
@ -9,6 +10,7 @@ import (
"strings"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/fsutil"
)
// BuildFrontendPackages will bump the version for the package to the latest canary build
@ -56,3 +58,29 @@ func GetLernaVersion(grafanaDir string) (string, error) {
}
return strings.TrimSpace(version), nil
}
func PackFrontendPackages(ctx context.Context, tag, grafanaDir, artifactsDir string) error {
exists, err := fsutil.Exists(artifactsDir)
if err != nil {
return err
}
if exists {
err = os.RemoveAll(artifactsDir)
if err != nil {
return err
}
}
// nolint:gosec
if err = os.MkdirAll(artifactsDir, 0755); err != nil {
return err
}
// nolint:gosec
cmd := exec.CommandContext(ctx, "yarn", "lerna", "exec", "--no-private", "--", "yarn", "pack", "--out", fmt.Sprintf("../../npm-artifacts/%%s-%v.tgz", tag))
cmd.Dir = grafanaDir
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), output, err)
}
return nil
}

240
pkg/build/npm/npm.go Normal file
View File

@ -0,0 +1,240 @@
package npm
import (
"context"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/build/gcloud/storage"
"github.com/grafana/grafana/pkg/build/lerna"
"github.com/grafana/grafana/pkg/build/versions"
)
const GrafanaDir = "."
const NpmArtifactDir = "./npm-artifacts"
// TODO: could this be replaced by `yarn lerna list -p` ?
var packages = []string{
"@grafana/ui",
"@grafana/data",
"@grafana/toolkit",
"@grafana/runtime",
"@grafana/e2e",
"@grafana/e2e-selectors",
"@grafana/schema",
}
// PublishNpmPackages will publish local NPM packages to NPM registry.
func PublishNpmPackages(ctx context.Context, tag string) error {
version, err := versions.GetVersion(tag)
if err != nil {
return err
}
log.Printf("Grafana version: %s", version.Version)
if err := setNpmCredentials(); err != nil {
return err
}
npmArtifacts, err := storage.ListLocalFiles(NpmArtifactDir)
if err != nil {
return err
}
for _, packedFile := range npmArtifacts {
// nolint:gosec
cmd := exec.CommandContext(ctx, "npm", "publish", packedFile.FullPath, "--tag", version.Channel)
cmd.Dir = GrafanaDir
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err)
}
}
return updateTag(ctx, version, tag)
}
// StoreNpmPackages will store local NPM packages in GCS bucket `bucketName`.
func StoreNpmPackages(ctx context.Context, tag, bucketName string) error {
err := lerna.PackFrontendPackages(ctx, tag, GrafanaDir, NpmArtifactDir)
if err != nil {
return err
}
gcs, err := storage.New()
if err != nil {
return err
}
bucket := gcs.Bucket(bucketName)
bucketPath := fmt.Sprintf("artifacts/npm/%s/", tag)
if err = gcs.CopyLocalDir(ctx, NpmArtifactDir, bucket, bucketPath, true); err != nil {
return err
}
log.Print("Successfully stored npm packages!")
return nil
}
// FetchNpmPackages will store NPM packages stored in GCS bucket `bucketName` on local disk in `frontend.NpmArtifactDir`.
func FetchNpmPackages(ctx context.Context, tag, bucketName string) error {
gcs, err := storage.New()
if err != nil {
return err
}
bucketPath := fmt.Sprintf("artifacts/npm/%s/", tag)
bucket := gcs.Bucket(bucketName)
err = gcs.DownloadDirectory(ctx, bucket, NpmArtifactDir, storage.FilesFilter{
Prefix: bucketPath,
FileExts: []string{".tgz"},
})
if err != nil {
return err
}
return nil
}
// updateTag will move next or latest npm dist-tags, if needed.
//
// Note: This function makes the assumption that npm dist-tags has already
// been updated and hence why move of dist-tags not always happens:
//
// If stable the dist-tag latest was used.
// If beta the dist-tag next was used.
//
// Scenarios:
//
// 1. Releasing a newer stable than the current stable
// Latest and next is 9.1.5.
// 9.1.6 is released, latest and next should point to 9.1.6.
// The next dist-tag is moved to point to 9.1.6.
//
// 2. Releasing during an active beta period:
// Latest and next is 9.1.6.
// 9.2.0-beta1 is released, the latest should stay on 9.1.6, next should point to 9.2.0-beta1
// No move of dist-tags
// 9.1.7 is relased, the latest should point to 9.1.7, next should stay to 9.2.0-beta1
// No move of dist-tags
// Next week 9.2.0-beta2 is released, the latest should point to 9.1.7, next should point to 9.2.0-beta2
// No move of dist-tags
// In two weeks 9.2.0 stable is relased, the latest and next should point to 9.2.0.
// The next dist-tag is moved to point to 9.2.0.
//
// 3. Releasing an older stable than the current stable
// Latest and next is 9.2.0.
// Next 9.1.8 is released, latest should point to 9.2.0, next should point to 9.2.0
// The latest dist-tag is moved to point to 9.2.0.
func updateTag(ctx context.Context, version *versions.Version, releaseVersion string) error {
if version.Channel != versions.Latest {
return nil
}
latestStableVersion, err := getLatestStableVersion()
if err != nil {
return err
}
betaVersion, err := getLatestBetaVersion()
if err != nil {
return err
}
isLatest, err := versions.IsGreaterThanOrEqual(releaseVersion, latestStableVersion)
if err != nil {
return err
}
isNewerThanLatestBeta, err := versions.IsGreaterThanOrEqual(releaseVersion, betaVersion)
if err != nil {
return err
}
for _, pkg := range packages {
if !isLatest {
err = runMoveLatestNPMTagCommand(ctx, pkg, latestStableVersion)
if err != nil {
return err
}
}
if isLatest && isNewerThanLatestBeta {
err = runMoveNextNPMTagCommand(ctx, pkg, version.Version)
if err != nil {
return err
}
}
}
return nil
}
func getLatestStableVersion() (string, error) {
return versions.GetLatestVersion(versions.LatestStableVersionURL)
}
func getLatestBetaVersion() (string, error) {
return versions.GetLatestVersion(versions.LatestBetaVersionURL)
}
func runMoveNextNPMTagCommand(ctx context.Context, pkg string, packageVersion string) error {
// nolint:gosec
cmd := exec.CommandContext(ctx, "npm", "dist-tag", "add", fmt.Sprintf("%s@%s", pkg, packageVersion), "next")
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err)
}
return nil
}
func runMoveLatestNPMTagCommand(ctx context.Context, pkg string, latestStableVersion string) error {
// nolint:gosec
cmd := exec.CommandContext(ctx, "npm", "dist-tag", "add", fmt.Sprintf("%s@%s", pkg, latestStableVersion), "latest")
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err)
}
return nil
}
// setNpmCredentials Creates a .npmrc file in the users home folder and writes the
// necessary credentials to it for publishing packages to the NPM registry.
func setNpmCredentials() error {
npmToken := strings.TrimSpace(os.Getenv("NPM_TOKEN"))
if npmToken == "" {
return fmt.Errorf("npm token is not set")
}
homeDir, err := os.UserHomeDir()
if err != nil {
return fmt.Errorf("failed to obtain home directory, err: %q", err)
}
npmPath := filepath.Join(homeDir, ".npmrc")
registry := []byte(fmt.Sprintf("//registry.npmjs.org/:_authToken=%s", npmToken))
if _, err = os.Stat(npmPath); os.IsNotExist(err) {
// nolint:gosec
f, err := os.Create(npmPath)
if err != nil {
return fmt.Errorf("couldn't create npmrc file, err: %q", err)
}
_, err = f.Write(registry)
if err != nil {
return fmt.Errorf("failed to write to file, err: %q", err)
}
defer func() {
if err := f.Close(); err != nil {
log.Printf("Failed to close file: %s", err.Error())
}
}()
} else {
err = os.WriteFile(npmPath, registry, 0644)
if err != nil {
return fmt.Errorf("error writing to file, err: %q", err)
}
}
return nil
}

View File

@ -0,0 +1,160 @@
package versions
import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"regexp"
"strconv"
"github.com/Masterminds/semver/v3"
)
var (
reGrafanaTag = regexp.MustCompile(`^v(\d+\.\d+\.\d+$)`)
reGrafanaTagBeta = regexp.MustCompile(`^v(\d+\.\d+\.\d+-beta)`)
reGrafanaTagCustom = regexp.MustCompile(`^v(\d+\.\d+\.\d+-\w+)`)
)
const (
Latest = "latest"
Next = "next"
Test = "test"
)
type Version struct {
Version string
Channel string
}
type VersionFromAPI struct {
Version string `json:"version"`
}
type LatestGcomAPI = string
const (
LatestStableVersionURL LatestGcomAPI = "https://grafana.com/api/grafana/versions/stable"
LatestBetaVersionURL LatestGcomAPI = "https://grafana.com/api/grafana/versions/beta"
)
func GetLatestVersion(url LatestGcomAPI) (string, error) {
// nolint:gosec
resp, err := http.Get(url)
if err != nil {
return "", err
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Printf("Failed to close body: %s", err.Error())
}
}()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("server returned non 200 status code: %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
var apiResponse VersionFromAPI
err = json.Unmarshal(body, &apiResponse)
if err != nil {
return "", err
}
return apiResponse.Version, nil
}
// IsGreaterThanOrEqual semantically checks whether newVersion is greater than or equal to stableVersion.
func IsGreaterThanOrEqual(newVersion, stableVersion string) (bool, error) {
v1SemVer, err := semver.NewVersion(newVersion)
if err != nil {
return isGreaterThanOrEqualFourDigit(newVersion, stableVersion)
}
v2SemVer, err := semver.NewVersion(stableVersion)
if err != nil {
return isGreaterThanOrEqualFourDigit(newVersion, stableVersion)
}
comp := v1SemVer.Compare(v2SemVer)
switch comp {
case -1:
return false, nil
case 1, 0:
return true, nil
default:
return true, fmt.Errorf("unknown comparison value between scemantic versions, err: %q", err)
}
}
var fourDigitRe = regexp.MustCompile(`(\d+\.\d+\.\d+)\.(\d+)`)
func parseFourDigit(version string) (*semver.Version, int, error) {
matches := fourDigitRe.FindStringSubmatch(version)
if len(matches) < 2 {
semVer, err := semver.NewVersion(version)
if err != nil {
return nil, 0, err
}
return semVer, 0, nil
}
semVer, err := semver.NewVersion(matches[1])
if err != nil {
return nil, 0, err
}
i, err := strconv.Atoi(matches[2])
if err != nil {
return nil, 0, err
}
return semVer, i, nil
}
func isGreaterThanOrEqualFourDigit(newVersion, stableVersion string) (bool, error) {
newVersionSemVer, newVersionSemVerNo, err := parseFourDigit(newVersion)
if err != nil {
return false, err
}
stableVersionSemVer, stableVersionSemVerNo, err := parseFourDigit(stableVersion)
if err != nil {
return false, err
}
if stableVersionSemVer.Original() != newVersionSemVer.Original() {
return IsGreaterThanOrEqual(newVersionSemVer.Original(), stableVersionSemVer.Original())
}
return newVersionSemVerNo >= stableVersionSemVerNo, nil
}
func GetVersion(tag string) (*Version, error) {
var version Version
switch {
case reGrafanaTag.MatchString(tag):
version = Version{
Version: reGrafanaTag.FindStringSubmatch(tag)[1],
Channel: Latest,
}
case reGrafanaTagBeta.MatchString(tag):
version = Version{
Version: reGrafanaTagBeta.FindStringSubmatch(tag)[1],
Channel: Next,
}
case reGrafanaTagCustom.MatchString(tag):
version = Version{
Version: reGrafanaTagCustom.FindStringSubmatch(tag)[1],
Channel: Test,
}
default:
return nil, fmt.Errorf("%s not a supported Grafana version, exitting", tag)
}
return &version, nil
}

View File

@ -0,0 +1,69 @@
package versions
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/require"
)
func TestIsGreaterThanOrEqual(t *testing.T) {
testCases := []struct {
newVersion string
stableVersion string
expected bool
}{
{newVersion: "9.0.0", stableVersion: "8.0.0", expected: true},
{newVersion: "6.0.0", stableVersion: "6.0.0", expected: true},
{newVersion: "7.0.0", stableVersion: "8.0.0", expected: false},
{newVersion: "8.5.0-beta1", stableVersion: "8.0.0", expected: true},
{newVersion: "8.5.0", stableVersion: "8.5.0-beta1", expected: true},
{newVersion: "9.0.0.1", stableVersion: "9.0.0", expected: true},
{newVersion: "9.0.0.2", stableVersion: "9.0.0.1", expected: true},
{newVersion: "9.1.0", stableVersion: "9.0.0.1", expected: true},
{newVersion: "9.1-0-beta1", stableVersion: "9.0.0.1", expected: true},
{newVersion: "9.0.0.1", stableVersion: "9.0.1.1", expected: false},
{newVersion: "9.0.1.1", stableVersion: "9.0.0.1", expected: true},
{newVersion: "9.0.0.1", stableVersion: "9.0.0.1", expected: true},
{newVersion: "7.0.0.1", stableVersion: "8.0.0", expected: false},
{newVersion: "9.1-0-beta1", stableVersion: "9.1-0-beta2", expected: false},
{newVersion: "9.1-0-beta3", stableVersion: "9.1-0-beta2", expected: true},
}
for _, tc := range testCases {
name := fmt.Sprintf("newVersion %s greater than or equal stableVersion %s = %v", tc.newVersion, tc.stableVersion, tc.expected)
t.Run(name, func(t *testing.T) {
result, err := IsGreaterThanOrEqual(tc.newVersion, tc.stableVersion)
require.NoError(t, err)
require.Equal(t, tc.expected, result)
})
}
}
func TestGetLatestVersion(t *testing.T) {
t.Run("it returns a version", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
response := VersionFromAPI{
Version: "8.4.0",
}
jsonRes, err := json.Marshal(&response)
require.NoError(t, err)
_, err = w.Write(jsonRes)
require.NoError(t, err)
}))
version, err := GetLatestVersion(server.URL)
require.NoError(t, err)
require.Equal(t, "8.4.0", version)
})
t.Run("it handles non 200 responses", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
_, err := GetLatestVersion(server.URL)
require.Error(t, err)
})
}

View File

@ -95,13 +95,14 @@ def store_npm_packages_step():
'name': 'store-npm-packages',
'image': build_image,
'depends_on': [
'compile-build-cmd',
'build-frontend-packages',
],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket),
},
'commands': ['./bin/grabpl artifacts npm store --tag ${DRONE_TAG}'],
'commands': ['./bin/build artifacts npm store --tag ${DRONE_TAG}'],
}
@ -110,6 +111,7 @@ def retrieve_npm_packages_step():
'name': 'retrieve-npm-packages',
'image': publish_image,
'depends_on': [
'compile-build-cmd',
'yarn-install',
],
'failure': 'ignore',
@ -117,7 +119,7 @@ def retrieve_npm_packages_step():
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket),
},
'commands': ['./bin/grabpl artifacts npm retrieve --tag ${DRONE_TAG}'],
'commands': ['./bin/build artifacts npm retrieve --tag ${DRONE_TAG}'],
}
@ -126,13 +128,14 @@ def release_npm_packages_step():
'name': 'release-npm-packages',
'image': build_image,
'depends_on': [
'compile-build-cmd',
'retrieve-npm-packages',
],
'failure': 'ignore',
'environment': {
'NPM_TOKEN': from_secret('npm_token'),
},
'commands': ['./bin/grabpl artifacts npm release --tag ${DRONE_TAG}'],
'commands': ['./bin/build artifacts npm release --tag ${DRONE_TAG}'],
}
@ -594,7 +597,7 @@ def publish_npm_pipelines():
'target': ['public'],
}
steps = [
download_grabpl_step(),
compile_build_cmd(),
yarn_install_step(),
retrieve_npm_packages_step(),
release_npm_packages_step(),