diff --git a/.drone.yml b/.drone.yml index 0e291340662..d3f38298f79 100644 --- a/.drone.yml +++ b/.drone.yml @@ -2214,8 +2214,9 @@ steps: event: - tag - commands: - - ./bin/grabpl artifacts npm store --tag ${DRONE_TAG} + - ./bin/build artifacts npm store --tag ${DRONE_TAG} depends_on: + - compile-build-cmd - build-frontend-packages environment: GCP_KEY: @@ -4352,19 +4353,21 @@ platform: services: [] steps: - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.20/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd - commands: - yarn install --immutable depends_on: [] image: grafana/build-container:1.6.6 name: yarn-install - commands: - - ./bin/grabpl artifacts npm retrieve --tag ${DRONE_TAG} + - ./bin/build artifacts npm retrieve --tag ${DRONE_TAG} depends_on: + - compile-build-cmd - yarn-install environment: GCP_KEY: @@ -4375,8 +4378,9 @@ steps: image: grafana/grafana-ci-deploy:1.3.3 name: retrieve-npm-packages - commands: - - ./bin/grabpl artifacts npm release --tag ${DRONE_TAG} + - ./bin/build artifacts npm release --tag ${DRONE_TAG} depends_on: + - compile-build-cmd - retrieve-npm-packages environment: NPM_TOKEN: @@ -6439,6 +6443,6 @@ kind: secret name: aws_secret_access_key --- kind: signature -hmac: 7f74588652e48e6c51cd54fb9973289c144f09d2f232fe30fe66b5d51b9b317e +hmac: 59d802aec2892f9bdd89caaa68d0a563cf14cedefe9e76646f3b734e69afa40d ... diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index 6bc823e9e6d..69a326f5335 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -210,6 +210,46 @@ func main() { }, }, }, + { + Name: "npm", + Usage: "Handle Grafana npm packages", + Subcommands: cli.Commands{ + { + Name: "release", + Usage: "Release npm packages", + ArgsUsage: "[version]", + Action: NpmReleaseAction, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "tag", + Usage: "Grafana version tag", + }, + }, + }, + { + Name: "store", + Usage: "Store npm packages tarball", + Action: NpmStoreAction, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "tag", + Usage: "Grafana version tag", + }, + }, + }, + { + Name: "retrieve", + Usage: "Retrieve npm packages tarball", + Action: NpmRetrieveAction, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "tag", + Usage: "Grafana version tag", + }, + }, + }, + }, + }, }, }, { diff --git a/pkg/build/cmd/npm.go b/pkg/build/cmd/npm.go new file mode 100644 index 00000000000..07c2fe1365f --- /dev/null +++ b/pkg/build/cmd/npm.go @@ -0,0 +1,88 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "strings" + + "github.com/grafana/grafana/pkg/build/npm" + "github.com/urfave/cli/v2" +) + +func NpmRetrieveAction(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + tag := c.String("tag") + if tag == "" { + return fmt.Errorf("no tag version specified, exitting") + } + + prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET")) + if prereleaseBucket == "" { + return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1) + } + + err := npm.FetchNpmPackages(c.Context, tag, prereleaseBucket) + if err != nil { + return err + } + return nil +} + +func NpmStoreAction(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + tag := c.String("tag") + if tag == "" { + return fmt.Errorf("no tag version specified, exiting") + } + + prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET")) + if prereleaseBucket == "" { + return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1) + } + + err := npm.StoreNpmPackages(c.Context, tag, prereleaseBucket) + if err != nil { + return err + } + return nil +} + +func NpmReleaseAction(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + tag := c.String("tag") + if tag == "" { + return fmt.Errorf("no tag version specified, exitting") + } + + cmd := exec.Command("git", "checkout", ".") + if err := cmd.Run(); err != nil { + fmt.Println("command failed to run, err: ", err) + return err + } + + err := npm.PublishNpmPackages(c.Context, tag) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/build/lerna/lerna.go b/pkg/build/lerna/lerna.go index de0cbbfe461..de04f5f0e88 100644 --- a/pkg/build/lerna/lerna.go +++ b/pkg/build/lerna/lerna.go @@ -1,6 +1,7 @@ package lerna import ( + "context" "encoding/json" "fmt" "os" @@ -9,6 +10,7 @@ import ( "strings" "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/fsutil" ) // BuildFrontendPackages will bump the version for the package to the latest canary build @@ -56,3 +58,29 @@ func GetLernaVersion(grafanaDir string) (string, error) { } return strings.TrimSpace(version), nil } + +func PackFrontendPackages(ctx context.Context, tag, grafanaDir, artifactsDir string) error { + exists, err := fsutil.Exists(artifactsDir) + if err != nil { + return err + } + if exists { + err = os.RemoveAll(artifactsDir) + if err != nil { + return err + } + } + // nolint:gosec + if err = os.MkdirAll(artifactsDir, 0755); err != nil { + return err + } + + // nolint:gosec + cmd := exec.CommandContext(ctx, "yarn", "lerna", "exec", "--no-private", "--", "yarn", "pack", "--out", fmt.Sprintf("../../npm-artifacts/%%s-%v.tgz", tag)) + cmd.Dir = grafanaDir + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), output, err) + } + + return nil +} diff --git a/pkg/build/npm/npm.go b/pkg/build/npm/npm.go new file mode 100644 index 00000000000..c2b93aef46b --- /dev/null +++ b/pkg/build/npm/npm.go @@ -0,0 +1,240 @@ +package npm + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/grafana/grafana/pkg/build/gcloud/storage" + "github.com/grafana/grafana/pkg/build/lerna" + "github.com/grafana/grafana/pkg/build/versions" +) + +const GrafanaDir = "." +const NpmArtifactDir = "./npm-artifacts" + +// TODO: could this be replaced by `yarn lerna list -p` ? +var packages = []string{ + "@grafana/ui", + "@grafana/data", + "@grafana/toolkit", + "@grafana/runtime", + "@grafana/e2e", + "@grafana/e2e-selectors", + "@grafana/schema", +} + +// PublishNpmPackages will publish local NPM packages to NPM registry. +func PublishNpmPackages(ctx context.Context, tag string) error { + version, err := versions.GetVersion(tag) + if err != nil { + return err + } + + log.Printf("Grafana version: %s", version.Version) + + if err := setNpmCredentials(); err != nil { + return err + } + + npmArtifacts, err := storage.ListLocalFiles(NpmArtifactDir) + if err != nil { + return err + } + for _, packedFile := range npmArtifacts { + // nolint:gosec + cmd := exec.CommandContext(ctx, "npm", "publish", packedFile.FullPath, "--tag", version.Channel) + cmd.Dir = GrafanaDir + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err) + } + } + + return updateTag(ctx, version, tag) +} + +// StoreNpmPackages will store local NPM packages in GCS bucket `bucketName`. +func StoreNpmPackages(ctx context.Context, tag, bucketName string) error { + err := lerna.PackFrontendPackages(ctx, tag, GrafanaDir, NpmArtifactDir) + if err != nil { + return err + } + + gcs, err := storage.New() + if err != nil { + return err + } + + bucket := gcs.Bucket(bucketName) + bucketPath := fmt.Sprintf("artifacts/npm/%s/", tag) + if err = gcs.CopyLocalDir(ctx, NpmArtifactDir, bucket, bucketPath, true); err != nil { + return err + } + + log.Print("Successfully stored npm packages!") + return nil +} + +// FetchNpmPackages will store NPM packages stored in GCS bucket `bucketName` on local disk in `frontend.NpmArtifactDir`. +func FetchNpmPackages(ctx context.Context, tag, bucketName string) error { + gcs, err := storage.New() + if err != nil { + return err + } + + bucketPath := fmt.Sprintf("artifacts/npm/%s/", tag) + bucket := gcs.Bucket(bucketName) + err = gcs.DownloadDirectory(ctx, bucket, NpmArtifactDir, storage.FilesFilter{ + Prefix: bucketPath, + FileExts: []string{".tgz"}, + }) + if err != nil { + return err + } + return nil +} + +// updateTag will move next or latest npm dist-tags, if needed. +// +// Note: This function makes the assumption that npm dist-tags has already +// been updated and hence why move of dist-tags not always happens: +// +// If stable the dist-tag latest was used. +// If beta the dist-tag next was used. +// +// Scenarios: +// +// 1. Releasing a newer stable than the current stable +// Latest and next is 9.1.5. +// 9.1.6 is released, latest and next should point to 9.1.6. +// The next dist-tag is moved to point to 9.1.6. +// +// 2. Releasing during an active beta period: +// Latest and next is 9.1.6. +// 9.2.0-beta1 is released, the latest should stay on 9.1.6, next should point to 9.2.0-beta1 +// No move of dist-tags +// 9.1.7 is relased, the latest should point to 9.1.7, next should stay to 9.2.0-beta1 +// No move of dist-tags +// Next week 9.2.0-beta2 is released, the latest should point to 9.1.7, next should point to 9.2.0-beta2 +// No move of dist-tags +// In two weeks 9.2.0 stable is relased, the latest and next should point to 9.2.0. +// The next dist-tag is moved to point to 9.2.0. +// +// 3. Releasing an older stable than the current stable +// Latest and next is 9.2.0. +// Next 9.1.8 is released, latest should point to 9.2.0, next should point to 9.2.0 +// The latest dist-tag is moved to point to 9.2.0. +func updateTag(ctx context.Context, version *versions.Version, releaseVersion string) error { + if version.Channel != versions.Latest { + return nil + } + + latestStableVersion, err := getLatestStableVersion() + if err != nil { + return err + } + + betaVersion, err := getLatestBetaVersion() + if err != nil { + return err + } + + isLatest, err := versions.IsGreaterThanOrEqual(releaseVersion, latestStableVersion) + if err != nil { + return err + } + + isNewerThanLatestBeta, err := versions.IsGreaterThanOrEqual(releaseVersion, betaVersion) + if err != nil { + return err + } + + for _, pkg := range packages { + if !isLatest { + err = runMoveLatestNPMTagCommand(ctx, pkg, latestStableVersion) + if err != nil { + return err + } + } + + if isLatest && isNewerThanLatestBeta { + err = runMoveNextNPMTagCommand(ctx, pkg, version.Version) + if err != nil { + return err + } + } + } + + return nil +} + +func getLatestStableVersion() (string, error) { + return versions.GetLatestVersion(versions.LatestStableVersionURL) +} + +func getLatestBetaVersion() (string, error) { + return versions.GetLatestVersion(versions.LatestBetaVersionURL) +} + +func runMoveNextNPMTagCommand(ctx context.Context, pkg string, packageVersion string) error { + // nolint:gosec + cmd := exec.CommandContext(ctx, "npm", "dist-tag", "add", fmt.Sprintf("%s@%s", pkg, packageVersion), "next") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err) + } + + return nil +} + +func runMoveLatestNPMTagCommand(ctx context.Context, pkg string, latestStableVersion string) error { + // nolint:gosec + cmd := exec.CommandContext(ctx, "npm", "dist-tag", "add", fmt.Sprintf("%s@%s", pkg, latestStableVersion), "latest") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err) + } + + return nil +} + +// setNpmCredentials Creates a .npmrc file in the users home folder and writes the +// necessary credentials to it for publishing packages to the NPM registry. +func setNpmCredentials() error { + npmToken := strings.TrimSpace(os.Getenv("NPM_TOKEN")) + if npmToken == "" { + return fmt.Errorf("npm token is not set") + } + + homeDir, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("failed to obtain home directory, err: %q", err) + } + + npmPath := filepath.Join(homeDir, ".npmrc") + registry := []byte(fmt.Sprintf("//registry.npmjs.org/:_authToken=%s", npmToken)) + if _, err = os.Stat(npmPath); os.IsNotExist(err) { + // nolint:gosec + f, err := os.Create(npmPath) + if err != nil { + return fmt.Errorf("couldn't create npmrc file, err: %q", err) + } + _, err = f.Write(registry) + if err != nil { + return fmt.Errorf("failed to write to file, err: %q", err) + } + defer func() { + if err := f.Close(); err != nil { + log.Printf("Failed to close file: %s", err.Error()) + } + }() + } else { + err = os.WriteFile(npmPath, registry, 0644) + if err != nil { + return fmt.Errorf("error writing to file, err: %q", err) + } + } + return nil +} diff --git a/pkg/build/versions/version.go b/pkg/build/versions/version.go new file mode 100644 index 00000000000..c1ee0b33d67 --- /dev/null +++ b/pkg/build/versions/version.go @@ -0,0 +1,160 @@ +package versions + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "regexp" + "strconv" + + "github.com/Masterminds/semver/v3" +) + +var ( + reGrafanaTag = regexp.MustCompile(`^v(\d+\.\d+\.\d+$)`) + reGrafanaTagBeta = regexp.MustCompile(`^v(\d+\.\d+\.\d+-beta)`) + reGrafanaTagCustom = regexp.MustCompile(`^v(\d+\.\d+\.\d+-\w+)`) +) + +const ( + Latest = "latest" + Next = "next" + Test = "test" +) + +type Version struct { + Version string + Channel string +} + +type VersionFromAPI struct { + Version string `json:"version"` +} + +type LatestGcomAPI = string + +const ( + LatestStableVersionURL LatestGcomAPI = "https://grafana.com/api/grafana/versions/stable" + LatestBetaVersionURL LatestGcomAPI = "https://grafana.com/api/grafana/versions/beta" +) + +func GetLatestVersion(url LatestGcomAPI) (string, error) { + // nolint:gosec + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer func() { + if err := resp.Body.Close(); err != nil { + log.Printf("Failed to close body: %s", err.Error()) + } + }() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("server returned non 200 status code: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + var apiResponse VersionFromAPI + err = json.Unmarshal(body, &apiResponse) + if err != nil { + return "", err + } + + return apiResponse.Version, nil +} + +// IsGreaterThanOrEqual semantically checks whether newVersion is greater than or equal to stableVersion. +func IsGreaterThanOrEqual(newVersion, stableVersion string) (bool, error) { + v1SemVer, err := semver.NewVersion(newVersion) + if err != nil { + return isGreaterThanOrEqualFourDigit(newVersion, stableVersion) + } + + v2SemVer, err := semver.NewVersion(stableVersion) + if err != nil { + return isGreaterThanOrEqualFourDigit(newVersion, stableVersion) + } + + comp := v1SemVer.Compare(v2SemVer) + switch comp { + case -1: + return false, nil + case 1, 0: + return true, nil + default: + return true, fmt.Errorf("unknown comparison value between scemantic versions, err: %q", err) + } +} + +var fourDigitRe = regexp.MustCompile(`(\d+\.\d+\.\d+)\.(\d+)`) + +func parseFourDigit(version string) (*semver.Version, int, error) { + matches := fourDigitRe.FindStringSubmatch(version) + if len(matches) < 2 { + semVer, err := semver.NewVersion(version) + if err != nil { + return nil, 0, err + } + return semVer, 0, nil + } + semVer, err := semver.NewVersion(matches[1]) + if err != nil { + return nil, 0, err + } + i, err := strconv.Atoi(matches[2]) + if err != nil { + return nil, 0, err + } + + return semVer, i, nil +} + +func isGreaterThanOrEqualFourDigit(newVersion, stableVersion string) (bool, error) { + newVersionSemVer, newVersionSemVerNo, err := parseFourDigit(newVersion) + if err != nil { + return false, err + } + + stableVersionSemVer, stableVersionSemVerNo, err := parseFourDigit(stableVersion) + if err != nil { + return false, err + } + + if stableVersionSemVer.Original() != newVersionSemVer.Original() { + return IsGreaterThanOrEqual(newVersionSemVer.Original(), stableVersionSemVer.Original()) + } + + return newVersionSemVerNo >= stableVersionSemVerNo, nil +} + +func GetVersion(tag string) (*Version, error) { + var version Version + switch { + case reGrafanaTag.MatchString(tag): + version = Version{ + Version: reGrafanaTag.FindStringSubmatch(tag)[1], + Channel: Latest, + } + case reGrafanaTagBeta.MatchString(tag): + version = Version{ + Version: reGrafanaTagBeta.FindStringSubmatch(tag)[1], + Channel: Next, + } + case reGrafanaTagCustom.MatchString(tag): + version = Version{ + Version: reGrafanaTagCustom.FindStringSubmatch(tag)[1], + Channel: Test, + } + default: + return nil, fmt.Errorf("%s not a supported Grafana version, exitting", tag) + } + + return &version, nil +} diff --git a/pkg/build/versions/version_test.go b/pkg/build/versions/version_test.go new file mode 100644 index 00000000000..8d80ace1429 --- /dev/null +++ b/pkg/build/versions/version_test.go @@ -0,0 +1,69 @@ +package versions + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsGreaterThanOrEqual(t *testing.T) { + testCases := []struct { + newVersion string + stableVersion string + expected bool + }{ + {newVersion: "9.0.0", stableVersion: "8.0.0", expected: true}, + {newVersion: "6.0.0", stableVersion: "6.0.0", expected: true}, + {newVersion: "7.0.0", stableVersion: "8.0.0", expected: false}, + {newVersion: "8.5.0-beta1", stableVersion: "8.0.0", expected: true}, + {newVersion: "8.5.0", stableVersion: "8.5.0-beta1", expected: true}, + {newVersion: "9.0.0.1", stableVersion: "9.0.0", expected: true}, + {newVersion: "9.0.0.2", stableVersion: "9.0.0.1", expected: true}, + {newVersion: "9.1.0", stableVersion: "9.0.0.1", expected: true}, + {newVersion: "9.1-0-beta1", stableVersion: "9.0.0.1", expected: true}, + {newVersion: "9.0.0.1", stableVersion: "9.0.1.1", expected: false}, + {newVersion: "9.0.1.1", stableVersion: "9.0.0.1", expected: true}, + {newVersion: "9.0.0.1", stableVersion: "9.0.0.1", expected: true}, + {newVersion: "7.0.0.1", stableVersion: "8.0.0", expected: false}, + {newVersion: "9.1-0-beta1", stableVersion: "9.1-0-beta2", expected: false}, + {newVersion: "9.1-0-beta3", stableVersion: "9.1-0-beta2", expected: true}, + } + + for _, tc := range testCases { + name := fmt.Sprintf("newVersion %s greater than or equal stableVersion %s = %v", tc.newVersion, tc.stableVersion, tc.expected) + t.Run(name, func(t *testing.T) { + result, err := IsGreaterThanOrEqual(tc.newVersion, tc.stableVersion) + require.NoError(t, err) + require.Equal(t, tc.expected, result) + }) + } +} + +func TestGetLatestVersion(t *testing.T) { + t.Run("it returns a version", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + response := VersionFromAPI{ + Version: "8.4.0", + } + jsonRes, err := json.Marshal(&response) + require.NoError(t, err) + _, err = w.Write(jsonRes) + require.NoError(t, err) + })) + version, err := GetLatestVersion(server.URL) + require.NoError(t, err) + require.Equal(t, "8.4.0", version) + }) + + t.Run("it handles non 200 responses", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + _, err := GetLatestVersion(server.URL) + require.Error(t, err) + }) +} diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index 759b5d99124..c1983d7dee9 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -95,13 +95,14 @@ def store_npm_packages_step(): 'name': 'store-npm-packages', 'image': build_image, 'depends_on': [ + 'compile-build-cmd', 'build-frontend-packages', ], 'environment': { 'GCP_KEY': from_secret('gcp_key'), 'PRERELEASE_BUCKET': from_secret(prerelease_bucket), }, - 'commands': ['./bin/grabpl artifacts npm store --tag ${DRONE_TAG}'], + 'commands': ['./bin/build artifacts npm store --tag ${DRONE_TAG}'], } @@ -110,6 +111,7 @@ def retrieve_npm_packages_step(): 'name': 'retrieve-npm-packages', 'image': publish_image, 'depends_on': [ + 'compile-build-cmd', 'yarn-install', ], 'failure': 'ignore', @@ -117,7 +119,7 @@ def retrieve_npm_packages_step(): 'GCP_KEY': from_secret('gcp_key'), 'PRERELEASE_BUCKET': from_secret(prerelease_bucket), }, - 'commands': ['./bin/grabpl artifacts npm retrieve --tag ${DRONE_TAG}'], + 'commands': ['./bin/build artifacts npm retrieve --tag ${DRONE_TAG}'], } @@ -126,13 +128,14 @@ def release_npm_packages_step(): 'name': 'release-npm-packages', 'image': build_image, 'depends_on': [ + 'compile-build-cmd', 'retrieve-npm-packages', ], 'failure': 'ignore', 'environment': { 'NPM_TOKEN': from_secret('npm_token'), }, - 'commands': ['./bin/grabpl artifacts npm release --tag ${DRONE_TAG}'], + 'commands': ['./bin/build artifacts npm release --tag ${DRONE_TAG}'], } @@ -594,7 +597,7 @@ def publish_npm_pipelines(): 'target': ['public'], } steps = [ - download_grabpl_step(), + compile_build_cmd(), yarn_install_step(), retrieve_npm_packages_step(), release_npm_packages_step(),