diff --git a/.drone.yml b/.drone.yml index ac92902bae0..52ae62655ac 100644 --- a/.drone.yml +++ b/.drone.yml @@ -334,7 +334,7 @@ steps: image: grafana/build-container:1.5.9 name: build-frontend-packages - commands: - - ./bin/grabpl build-plugins --jobs 8 --edition oss + - ./bin/build build-plugins --jobs 8 --edition oss depends_on: - gen-version - yarn-install @@ -1123,7 +1123,7 @@ steps: image: grafana/build-container:1.5.9 name: build-frontend-packages - commands: - - ./bin/grabpl build-plugins --jobs 8 --edition oss + - ./bin/build build-plugins --jobs 8 --edition oss depends_on: - gen-version - yarn-install @@ -1831,7 +1831,7 @@ steps: image: grafana/build-container:1.5.9 name: build-frontend-packages - commands: - - ./bin/grabpl build-plugins --jobs 8 --edition oss + - ./bin/build build-plugins --jobs 8 --edition oss depends_on: - gen-version - yarn-install @@ -2468,7 +2468,7 @@ steps: image: grafana/build-container:1.5.9 name: build-frontend-packages - commands: - - ./bin/grabpl build-plugins --jobs 8 --edition enterprise + - ./bin/build build-plugins --jobs 8 --edition enterprise depends_on: - gen-version - yarn-install @@ -3778,7 +3778,7 @@ steps: image: grafana/build-container:1.5.9 name: build-frontend-packages - commands: - - ./bin/grabpl build-plugins --jobs 8 --edition oss + - ./bin/build build-plugins --jobs 8 --edition oss depends_on: - gen-version - yarn-install @@ -4361,7 +4361,7 @@ steps: image: grafana/build-container:1.5.9 name: build-frontend-packages - commands: - - ./bin/grabpl build-plugins --jobs 8 --edition enterprise + - ./bin/build build-plugins --jobs 8 --edition enterprise depends_on: - gen-version - yarn-install @@ -5133,6 +5133,6 @@ kind: secret name: gcp_upload_artifacts_key --- kind: signature -hmac: cf7b07bd7aa1c46665d485693553ff98e7dacf0e4853c5d640437d0218c91db7 +hmac: 7d799b8c9888eeb4b72f6aa1721fe335d4eb74c6fb2db9612970adbb9462d395 ... diff --git a/pkg/build/cmd/buildinternalplugins.go b/pkg/build/cmd/buildinternalplugins.go new file mode 100644 index 00000000000..66429f5618c --- /dev/null +++ b/pkg/build/cmd/buildinternalplugins.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "log" + "path/filepath" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/errutil" + "github.com/grafana/grafana/pkg/build/plugins" + "github.com/grafana/grafana/pkg/build/syncutil" + "github.com/urfave/cli/v2" +) + +func BuildInternalPlugins(c *cli.Context) error { + cfg := config.Config{ + NumWorkers: c.Int("jobs"), + } + + const grafanaDir = "." + metadata, err := config.GetMetadata(filepath.Join("dist", "version.json")) + if err != nil { + return err + } + verMode, err := config.GetVersion(metadata.ReleaseMode) + if err != nil { + return err + } + + log.Println("Building internal Grafana plug-ins...") + + ctx := context.Background() + + p := syncutil.NewWorkerPool(cfg.NumWorkers) + defer p.Close() + + var g *errutil.Group + g, ctx = errutil.GroupWithContext(ctx) + if err := plugins.Build(ctx, grafanaDir, p, g, verMode); err != nil { + return cli.Exit(err.Error(), 1) + } + if err := g.Wait(); err != nil { + return cli.Exit(err.Error(), 1) + } + + if err := plugins.Download(ctx, grafanaDir, p); err != nil { + return cli.Exit(err.Error(), 1) + } + + log.Println("Successfully built Grafana plug-ins!") + + return nil +} diff --git a/pkg/build/cmd/flags.go b/pkg/build/cmd/flags.go index 89af88a0b90..9d8451dfe4b 100644 --- a/pkg/build/cmd/flags.go +++ b/pkg/build/cmd/flags.go @@ -20,4 +20,16 @@ var ( Name: "variants", Usage: "Comma-separated list of variants to build", } + noInstallDepsFlag = cli.BoolFlag{ + Name: "no-install-deps", + Usage: "Don't install dependencies", + } + signingAdminFlag = cli.BoolFlag{ + Name: "signing-admin", + Usage: "Use manifest signing admin API endpoint?", + } + signFlag = cli.BoolFlag{ + Name: "sign", + Usage: "Enable plug-in signing (you must set GRAFANA_API_KEY)", + } ) diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index 6ee2db3c950..584a9616f99 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -23,6 +23,18 @@ func main() { &buildIDFlag, }, }, + { + Name: "build-plugins", + Usage: "Build internal plug-ins", + Action: ArgCountWrapper(1, BuildInternalPlugins), + Flags: []cli.Flag{ + &jobsFlag, + &editionFlag, + &signingAdminFlag, + &signFlag, + &noInstallDepsFlag, + }, + }, } if err := app.Run(os.Args); err != nil { diff --git a/pkg/build/config/version.go b/pkg/build/config/version.go index 171a2d18cab..04854ca018d 100644 --- a/pkg/build/config/version.go +++ b/pkg/build/config/version.go @@ -5,8 +5,12 @@ import ( "errors" "fmt" "io" + "io/ioutil" "log" "os" + "path/filepath" + "regexp" + "strings" ) type Metadata struct { @@ -83,5 +87,77 @@ func GetVersion(mode VersionMode) (*Version, error) { return &v, nil } - return nil, fmt.Errorf("mode not found in version list") + return nil, fmt.Errorf("mode '%s' not found in version list", mode) +} + +func shortenBuildID(buildID string) string { + buildID = strings.ReplaceAll(buildID, "-", "") + if len(buildID) < 9 { + return buildID + } + + return buildID[0:8] +} + +// GetGrafanaVersion gets the Grafana version from the package.json +func GetGrafanaVersion(buildID, grafanaDir string) (string, error) { + pkgJSONPath := filepath.Join(grafanaDir, "package.json") + //nolint:gosec + pkgJSONB, err := ioutil.ReadFile(pkgJSONPath) + if err != nil { + return "", fmt.Errorf("failed to read %q: %w", pkgJSONPath, err) + } + pkgObj := map[string]interface{}{} + if err := json.Unmarshal(pkgJSONB, &pkgObj); err != nil { + return "", fmt.Errorf("failed decoding %q: %w", pkgJSONPath, err) + } + + version := pkgObj["version"].(string) + if version == "" { + return "", fmt.Errorf("failed to read version from %q", pkgJSONPath) + } + if buildID != "" { + buildID = shortenBuildID(buildID) + verComponents := strings.Split(version, "-") + version = verComponents[0] + if len(verComponents) > 1 { + buildID = fmt.Sprintf("%s%s", buildID, verComponents[1]) + } + version = fmt.Sprintf("%s-%s", version, buildID) + } + + return version, nil +} + +func CheckDroneTargetBranch() (VersionMode, error) { + reRlsBranch := regexp.MustCompile(`^v\d+\.\d+\.x$`) + target := os.Getenv("DRONE_TARGET_BRANCH") + if target == "" { + return "", fmt.Errorf("failed to get DRONE_TARGET_BRANCH environmental variable") + } else if target == string(MainMode) { + return MainMode, nil + } + if reRlsBranch.MatchString(target) { + return ReleaseBranchMode, nil + } + return "", fmt.Errorf("unrecognized target branch: %s", target) +} + +func CheckSemverSuffix() (VersionMode, error) { + reBetaRls := regexp.MustCompile(`beta.*`) + reTestRls := regexp.MustCompile(`test.*`) + tagSuffix, ok := os.LookupEnv("DRONE_SEMVER_PRERELEASE") + if !ok || tagSuffix == "" { + fmt.Println("DRONE_SEMVER_PRERELEASE doesn't exist for a tag, this is a release event...") + return ReleaseMode, nil + } + switch { + case reBetaRls.MatchString(tagSuffix): + return BetaReleaseMode, nil + case reTestRls.MatchString(tagSuffix): + return TestReleaseMode, nil + default: + fmt.Printf("DRONE_SEMVER_PRERELEASE is custom string, release event with %s suffix\n", tagSuffix) + return ReleaseMode, nil + } } diff --git a/pkg/build/grafana/variant.go b/pkg/build/grafana/variant.go index 40c9a50c05b..e55d0accb48 100644 --- a/pkg/build/grafana/variant.go +++ b/pkg/build/grafana/variant.go @@ -46,9 +46,9 @@ func BuildVariant(ctx context.Context, opts BuildVariantOpts) error { stderr = bytes.NewBuffer(nil) ) - args.BuildOpts.Workdir = grafanaDir - args.BuildOpts.Stdout = stdout - args.BuildOpts.Stderr = stderr + args.Workdir = grafanaDir + args.Stdout = stdout + args.Stderr = stderr args.Package = pkg if err := BuildGrafanaBinary(ctx, binary, opts.Version, args, opts.Edition); err != nil { diff --git a/pkg/build/plugins/build.go b/pkg/build/plugins/build.go new file mode 100644 index 00000000000..b2c3836c199 --- /dev/null +++ b/pkg/build/plugins/build.go @@ -0,0 +1,66 @@ +package plugins + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "os/exec" + "path/filepath" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/errutil" + "github.com/grafana/grafana/pkg/build/syncutil" + "github.com/grafana/grafana/pkg/infra/fs" +) + +type PluginSigningMode = int + +// BuildPlugins builds internal plugins. +// The built plugins are placed in plugins-bundled/dist/. +func Build(ctx context.Context, grafanaDir string, p syncutil.WorkerPool, g *errutil.Group, verMode *config.Version) error { + log.Printf("Building plugins in %q...", grafanaDir) + + root := filepath.Join(grafanaDir, "plugins-bundled", "internal") + fis, err := ioutil.ReadDir(root) + if err != nil { + return err + } + + for i := range fis { + fi := fis[i] + if !fi.IsDir() { + continue + } + + dpath := filepath.Join(root, fi.Name()) + + p.Schedule(g.Wrap(func() error { + log.Printf("Building plugin %q...", dpath) + + cmd := exec.Command("yarn", "build") + cmd.Dir = dpath + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("yarn build failed: %s", output) + } + + dstPath := filepath.Join("plugins-bundled", "dist", fi.Name()) + if err := fs.CopyRecursive(filepath.Join(dpath, "dist"), dstPath); err != nil { + return err + } + if !verMode.PluginSignature.Sign { + return nil + } + + return BuildManifest(ctx, dstPath, verMode.PluginSignature.AdminSign) + })) + } + + if err := g.Wait(); err != nil { + return err + } + + log.Printf("Built all plug-ins successfully!") + + return nil +} diff --git a/pkg/build/plugins/download.go b/pkg/build/plugins/download.go new file mode 100644 index 00000000000..3c3fc56a6ad --- /dev/null +++ b/pkg/build/plugins/download.go @@ -0,0 +1,118 @@ +package plugins + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + + "github.com/grafana/grafana/pkg/build/errutil" + "github.com/grafana/grafana/pkg/build/syncutil" +) + +// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package. +func logCloseError(closeFunc func() error) { + if err := closeFunc(); err != nil { + log.Println(err) + } +} + +// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package. +func logError(err error) { + if err != nil { + log.Println(err) + } +} + +// pluginManifest has details of an external plugin package. +type pluginManifest struct { + Name string `json:"name"` + Version string `json:"version"` + Checksum string `json:"checksum"` +} + +// pluginsManifest represents a manifest of Grafana's external plugins. +type pluginsManifest struct { + Plugins []pluginManifest `json:"plugins"` +} + +// downloadPlugins downloads Grafana plugins that should be bundled into packages. +// +// The plugin archives are downloaded into /plugins-bundled. +func Download(ctx context.Context, grafanaDir string, p syncutil.WorkerPool) error { + g, _ := errutil.GroupWithContext(ctx) + + log.Println("Downloading external plugins...") + + var m pluginsManifest + manifestPath := filepath.Join(grafanaDir, "plugins-bundled", "external.json") + //nolint:gosec + manifestB, err := ioutil.ReadFile(manifestPath) + if err != nil { + return fmt.Errorf("failed to open plugins manifest %q: %w", manifestPath, err) + } + if err := json.Unmarshal(manifestB, &m); err != nil { + return err + } + + for i := range m.Plugins { + pm := m.Plugins[i] + p.Schedule(g.Wrap(func() error { + tgt := filepath.Join(grafanaDir, "plugins-bundled", fmt.Sprintf("%s-%s.zip", pm.Name, pm.Version)) + out, err := os.Create(tgt) + if err != nil { + return err + } + defer logCloseError(out.Close) + + u := fmt.Sprintf("http://storage.googleapis.com/plugins-ci/plugins/%s/%s-%s.zip", pm.Name, pm.Name, + pm.Version) + log.Printf("Downloading plugin %q to %q...", u, tgt) + // nolint:gosec + resp, err := http.Get(u) + if err != nil { + return fmt.Errorf("downloading %q failed: %w", u, err) + } + defer logError(resp.Body.Close()) + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to download %q, status code %d", u, resp.StatusCode) + } + + if _, err := io.Copy(out, resp.Body); err != nil { + return fmt.Errorf("downloading %q failed: %w", u, err) + } + if err := out.Close(); err != nil { + return fmt.Errorf("downloading %q failed: %w", u, err) + } + + //nolint:gosec + fd, err := os.Open(tgt) + if err != nil { + return err + } + defer logCloseError(fd.Close) + + h := sha256.New() + if _, err := io.Copy(h, fd); err != nil { + return err + } + + chksum := hex.EncodeToString(h.Sum(nil)) + if chksum != pm.Checksum { + return fmt.Errorf("plugin %q has bad checksum: %s (expected %s)", u, chksum, pm.Checksum) + } + + return Unzip(tgt, filepath.Join(grafanaDir, "plugins-bundled")) + })) + } + + return g.Wait() +} diff --git a/pkg/build/plugins/manifest.go b/pkg/build/plugins/manifest.go new file mode 100644 index 00000000000..1166a0d41f6 --- /dev/null +++ b/pkg/build/plugins/manifest.go @@ -0,0 +1,196 @@ +package plugins + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" +) + +type manifest struct { + Plugin string `json:"plugin"` + Version string `json:"version"` + Files map[string]string `json:"files"` +} + +func getManifest(dpath string, chksums map[string]string) (manifest, error) { + m := manifest{} + + type pluginInfo struct { + Version string `json:"version"` + } + + type plugin struct { + ID string `json:"id"` + Info pluginInfo `json:"info"` + } + + //nolint:gosec + f, err := os.Open(filepath.Join(dpath, "plugin.json")) + if err != nil { + return m, err + } + decoder := json.NewDecoder(f) + var p plugin + if err := decoder.Decode(&p); err != nil { + return m, err + } + + if p.ID == "" { + return m, fmt.Errorf("plugin.json doesn't define id") + } + if p.Info.Version == "" { + return m, fmt.Errorf("plugin.json doesn't define info.version") + } + + return manifest{ + Plugin: p.ID, + Version: p.Info.Version, + Files: chksums, + }, nil +} + +// BuildManifest requests a plugin's signed manifest file fromt he Grafana API. +// If signingAdmin is true, the manifest signing admin endpoint (without plugin ID) will be used, and requires +// an admin API key. +func BuildManifest(ctx context.Context, dpath string, signingAdmin bool) error { + log.Printf("Building manifest for plug-in at %q", dpath) + + apiKey := os.Getenv("GRAFANA_API_KEY") + if apiKey == "" { + return fmt.Errorf("GRAFANA_API_KEY must be set") + } + + manifestPath := filepath.Join(dpath, "MANIFEST.txt") + chksums, err := getChksums(dpath, manifestPath) + if err != nil { + return err + } + m, err := getManifest(dpath, chksums) + if err != nil { + return err + } + + b := bytes.NewBuffer(nil) + encoder := json.NewEncoder(b) + if err := encoder.Encode(&m); err != nil { + return err + } + jsonB := b.Bytes() + u := "https://grafana.com/api/plugins/ci/sign" + if !signingAdmin { + u = fmt.Sprintf("https://grafana.com/api/plugins/%s/ci/sign", m.Plugin) + } + log.Printf("Requesting signed manifest from Grafana API...") + req, err := http.NewRequestWithContext(ctx, "POST", u, bytes.NewReader(jsonB)) + if err != nil { + return err + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey)) + req.Header.Add("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to get signed manifest from Grafana API: %w", err) + } + defer logError(resp.Body.Close()) + if resp.StatusCode != 200 { + msg, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Printf("Failed to read response body: %s", err) + msg = []byte("") + } + return fmt.Errorf("request for signed manifest failed with status code %d: %s", resp.StatusCode, string(msg)) + } + + log.Printf("Successfully signed manifest via Grafana API, writing to %q", manifestPath) + f, err := os.Create(manifestPath) + if err != nil { + return fmt.Errorf("failed to create %s: %w", manifestPath, err) + } + defer logCloseError(f.Close) + if _, err := io.Copy(f, resp.Body); err != nil { + return fmt.Errorf("failed to write %s: %w", manifestPath, err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("failed to write %s: %w", manifestPath, err) + } + + return nil +} + +func getChksums(dpath, manifestPath string) (map[string]string, error) { + manifestPath = filepath.Clean(manifestPath) + + chksums := map[string]string{} + if err := filepath.Walk(dpath, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + if fi.IsDir() { + return nil + } + + path = filepath.Clean(path) + + // Handle symbolic links + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + finalPath, err := filepath.EvalSymlinks(path) + if err != nil { + return err + } + + log.Printf("Handling symlink %q, pointing to %q", path, finalPath) + + info, err := os.Stat(finalPath) + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + if _, err := filepath.Rel(dpath, finalPath); err != nil { + return fmt.Errorf("symbolic link %q targets a file outside of the plugin directory: %q", path, finalPath) + } + + if finalPath == manifestPath { + return nil + } + } + + if path == manifestPath { + return nil + } + + h := sha256.New() + //nolint:gosec + f, err := os.Open(path) + if err != nil { + return err + } + defer logCloseError(f.Close) + if _, err := io.Copy(h, f); err != nil { + return err + } + + relPath, err := filepath.Rel(dpath, path) + if err != nil { + return err + } + chksums[relPath] = fmt.Sprintf("%x", h.Sum(nil)) + + return nil + }); err != nil { + return nil, err + } + + return chksums, nil +} diff --git a/pkg/build/plugins/zip.go b/pkg/build/plugins/zip.go new file mode 100644 index 00000000000..73f8e8d82f6 --- /dev/null +++ b/pkg/build/plugins/zip.go @@ -0,0 +1,64 @@ +package plugins + +import ( + "archive/zip" + "io" + "log" + "os" + "path/filepath" +) + +// Unzip unzips a plugin. +func Unzip(fpath, tgtDir string) error { + log.Printf("Unzipping plugin %q into %q...", fpath, tgtDir) + + r, err := zip.OpenReader(fpath) + if err != nil { + return err + } + defer logCloseError(r.Close) + + // Closure to address file descriptors issue with all the deferred .Close() methods + extractAndWriteFile := func(f *zip.File) error { + log.Printf("Extracting zip member %q...", f.Name) + + rc, err := f.Open() + if err != nil { + return err + } + defer logCloseError(rc.Close) + + //nolint:gosec + dstPath := filepath.Join(tgtDir, f.Name) + + if f.FileInfo().IsDir() { + return os.MkdirAll(dstPath, f.Mode()) + } + + if err := os.MkdirAll(filepath.Dir(dstPath), f.Mode()); err != nil { + return err + } + + //nolint:gosec + fd, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + defer logCloseError(fd.Close) + + // nolint:gosec + if _, err := io.Copy(fd, rc); err != nil { + return err + } + + return fd.Close() + } + + for _, f := range r.File { + if err := extractAndWriteFile(f); err != nil { + return err + } + } + + return nil +} diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index eb03f95b3cd..c7e28b364a0 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -488,7 +488,7 @@ def build_plugins_step(edition, ver_mode): ], 'commands': [ # TODO: Use percentage for num jobs - './bin/grabpl build-plugins --jobs 8 --edition {}'.format(edition), + './bin/build build-plugins --jobs 8 --edition {}'.format(edition), ], }