IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

CI: Move grafanacom command to OSS (#55853) (#56168)

Browse Source 
* Move publish-packages command over from

* Fix lint

* Move grafanacom command to OSS

* Add GetLatestMainBuild to gsutil

* Fix lint

* More lint fixes

* Add tests for grafanacom

* Fix lint

(cherry picked from commit 947838cca0)

Co-authored-by: Dimitris Sotirakis <dimitrios.sotirakis@grafana.com>
This commit is contained in:
Grot (@grafanabot) 2022-10-03 13:30:11 +02:00 committed by GitHub
parent fcf605e07f
commit 29bb039c94
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 529 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
pkg/build/cmd/artifacts.go
View File

@ -1,13 +1,24 @@
package main
import (
"fmt"
"strings"
"github.com/grafana/grafana/pkg/build/config"
)
const ReleaseFolder = "release"
const MainFolder = "main"
const EnterpriseSfx = "-enterprise"
const CacheSettings = "Cache-Control:public, max-age="
type buildArtifact struct {
Os string
Arch string
urlPostfix string
packagePostfix string
}
type PublishConfig struct {
config.Config
@ -20,3 +31,110 @@ type PublishConfig struct {
TTL string
SimulateRelease bool
}
const rhelOS = "rhel"
const debOS = "deb"
func (t buildArtifact) GetURL(baseArchiveURL string, cfg PublishConfig) string {
rev := ""
prefix := "-"
if t.Os == debOS {
prefix = "_"
} else if t.Os == rhelOS {
rev = "-1"
}
version := cfg.Version
verComponents := strings.Split(version, "-")
if len(verComponents) > 2 {
panic(fmt.Sprintf("Version string contains more than one hyphen: %q", version))
}
switch t.Os {
case debOS, rhelOS:
if len(verComponents) > 1 {
// With Debian and RPM packages, it's customary to prefix any pre-release component with a ~, since this
// is considered of lower lexical value than the empty character, and this way pre-release versions are
// considered to be of a lower version than the final version (which lacks this suffix).
version = fmt.Sprintf("%s~%s", verComponents[0], verComponents[1])
}
}
// https://dl.grafana.com/oss/main/grafana_8.5.0~54094pre_armhf.deb: 404 Not Found
url := fmt.Sprintf("%s%s%s%s%s%s", baseArchiveURL, t.packagePostfix, prefix, version, rev, t.urlPostfix)
return url
}
var ArtifactConfigs = []buildArtifact{
{
Os: debOS,
Arch: "arm64",
urlPostfix: "_arm64.deb",
},
{
Os: rhelOS,
Arch: "arm64",
urlPostfix: ".aarch64.rpm",
},
{
Os: "linux",
Arch: "arm64",
urlPostfix: ".linux-arm64.tar.gz",
},
{
Os: debOS,
Arch: "armv7",
urlPostfix: "_armhf.deb",
},
{
Os: debOS,
Arch: "armv6",
packagePostfix: "-rpi",
urlPostfix: "_armhf.deb",
},
{
Os: rhelOS,
Arch: "armv7",
urlPostfix: ".armhfp.rpm",
},
{
Os: "linux",
Arch: "armv6",
urlPostfix: ".linux-armv6.tar.gz",
},
{
Os: "linux",
Arch: "armv7",
urlPostfix: ".linux-armv7.tar.gz",
},
{
Os: "darwin",
Arch: "amd64",
urlPostfix: ".darwin-amd64.tar.gz",
},
{
Os: "deb",
Arch: "amd64",
urlPostfix: "_amd64.deb",
},
{
Os: rhelOS,
Arch: "amd64",
urlPostfix: ".x86_64.rpm",
},
{
Os: "linux",
Arch: "amd64",
urlPostfix: ".linux-amd64.tar.gz",
},
{
Os: "win",
Arch: "amd64",
urlPostfix: ".windows-amd64.zip",
},
{
Os: "win-installer",
Arch: "amd64",
urlPostfix: ".windows-amd64.msi",
},
}

323
pkg/build/cmd/grafanacom.go Normal file
View File

@ -0,0 +1,323 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/grafana/grafana/pkg/build/gcloud/storage"
"github.com/urfave/cli/v2"
)
const grafanaAPI = "https://grafana.com/api"
// GrafanaCom implements the sub-command "grafana-com".
func GrafanaCom(c *cli.Context) error {
bucketStr := c.String("src-bucket")
edition := config.Edition(c.String("edition"))
if err := gcloud.ActivateServiceAccount(); err != nil {
return fmt.Errorf("couldn't activate service account, err: %w", err)
}
metadata, err := GenerateMetadata(c)
if err != nil {
return err
}
releaseMode, err := metadata.GetReleaseMode()
if err != nil {
return err
}
version := metadata.GrafanaVersion
if releaseMode.Mode == config.Cronjob {
gcs, err := storage.New()
if err != nil {
return err
}
bucket := gcs.Bucket(bucketStr)
latestMainVersion, err := storage.GetLatestMainBuild(c.Context, bucket, filepath.Join(string(edition), "main"))
if err != nil {
return err
}
version = latestMainVersion
}
dryRun := c.Bool("dry-run")
simulateRelease := c.Bool("simulate-release")
// Test release mode and dryRun imply simulateRelease
if releaseMode.IsTest || dryRun {
simulateRelease = true
}
grafanaAPIKey := strings.TrimSpace(os.Getenv("GRAFANA_COM_API_KEY"))
if grafanaAPIKey == "" {
return cli.NewExitError("the environment variable GRAFANA_COM_API_KEY must be set", 1)
}
whatsNewURL, releaseNotesURL, err := getReleaseURLs()
if err != nil {
return cli.NewExitError(err.Error(), 1)
}
// TODO: Verify config values
cfg := PublishConfig{
Config: config.Config{
Version: version,
},
Edition: edition,
ReleaseMode: releaseMode,
GrafanaAPIKey: grafanaAPIKey,
WhatsNewURL: whatsNewURL,
ReleaseNotesURL: releaseNotesURL,
DryRun: dryRun,
TTL: c.String("ttl"),
SimulateRelease: simulateRelease,
}
if err := publishPackages(cfg); err != nil {
return cli.NewExitError(err.Error(), 1)
}
log.Println("Successfully published packages to grafana.com!")
return nil
}
func getReleaseURLs() (string, string, error) {
type grafanaConf struct {
WhatsNewURL string `json:"whatsNewUrl"`
ReleaseNotesURL string `json:"releaseNotesUrl"`
}
type packageConf struct {
Grafana grafanaConf `json:"grafana"`
}
pkgB, err := os.ReadFile("package.json")
if err != nil {
return "", "", fmt.Errorf("failed to read package.json: %w", err)
}
var pconf packageConf
if err := json.Unmarshal(pkgB, &pconf); err != nil {
return "", "", fmt.Errorf("failed to decode package.json: %w", err)
}
if _, err := url.ParseRequestURI(pconf.Grafana.WhatsNewURL); err != nil {
return "", "", fmt.Errorf("grafana.whatsNewUrl is invalid in package.json: %q", pconf.Grafana.WhatsNewURL)
}
if _, err := url.ParseRequestURI(pconf.Grafana.ReleaseNotesURL); err != nil {
return "", "", fmt.Errorf("grafana.releaseNotesUrl is invalid in package.json: %q",
pconf.Grafana.ReleaseNotesURL)
}
return pconf.Grafana.WhatsNewURL, pconf.Grafana.ReleaseNotesURL, nil
}
// publishPackages publishes packages to grafana.com.
func publishPackages(cfg PublishConfig) error {
log.Printf("Publishing Grafana packages, version %s, %s edition, %s mode, dryRun: %v, simulating: %v...\n",
cfg.Version, cfg.Edition, cfg.ReleaseMode.Mode, cfg.DryRun, cfg.SimulateRelease)
versionStr := fmt.Sprintf("v%s", cfg.Version)
log.Printf("Creating release %s at grafana.com...\n", versionStr)
var sfx string
var pth string
switch cfg.Edition {
case config.EditionOSS:
pth = "oss"
case config.EditionEnterprise:
pth = "enterprise"
sfx = EnterpriseSfx
default:
return fmt.Errorf("unrecognized edition %q", cfg.Edition)
}
switch cfg.ReleaseMode.Mode {
case config.MainMode, config.CustomMode, config.CronjobMode:
pth = path.Join(pth, MainFolder)
default:
pth = path.Join(pth, ReleaseFolder)
}
product := fmt.Sprintf("grafana%s", sfx)
pth = path.Join(pth, product)
baseArchiveURL := fmt.Sprintf("https://dl.grafana.com/%s", pth)
var builds []buildRepr
for _, ba := range ArtifactConfigs {
u := ba.GetURL(baseArchiveURL, cfg)
sha256, err := getSHA256(u)
if err != nil {
return err
}
builds = append(builds, buildRepr{
OS: ba.Os,
URL: u,
SHA256: string(sha256),
Arch: ba.Arch,
})
}
r := releaseRepr{
Version: cfg.Version,
ReleaseDate: time.Now().UTC(),
Builds: builds,
Stable: cfg.ReleaseMode.Mode == config.TagMode,
Beta: cfg.ReleaseMode.IsBeta,
Nightly: cfg.ReleaseMode.Mode == config.CronjobMode,
}
if cfg.ReleaseMode.Mode == config.TagMode || r.Beta {
r.WhatsNewURL = cfg.WhatsNewURL
r.ReleaseNotesURL = cfg.ReleaseNotesURL
}
if err := postRequest(cfg, "versions", r, fmt.Sprintf("create release %s", r.Version)); err != nil {
return err
}
if err := postRequest(cfg, fmt.Sprintf("versions/%s", cfg.Version), r,
fmt.Sprintf("update release %s", cfg.Version)); err != nil {
return err
}
for _, b := range r.Builds {
if err := postRequest(cfg, fmt.Sprintf("versions/%s/packages", cfg.Version), b,
fmt.Sprintf("create build %s %s", b.OS, b.Arch)); err != nil {
return err
}
if err := postRequest(cfg, fmt.Sprintf("versions/%s/packages/%s/%s", cfg.Version, b.Arch, b.OS), b,
fmt.Sprintf("update build %s %s", b.OS, b.Arch)); err != nil {
return err
}
}
return nil
}
func getSHA256(u string) ([]byte, error) {
shaURL := fmt.Sprintf("%s.sha256", u)
// nolint:gosec
resp, err := http.Get(shaURL)
if err != nil {
return nil, err
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Println("failed to close response body, err: %w", err)
}
}()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("failed downloading %s: %s", u, resp.Status)
}
var sha256 []byte
if err := json.NewDecoder(resp.Body).Decode(&sha256); err != nil {
return nil, err
}
return sha256, nil
}
func postRequest(cfg PublishConfig, pth string, obj interface{}, descr string) error {
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
default:
return fmt.Errorf("unrecognized edition %q", cfg.Edition)
}
product := fmt.Sprintf("grafana%s", sfx)
jsonB, err := json.Marshal(obj)
if err != nil {
return fmt.Errorf("failed to JSON encode release: %w", err)
}
u, err := constructURL(product, pth)
if err != nil {
return err
}
req, err := http.NewRequest(http.MethodPost, u, bytes.NewReader(jsonB))
if err != nil {
return err
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", cfg.GrafanaAPIKey))
req.Header.Add("Content-Type", "application/json")
log.Printf("Posting to grafana.com API, %s - JSON: %s\n", u, string(jsonB))
if cfg.SimulateRelease {
log.Println("Only simulating request")
return nil
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("failed posting to %s (%s): %s", u, descr, err)
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Println("failed to close response body, err: %w", err)
}
}()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
var body []byte
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
return err
}
if err != nil {
return err
}
if strings.Contains(string(body), "already exists") || strings.Contains(string(body), "Nothing to update") {
log.Printf("Already exists: %s\n", descr)
return nil
}
return fmt.Errorf("failed posting to %s (%s): %s", u, descr, resp.Status)
}
log.Printf("Successfully posted to grafana.com API, %s\n", u)
return nil
}
func constructURL(product string, pth string) (string, error) {
productPath := filepath.Clean(filepath.Join("/", product, pth))
u, err := url.Parse(grafanaAPI)
if err != nil {
return "", err
}
u.Path = path.Join(u.Path, productPath)
return u.String(), err
}
type buildRepr struct {
OS string `json:"os"`
URL string `json:"url"`
SHA256 string `json:"sha256"`
Arch string `json:"arch"`
}
type releaseRepr struct {
Version string `json:"version"`
ReleaseDate time.Time `json:"releaseDate"`
Stable bool `json:"stable"`
Beta bool `json:"beta"`
Nightly bool `json:"nightly"`
WhatsNewURL string `json:"whatsNewUrl"`
ReleaseNotesURL string `json:"releaseNotesUrl"`
Builds []buildRepr `json:"-"`
}

35
pkg/build/cmd/grafanacom_test.go Normal file
View File

@ -0,0 +1,35 @@
package main
import (
"testing"
)
func Test_constructURL(t *testing.T) {
type args struct {
product string
pth string
}
tests := []struct {
name string
args args
want string
wantErr bool
}{
{name: "cleans .. sequence", args: args{"..", ".."}, want: "https://grafana.com/api", wantErr: false},
{name: "doesn't clean anything - non malicious url", args: args{"foo", "bar"}, want: "https://grafana.com/api/foo/bar", wantErr: false},
{name: "doesn't clean anything - three dots", args: args{"...", "..."}, want: "https://grafana.com/api/.../...", wantErr: false},
{name: "cleans .", args: args{"..", ".."}, want: "https://grafana.com/api", wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := constructURL(tt.args.product, tt.args.pth)
if (err != nil) != tt.wantErr {
t.Errorf("constructURL() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("constructURL() got = %v, want %v", got, tt.want)
}
})
}
}

15
pkg/build/cmd/main.go
View File

@ -216,6 +216,21 @@ func main() {
},
},
},
{
Name: "grafana-com",
Usage: "Publish packages to grafana.com",
Action: GrafanaCom,
Flags: []cli.Flag{
&editionFlag,
&buildIDFlag,
&dryRunFlag,
&cli.StringFlag{
Name: "src-bucket",
Value: "grafana-downloads",
Usage: "Google Cloud Storage bucket",
},
},
},
},
},
}

2
pkg/build/config/version_mode.go
View File

@ -9,6 +9,7 @@ const (
ReleaseBranchMode VersionMode = "branch"
PullRequestMode VersionMode = "pull_request"
CustomMode VersionMode = "custom"
CronjobMode VersionMode = "cron"
)
const (
@ -17,6 +18,7 @@ const (
Push = "push"
Custom = "custom"
Promote = "promote"
Cronjob = "cron"
)
const (

36
pkg/build/gcloud/storage/gsutil.go
View File

@ -10,6 +10,7 @@ import (
"os"
"path"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
@ -358,6 +359,41 @@ func (client *Client) DownloadDirectory(ctx context.Context, bucket *storage.Buc
return nil
}
// GetLatestMainBuild gets the latest main build which is successfully uploaded to the gcs bucket.
func GetLatestMainBuild(ctx context.Context, bucket *storage.BucketHandle, path string) (string, error) {
if bucket == nil {
return "", ErrorNilBucket
}
it := bucket.Objects(ctx, &storage.Query{
Prefix: path,
})
var files []string
for {
attrs, err := it.Next()
if errors.Is(err, iterator.Done) {
break
}
if err != nil {
return "", fmt.Errorf("failed to iterate through bucket, err: %w", err)
}
files = append(files, attrs.Name)
}
var latestVersion string
for i := len(files) - 1; i >= 0; i-- {
captureVersion := regexp.MustCompile(`(\d+\.\d+\.\d+-\d+pre)`)
if captureVersion.MatchString(files[i]) {
latestVersion = captureVersion.FindString(files[i])
break
}
}
return latestVersion, nil
}
// downloadFile downloads an object to a file.
func (client *Client) downloadFile(ctx context.Context, bucket *storage.BucketHandle, objectName, destFileName string) error {
if bucket == nil {