IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2024-12-28 18:01:40 -06:00
Code Issues Packages Projects Releases Wiki Activity

CI: Move fetch-images sub-command from build-pipeline repo (#55298)

Browse Source 
* Move fetch images from build-pipeline

* Modify drone

* Replace zerolog with builtin log

* Fix lint
This commit is contained in:
Dimitris Sotirakis 2022-09-19 11:19:31 +03:00 committed by GitHub
parent 754eea20b3
commit 93e78e2146
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 197 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
.drone.yml
View File

@ -3038,10 +3038,16 @@ steps:
image: byrnedo/alpine-curl:0.1.8
name: grabpl
- commands:
- ./bin/grabpl artifacts docker fetch --version-tag ${TAG} --edition oss --base
alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7
- go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
depends_on: []
environment:
CGO_ENABLED: 0
image: golang:1.19.1
name: compile-build-cmd
- commands:
- ./bin/build artifacts docker fetch --edition oss
depends_on:
- grabpl
- compile-build-cmd
environment:
DOCKER_PASSWORD:
from_secret: docker_password
@ -3118,10 +3124,16 @@ steps:
image: byrnedo/alpine-curl:0.1.8
name: grabpl
- commands:
- ./bin/grabpl artifacts docker fetch --version-tag ${TAG} --edition enterprise
--base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7
- go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
depends_on: []
environment:
CGO_ENABLED: 0
image: golang:1.19.1
name: compile-build-cmd
- commands:
- ./bin/build artifacts docker fetch --edition enterprise
depends_on:
- grabpl
- compile-build-cmd
environment:
DOCKER_PASSWORD:
from_secret: docker_password
@ -3181,10 +3193,16 @@ steps:
image: byrnedo/alpine-curl:0.1.8
name: grabpl
- commands:
- ./bin/grabpl artifacts docker fetch --version-tag ${TAG} --edition oss --base
alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7
- go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
depends_on: []
environment:
CGO_ENABLED: 0
image: golang:1.19.1
name: compile-build-cmd
- commands:
- ./bin/build artifacts docker fetch --edition oss
depends_on:
- grabpl
- compile-build-cmd
environment:
DOCKER_PASSWORD:
from_secret: docker_password
@ -3262,10 +3280,16 @@ steps:
image: byrnedo/alpine-curl:0.1.8
name: grabpl
- commands:
- ./bin/grabpl artifacts docker fetch --version-tag ${TAG} --edition enterprise
--base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7
- go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
depends_on: []
environment:
CGO_ENABLED: 0
image: golang:1.19.1
name: compile-build-cmd
- commands:
- ./bin/build artifacts docker fetch --edition enterprise
depends_on:
- grabpl
- compile-build-cmd
environment:
DOCKER_PASSWORD:
from_secret: docker_password
@ -5060,6 +5084,6 @@ kind: secret
name: packages_secret_access_key
---
kind: signature
hmac: 1bbfd995ded7c2d1c0330ef2009691577a6613ec98df296fc8ec4388b9898e2d
hmac: f5d430e8ea6ad2f889f24ffcaa5a2555e77b45fab364b6db85efa591bedc0a6b
...

116
pkg/build/cmd/fetchimages.go Normal file
View File

@ -0,0 +1,116 @@
package main
import (
"fmt"
"log"
"os/exec"
"strings"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/docker"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/urfave/cli/v2"
)
const (
alpine = "alpine"
ubuntu = "ubuntu"
)
func FetchImages(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.NewExitError(err.Error(), 1)
}
return cli.NewExitError("", 1)
}
metadata, err := GenerateMetadata(c)
if err != nil {
return err
}
buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode)
if err != nil {
return err
}
cfg := docker.Config{
Archs: buildConfig.Docker.Architectures,
Distribution: buildConfig.Docker.Distribution,
Bucket: buildConfig.Docker.PrereleaseBucket,
Edition: c.String("edition"),
Tag: metadata.GrafanaVersion,
}
edition := fmt.Sprintf("-%s", cfg.Edition)
err = gcloud.ActivateServiceAccount()
if err != nil {
return err
}
var basesStr []string
for _, base := range cfg.Distribution {
switch base {
case alpine:
basesStr = append(basesStr, "")
case ubuntu:
basesStr = append(basesStr, "-ubuntu")
default:
return fmt.Errorf("unrecognized base %q", base)
}
}
err = downloadFromGCS(cfg, basesStr, edition)
if err != nil {
return err
}
err = loadImages(cfg, basesStr, edition)
if err != nil {
return err
}
return nil
}
func loadImages(cfg docker.Config, basesStr []string, edition string) error {
log.Println("Loading fetched image files to local docker registry...")
log.Printf("Number of images to be loaded: %d\n", len(basesStr)*len(cfg.Archs))
for _, base := range basesStr {
for _, arch := range cfg.Archs {
imageFilename := fmt.Sprintf("grafana%s-%s%s-%s.img", edition, cfg.Tag, base, arch)
log.Printf("image file name: %s\n", imageFilename)
//nolint:gosec
cmd := exec.Command("docker", "load", "-i", imageFilename)
cmd.Dir = "."
out, err := cmd.CombinedOutput()
if err != nil {
log.Printf("out: %s\n", out)
return fmt.Errorf("error loading image: %q", err)
}
log.Printf("Successfully loaded %s!\n %s\n", fmt.Sprintf("grafana%s-%s%s-%s", edition, cfg.Tag, base, arch), out)
}
}
log.Println("Images successfully loaded!")
return nil
}
func downloadFromGCS(cfg docker.Config, basesStr []string, edition string) error {
log.Printf("Downloading Docker images from GCS bucket: %s\n", cfg.Bucket)
for _, base := range basesStr {
for _, arch := range cfg.Archs {
src := fmt.Sprintf("gs://%s/%s/grafana%s-%s%s-%s.img", cfg.Bucket, cfg.Tag, edition, cfg.Tag, base, arch)
args := strings.Split(fmt.Sprintf("-m cp -r %s .", src), " ")
//nolint:gosec
cmd := exec.Command("gsutil", args...)
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("failed to download: %w\n%s", err, out)
}
}
}
log.Printf("Successfully fetched image files from %s bucket!\n", cfg.Bucket)
return nil
}

15
pkg/build/cmd/main.go
View File

@ -135,6 +135,21 @@ func main() {
},
},
},
{
Name: "docker",
Usage: "Handle Grafana Docker images",
Subcommands: cli.Commands{
{
Name: "fetch",
Usage: "Fetch Grafana Docker images",
ArgsUsage: "[version]",
Action: ArgCountWrapper(1, FetchImages),
Flags: []cli.Flag{
&editionFlag,
},
},
},
},
}
if err := app.Run(os.Args); err != nil {

3
pkg/build/cmd/storestorybook.go
View File

@ -22,9 +22,6 @@ func StoreStorybook(c *cli.Context) error {
if err != nil {
return err
}
if err != nil {
return err
}
storybookBucket := buildConfig.Buckets.Storybook
srcPath := buildConfig.Buckets.StorybookSrcDir

7
pkg/build/config/version.go
View File

@ -28,9 +28,10 @@ type PluginSignature struct {
}
type Docker struct {
ShouldSave bool `json:"shouldSave,omitempty"`
Distribution []Distribution `json:"distribution,omitempty"`
Architectures []Architecture `json:"archs,omitempty"`
ShouldSave bool `json:"shouldSave,omitempty"`
Distribution []Distribution `json:"distribution,omitempty"`
Architectures []Architecture `json:"archs,omitempty"`
PrereleaseBucket string `json:"prereleaseBucket,omitempty"`
}
type Buckets struct {

1
pkg/build/config/versions.go
View File

@ -151,6 +151,7 @@ var Versions = VersionMap{
Alpine,
Ubuntu,
},
PrereleaseBucket: "grafana-prerelease/artifacts/docker",
},
Buckets: Buckets{
Artifacts: "grafana-prerelease/artifacts/downloads",

15
pkg/build/docker/dockerconfig.go Normal file
View File

@ -0,0 +1,15 @@
package docker
import (
"github.com/grafana/grafana/pkg/build/config"
)
type Config struct {
Bucket string
Edition string
Tag string
Distribution []config.Distribution
Archs []config.Architecture
DockerHubRepo string
Security bool
}

17
scripts/drone/events/release.star
View File

@ -120,22 +120,23 @@ def fetch_images_step(edition):
'DOCKER_USER': from_secret('docker_username'),
'DOCKER_PASSWORD': from_secret('docker_password'),
},
'commands': ['./bin/grabpl artifacts docker fetch --version-tag ${{TAG}} --edition {} --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7'.format(edition)],
'depends_on': ['grabpl'],
'commands': ['./bin/build artifacts docker fetch --edition {}'.format(edition)],
'depends_on': ['compile-build-cmd'],
'volumes': [{
'name': 'docker',
'path': '/var/run/docker.sock'
}],
}
def publish_image_steps(version, mode, docker_repo, additional_docker_repo=""):
def publish_image_steps(edition, mode, docker_repo, additional_docker_repo=""):
steps = [
download_grabpl_step(),
fetch_images_step(version),
publish_images_step(version, 'release', mode, docker_repo),
compile_build_cmd(),
fetch_images_step(edition),
publish_images_step(edition, 'release', mode, docker_repo),
]
if additional_docker_repo != "":
steps.extend([publish_images_step(version, 'release', mode, additional_docker_repo)])
steps.extend([publish_images_step(edition, 'release', mode, additional_docker_repo)])
return steps
@ -146,9 +147,9 @@ def publish_image_pipelines(mode):
}
return [pipeline(
name='publish-docker-oss-{}'.format(mode), trigger=trigger, steps=publish_image_steps(version='oss', mode=mode, docker_repo='grafana', additional_docker_repo='grafana-oss'), edition=""
name='publish-docker-oss-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='oss', mode=mode, docker_repo='grafana', additional_docker_repo='grafana-oss'), edition=""
), pipeline(
name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(version='enterprise', mode=mode, docker_repo='grafana-enterprise'), edition=""
name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana-enterprise'), edition=""
),]
def get_oss_pipelines(trigger, ver_mode):