diff --git a/pkg/build/cmd/deb.go b/pkg/build/cmd/deb.go deleted file mode 100644 index 8ec651a91ec..00000000000 --- a/pkg/build/cmd/deb.go +++ /dev/null @@ -1,243 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "os/exec" - "path" - "path/filepath" - "strings" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/packaging" - "github.com/grafana/grafana/pkg/infra/fs" - "github.com/urfave/cli/v2" -) - -func writeAptlyConf(dbDir, repoDir string) error { - aptlyConf := fmt.Sprintf(`{ - "rootDir": "%s", - "downloadConcurrency": 4, - "downloadSpeedLimit": 0, - "architectures": [], - "dependencyFollowSuggests": false, - "dependencyFollowRecommends": false, - "dependencyFollowAllVariants": false, - "dependencyFollowSource": false, - "dependencyVerboseResolve": false, - "gpgDisableSign": false, - "gpgDisableVerify": false, - "gpgProvider": "gpg2", - "downloadSourcePackages": false, - "skipLegacyPool": true, - "ppaDistributorID": "ubuntu", - "ppaCodename": "", - "skipContentsPublishing": false, - "FileSystemPublishEndpoints": { - "repo": { - "rootDir": "%s", - "linkMethod": "copy" - } - }, - "S3PublishEndpoints": {}, - "SwiftPublishEndpoints": {} -} -`, dbDir, repoDir) - home, err := os.UserHomeDir() - if err != nil { - return err - } - return os.WriteFile(filepath.Join(home, ".aptly.conf"), []byte(aptlyConf), 0600) -} - -// downloadDebs downloads Deb packages. -func downloadDebs(cfg PublishConfig, workDir string) error { - if cfg.Bucket == "" { - panic("cfg.Bucket has to be set") - } - if !strings.HasSuffix(workDir, string(filepath.Separator)) { - workDir += string(filepath.Separator) - } - - var version string - if cfg.ReleaseMode.Mode == config.TagMode { - if cfg.ReleaseMode.IsBeta { - version = strings.ReplaceAll(cfg.Version, "-", "~") - } else { - version = cfg.Version - } - } - if version == "" { - panic(fmt.Sprintf("Unrecognized version mode %s", cfg.ReleaseMode.Mode)) - } - - var sfx string - switch cfg.Edition { - case config.EditionOSS: - case config.EditionEnterprise: - sfx = EnterpriseSfx - default: - return fmt.Errorf("unrecognized edition %q", cfg.Edition) - } - - u := fmt.Sprintf("gs://%s/%s/%s/grafana%s_%s_*.deb*", cfg.Bucket, - strings.ToLower(string(cfg.Edition)), ReleaseFolder, sfx, version) - log.Printf("Downloading Deb packages %q...\n", u) - args := []string{ - "-m", - "cp", - u, - workDir, - } - //nolint:gosec - cmd := exec.Command("gsutil", args...) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to download Deb packages %q: %w\n%s", u, err, output) - } - - return nil -} - -// updateDebRepo updates the Debian repository with the new release. -func updateDebRepo(cfg PublishConfig, workDir string) error { - if cfg.ReleaseMode.Mode != config.TagMode { - panic(fmt.Sprintf("Unsupported version mode: %s", cfg.ReleaseMode.Mode)) - } - - if cfg.ReleaseMode.IsTest { - if cfg.Config.DebDBBucket == packaging.DefaultDebDBBucket { - return fmt.Errorf("in test-release mode, the default Deb DB bucket shouldn't be used") - } - if cfg.Config.DebRepoBucket == packaging.DefaultDebRepoBucket { - return fmt.Errorf("in test-release mode, the default Deb repo bucket shouldn't be used") - } - } - - if err := downloadDebs(cfg, workDir); err != nil { - return err - } - - repoName := "grafana" - if cfg.ReleaseMode.IsBeta { - repoName = "beta" - } - - repoRoot := path.Join(os.TempDir(), "deb-repo") - defer func() { - if err := os.RemoveAll(repoRoot); err != nil { - log.Printf("Failed to remove temporary directory %q: %s\n", repoRoot, err.Error()) - } - }() - - dbDir := filepath.Join(repoRoot, "db") - repoDir := filepath.Join(repoRoot, "repo") - tmpDir := filepath.Join(repoRoot, "tmp") - for _, dpath := range []string{dbDir, repoDir, tmpDir} { - if err := os.MkdirAll(dpath, 0750); err != nil { - return err - } - } - - if err := writeAptlyConf(dbDir, repoDir); err != nil { - return err - } - - // Download the Debian repo database - u := fmt.Sprintf("gs://%s/%s", cfg.DebDBBucket, strings.ToLower(string(cfg.Edition))) - log.Printf("Downloading Debian repo database from %s...\n", u) - //nolint:gosec - cmd := exec.Command("gsutil", "-m", "rsync", "-r", "-d", u, dbDir) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to download Debian repo database: %w\n%s", err, output) - } - - if err := addPkgsToRepo(cfg, workDir, tmpDir, repoName); err != nil { - return err - } - - log.Println("Updating local Debian package repository...") - // Update published local repository. This assumes that there exists already a local, published repo. - for _, tp := range []string{"stable", "beta"} { - passArg := fmt.Sprintf("-passphrase-file=%s", cfg.GPGPassPath) - //nolint:gosec - cmd := exec.Command("aptly", "publish", "update", "-batch", passArg, "-force-overwrite", tp, - "filesystem:repo:grafana") - if output, err := cmd.CombinedOutput(); err != nil { - return cli.NewExitError(fmt.Sprintf("failed to update Debian %q repository: %s", tp, output), 1) - } - } - - // Update database in GCS - u = fmt.Sprintf("gs://%s/%s", cfg.DebDBBucket, strings.ToLower(string(cfg.Edition))) - if cfg.DryRun { - log.Printf("Simulating upload of Debian repo database to GCS (%s)\n", u) - } else { - log.Printf("Uploading Debian repo database to GCS (%s)...\n", u) - //nolint:gosec - cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", dbDir, u) - if output, err := cmd.CombinedOutput(); err != nil { - return cli.NewExitError(fmt.Sprintf("failed to upload Debian repo database to GCS: %s", output), 1) - } - } - - // Update metadata and binaries in repository bucket - u = fmt.Sprintf("gs://%s/%s/deb", cfg.DebRepoBucket, strings.ToLower(string(cfg.Edition))) - grafDir := filepath.Join(repoDir, "grafana") - if cfg.DryRun { - log.Printf("Simulating upload of Debian repo resources to GCS (%s)\n", u) - } else { - log.Printf("Uploading Debian repo resources to GCS (%s)...\n", u) - //nolint:gosec - cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", grafDir, u) - if output, err := cmd.CombinedOutput(); err != nil { - return cli.NewExitError(fmt.Sprintf("failed to upload Debian repo resources to GCS: %s", output), 1) - } - allRepoResources := fmt.Sprintf("%s/**/*", u) - log.Printf("Setting cache ttl for Debian repo resources on GCS (%s)...\n", allRepoResources) - //nolint:gosec - cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources) - if output, err := cmd.CombinedOutput(); err != nil { - return cli.NewExitError(fmt.Sprintf("failed to set cache ttl for Debian repo resources on GCS: %s", output), 1) - } - } - - return nil -} - -func addPkgsToRepo(cfg PublishConfig, workDir, tmpDir, repoName string) error { - var sfx string - switch cfg.Edition { - case config.EditionOSS: - case config.EditionEnterprise: - sfx = EnterpriseSfx - default: - return fmt.Errorf("unsupported edition %q", cfg.Edition) - } - - log.Printf("Adding packages to Debian %q repo...\n", repoName) - // TODO: Be more specific about filename pattern - debs, err := filepath.Glob(filepath.Join(workDir, fmt.Sprintf("grafana%s*.deb", sfx))) - if err != nil { - return err - } - for _, deb := range debs { - basename := filepath.Base(deb) - if strings.Contains(basename, "latest") { - continue - } - - tgt := filepath.Join(tmpDir, basename) - if err := fs.CopyFile(deb, tgt); err != nil { - return err - } - } - // XXX: Adds too many packages in enterprise (Arve: What does this mean exactly?) - //nolint:gosec - cmd := exec.Command("aptly", "repo", "add", "-force-replace", repoName, tmpDir) - if output, err := cmd.CombinedOutput(); err != nil { - return cli.NewExitError(fmt.Sprintf("failed to add packages to local Debian repository: %s", output), 1) - } - - return nil -} diff --git a/pkg/build/cmd/grafanacom.go b/pkg/build/cmd/grafanacom.go index 87358419b22..988347a14b9 100644 --- a/pkg/build/cmd/grafanacom.go +++ b/pkg/build/cmd/grafanacom.go @@ -17,6 +17,7 @@ import ( "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/gcloud" "github.com/grafana/grafana/pkg/build/gcloud/storage" + "github.com/grafana/grafana/pkg/build/packaging" "github.com/urfave/cli/v2" ) @@ -72,7 +73,7 @@ func GrafanaCom(c *cli.Context) error { } // TODO: Verify config values - cfg := PublishConfig{ + cfg := packaging.PublishConfig{ Config: config.Config{ Version: version, }, @@ -124,7 +125,7 @@ func getReleaseURLs() (string, string, error) { } // publishPackages publishes packages to grafana.com. -func publishPackages(cfg PublishConfig) error { +func publishPackages(cfg packaging.PublishConfig) error { log.Printf("Publishing Grafana packages, version %s, %s edition, %s mode, dryRun: %v, simulating: %v...\n", cfg.Version, cfg.Edition, cfg.ReleaseMode.Mode, cfg.DryRun, cfg.SimulateRelease) @@ -138,16 +139,16 @@ func publishPackages(cfg PublishConfig) error { pth = "oss" case config.EditionEnterprise: pth = "enterprise" - sfx = EnterpriseSfx + sfx = packaging.EnterpriseSfx default: return fmt.Errorf("unrecognized edition %q", cfg.Edition) } switch cfg.ReleaseMode.Mode { case config.MainMode, config.CustomMode, config.CronjobMode: - pth = path.Join(pth, MainFolder) + pth = path.Join(pth, packaging.MainFolder) default: - pth = path.Join(pth, ReleaseFolder) + pth = path.Join(pth, packaging.ReleaseFolder) } product := fmt.Sprintf("grafana%s", sfx) @@ -155,7 +156,7 @@ func publishPackages(cfg PublishConfig) error { baseArchiveURL := fmt.Sprintf("https://dl.grafana.com/%s", pth) var builds []buildRepr - for _, ba := range ArtifactConfigs { + for _, ba := range packaging.ArtifactConfigs { u := ba.GetURL(baseArchiveURL, cfg) sha256, err := getSHA256(u) @@ -230,12 +231,12 @@ func getSHA256(u string) ([]byte, error) { return sha256, nil } -func postRequest(cfg PublishConfig, pth string, obj interface{}, descr string) error { +func postRequest(cfg packaging.PublishConfig, pth string, obj interface{}, descr string) error { var sfx string switch cfg.Edition { case config.EditionOSS: case config.EditionEnterprise: - sfx = EnterpriseSfx + sfx = packaging.EnterpriseSfx default: return fmt.Errorf("unrecognized edition %q", cfg.Edition) } diff --git a/pkg/build/cmd/publishpackages.go b/pkg/build/cmd/publishpackages.go index b640688f261..d66067d99c4 100644 --- a/pkg/build/cmd/publishpackages.go +++ b/pkg/build/cmd/publishpackages.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/gcloud" "github.com/grafana/grafana/pkg/build/gpg" + "github.com/grafana/grafana/pkg/build/packaging" "github.com/urfave/cli/v2" ) @@ -43,7 +44,7 @@ func PublishPackages(c *cli.Context) error { edition := config.Edition(c.String("edition")) // TODO: Verify config values - cfg := PublishConfig{ + cfg := packaging.PublishConfig{ Config: config.Config{ Version: metadata.GrafanaVersion, Bucket: c.String("packages-bucket"), @@ -83,18 +84,18 @@ func PublishPackages(c *cli.Context) error { } // updatePkgRepos updates package manager repositories. -func updatePkgRepos(cfg PublishConfig, workDir string) error { +func updatePkgRepos(cfg packaging.PublishConfig, workDir string) error { if err := gpg.Import(cfg.Config); err != nil { return err } // If updating the Deb repo fails, still continue with the RPM repo, so we don't have to retry // both by hand - debErr := updateDebRepo(cfg, workDir) + debErr := packaging.UpdateDebRepo(cfg, workDir) if debErr != nil { log.Printf("Updating Deb repo failed: %s\n", debErr) } - rpmErr := updateRPMRepo(cfg, workDir) + rpmErr := packaging.UpdateRPMRepo(cfg, workDir) if rpmErr != nil { log.Printf("Updating RPM repo failed: %s\n", rpmErr) } diff --git a/pkg/build/cmd/rpm.go b/pkg/build/cmd/rpm.go deleted file mode 100644 index ef5725c959c..00000000000 --- a/pkg/build/cmd/rpm.go +++ /dev/null @@ -1,365 +0,0 @@ -package main - -import ( - "bytes" - "crypto" - "fmt" - "log" - "os" - "os/exec" - "path" - "path/filepath" - "strings" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/packaging" - "github.com/grafana/grafana/pkg/infra/fs" - "golang.org/x/crypto/openpgp" - "golang.org/x/crypto/openpgp/armor" - "golang.org/x/crypto/openpgp/packet" -) - -// updateRPMRepo updates the RPM repository with the new release. -func updateRPMRepo(cfg PublishConfig, workDir string) error { - if cfg.ReleaseMode.Mode != config.TagMode { - panic(fmt.Sprintf("Unsupported version mode %s", cfg.ReleaseMode.Mode)) - } - - if cfg.ReleaseMode.IsTest && cfg.Config.RPMRepoBucket == packaging.DefaultRPMRepoBucket { - return fmt.Errorf("in test-release mode, the default RPM repo bucket shouldn't be used") - } - - if err := downloadRPMs(cfg, workDir); err != nil { - return err - } - - repoRoot := path.Join(os.TempDir(), "rpm-repo") - defer func() { - if err := os.RemoveAll(repoRoot); err != nil { - log.Printf("Failed to remove %q: %s\n", repoRoot, err.Error()) - } - }() - - repoName := "rpm" - if cfg.ReleaseMode.IsBeta { - repoName = "rpm-beta" - } - folderURI := fmt.Sprintf("gs://%s/%s/%s", cfg.RPMRepoBucket, strings.ToLower(string(cfg.Edition)), repoName) - - // Download the RPM database - log.Printf("Downloading RPM database from GCS (%s)...\n", folderURI) - //nolint:gosec - cmd := exec.Command("gsutil", "-m", "rsync", "-r", "-d", folderURI, repoRoot) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to download RPM database from GCS: %w\n%s", err, output) - } - - // Add the new release to the repo - var sfx string - switch cfg.Edition { - case config.EditionOSS: - case config.EditionEnterprise: - sfx = EnterpriseSfx - default: - return fmt.Errorf("unsupported edition %q", cfg.Edition) - } - allRPMs, err := filepath.Glob(filepath.Join(workDir, fmt.Sprintf("grafana%s-*.rpm", sfx))) - if err != nil { - return fmt.Errorf("failed to list RPMs in %q: %w", workDir, err) - } - rpms := []string{} - for _, rpm := range allRPMs { - if strings.Contains(rpm, "-latest") { - continue - } - - rpms = append(rpms, rpm) - } - // XXX: What does the following comment mean? - // adds to many files for enterprise - for _, rpm := range rpms { - if err := fs.CopyFile(rpm, filepath.Join(repoRoot, filepath.Base(rpm))); err != nil { - return err - } - } - - //nolint:gosec - cmd = exec.Command("createrepo", repoRoot) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to create repo at %q: %w\n%s", repoRoot, err, output) - } - - if err := signRPMRepo(repoRoot, cfg); err != nil { - return err - } - - // Update the repo in GCS - - // Sync packages first to avoid cache misses - if cfg.DryRun { - log.Printf("Simulating upload of RPMs to GCS (%s)\n", folderURI) - } else { - log.Printf("Uploading RPMs to GCS (%s)...\n", folderURI) - args := []string{"-m", "cp"} - args = append(args, rpms...) - args = append(args, folderURI) - //nolint:gosec - cmd = exec.Command("gsutil", args...) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to upload RPMs to GCS: %w\n%s", err, output) - } - } - - if cfg.DryRun { - log.Printf("Simulating upload of RPM repo metadata to GCS (%s)\n", folderURI) - } else { - log.Printf("Uploading RPM repo metadata to GCS (%s)...\n", folderURI) - //nolint:gosec - cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", repoRoot, folderURI) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to upload RPM repo metadata to GCS: %w\n%s", err, output) - } - allRepoResources := fmt.Sprintf("%s/**/*", folderURI) - log.Printf("Setting cache ttl for RPM repo resources on GCS (%s)...\n", allRepoResources) - //nolint:gosec - cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to set cache ttl for RPM repo resources on GCS: %w\n%s", err, output) - } - } - - return nil -} - -// downloadRPMs downloads RPM packages. -func downloadRPMs(cfg PublishConfig, workDir string) error { - if !strings.HasSuffix(workDir, string(filepath.Separator)) { - workDir += string(filepath.Separator) - } - var version string - if cfg.ReleaseMode.Mode == config.TagMode { - if cfg.ReleaseMode.IsBeta { - version = strings.ReplaceAll(cfg.Version, "-", "~") - } else { - version = cfg.Version - } - } - if version == "" { - panic(fmt.Sprintf("Unrecognized version mode %s", cfg.ReleaseMode.Mode)) - } - - var sfx string - switch cfg.Edition { - case config.EditionOSS: - case config.EditionEnterprise: - sfx = EnterpriseSfx - default: - return fmt.Errorf("unrecognized edition %q", cfg.Edition) - } - - u := fmt.Sprintf("gs://%s/%s/%s/grafana%s-%s-*.*.rpm*", cfg.Bucket, - strings.ToLower(string(cfg.Edition)), ReleaseFolder, sfx, version) - log.Printf("Downloading RPM packages %q...\n", u) - args := []string{ - "-m", - "cp", - u, - workDir, - } - //nolint:gosec - cmd := exec.Command("gsutil", args...) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to download RPM packages %q: %w\n%s", u, err, output) - } - - return nil -} - -func getPublicKey(cfg PublishConfig) (*packet.PublicKey, error) { - f, err := os.Open(cfg.GPGPublicKey) - if err != nil { - return nil, fmt.Errorf("failed to open %q: %w", cfg.GPGPublicKey, err) - } - defer func(f *os.File) { - err := f.Close() - if err != nil { - return - } - }(f) - - block, err := armor.Decode(f) - if err != nil { - return nil, err - } - - if block.Type != openpgp.PublicKeyType { - return nil, fmt.Errorf("invalid public key block type: %q", block.Type) - } - - packetReader := packet.NewReader(block.Body) - pkt, err := packetReader.Next() - if err != nil { - return nil, err - } - - key, ok := pkt.(*packet.PublicKey) - if !ok { - return nil, fmt.Errorf("got non-public key from packet reader: %T", pkt) - } - - return key, nil -} - -func getPrivateKey(cfg PublishConfig) (*packet.PrivateKey, error) { - f, err := os.Open(cfg.GPGPrivateKey) - if err != nil { - return nil, fmt.Errorf("failed to open %q: %w", cfg.GPGPrivateKey, err) - } - defer func(f *os.File) { - err := f.Close() - if err != nil { - return - } - }(f) - - passphraseB, err := os.ReadFile(cfg.GPGPassPath) - if err != nil { - return nil, fmt.Errorf("failed to read %q: %w", cfg.GPGPrivateKey, err) - } - passphraseB = bytes.TrimSuffix(passphraseB, []byte("\n")) - - block, err := armor.Decode(f) - if err != nil { - return nil, err - } - - if block.Type != openpgp.PrivateKeyType { - return nil, fmt.Errorf("invalid private key block type: %q", block.Type) - } - - packetReader := packet.NewReader(block.Body) - pkt, err := packetReader.Next() - if err != nil { - return nil, err - } - - key, ok := pkt.(*packet.PrivateKey) - if !ok { - return nil, fmt.Errorf("got non-private key from packet reader: %T", pkt) - } - - if err := key.Decrypt(passphraseB); err != nil { - return nil, fmt.Errorf("failed to decrypt private key: %w", err) - } - return key, nil -} - -// signRPMRepo signs an RPM repository using PGP. -// The signature gets written to the file repodata/repomd.xml.asc. -func signRPMRepo(repoRoot string, cfg PublishConfig) error { - if cfg.GPGPublicKey == "" || cfg.GPGPrivateKey == "" { - return fmt.Errorf("private or public key is empty") - } - - log.Printf("Signing RPM repo") - - pubKey, err := getPublicKey(cfg) - if err != nil { - return err - } - - privKey, err := getPrivateKey(cfg) - if err != nil { - return err - } - - pcfg := packet.Config{ - DefaultHash: crypto.SHA256, - DefaultCipher: packet.CipherAES256, - DefaultCompressionAlgo: packet.CompressionZLIB, - CompressionConfig: &packet.CompressionConfig{ - Level: 9, - }, - RSABits: 4096, - } - currentTime := pcfg.Now() - uid := packet.NewUserId("", "", "") - - isPrimaryID := false - keyLifetimeSecs := uint32(86400 * 365) - signer := openpgp.Entity{ - PrimaryKey: pubKey, - PrivateKey: privKey, - Identities: map[string]*openpgp.Identity{ - uid.Id: { - Name: uid.Name, - UserId: uid, - SelfSignature: &packet.Signature{ - CreationTime: currentTime, - SigType: packet.SigTypePositiveCert, - PubKeyAlgo: packet.PubKeyAlgoRSA, - Hash: pcfg.Hash(), - IsPrimaryId: &isPrimaryID, - FlagsValid: true, - FlagSign: true, - FlagCertify: true, - IssuerKeyId: &pubKey.KeyId, - }, - }, - }, - Subkeys: []openpgp.Subkey{ - { - PublicKey: pubKey, - PrivateKey: privKey, - Sig: &packet.Signature{ - CreationTime: currentTime, - SigType: packet.SigTypeSubkeyBinding, - PubKeyAlgo: packet.PubKeyAlgoRSA, - Hash: pcfg.Hash(), - PreferredHash: []uint8{8}, // SHA-256 - FlagsValid: true, - FlagEncryptStorage: true, - FlagEncryptCommunications: true, - IssuerKeyId: &pubKey.KeyId, - KeyLifetimeSecs: &keyLifetimeSecs, - }, - }, - }, - } - - // Ignore gosec G304 as this function is only used in the build process. - //nolint:gosec - freader, err := os.Open(filepath.Join(repoRoot, "repodata", "repomd.xml")) - if err != nil { - return err - } - defer func(freader *os.File) { - err := freader.Close() - if err != nil { - return - } - }(freader) - - // Ignore gosec G304 as this function is only used in the build process. - //nolint:gosec - sigwriter, err := os.Create(filepath.Join(repoRoot, "repodata", "repomd.xml.asc")) - if err != nil { - return err - } - defer func(sigwriter *os.File) { - err := sigwriter.Close() - if err != nil { - return - } - }(sigwriter) - - if err := openpgp.ArmoredDetachSignText(sigwriter, &signer, freader, nil); err != nil { - return fmt.Errorf("failed to write PGP signature: %w", err) - } - - if err := sigwriter.Close(); err != nil { - return fmt.Errorf("failed to write PGP signature: %w", err) - } - - return nil -} diff --git a/pkg/build/cmd/artifacts.go b/pkg/build/packaging/artifacts.go similarity index 99% rename from pkg/build/cmd/artifacts.go rename to pkg/build/packaging/artifacts.go index f93d002b107..bdce67170e1 100644 --- a/pkg/build/cmd/artifacts.go +++ b/pkg/build/packaging/artifacts.go @@ -1,4 +1,4 @@ -package main +package packaging import ( "fmt" diff --git a/pkg/build/packaging/deb.go b/pkg/build/packaging/deb.go index c20b9edbfae..bae93282e69 100644 --- a/pkg/build/packaging/deb.go +++ b/pkg/build/packaging/deb.go @@ -1 +1,242 @@ package packaging + +import ( + "fmt" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/infra/fs" + "github.com/urfave/cli/v2" +) + +func writeAptlyConf(dbDir, repoDir string) error { + aptlyConf := fmt.Sprintf(`{ + "rootDir": "%s", + "downloadConcurrency": 4, + "downloadSpeedLimit": 0, + "architectures": [], + "dependencyFollowSuggests": false, + "dependencyFollowRecommends": false, + "dependencyFollowAllVariants": false, + "dependencyFollowSource": false, + "dependencyVerboseResolve": false, + "gpgDisableSign": false, + "gpgDisableVerify": false, + "gpgProvider": "gpg2", + "downloadSourcePackages": false, + "skipLegacyPool": true, + "ppaDistributorID": "ubuntu", + "ppaCodename": "", + "skipContentsPublishing": false, + "FileSystemPublishEndpoints": { + "repo": { + "rootDir": "%s", + "linkMethod": "copy" + } + }, + "S3PublishEndpoints": {}, + "SwiftPublishEndpoints": {} +} +`, dbDir, repoDir) + home, err := os.UserHomeDir() + if err != nil { + return err + } + return os.WriteFile(filepath.Join(home, ".aptly.conf"), []byte(aptlyConf), 0600) +} + +// downloadDebs downloads Deb packages. +func downloadDebs(cfg PublishConfig, workDir string) error { + if cfg.Bucket == "" { + panic("cfg.Bucket has to be set") + } + if !strings.HasSuffix(workDir, string(filepath.Separator)) { + workDir += string(filepath.Separator) + } + + var version string + if cfg.ReleaseMode.Mode == config.TagMode { + if cfg.ReleaseMode.IsBeta { + version = strings.ReplaceAll(cfg.Version, "-", "~") + } else { + version = cfg.Version + } + } + if version == "" { + panic(fmt.Sprintf("Unrecognized version mode %s", cfg.ReleaseMode.Mode)) + } + + var sfx string + switch cfg.Edition { + case config.EditionOSS: + case config.EditionEnterprise: + sfx = EnterpriseSfx + default: + return fmt.Errorf("unrecognized edition %q", cfg.Edition) + } + + u := fmt.Sprintf("gs://%s/%s/%s/grafana%s_%s_*.deb*", cfg.Bucket, + strings.ToLower(string(cfg.Edition)), ReleaseFolder, sfx, version) + log.Printf("Downloading Deb packages %q...\n", u) + args := []string{ + "-m", + "cp", + u, + workDir, + } + //nolint:gosec + cmd := exec.Command("gsutil", args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to download Deb packages %q: %w\n%s", u, err, output) + } + + return nil +} + +// UpdateDebRepo updates the Debian repository with the new release. +func UpdateDebRepo(cfg PublishConfig, workDir string) error { + if cfg.ReleaseMode.Mode != config.TagMode { + panic(fmt.Sprintf("Unsupported version mode: %s", cfg.ReleaseMode.Mode)) + } + + if cfg.ReleaseMode.IsTest { + if cfg.Config.DebDBBucket == DefaultDebDBBucket { + return fmt.Errorf("in test-release mode, the default Deb DB bucket shouldn't be used") + } + if cfg.Config.DebRepoBucket == DefaultDebRepoBucket { + return fmt.Errorf("in test-release mode, the default Deb repo bucket shouldn't be used") + } + } + + if err := downloadDebs(cfg, workDir); err != nil { + return err + } + + repoName := "grafana" + if cfg.ReleaseMode.IsBeta { + repoName = "beta" + } + + repoRoot := path.Join(os.TempDir(), "deb-repo") + defer func() { + if err := os.RemoveAll(repoRoot); err != nil { + log.Printf("Failed to remove temporary directory %q: %s\n", repoRoot, err.Error()) + } + }() + + dbDir := filepath.Join(repoRoot, "db") + repoDir := filepath.Join(repoRoot, "repo") + tmpDir := filepath.Join(repoRoot, "tmp") + for _, dpath := range []string{dbDir, repoDir, tmpDir} { + if err := os.MkdirAll(dpath, 0750); err != nil { + return err + } + } + + if err := writeAptlyConf(dbDir, repoDir); err != nil { + return err + } + + // Download the Debian repo database + u := fmt.Sprintf("gs://%s/%s", cfg.DebDBBucket, strings.ToLower(string(cfg.Edition))) + log.Printf("Downloading Debian repo database from %s...\n", u) + //nolint:gosec + cmd := exec.Command("gsutil", "-m", "rsync", "-r", "-d", u, dbDir) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to download Debian repo database: %w\n%s", err, output) + } + + if err := addPkgsToRepo(cfg, workDir, tmpDir, repoName); err != nil { + return err + } + + log.Println("Updating local Debian package repository...") + // Update published local repository. This assumes that there exists already a local, published repo. + for _, tp := range []string{"stable", "beta"} { + passArg := fmt.Sprintf("-passphrase-file=%s", cfg.GPGPassPath) + //nolint:gosec + cmd := exec.Command("aptly", "publish", "update", "-batch", passArg, "-force-overwrite", tp, + "filesystem:repo:grafana") + if output, err := cmd.CombinedOutput(); err != nil { + return cli.NewExitError(fmt.Sprintf("failed to update Debian %q repository: %s", tp, output), 1) + } + } + + // Update database in GCS + u = fmt.Sprintf("gs://%s/%s", cfg.DebDBBucket, strings.ToLower(string(cfg.Edition))) + if cfg.DryRun { + log.Printf("Simulating upload of Debian repo database to GCS (%s)\n", u) + } else { + log.Printf("Uploading Debian repo database to GCS (%s)...\n", u) + //nolint:gosec + cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", dbDir, u) + if output, err := cmd.CombinedOutput(); err != nil { + return cli.NewExitError(fmt.Sprintf("failed to upload Debian repo database to GCS: %s", output), 1) + } + } + + // Update metadata and binaries in repository bucket + u = fmt.Sprintf("gs://%s/%s/deb", cfg.DebRepoBucket, strings.ToLower(string(cfg.Edition))) + grafDir := filepath.Join(repoDir, "grafana") + if cfg.DryRun { + log.Printf("Simulating upload of Debian repo resources to GCS (%s)\n", u) + } else { + log.Printf("Uploading Debian repo resources to GCS (%s)...\n", u) + //nolint:gosec + cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", grafDir, u) + if output, err := cmd.CombinedOutput(); err != nil { + return cli.NewExitError(fmt.Sprintf("failed to upload Debian repo resources to GCS: %s", output), 1) + } + allRepoResources := fmt.Sprintf("%s/**/*", u) + log.Printf("Setting cache ttl for Debian repo resources on GCS (%s)...\n", allRepoResources) + //nolint:gosec + cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources) + if output, err := cmd.CombinedOutput(); err != nil { + return cli.NewExitError(fmt.Sprintf("failed to set cache ttl for Debian repo resources on GCS: %s", output), 1) + } + } + + return nil +} + +func addPkgsToRepo(cfg PublishConfig, workDir, tmpDir, repoName string) error { + var sfx string + switch cfg.Edition { + case config.EditionOSS: + case config.EditionEnterprise: + sfx = EnterpriseSfx + default: + return fmt.Errorf("unsupported edition %q", cfg.Edition) + } + + log.Printf("Adding packages to Debian %q repo...\n", repoName) + // TODO: Be more specific about filename pattern + debs, err := filepath.Glob(filepath.Join(workDir, fmt.Sprintf("grafana%s*.deb", sfx))) + if err != nil { + return err + } + for _, deb := range debs { + basename := filepath.Base(deb) + if strings.Contains(basename, "latest") { + continue + } + + tgt := filepath.Join(tmpDir, basename) + if err := fs.CopyFile(deb, tgt); err != nil { + return err + } + } + // XXX: Adds too many packages in enterprise (Arve: What does this mean exactly?) + //nolint:gosec + cmd := exec.Command("aptly", "repo", "add", "-force-replace", repoName, tmpDir) + if output, err := cmd.CombinedOutput(); err != nil { + return cli.NewExitError(fmt.Sprintf("failed to add packages to local Debian repository: %s", output), 1) + } + + return nil +} diff --git a/pkg/build/packaging/rpm.go b/pkg/build/packaging/rpm.go index c20b9edbfae..a75ba9d6b1e 100644 --- a/pkg/build/packaging/rpm.go +++ b/pkg/build/packaging/rpm.go @@ -1 +1,364 @@ package packaging + +import ( + "bytes" + "crypto" + "fmt" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/infra/fs" + "golang.org/x/crypto/openpgp" + "golang.org/x/crypto/openpgp/armor" + "golang.org/x/crypto/openpgp/packet" +) + +// UpdateRPMRepo updates the RPM repository with the new release. +func UpdateRPMRepo(cfg PublishConfig, workDir string) error { + if cfg.ReleaseMode.Mode != config.TagMode { + panic(fmt.Sprintf("Unsupported version mode %s", cfg.ReleaseMode.Mode)) + } + + if cfg.ReleaseMode.IsTest && cfg.Config.RPMRepoBucket == DefaultRPMRepoBucket { + return fmt.Errorf("in test-release mode, the default RPM repo bucket shouldn't be used") + } + + if err := downloadRPMs(cfg, workDir); err != nil { + return err + } + + repoRoot := path.Join(os.TempDir(), "rpm-repo") + defer func() { + if err := os.RemoveAll(repoRoot); err != nil { + log.Printf("Failed to remove %q: %s\n", repoRoot, err.Error()) + } + }() + + repoName := "rpm" + if cfg.ReleaseMode.IsBeta { + repoName = "rpm-beta" + } + folderURI := fmt.Sprintf("gs://%s/%s/%s", cfg.RPMRepoBucket, strings.ToLower(string(cfg.Edition)), repoName) + + // Download the RPM database + log.Printf("Downloading RPM database from GCS (%s)...\n", folderURI) + //nolint:gosec + cmd := exec.Command("gsutil", "-m", "rsync", "-r", "-d", folderURI, repoRoot) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to download RPM database from GCS: %w\n%s", err, output) + } + + // Add the new release to the repo + var sfx string + switch cfg.Edition { + case config.EditionOSS: + case config.EditionEnterprise: + sfx = EnterpriseSfx + default: + return fmt.Errorf("unsupported edition %q", cfg.Edition) + } + allRPMs, err := filepath.Glob(filepath.Join(workDir, fmt.Sprintf("grafana%s-*.rpm", sfx))) + if err != nil { + return fmt.Errorf("failed to list RPMs in %q: %w", workDir, err) + } + rpms := []string{} + for _, rpm := range allRPMs { + if strings.Contains(rpm, "-latest") { + continue + } + + rpms = append(rpms, rpm) + } + // XXX: What does the following comment mean? + // adds to many files for enterprise + for _, rpm := range rpms { + if err := fs.CopyFile(rpm, filepath.Join(repoRoot, filepath.Base(rpm))); err != nil { + return err + } + } + + //nolint:gosec + cmd = exec.Command("createrepo", repoRoot) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create repo at %q: %w\n%s", repoRoot, err, output) + } + + if err := signRPMRepo(repoRoot, cfg); err != nil { + return err + } + + // Update the repo in GCS + + // Sync packages first to avoid cache misses + if cfg.DryRun { + log.Printf("Simulating upload of RPMs to GCS (%s)\n", folderURI) + } else { + log.Printf("Uploading RPMs to GCS (%s)...\n", folderURI) + args := []string{"-m", "cp"} + args = append(args, rpms...) + args = append(args, folderURI) + //nolint:gosec + cmd = exec.Command("gsutil", args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to upload RPMs to GCS: %w\n%s", err, output) + } + } + + if cfg.DryRun { + log.Printf("Simulating upload of RPM repo metadata to GCS (%s)\n", folderURI) + } else { + log.Printf("Uploading RPM repo metadata to GCS (%s)...\n", folderURI) + //nolint:gosec + cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", repoRoot, folderURI) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to upload RPM repo metadata to GCS: %w\n%s", err, output) + } + allRepoResources := fmt.Sprintf("%s/**/*", folderURI) + log.Printf("Setting cache ttl for RPM repo resources on GCS (%s)...\n", allRepoResources) + //nolint:gosec + cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to set cache ttl for RPM repo resources on GCS: %w\n%s", err, output) + } + } + + return nil +} + +// downloadRPMs downloads RPM packages. +func downloadRPMs(cfg PublishConfig, workDir string) error { + if !strings.HasSuffix(workDir, string(filepath.Separator)) { + workDir += string(filepath.Separator) + } + var version string + if cfg.ReleaseMode.Mode == config.TagMode { + if cfg.ReleaseMode.IsBeta { + version = strings.ReplaceAll(cfg.Version, "-", "~") + } else { + version = cfg.Version + } + } + if version == "" { + panic(fmt.Sprintf("Unrecognized version mode %s", cfg.ReleaseMode.Mode)) + } + + var sfx string + switch cfg.Edition { + case config.EditionOSS: + case config.EditionEnterprise: + sfx = EnterpriseSfx + default: + return fmt.Errorf("unrecognized edition %q", cfg.Edition) + } + + u := fmt.Sprintf("gs://%s/%s/%s/grafana%s-%s-*.*.rpm*", cfg.Bucket, + strings.ToLower(string(cfg.Edition)), ReleaseFolder, sfx, version) + log.Printf("Downloading RPM packages %q...\n", u) + args := []string{ + "-m", + "cp", + u, + workDir, + } + //nolint:gosec + cmd := exec.Command("gsutil", args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to download RPM packages %q: %w\n%s", u, err, output) + } + + return nil +} + +func getPublicKey(cfg PublishConfig) (*packet.PublicKey, error) { + f, err := os.Open(cfg.GPGPublicKey) + if err != nil { + return nil, fmt.Errorf("failed to open %q: %w", cfg.GPGPublicKey, err) + } + defer func(f *os.File) { + err := f.Close() + if err != nil { + return + } + }(f) + + block, err := armor.Decode(f) + if err != nil { + return nil, err + } + + if block.Type != openpgp.PublicKeyType { + return nil, fmt.Errorf("invalid public key block type: %q", block.Type) + } + + packetReader := packet.NewReader(block.Body) + pkt, err := packetReader.Next() + if err != nil { + return nil, err + } + + key, ok := pkt.(*packet.PublicKey) + if !ok { + return nil, fmt.Errorf("got non-public key from packet reader: %T", pkt) + } + + return key, nil +} + +func getPrivateKey(cfg PublishConfig) (*packet.PrivateKey, error) { + f, err := os.Open(cfg.GPGPrivateKey) + if err != nil { + return nil, fmt.Errorf("failed to open %q: %w", cfg.GPGPrivateKey, err) + } + defer func(f *os.File) { + err := f.Close() + if err != nil { + return + } + }(f) + + passphraseB, err := os.ReadFile(cfg.GPGPassPath) + if err != nil { + return nil, fmt.Errorf("failed to read %q: %w", cfg.GPGPrivateKey, err) + } + passphraseB = bytes.TrimSuffix(passphraseB, []byte("\n")) + + block, err := armor.Decode(f) + if err != nil { + return nil, err + } + + if block.Type != openpgp.PrivateKeyType { + return nil, fmt.Errorf("invalid private key block type: %q", block.Type) + } + + packetReader := packet.NewReader(block.Body) + pkt, err := packetReader.Next() + if err != nil { + return nil, err + } + + key, ok := pkt.(*packet.PrivateKey) + if !ok { + return nil, fmt.Errorf("got non-private key from packet reader: %T", pkt) + } + + if err := key.Decrypt(passphraseB); err != nil { + return nil, fmt.Errorf("failed to decrypt private key: %w", err) + } + return key, nil +} + +// signRPMRepo signs an RPM repository using PGP. +// The signature gets written to the file repodata/repomd.xml.asc. +func signRPMRepo(repoRoot string, cfg PublishConfig) error { + if cfg.GPGPublicKey == "" || cfg.GPGPrivateKey == "" { + return fmt.Errorf("private or public key is empty") + } + + log.Printf("Signing RPM repo") + + pubKey, err := getPublicKey(cfg) + if err != nil { + return err + } + + privKey, err := getPrivateKey(cfg) + if err != nil { + return err + } + + pcfg := packet.Config{ + DefaultHash: crypto.SHA256, + DefaultCipher: packet.CipherAES256, + DefaultCompressionAlgo: packet.CompressionZLIB, + CompressionConfig: &packet.CompressionConfig{ + Level: 9, + }, + RSABits: 4096, + } + currentTime := pcfg.Now() + uid := packet.NewUserId("", "", "") + + isPrimaryID := false + keyLifetimeSecs := uint32(86400 * 365) + signer := openpgp.Entity{ + PrimaryKey: pubKey, + PrivateKey: privKey, + Identities: map[string]*openpgp.Identity{ + uid.Id: { + Name: uid.Name, + UserId: uid, + SelfSignature: &packet.Signature{ + CreationTime: currentTime, + SigType: packet.SigTypePositiveCert, + PubKeyAlgo: packet.PubKeyAlgoRSA, + Hash: pcfg.Hash(), + IsPrimaryId: &isPrimaryID, + FlagsValid: true, + FlagSign: true, + FlagCertify: true, + IssuerKeyId: &pubKey.KeyId, + }, + }, + }, + Subkeys: []openpgp.Subkey{ + { + PublicKey: pubKey, + PrivateKey: privKey, + Sig: &packet.Signature{ + CreationTime: currentTime, + SigType: packet.SigTypeSubkeyBinding, + PubKeyAlgo: packet.PubKeyAlgoRSA, + Hash: pcfg.Hash(), + PreferredHash: []uint8{8}, // SHA-256 + FlagsValid: true, + FlagEncryptStorage: true, + FlagEncryptCommunications: true, + IssuerKeyId: &pubKey.KeyId, + KeyLifetimeSecs: &keyLifetimeSecs, + }, + }, + }, + } + + // Ignore gosec G304 as this function is only used in the build process. + //nolint:gosec + freader, err := os.Open(filepath.Join(repoRoot, "repodata", "repomd.xml")) + if err != nil { + return err + } + defer func(freader *os.File) { + err := freader.Close() + if err != nil { + return + } + }(freader) + + // Ignore gosec G304 as this function is only used in the build process. + //nolint:gosec + sigwriter, err := os.Create(filepath.Join(repoRoot, "repodata", "repomd.xml.asc")) + if err != nil { + return err + } + defer func(sigwriter *os.File) { + err := sigwriter.Close() + if err != nil { + return + } + }(sigwriter) + + if err := openpgp.ArmoredDetachSignText(sigwriter, &signer, freader, nil); err != nil { + return fmt.Errorf("failed to write PGP signature: %w", err) + } + + if err := sigwriter.Close(); err != nil { + return fmt.Errorf("failed to write PGP signature: %w", err) + } + + return nil +} diff --git a/pkg/build/cmd/rpm_test.go b/pkg/build/packaging/rpm_test.go similarity index 99% rename from pkg/build/cmd/rpm_test.go rename to pkg/build/packaging/rpm_test.go index 60d8ac5f50a..5a030a3093b 100644 --- a/pkg/build/cmd/rpm_test.go +++ b/pkg/build/packaging/rpm_test.go @@ -1,4 +1,4 @@ -package main +package packaging import ( "os"