Kinds: rework verify-kinds to reduce GH API calls (#71304)

* Kinds: rework verify-kinds to reduce GH API calls

* Clean up and apply review feedback

.github/workflows/publish-kinds-next.yml

@@ -24,8 +24,6 @@ jobs:
 
       - name: "Verify kinds"
         run: go run .github/workflows/scripts/kinds/verify-kinds.go
-        env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
 
       - name: "Generate token"
         id: generate_token

.github/workflows/publish-kinds-release.yml

@@ -17,6 +17,7 @@ jobs:
       - name: "Checkout Grafana repo"
         uses: "actions/checkout@v3"
         with:
+          # required for the `grafana/grafana-github-actions/has-matching-release-tag` action to work
           fetch-depth: 0
 
       - name: "Setup Go"
@@ -26,8 +27,6 @@ jobs:
 
       - name: "Verify kinds"
         run: go run .github/workflows/scripts/kinds/verify-kinds.go
-        env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
 
       - name: "Checkout Actions library"
         uses: "actions/checkout@v3"

.github/workflows/scripts/kinds/verify-kinds.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"archive/zip"
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -37,31 +39,17 @@ const (
 // If kind names are given as parameters, the script will make the above actions only for the
 // given kinds.
 func main() {
-	var kindArgs []string
-	if len(os.Args) > 1 {
-		kindArgs = os.Args[1:]
-	}
-
 	var corek []kindsys.Kind
 	var compok []kindsys.Composable
 
-	// This script will reach the GH API rate limit if ran for all kinds without token.
-	// If you don't have a GH token, run the script with kind names as parameters to run
-	// it only for a limited set of kinds.
-	var ts oauth2.TokenSource
-	token, ok := os.LookupEnv("GITHUB_TOKEN")
-	if ok {
-		ts = oauth2.StaticTokenSource(
-			&oauth2.Token{AccessToken: token},
-		)
+	kindRegistry, err := NewKindRegistry()
+	defer kindRegistry.cleanUp()
+	if err != nil {
+		die(err)
 	}
 
-	ctx := context.Background()
-	tc := oauth2.NewClient(ctx, ts)
-	client := github.NewClient(tc)
-
 	// Search for the latest version directory present in the kind-registry repo
-	latestRegistryDir, err := findLatestDir(ctx, client)
+	latestRegistryDir, err := kindRegistry.findLatestDir()
 	if err != nil {
 		die(fmt.Errorf("failed to get latest directory for published kinds: %s", err))
 	}
@@ -70,12 +58,8 @@ func main() {
 
 	// Kind verification
 	for _, kind := range corekind.NewBase(nil).All() {
-		if len(kindArgs) > 0 && !contains(kindArgs, kind.Name()) {
-			continue
-		}
-
 		name := kind.Props().Common().MachineName
-		err := verifyKind(kind, name, "core", latestRegistryDir)
+		err := verifyKind(kindRegistry, kind, name, "core", latestRegistryDir)
 		if err != nil {
 			errs = append(errs, err)
 			continue
@@ -86,10 +70,6 @@ func main() {
 
 	for _, pp := range corelist.New(nil) {
 		for _, kind := range pp.ComposableKinds {
-			if len(kindArgs) > 0 && !contains(kindArgs, kind.Name()) {
-				continue
-			}
-
 			si, err := kindsys.FindSchemaInterface(kind.Def().Properties.SchemaInterface)
 			if err != nil {
 				errs = append(errs, err)
@@ -97,7 +77,7 @@ func main() {
 			}
 
 			name := strings.ToLower(fmt.Sprintf("%s/%s", strings.TrimSuffix(kind.Lineage().Name(), si.Name()), si.Name()))
-			err = verifyKind(kind, name, "composable", latestRegistryDir)
+			err = verifyKind(kindRegistry, kind, name, "composable", latestRegistryDir)
 			if err != nil {
 				errs = append(errs, err)
 				continue
@@ -115,11 +95,11 @@ func main() {
 
 	// File generation
 	jfs := codejen.NewFS()
-	registryPath := filepath.Join(".github", "workflows", "scripts", "kinds")
+	outputPath := filepath.Join(".github", "workflows", "scripts", "kinds")
 
 	coreJennies := codejen.JennyList[kindsys.Kind]{}
 	coreJennies.Append(
-		KindRegistryJenny(registryPath, kindArgs),
+		KindRegistryJenny(outputPath),
 	)
 	corefs, err := coreJennies.GenerateFS(corek...)
 	die(err)
@@ -127,7 +107,7 @@ func main() {
 
 	composableJennies := codejen.JennyList[kindsys.Composable]{}
 	composableJennies.Append(
-		ComposableKindRegistryJenny(registryPath, kindArgs),
+		ComposableKindRegistryJenny(outputPath),
 	)
 	composablefs, err := composableJennies.GenerateFS(compok...)
 	die(err)
@@ -143,15 +123,14 @@ func die(errs ...error) {
 		for _, err := range errs {
 			fmt.Fprint(os.Stderr, err, "\n")
 		}
-		fmt.Println("Run `go run verify-kinds.go <kind name> <kind name>` to run the script on a limited set of kinds.")
 		os.Exit(1)
 	}
 }
 
 // verifyKind verifies that stable kinds are not updated once published (new schemas
 // can be added but existing ones cannot be updated)
-func verifyKind(kind kindsys.Kind, name string, category string, latestRegistryDir string) error {
-	oldKindString, err := getPublishedKind(name, category, latestRegistryDir)
+func verifyKind(registry *kindRegistry, kind kindsys.Kind, name string, category string, latestRegistryDir string) error {
+	oldKindString, err := registry.getPublishedKind(name, category, latestRegistryDir)
 	if err != nil {
 		return err
 	}
@@ -194,77 +173,6 @@ func verifyKind(kind kindsys.Kind, name string, category string, latestRegistryD
 	return nil
 }
 
-// getPublishedKind retrieves the latest published kind from the kind registry
-func getPublishedKind(name string, category string, latestRegistryDir string) (string, error) {
-	var ts oauth2.TokenSource
-	token, ok := os.LookupEnv("GITHUB_TOKEN")
-	if ok {
-		ts = oauth2.StaticTokenSource(
-			&oauth2.Token{AccessToken: token},
-		)
-	}
-
-	ctx := context.Background()
-	tc := oauth2.NewClient(ctx, ts)
-	client := github.NewClient(tc)
-
-	if latestRegistryDir == "" {
-		return "", nil
-	}
-
-	file, _, resp, err := client.Repositories.GetContents(ctx, GITHUB_OWNER, GITHUB_REPO, fmt.Sprintf("grafana/%s/%s/%s.cue", latestRegistryDir, category, name), nil)
-	if err != nil {
-		if resp.StatusCode == http.StatusNotFound {
-			return "", nil
-		}
-		return "", fmt.Errorf("error retrieving published kind from GH, %d: %w", resp.StatusCode, err)
-	}
-	content, err := file.GetContent()
-	if err != nil {
-		return "", fmt.Errorf("error decoding published kind content: %w", err)
-	}
-
-	return content, nil
-}
-
-// findLatestDir get the latest version directory published in the kind registry
-func findLatestDir(ctx context.Context, client *github.Client) (string, error) {
-	re := regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`)
-	latestVersion := []uint64{0, 0, 0}
-	latestDir := ""
-
-	_, dir, resp, err := client.Repositories.GetContents(ctx, GITHUB_OWNER, GITHUB_REPO, "grafana", nil)
-	if err != nil {
-		if resp.StatusCode == http.StatusNotFound {
-			return "", nil
-		}
-		return "", err
-	}
-
-	for _, content := range dir {
-		if content.GetType() != "dir" {
-			continue
-		}
-
-		parts := re.FindStringSubmatch(content.GetName())
-		if parts == nil || len(parts) < 4 {
-			continue
-		}
-
-		version := make([]uint64, len(parts)-1)
-		for i := 1; i < len(parts); i++ {
-			version[i-1], _ = strconv.ParseUint(parts[i], 10, 32)
-		}
-
-		if isLess(latestVersion, version) {
-			latestVersion = version
-			latestDir = content.GetName()
-		}
-	}
-
-	return latestDir, nil
-}
-
 func isLess(v1 []uint64, v2 []uint64) bool {
 	if len(v1) == 1 || len(v2) == 1 {
 		return v1[0] < v2[0]
@@ -313,16 +221,14 @@ func loadComposableKind(name string, kind string) (kindsys.Kind, error) {
 }
 
 // KindRegistryJenny generates kind files into the "next" folder of the local kind registry.
-func KindRegistryJenny(path string, kindSet []string) codegen.OneToOne {
+func KindRegistryJenny(path string) codegen.OneToOne {
 	return &kindregjenny{
-		path:    path,
-		kindSet: kindSet,
+		path: path,
 	}
 }
 
 type kindregjenny struct {
-	path    string
-	kindSet []string
+	path string
 }
 
 func (j *kindregjenny) JennyName() string {
@@ -330,10 +236,6 @@ func (j *kindregjenny) JennyName() string {
 }
 
 func (j *kindregjenny) Generate(kind kindsys.Kind) (*codejen.File, error) {
-	if len(j.kindSet) > 0 && !contains(j.kindSet, kind.Name()) {
-		return nil, nil
-	}
-
 	name := kind.Props().Common().MachineName
 	core, ok := kind.(kindsys.Core)
 	if !ok {
@@ -361,16 +263,14 @@ func kindToBytes(kind cue.Value) ([]byte, error) {
 }
 
 // ComposableKindRegistryJenny generates kind files into the "next" folder of the local kind registry.
-func ComposableKindRegistryJenny(path string, kindSet []string) codejen.OneToOne[kindsys.Composable] {
+func ComposableKindRegistryJenny(path string) codejen.OneToOne[kindsys.Composable] {
 	return &ckrJenny{
-		path:    path,
-		kindSet: kindSet,
+		path: path,
 	}
 }
 
 type ckrJenny struct {
-	path    string
-	kindSet []string
+	path string
 }
 
 func (j *ckrJenny) JennyName() string {
@@ -378,10 +278,6 @@ func (j *ckrJenny) JennyName() string {
 }
 
 func (j *ckrJenny) Generate(k kindsys.Composable) (*codejen.File, error) {
-	if len(j.kindSet) > 0 && !contains(j.kindSet, k.Name()) {
-		return nil, nil
-	}
-
 	si, err := kindsys.FindSchemaInterface(k.Def().Properties.SchemaInterface)
 	if err != nil {
 		panic(err)
@@ -399,12 +295,124 @@ func (j *ckrJenny) Generate(k kindsys.Composable) (*codejen.File, error) {
 	return codejen.NewFile(filepath.Join(j.path, "next", "composable", name+".cue"), newKindBytes, j), nil
 }
 
-func contains(array []string, value string) bool {
-	for _, v := range array {
-		if v == value {
-			return true
+type kindRegistry struct {
+	zipDir  string
+	zipFile *zip.ReadCloser
+}
+
+// NewKindRegistry downloads the archive of the kind-registry GH repository and open it
+func NewKindRegistry() (*kindRegistry, error) {
+	ctx := context.Background()
+	tc := oauth2.NewClient(ctx, nil)
+	client := github.NewClient(tc)
+
+	// Create a temporary file to store the downloaded archive
+	file, err := os.CreateTemp("", "*.zip")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temporary file: %w", err)
+	}
+	defer file.Close()
+
+	// Get the repository archive URL
+	archiveURL, _, err := client.Repositories.GetArchiveLink(ctx, GITHUB_OWNER, GITHUB_REPO, github.Zipball, &github.RepositoryContentGetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get archive URL: %w", err)
+	}
+
+	// Download the archive file
+	httpClient := http.DefaultClient
+	resp, err := httpClient.Get(archiveURL.String())
+	if err != nil {
+		return nil, fmt.Errorf("failed to download archive: %w", err)
+	}
+	defer resp.Body.Close()
+
+	// Save the downloaded archive to the temporary file
+	_, err = io.Copy(file, resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to save archive: %w", err)
+	}
+
+	// Open the zip file for reading
+	zipDir := file.Name()
+	zipFile, err := zip.OpenReader(zipDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open zip file %s: %w", zipDir, err)
+	}
+
+	return &kindRegistry{
+		zipDir:  zipDir,
+		zipFile: zipFile,
+	}, nil
+}
+
+// cleanUp removes the archive from the temporary files and closes the zip reader
+func (registry *kindRegistry) cleanUp() {
+	if registry.zipDir != "" {
+		err := os.Remove(registry.zipDir)
+		if err != nil {
+			fmt.Fprint(os.Stderr, fmt.Errorf("failed to remove zip archive: %w", err))
 		}
 	}
 
-	return false
+	if registry.zipFile != nil {
+		err := registry.zipFile.Close()
+		if err != nil {
+			fmt.Fprint(os.Stderr, fmt.Errorf("failed to close zip file reader: %w", err))
+		}
+	}
+}
+
+// findLatestDir get the latest version directory published in the kind registry
+func (registry *kindRegistry) findLatestDir() (string, error) {
+	re := regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`)
+	latestVersion := []uint64{0, 0, 0}
+	latestDir := ""
+
+	for _, file := range registry.zipFile.File {
+		if !file.FileInfo().IsDir() {
+			continue
+		}
+
+		parts := re.FindStringSubmatch(file.Name)
+		if parts == nil || len(parts) < 4 {
+			continue
+		}
+
+		version := make([]uint64, len(parts)-1)
+		for i := 1; i < len(parts); i++ {
+			version[i-1], _ = strconv.ParseUint(parts[i], 10, 32)
+		}
+
+		if isLess(latestVersion, version) {
+			latestVersion = version
+			latestDir = file.Name
+		}
+	}
+
+	return latestDir, nil
+}
+
+// getPublishedKind retrieves the latest published kind from the kind registry
+func (registry *kindRegistry) getPublishedKind(name string, category string, latestRegistryDir string) (string, error) {
+	if latestRegistryDir == "" {
+		return "", nil
+	}
+
+	kindPath := filepath.Join(
+		registry.zipDir,
+		fmt.Sprintf("grafana/%s/%s/%s.cue", latestRegistryDir, category, name),
+	)
+	file, err := registry.zipFile.Open(kindPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to open file: %w", err)
+	}
+	defer file.Close()
+
+	data, err := io.ReadAll(file)
+	if err != nil {
+		return "", fmt.Errorf("failed to read file: %w", err)
+	}
+
+	return string(data), nil
 }