schema: Migrate from scuemata to thema (#49805)

* Remove crufty scuemata bits

Buhbye to: cue/ dir with old definitions, CI steps for checking unnecessary
things, and the original dashboard scuemata file.

* Remove grafana-cli cue subcommand

* Remove old testdata

* Don't swallow errors from codegen

* Small nits and tweaks to cuectx package

* WIP - refactor pluggen to use Thema

Also consolidate the embed.FS in the repo root.

* Finish halfway rename

* Convert all panel models.cue to thema

* Rewrite pluggen to use Thema

* Remove pkg/schema, and trim command

* Remove schemaloader service and usages

Will be replaced by coremodel-centric hydrate/dehydrate system Soon™.

* Remove schemaloader from wire

* Remove hangover field on histogram models.cue

* Fix lint errors, some vestiges of trim service

* Remove unused cuetsify cli command

pkg/codegen/pluggen.go

@@ -6,26 +6,32 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	"os"
 	"path/filepath"
 	"strings"
-	"testing/fstest"
 	"text/template"
 
 	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/ast"
+	"cuelang.org/go/cue/build"
+	"cuelang.org/go/cue/cuecontext"
 	"cuelang.org/go/cue/errors"
-	cload "cuelang.org/go/cue/load"
+	"cuelang.org/go/cue/load"
 	"cuelang.org/go/cue/parser"
 	"github.com/grafana/cuetsy"
-	"github.com/grafana/grafana/pkg/schema/load"
+	"github.com/grafana/grafana"
+	"github.com/grafana/thema"
+	tload "github.com/grafana/thema/load"
 )
 
 // The only import statement we currently allow in any models.cue file
-const allowedImport = "github.com/grafana/grafana/packages/grafana-schema/src/schema"
+const schemasPath = "github.com/grafana/grafana/packages/grafana-schema/src/schema"
 
+// CUE import paths, mapped to corresponding TS import paths. An empty value
+// indicates the import path should be dropped in the conversion to TS. Imports
+// not present in the list are not not allowed, and code generation will fail.
 var importMap = map[string]string{
-	allowedImport: "@grafana/schema",
+	"github.com/grafana/thema": "",
+	schemasPath:                "@grafana/schema",
 }
 
 // Hard-coded list of paths to skip. Remove a particular file as we're ready
@@ -44,55 +50,33 @@ var skipPaths = []string{
 
 const prefix = "/"
 
-var paths = load.GetDefaultLoadPaths()
-
 // CuetsifyPlugins runs cuetsy against plugins' models.cue files.
 func CuetsifyPlugins(ctx *cue.Context, root string) (WriteDiffer, error) {
+	lib := thema.NewLibrary(ctx)
 	// TODO this whole func has a lot of old, crufty behavior from the scuemata era; needs TLC
-	var fspaths load.BaseLoadPaths
-	var err error
-
-	fspaths.BaseCueFS, err = populateMapFSFromRoot(paths.BaseCueFS, root, "")
+	overlay := make(map[string]load.Source)
+	err := toOverlay(prefix, grafana.CueSchemaFS, overlay)
+	// err := tload.ToOverlay(prefix, grafana.CueSchemaFS, overlay)
 	if err != nil {
 		return nil, err
 	}
-	fspaths.DistPluginCueFS, err = populateMapFSFromRoot(paths.DistPluginCueFS, root, "")
-	if err != nil {
-		return nil, err
-	}
-	overlay, err := defaultOverlay(fspaths)
-	if err != nil {
-		return nil, err
-	}
-
-	// Prep the cue load config
-	clcfg := &cload.Config{
-		Overlay: overlay,
-		// FIXME these module paths won't work for things not under our cue.mod - AKA third-party plugins
-		ModuleRoot: prefix,
-		Module:     "github.com/grafana/grafana",
-	}
-
-	// FIXME hardcoding paths to exclude is not the way to handle this
-	excl := map[string]bool{
-		"cue.mod":      true,
-		"cue/scuemata": true,
-		"packages/grafana-schema/src/scuemata/dashboard":      true,
-		"packages/grafana-schema/src/scuemata/dashboard/dist": true,
-	}
 
 	exclude := func(path string) bool {
-		dir := filepath.Dir(path)
-		if excl[dir] {
-			return true
-		}
 		for _, p := range skipPaths {
 			if path == p {
 				return true
 			}
 		}
 
-		return false
+		return filepath.Dir(path) == "cue.mod"
+	}
+
+	// Prep the cue load config
+	clcfg := &load.Config{
+		Overlay: overlay,
+		// FIXME these module paths won't work for things not under our cue.mod - AKA third-party plugins
+		ModuleRoot: prefix,
+		Module:     "github.com/grafana/grafana",
 	}
 
 	outfiles := NewWriteDiffer()
@@ -110,88 +94,74 @@ func CuetsifyPlugins(ctx *cue.Context, root string) (WriteDiffer, error) {
 			}
 			seen[dir] = true
 			clcfg.Dir = filepath.Join(root, dir)
-			// FIXME Horrible hack to figure out the identifier used for
-			// imported packages - intercept the parser called by the loader to
-			// look at the ast.Files on their way in to building.
-			// Much better if we could work backwards from the cue.Value,
-			// maybe even directly in cuetsy itself, and figure out when a
-			// referenced object is "out of bounds".
-			// var imports sync.Map
-			var imports []*ast.ImportSpec
-			clcfg.ParseFile = func(name string, src interface{}) (*ast.File, error) {
-				f, err := parser.ParseFile(name, src, parser.ParseComments)
-				if err != nil {
-					return nil, err
-				}
-				imports = append(imports, f.Imports...)
-				return f, nil
-			}
-			if strings.Contains(path, "public/app/plugins") {
-				clcfg.Package = "grafanaschema"
-			} else {
-				clcfg.Package = ""
-			}
-
-			// FIXME loading in this way causes all files in a dir to be loaded
-			// as a single cue.Instance or cue.Value, which makes it quite
-			// difficult to map them _back_ onto the original file and generate
-			// discrete .gen.ts files for each .cue input.  However, going one
-			// .cue file at a time and passing it as the first arg to
-			// load.Instances() means that the other files are ignored
-			// completely, causing references between these files to be
-			// unresolved, and thus encounter a different kind of error.
-			insts := cload.Instances(nil, clcfg)
-			if len(insts) > 1 {
-				panic("extra instances")
-			}
-			bi := insts[0]
-
-			v := ctx.BuildInstance(bi)
-			if v.Err() != nil {
-				return v.Err()
-			}
 
 			var b []byte
 			f := &tsFile{}
-			seen := make(map[string]bool)
-			// FIXME explicitly mapping path patterns to conversion patterns
-			// is exactly what we want to avoid
+
 			switch {
-			// panel plugin models.cue files
-			case strings.Contains(path, "public/app/plugins"):
-				for _, im := range imports {
+			default:
+				insts := load.Instances(nil, clcfg)
+				if len(insts) > 1 {
+					return fmt.Errorf("%s: resulted in more than one instance", path)
+				}
+				v := ctx.BuildInstance(insts[0])
+
+				b, err = cuetsy.Generate(v, cuetsy.Config{})
+				if err != nil {
+					return err
+				}
+
+			case strings.Contains(path, "public/app/plugins"): // panel plugin models.cue files
+				// The simple - and preferable - thing would be to have plugins use the same
+				// package name for their models.cue as their containing dir. That's not
+				// possible, though, because we allow dashes in plugin names, but CUE does not
+				// allow them in package names. Yuck.
+				inst, err := loadInstancesWithThema(in, dir, "grafanaschema")
+				if err != nil {
+					return fmt.Errorf("could not load CUE instance for %s: %w", dir, err)
+				}
+
+				// Also parse file directly to extract imports.
+				// NOTE this will need refactoring to support working with more than one file at a time
+				of, _ := in.Open(path)
+				pf, _ := parser.ParseFile(filepath.Base(path), of, parser.ParseComments)
+
+				iseen := make(map[string]bool)
+				for _, im := range pf.Imports {
 					ip := strings.Trim(im.Path.Value, "\"")
-					if ip != allowedImport {
+					mappath, has := importMap[ip]
+					if !has {
 						// TODO make a specific error type for this
-						return errors.Newf(im.Pos(), "import %q not allowed, panel plugins may only import from %q", ip, allowedImport)
+						var all []string
+						for im := range importMap {
+							all = append(all, fmt.Sprintf("\t%s", im))
+						}
+						return errors.Newf(im.Pos(), "%s: import %q not allowed, panel plugins may only import from:\n%s\n", path, ip, strings.Join(all, "\n"))
 					}
 					// TODO this approach will silently swallow the unfixable
 					// error case where multiple files in the same dir import
 					// the same package to a different ident
-					if !seen[ip] {
-						seen[ip] = true
+					if mappath != "" && !iseen[ip] {
+						iseen[ip] = true
 						f.Imports = append(f.Imports, convertImport(im))
 					}
 				}
 
-				// Extract the latest schema and its version number. (All of this goes away with Thema, whew)
-				f.V = &tsModver{}
-				lins := v.LookupPath(cue.ParsePath("Panel.lineages"))
-				f.V.Lin, _ = lins.Len().Int64()
-				f.V.Lin = f.V.Lin - 1
-				schs := lins.LookupPath(cue.MakePath(cue.Index(int(f.V.Lin))))
-				f.V.Sch, _ = schs.Len().Int64()
-				f.V.Sch = f.V.Sch - 1
-				latest := schs.LookupPath(cue.MakePath(cue.Index(int(f.V.Sch))))
+				v := ctx.BuildInstance(inst)
 
-				b, err = cuetsy.Generate(latest, cuetsy.Config{})
-			default:
-				b, err = cuetsy.Generate(v, cuetsy.Config{})
+				lin, err := thema.BindLineage(v.LookupPath(cue.ParsePath("Panel")), lib)
+				if err != nil {
+					return fmt.Errorf("%s: failed to bind lineage: %w", path, err)
+				}
+				f.V = thema.LatestVersion(lin)
+				f.WriteModelVersion = true
+
+				b, err = cuetsy.Generate(thema.SchemaP(lin, f.V).UnwrapCUE(), cuetsy.Config{})
+				if err != nil {
+					return err
+				}
 			}
 
-			if err != nil {
-				return err
-			}
 			f.Body = string(b)
 
 			var buf bytes.Buffer
@@ -201,11 +171,7 @@ func CuetsifyPlugins(ctx *cue.Context, root string) (WriteDiffer, error) {
 		})
 	}
 
-	err = cuetsify(fspaths.BaseCueFS)
-	if err != nil {
-		return nil, gerrors.New(errors.Details(err, nil))
-	}
-	err = cuetsify(fspaths.DistPluginCueFS)
+	err = cuetsify(grafana.CueSchemaFS)
 	if err != nil {
 		return nil, gerrors.New(errors.Details(err, nil))
 	}
@@ -215,7 +181,7 @@ func CuetsifyPlugins(ctx *cue.Context, root string) (WriteDiffer, error) {
 
 func convertImport(im *ast.ImportSpec) *tsImport {
 	tsim := &tsImport{
-		Pkg: importMap[allowedImport],
+		Pkg: importMap[schemasPath],
 	}
 	if im.Name != nil && im.Name.String() != "" {
 		tsim.Ident = im.Name.String()
@@ -231,21 +197,101 @@ func convertImport(im *ast.ImportSpec) *tsImport {
 	return tsim
 }
 
-func defaultOverlay(p load.BaseLoadPaths) (map[string]cload.Source, error) {
-	overlay := make(map[string]cload.Source)
+var themamodpath string = filepath.Join("cue.mod", "pkg", "github.com", "grafana", "thema")
 
-	if err := toOverlay(prefix, p.BaseCueFS, overlay); err != nil {
+// all copied and hacked up from Thema's LoadInstancesWithThema, simply to allow setting the
+// package name
+func loadInstancesWithThema(modFS fs.FS, dir string, pkgname string) (*build.Instance, error) {
+	var modname string
+	err := fs.WalkDir(modFS, "cue.mod", func(path string, d fs.DirEntry, err error) error {
+		// fs.FS implementations tend to not use path separators as expected. Use a
+		// normalized one for comparisons, but retain the original for calls back into modFS.
+		normpath := filepath.FromSlash(path)
+		if err != nil {
+			return err
+		}
+
+		if d.IsDir() {
+			switch normpath {
+			case filepath.Join("cue.mod", "gen"), filepath.Join("cue.mod", "usr"):
+				return fs.SkipDir
+			case themamodpath:
+				return fmt.Errorf("path %q already exists in modFS passed to InstancesWithThema, must be absent for dynamic dependency injection", themamodpath)
+			}
+			return nil
+		} else if normpath == filepath.Join("cue.mod", "module.cue") {
+			modf, err := modFS.Open(path)
+			if err != nil {
+				return err
+			}
+			defer modf.Close() // nolint: errcheck
+
+			b, err := io.ReadAll(modf)
+			if err != nil {
+				return err
+			}
+
+			modname, err = cuecontext.New().CompileBytes(b).LookupPath(cue.MakePath(cue.Str("module"))).String()
+			if err != nil {
+				return err
+			}
+			if modname == "" {
+				return fmt.Errorf("InstancesWithThema requires non-empty module name in modFS' cue.mod/module.cue")
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
 		return nil, err
 	}
 
-	if err := toOverlay(prefix, p.DistPluginCueFS, overlay); err != nil {
+	if modname == "" {
+		return nil, errors.New("cue.mod/module.cue did not exist")
+	}
+
+	modroot := filepath.FromSlash(filepath.Join("/", modname))
+	overlay := make(map[string]load.Source)
+	if err := tload.ToOverlay(modroot, modFS, overlay); err != nil {
 		return nil, err
 	}
 
-	return overlay, nil
+	// Special case for when we're calling this loader with paths inside the thema module
+	if modname == "github.com/grafana/thema" {
+		if err := tload.ToOverlay(modroot, thema.CueJointFS, overlay); err != nil {
+			return nil, err
+		}
+	} else {
+		if err := tload.ToOverlay(filepath.Join(modroot, themamodpath), thema.CueFS, overlay); err != nil {
+			return nil, err
+		}
+	}
+
+	if dir == "" {
+		dir = "."
+	}
+
+	cfg := &load.Config{
+		Overlay:    overlay,
+		ModuleRoot: modroot,
+		Module:     modname,
+		Dir:        filepath.Join(modroot, dir),
+		Package:    pkgname,
+	}
+	if dir == "." {
+		cfg.Package = filepath.Base(modroot)
+		cfg.Dir = modroot
+	}
+
+	inst := load.Instances(nil, cfg)[0]
+	if inst.Err != nil {
+		return nil, inst.Err
+	}
+
+	return inst, nil
 }
 
-func toOverlay(prefix string, vfs fs.FS, overlay map[string]cload.Source) error {
+func toOverlay(prefix string, vfs fs.FS, overlay map[string]load.Source) error {
 	if !filepath.IsAbs(prefix) {
 		return fmt.Errorf("must provide absolute path prefix when generating cue overlay, got %q", prefix)
 	}
@@ -274,7 +320,7 @@ func toOverlay(prefix string, vfs fs.FS, overlay map[string]cload.Source) error
 			return err
 		}
 
-		overlay[filepath.Join(prefix, path)] = cload.FromBytes(b)
+		overlay[filepath.Join(prefix, path)] = load.FromBytes(b)
 		return nil
 	})
 
@@ -285,44 +331,11 @@ func toOverlay(prefix string, vfs fs.FS, overlay map[string]cload.Source) error
 	return nil
 }
 
-// Helper function that populates an fs.FS by walking over a virtual filesystem,
-// and reading files from disk corresponding to each file encountered.
-func populateMapFSFromRoot(in fs.FS, root, join string) (fs.FS, error) {
-	out := make(fstest.MapFS)
-	err := fs.WalkDir(in, ".", func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if d.IsDir() {
-			return nil
-		}
-		// Ignore gosec warning G304. The input set here is necessarily
-		// constrained to files specified in embed.go
-		// nolint:gosec
-		b, err := os.Open(filepath.Join(root, join, path))
-		if err != nil {
-			return err
-		}
-		byt, err := io.ReadAll(b)
-		if err != nil {
-			return err
-		}
-
-		out[path] = &fstest.MapFile{Data: byt}
-		return nil
-	})
-	return out, err
-}
-
 type tsFile struct {
-	V       *tsModver
-	Imports []*tsImport
-	Body    string
-}
-
-type tsModver struct {
-	Lin, Sch int64
+	V                 thema.SyntacticVersion
+	WriteModelVersion bool
+	Imports           []*tsImport
+	Body              string
 }
 
 type tsImport struct {
@@ -337,7 +350,7 @@ var tsTemplate = template.Must(template.New("cuetsygen").Parse(`//~~~~~~~~~~~~~~
 //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 {{range .Imports}}
 import * as {{.Ident}} from '{{.Pkg}}';{{end}}
-{{if .V}}
-export const modelVersion = Object.freeze([{{ .V.Lin }}, {{ .V.Sch }}]);
+{{if .WriteModelVersion }}
+export const modelVersion = Object.freeze([{{index .V 0}}, {{index .V 1}}]);
 {{end}}
 {{.Body}}`))