plugins: Introduce generated, static core plugin registry (#54118)

* Refactor towards template/codegen framework

* Add templates for plugin gen

* Add Go codegen for plugins; overhaul framework, too

* Add new codegen output; assorted framework fixes

* Regenerate after merge

* Remove accidental commit file, update templates

* Export the pfs.Tree loader from plugin types

* Print details from cuetsy errors

* Generate loaders for all plugins and list in registry

* Use pfs_gen.go over lineage_gen.go

* Un-un-ignore main file

* Introduce simple List static registry for plugins

* Last tweaks to codegen

* remove unused tvars

* Ensure loop-local instances for both vars

* Generate pfs parsing in-place in registry

* Stop generating pfs_gen.go

* Move Tree into pfs, rename subdir

* Change package name to match dir

* Ignore gocyclo on HTTPServer.getNavTree

pkg/codegen/coremodel.go

@@ -5,17 +5,14 @@ import (
 	"errors"
 	"fmt"
 	"go/ast"
-	"go/format"
-	"go/parser"
-	"go/token"
 	"io"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 	"testing/fstest"
-	"text/template"
 
+	cerrors "cuelang.org/go/cue/errors"
 	"cuelang.org/go/pkg/encoding/yaml"
 	"github.com/deepmap/oapi-codegen/pkg/codegen"
 	"github.com/getkin/kin-openapi/openapi3"
@@ -23,7 +20,6 @@ import (
 	"github.com/grafana/grafana/pkg/cuectx"
 	"github.com/grafana/thema"
 	"github.com/grafana/thema/encoding/openapi"
-	"golang.org/x/tools/imports"
 )
 
 // ExtractedLineage contains the results of statically analyzing a Grafana
@@ -94,12 +90,14 @@ func ExtractLineage(path string, lib thema.Library) (*ExtractedLineage, error) {
 		},
 	}
 
-	ec.RelativePath, err = filepath.Rel(groot, filepath.Dir(path))
+	// ec.RelativePath, err = filepath.Rel(groot, filepath.Dir(path))
+	ec.RelativePath, err = filepath.Rel(groot, path)
 	if err != nil {
 		// should be unreachable, since we rootclimbed to find groot above
 		panic(err)
 	}
-	ec.Lineage, err = cuectx.LoadGrafanaInstancesWithThema(ec.RelativePath, fs, lib)
+	ec.RelativePath = filepath.ToSlash(ec.RelativePath)
+	ec.Lineage, err = cuectx.LoadGrafanaInstancesWithThema(filepath.Dir(ec.RelativePath), fs, lib)
 	if err != nil {
 		return ec, err
 	}
@@ -116,7 +114,7 @@ func (ls *ExtractedLineage) toTemplateObj() tplVars {
 	return tplVars{
 		Name:        lin.Name(),
 		LineagePath: ls.RelativePath,
-		PkgPath:     filepath.ToSlash(filepath.Join("github.com/grafana/grafana", ls.RelativePath)),
+		PkgPath:     filepath.ToSlash(filepath.Join("github.com/grafana/grafana", filepath.Dir(ls.RelativePath))),
 		TitleName:   strings.Title(lin.Name()), // nolint
 		LatestSeqv:  sch.Version()[0],
 		LatestSchv:  sch.Version()[1],
@@ -170,12 +168,19 @@ func (ls *ExtractedLineage) GenerateGoCoremodel(path string) (WriteDiffer, error
 		return nil, fmt.Errorf("loading generated openapi failed; %w", err)
 	}
 
+	var importbuf bytes.Buffer
+	if err = tmpls.Lookup("coremodel_imports.tmpl").Execute(&importbuf, tvars_coremodel_imports{
+		PackageName: lin.Name(),
+	}); err != nil {
+		return nil, fmt.Errorf("error executing imports template: %w", err)
+	}
+
 	gostr, err := codegen.Generate(oT, lin.Name(), codegen.Options{
 		GenerateTypes: true,
 		SkipPrune:     true,
 		SkipFmt:       true,
 		UserTemplates: map[string]string{
-			"imports.tmpl": fmt.Sprintf(tmplImports, ls.RelativePath),
+			"imports.tmpl": importbuf.String(),
 			"typedef.tmpl": tmplTypedef,
 		},
 	})
@@ -183,34 +188,34 @@ func (ls *ExtractedLineage) GenerateGoCoremodel(path string) (WriteDiffer, error
 		return nil, fmt.Errorf("openapi generation failed: %w", err)
 	}
 
+	buf := new(bytes.Buffer)
+	if err = tmpls.Lookup("autogen_header.tmpl").Execute(buf, tvars_autogen_header{
+		LineagePath:   ls.RelativePath,
+		GeneratorPath: "pkg/framework/coremodel/gen.go", // FIXME hardcoding is not OK
+	}); err != nil {
+		return nil, fmt.Errorf("error executing header template: %w", err)
+	}
+
+	fmt.Fprint(buf, "\n", gostr)
+
 	vars := ls.toTemplateObj()
-	var buuf bytes.Buffer
-	err = tmplAddenda.Execute(&buuf, vars)
+	err = tmpls.Lookup("addenda.tmpl").Execute(buf, vars)
 	if err != nil {
 		panic(err)
 	}
 
-	fset := token.NewFileSet()
-	fname := fmt.Sprintf("%s_gen.go", lin.Name())
-	gf, err := parser.ParseFile(fset, fname, gostr+buuf.String(), parser.ParseComments)
+	fullp := filepath.Join(path, fmt.Sprintf("%s_gen.go", lin.Name()))
+	byt, err := postprocessGoFile(genGoFile{
+		path:   fullp,
+		walker: makePrefixDropper(strings.Title(lin.Name()), "Model"),
+		in:     buf.Bytes(),
+	})
 	if err != nil {
-		return nil, fmt.Errorf("generated go file parsing failed: %w", err)
-	}
-	ast.Walk(prefixDropper(strings.Title(lin.Name())), gf)
-
-	var buf bytes.Buffer
-	err = format.Node(&buf, fset, gf)
-	if err != nil {
-		return nil, fmt.Errorf("ast printing failed: %w", err)
-	}
-
-	byt, err := imports.Process(fname, buf.Bytes(), nil)
-	if err != nil {
-		return nil, fmt.Errorf("goimports processing failed: %w", err)
+		return nil, err
 	}
 
 	wd := NewWriteDiffer()
-	wd[filepath.Join(path, fname)] = byt
+	wd[fullp] = byt
 
 	return wd, nil
 }
@@ -238,7 +243,7 @@ func (ls *ExtractedLineage) GenerateTypescriptCoremodel(path string) (WriteDiffe
 
 	top, err := cuetsy.GenerateSingleAST(strings.Title(ls.Lineage.Name()), schv, cuetsy.TypeInterface)
 	if err != nil {
-		return nil, fmt.Errorf("cuetsy top gen failed: %w", err)
+		return nil, fmt.Errorf("cuetsy top gen failed: %s", cerrors.Details(err, nil))
 	}
 
 	// TODO until cuetsy can toposort its outputs, put the top/parent type at the bottom of the file.
@@ -250,7 +255,12 @@ func (ls *ExtractedLineage) GenerateTypescriptCoremodel(path string) (WriteDiffe
 	var strb strings.Builder
 	var str string
 	fpath := ls.Lineage.Name() + ".gen.ts"
-	strb.WriteString(fmt.Sprintf(genHeader, ls.RelativePath))
+	if err := tmpls.Lookup("autogen_header.tmpl").Execute(&strb, tvars_autogen_header{
+		LineagePath:   ls.RelativePath,
+		GeneratorPath: "pkg/framework/coremodel/gen.go", // FIXME hardcoding is not OK
+	}); err != nil {
+		return nil, fmt.Errorf("error executing header template: %w", err)
+	}
 
 	if !ls.IsCanonical {
 		fpath = fmt.Sprintf("%s_experimental.gen.ts", ls.Lineage.Name())
@@ -273,16 +283,34 @@ func (ls *ExtractedLineage) GenerateTypescriptCoremodel(path string) (WriteDiffe
 	return wd, nil
 }
 
-type prefixDropper string
+type prefixDropper struct {
+	str     string
+	base    string
+	rxp     *regexp.Regexp
+	rxpsuff *regexp.Regexp
+}
+
+func makePrefixDropper(str, base string) prefixDropper {
+	return prefixDropper{
+		str:     str,
+		base:    base,
+		rxpsuff: regexp.MustCompile(fmt.Sprintf(`%s([a-zA-Z_]*)`, str)),
+		rxp:     regexp.MustCompile(fmt.Sprintf(`%s([\s.,;-])`, str)),
+	}
+}
 
 func (d prefixDropper) Visit(n ast.Node) ast.Visitor {
-	asstr := string(d)
 	switch x := n.(type) {
 	case *ast.Ident:
-		if x.Name != asstr {
-			x.Name = strings.TrimPrefix(x.Name, asstr)
+		if x.Name != d.str {
+			x.Name = strings.TrimPrefix(x.Name, d.str)
 		} else {
-			x.Name = "Model"
+			x.Name = d.base
+		}
+	case *ast.CommentGroup:
+		for _, c := range x.List {
+			c.Text = d.rxp.ReplaceAllString(c.Text, d.base+"$1")
+			c.Text = d.rxpsuff.ReplaceAllString(c.Text, "$1")
 		}
 	}
 	return d
@@ -297,177 +325,33 @@ func GenerateCoremodelRegistry(path string, ecl []*ExtractedLineage) (WriteDiffe
 		cml = append(cml, ec.toTemplateObj())
 	}
 
-	var buf bytes.Buffer
-	err := tmplRegistry.Execute(&buf, struct {
-		Coremodels []tplVars
-	}{
+	buf := new(bytes.Buffer)
+	if err := tmpls.Lookup("coremodel_registry.tmpl").Execute(buf, tvars_coremodel_registry{
+		Header: tvars_autogen_header{
+			GeneratorPath: "pkg/framework/coremodel/gen.go", // FIXME hardcoding is not OK
+		},
 		Coremodels: cml,
+	}); err != nil {
+		return nil, fmt.Errorf("failed executing coremodel registry template: %w", err)
+	}
+
+	byt, err := postprocessGoFile(genGoFile{
+		path: path,
+		in:   buf.Bytes(),
 	})
 	if err != nil {
-		return nil, fmt.Errorf("failed generating template: %w", err)
+		return nil, err
 	}
-
-	byt, err := imports.Process(path, buf.Bytes(), nil)
-	if err != nil {
-		return nil, fmt.Errorf("goimports processing failed: %w", err)
-	}
-
 	wd := NewWriteDiffer()
 	wd[path] = byt
 	return wd, nil
 }
 
-var genHeader = `// This file is autogenerated. DO NOT EDIT.
-//
-// Run "make gen-cue" from repository root to regenerate.
-//
-// Derived from the Thema lineage at %s
-
-`
-
-var tmplImports = genHeader + `package {{ .PackageName }}
-
-import (
-	"embed"
-	"path/filepath"
-
-	"github.com/grafana/grafana/pkg/cuectx"
-	"github.com/grafana/grafana/pkg/framework/coremodel"
-	"github.com/grafana/thema"
-)
-`
-
-var tmplAddenda = template.Must(template.New("addenda").Parse(`
-//go:embed coremodel.cue
-var cueFS embed.FS
-
-// codegen ensures that this is always the latest Thema schema version
-var currentVersion = thema.SV({{ .LatestSeqv }}, {{ .LatestSchv }})
-
-// Lineage returns the Thema lineage representing a Grafana {{ .Name }}.
-//
-// The lineage is the canonical specification of the current {{ .Name }} schema,
-// all prior schema versions, and the mappings that allow migration between
-// schema versions.
-{{- if .IsComposed }}//
-// This is the base variant of the schema. It does not include any composed
-// plugin schemas.{{ end }}
-func Lineage(lib thema.Library, opts ...thema.BindOption) (thema.Lineage, error) {
-	return cuectx.LoadGrafanaInstancesWithThema(filepath.Join("pkg", "coremodel", "{{ .Name }}"), cueFS, lib, opts...)
-}
-
-var _ thema.LineageFactory = Lineage
-var _ coremodel.Interface = &Coremodel{}
-
-// Coremodel contains the foundational schema declaration for {{ .Name }}s.
-// It implements coremodel.Interface.
-type Coremodel struct {
-	lin thema.Lineage
-}
-
-// Lineage returns the canonical {{ .Name }} Lineage.
-func (c *Coremodel) Lineage() thema.Lineage {
-	return c.lin
-}
-
-// CurrentSchema returns the current (latest) {{ .Name }} Thema schema.
-func (c *Coremodel) CurrentSchema() thema.Schema {
-	return thema.SchemaP(c.lin, currentVersion)
-}
-
-// GoType returns a pointer to an empty Go struct that corresponds to
-// the current Thema schema.
-func (c *Coremodel) GoType() interface{} {
-	return &Model{}
-}
-
-// New returns a new instance of the {{ .Name }} coremodel.
-//
-// Note that this function does not cache, and initially loading a Thema lineage
-// can be expensive. As such, the Grafana backend should prefer to access this
-// coremodel through a registry (pkg/framework/coremodel/registry), which does cache.
-func New(lib thema.Library) (*Coremodel, error) {
-	lin, err := Lineage(lib)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Coremodel{
-		lin: lin,
-	}, nil
-}
-`))
-
 var tmplTypedef = `{{range .Types}}
-{{ with .Schema.Description }}{{ . }}{{ else }}// {{.TypeName}} defines model for {{.JsonName}}.{{ end }}
+{{ with .Schema.Description }}{{ . }}{{ else }}// {{.TypeName}} is the Go representation of a {{.JsonName}}.{{ end }}
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
 type {{.TypeName}} {{if and (opts.AliasTypes) (.CanAlias)}}={{end}} {{.Schema.TypeDecl}}
 {{end}}
 `
-
-var tmplRegistry = template.Must(template.New("registry").Parse(`
-// This file is autogenerated. DO NOT EDIT.
-//
-// Generated by pkg/framework/coremodel/gen.go
-// Run "make gen-cue" from repository root to regenerate.
-
-package registry
-
-import (
-	"fmt"
-	"sync"
-
-	"github.com/google/wire"
-	{{range .Coremodels }}
-	"{{ .PkgPath }}"{{end}}
-	"github.com/grafana/grafana/pkg/cuectx"
-	"github.com/grafana/grafana/pkg/framework/coremodel"
-	"github.com/grafana/thema"
-)
-
-// Base is a registry of coremodel.Interface. It provides two modes for accessing
-// coremodels: individually via literal named methods, or as a slice returned from All().
-//
-// Prefer the individual named methods for use cases where the particular coremodel(s) that
-// are needed are known to the caller. For example, a dashboard linter can know that it
-// specifically wants the dashboard coremodel.
-//
-// Prefer All() when performing operations generically across all coremodels. For example,
-// a validation HTTP middleware for any coremodel-schematized object type.
-type Base struct {
-	all []coremodel.Interface
-	{{- range .Coremodels }}
-	{{ .Name }} *{{ .Name }}.Coremodel{{end}}
-}
-
-// type guards
-var (
-{{- range .Coremodels }}
-	_ coremodel.Interface = &{{ .Name }}.Coremodel{}{{end}}
-)
-
-{{range .Coremodels }}
-// {{ .TitleName }} returns the {{ .Name }} coremodel. The return value is guaranteed to
-// implement coremodel.Interface.
-func (s *Base) {{ .TitleName }}() *{{ .Name }}.Coremodel {
-	return s.{{ .Name }}
-}
-{{end}}
-
-func doProvideBase(lib thema.Library) *Base {
-	var err error
-	reg := &Base{}
-
-{{range .Coremodels }}
-	reg.{{ .Name }}, err = {{ .Name }}.New(lib)
-	if err != nil {
-		panic(fmt.Sprintf("error while initializing {{ .Name }} coremodel: %s", err))
-	}
-    reg.all = append(reg.all, reg.{{ .Name }})
-{{end}}
-
-	return reg
-}
-`))

pkg/codegen/pluggen.go

@@ -4,15 +4,20 @@ import (
 	"bytes"
 	"fmt"
 	"io/fs"
+	"path"
 	"path/filepath"
 	"sort"
 	"strings"
-	"text/template"
 
 	"cuelang.org/go/cue/ast"
+	"cuelang.org/go/pkg/encoding/yaml"
+	"github.com/deepmap/oapi-codegen/pkg/codegen"
+	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/grafana/cuetsy"
+	"github.com/grafana/grafana/pkg/framework/coremodel"
 	"github.com/grafana/grafana/pkg/plugins/pfs"
 	"github.com/grafana/thema"
+	"github.com/grafana/thema/encoding/openapi"
 )
 
 // CUE import paths, mapped to corresponding TS import paths. An empty value
@@ -92,13 +97,20 @@ type PluginTreeOrErr struct {
 }
 
 // PluginTree is a pfs.Tree. It exists so we can add methods for code generation to it.
+//
+// It is, for now, tailored specifically to Grafana core's codegen needs.
 type PluginTree pfs.Tree
 
 func (pt *PluginTree) GenerateTS(path string) (WriteDiffer, error) {
 	t := (*pfs.Tree)(pt)
 
 	// TODO replace with cuetsy's TS AST
-	f := &tsFile{}
+	f := &tvars_cuetsy_multi{
+		Header: tvars_autogen_header{
+			GeneratorPath: "public/app/plugins/gen.go", // FIXME hardcoding is not OK
+			LineagePath:   "models.cue",
+		},
+	}
 
 	pi := t.RootPlugin()
 	slotimps := pi.SlotImplementations()
@@ -120,24 +132,18 @@ func (pt *PluginTree) GenerateTS(path string) (WriteDiffer, error) {
 			ModelName: slotname,
 		}
 
-		// TODO this is hardcoded for now, but should ultimately be a property of
-		// whether the slot is a grouped lineage:
-		// https://github.com/grafana/thema/issues/62
-		switch slotname {
-		case "Panel", "DSConfig":
+		if isGroupLineage(slotname) {
 			b, err := cuetsy.Generate(sch.UnwrapCUE(), cuetsy.Config{})
 			if err != nil {
 				return nil, fmt.Errorf("%s: error translating %s lineage to TypeScript: %w", path, slotname, err)
 			}
 			sec.Body = string(b)
-		case "Query":
+		} else {
 			a, err := cuetsy.GenerateSingleAST(strings.Title(lin.Name()), sch.UnwrapCUE(), cuetsy.TypeInterface)
 			if err != nil {
 				return nil, fmt.Errorf("%s: error translating %s lineage to TypeScript: %w", path, slotname, err)
 			}
 			sec.Body = fmt.Sprint(a)
-		default:
-			panic("unrecognized slot name: " + slotname)
 		}
 
 		f.Sections = append(f.Sections, sec)
@@ -145,7 +151,7 @@ func (pt *PluginTree) GenerateTS(path string) (WriteDiffer, error) {
 
 	wd := NewWriteDiffer()
 	var buf bytes.Buffer
-	err := tsSectionTemplate.Execute(&buf, f)
+	err := tmpls.Lookup("cuetsy_multi.tmpl").Execute(&buf, f)
 	if err != nil {
 		return nil, fmt.Errorf("%s: error executing plugin TS generator template: %w", path, err)
 	}
@@ -153,6 +159,260 @@ func (pt *PluginTree) GenerateTS(path string) (WriteDiffer, error) {
 	return wd, nil
 }
 
+func isGroupLineage(slotname string) bool {
+	sl, has := coremodel.AllSlots()[slotname]
+	if !has {
+		panic("unknown slotname name: " + slotname)
+	}
+	return sl.IsGroup()
+}
+
+type GoGenConfig struct {
+	// Types indicates whether corresponding Go types should be generated from the
+	// latest version in the lineage(s).
+	Types bool
+
+	// ThemaBindings indicates whether Thema bindings (an implementation of
+	// ["github.com/grafana/thema".LineageFactory]) should be generated for
+	// lineage(s).
+	ThemaBindings bool
+
+	// DocPathPrefix allows the caller to optionally specify a path to be prefixed
+	// onto paths generated for documentation. This is useful for io/fs-based code
+	// generators, which typically only have knowledge of paths relative to the fs.FS
+	// root, typically an encapsulated subpath, but docs are easier to understand when
+	// paths are relative to a repository root.
+	//
+	// Note that all paths are normalized to use slashes, regardless of the
+	// OS running the code generator.
+	DocPathPrefix string
+}
+
+func (pt *PluginTree) GenerateGo(path string, cfg GoGenConfig) (WriteDiffer, error) {
+	t := (*pfs.Tree)(pt)
+	wd := NewWriteDiffer()
+
+	all := t.SubPlugins()
+	if all == nil {
+		all = make(map[string]pfs.PluginInfo)
+	}
+	all[""] = t.RootPlugin()
+	for subpath, plug := range all {
+		fullp := filepath.Join(path, subpath)
+		if cfg.Types {
+			gwd, err := genGoTypes(plug, path, subpath, cfg.DocPathPrefix)
+			if err != nil {
+				return nil, fmt.Errorf("error generating go types for %s: %w", fullp, err)
+			}
+			if err = wd.Merge(gwd); err != nil {
+				return nil, fmt.Errorf("error merging file set to generate for %s: %w", fullp, err)
+			}
+		}
+		if cfg.ThemaBindings {
+			twd, err := genThemaBindings(plug, path, subpath, cfg.DocPathPrefix)
+			if err != nil {
+				return nil, fmt.Errorf("error generating thema bindings for %s: %w", fullp, err)
+			}
+			if err = wd.Merge(twd); err != nil {
+				return nil, fmt.Errorf("error merging file set to generate for %s: %w", fullp, err)
+			}
+		}
+	}
+
+	return wd, nil
+}
+
+func genGoTypes(plug pfs.PluginInfo, path, subpath, prefix string) (WriteDiffer, error) {
+	wd := NewWriteDiffer()
+	for slotname, lin := range plug.SlotImplementations() {
+		lowslot := strings.ToLower(slotname)
+		lib := lin.Library()
+		sch := thema.SchemaP(lin, thema.LatestVersion(lin))
+
+		// FIXME gotta hack this out of thema in order to deal with our custom imports :scream:
+		f, err := openapi.GenerateSchema(sch, nil)
+		if err != nil {
+			return nil, fmt.Errorf("thema openapi generation failed: %w", err)
+		}
+
+		str, err := yaml.Marshal(lib.Context().BuildFile(f))
+		if err != nil {
+			return nil, fmt.Errorf("cue-yaml marshaling failed: %w", err)
+		}
+
+		loader := openapi3.NewLoader()
+		oT, err := loader.LoadFromData([]byte(str))
+		if err != nil {
+			return nil, fmt.Errorf("loading generated openapi failed; %w", err)
+		}
+
+		buf := new(bytes.Buffer)
+		if err = tmpls.Lookup("autogen_header.tmpl").Execute(buf, tvars_autogen_header{
+			GeneratorPath:  "public/app/plugins/gen.go", // FIXME hardcoding is not OK
+			LineagePath:    filepath.ToSlash(filepath.Join(prefix, subpath, "models.cue")),
+			LineageCUEPath: slotname,
+			GenLicense:     true,
+		}); err != nil {
+			return nil, fmt.Errorf("error generating file header: %w", err)
+		}
+
+		cgopt := codegen.Options{
+			GenerateTypes: true,
+			SkipPrune:     true,
+			SkipFmt:       true,
+			UserTemplates: map[string]string{
+				"imports.tmpl": "package {{ .PackageName }}",
+				"typedef.tmpl": tmplTypedef,
+			},
+		}
+		if isGroupLineage(slotname) {
+			cgopt.ExcludeSchemas = []string{lin.Name()}
+		}
+
+		gostr, err := codegen.Generate(oT, lin.Name(), cgopt)
+		if err != nil {
+			return nil, fmt.Errorf("openapi generation failed: %w", err)
+		}
+		fmt.Fprint(buf, gostr)
+
+		finalpath := filepath.Join(path, subpath, fmt.Sprintf("types_%s_gen.go", lowslot))
+		byt, err := postprocessGoFile(genGoFile{
+			path:   finalpath,
+			walker: makePrefixDropper(strings.Title(lin.Name()), slotname),
+			in:     buf.Bytes(),
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		wd[finalpath] = byt
+	}
+
+	return wd, nil
+}
+
+func genThemaBindings(plug pfs.PluginInfo, path, subpath, prefix string) (WriteDiffer, error) {
+	wd := NewWriteDiffer()
+	bindings := make([]tvars_plugin_lineage_binding, 0)
+	for slotname, lin := range plug.SlotImplementations() {
+		lv := thema.LatestVersion(lin)
+		bindings = append(bindings, tvars_plugin_lineage_binding{
+			SlotName:   slotname,
+			LatestMajv: lv[0],
+			LatestMinv: lv[1],
+		})
+	}
+
+	buf := new(bytes.Buffer)
+	if err := tmpls.Lookup("plugin_lineage_file.tmpl").Execute(buf, tvars_plugin_lineage_file{
+		PackageName: sanitizePluginId(plug.Meta().Id),
+		PluginType:  string(plug.Meta().Type),
+		PluginID:    plug.Meta().Id,
+		SlotImpls:   bindings,
+		HasModels:   len(bindings) != 0,
+		Header: tvars_autogen_header{
+			GeneratorPath: "public/app/plugins/gen.go", // FIXME hardcoding is not OK
+			GenLicense:    true,
+			LineagePath:   filepath.Join(prefix, subpath),
+		},
+	}); err != nil {
+		return nil, fmt.Errorf("error executing plugin lineage file template: %w", err)
+	}
+
+	fullpath := filepath.Join(path, subpath, "pfs_gen.go")
+	if byt, err := postprocessGoFile(genGoFile{
+		path: fullpath,
+		in:   buf.Bytes(),
+	}); err != nil {
+		return nil, err
+	} else {
+		wd[fullpath] = byt
+	}
+
+	return wd, nil
+}
+
+// Plugin IDs are allowed to contain characters that aren't allowed in CUE
+// package names, Go package names, TS or Go type names, etc.
+// TODO expose this as standard
+func sanitizePluginId(s string) string {
+	return strings.Map(func(r rune) rune {
+		switch {
+		case r >= 'a' && r <= 'z':
+			fallthrough
+		case r >= 'A' && r <= 'Z':
+			fallthrough
+		case r >= '0' && r <= '9':
+			fallthrough
+		case r == '_':
+			return r
+		case r == '-':
+			return '_'
+		default:
+			return -1
+		}
+	}, s)
+}
+
+// FIXME unexport this and refactor, this is way too one-off to be in here
+func GenPluginTreeList(trees []TreeAndPath, prefix, target string, ref bool) (WriteDiffer, error) {
+	buf := new(bytes.Buffer)
+	vars := tvars_plugin_registry{
+		Header: tvars_autogen_header{
+			GenLicense: true,
+		},
+		Plugins: make([]struct {
+			PkgName, Path, ImportPath string
+			NoAlias                   bool
+		}, 0, len(trees)),
+	}
+
+	type tpl struct {
+		PkgName, Path, ImportPath string
+		NoAlias                   bool
+	}
+
+	// No sub-plugin support here. If we never allow subplugins in core, that's probably fine.
+	// But still worth noting.
+	for _, pt := range trees {
+		rp := (*pfs.Tree)(pt.Tree).RootPlugin()
+		vars.Plugins = append(vars.Plugins, tpl{
+			PkgName:    sanitizePluginId(rp.Meta().Id),
+			NoAlias:    sanitizePluginId(rp.Meta().Id) != filepath.Base(pt.Path),
+			ImportPath: filepath.ToSlash(filepath.Join(prefix, pt.Path)),
+			Path:       path.Join(append(strings.Split(prefix, "/")[3:], pt.Path)...),
+		})
+	}
+
+	tmplname := "plugin_registry.tmpl"
+	if ref {
+		tmplname = "plugin_registry_ref.tmpl"
+	}
+
+	if err := tmpls.Lookup(tmplname).Execute(buf, vars); err != nil {
+		return nil, fmt.Errorf("failed executing plugin registry template: %w", err)
+	}
+
+	byt, err := postprocessGoFile(genGoFile{
+		path: target,
+		in:   buf.Bytes(),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error postprocessing plugin registry: %w", err)
+	}
+
+	wd := NewWriteDiffer()
+	wd[target] = byt
+	return wd, nil
+}
+
+// FIXME unexport this and refactor, this is way too one-off to be in here
+type TreeAndPath struct {
+	Tree *PluginTree
+	// path relative to path prefix UUUGHHH (basically {panel,datasource}/<dir>}
+	Path string
+}
+
 // TODO convert this to use cuetsy ts types, once import * form is supported
 func convertImport(im *ast.ImportSpec) *tsImport {
 	var err error
@@ -182,11 +442,6 @@ func convertImport(im *ast.ImportSpec) *tsImport {
 	return tsim
 }
 
-type tsFile struct {
-	Imports  []*tsImport
-	Sections []tsSection
-}
-
 type tsSection struct {
 	V         thema.SyntacticVersion
 	ModelName string
@@ -197,15 +452,3 @@ type tsImport struct {
 	Ident string
 	Pkg   string
 }
-
-var tsSectionTemplate = template.Must(template.New("cuetsymulti").Parse(`//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-// This file is autogenerated. DO NOT EDIT.
-//
-// To regenerate, run "make gen-cue" from the repository root.
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-{{range .Imports}}
-import * as {{.Ident}} from '{{.Pkg}}';{{end}}
-{{range .Sections}}{{if ne .ModelName "" }}
-export const {{.ModelName}}ModelVersion = Object.freeze([{{index .V 0}}, {{index .V 1}}]);
-{{end}}
-{{.Body}}{{end}}`))

pkg/codegen/tmpl.go

@@ -0,0 +1,65 @@
+package codegen
+
+import (
+	"embed"
+	"text/template"
+	"time"
+)
+
+// All the parsed templates in the tmpl subdirectory
+var tmpls *template.Template
+
+func init() {
+	base := template.New("codegen").Funcs(template.FuncMap{
+		"now": time.Now,
+	})
+	tmpls = template.Must(base.ParseFS(tmplFS, "tmpl/*.tmpl"))
+}
+
+//go:embed tmpl/*.tmpl
+var tmplFS embed.FS
+
+// The following group of types, beginning with tvars_*, all contain the set
+// of variables expected by the corresponding named template file under tmpl/
+type (
+	tvars_autogen_header struct {
+		GeneratorPath  string
+		LineagePath    string
+		LineageCUEPath string
+		GenLicense     bool
+	}
+	tvars_coremodel_registry struct {
+		Header     tvars_autogen_header
+		Coremodels []tplVars
+	}
+	tvars_coremodel_imports struct {
+		PackageName string
+	}
+	tvars_plugin_lineage_binding struct {
+		SlotName               string
+		LatestMajv, LatestMinv uint
+	}
+	tvars_plugin_lineage_file struct {
+		PackageName string
+		PluginID    string
+		PluginType  string
+		HasModels   bool
+		RootCUE     bool
+		SlotImpls   []tvars_plugin_lineage_binding
+		Header      tvars_autogen_header
+	}
+	tvars_cuetsy_multi struct {
+		Header   tvars_autogen_header
+		Imports  []*tsImport
+		Sections []tsSection
+	}
+	tvars_plugin_registry struct {
+		Header  tvars_autogen_header
+		Plugins []struct {
+			PkgName    string
+			Path       string
+			ImportPath string
+			NoAlias    bool
+		}
+	}
+)

pkg/codegen/tmpl/addenda.tmpl

@@ -0,0 +1,64 @@
+
+
+//go:embed coremodel.cue
+var cueFS embed.FS
+
+// The current version of the coremodel schema, as declared in coremodel.cue.
+// This version determines what schema version is returned from [Coremodel.CurrentSchema],
+// and which schema version is used for code generation within the grafana/grafana repository.
+//
+// The code generator ensures that this is always the latest Thema schema version.
+var currentVersion = thema.SV({{ .LatestSeqv }}, {{ .LatestSchv }})
+
+// Lineage returns the Thema lineage representing a Grafana {{ .Name }}.
+//
+// The lineage is the canonical specification of the current {{ .Name }} schema,
+// all prior schema versions, and the mappings that allow migration between
+// schema versions.
+{{- if .IsComposed }}//
+// This is the base variant of the schema. It does not include any composed
+// plugin schemas.{{ end }}
+func Lineage(lib thema.Library, opts ...thema.BindOption) (thema.Lineage, error) {
+	return cuectx.LoadGrafanaInstancesWithThema(filepath.Join("pkg", "coremodel", "{{ .Name }}"), cueFS, lib, opts...)
+}
+
+var _ thema.LineageFactory = Lineage
+var _ coremodel.Interface = &Coremodel{}
+
+// Coremodel contains the foundational schema declaration for {{ .Name }}s.
+// It implements coremodel.Interface.
+type Coremodel struct {
+	lin thema.Lineage
+}
+
+// Lineage returns the canonical {{ .Name }} Lineage.
+func (c *Coremodel) Lineage() thema.Lineage {
+	return c.lin
+}
+
+// CurrentSchema returns the current (latest) {{ .Name }} Thema schema.
+func (c *Coremodel) CurrentSchema() thema.Schema {
+	return thema.SchemaP(c.lin, currentVersion)
+}
+
+// GoType returns a pointer to an empty Go struct that corresponds to
+// the current Thema schema.
+func (c *Coremodel) GoType() interface{} {
+	return &Model{}
+}
+
+// New returns a new instance of the {{ .Name }} coremodel.
+//
+// Note that this function does not cache, and initially loading a Thema lineage
+// can be expensive. As such, the Grafana backend should prefer to access this
+// coremodel through a registry (pkg/framework/coremodel/registry), which does cache.
+func New(lib thema.Library) (*Coremodel, error) {
+	lin, err := Lineage(lib)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Coremodel{
+		lin: lin,
+	}, nil
+}

pkg/codegen/tmpl/autogen_header.tmpl

@@ -0,0 +1,28 @@
+{{ if .GenLicense -}}
+// Copyright {{ now.Year }} Grafana Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{{ end -}}
+// This file is autogenerated. DO NOT EDIT.
+{{- if ne .GeneratorPath "" }}
+//
+// Generated by {{ .GeneratorPath }}
+{{- end }}
+{{- if ne .LineagePath "" }}
+//
+// Derived from the Thema lineage declared in {{ .LineagePath }}{{ if ne .LineageCUEPath "" }} at CUE path "{{ .LineageCUEPath }}"{{ end }}
+{{- end }}
+//
+// Run `make gen-cue` from repository root to regenerate.
+

pkg/codegen/tmpl/coremodel_imports.tmpl

@@ -0,0 +1,10 @@
+package {{ .PackageName }}
+
+import (
+	"embed"
+	"path/filepath"
+
+	"github.com/grafana/grafana/pkg/cuectx"
+	"github.com/grafana/grafana/pkg/framework/coremodel"
+	"github.com/grafana/thema"
+)

pkg/codegen/tmpl/coremodel_registry.tmpl

@@ -0,0 +1,58 @@
+{{ template "autogen_header.tmpl" .Header }}
+package registry
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/google/wire"
+	{{range .Coremodels }}
+	"{{ .PkgPath }}"{{end}}
+	"github.com/grafana/grafana/pkg/cuectx"
+	"github.com/grafana/grafana/pkg/framework/coremodel"
+	"github.com/grafana/thema"
+)
+
+// Base is a registry of coremodel.Interface. It provides two modes for accessing
+// coremodels: individually via literal named methods, or as a slice returned from All().
+//
+// Prefer the individual named methods for use cases where the particular coremodel(s) that
+// are needed are known to the caller. For example, a dashboard linter can know that it
+// specifically wants the dashboard coremodel.
+//
+// Prefer All() when performing operations generically across all coremodels. For example,
+// a validation HTTP middleware for any coremodel-schematized object type.
+type Base struct {
+	all []coremodel.Interface
+	{{- range .Coremodels }}
+	{{ .Name }} *{{ .Name }}.Coremodel{{end}}
+}
+
+// type guards
+var (
+{{- range .Coremodels }}
+	_ coremodel.Interface = &{{ .Name }}.Coremodel{}{{end}}
+)
+
+{{range .Coremodels }}
+// {{ .TitleName }} returns the {{ .Name }} coremodel. The return value is guaranteed to
+// implement coremodel.Interface.
+func (b *Base) {{ .TitleName }}() *{{ .Name }}.Coremodel {
+	return b.{{ .Name }}
+}
+{{end}}
+
+func doProvideBase(lib thema.Library) *Base {
+	var err error
+	reg := &Base{}
+
+{{range .Coremodels }}
+	reg.{{ .Name }}, err = {{ .Name }}.New(lib)
+	if err != nil {
+		panic(fmt.Sprintf("error while initializing {{ .Name }} coremodel: %s", err))
+	}
+    reg.all = append(reg.all, reg.{{ .Name }})
+{{end}}
+
+	return reg
+}

pkg/codegen/tmpl/cuetsy_multi.tmpl

@@ -0,0 +1,7 @@
+{{ template "autogen_header.tmpl" .Header -}}
+{{range .Imports}}
+import * as {{.Ident}} from '{{.Pkg}}';{{end}}
+{{range .Sections}}{{if ne .ModelName "" }}
+export const {{.ModelName}}ModelVersion = Object.freeze([{{index .V 0}}, {{index .V 1}}]);
+{{end}}
+{{.Body}}{{end}}

pkg/codegen/tmpl/plugin_lineage_binding.tmpl

@@ -0,0 +1,12 @@
+// The current version of the coremodel schema, as declared in coremodel.cue.
+// This version determines what schema version is returned from [Coremodel.CurrentSchema],
+// and which schema version is used for code generation within the grafana/grafana repository.
+//
+// The code generator ensures that this is always the latest Thema schema version.
+var currentVersion{{ .SlotName }} = thema.SV({{ .LatestSeqv }}, {{ .LatestSchv }})
+
+// {{ .SlotName }}Lineage returns the Thema lineage for the {{ .PluginID }} {{ .PluginType }} plugin's
+// {{ .SlotName }} ["github.com/grafana/grafana/pkg/framework/coremodel".Slot] implementation.
+func {{ .SlotName }}Lineage(lib thema.Library, opts ...thema.BindOption) (thema.Lineage, error) {
+	return cuectx.LoadGrafanaInstancesWithThema(filepath.Join("public", "app", "{{ .Name }}"), cueFS, lib, opts...)
+}

pkg/codegen/tmpl/plugin_lineage_file.tmpl

@@ -0,0 +1,58 @@
+{{ template "autogen_header.tmpl" .Header -}}
+package {{ .PackageName }}
+
+import (
+	"embed"
+	"fmt"
+	"path/filepath"
+	"sync"
+
+	"github.com/grafana/grafana/pkg/cuectx"
+	"github.com/grafana/grafana/pkg/plugins/pfs"
+	"github.com/grafana/thema"
+)
+
+var parseOnce sync.Once
+var ptree *pfs.Tree
+
+//go:embed plugin.{{ if .RootCUE }}cue{{ else }}json{{ end }}{{ if .HasModels }} models.cue{{ end }}
+var plugFS embed.FS
+
+// PluginTree returns the plugin tree representing the statically analyzable contents of the {{ .PluginID }} plugin.
+func PluginTree(lib *thema.Library) *pfs.Tree {
+	var err error
+	if lib == nil {
+		parseOnce.Do(func() {
+			ptree, err = pfs.ParsePluginFS(plugFS, cuectx.ProvideThemaLibrary())
+		})
+	} else {
+		ptree, err = pfs.ParsePluginFS(plugFS, cuectx.ProvideThemaLibrary())
+	}
+
+	if err != nil {
+		// Even the most rudimentary testing in CI ensures this is unreachable
+		panic(fmt.Errorf("error parsing plugin fs tree: %w", err))
+	}
+
+	return ptree
+}
+
+{{ $pluginfo := . }}{{ range $slot := .SlotImpls }}
+// {{ .SlotName }}Lineage returns the Thema lineage for the {{ $pluginfo.PluginID }} {{ $pluginfo.PluginType }} plugin's
+// {{ .SlotName }} ["github.com/grafana/grafana/pkg/framework/coremodel".Slot] implementation.
+func {{ .SlotName }}Lineage(lib *thema.Library, opts ...thema.BindOption) (thema.Lineage, error) {
+  t := PluginTree(lib)
+  lin, has := t.RootPlugin().SlotImplementations()["{{ .SlotName }}"]
+  if !has {
+    panic("unreachable: lineage for {{ .SlotName }} does not exist, but code is only generated for existing lineages")
+  }
+  return lin, nil
+}
+
+// The current schema version of the {{ .SlotName }} slot implementation.
+//
+// Code generation ensures that this is always the version number for the latest schema
+// in the {{ .SlotName }} Thema lineage.
+var currentVersion{{ .SlotName }} = thema.SV({{ .LatestMajv }}, {{ .LatestMinv }})
+
+{{ end }}

pkg/codegen/tmpl/plugin_registry.tmpl

@@ -0,0 +1,31 @@
+{{ template "autogen_header.tmpl" .Header -}}
+package corelist
+
+import (
+  "fmt"
+  "io/fs"
+  "sync"
+  "github.com/grafana/grafana"
+	"github.com/grafana/grafana/pkg/plugins/pfs"
+	"github.com/grafana/thema"
+)
+
+func makeTreeOrPanic(path string, pkgname string, lib thema.Library) *pfs.Tree {
+  sub, err := fs.Sub(grafana.CueSchemaFS, path)
+  if err != nil {
+    panic("could not create fs sub to " + path)
+  }
+  tree, err := pfs.ParsePluginFS(sub, lib)
+  if err != nil {
+    panic(fmt.Sprintf("error parsing plugin metadata for %s: %s", pkgname, err))
+  }
+  return tree
+}
+
+func coreTreeList(lib thema.Library) pfs.TreeList{
+  return pfs.TreeList{
+    {{- range .Plugins }}
+    makeTreeOrPanic("{{ .Path }}", "{{ .PkgName }}", lib),
+    {{- end }}
+  }
+}

pkg/codegen/tmpl/plugin_registry_ref.tmpl

@@ -0,0 +1,16 @@
+{{ template "autogen_header.tmpl" .Header -}}
+package registry
+
+import (
+	"github.com/grafana/grafana/pkg/plugins/pfs"
+	"github.com/grafana/thema"
+  {{ range .Plugins }}
+  {{ if .NoAlias }}{{ .PkgName }} {{end}}"{{ .Path }}"{{ end }}
+)
+
+func coreTreeLoaders() []func(*thema.Library) *pfs.Tree{
+  return []func(*thema.Library) *pfs.Tree{
+    {{- range .Plugins }}
+    {{ .PkgName }}.PluginTree,{{ end }}
+  }
+}

pkg/codegen/util_go.go

@@ -0,0 +1,67 @@
+package codegen
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/format"
+	"go/parser"
+	"go/token"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/tools/imports"
+)
+
+type genGoFile struct {
+	path   string
+	walker ast.Visitor
+	in     []byte
+}
+
+func postprocessGoFile(cfg genGoFile) ([]byte, error) {
+	fname := filepath.Base(cfg.path)
+	buf := new(bytes.Buffer)
+	fset := token.NewFileSet()
+	gf, err := parser.ParseFile(fset, fname, string(cfg.in), parser.ParseComments)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing generated file: %w", err)
+	}
+
+	if cfg.walker != nil {
+		ast.Walk(cfg.walker, gf)
+
+		err = format.Node(buf, fset, gf)
+		if err != nil {
+			return nil, fmt.Errorf("error formatting Go AST: %w", err)
+		}
+	} else {
+		buf = bytes.NewBuffer(cfg.in)
+	}
+
+	byt, err := imports.Process(fname, buf.Bytes(), nil)
+	if err != nil {
+		return nil, fmt.Errorf("goimports processing failed: %w", err)
+	}
+
+	// Compare imports before and after; warn about performance if some were added
+	gfa, _ := parser.ParseFile(fset, fname, string(byt), parser.ParseComments)
+	imap := make(map[string]bool)
+	for _, im := range gf.Imports {
+		imap[im.Path.Value] = true
+	}
+	var added []string
+	for _, im := range gfa.Imports {
+		if !imap[im.Path.Value] {
+			added = append(added, im.Path.Value)
+		}
+	}
+
+	if len(added) != 0 {
+		// TODO improve the guidance in this error if/when we better abstract over imports to generate
+		fmt.Fprintf(os.Stderr, "The following imports were added by goimports while generating %s: \n\t%s\nRelying on goimports to find imports significantly slows down code generation. Consider adding these to the relevant template.\n", cfg.path, strings.Join(added, "\n\t"))
+	}
+
+	return byt, nil
+}