codegen: Introduce TS codegen veneer (#54816)

* Split all named types out into defs, etc.

* Use latest cuetsy, refactor generators accordingly

* Return AST type from plugin TS generator

* Near-complete checkin of TS veneer code generator

* First full completed pass

* Improve the attribute name

* Defer use of the dashboard veneer type to follow-up

* Remove dummy index, prettier on veneer

* Fix merge errors in gen.go

* Add match field to SpecialValueMap

* Fix backend lint errors

pkg/codegen/coremodel.go

@@ -17,14 +17,15 @@ import (
 	"github.com/deepmap/oapi-codegen/pkg/codegen"
 	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/grafana/cuetsy"
+	tsast "github.com/grafana/cuetsy/ts/ast"
 	"github.com/grafana/grafana/pkg/cuectx"
 	"github.com/grafana/thema"
 	"github.com/grafana/thema/encoding/openapi"
 )
 
-// ExtractedLineage contains the results of statically analyzing a Grafana
+// CoremodelDeclaration contains the results of statically analyzing a Grafana
 // directory for a Thema lineage.
-type ExtractedLineage struct {
+type CoremodelDeclaration struct {
 	Lineage thema.Lineage
 	// Absolute path to the coremodel's coremodel.cue file.
 	LineagePath string
@@ -48,12 +49,12 @@ type ExtractedLineage struct {
 // This loading approach is intended primarily for use with code generators, or
 // other use cases external to grafana-server backend. For code within
 // grafana-server, prefer lineage loaders provided in e.g. pkg/coremodel/*.
-func ExtractLineage(path string, lib thema.Library) (*ExtractedLineage, error) {
+func ExtractLineage(path string, lib thema.Library) (*CoremodelDeclaration, error) {
 	if !filepath.IsAbs(path) {
 		return nil, fmt.Errorf("must provide an absolute path, got %q", path)
 	}
 
-	ec := &ExtractedLineage{
+	ec := &CoremodelDeclaration{
 		LineagePath: path,
 	}
 
@@ -107,14 +108,14 @@ func ExtractLineage(path string, lib thema.Library) (*ExtractedLineage, error) {
 }
 
 // toTemplateObj extracts creates a struct with all the useful strings for template generation.
-func (ls *ExtractedLineage) toTemplateObj() tplVars {
-	lin := ls.Lineage
+func (cd *CoremodelDeclaration) toTemplateObj() tplVars {
+	lin := cd.Lineage
 	sch := thema.SchemaP(lin, thema.LatestVersion(lin))
 
 	return tplVars{
 		Name:        lin.Name(),
-		LineagePath: ls.RelativePath,
-		PkgPath:     filepath.ToSlash(filepath.Join("github.com/grafana/grafana", filepath.Dir(ls.RelativePath))),
+		LineagePath: cd.RelativePath,
+		PkgPath:     filepath.ToSlash(filepath.Join("github.com/grafana/grafana", filepath.Dir(cd.RelativePath))),
 		TitleName:   strings.Title(lin.Name()), // nolint
 		LatestSeqv:  sch.Version()[0],
 		LatestSchv:  sch.Version()[1],
@@ -139,13 +140,22 @@ var nonAPITypes = map[string]bool{
 	"pluginmeta": true,
 }
 
+// PathVersion returns the string path element to use for the latest schema.
+// "x" if not yet canonical, otherwise, "v<major>"
+func (cd *CoremodelDeclaration) PathVersion() string {
+	if !cd.IsCanonical {
+		return "x"
+	}
+	return fmt.Sprintf("v%v", thema.LatestVersion(cd.Lineage)[0])
+}
+
 // GenerateGoCoremodel generates a standard Go model struct and coremodel
 // implementation from a coremodel CUE declaration.
 //
 // The provided path must be a directory. Generated code files will be written
 // to that path. The final element of the path must match the Lineage.Name().
-func (ls *ExtractedLineage) GenerateGoCoremodel(path string) (WriteDiffer, error) {
-	lin, lib := ls.Lineage, ls.Lineage.Library()
+func (cd *CoremodelDeclaration) GenerateGoCoremodel(path string) (WriteDiffer, error) {
+	lin, lib := cd.Lineage, cd.Lineage.Library()
 	_, name := filepath.Split(path)
 	if name != lin.Name() {
 		return nil, fmt.Errorf("lineage name %q must match final element of path, got %q", lin.Name(), path)
@@ -190,7 +200,7 @@ func (ls *ExtractedLineage) GenerateGoCoremodel(path string) (WriteDiffer, error
 
 	buf := new(bytes.Buffer)
 	if err = tmpls.Lookup("autogen_header.tmpl").Execute(buf, tvars_autogen_header{
-		LineagePath:   ls.RelativePath,
+		LineagePath:   cd.RelativePath,
 		GeneratorPath: "pkg/framework/coremodel/gen.go", // FIXME hardcoding is not OK
 	}); err != nil {
 		return nil, fmt.Errorf("error executing header template: %w", err)
@@ -198,7 +208,7 @@ func (ls *ExtractedLineage) GenerateGoCoremodel(path string) (WriteDiffer, error
 
 	fmt.Fprint(buf, "\n", gostr)
 
-	vars := ls.toTemplateObj()
+	vars := cd.toTemplateObj()
 	err = tmpls.Lookup("addenda.tmpl").Execute(buf, vars)
 	if err != nil {
 		panic(err)
@@ -228,59 +238,38 @@ type tplVars struct {
 	IsComposed             bool
 }
 
-func (ls *ExtractedLineage) GenerateTypescriptCoremodel(path string) (WriteDiffer, error) {
-	_, name := filepath.Split(path)
-	if name != ls.Lineage.Name() {
-		return nil, fmt.Errorf("lineage name %q must match final element of path, got %q", ls.Lineage.Name(), path)
-	}
+func (cd *CoremodelDeclaration) GenerateTypescriptCoremodel() (*tsast.File, error) {
+	schv := thema.SchemaP(cd.Lineage, thema.LatestVersion(cd.Lineage)).UnwrapCUE()
 
-	schv := thema.SchemaP(ls.Lineage, thema.LatestVersion(ls.Lineage)).UnwrapCUE()
-
-	parts, err := cuetsy.GenerateAST(schv, cuetsy.Config{})
+	tf, err := cuetsy.GenerateAST(schv, cuetsy.Config{
+		Export: true,
+	})
 	if err != nil {
-		return nil, fmt.Errorf("cuetsy parts gen failed: %w", err)
+		return nil, fmt.Errorf("cuetsy tf gen failed: %w", err)
 	}
 
-	top, err := cuetsy.GenerateSingleAST(strings.Title(ls.Lineage.Name()), schv, cuetsy.TypeInterface)
+	top, err := cuetsy.GenerateSingleAST(strings.Title(cd.Lineage.Name()), schv, cuetsy.TypeInterface)
 	if err != nil {
 		return nil, fmt.Errorf("cuetsy top gen failed: %s", cerrors.Details(err, nil))
 	}
 
-	// TODO until cuetsy can toposort its outputs, put the top/parent type at the bottom of the file.
-	parts.Nodes = append(parts.Nodes, top.T)
-	if top.D != nil {
-		parts.Nodes = append(parts.Nodes, top.D)
-	}
-
-	var strb strings.Builder
-	var str string
-	fpath := ls.Lineage.Name() + ".gen.ts"
-	if err := tmpls.Lookup("autogen_header.tmpl").Execute(&strb, tvars_autogen_header{
-		LineagePath:   ls.RelativePath,
+	buf := new(bytes.Buffer)
+	if err := tmpls.Lookup("autogen_header.tmpl").Execute(buf, tvars_autogen_header{
+		LineagePath:   cd.RelativePath,
 		GeneratorPath: "pkg/framework/coremodel/gen.go", // FIXME hardcoding is not OK
 	}); err != nil {
 		return nil, fmt.Errorf("error executing header template: %w", err)
 	}
-
-	if !ls.IsCanonical {
-		fpath = fmt.Sprintf("%s_experimental.gen.ts", ls.Lineage.Name())
-		strb.WriteString(`
-// This model is a WIP and not yet canonical. Consequently, its members are
-// not exported to exclude it from grafana-schema's public API surface.
-
-`)
-		strb.WriteString(fmt.Sprint(parts))
-		// TODO replace this regexp with cuetsy config for whether members are exported
-		re := regexp.MustCompile(`(?m)^export `)
-		str = re.ReplaceAllLiteralString(strb.String(), "")
-	} else {
-		strb.WriteString(fmt.Sprint(parts))
-		str = strb.String()
+	tf.Doc = &tsast.Comment{
+		Text: buf.String(),
 	}
 
-	wd := NewWriteDiffer()
-	wd[filepath.Join(path, fpath)] = []byte(str)
-	return wd, nil
+	// TODO until cuetsy can toposort its outputs, put the top/parent type at the bottom of the file.
+	tf.Nodes = append(tf.Nodes, top.T)
+	if top.D != nil {
+		tf.Nodes = append(tf.Nodes, top.D)
+	}
+	return tf, nil
 }
 
 type prefixDropper struct {
@@ -319,7 +308,7 @@ func (d prefixDropper) Visit(n ast.Node) ast.Visitor {
 // GenerateCoremodelRegistry produces Go files that define a registry with
 // references to all the Go code that is expected to be generated from the
 // provided lineages.
-func GenerateCoremodelRegistry(path string, ecl []*ExtractedLineage) (WriteDiffer, error) {
+func GenerateCoremodelRegistry(path string, ecl []*CoremodelDeclaration) (WriteDiffer, error) {
 	var cml []tplVars
 	for _, ec := range ecl {
 		cml = append(cml, ec.toTemplateObj())

pkg/codegen/pluggen.go

@@ -14,6 +14,7 @@ import (
 	"github.com/deepmap/oapi-codegen/pkg/codegen"
 	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/grafana/cuetsy"
+	tsast "github.com/grafana/cuetsy/ts/ast"
 	"github.com/grafana/grafana/pkg/framework/coremodel"
 	"github.com/grafana/grafana/pkg/plugins/pfs"
 	"github.com/grafana/thema"
@@ -101,15 +102,22 @@ type PluginTreeOrErr struct {
 // It is, for now, tailored specifically to Grafana core's codegen needs.
 type PluginTree pfs.Tree
 
-func (pt *PluginTree) GenerateTS(path string) (WriteDiffer, error) {
+func (pt *PluginTree) GenerateTypeScriptAST() (*tsast.File, error) {
 	t := (*pfs.Tree)(pt)
+	f := &tsast.File{}
 
-	// TODO replace with cuetsy's TS AST
-	f := &tvars_cuetsy_multi{
-		Header: tvars_autogen_header{
-			GeneratorPath: "public/app/plugins/gen.go", // FIXME hardcoding is not OK
-			LineagePath:   "models.cue",
-		},
+	tf := tvars_autogen_header{
+		GeneratorPath: "public/app/plugins/gen.go", // FIXME hardcoding is not OK
+		LineagePath:   "models.cue",
+	}
+	var buf bytes.Buffer
+	err := tmpls.Lookup("autogen_header.tmpl").Execute(&buf, tf)
+	if err != nil {
+		return nil, fmt.Errorf("error executing header template: %w", err)
+	}
+
+	f.Doc = &tsast.Comment{
+		Text: buf.String(),
 	}
 
 	pi := t.RootPlugin()
@@ -118,7 +126,9 @@ func (pt *PluginTree) GenerateTS(path string) (WriteDiffer, error) {
 		return nil, nil
 	}
 	for _, im := range pi.CUEImports() {
-		if tsim := convertImport(im); tsim != nil {
+		if tsim, err := convertImport(im); err != nil {
+			return nil, err
+		} else if tsim.From.Value != "" {
 			f.Imports = append(f.Imports, tsim)
 		}
 	}
@@ -126,37 +136,36 @@ func (pt *PluginTree) GenerateTS(path string) (WriteDiffer, error) {
 	for slotname, lin := range slotimps {
 		v := thema.LatestVersion(lin)
 		sch := thema.SchemaP(lin, v)
-		// TODO need call expressions in cuetsy tsast to be able to do these
-		sec := tsSection{
-			V:         v,
-			ModelName: slotname,
-		}
+		// Inject a node for the const with the version
+		f.Nodes = append(f.Nodes, tsast.Raw{
+			// TODO need call expressions in cuetsy tsast to be able to do these properly
+			Data: fmt.Sprintf("export const %sModelVersion = Object.freeze([%v, %v]);", slotname, v[0], v[1]),
+		})
 
+		// TODO this is hardcoded for now, but should ultimately be a property of
+		// whether the slot is a grouped lineage:
+		// https://github.com/grafana/thema/issues/62
 		if isGroupLineage(slotname) {
-			b, err := cuetsy.Generate(sch.UnwrapCUE(), cuetsy.Config{})
+			tsf, err := cuetsy.GenerateAST(sch.UnwrapCUE(), cuetsy.Config{
+				Export: true,
+			})
 			if err != nil {
-				return nil, fmt.Errorf("%s: error translating %s lineage to TypeScript: %w", path, slotname, err)
+				return nil, fmt.Errorf("error translating %s lineage to TypeScript: %w", slotname, err)
 			}
-			sec.Body = string(b)
+			f.Nodes = append(f.Nodes, tsf.Nodes...)
 		} else {
-			a, err := cuetsy.GenerateSingleAST(strings.Title(lin.Name()), sch.UnwrapCUE(), cuetsy.TypeInterface)
+			pair, err := cuetsy.GenerateSingleAST(strings.Title(lin.Name()), sch.UnwrapCUE(), cuetsy.TypeInterface)
 			if err != nil {
-				return nil, fmt.Errorf("%s: error translating %s lineage to TypeScript: %w", path, slotname, err)
+				return nil, fmt.Errorf("error translating %s lineage to TypeScript: %w", slotname, err)
+			}
+			f.Nodes = append(f.Nodes, pair.T)
+			if pair.D != nil {
+				f.Nodes = append(f.Nodes, pair.D)
 			}
-			sec.Body = fmt.Sprint(a)
 		}
-
-		f.Sections = append(f.Sections, sec)
 	}
 
-	wd := NewWriteDiffer()
-	var buf bytes.Buffer
-	err := tmpls.Lookup("cuetsy_multi.tmpl").Execute(&buf, f)
-	if err != nil {
-		return nil, fmt.Errorf("%s: error executing plugin TS generator template: %w", path, err)
-	}
-	wd[filepath.Join(path, "models.gen.ts")] = buf.Bytes()
-	return wd, nil
+	return f, nil
 }
 
 func isGroupLineage(slotname string) bool {
@@ -414,41 +423,27 @@ type TreeAndPath struct {
 }
 
 // TODO convert this to use cuetsy ts types, once import * form is supported
-func convertImport(im *ast.ImportSpec) *tsImport {
-	var err error
-	tsim := &tsImport{}
-	tsim.Pkg, err = MapCUEImportToTS(strings.Trim(im.Path.Value, "\""))
-	if err != nil {
-		// should be unreachable if paths has been verified already
-		panic(err)
+func convertImport(im *ast.ImportSpec) (tsast.ImportSpec, error) {
+	tsim := tsast.ImportSpec{}
+	pkg, err := MapCUEImportToTS(strings.Trim(im.Path.Value, "\""))
+	if err != nil || pkg == "" {
+		// err should be unreachable if paths has been verified already
+		// Empty string mapping means skip it
+		return tsim, err
 	}
 
-	if tsim.Pkg == "" {
-		// Empty string mapping means skip it
-		return nil
-	}
+	tsim.From = tsast.Str{Value: pkg}
 
 	if im.Name != nil && im.Name.String() != "" {
-		tsim.Ident = im.Name.String()
+		tsim.AsName = im.Name.String()
 	} else {
 		sl := strings.Split(im.Path.Value, "/")
 		final := sl[len(sl)-1]
 		if idx := strings.Index(final, ":"); idx != -1 {
-			tsim.Pkg = final[idx:]
+			tsim.AsName = final[idx:]
 		} else {
-			tsim.Pkg = final
+			tsim.AsName = final
 		}
 	}
-	return tsim
-}
-
-type tsSection struct {
-	V         thema.SyntacticVersion
-	ModelName string
-	Body      string
-}
-
-type tsImport struct {
-	Ident string
-	Pkg   string
+	return tsim, nil
 }

pkg/codegen/tmpl.go

@@ -1,6 +1,7 @@
 package codegen
 
 import (
+	"bytes"
 	"embed"
 	"text/template"
 	"time"
@@ -48,11 +49,6 @@ type (
 		SlotImpls   []tvars_plugin_lineage_binding
 		Header      tvars_autogen_header
 	}
-	tvars_cuetsy_multi struct {
-		Header   tvars_autogen_header
-		Imports  []*tsImport
-		Sections []tsSection
-	}
 	tvars_plugin_registry struct {
 		Header  tvars_autogen_header
 		Plugins []struct {
@@ -63,3 +59,14 @@ type (
 		}
 	}
 )
+
+type HeaderVars = tvars_autogen_header
+
+// GenGrafanaHeader creates standard header elements for generated Grafana files.
+func GenGrafanaHeader(vars HeaderVars) string {
+	buf := new(bytes.Buffer)
+	if err := tmpls.Lookup("autogen_header.tmpl").Execute(buf, vars); err != nil {
+		panic(err)
+	}
+	return buf.String()
+}

pkg/codegen/tmpl/cuetsy_multi.tmpl

@@ -1,7 +1,2 @@
 {{ template "autogen_header.tmpl" .Header -}}
-{{range .Imports}}
-import * as {{.Ident}} from '{{.Pkg}}';{{end}}
-{{range .Sections}}{{if ne .ModelName "" }}
-export const {{.ModelName}}ModelVersion = Object.freeze([{{index .V 0}}, {{index .V 1}}]);
-{{end}}
-{{.Body}}{{end}}
+{{ .Body }}