Core: Remove thema and kindsys dependencies (#84499)

* Move some thema code inside grafana

* Use new codegen instead of thema for core kinds

* Replace TS generator

* Use new generator for go types

* Remove thema from oapi generator

* Remove thema from generators

* Don't use kindsys/thema for core kinds

* Remove kindsys/thema from plugins

* Remove last thema related

* Remove most of cuectx and move utils_ts into codegen. It also deletes wire dependency

* Merge plugins generators

* Delete thema dependency 🎉

* Fix CODEOWNERS

* Fix package name

* Fix TS output names

* More path fixes

* Fix mod codeowners

* Use original plugin's name

* Remove kindsys dependency 🎉

* Modify oapi schema and create an apply function to fix elasticsearch errors

* cue.mod was deleted by mistake

* Fix TS panels

* sort imports

* Fixing elasticsearch output

* Downgrade oapi-codegen library

* Update output ts files

* More fixes

* Restore old elasticsearch generated file and skip its generation. Remove core imports into plugins

* More lint fixes

* Add codeowners

* restore embed.go file

* Fix embed.go
This commit is contained in:
Selene
2024-03-21 11:11:29 +01:00
committed by GitHub
parent 856e410480
commit 473898e47c
56 changed files with 1433 additions and 1631 deletions

View File

@@ -7,14 +7,12 @@ import (
"cuelang.org/go/cue"
"github.com/grafana/codejen"
"github.com/grafana/kindsys"
"github.com/grafana/thema"
)
type OneToOne codejen.OneToOne[kindsys.Kind]
type OneToMany codejen.OneToMany[kindsys.Kind]
type ManyToOne codejen.ManyToOne[kindsys.Kind]
type ManyToMany codejen.ManyToMany[kindsys.Kind]
type OneToOne codejen.OneToOne[SchemaForGen]
type OneToMany codejen.OneToMany[SchemaForGen]
type ManyToOne codejen.ManyToOne[SchemaForGen]
type ManyToMany codejen.ManyToMany[SchemaForGen]
// SlashHeaderMapper produces a FileMapper that injects a comment header onto
// a [codejen.File] indicating the main generator that produced it (via the provided
@@ -47,21 +45,10 @@ func SlashHeaderMapper(maingen string) codejen.FileMapper {
}
}
// SchemaForGen is an intermediate values type for jennies that holds both a thema.Schema,
// and values relevant to generating the schema that should properly, eventually, be in
// thema itself.
//
// TODO this will be replaced by thema-native constructs
type SchemaForGen struct {
// The PascalCase name of the schematized type.
Name string
// The schema to be rendered for the type itself.
Schema thema.Schema
// Whether the schema is grouped. See https://github.com/grafana/thema/issues/62
IsGroup bool
}
type CueSchema struct {
CueFile cue.Value
FilePath string
Name string
CueFile cue.Value
FilePath string
IsGroup bool
OutputName string // Some TS output files are capitalised and others not.
}

View File

@@ -0,0 +1,282 @@
package generators
import (
"fmt"
"regexp"
"strings"
"unicode"
"github.com/dave/dst"
"github.com/dave/dst/dstutil"
)
// depointerizer returns an AST manipulator that removes redundant
// pointer indirection from the defined types.
func depointerizer() dstutil.ApplyFunc {
return func(c *dstutil.Cursor) bool {
switch x := c.Node().(type) {
case *dst.Field:
if s, is := x.Type.(*dst.StarExpr); is {
switch deref := depoint(s).(type) {
case *dst.ArrayType, *dst.MapType:
x.Type = deref
}
}
}
return true
}
}
func depoint(e dst.Expr) dst.Expr {
if star, is := e.(*dst.StarExpr); is {
return star.X
}
return e
}
func setStar(e dst.Expr) string {
if _, is := e.(*dst.StarExpr); is {
return "*"
}
return ""
}
func fixTODOComments() dstutil.ApplyFunc {
return func(cursor *dstutil.Cursor) bool {
switch f := cursor.Node().(type) {
case *dst.File:
for _, d := range f.Decls {
if isTypeSpec(d) {
removeGoFieldComment(d.Decorations().Start.All())
}
fixTODOComment(d.Decorations().Start.All())
}
case *dst.Field:
if len(f.Names) > 0 {
removeGoFieldComment(f.Decorations().Start.All())
}
}
return true
}
}
func fixTODOComment(comments []string) {
todoRegex := regexp.MustCompile("(//) (.*) (TODO.*)")
if len(comments) > 0 {
comments[0] = todoRegex.ReplaceAllString(comments[0], "$1 $3")
}
}
func removeGoFieldComment(comments []string) {
todoRegex := regexp.MustCompile("(//) ([A-Z].*?) ([A-Z]?.*?) (.*)")
if len(comments) > 0 {
matches := todoRegex.FindAllStringSubmatch(comments[0], -1)
if len(matches) > 0 {
if strings.EqualFold(matches[0][3], matches[0][2]) {
comments[0] = fmt.Sprintf("%s %s %s", matches[0][1], matches[0][3], matches[0][4])
} else {
r := []rune(matches[0][3])
if !unicode.IsLower(r[0]) {
comments[0] = fmt.Sprintf("%s %s %s", matches[0][1], matches[0][3], matches[0][4])
}
}
}
}
}
func isTypeSpec(d dst.Decl) bool {
gd, ok := d.(*dst.GenDecl)
if !ok {
return false
}
_, is := gd.Specs[0].(*dst.TypeSpec)
return is
}
// It fixes the "generic" fields. It happens when a value in cue could be different structs.
// For Go it generates a struct with a json.RawMessage field inside and multiple functions to map it between the different possibilities.
func fixRawData() dstutil.ApplyFunc {
return func(c *dstutil.Cursor) bool {
f, is := c.Node().(*dst.File)
if !is {
return false
}
rawFields := make(map[string]bool)
existingRawFields := make(map[string]bool)
for _, decl := range f.Decls {
switch x := decl.(type) {
// Find the structs that only contains one json.RawMessage inside
case *dst.GenDecl:
for _, t := range x.Specs {
if ts, ok := t.(*dst.TypeSpec); ok {
if tp, ok := ts.Type.(*dst.StructType); ok && len(tp.Fields.List) == 1 {
if fn, ok := tp.Fields.List[0].Type.(*dst.SelectorExpr); ok {
if fmt.Sprintf("%s.%s", fn.X, fn.Sel.Name) == "json.RawMessage" {
rawFields[ts.Name.Name] = true
}
}
}
}
}
// Find the functions of the previous structs to verify that are the ones that we are looking for.
case *dst.FuncDecl:
for _, recv := range x.Recv.List {
fnType := depoint(recv.Type).(*dst.Ident).Name
if rawFields[fnType] {
existingRawFields[fnType] = true
}
}
}
}
dstutil.Apply(f, func(c *dstutil.Cursor) bool {
switch x := c.Node().(type) {
// Delete the functions
case *dst.FuncDecl:
c.Delete()
case *dst.GenDecl:
// Deletes all "generics" generated for these json.RawMessage structs
comments := x.Decorations().Start.All()
if len(comments) > 0 {
if strings.HasSuffix(comments[0], "defines model for .") {
c.Delete()
}
}
for _, spec := range x.Specs {
if tp, ok := spec.(*dst.TypeSpec); ok {
// Delete structs with only json.RawMessage
if existingRawFields[tp.Name.Name] && tp.Name.Name != "MetricAggregation2" {
c.Delete()
continue
}
// Set types that was using these structs as interface{}
if st, ok := tp.Type.(*dst.StructType); ok {
iterateStruct(st, withoutRawData(existingRawFields))
}
if mt, ok := tp.Type.(*dst.MapType); ok {
iterateMap(mt, withoutRawData(existingRawFields))
}
if at, ok := tp.Type.(*dst.ArrayType); ok {
iterateArray(at, withoutRawData(existingRawFields))
}
}
}
}
return true
}, nil)
return true
}
}
// Fixes type name containing underscores in the generated Go files
func fixUnderscoreInTypeName() dstutil.ApplyFunc {
return func(c *dstutil.Cursor) bool {
switch x := c.Node().(type) {
case *dst.GenDecl:
if specs, isType := x.Specs[0].(*dst.TypeSpec); isType {
if strings.Contains(specs.Name.Name, "_") {
oldName := specs.Name.Name
specs.Name.Name = strings.ReplaceAll(specs.Name.Name, "_", "")
x.Decs.Start[0] = strings.ReplaceAll(x.Decs.Start[0], oldName, specs.Name.Name)
}
if st, ok := specs.Type.(*dst.StructType); ok {
iterateStruct(st, withoutUnderscore)
}
if mt, ok := specs.Type.(*dst.MapType); ok {
iterateMap(mt, withoutUnderscore)
}
if at, ok := specs.Type.(*dst.ArrayType); ok {
iterateArray(at, withoutUnderscore)
}
}
case *dst.Field:
findFieldsWithUnderscores(x)
}
return true
}
}
func findFieldsWithUnderscores(x *dst.Field) {
switch t := x.Type.(type) {
case *dst.Ident:
withoutUnderscore(t)
case *dst.StarExpr:
if i, is := t.X.(*dst.Ident); is {
withoutUnderscore(i)
}
case *dst.ArrayType:
iterateArray(t, withoutUnderscore)
case *dst.MapType:
iterateMap(t, withoutUnderscore)
}
}
func withoutUnderscore(i *dst.Ident) {
if strings.Contains(i.Name, "_") {
i.Name = strings.ReplaceAll(i.Name, "_", "")
}
}
func withoutRawData(existingFields map[string]bool) func(ident *dst.Ident) {
return func(i *dst.Ident) {
if existingFields[i.Name] {
i.Name = setStar(i) + "any"
}
}
}
func iterateStruct(s *dst.StructType, fn func(i *dst.Ident)) {
for i, f := range s.Fields.List {
switch mx := depoint(f.Type).(type) {
case *dst.Ident:
fn(mx)
case *dst.ArrayType:
iterateArray(mx, fn)
case *dst.MapType:
iterateMap(mx, fn)
case *dst.StructType:
iterateStruct(mx, fn)
case *dst.InterfaceType:
s.Fields.List[i].Type = interfaceToAny(f.Type)
}
}
}
func iterateMap(s *dst.MapType, fn func(i *dst.Ident)) {
switch mx := s.Value.(type) {
case *dst.Ident:
fn(mx)
case *dst.ArrayType:
iterateArray(mx, fn)
case *dst.MapType:
iterateMap(mx, fn)
case *dst.InterfaceType:
s.Value = interfaceToAny(s.Value)
}
}
func iterateArray(a *dst.ArrayType, fn func(i *dst.Ident)) {
switch mx := a.Elt.(type) {
case *dst.Ident:
fn(mx)
case *dst.ArrayType:
iterateArray(mx, fn)
case *dst.StructType:
iterateStruct(mx, fn)
case *dst.InterfaceType:
a.Elt = interfaceToAny(a.Elt)
}
}
func interfaceToAny(i dst.Expr) dst.Expr {
star := ""
if _, is := i.(*dst.StarExpr); is {
star = "*"
}
return &dst.Ident{Name: star + "any"}
}

View File

@@ -0,0 +1,193 @@
package generators
import (
"bytes"
"fmt"
"go/parser"
"go/token"
"path/filepath"
"strings"
"cuelang.org/go/cue"
"cuelang.org/go/pkg/encoding/yaml"
"github.com/dave/dst/decorator"
"github.com/dave/dst/dstutil"
"github.com/deepmap/oapi-codegen/pkg/codegen"
"github.com/getkin/kin-openapi/openapi3"
"golang.org/x/tools/imports"
)
type GoConfig struct {
Config *OpenApiConfig
PackageName string
ApplyFuncs []dstutil.ApplyFunc
}
func GenerateTypesGo(v cue.Value, cfg *GoConfig) ([]byte, error) {
if cfg == nil {
return nil, fmt.Errorf("configuration cannot be nil")
}
applyFuncs := []dstutil.ApplyFunc{depointerizer(), fixRawData(), fixUnderscoreInTypeName(), fixTODOComments()}
applyFuncs = append(applyFuncs, cfg.ApplyFuncs...)
f, err := generateOpenAPI(v, cfg.Config)
if err != nil {
return nil, err
}
str, err := yaml.Marshal(v.Context().BuildFile(f))
if err != nil {
return nil, fmt.Errorf("cue-yaml marshaling failed: %w", err)
}
loader := openapi3.NewLoader()
oT, err := loader.LoadFromData([]byte(str))
if err != nil {
return nil, fmt.Errorf("loading generated openapi failed: %w", err)
}
schemaName, err := getSchemaName(v)
if err != nil {
return nil, err
}
if cfg.PackageName == "" {
cfg.PackageName = schemaName
}
// Hack to fix https://github.com/grafana/thema/pull/127 issue without importing
// to avoid to add the whole vendor in Grafana code
if cfg.PackageName == "dataquery" {
fixDataQuery(oT)
}
ccfg := codegen.Configuration{
PackageName: cfg.PackageName,
Compatibility: codegen.CompatibilityOptions{
AlwaysPrefixEnumValues: true,
},
Generate: codegen.GenerateOptions{
Models: true,
},
OutputOptions: codegen.OutputOptions{
SkipPrune: true,
UserTemplates: map[string]string{
"imports.tmpl": importstmpl,
},
},
}
gostr, err := codegen.Generate(oT, ccfg)
if err != nil {
return nil, fmt.Errorf("openapi generation failed: %w", err)
}
return postprocessGoFile(genGoFile{
path: fmt.Sprintf("%s_type_gen.go", schemaName),
appliers: applyFuncs,
in: []byte(gostr),
})
}
type genGoFile struct {
path string
appliers []dstutil.ApplyFunc
in []byte
}
func postprocessGoFile(cfg genGoFile) ([]byte, error) {
fname := sanitizeLabelString(filepath.Base(cfg.path))
buf := new(bytes.Buffer)
fset := token.NewFileSet()
gf, err := decorator.ParseFile(fset, fname, string(cfg.in), parser.ParseComments)
if err != nil {
return nil, fmt.Errorf("error parsing generated file: %w", err)
}
for _, af := range cfg.appliers {
dstutil.Apply(gf, af, nil)
}
err = decorator.Fprint(buf, gf)
if err != nil {
return nil, fmt.Errorf("error formatting generated file: %w", err)
}
byt, err := imports.Process(fname, buf.Bytes(), nil)
if err != nil {
return nil, fmt.Errorf("goimports processing of generated file failed: %w", err)
}
// Compare imports before and after; warn about performance if some were added
gfa, _ := parser.ParseFile(fset, fname, string(byt), parser.ParseComments)
imap := make(map[string]bool)
for _, im := range gf.Imports {
imap[im.Path.Value] = true
}
var added []string
for _, im := range gfa.Imports {
if !imap[im.Path.Value] {
added = append(added, im.Path.Value)
}
}
if len(added) != 0 {
// TODO improve the guidance in this error if/when we better abstract over imports to generate
return nil, fmt.Errorf("goimports added the following import statements to %s: \n\t%s\nRelying on goimports to find imports significantly slows down code generation. Either add these imports with an AST manipulation in cfg.ApplyFuncs, or set cfg.IgnoreDiscoveredImports to true", cfg.path, strings.Join(added, "\n\t"))
}
return byt, nil
}
// fixDataQuery extends the properties for the AllOf schemas when a DataQuery exists.
// deep/oapi-codegen library ignores the properties of the models and only ones have references.
// It doesn't apply this change https://github.com/grafana/thema/pull/154 since it modifies the
// vendor implementation, and we don't import it.
func fixDataQuery(spec *openapi3.T) *openapi3.T {
for _, sch := range spec.Components.Schemas {
if sch.Value != nil && len(sch.Value.AllOf) > 0 {
for _, allOf := range sch.Value.AllOf {
for n, p := range allOf.Value.Properties {
sch.Value.Properties[n] = p
}
}
sch.Value.AllOf = nil
}
}
return spec
}
// Almost all of the below imports are eliminated by dst transformers and calls
// to goimports - but if they're not present in the template, then the internal
// call to goimports that oapi-codegen makes will trigger a search for them,
// which can slow down codegen by orders of magnitude.
var importstmpl = `package {{ .PackageName }}
import (
"bytes"
"compress/gzip"
"context"
"encoding/base64"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"gopkg.in/yaml.v2"
"io"
"io/ioutil"
"os"
"net/http"
"net/url"
"path"
"strings"
"time"
"github.com/deepmap/oapi-codegen/pkg/runtime"
openapi_types "github.com/deepmap/oapi-codegen/pkg/types"
"github.com/getkin/kin-openapi/openapi3"
"github.com/go-chi/chi/v5"
"github.com/labstack/echo/v4"
"github.com/gin-gonic/gin"
"github.com/gorilla/mux"
)
`

View File

@@ -0,0 +1,199 @@
package generators
import (
"fmt"
"strings"
"cuelang.org/go/cue"
"cuelang.org/go/cue/ast"
"cuelang.org/go/encoding/openapi"
)
type OpenApiConfig struct {
Config *openapi.Config
IsGroup bool
RootName string
SubPath cue.Path
}
func generateOpenAPI(v cue.Value, cfg *OpenApiConfig) (*ast.File, error) {
if cfg == nil {
return nil, fmt.Errorf("missing openapi configuration")
}
if cfg.Config == nil {
cfg.Config = &openapi.Config{}
}
name, err := getSchemaName(v)
if err != nil {
return nil, err
}
gen := &oapiGen{
cfg: cfg,
name: name,
val: v.LookupPath(cue.ParsePath("lineage.schemas[0].schema")),
subpath: cfg.SubPath,
bpath: v.LookupPath(cue.ParsePath("lineage.schemas[0]")).Path(),
}
declFunc := genSchema
if cfg.IsGroup {
declFunc = genGroup
}
decls, err := declFunc(gen)
if err != nil {
return nil, err
}
// TODO recursively sort output to improve stability of output
return &ast.File{
Decls: []ast.Decl{
ast.NewStruct(
"openapi", ast.NewString("3.0.0"),
"paths", ast.NewStruct(),
"components", ast.NewStruct(
"schemas", &ast.StructLit{Elts: decls},
),
),
},
}, nil
}
type oapiGen struct {
cfg *OpenApiConfig
val cue.Value
subpath cue.Path
// overall name for the generated oapi doc
name string
// original NameFunc
onf func(cue.Value, cue.Path) string
// full prefix path that leads up to the #SchemaDef, e.g. lin._sortedSchemas[0]
bpath cue.Path
}
func genGroup(gen *oapiGen) ([]ast.Decl, error) {
ctx := gen.val.Context()
iter, err := gen.val.Fields(cue.Definitions(true), cue.Optional(true))
if err != nil {
panic(fmt.Errorf("unreachable - should always be able to get iter for struct kinds: %w", err))
}
var decls []ast.Decl
for iter.Next() {
val, sel := iter.Value(), iter.Selector()
name := strings.Trim(sel.String(), "?#")
v := ctx.CompileString(fmt.Sprintf("#%s: _", name))
defpath := cue.MakePath(cue.Def(name))
defsch := v.FillPath(defpath, val)
cfgi := *gen.cfg.Config
cfgi.NameFunc = func(val cue.Value, path cue.Path) string {
return gen.nfSingle(val, path, defpath, name)
}
part, err := openapi.Generate(defsch, &cfgi)
if err != nil {
return nil, fmt.Errorf("failed generation for grouped field %s: %w", sel, err)
}
decls = append(decls, getSchemas(part)...)
}
return decls, nil
}
func genSchema(gen *oapiGen) ([]ast.Decl, error) {
hasSubpath := len(gen.cfg.SubPath.Selectors()) > 0
name := sanitizeLabelString(gen.name)
if gen.cfg.RootName != "" {
name = gen.cfg.RootName
} else if hasSubpath {
sel := gen.cfg.SubPath.Selectors()
name = sel[len(sel)-1].String()
}
val := gen.val
if hasSubpath {
for i, sel := range gen.cfg.SubPath.Selectors() {
if !gen.val.Allows(sel) {
return nil, fmt.Errorf("subpath %q not present in schema", cue.MakePath(gen.cfg.SubPath.Selectors()[:i+1]...))
}
}
val = val.LookupPath(gen.cfg.SubPath)
}
v := gen.val.Context().CompileString(fmt.Sprintf("#%s: _", name))
defpath := cue.MakePath(cue.Def(name))
defsch := v.FillPath(defpath, val)
gen.cfg.Config.NameFunc = func(val cue.Value, path cue.Path) string {
return gen.nfSingle(val, path, defpath, name)
}
f, err := openapi.Generate(defsch.Eval(), gen.cfg.Config)
if err != nil {
return nil, err
}
return getSchemas(f), nil
}
// For generating a single, our NameFunc must:
// - Eliminate any path prefixes on the element, both internal lineage and wrapping
// - Replace the name "_#schema" with the desired name
// - Call the user-provided NameFunc, if any
// - Remove CUE markers like #, !, ?
func (gen *oapiGen) nfSingle(val cue.Value, path, defpath cue.Path, name string) string {
tpath := trimPathPrefix(trimThemaPathPrefix(path, gen.bpath), defpath)
if path.String() == "" || tpath.String() == defpath.String() {
return name
}
if val == gen.val {
return ""
}
if gen.onf != nil {
return gen.onf(val, tpath)
}
return strings.Trim(tpath.String(), "?#")
}
func getSchemas(f *ast.File) []ast.Decl {
compos := orp(getFieldByLabel(f, "components"))
schemas := orp(getFieldByLabel(compos.Value, "schemas"))
return schemas.Value.(*ast.StructLit).Elts
}
func orp[T any](t T, err error) T {
if err != nil {
panic(err)
}
return t
}
func trimThemaPathPrefix(p, base cue.Path) cue.Path {
if !pathHasPrefix(p, base) {
return p
}
rest := p.Selectors()[len(base.Selectors()):]
if len(rest) == 0 {
return cue.Path{}
}
switch rest[0].String() {
case "schema", "_#schema", "_join", "joinSchema":
return cue.MakePath(rest[1:]...)
default:
return cue.MakePath(rest...)
}
}

View File

@@ -0,0 +1,56 @@
package generators
import (
"fmt"
"cuelang.org/go/cue"
"cuelang.org/go/cue/cuecontext"
"github.com/grafana/cuetsy"
"github.com/grafana/cuetsy/ts"
"github.com/grafana/cuetsy/ts/ast"
)
type TSConfig struct {
CuetsyConfig *cuetsy.Config
IsGroup bool
RootName string
}
// GenerateTypesTS generates native TypeScript types and defaults corresponding to
// the provided Schema.
func GenerateTypesTS(v cue.Value, cfg *TSConfig) (*ast.File, error) {
if cfg == nil {
return nil, fmt.Errorf("configuration cannot be empty")
}
if cfg.CuetsyConfig == nil {
cfg.CuetsyConfig = &cuetsy.Config{
Export: true,
}
}
// Thema #schema was a unification between the schema and cue.TopKind(_). For any reason, without this unification,
// cuetsy fails finding some enums ¯\_(ツ)_/¯.
oldSchema := cuecontext.New().CompileBytes([]byte("_"))
schdef := v.LookupPath(cue.ParsePath("lineage.schemas[0].schema")).Unify(oldSchema)
tf, err := cuetsy.GenerateAST(schdef, *cfg.CuetsyConfig)
if err != nil {
return nil, fmt.Errorf("generating TS for child elements of schema failed: %w", err)
}
file := &ts.File{
Nodes: tf.Nodes,
}
if !cfg.IsGroup {
top, err := cuetsy.GenerateSingleAST(cfg.RootName, schdef, cuetsy.TypeInterface)
if err != nil {
return nil, fmt.Errorf("generating TS for schema root failed: %w", err)
}
file.Nodes = append(file.Nodes, top.T)
if top.D != nil {
file.Nodes = append(file.Nodes, top.D)
}
}
return file, nil
}

View File

@@ -0,0 +1,130 @@
package generators
import (
"fmt"
"strconv"
"strings"
"cuelang.org/go/cue"
"cuelang.org/go/cue/ast"
"cuelang.org/go/cue/token"
)
// sanitizeLabelString strips characters from a string that are not allowed for
// use in a CUE label.
func sanitizeLabelString(s string) string {
return strings.Map(func(r rune) rune {
switch {
case r >= 'a' && r <= 'z':
fallthrough
case r >= 'A' && r <= 'Z':
fallthrough
case r >= '0' && r <= '9':
fallthrough
case r == '_':
return r
default:
return -1
}
}, s)
}
// trimPathPrefix strips the provided prefix from the provided path, if the
// prefix exists.
//
// If path and prefix are equivalent, and there is at least one additional
// selector in the provided path.
func trimPathPrefix(path, prefix cue.Path) cue.Path {
sels, psels := path.Selectors(), prefix.Selectors()
if len(sels) == 1 {
return path
}
var i int
for ; i < len(psels) && i < len(sels); i++ {
if !selEq(psels[i], sels[i]) {
break
}
}
return cue.MakePath(sels[i:]...)
}
// selEq indicates whether two selectors are equivalent. Selectors are equivalent if
// they are either exactly equal, or if they are equal ignoring path optionality.
func selEq(s1, s2 cue.Selector) bool {
return s1 == s2 || s1.Optional() == s2.Optional()
}
// getFieldByLabel returns the ast.Field with a given label from a struct-ish input.
func getFieldByLabel(n ast.Node, label string) (*ast.Field, error) {
var d []ast.Decl
switch x := n.(type) {
case *ast.File:
d = x.Decls
case *ast.StructLit:
d = x.Elts
default:
return nil, fmt.Errorf("not an *ast.File or *ast.StructLit")
}
for _, el := range d {
if isFieldWithLabel(el, label) {
return el.(*ast.Field), nil
}
}
return nil, fmt.Errorf("no field with label %q", label)
}
func isFieldWithLabel(n ast.Node, label string) bool {
if x, is := n.(*ast.Field); is {
if l, is := x.Label.(*ast.BasicLit); is {
return strEq(l, label)
}
if l, is := x.Label.(*ast.Ident); is {
return identStrEq(l, label)
}
}
return false
}
func strEq(lit *ast.BasicLit, str string) bool {
if lit.Kind != token.STRING {
return false
}
ls, _ := strconv.Unquote(lit.Value)
return str == ls || str == lit.Value
}
func identStrEq(id *ast.Ident, str string) bool {
if str == id.Name {
return true
}
ls, _ := strconv.Unquote(id.Name)
return str == ls
}
// pathHasPrefix tests whether the [cue.Path] p begins with prefix.
func pathHasPrefix(p, prefix cue.Path) bool {
ps, pres := p.Selectors(), prefix.Selectors()
if len(pres) > len(ps) {
return false
}
return pathsAreEq(ps[:len(pres)], pres)
}
func pathsAreEq(p1s, p2s []cue.Selector) bool {
if len(p1s) != len(p2s) {
return false
}
for i := 0; i < len(p2s); i++ {
if !selEq(p2s[i], p1s[i]) {
return false
}
}
return true
}
func getSchemaName(v cue.Value) (string, error) {
nameValue := v.LookupPath(cue.ParsePath("name"))
return nameValue.String()
}

View File

@@ -7,7 +7,6 @@ import (
"path/filepath"
"strings"
"cuelang.org/go/cue"
"github.com/grafana/codejen"
)
@@ -21,10 +20,10 @@ func (jenny *CoreRegistryJenny) JennyName() string {
return "CoreRegistryJenny"
}
func (jenny *CoreRegistryJenny) Generate(cueFiles []CueSchema) (codejen.Files, error) {
func (jenny *CoreRegistryJenny) Generate(cueFiles ...SchemaForGen) (codejen.Files, error) {
schemas := make([]Schema, len(cueFiles))
for i, v := range cueFiles {
name, err := getSchemaName(v.CueFile)
name, err := getSchemaName(v.Name)
if err != nil {
return nil, err
}
@@ -51,12 +50,7 @@ func (jenny *CoreRegistryJenny) Generate(cueFiles []CueSchema) (codejen.Files, e
return codejen.Files{*file}, nil
}
func getSchemaName(v cue.Value) (string, error) {
name, err := getPackageName(v)
if err != nil {
return "", err
}
name = strings.Replace(name, "-", "_", -1)
return strings.ToLower(name), nil
func getSchemaName(pkg string) (string, error) {
pkg = strings.Replace(pkg, "-", "_", -1)
return strings.ToLower(pkg), nil
}

View File

@@ -7,7 +7,6 @@ import (
"github.com/grafana/codejen"
"github.com/grafana/cuetsy/ts"
"github.com/grafana/cuetsy/ts/ast"
"github.com/grafana/kindsys"
)
// LatestMajorsOrXJenny returns a jenny that repeats the input for the latest in each major version.
@@ -27,28 +26,17 @@ func (j *lmox) JennyName() string {
return "LatestMajorsOrXJenny"
}
func (j *lmox) Generate(kind kindsys.Kind) (codejen.Files, error) {
// TODO remove this once codejen catches nils https://github.com/grafana/codejen/issues/5
if kind == nil {
return nil, nil
}
comm := kind.Props().Common()
sfg := SchemaForGen{
Name: comm.Name,
IsGroup: true,
Schema: kind.Lineage().Latest(),
}
func (j *lmox) Generate(sfg SchemaForGen) (codejen.Files, error) {
sfg.IsGroup = true
f, err := j.inner.Generate(sfg)
if err != nil {
return nil, fmt.Errorf("%s jenny failed on %s schema for %s: %w", j.inner.JennyName(), sfg.Schema.Version(), kind.Props().Common().Name, err)
return nil, fmt.Errorf("%s jenny failed for %s: %w", j.inner.JennyName(), sfg.Name, err)
}
if f == nil || !f.Exists() {
return nil, nil
}
f.RelativePath = filepath.Join(j.parentdir, comm.MachineName, "x", f.RelativePath)
f.RelativePath = filepath.Join(j.parentdir, sfg.OutputName, "x", f.RelativePath)
f.From = append(f.From, j)
return codejen.Files{*f}, nil
}

View File

@@ -2,13 +2,12 @@ package codegen
import (
"fmt"
"strings"
"cuelang.org/go/cue"
"github.com/dave/dst/dstutil"
"github.com/grafana/codejen"
"github.com/grafana/kindsys"
"github.com/grafana/thema/encoding/gocode"
"github.com/grafana/thema/encoding/openapi"
"github.com/grafana/grafana/pkg/codegen/generators"
)
type GoSpecJenny struct {
@@ -19,19 +18,19 @@ func (jenny *GoSpecJenny) JennyName() string {
return "GoResourceTypes"
}
func (jenny *GoSpecJenny) Generate(kinds ...kindsys.Kind) (codejen.Files, error) {
files := make(codejen.Files, len(kinds))
for i, v := range kinds {
name := v.Lineage().Name()
b, err := gocode.GenerateTypesOpenAPI(v.Lineage().Latest(),
&gocode.TypeConfigOpenAPI{
Config: &openapi.Config{
Group: false,
func (jenny *GoSpecJenny) Generate(sfg ...SchemaForGen) (codejen.Files, error) {
files := make(codejen.Files, len(sfg))
for i, v := range sfg {
packageName := strings.ToLower(v.Name)
b, err := generators.GenerateTypesGo(v.CueFile,
&generators.GoConfig{
Config: &generators.OpenApiConfig{
IsGroup: false,
RootName: "Spec",
Subpath: cue.MakePath(cue.Str("spec")),
SubPath: cue.MakePath(cue.Str("spec")),
},
PackageName: name,
ApplyFuncs: append(jenny.ApplyFuncs, PrefixDropper(v.Props().Common().Name)),
PackageName: packageName,
ApplyFuncs: append(jenny.ApplyFuncs, PrefixDropper(v.Name)),
},
)
@@ -39,7 +38,7 @@ func (jenny *GoSpecJenny) Generate(kinds ...kindsys.Kind) (codejen.Files, error)
return nil, err
}
files[i] = *codejen.NewFile(fmt.Sprintf("pkg/kinds/%s/%s_spec_gen.go", name, name), b, jenny)
files[i] = *codejen.NewFile(fmt.Sprintf("pkg/kinds/%s/%s_spec_gen.go", packageName, packageName), b, jenny)
}
return files, nil

View File

@@ -18,25 +18,20 @@ func (jenny *K8ResourcesJenny) JennyName() string {
return "K8ResourcesJenny"
}
func (jenny *K8ResourcesJenny) Generate(cueFiles []CueSchema) (codejen.Files, error) {
func (jenny *K8ResourcesJenny) Generate(cueFiles ...SchemaForGen) (codejen.Files, error) {
files := make(codejen.Files, 0)
for _, val := range cueFiles {
pkg, err := getPackageName(val.CueFile)
resource, err := jenny.genResource(val.Name, val.CueFile)
if err != nil {
return nil, err
}
resource, err := jenny.genResource(pkg, val.CueFile)
metadata, err := jenny.genMetadata(val.Name)
if err != nil {
return nil, err
}
metadata, err := jenny.genMetadata(pkg)
if err != nil {
return nil, err
}
status, err := jenny.genStatus(pkg)
status, err := jenny.genStatus(val.Name)
if err != nil {
return nil, err
}
@@ -100,15 +95,6 @@ func (jenny *K8ResourcesJenny) genStatus(pkg string) (codejen.File, error) {
return *codejen.NewFile(fmt.Sprintf("pkg/kinds/%s/%s_status_gen.go", pkg, pkg), buf.Bytes(), jenny), nil
}
func getPackageName(val cue.Value) (string, error) {
name := val.LookupPath(cue.ParsePath("name"))
pkg, err := name.String()
if err != nil {
return "", fmt.Errorf("file doesn't have name field set: %s", err)
}
return pkg, nil
}
func getVersion(val cue.Value) (string, error) {
val = val.LookupPath(cue.ParsePath("lineage.schemas[0].version"))
versionValues, err := val.List()

View File

@@ -4,17 +4,12 @@ import (
"github.com/grafana/codejen"
"github.com/grafana/cuetsy"
"github.com/grafana/cuetsy/ts/ast"
"github.com/grafana/grafana/pkg/cuectx"
"github.com/grafana/thema/encoding/typescript"
"github.com/grafana/grafana/pkg/codegen/generators"
)
type ApplyFunc func(sfg SchemaForGen, file *ast.File)
// TSTypesJenny is a [OneToOne] that produces TypeScript types and
// defaults for a Thema schema.
//
// Thema's generic TS jenny will be able to replace this one once
// https://github.com/grafana/thema/issues/89 is complete.
// TSTypesJenny is a [OneToOne] that produces TypeScript types and defaults.
type TSTypesJenny struct {
ApplyFuncs []ApplyFunc
}
@@ -26,14 +21,13 @@ func (j TSTypesJenny) JennyName() string {
}
func (j TSTypesJenny) Generate(sfg SchemaForGen) (*codejen.File, error) {
// TODO allow using name instead of machine name in thema generator
f, err := typescript.GenerateTypes(sfg.Schema, &typescript.TypeConfig{
f, err := generators.GenerateTypesTS(sfg.CueFile, &generators.TSConfig{
CuetsyConfig: &cuetsy.Config{
Export: true,
ImportMapper: cuectx.MapCUEImportToTS,
ImportMapper: MapCUEImportToTS,
},
RootName: sfg.Name,
Group: sfg.IsGroup,
IsGroup: sfg.IsGroup,
})
for _, renameFunc := range j.ApplyFuncs {
@@ -44,5 +38,10 @@ func (j TSTypesJenny) Generate(sfg SchemaForGen) (*codejen.File, error) {
return nil, err
}
return codejen.NewFile(sfg.Schema.Lineage().Name()+"_types.gen.ts", []byte(f.String()), j), nil
outputName := sfg.Name
if sfg.OutputName != "" {
outputName = sfg.OutputName
}
return codejen.NewFile(outputName+"_types.gen.ts", []byte(f.String()), j), nil
}

View File

@@ -12,14 +12,9 @@ import (
"github.com/grafana/cuetsy"
"github.com/grafana/cuetsy/ts"
"github.com/grafana/cuetsy/ts/ast"
"github.com/grafana/grafana/pkg/cuectx"
"github.com/grafana/kindsys"
"github.com/grafana/thema"
"github.com/grafana/thema/encoding/typescript"
"github.com/grafana/grafana/pkg/codegen/generators"
)
var schPath = cue.MakePath(cue.Hid("_#schema", "github.com/grafana/thema"))
// TSVeneerIndexJenny generates an index.gen.ts file with references to all
// generated TS types. Elements with the attribute @grafana(TSVeneer="type") are
// exported from a handwritten file, rather than the raw generated types.
@@ -43,19 +38,18 @@ func (gen *genTSVeneerIndex) JennyName() string {
return "TSVeneerIndexJenny"
}
func (gen *genTSVeneerIndex) Generate(kinds ...kindsys.Kind) (*codejen.File, error) {
func (gen *genTSVeneerIndex) Generate(sfg ...SchemaForGen) (*codejen.File, error) {
tsf := new(ast.File)
for _, def := range kinds {
sch := def.Lineage().Latest()
f, err := typescript.GenerateTypes(sch, &typescript.TypeConfig{
for _, def := range sfg {
f, err := generators.GenerateTypesTS(def.CueFile, &generators.TSConfig{
CuetsyConfig: &cuetsy.Config{
ImportMapper: cuectx.MapCUEImportToTS,
ImportMapper: MapCUEImportToTS,
},
RootName: def.Props().Common().Name,
Group: def.Props().Common().LineageIsGroup,
RootName: def.Name,
IsGroup: def.IsGroup,
})
if err != nil {
return nil, fmt.Errorf("%s: %w", def.Props().Common().Name, err)
return nil, fmt.Errorf("%s: %w", def.Name, err)
}
// The obvious approach would be calling renameSpecNode() here, same as in the ts resource jenny,
// to rename the "spec" field to the name of the kind. But that was causing extra
@@ -66,7 +60,7 @@ func (gen *genTSVeneerIndex) Generate(kinds ...kindsys.Kind) (*codejen.File, err
elems, err := gen.extractTSIndexVeneerElements(def, f)
if err != nil {
return nil, fmt.Errorf("%s: %w", def.Props().Common().Name, err)
return nil, fmt.Errorf("%s: %w", def.Name, err)
}
tsf.Nodes = append(tsf.Nodes, elems...)
}
@@ -74,12 +68,9 @@ func (gen *genTSVeneerIndex) Generate(kinds ...kindsys.Kind) (*codejen.File, err
return codejen.NewFile(filepath.Join(gen.dir, "index.gen.ts"), []byte(tsf.String()), gen), nil
}
func (gen *genTSVeneerIndex) extractTSIndexVeneerElements(def kindsys.Kind, tf *ast.File) ([]ast.Decl, error) {
lin := def.Lineage()
comm := def.Props().Common()
func (gen *genTSVeneerIndex) extractTSIndexVeneerElements(def SchemaForGen, tf *ast.File) ([]ast.Decl, error) {
// Check the root, then walk the tree
rootv := lin.Latest().Underlying().LookupPath(schPath)
rootv := def.CueFile.LookupPath(cue.ParsePath("lineage.schemas[0].schema"))
var raw, custom, rawD, customD ast.Idents
@@ -105,7 +96,7 @@ func (gen *genTSVeneerIndex) extractTSIndexVeneerElements(def kindsys.Kind, tf *
}
// Search the generated TS AST for the type and default def nodes
pair := findDeclNode(name, comm.Name, tf)
pair := findDeclNode(name, def.Name, tf)
if pair.T == nil {
// No generated type for this item, skip it
return false
@@ -152,29 +143,32 @@ func (gen *genTSVeneerIndex) extractTSIndexVeneerElements(def kindsys.Kind, tf *
return nil, terr
}
vpath := fmt.Sprintf("v%v", thema.Lineage.Latest(lin).Version())
if def.Props().Common().Maturity.Less(kindsys.MaturityStable) {
vpath = "x"
}
vpath := "x"
machineName := strings.ToLower(def.Name)
ret := make([]ast.Decl, 0)
if len(raw) > 0 {
ret = append(ret, ast.ExportSet{
CommentList: []ast.Comment{ts.CommentFromString(fmt.Sprintf("Raw generated types from %s kind.", comm.Name), 80, false)},
CommentList: []ast.Comment{ts.CommentFromString(fmt.Sprintf("Raw generated types from %s kind.", def.Name), 80, false)},
TypeOnly: true,
Exports: raw,
From: ast.Str{Value: fmt.Sprintf("./raw/%s/%s/%s_types.gen", comm.MachineName, vpath, comm.MachineName)},
From: ast.Str{Value: fmt.Sprintf("./raw/%s/%s/%s_types.gen", machineName, vpath, machineName)},
})
}
if len(rawD) > 0 {
ret = append(ret, ast.ExportSet{
CommentList: []ast.Comment{ts.CommentFromString(fmt.Sprintf("Raw generated enums and default consts from %s kind.", lin.Name()), 80, false)},
CommentList: []ast.Comment{ts.CommentFromString(fmt.Sprintf("Raw generated enums and default consts from %s kind.", machineName), 80, false)},
TypeOnly: false,
Exports: rawD,
From: ast.Str{Value: fmt.Sprintf("./raw/%s/%s/%s_types.gen", comm.MachineName, vpath, comm.MachineName)},
From: ast.Str{Value: fmt.Sprintf("./raw/%s/%s/%s_types.gen", machineName, vpath, machineName)},
})
}
vtfile := fmt.Sprintf("./veneer/%s.types", lin.Name())
vtfile := fmt.Sprintf("./veneer/%s.types", machineName)
version, err := getVersion(def.CueFile)
if err != nil {
return nil, err
}
customstr := fmt.Sprintf(`// The following exported declarations correspond to types in the %s@%s kind's
// schema with attribute @grafana(TSVeneer="type").
//
@@ -184,7 +178,7 @@ func (gen *genTSVeneerIndex) extractTSIndexVeneerElements(def kindsys.Kind, tf *
// and exports all the symbols in the list.
//
// TODO generate code such that tsc enforces type compatibility between raw and veneer decls`,
lin.Name(), thema.Lineage.Latest(lin).Version(), filepath.ToSlash(filepath.Join(gen.dir, vtfile)))
machineName, strings.ReplaceAll(version, "-", "."), filepath.ToSlash(filepath.Join(gen.dir, vtfile)))
customComments := []ast.Comment{{Text: customstr}}
if len(custom) > 0 {

View File

@@ -27,21 +27,6 @@ func PrefixDropper(prefix string) dstutil.ApplyFunc {
}).applyfunc
}
// PrefixReplacer returns a dstutil.ApplyFunc that removes the provided prefix
// string when it appears as a leading sequence in type names, var names, and
// comments in a generated Go file.
//
// When an exact match for prefix is found, the provided replace string
// is substituted.
func PrefixReplacer(prefix, replace string) dstutil.ApplyFunc {
return (&prefixmod{
prefix: prefix,
replace: replace,
rxpsuff: regexp.MustCompile(fmt.Sprintf(`%s([a-zA-Z_]+)`, prefix)),
rxp: regexp.MustCompile(fmt.Sprintf(`%s([\s.,;-])`, prefix)),
}).applyfunc
}
func depoint(e dst.Expr) dst.Expr {
if star, is := e.(*dst.StarExpr); is {
return star.X

85
pkg/codegen/util_ts.go Normal file
View File

@@ -0,0 +1,85 @@
package codegen
import (
"fmt"
"sort"
"strings"
"cuelang.org/go/cue/ast"
tsast "github.com/grafana/cuetsy/ts/ast"
)
// CUE import paths, mapped to corresponding TS import paths. An empty value
// indicates the import path should be dropped in the conversion to TS. Imports
// not present in the list are not allowed, and code generation will fail.
var importMap = map[string]string{
"github.com/grafana/grafana/pkg/plugins/pfs": "",
"github.com/grafana/grafana/packages/grafana-schema/src/common": "@grafana/schema",
}
func init() {
allow := PermittedCUEImports()
strsl := make([]string, 0, len(importMap))
for p := range importMap {
strsl = append(strsl, p)
}
sort.Strings(strsl)
sort.Strings(allow)
if strings.Join(strsl, "") != strings.Join(allow, "") {
panic("CUE import map is not the same as permitted CUE import list - these files must be kept in sync!")
}
}
// PermittedCUEImports returns the list of import paths that may be imported in
// Grafana kind definitions.
func PermittedCUEImports() []string {
return []string{
"github.com/grafana/grafana/pkg/plugins/pfs",
"github.com/grafana/grafana/packages/grafana-schema/src/common",
}
}
// MapCUEImportToTS maps the provided CUE import path to the corresponding
// TypeScript import path in generated code.
//
// Providing an import path that is not allowed results in an error. If a nil
// error and empty string are returned, the import path should be dropped in
// generated code.
func MapCUEImportToTS(path string) (string, error) {
i, has := importMap[path]
if !has {
return "", fmt.Errorf("import %q in models.cue is not allowed", path)
}
return i, nil
}
// ConvertImport converts a CUE import statement, represented in its AST form,
// to the corresponding TS import, if the CUE import is allowed.
//
// Some CUE imports are allowed but have no corresponding TS import - the CUE
// types from that package are expected to be inlined.
func ConvertImport(im *ast.ImportSpec) (tsast.ImportSpec, error) {
tsim := tsast.ImportSpec{}
pkg, err := MapCUEImportToTS(strings.Trim(im.Path.Value, "\""))
if err != nil || pkg == "" {
// err should be unreachable if paths has been verified already
// Empty string mapping means skip it
return tsim, err
}
tsim.From = tsast.Str{Value: pkg}
if im.Name != nil && im.Name.String() != "" {
tsim.AsName = im.Name.String()
} else {
sl := strings.Split(strings.Trim(im.Path.Value, "\""), "/")
final := sl[len(sl)-1]
if idx := strings.Index(final, ":"); idx != -1 {
tsim.AsName = final[idx:]
} else {
tsim.AsName = final
}
}
return tsim, nil
}