IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2026-08-12 06:05:02 -05:00
Code Issues Packages Projects Releases Wiki Activity

Reconcile coremodels, entities, objects under new kind framework (#56492)

Browse Source 
* Update thema to latest

* Deal with s/Library/*Runtime/

* Commit new, working results of codegen

* We like pointers now

* Always take runtime arg for NewBase()

* Sketchy handwavy pass at entity meta framework

* Little nibbles

* Update pkg/framework/coremodel/entityframework.cue

Co-authored-by: Artur Wierzbicki <wierzbicki.artur.94@gmail.com>

* Move file into new framework location

* Introduce loaders, Go code

* Complete rename to kind

* Flesh out framework, add svg/dashboard examples

* Cruft removal

* Remove generated kind go files from gitignore

* Refine maturity concept, add SlotKind

* Update embed and go deps

* Export PrefixWithGrafanaCUE

* Make the loader actually work, holy crap

* Many small tweaks to type.cue

* Add Apache 2 licensing exceptions for kinds

* Add new kinds dir, start of generator

* Roll back to earlier oapi-codegen

* Introduce new grafana-specific CUE loaders

* Introduce new tidy code generators framework

* Catch up kind framework with tinkering

* Add slices for the generators

* Add write/verify step to main generator

* Many renames

* Split up kind framework cue files

* Use kind.Decl within generated kinds

* Create kind.SomeDecl wrapper type to cache lineages

* Better names again

* Get one generated implemented, hopefully

* Copy dashboard schema into new kind.cue

* Small fixes to make the initial gen work

* Put svg kind in its new home

* Add generated Go dashboard type

* More renames and cleanups

* Add base kind registry and generator

* Stop blacklisting *_gen.go files

This is not the Go best practice, anyway. All we actually want to ignore
for enterprise is generated wire files.

* Change codegen output directories

pkg/kind -> pkg/kinds
pkg/registry/kindreg -> pkg/registry/corekind

* Rename pkg/framework/kind to pkg/kindsys

* Add core structured kind generator

* Add plural and machine names to kind spec

* Copy playlist over to kind system

* Consolidate kindsys files

* Add raw kind generator

* Update CODEOWNERS for kind framework

* Touch up comments a bit

* More docs tweaks

* Remove generated types to reduce noise for review

* Split each generator into its own file

* Rename Slot kind to Composable kind

* Add handwavy types for customkind loading

* Guard against init calls to framework loader

* First pass at doc on extending the kind system

* Improve attribute example in docs

* Fix wire imports

* Add basic TS types generator

* Fix composable kind category def

* No need for a separate file with generate directive

* Catch dashboard schema up

* Rename generator types to something saner and generic

* Make version configurable in ts/go generators

* Add CommonMeta to ease property access

* Add kindsys prop indicating whether lineage is group

* Put all kind categories back in a single file

* Finish with kindsys group props

* Refactor maturity progression per discussion

- Replace "committed" with "merged"
- All kindcats can use all maturity levels, at least for now

* Convert ts veneer index generator to modular system

* Move over to new jennywrites framework

* Strip down old coremodel generator

* Use public version of jennywrites

* Pull latest thema

* Commit generated Go types

* Add header injection postprocessor

* Move sdboyer/jennywrites to grafana/codejen

* Tweak header output

* Remove dashboard and playlist coremodels

* Fix up backend dashboards devenv test

* Fix TS import patterns to new gen filename

* Update internal imports, remove coremodel registry

* Fix compilation errors, wire generation

* Export and replace the prefix dropper

* More Go struct and field name changes

* Last name fixes, hopefully

* Fix lint errors

* Last lint error

Co-authored-by: Artur Wierzbicki <wierzbicki.artur.94@gmail.com>
This commit is contained in:
sam boyer
2022-11-10 12:36:40 -08:00
committed by GitHub
co-authored by Artur Wierzbicki
parent f92d978386
commit 07e5f8117f
68 changed files with 2703 additions and 1209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/framework/coremodel/gen.go
+1 -348
View File
@@ -7,21 +7,12 @@ package main
import (
"fmt"
"os"
"path"
"path/filepath"
"sort"
"strings"
"cuelang.org/go/cue"
"cuelang.org/go/cue/cuecontext"
"cuelang.org/go/cue/errors"
"cuelang.org/go/cue/load"
"github.com/grafana/cuetsy"
"github.com/grafana/cuetsy/ts"
"github.com/grafana/cuetsy/ts/ast"
gcgen "github.com/grafana/grafana/pkg/codegen"
"github.com/grafana/grafana/pkg/cuectx"
"github.com/grafana/thema"
)
const sep = string(filepath.Separator)
@@ -45,74 +36,11 @@ func init() {
// Generate Go and Typescript implementations for all coremodels, and populate the
// coremodel static registry.
func main() {
rt := cuectx.GrafanaThemaRuntime()
if len(os.Args) > 1 {
fmt.Fprintf(os.Stderr, "coremodel code generator does not currently accept any arguments\n, got %q", os.Args)
os.Exit(1)
}
items, err := os.ReadDir(cmroot)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read coremodels parent dir %s: %s\n", cmroot, err)
os.Exit(1)
}
var lins []*gcgen.CoremodelDeclaration
for _, item := range items {
if item.IsDir() {
lin, err := gcgen.ExtractLineage(filepath.Join(cmroot, item.Name(), "coremodel.cue"), rt)
if err != nil {
fmt.Fprintf(os.Stderr, "could not process coremodel dir %s: %s\n", filepath.Join(cmroot, item.Name()), err)
os.Exit(1)
}
lins = append(lins, lin)
}
}
sort.Slice(lins, func(i, j int) bool {
return lins[i].Lineage.Name() < lins[j].Lineage.Name()
})
// The typescript veneer index.gen.ts file, which we'll build up over time
// from the exported types.
tsvidx := new(ast.File)
wd := gcgen.NewWriteDiffer()
for _, ls := range lins {
gofiles, err := ls.GenerateGoCoremodel(filepath.Join(cmroot, ls.Lineage.Name()))
if err != nil {
fmt.Fprintf(os.Stderr, "failed to generate Go for %s: %s\n", ls.Lineage.Name(), err)
os.Exit(1)
}
wd.Merge(gofiles)
// Only generate TS for API types
if ls.IsAPIType {
tsf, err := ls.GenerateTypescriptCoremodel()
if err != nil {
fmt.Fprintf(os.Stderr, "error generating TypeScript for %s: %s\n", ls.Lineage.Name(), err)
os.Exit(1)
}
tsf.Doc = mkTSHeader(ls)
wd[filepath.FromSlash(filepath.Join(tsroot, rawTSGenPath(ls)))] = []byte(tsf.String())
decls, err := extractTSIndexVeneerElements(ls, tsf)
if err != nil {
fmt.Fprintf(os.Stderr, "error generating TypeScript veneer for %s: %s\n", ls.Lineage.Name(), errors.Details(err, nil))
os.Exit(1)
}
tsvidx.Nodes = append(tsvidx.Nodes, decls...)
}
}
tsvidx.Doc = mkTSHeader(nil)
wd[filepath.Join(tsroot, "index.gen.ts")] = []byte(tsvidx.String())
regfiles, err := gcgen.GenerateCoremodelRegistry(filepath.Join(groot, "pkg", "framework", "coremodel", "registry", "registry_gen.go"), lins)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to generate coremodel registry: %s\n", err)
os.Exit(1)
}
wd.Merge(regfiles)
// TODO generating these is here temporarily until we make a more permanent home
wdsh, err := genSharedSchemas(groot)
@@ -137,25 +65,6 @@ func main() {
}
}
// generates the path relative to packages/grafana-schema/src at which the raw
// type definitions should be exported for the latest schema of this type
func rawTSGenPath(cm *gcgen.CoremodelDeclaration) string {
return fmt.Sprintf("raw/%s/%s/%s.gen.ts", cm.Lineage.Name(), cm.PathVersion(), cm.Lineage.Name())
}
func mkTSHeader(cm *gcgen.CoremodelDeclaration) *ast.Comment {
v := gcgen.HeaderVars{
GeneratorPath: "pkg/framework/coremodel/gen.go",
}
if cm != nil {
v.LineagePath = cm.RelativePath
}
v.GeneratorPath = "pkg/framework/coremodel/gen.go"
return &ast.Comment{
Text: strings.TrimSpace(gcgen.GenGrafanaHeader(v)),
}
}
func genSharedSchemas(groot string) (gcgen.WriteDiffer, error) {
abspath := filepath.Join(groot, "packages", "grafana-schema", "src", "schema")
cfg := &load.Config{
@@ -183,7 +92,7 @@ func genSharedSchemas(groot string) (gcgen.WriteDiffer, error) {
}
wd := gcgen.NewWriteDiffer()
wd[filepath.Join(abspath, "mudball.gen.ts")] = append([]byte(`//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
wd[filepath.Join(abspath, "mudball.gen.ts")] = append([]byte(`//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~
// This file is autogenerated. DO NOT EDIT.
//
// To regenerate, run "make gen-cue" from the repository root.
@@ -191,259 +100,3 @@ func genSharedSchemas(groot string) (gcgen.WriteDiffer, error) {
`), b...)
return wd, nil
}
// TODO make this more generic and reusable
func extractTSIndexVeneerElements(cm *gcgen.CoremodelDeclaration, tf *ast.File) ([]ast.Decl, error) {
lin := cm.Lineage
sch := thema.SchemaP(lin, thema.LatestVersion(lin))
// Check the root, then walk the tree
rootv := sch.UnwrapCUE()
var raw, custom, rawD, customD ast.Idents
var terr errors.Error
visit := func(p cue.Path, wv cue.Value) bool {
var name string
sels := p.Selectors()
switch len(sels) {
case 0:
name = strings.Title(cm.Lineage.Name())
fallthrough
case 1:
// Only deal with subpaths that are definitions, for now
// TODO incorporate smarts about grouped lineages here
if name == "" {
if !sels[0].IsDefinition() {
return false
}
// It might seem to make sense that we'd strip out the leading # here for
// definitions. However, cuetsy's tsast actually has the # still present in its
// Ident types, stripping it out on the fly when stringifying.
name = sels[0].String()
}
// Search the generated TS AST for the type and default decl nodes
pair := findDeclNode(name, tf)
if pair.T == nil {
// No generated type for this item, skip it
return false
}
cust, perr := getCustomVeneerAttr(wv)
if perr != nil {
terr = errors.Append(terr, errors.Promote(perr, fmt.Sprintf("%s: ", p.String())))
}
var has bool
for _, tgt := range cust {
has = has || tgt.target == "type"
}
if has {
custom = append(custom, *pair.T)
if pair.D != nil {
customD = append(customD, *pair.D)
}
} else {
raw = append(raw, *pair.T)
if pair.D != nil {
rawD = append(rawD, *pair.D)
}
}
}
return true
}
walk(rootv, visit, nil)
if len(errors.Errors(terr)) != 0 {
return nil, terr
}
ret := make([]ast.Decl, 0)
if len(raw) > 0 {
ret = append(ret, ast.ExportSet{
CommentList: []ast.Comment{ts.CommentFromString(fmt.Sprintf("Raw generated types from %s entity type.", cm.Lineage.Name()), 80, false)},
TypeOnly: true,
Exports: raw,
From: ast.Str{Value: fmt.Sprintf("./raw/%s/%s/%s.gen", cm.Lineage.Name(), cm.PathVersion(), cm.Lineage.Name())},
})
}
if len(rawD) > 0 {
ret = append(ret, ast.ExportSet{
CommentList: []ast.Comment{ts.CommentFromString(fmt.Sprintf("Raw generated default consts from %s entity type.", cm.Lineage.Name()), 80, false)},
TypeOnly: false,
Exports: rawD,
From: ast.Str{Value: fmt.Sprintf("./raw/%s/%s/%s.gen", cm.Lineage.Name(), cm.PathVersion(), cm.Lineage.Name())},
})
}
vtfile := fmt.Sprintf("./veneer/%s.types", cm.Lineage.Name())
customstr := fmt.Sprintf(`// The following exported declarations correspond to types in the %s@%s schema with
// attribute @grafana(TSVeneer="type"). (lineage declared in file: %s)
//
// The handwritten file for these type and default veneers is expected to be at
// %s.ts.
// This re-export declaration enforces that the handwritten veneer file exists,
// and exports all the symbols in the list.
//
// TODO generate code such that tsc enforces type compatibility between raw and veneer decls`,
cm.Lineage.Name(), thema.LatestVersion(cm.Lineage), cm.RelativePath, filepath.Clean(path.Join("packages", "grafana-schema", "src", vtfile)))
customComments := []ast.Comment{{Text: customstr}}
if len(custom) > 0 {
ret = append(ret, ast.ExportSet{
CommentList: customComments,
TypeOnly: true,
Exports: custom,
From: ast.Str{Value: vtfile},
})
}
if len(customD) > 0 {
ret = append(ret, ast.ExportSet{
CommentList: customComments,
TypeOnly: false,
Exports: customD,
From: ast.Str{Value: vtfile},
})
}
// TODO emit a decl in the index.gen.ts that ensures any custom veneer types are "compatible" with current version raw types
return ret, nil
}
type declPair struct {
T, D *ast.Ident
}
func findDeclNode(name string, tf *ast.File) declPair {
var p declPair
for _, decl := range tf.Nodes {
// Peer through export keywords
if ex, is := decl.(ast.ExportKeyword); is {
decl = ex.Decl
}
switch x := decl.(type) {
case ast.TypeDecl:
if x.Name.Name == name {
p.T = &x.Name
}
case ast.VarDecl:
if x.Names.Idents[0].Name == "default"+name {
p.D = &x.Names.Idents[0]
}
}
}
return p
}
type tsVeneerAttr struct {
target string
}
func walk(v cue.Value, before func(cue.Path, cue.Value) bool, after func(cue.Path, cue.Value)) {
innerWalk(cue.MakePath(), v, before, after)
}
func innerWalk(p cue.Path, v cue.Value, before func(cue.Path, cue.Value) bool, after func(cue.Path, cue.Value)) {
// switch v.IncompleteKind() {
switch v.Kind() {
default:
if before != nil && !before(p, v) {
return
}
case cue.StructKind:
if before != nil && !before(p, v) {
return
}
iter, err := v.Fields(cue.All())
if err != nil {
panic(err)
}
for iter.Next() {
innerWalk(appendPath(p, iter.Selector()), iter.Value(), before, after)
}
if lv := v.LookupPath(cue.MakePath(cue.AnyString)); lv.Exists() {
innerWalk(appendPath(p, cue.AnyString), lv, before, after)
}
case cue.ListKind:
if before != nil && !before(p, v) {
return
}
list, err := v.List()
if err != nil {
panic(err)
}
for i := 0; list.Next(); i++ {
innerWalk(appendPath(p, cue.Index(i)), list.Value(), before, after)
}
if lv := v.LookupPath(cue.MakePath(cue.AnyIndex)); lv.Exists() {
innerWalk(appendPath(p, cue.AnyString), lv, before, after)
}
}
if after != nil {
after(p, v)
}
}
func appendPath(p cue.Path, sel cue.Selector) cue.Path {
return cue.MakePath(append(p.Selectors(), sel)...)
}
var allowedTSVeneers = map[string]bool{
"type": true,
}
func allowedTSVeneersString() string {
var list []string
for tgt := range allowedTSVeneers {
list = append(list, tgt)
}
sort.Strings(list)
return strings.Join(list, "|")
}
func getCustomVeneerAttr(v cue.Value) ([]tsVeneerAttr, error) {
var attrs []tsVeneerAttr
for _, a := range v.Attributes(cue.ValueAttr) {
if a.Name() != "grafana" {
continue
}
for i := 0; i < a.NumArgs(); i++ {
key, av := a.Arg(i)
if key != "TSVeneer" {
return nil, valError(v, "attribute 'grafana' only allows the arg 'TSVeneer'")
}
aterr := valError(v, "@grafana(TSVeneer=\"x\") requires one or more of the following separated veneer types for x: %s", allowedTSVeneersString())
var some bool
for _, tgt := range strings.Split(av, "|") {
some = true
if !allowedTSVeneers[tgt] {
return nil, aterr
}
attrs = append(attrs, tsVeneerAttr{
target: tgt,
})
}
if !some {
return nil, aterr
}
}
}
sort.Slice(attrs, func(i, j int) bool {
return attrs[i].target < attrs[j].target
})
return attrs, nil
}
func valError(v cue.Value, format string, args ...interface{}) error {
s := v.Source()
if s == nil {
return fmt.Errorf(format, args...)
}
return errors.Newf(s.Pos(), format, args...)
}
pkg/framework/coremodel/registry/assignability_test.go
-22
View File
@@ -1,22 +0,0 @@
package registry_test
import (
"testing"
"github.com/grafana/grafana/pkg/framework/coremodel/registry"
"github.com/grafana/thema"
)
func TestSchemaAssignability(t *testing.T) {
reg := registry.NewBase(nil)
for _, cm := range reg.All() {
tcm := cm
t.Run(tcm.Lineage().Name(), func(t *testing.T) {
err := thema.AssignableTo(tcm.CurrentSchema(), tcm.GoType())
if err != nil {
t.Fatal(err)
}
})
}
}
pkg/framework/coremodel/registry/provide.go
-48
View File
@@ -1,48 +0,0 @@
package registry
import (
"sync"
"github.com/google/wire"
"github.com/grafana/grafana/pkg/cuectx"
"github.com/grafana/grafana/pkg/framework/coremodel"
"github.com/grafana/thema"
)
// CoremodelSet contains all of the wire-style providers related to coremodels.
var CoremodelSet = wire.NewSet(
NewBase,
)
var (
baseOnce sync.Once
defaultBase *Base
)
// NewBase provides a registry of all coremodels, without any composition of
// plugin-defined schemas.
//
// All calling code within grafana/grafana is expected to use Grafana's
// singleton [thema.Runtime], returned from [cuectx.GrafanaThemaRuntime]. If nil
// is passed, the singleton will be used.
func NewBase(rt *thema.Runtime) *Base {
allrt := cuectx.GrafanaThemaRuntime()
if rt == nil || rt == allrt {
baseOnce.Do(func() {
defaultBase = doProvideBase(allrt)
})
return defaultBase
}
return doProvideBase(rt)
}
// All returns a slice of all registered coremodels.
//
// Prefer this method when operating generically across all coremodels.
//
// The returned slice is sorted lexicographically by coremodel name. It should
// not be modified.
func (b *Base) All() []coremodel.Interface {
return b.all
}
pkg/framework/coremodel/registry/registry_gen.go
-83
View File
@@ -1,83 +0,0 @@
// This file is autogenerated. DO NOT EDIT.
//
// Generated by pkg/framework/coremodel/gen.go
//
// Run `make gen-cue` from repository root to regenerate.
package registry
import (
"fmt"
"github.com/grafana/grafana/pkg/coremodel/dashboard"
"github.com/grafana/grafana/pkg/coremodel/playlist"
"github.com/grafana/grafana/pkg/coremodel/pluginmeta"
"github.com/grafana/grafana/pkg/framework/coremodel"
"github.com/grafana/thema"
)
// Base is a registry of coremodel.Interface. It provides two modes for accessing
// coremodels: individually via literal named methods, or as a slice returned from All().
//
// Prefer the individual named methods for use cases where the particular coremodel(s) that
// are needed are known to the caller. For example, a dashboard linter can know that it
// specifically wants the dashboard coremodel.
//
// Prefer All() when performing operations generically across all coremodels. For example,
// a validation HTTP middleware for any coremodel-schematized object type.
type Base struct {
all []coremodel.Interface
dashboard *dashboard.Coremodel
playlist *playlist.Coremodel
pluginmeta *pluginmeta.Coremodel
}
// type guards
var (
_ coremodel.Interface = &dashboard.Coremodel{}
_ coremodel.Interface = &playlist.Coremodel{}
_ coremodel.Interface = &pluginmeta.Coremodel{}
)
// Dashboard returns the dashboard coremodel. The return value is guaranteed to
// implement coremodel.Interface.
func (b *Base) Dashboard() *dashboard.Coremodel {
return b.dashboard
}
// Playlist returns the playlist coremodel. The return value is guaranteed to
// implement coremodel.Interface.
func (b *Base) Playlist() *playlist.Coremodel {
return b.playlist
}
// Pluginmeta returns the pluginmeta coremodel. The return value is guaranteed to
// implement coremodel.Interface.
func (b *Base) Pluginmeta() *pluginmeta.Coremodel {
return b.pluginmeta
}
func doProvideBase(rt *thema.Runtime) *Base {
var err error
reg := &Base{}
reg.dashboard, err = dashboard.New(rt)
if err != nil {
panic(fmt.Sprintf("error while initializing dashboard coremodel: %s", err))
}
reg.all = append(reg.all, reg.dashboard)
reg.playlist, err = playlist.New(rt)
if err != nil {
panic(fmt.Sprintf("error while initializing playlist coremodel: %s", err))
}
reg.all = append(reg.all, reg.playlist)
reg.pluginmeta, err = pluginmeta.New(rt)
if err != nil {
panic(fmt.Sprintf("error while initializing pluginmeta coremodel: %s", err))
}
reg.all = append(reg.all, reg.pluginmeta)
return reg
}
pkg/framework/coremodel/slot/doc.go
-2
View File
@@ -1,2 +0,0 @@
// Package Slot exposes Grafana's coremodel composition Slot definitions for use in Go.
package slot