IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Schema: Refactor plugin code generation (#58901)

Browse Source 
* wip

* wip

* almost there..

* wip - change so it can run.

* treelist is working.

* support CODEGEN_VERIFY env variable

* use log.fatal

* comment out old PluginTreeList code generation

* cleanup

* rename corelist package files

* fix makefile

* move pkg/codegen/pluggen.go to pkg/plugins/codegen

* copy and refactor files to pkg/plugins/codegen

* use pkg/plugins/codegen instead of pkg/codegen for core plugins code gen

* remove unneeded files

* remove unused code to resolve linting errors

* adapters first hack

* added flattener

* add back ignore build tags to go generate file

* cleaned up the code a bit.

* seems to work, needs to do some refactoring of the GoTypesJenns and TSTypesJenny.

* one more step, going to get upstream changes in this branch.

* working but need to run import tmpl in jenny_schemapath to have the proper imports.

* added header to generated files.

* added missing jenny.

* preventing plugins with multiple decls/schemas to insert multiple lines in corelist.

* fixed so we use Slot type from kindsys to detect if its group.

* adding a go jenny that only runs if the plugin has a backend.

* added version object to generated ts.

* generating the ts types with the same output as prior to this refactoring.

* removed code that is replaced by the jenny pattern.

* removed the go code that isn't used anymore.

* removed some more unused code and renamed pluggen to util_ts

* fixed linting issue.

* removed unused vars.

* use a jenny list postprocessor for header injection

* moved decl and decl_parser to pfs.

* removed the pre-pended header in the gotypes jenny since it is done in the postprocess.

* moved decl to pfs.

* removed unused template.

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>
This commit is contained in:
Marcus Andersson 2022-12-02 08:22:28 +01:00 committed by GitHub
parent 0c560b8b0d
commit 7f92f1df00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 644 additions and 646 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -69,7 +69,7 @@ gen-cue: ## Do all CUE/Thema code generation
go generate ./pkg/plugins/plugindef go generate ./pkg/plugins/plugindef
go generate ./kinds/gen.go go generate ./kinds/gen.go
go generate ./pkg/framework/coremodel go generate ./pkg/framework/coremodel
go generate ./public/app/plugins go generate ./public/app/plugins/gen.go
gen-go: $(WIRE) gen-cue gen-go: $(WIRE) gen-cue
@echo "generate go files" @echo "generate go files"

2
kinds/gen.go
View File

@ -29,7 +29,7 @@ func main() {
// Core kinds composite code generator. Produces all generated code in // Core kinds composite code generator. Produces all generated code in
// grafana/grafana that derives from raw and structured core kinds. // grafana/grafana that derives from raw and structured core kinds.
coreKindsGen := codejen.JennyListWithNamer[*codegen.DeclForGen](func(decl *codegen.DeclForGen) string { coreKindsGen := codejen.JennyListWithNamer(func(decl *codegen.DeclForGen) string {
return decl.Meta.Common().MachineName return decl.Meta.Common().MachineName
}) })

449
pkg/codegen/pluggen.go
View File

@ -1,449 +0,0 @@
package codegen
import (
"bytes"
"fmt"
"io/fs"
"path"
"path/filepath"
"sort"
"strings"
"cuelang.org/go/cue/ast"
"cuelang.org/go/pkg/encoding/yaml"
"github.com/deepmap/oapi-codegen/pkg/codegen"
"github.com/getkin/kin-openapi/openapi3"
"github.com/grafana/cuetsy"
tsast "github.com/grafana/cuetsy/ts/ast"
"github.com/grafana/grafana/pkg/kindsys"
"github.com/grafana/grafana/pkg/plugins/pfs"
"github.com/grafana/thema"
"github.com/grafana/thema/encoding/openapi"
)
// CUE import paths, mapped to corresponding TS import paths. An empty value
// indicates the import path should be dropped in the conversion to TS. Imports
// not present in the list are not not allowed, and code generation will fail.
var importMap = map[string]string{
"github.com/grafana/thema": "",
"github.com/grafana/grafana/packages/grafana-schema/src/schema": "@grafana/schema",
}
func init() {
allow := pfs.PermittedCUEImports()
strsl := make([]string, 0, len(importMap))
for p := range importMap {
strsl = append(strsl, p)
}
sort.Strings(strsl)
sort.Strings(allow)
if strings.Join(strsl, "") != strings.Join(allow, "") {
panic("CUE import map is not the same as permitted CUE import list - these files must be kept in sync!")
}
}
// MapCUEImportToTS maps the provided CUE import path to the corresponding
// TypeScript import path in generated code.
//
// Providing an import path that is not allowed results in an error. If a nil
// error and empty string are returned, the import path should be dropped in
// generated code.
func MapCUEImportToTS(path string) (string, error) {
i, has := importMap[path]
if !has {
return "", fmt.Errorf("import %q in models.cue is not allowed", path)
}
return i, nil
}
// ExtractPluginTrees attempts to create a *pfs.Tree for each of the top-level child
// directories in the provided fs.FS.
//
// Errors returned from [pfs.ParsePluginFS] are placed in the option map. Only
// filesystem traversal and read errors will result in a non-nil second return
// value.
func ExtractPluginTrees(parent fs.FS, rt *thema.Runtime) (map[string]PluginTreeOrErr, error) {
ents, err := fs.ReadDir(parent, ".")
if err != nil {
return nil, fmt.Errorf("error reading fs root directory: %w", err)
}
ptrees := make(map[string]PluginTreeOrErr)
for _, plugdir := range ents {
subpath := plugdir.Name()
sub, err := fs.Sub(parent, subpath)
if err != nil {
return nil, fmt.Errorf("error creating subfs for path %s: %w", subpath, err)
}
var either PluginTreeOrErr
if ptree, err := pfs.ParsePluginFS(sub, rt); err == nil {
either.Tree = (*PluginTree)(ptree)
} else {
either.Err = err
}
ptrees[subpath] = either
}
return ptrees, nil
}
// PluginTreeOrErr represents either a *pfs.Tree, or the error that occurred
// while trying to create one.
// TODO replace with generic option type after go 1.18
type PluginTreeOrErr struct {
Err error
Tree *PluginTree
}
// PluginTree is a pfs.Tree. It exists so we can add methods for code generation to it.
//
// It is, for now, tailored specifically to Grafana core's codegen needs.
type PluginTree pfs.Tree
func (pt *PluginTree) GenerateTypeScriptAST() (*tsast.File, error) {
t := (*pfs.Tree)(pt)
f := &tsast.File{}
tf := tvars_autogen_header{
GeneratorPath: "public/app/plugins/gen.go", // FIXME hardcoding is not OK
LineagePath: "models.cue",
}
var buf bytes.Buffer
err := tmpls.Lookup("autogen_header.tmpl").Execute(&buf, tf)
if err != nil {
return nil, fmt.Errorf("error executing header template: %w", err)
}
f.Doc = &tsast.Comment{
Text: buf.String(),
}
pi := t.RootPlugin()
slotimps := pi.SlotImplementations()
if len(slotimps) == 0 {
return nil, nil
}
for _, im := range pi.CUEImports() {
if tsim, err := convertImport(im); err != nil {
return nil, err
} else if tsim.From.Value != "" {
f.Imports = append(f.Imports, tsim)
}
}
for slotname, lin := range slotimps {
v := thema.LatestVersion(lin)
sch := thema.SchemaP(lin, v)
// Inject a node for the const with the version
f.Nodes = append(f.Nodes, tsast.Raw{
// TODO need call expressions in cuetsy tsast to be able to do these properly
Data: fmt.Sprintf("export const %sModelVersion = Object.freeze([%v, %v]);", slotname, v[0], v[1]),
})
// TODO this is hardcoded for now, but should ultimately be a property of
// whether the slot is a grouped lineage:
// https://github.com/grafana/thema/issues/62
if isGroupLineage(slotname) {
tsf, err := cuetsy.GenerateAST(sch.Underlying(), cuetsy.Config{
Export: true,
})
if err != nil {
return nil, fmt.Errorf("error translating %s lineage to TypeScript: %w", slotname, err)
}
f.Nodes = append(f.Nodes, tsf.Nodes...)
} else {
pair, err := cuetsy.GenerateSingleAST(strings.Title(lin.Name()), sch.Underlying(), cuetsy.TypeInterface)
if err != nil {
return nil, fmt.Errorf("error translating %s lineage to TypeScript: %w", slotname, err)
}
f.Nodes = append(f.Nodes, pair.T)
if pair.D != nil {
f.Nodes = append(f.Nodes, pair.D)
}
}
}
return f, nil
}
func isGroupLineage(slotname string) bool {
sl, has := kindsys.AllSlots(nil)[slotname]
if !has {
panic("unknown slotname name: " + slotname)
}
return sl.IsGroup()
}
type GoGenConfig struct {
// Types indicates whether corresponding Go types should be generated from the
// latest version in the lineage(s).
Types bool
// ThemaBindings indicates whether Thema bindings (an implementation of
// ["github.com/grafana/thema".LineageFactory]) should be generated for
// lineage(s).
ThemaBindings bool
// DocPathPrefix allows the caller to optionally specify a path to be prefixed
// onto paths generated for documentation. This is useful for io/fs-based code
// generators, which typically only have knowledge of paths relative to the fs.FS
// root, typically an encapsulated subpath, but docs are easier to understand when
// paths are relative to a repository root.
//
// Note that all paths are normalized to use slashes, regardless of the
// OS running the code generator.
DocPathPrefix string
}
func (pt *PluginTree) GenerateGo(path string, cfg GoGenConfig) (WriteDiffer, error) {
t := (*pfs.Tree)(pt)
wd := NewWriteDiffer()
all := t.SubPlugins()
if all == nil {
all = make(map[string]pfs.PluginInfo)
}
all[""] = t.RootPlugin()
for subpath, plug := range all {
fullp := filepath.Join(path, subpath)
if cfg.Types {
gwd, err := pgenGoTypes(plug, path, subpath, cfg.DocPathPrefix)
if err != nil {
return nil, fmt.Errorf("error generating go types for %s: %w", fullp, err)
}
if err = wd.Merge(gwd); err != nil {
return nil, fmt.Errorf("error merging file set to generate for %s: %w", fullp, err)
}
}
if cfg.ThemaBindings {
twd, err := pgenThemaBindings(plug, path, subpath, cfg.DocPathPrefix)
if err != nil {
return nil, fmt.Errorf("error generating thema bindings for %s: %w", fullp, err)
}
if err = wd.Merge(twd); err != nil {
return nil, fmt.Errorf("error merging file set to generate for %s: %w", fullp, err)
}
}
}
return wd, nil
}
func pgenGoTypes(plug pfs.PluginInfo, path, subpath, prefix string) (WriteDiffer, error) {
wd := NewWriteDiffer()
for slotname, lin := range plug.SlotImplementations() {
lowslot := strings.ToLower(slotname)
rt := lin.Runtime()
sch := thema.SchemaP(lin, thema.LatestVersion(lin))
// FIXME gotta hack this out of thema in order to deal with our custom imports :scream:
f, err := openapi.GenerateSchema(sch, nil)
if err != nil {
return nil, fmt.Errorf("thema openapi generation failed: %w", err)
}
str, err := yaml.Marshal(rt.Context().BuildFile(f))
if err != nil {
return nil, fmt.Errorf("cue-yaml marshaling failed: %w", err)
}
loader := openapi3.NewLoader()
oT, err := loader.LoadFromData([]byte(str))
if err != nil {
return nil, fmt.Errorf("loading generated openapi failed; %w", err)
}
buf := new(bytes.Buffer)
if err = tmpls.Lookup("autogen_header.tmpl").Execute(buf, tvars_autogen_header{
GeneratorPath: "public/app/plugins/gen.go", // FIXME hardcoding is not OK
LineagePath: filepath.ToSlash(filepath.Join(prefix, subpath, "models.cue")),
LineageCUEPath: slotname,
GenLicense: true,
}); err != nil {
return nil, fmt.Errorf("error generating file header: %w", err)
}
cgopt := codegen.Options{
GenerateTypes: true,
SkipPrune: true,
SkipFmt: true,
UserTemplates: map[string]string{
"imports.tmpl": "package {{ .PackageName }}",
"typedef.tmpl": tmplTypedef,
},
}
if isGroupLineage(slotname) {
cgopt.ExcludeSchemas = []string{lin.Name()}
}
gostr, err := codegen.Generate(oT, lin.Name(), cgopt)
if err != nil {
return nil, fmt.Errorf("openapi generation failed: %w", err)
}
fmt.Fprint(buf, gostr)
finalpath := filepath.Join(path, subpath, fmt.Sprintf("types_%s_gen.go", lowslot))
byt, err := postprocessGoFile(genGoFile{
path: finalpath,
walker: PrefixDropper(strings.Title(lin.Name())),
in: buf.Bytes(),
})
if err != nil {
return nil, err
}
wd[finalpath] = byt
}
return wd, nil
}
func pgenThemaBindings(plug pfs.PluginInfo, path, subpath, prefix string) (WriteDiffer, error) {
wd := NewWriteDiffer()
bindings := make([]tvars_plugin_lineage_binding, 0)
for slotname, lin := range plug.SlotImplementations() {
lv := thema.LatestVersion(lin)
bindings = append(bindings, tvars_plugin_lineage_binding{
SlotName: slotname,
LatestMajv: lv[0],
LatestMinv: lv[1],
})
}
buf := new(bytes.Buffer)
if err := tmpls.Lookup("plugin_lineage_file.tmpl").Execute(buf, tvars_plugin_lineage_file{
PackageName: sanitizePluginId(plug.Meta().Id),
PluginType: string(plug.Meta().Type),
PluginID: plug.Meta().Id,
SlotImpls: bindings,
HasModels: len(bindings) != 0,
Header: tvars_autogen_header{
GeneratorPath: "public/app/plugins/gen.go", // FIXME hardcoding is not OK
GenLicense: true,
LineagePath: filepath.Join(prefix, subpath),
},
}); err != nil {
return nil, fmt.Errorf("error executing plugin lineage file template: %w", err)
}
fullpath := filepath.Join(path, subpath, "pfs_gen.go")
if byt, err := postprocessGoFile(genGoFile{
path: fullpath,
in: buf.Bytes(),
}); err != nil {
return nil, err
} else {
wd[fullpath] = byt
}
return wd, nil
}
// Plugin IDs are allowed to contain characters that aren't allowed in CUE
// package names, Go package names, TS or Go type names, etc.
// TODO expose this as standard
func sanitizePluginId(s string) string {
return strings.Map(func(r rune) rune {
switch {
case r >= 'a' && r <= 'z':
fallthrough
case r >= 'A' && r <= 'Z':
fallthrough
case r >= '0' && r <= '9':
fallthrough
case r == '_':
return r
case r == '-':
return '_'
default:
return -1
}
}, s)
}
// FIXME unexport this and refactor, this is way too one-off to be in here
func GenPluginTreeList(trees []TreeAndPath, prefix, target string, ref bool) (WriteDiffer, error) {
buf := new(bytes.Buffer)
vars := tvars_plugin_registry{
Header: tvars_autogen_header{
GenLicense: true,
},
Plugins: make([]struct {
PkgName, Path, ImportPath string
NoAlias bool
}, 0, len(trees)),
}
type tpl struct {
PkgName, Path, ImportPath string
NoAlias bool
}
// No sub-plugin support here. If we never allow subplugins in core, that's probably fine.
// But still worth noting.
for _, pt := range trees {
rp := (*pfs.Tree)(pt.Tree).RootPlugin()
vars.Plugins = append(vars.Plugins, tpl{
PkgName: sanitizePluginId(rp.Meta().Id),
NoAlias: sanitizePluginId(rp.Meta().Id) != filepath.Base(pt.Path),
ImportPath: filepath.ToSlash(filepath.Join(prefix, pt.Path)),
Path: path.Join(append(strings.Split(prefix, "/")[3:], pt.Path)...),
})
}
tmplname := "plugin_registry.tmpl"
if ref {
tmplname = "plugin_registry_ref.tmpl"
}
if err := tmpls.Lookup(tmplname).Execute(buf, vars); err != nil {
return nil, fmt.Errorf("failed executing plugin registry template: %w", err)
}
byt, err := postprocessGoFile(genGoFile{
path: target,
in: buf.Bytes(),
})
if err != nil {
return nil, fmt.Errorf("error postprocessing plugin registry: %w", err)
}
wd := NewWriteDiffer()
wd[target] = byt
return wd, nil
}
// FIXME unexport this and refactor, this is way too one-off to be in here
type TreeAndPath struct {
Tree *PluginTree
// path relative to path prefix UUUGHHH (basically {panel,datasource}/<dir>}
Path string
}
// TODO convert this to use cuetsy ts types, once import * form is supported
func convertImport(im *ast.ImportSpec) (tsast.ImportSpec, error) {
tsim := tsast.ImportSpec{}
pkg, err := MapCUEImportToTS(strings.Trim(im.Path.Value, "\""))
if err != nil || pkg == "" {
// err should be unreachable if paths has been verified already
// Empty string mapping means skip it
return tsim, err
}
tsim.From = tsast.Str{Value: pkg}
if im.Name != nil && im.Name.String() != "" {
tsim.AsName = im.Name.String()
} else {
sl := strings.Split(im.Path.Value, "/")
final := sl[len(sl)-1]
if idx := strings.Index(final, ":"); idx != -1 {
tsim.AsName = final[idx:]
} else {
tsim.AsName = final
}
}
return tsim, nil
}

22
pkg/codegen/tmpl.go
View File

@ -46,28 +46,6 @@ type (
tvars_coremodel_imports struct { tvars_coremodel_imports struct {
PackageName string PackageName string
} }
tvars_plugin_lineage_binding struct {
SlotName string
LatestMajv, LatestMinv uint
}
tvars_plugin_lineage_file struct {
PackageName string
PluginID string
PluginType string
HasModels bool
RootCUE bool
SlotImpls []tvars_plugin_lineage_binding
Header tvars_autogen_header
}
tvars_plugin_registry struct {
Header tvars_autogen_header
Plugins []struct {
PkgName string
Path string
ImportPath string
NoAlias bool
}
}
) )
type HeaderVars = tvars_autogen_header type HeaderVars = tvars_autogen_header

10
pkg/kindsys/slot.go
View File

@ -1,6 +1,8 @@
package kindsys package kindsys
import ( import (
"fmt"
"cuelang.org/go/cue" "cuelang.org/go/cue"
) )
@ -62,6 +64,14 @@ func (s Slot) IsGroup() bool {
} }
} }
func FindSlot(name string) (*Slot, error) {
sl, has := AllSlots(nil)[name]
if !has {
return nil, fmt.Errorf("unsupported slot: %s", name)
}
return sl, nil
}
// AllSlots returns a map of all [Slot]s defined in the Grafana kindsys // AllSlots returns a map of all [Slot]s defined in the Grafana kindsys
// framework. // framework.
// //

46
pkg/plugins/codegen/jenny_plugingotypes.go Normal file
View File

@ -0,0 +1,46 @@
package codegen
import (
"fmt"
"path/filepath"
"strings"
"github.com/grafana/codejen"
"github.com/grafana/grafana/pkg/plugins/pfs"
)
func PluginGoTypesJenny(root string, inner codejen.OneToOne[*pfs.PluginDecl]) codejen.OneToOne[*pfs.PluginDecl] {
return &pgoJenny{
inner: inner,
root: root,
}
}
type pgoJenny struct {
inner codejen.OneToOne[*pfs.PluginDecl]
root string
}
func (j *pgoJenny) JennyName() string {
return "PluginGoTypesJenny"
}
func (j *pgoJenny) Generate(decl *pfs.PluginDecl) (*codejen.File, error) {
b := decl.PluginMeta.Backend
if b == nil || !*b || !decl.HasSchema() {
return nil, nil
}
f, err := j.inner.Generate(decl)
if err != nil {
return nil, err
}
pluginfolder := filepath.Base(decl.PluginPath)
slotname := strings.ToLower(decl.Slot.Name())
filename := fmt.Sprintf("types_%s_gen.go", slotname)
f.RelativePath = filepath.Join(j.root, pluginfolder, filename)
f.From = append(f.From, j)
return f, nil
}

100
pkg/plugins/codegen/jenny_plugintreelist.go Normal file
View File

@ -0,0 +1,100 @@
package codegen
import (
"bytes"
"fmt"
"path"
"path/filepath"
"strings"
"github.com/grafana/codejen"
"github.com/grafana/grafana/pkg/plugins/pfs"
)
const prefix = "github.com/grafana/grafana/public/app/plugins"
// PluginTreeListJenny creates a [codejen.ManyToOne] that produces Go code
// for loading a [pfs.TreeList] given [*kindsys.PluginDecl] as inputs.
func PluginTreeListJenny() codejen.ManyToOne[*pfs.PluginDecl] {
outputFile := filepath.Join("pkg", "plugins", "pfs", "corelist", "corelist_load_gen.go")
return &ptlJenny{
outputFile: outputFile,
plugins: make(map[string]bool, 0),
}
}
type ptlJenny struct {
outputFile string
plugins map[string]bool
}
func (j *ptlJenny) JennyName() string {
return "PluginTreeListJenny"
}
func (j *ptlJenny) Generate(decls ...*pfs.PluginDecl) (*codejen.File, error) {
buf := new(bytes.Buffer)
vars := templateVars_plugin_registry{
Plugins: make([]struct {
PkgName, Path, ImportPath string
NoAlias bool
}, 0, len(decls)),
}
type tpl struct {
PkgName, Path, ImportPath string
NoAlias bool
}
for _, decl := range decls {
meta := decl.PluginMeta
if _, exists := j.plugins[meta.Id]; exists {
continue
}
pluginId := j.sanitizePluginId(meta.Id)
vars.Plugins = append(vars.Plugins, tpl{
PkgName: pluginId,
NoAlias: pluginId != filepath.Base(decl.PluginPath),
ImportPath: filepath.ToSlash(filepath.Join(prefix, decl.PluginPath)),
Path: path.Join(append(strings.Split(prefix, "/")[3:], decl.PluginPath)...),
})
j.plugins[meta.Id] = true
}
if err := tmpls.Lookup("plugin_registry.tmpl").Execute(buf, vars); err != nil {
return nil, fmt.Errorf("failed executing plugin registry template: %w", err)
}
byt, err := postprocessGoFile(genGoFile{
path: j.outputFile,
in: buf.Bytes(),
})
if err != nil {
return nil, fmt.Errorf("error postprocessing plugin registry: %w", err)
}
return codejen.NewFile(j.outputFile, byt, j), nil
}
func (j *ptlJenny) sanitizePluginId(s string) string {
return strings.Map(func(r rune) rune {
switch {
case r >= 'a' && r <= 'z':
fallthrough
case r >= 'A' && r <= 'Z':
fallthrough
case r >= '0' && r <= '9':
fallthrough
case r == '_':
return r
case r == '-':
return '_'
default:
return -1
}
}, s)
}

64
pkg/plugins/codegen/jenny_plugintstypes.go Normal file
View File

@ -0,0 +1,64 @@
package codegen
import (
"fmt"
"path/filepath"
"github.com/grafana/codejen"
tsast "github.com/grafana/cuetsy/ts/ast"
"github.com/grafana/grafana/pkg/plugins/pfs"
)
func PluginTSTypesJenny(root string, inner codejen.OneToOne[*pfs.PluginDecl]) codejen.OneToOne[*pfs.PluginDecl] {
return &ptsJenny{
root: root,
inner: inner,
}
}
type ptsJenny struct {
root string
inner codejen.OneToOne[*pfs.PluginDecl]
}
func (j *ptsJenny) JennyName() string {
return "PluginTSTypesJenny"
}
func (j *ptsJenny) Generate(decl *pfs.PluginDecl) (*codejen.File, error) {
if !decl.HasSchema() {
return nil, nil
}
tsf := &tsast.File{}
for _, im := range decl.Imports {
if tsim, err := convertImport(im); err != nil {
return nil, err
} else if tsim.From.Value != "" {
tsf.Imports = append(tsf.Imports, tsim)
}
}
slotname := decl.Slot.Name()
v := decl.Lineage.Latest().Version()
tsf.Nodes = append(tsf.Nodes, tsast.Raw{
Data: fmt.Sprintf("export const %sModelVersion = Object.freeze([%v, %v]);", slotname, v[0], v[1]),
})
jf, err := j.inner.Generate(decl)
if err != nil {
return nil, err
}
tsf.Nodes = append(tsf.Nodes, tsast.Raw{
Data: string(jf.Data),
})
path := filepath.Join(j.root, decl.PluginPath, "models.gen.ts")
data := []byte(tsf.String())
data = data[:len(data)-1] // remove the additional line break added by the inner jenny
return codejen.NewFile(path, data, append(jf.From, j)...), nil
}

33
pkg/plugins/codegen/tmpl.go Normal file
View File

@ -0,0 +1,33 @@
package codegen
import (
"embed"
"text/template"
"time"
)
// All the parsed templates in the tmpl subdirectory
var tmpls *template.Template
func init() {
base := template.New("codegen").Funcs(template.FuncMap{
"now": time.Now,
})
tmpls = template.Must(base.ParseFS(tmplFS, "tmpl/*.tmpl"))
}
//go:embed tmpl/*.tmpl
var tmplFS embed.FS
// The following group of types, beginning with templateVars_*, all contain the set
// of variables expected by the corresponding named template file under tmpl/
type (
templateVars_plugin_registry struct {
Plugins []struct {
PkgName string
Path string
ImportPath string
NoAlias bool
}
}
)

30
pkg/plugins/codegen/tmpl/plugin_registry.tmpl Normal file
View File

@ -0,0 +1,30 @@
package corelist
import (
"fmt"
"io/fs"
"sync"
"github.com/grafana/grafana"
"github.com/grafana/grafana/pkg/plugins/pfs"
"github.com/grafana/thema"
)
func makeTreeOrPanic(path string, pkgname string, rt *thema.Runtime) *pfs.Tree {
sub, err := fs.Sub(grafana.CueSchemaFS, path)
if err != nil {
panic("could not create fs sub to " + path)
}
tree, err := pfs.ParsePluginFS(sub, rt)
if err != nil {
panic(fmt.Sprintf("error parsing plugin metadata for %s: %s", pkgname, err))
}
return tree
}
func coreTreeList(rt *thema.Runtime) pfs.TreeList{
return pfs.TreeList{
{{- range .Plugins }}
makeTreeOrPanic("{{ .Path }}", "{{ .PkgName }}", rt),
{{- end }}
}
}

67
pkg/plugins/codegen/util_go.go Normal file
View File

@ -0,0 +1,67 @@
package codegen
import (
"bytes"
"fmt"
"go/format"
"go/parser"
"go/token"
"os"
"path/filepath"
"strings"
"golang.org/x/tools/go/ast/astutil"
"golang.org/x/tools/imports"
)
type genGoFile struct {
path string
walker astutil.ApplyFunc
in []byte
}
func postprocessGoFile(cfg genGoFile) ([]byte, error) {
fname := filepath.Base(cfg.path)
buf := new(bytes.Buffer)
fset := token.NewFileSet()
gf, err := parser.ParseFile(fset, fname, string(cfg.in), parser.ParseComments)
if err != nil {
return nil, fmt.Errorf("error parsing generated file: %w", err)
}
if cfg.walker != nil {
astutil.Apply(gf, cfg.walker, nil)
err = format.Node(buf, fset, gf)
if err != nil {
return nil, fmt.Errorf("error formatting Go AST: %w", err)
}
} else {
buf = bytes.NewBuffer(cfg.in)
}
byt, err := imports.Process(fname, buf.Bytes(), nil)
if err != nil {
return nil, fmt.Errorf("goimports processing failed: %w", err)
}
// Compare imports before and after; warn about performance if some were added
gfa, _ := parser.ParseFile(fset, fname, string(byt), parser.ParseComments)
imap := make(map[string]bool)
for _, im := range gf.Imports {
imap[im.Path.Value] = true
}
var added []string
for _, im := range gfa.Imports {
if !imap[im.Path.Value] {
added = append(added, im.Path.Value)
}
}
if len(added) != 0 {
// TODO improve the guidance in this error if/when we better abstract over imports to generate
fmt.Fprintf(os.Stderr, "The following imports were added by goimports while generating %s: \n\t%s\nRelying on goimports to find imports significantly slows down code generation. Consider adding these to the relevant template.\n", cfg.path, strings.Join(added, "\n\t"))
}
return byt, nil
}

73
pkg/plugins/codegen/util_ts.go Normal file
View File

@ -0,0 +1,73 @@
package codegen
import (
"fmt"
"sort"
"strings"
"cuelang.org/go/cue/ast"
tsast "github.com/grafana/cuetsy/ts/ast"
"github.com/grafana/grafana/pkg/plugins/pfs"
)
// CUE import paths, mapped to corresponding TS import paths. An empty value
// indicates the import path should be dropped in the conversion to TS. Imports
// not present in the list are not not allowed, and code generation will fail.
var importMap = map[string]string{
"github.com/grafana/thema": "",
"github.com/grafana/grafana/packages/grafana-schema/src/schema": "@grafana/schema",
}
func init() {
allow := pfs.PermittedCUEImports()
strsl := make([]string, 0, len(importMap))
for p := range importMap {
strsl = append(strsl, p)
}
sort.Strings(strsl)
sort.Strings(allow)
if strings.Join(strsl, "") != strings.Join(allow, "") {
panic("CUE import map is not the same as permitted CUE import list - these files must be kept in sync!")
}
}
// mapCUEImportToTS maps the provided CUE import path to the corresponding
// TypeScript import path in generated code.
//
// Providing an import path that is not allowed results in an error. If a nil
// error and empty string are returned, the import path should be dropped in
// generated code.
func mapCUEImportToTS(path string) (string, error) {
i, has := importMap[path]
if !has {
return "", fmt.Errorf("import %q in models.cue is not allowed", path)
}
return i, nil
}
// TODO convert this to use cuetsy ts types, once import * form is supported
func convertImport(im *ast.ImportSpec) (tsast.ImportSpec, error) {
tsim := tsast.ImportSpec{}
pkg, err := mapCUEImportToTS(strings.Trim(im.Path.Value, "\""))
if err != nil || pkg == "" {
// err should be unreachable if paths has been verified already
// Empty string mapping means skip it
return tsim, err
}
tsim.From = tsast.Str{Value: pkg}
if im.Name != nil && im.Name.String() != "" {
tsim.AsName = im.Name.String()
} else {
sl := strings.Split(im.Path.Value, "/")
final := sl[len(sl)-1]
if idx := strings.Index(final, ":"); idx != -1 {
tsim.AsName = final[idx:]
} else {
tsim.AsName = final
}
}
return tsim, nil
}

0
pkg/plugins/pfs/corelist/new.go → pkg/plugins/pfs/corelist/corelist.go
View File

21
pkg/plugins/pfs/corelist/loadlist_gen.go → pkg/plugins/pfs/corelist/corelist_load_gen.go
View File

@ -1,20 +1,11 @@
// Copyright 2022 Grafana Labs // THIS FILE IS GENERATED. EDITING IS FUTILE.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Generated by:
// you may not use this file except in compliance with the License. // public/app/plugins/gen.go
// You may obtain a copy of the License at // Using jennies:
// PluginTreeListJenny
// //
// http://www.apache.org/licenses/LICENSE-2.0 // Run 'make gen-cue' from repository root to regenerate.
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This file is autogenerated. DO NOT EDIT.
//
// Run `make gen-cue` from repository root to regenerate.
package corelist package corelist

28
pkg/plugins/pfs/decl.go Normal file
View File

@ -0,0 +1,28 @@
package pfs
import (
"cuelang.org/go/cue/ast"
"github.com/grafana/grafana/pkg/kindsys"
"github.com/grafana/grafana/pkg/plugins/plugindef"
"github.com/grafana/thema"
)
type PluginDecl struct {
Slot *kindsys.Slot
Lineage thema.Lineage
Imports []*ast.ImportSpec
PluginPath string
PluginMeta plugindef.PluginDef
}
func EmptyPluginDecl(path string, meta plugindef.PluginDef) *PluginDecl {
return &PluginDecl{
PluginPath: path,
PluginMeta: meta,
Imports: make([]*ast.ImportSpec, 0),
}
}
func (decl *PluginDecl) HasSchema() bool {
return decl.Lineage != nil && decl.Slot != nil
}

77
pkg/plugins/pfs/decl_parser.go Normal file
View File

@ -0,0 +1,77 @@
package pfs
import (
"fmt"
"io/fs"
"log"
"os"
"path/filepath"
"sort"
"github.com/grafana/grafana/pkg/kindsys"
"github.com/grafana/thema"
)
type declParser struct {
rt *thema.Runtime
skip map[string]bool
}
func NewDeclParser(rt *thema.Runtime, skip map[string]bool) *declParser {
return &declParser{
rt: rt,
skip: skip,
}
}
func (psr *declParser) Parse(root fs.FS) ([]*PluginDecl, error) {
plugins, err := fs.Glob(root, "**/**/plugin.json")
if err != nil {
return nil, fmt.Errorf("error finding plugin dirs: %w", err)
}
decls := make([]*PluginDecl, 0)
for _, plugin := range plugins {
path := filepath.Dir(plugin)
base := filepath.Base(path)
if skip, ok := psr.skip[base]; ok && skip {
continue
}
dir := os.DirFS(path)
ptree, err := ParsePluginFS(dir, psr.rt)
if err != nil {
log.Println(fmt.Errorf("parsing plugin failed for %s: %s", dir, err))
continue
}
p := ptree.RootPlugin()
slots := p.SlotImplementations()
if len(slots) == 0 {
decls = append(decls, EmptyPluginDecl(path, p.Meta()))
continue
}
for slotName, lin := range slots {
slot, err := kindsys.FindSlot(slotName)
if err != nil {
log.Println(fmt.Errorf("parsing plugin failed for %s: %s", dir, err))
continue
}
decls = append(decls, &PluginDecl{
Slot: slot,
Lineage: lin,
Imports: p.CUEImports(),
PluginMeta: p.Meta(),
PluginPath: path,
})
}
}
sort.Slice(decls, func(i, j int) bool {
return decls[i].PluginPath < decls[j].PluginPath
})
return decls, nil
}

123
public/app/plugins/gen.go
View File

@ -1,18 +1,22 @@
//go:build ignore //go:build ignore
// +build ignore // +build ignore
//go:generate go run gen.go
package main package main
import ( import (
"errors" "context"
"fmt" "fmt"
"log"
"os" "os"
"path/filepath" "path/filepath"
"sort"
"strings" "strings"
"github.com/grafana/grafana/pkg/codegen" "github.com/grafana/codejen"
corecodegen "github.com/grafana/grafana/pkg/codegen"
"github.com/grafana/grafana/pkg/cuectx" "github.com/grafana/grafana/pkg/cuectx"
"github.com/grafana/grafana/pkg/plugins/codegen"
"github.com/grafana/grafana/pkg/plugins/pfs" "github.com/grafana/grafana/pkg/plugins/pfs"
) )
@ -33,106 +37,55 @@ const sep = string(filepath.Separator)
func main() { func main() {
if len(os.Args) > 1 { if len(os.Args) > 1 {
fmt.Fprintf(os.Stderr, "plugin thema code generator does not currently accept any arguments\n, got %q", os.Args) log.Fatal(fmt.Errorf("plugin thema code generator does not currently accept any arguments\n, got %q", os.Args))
os.Exit(1)
} }
cwd, err := os.Getwd() cwd, err := os.Getwd()
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "could not get working directory: %s", err) log.Fatal(fmt.Errorf("could not get working directory: %s", err))
os.Exit(1)
} }
grootp := strings.Split(cwd, sep) grootp := strings.Split(cwd, sep)
groot := filepath.Join(sep, filepath.Join(grootp[:len(grootp)-3]...)) groot := filepath.Join(sep, filepath.Join(grootp[:len(grootp)-3]...))
rt := cuectx.GrafanaThemaRuntime()
wd := codegen.NewWriteDiffer() pluginKindGen := codejen.JennyListWithNamer(func(d *pfs.PluginDecl) string {
lib := cuectx.GrafanaThemaRuntime() return d.PluginMeta.Id
type ptreepath struct {
Path string
Tree *codegen.PluginTree
}
var ptrees []codegen.TreeAndPath
for _, typ := range []string{"datasource", "panel"} {
dir := filepath.Join(cwd, typ)
treeor, err := codegen.ExtractPluginTrees(os.DirFS(dir), lib)
if err != nil {
fmt.Fprintf(os.Stderr, "extracting plugin trees failed for %s: %s\n", dir, err)
os.Exit(1)
}
for name, option := range treeor {
if skipPlugins[name] {
continue
}
if option.Tree != nil {
ptrees = append(ptrees, codegen.TreeAndPath{
Path: filepath.Join(typ, name),
Tree: option.Tree,
})
} else if !errors.Is(option.Err, pfs.ErrNoRootFile) {
fmt.Fprintf(os.Stderr, "error parsing plugin directory %s: %s\n", filepath.Join(dir, name), option.Err)
os.Exit(1)
}
}
}
// Ensure ptrees are sorted, so that visit order is deterministic. Otherwise
// having multiple core plugins with errors can cause confusing error
// flip-flopping
sort.Slice(ptrees, func(i, j int) bool {
return ptrees[i].Path < ptrees[j].Path
}) })
var wdm codegen.WriteDiffer pluginKindGen.Append(
for _, ptp := range ptrees { codegen.PluginTreeListJenny(),
tfast, err := ptp.Tree.GenerateTypeScriptAST() codegen.PluginGoTypesJenny("pkg/tsdb", adaptToPipeline(corecodegen.GoTypesJenny{})),
if err != nil { codegen.PluginTSTypesJenny("public/app/plugins", adaptToPipeline(corecodegen.TSTypesJenny{})),
fmt.Fprintf(os.Stderr, "generating typescript failed for %s: %s\n", ptp.Path, err) )
os.Exit(1)
}
// nil return if there was nothing to generate (no slot implementations)
if tfast != nil {
wd[filepath.Join(ptp.Path, "models.gen.ts")] = []byte(tfast.String())
}
relp, _ := filepath.Rel(groot, ptp.Path) pluginKindGen.AddPostprocessors(corecodegen.SlashHeaderMapper("public/app/plugins/gen.go"))
wdm, err = ptp.Tree.GenerateGo(ptp.Path, codegen.GoGenConfig{
Types: isDatasource(ptp.Tree),
// TODO false until we decide on a consistent codegen format for core and external plugins
ThemaBindings: false,
DocPathPrefix: relp,
})
if err != nil {
fmt.Fprintf(os.Stderr, "generating Go failed for %s: %s\n", ptp.Path, err)
os.Exit(1)
}
wd.Merge(wdm)
}
wdm, err = codegen.GenPluginTreeList(ptrees, "github.com/grafana/grafana/public/app/plugins", filepath.Join(groot, "pkg", "plugins", "pfs", "corelist", "loadlist_gen.go"), false) declParser := pfs.NewDeclParser(rt, skipPlugins)
decls, err := declParser.Parse(os.DirFS(cwd))
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "generating plugin loader registry failed: %s\n", err) log.Fatalln(fmt.Errorf("parsing plugins in dir failed %s: %s", cwd, err))
os.Exit(1) }
jfs, err := pluginKindGen.GenerateFS(decls...)
if err != nil {
log.Fatalln(fmt.Errorf("error writing files to disk: %s", err))
} }
wd.Merge(wdm)
if _, set := os.LookupEnv("CODEGEN_VERIFY"); set { if _, set := os.LookupEnv("CODEGEN_VERIFY"); set {
err = wd.Verify() if err = jfs.Verify(context.Background(), groot); err != nil {
if err != nil { log.Fatal(fmt.Errorf("generated code is out of sync with inputs:\n%s\nrun `make gen-cue` to regenerate", err))
fmt.Fprintf(os.Stderr, "generated code is out of sync with inputs:\n%s\nrun `make gen-cue` to regenerate\n\n", err)
os.Exit(1)
}
} else {
err = wd.Write()
if err != nil {
fmt.Fprintf(os.Stderr, "error while writing generated code to disk:\n%s\n", err)
os.Exit(1)
} }
} else if err = jfs.Write(context.Background(), groot); err != nil {
log.Fatal(fmt.Errorf("error while writing generated code to disk:\n%s", err))
} }
} }
func isDatasource(pt *codegen.PluginTree) bool { func adaptToPipeline(j codejen.OneToOne[corecodegen.SchemaForGen]) codejen.OneToOne[*pfs.PluginDecl] {
return string((*pfs.Tree)(pt).RootPlugin().Meta().Type) == "datasource" return codejen.AdaptOneToOne(j, func(pd *pfs.PluginDecl) corecodegen.SchemaForGen {
return corecodegen.SchemaForGen{
Name: pd.PluginMeta.Name,
Schema: pd.Lineage.Latest(),
IsGroup: pd.Slot.IsGroup(),
}
})
} }

3
public/app/plugins/igen.go
View File

@ -1,3 +0,0 @@
package plugins
//go:generate go run gen.go

14
public/app/plugins/panel/annolist/models.gen.ts
View File

@ -1,12 +1,12 @@
// This file is autogenerated. DO NOT EDIT. // THIS FILE IS GENERATED. EDITING IS FUTILE.
// //
// Generated by public/app/plugins/gen.go // Generated by:
// public/app/plugins/gen.go
// Using jennies:
// TSTypesJenny
// PluginTSTypesJenny
// //
// Derived from the Thema lineage declared in models.cue // Run 'make gen-cue' from repository root to regenerate.
//
// Run `make gen-cue` from repository root to regenerate.
export const PanelModelVersion = Object.freeze([0, 0]); export const PanelModelVersion = Object.freeze([0, 0]);

14
public/app/plugins/panel/barchart/models.gen.ts
View File

@ -1,12 +1,12 @@
// This file is autogenerated. DO NOT EDIT. // THIS FILE IS GENERATED. EDITING IS FUTILE.
// //
// Generated by public/app/plugins/gen.go // Generated by:
// public/app/plugins/gen.go
// Using jennies:
// TSTypesJenny
// PluginTSTypesJenny
// //
// Derived from the Thema lineage declared in models.cue // Run 'make gen-cue' from repository root to regenerate.
//
// Run `make gen-cue` from repository root to regenerate.
import * as ui from '@grafana/schema'; import * as ui from '@grafana/schema';

14
public/app/plugins/panel/bargauge/models.gen.ts
View File

@ -1,12 +1,12 @@
// This file is autogenerated. DO NOT EDIT. // THIS FILE IS GENERATED. EDITING IS FUTILE.
// //
// Generated by public/app/plugins/gen.go // Generated by:
// public/app/plugins/gen.go
// Using jennies:
// TSTypesJenny
// PluginTSTypesJenny
// //
// Derived from the Thema lineage declared in models.cue // Run 'make gen-cue' from repository root to regenerate.
//
// Run `make gen-cue` from repository root to regenerate.
import * as ui from '@grafana/schema'; import * as ui from '@grafana/schema';

14
public/app/plugins/panel/dashlist/models.gen.ts
View File

@ -1,12 +1,12 @@
// This file is autogenerated. DO NOT EDIT. // THIS FILE IS GENERATED. EDITING IS FUTILE.
// //
// Generated by public/app/plugins/gen.go // Generated by:
// public/app/plugins/gen.go
// Using jennies:
// TSTypesJenny
// PluginTSTypesJenny
// //
// Derived from the Thema lineage declared in models.cue // Run 'make gen-cue' from repository root to regenerate.
//
// Run `make gen-cue` from repository root to regenerate.
export const PanelModelVersion = Object.freeze([0, 0]); export const PanelModelVersion = Object.freeze([0, 0]);

14
public/app/plugins/panel/gauge/models.gen.ts
View File

@ -1,12 +1,12 @@
// This file is autogenerated. DO NOT EDIT. // THIS FILE IS GENERATED. EDITING IS FUTILE.
// //
// Generated by public/app/plugins/gen.go // Generated by:
// public/app/plugins/gen.go
// Using jennies:
// TSTypesJenny
// PluginTSTypesJenny
// //
// Derived from the Thema lineage declared in models.cue // Run 'make gen-cue' from repository root to regenerate.
//
// Run `make gen-cue` from repository root to regenerate.
import * as ui from '@grafana/schema'; import * as ui from '@grafana/schema';

14
public/app/plugins/panel/histogram/models.gen.ts
View File

@ -1,12 +1,12 @@
// This file is autogenerated. DO NOT EDIT. // THIS FILE IS GENERATED. EDITING IS FUTILE.
// //
// Generated by public/app/plugins/gen.go // Generated by:
// public/app/plugins/gen.go
// Using jennies:
// TSTypesJenny
// PluginTSTypesJenny
// //
// Derived from the Thema lineage declared in models.cue // Run 'make gen-cue' from repository root to regenerate.
//
// Run `make gen-cue` from repository root to regenerate.
import * as ui from '@grafana/schema'; import * as ui from '@grafana/schema';

14
public/app/plugins/panel/news/models.gen.ts
View File

@ -1,12 +1,12 @@
// This file is autogenerated. DO NOT EDIT. // THIS FILE IS GENERATED. EDITING IS FUTILE.
// //
// Generated by public/app/plugins/gen.go // Generated by:
// public/app/plugins/gen.go
// Using jennies:
// TSTypesJenny
// PluginTSTypesJenny
// //
// Derived from the Thema lineage declared in models.cue // Run 'make gen-cue' from repository root to regenerate.
//
// Run `make gen-cue` from repository root to regenerate.
export const PanelModelVersion = Object.freeze([0, 0]); export const PanelModelVersion = Object.freeze([0, 0]);

14
public/app/plugins/panel/piechart/models.gen.ts
View File

@ -1,12 +1,12 @@
// This file is autogenerated. DO NOT EDIT. // THIS FILE IS GENERATED. EDITING IS FUTILE.
// //
// Generated by public/app/plugins/gen.go // Generated by:
// public/app/plugins/gen.go
// Using jennies:
// TSTypesJenny
// PluginTSTypesJenny
// //
// Derived from the Thema lineage declared in models.cue // Run 'make gen-cue' from repository root to regenerate.
//
// Run `make gen-cue` from repository root to regenerate.
import * as ui from '@grafana/schema'; import * as ui from '@grafana/schema';

14
public/app/plugins/panel/stat/models.gen.ts
View File

@ -1,12 +1,12 @@
// This file is autogenerated. DO NOT EDIT. // THIS FILE IS GENERATED. EDITING IS FUTILE.
// //
// Generated by public/app/plugins/gen.go // Generated by:
// public/app/plugins/gen.go
// Using jennies:
// TSTypesJenny
// PluginTSTypesJenny
// //
// Derived from the Thema lineage declared in models.cue // Run 'make gen-cue' from repository root to regenerate.
//
// Run `make gen-cue` from repository root to regenerate.
import * as ui from '@grafana/schema'; import * as ui from '@grafana/schema';

14
public/app/plugins/panel/text/models.gen.ts
View File

@ -1,12 +1,12 @@
// This file is autogenerated. DO NOT EDIT. // THIS FILE IS GENERATED. EDITING IS FUTILE.
// //
// Generated by public/app/plugins/gen.go // Generated by:
// public/app/plugins/gen.go
// Using jennies:
// TSTypesJenny
// PluginTSTypesJenny
// //
// Derived from the Thema lineage declared in models.cue // Run 'make gen-cue' from repository root to regenerate.
//
// Run `make gen-cue` from repository root to regenerate.
export const PanelModelVersion = Object.freeze([0, 0]); export const PanelModelVersion = Object.freeze([0, 0]);