mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
plugins: New static scanner and validator, with Thema slot support (#53754)
* coremodels: Convert plugin-metadata schema to a coremodel * Newer cuetsy; try quoting field name * Add slot definitions * Start sketching out pfs package * Rerun codegen with fixes, new cuetsy * Catch up dashboard with new cuetsy * Update to go1.18 * Use new vmuxers in thema * Add slot system in Go * Draft finished implementation of pfs * Collapse slot pkg into coremodel dir; add PluginInfo * Add the mux type on top of kernel * Refactor plugin generator for extensibility * Change models.cue package, numerous debugs * Bring new output to parity with old * Remove old plugin generation logic * Misc tweaking * Reintroduce generation of shared schemas * Drop back to go1.17 * Add globbing to tsconfig exclude * Introduce pfs test on existing testdata * Make most existing testdata tests pass with pfs * coremodels: Convert plugin-metadata schema to a coremodel * Newer cuetsy; try quoting field name * Add APIType control concept, regen pluginmeta * Use proper numeric types for schema fields * Make pluginmeta schema follow Go type breakdown * More decomposition into distinct types * Add test case for no plugin.json file * Fix missing ref to #Dependencies * Remove generated TS for pluginmeta * Update dependencies, rearrange go.mod * Regenerate without Model prefix * Use updated thema loader; this is now runnable * Skip app plugin with weird include * Make plugin tree extractor reusable * Split out slot lineage load/validate logic * Add myriad tests for new plugin validation failures * Add test for zip fixtures * One last run of codegen * Proper delinting * Ensure validation order is deterministic * Let there actually be sorting * Undo reliance on builtIn field (#54009) * undo builtIn reliance * fix tests Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>
This commit is contained in:
sam boyer
GitHub
@@ -2,186 +2,172 @@ package codegen
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
gerrors "errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"cuelang.org/go/cue"
|
||||
"cuelang.org/go/cue/ast"
|
||||
"cuelang.org/go/cue/build"
|
||||
"cuelang.org/go/cue/cuecontext"
|
||||
"cuelang.org/go/cue/errors"
|
||||
"cuelang.org/go/cue/load"
|
||||
"cuelang.org/go/cue/parser"
|
||||
"github.com/grafana/cuetsy"
|
||||
"github.com/grafana/grafana"
|
||||
"github.com/grafana/grafana/pkg/plugins/pfs"
|
||||
"github.com/grafana/thema"
|
||||
tload "github.com/grafana/thema/load"
|
||||
)
|
||||
|
||||
// The only import statement we currently allow in any models.cue file
|
||||
const schemasPath = "github.com/grafana/grafana/packages/grafana-schema/src/schema"
|
||||
|
||||
// CUE import paths, mapped to corresponding TS import paths. An empty value
|
||||
// indicates the import path should be dropped in the conversion to TS. Imports
|
||||
// not present in the list are not not allowed, and code generation will fail.
|
||||
var importMap = map[string]string{
|
||||
"github.com/grafana/thema": "",
|
||||
schemasPath: "@grafana/schema",
|
||||
"github.com/grafana/thema": "",
|
||||
"github.com/grafana/grafana/packages/grafana-schema/src/schema": "@grafana/schema",
|
||||
}
|
||||
|
||||
// Hard-coded list of paths to skip. Remove a particular file as we're ready
|
||||
// to rely on the TypeScript auto-generated by cuetsy for that particular file.
|
||||
var skipPaths = []string{
|
||||
"public/app/plugins/panel/canvas/models.cue",
|
||||
"public/app/plugins/panel/heatmap/models.cue",
|
||||
"public/app/plugins/panel/heatmap-old/models.cue",
|
||||
"public/app/plugins/panel/candlestick/models.cue",
|
||||
"public/app/plugins/panel/state-timeline/models.cue",
|
||||
"public/app/plugins/panel/status-history/models.cue",
|
||||
"public/app/plugins/panel/table/models.cue",
|
||||
"public/app/plugins/panel/timeseries/models.cue",
|
||||
}
|
||||
|
||||
const prefix = "/"
|
||||
|
||||
// CuetsifyPlugins runs cuetsy against plugins' models.cue files.
|
||||
func CuetsifyPlugins(ctx *cue.Context, root string) (WriteDiffer, error) {
|
||||
lib := thema.NewLibrary(ctx)
|
||||
// TODO this whole func has a lot of old, crufty behavior from the scuemata era; needs TLC
|
||||
overlay := make(map[string]load.Source)
|
||||
err := toOverlay(prefix, grafana.CueSchemaFS, overlay)
|
||||
// err := tload.ToOverlay(prefix, grafana.CueSchemaFS, overlay)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
func init() {
|
||||
allow := pfs.PermittedCUEImports()
|
||||
strsl := make([]string, 0, len(importMap))
|
||||
for p := range importMap {
|
||||
strsl = append(strsl, p)
|
||||
}
|
||||
|
||||
exclude := func(path string) bool {
|
||||
for _, p := range skipPaths {
|
||||
if path == p {
|
||||
return true
|
||||
}
|
||||
sort.Strings(strsl)
|
||||
sort.Strings(allow)
|
||||
if strings.Join(strsl, "") != strings.Join(allow, "") {
|
||||
panic("CUE import map is not the same as permitted CUE import list - these files must be kept in sync!")
|
||||
}
|
||||
}
|
||||
|
||||
// MapCUEImportToTS maps the provided CUE import path to the corresponding
|
||||
// TypeScript import path in generated code.
|
||||
//
|
||||
// Providing an import path that is not allowed results in an error. If a nil
|
||||
// error and empty string are returned, the import path should be dropped in
|
||||
// generated code.
|
||||
func MapCUEImportToTS(path string) (string, error) {
|
||||
i, has := importMap[path]
|
||||
if !has {
|
||||
return "", fmt.Errorf("import %q in models.cue is not allowed", path)
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
|
||||
// ExtractPluginTrees attempts to create a *pfs.Tree for each of the top-level child
|
||||
// directories in the provided fs.FS.
|
||||
//
|
||||
// Errors returned from [pfs.ParsePluginFS] are placed in the option map. Only
|
||||
// filesystem traversal and read errors will result in a non-nil second return
|
||||
// value.
|
||||
func ExtractPluginTrees(parent fs.FS, lib thema.Library) (map[string]PluginTreeOrErr, error) {
|
||||
ents, err := fs.ReadDir(parent, ".")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading fs root directory: %w", err)
|
||||
}
|
||||
|
||||
ptrees := make(map[string]PluginTreeOrErr)
|
||||
for _, plugdir := range ents {
|
||||
subpath := plugdir.Name()
|
||||
sub, err := fs.Sub(parent, subpath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating subfs for path %s: %w", subpath, err)
|
||||
}
|
||||
|
||||
return filepath.Dir(path) == "cue.mod"
|
||||
var either PluginTreeOrErr
|
||||
if ptree, err := pfs.ParsePluginFS(sub, lib); err == nil {
|
||||
either.Tree = (*PluginTree)(ptree)
|
||||
} else {
|
||||
either.Err = err
|
||||
}
|
||||
ptrees[subpath] = either
|
||||
}
|
||||
|
||||
// Prep the cue load config
|
||||
clcfg := &load.Config{
|
||||
Overlay: overlay,
|
||||
// FIXME these module paths won't work for things not under our cue.mod - AKA third-party plugins
|
||||
ModuleRoot: prefix,
|
||||
Module: "github.com/grafana/grafana",
|
||||
}
|
||||
|
||||
outfiles := NewWriteDiffer()
|
||||
|
||||
cuetsify := func(in fs.FS) error {
|
||||
seen := make(map[string]bool)
|
||||
return fs.WalkDir(in, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
if d.IsDir() || filepath.Ext(d.Name()) != ".cue" || seen[dir] || exclude(path) {
|
||||
return nil
|
||||
}
|
||||
seen[dir] = true
|
||||
clcfg.Dir = filepath.Join(root, dir)
|
||||
|
||||
var b []byte
|
||||
f := &tsFile{}
|
||||
|
||||
switch {
|
||||
default:
|
||||
insts := load.Instances(nil, clcfg)
|
||||
if len(insts) > 1 {
|
||||
return fmt.Errorf("%s: resulted in more than one instance", path)
|
||||
}
|
||||
v := ctx.BuildInstance(insts[0])
|
||||
|
||||
b, err = cuetsy.Generate(v, cuetsy.Config{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
case strings.Contains(path, "public/app/plugins"): // panel plugin models.cue files
|
||||
// The simple - and preferable - thing would be to have plugins use the same
|
||||
// package name for their models.cue as their containing dir. That's not
|
||||
// possible, though, because we allow dashes in plugin names, but CUE does not
|
||||
// allow them in package names. Yuck.
|
||||
inst, err := loadInstancesWithThema(in, dir, "grafanaschema")
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not load CUE instance for %s: %w", dir, err)
|
||||
}
|
||||
|
||||
// Also parse file directly to extract imports.
|
||||
// NOTE this will need refactoring to support working with more than one file at a time
|
||||
of, _ := in.Open(path)
|
||||
pf, _ := parser.ParseFile(filepath.Base(path), of, parser.ParseComments)
|
||||
|
||||
iseen := make(map[string]bool)
|
||||
for _, im := range pf.Imports {
|
||||
ip := strings.Trim(im.Path.Value, "\"")
|
||||
mappath, has := importMap[ip]
|
||||
if !has {
|
||||
// TODO make a specific error type for this
|
||||
var all []string
|
||||
for im := range importMap {
|
||||
all = append(all, fmt.Sprintf("\t%s", im))
|
||||
}
|
||||
return errors.Newf(im.Pos(), "%s: import %q not allowed, panel plugins may only import from:\n%s\n", path, ip, strings.Join(all, "\n"))
|
||||
}
|
||||
// TODO this approach will silently swallow the unfixable
|
||||
// error case where multiple files in the same dir import
|
||||
// the same package to a different ident
|
||||
if mappath != "" && !iseen[ip] {
|
||||
iseen[ip] = true
|
||||
f.Imports = append(f.Imports, convertImport(im))
|
||||
}
|
||||
}
|
||||
|
||||
v := ctx.BuildInstance(inst)
|
||||
|
||||
lin, err := thema.BindLineage(v.LookupPath(cue.ParsePath("Panel")), lib)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: failed to bind lineage: %w", path, err)
|
||||
}
|
||||
f.V = thema.LatestVersion(lin)
|
||||
f.WriteModelVersion = true
|
||||
|
||||
b, err = cuetsy.Generate(thema.SchemaP(lin, f.V).UnwrapCUE(), cuetsy.Config{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
f.Body = string(b)
|
||||
|
||||
var buf bytes.Buffer
|
||||
err = tsTemplate.Execute(&buf, f)
|
||||
outfiles[filepath.Join(root, strings.Replace(path, ".cue", ".gen.ts", -1))] = buf.Bytes()
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
err = cuetsify(grafana.CueSchemaFS)
|
||||
if err != nil {
|
||||
return nil, gerrors.New(errors.Details(err, nil))
|
||||
}
|
||||
|
||||
return outfiles, nil
|
||||
return ptrees, nil
|
||||
}
|
||||
|
||||
func convertImport(im *ast.ImportSpec) *tsImport {
|
||||
tsim := &tsImport{
|
||||
Pkg: importMap[schemasPath],
|
||||
// PluginTreeOrErr represents either a *pfs.Tree, or the error that occurred
|
||||
// while trying to create one.
|
||||
// TODO replace with generic option type after go 1.18
|
||||
type PluginTreeOrErr struct {
|
||||
Err error
|
||||
Tree *PluginTree
|
||||
}
|
||||
|
||||
// PluginTree is a pfs.Tree. It exists so we can add methods for code generation to it.
|
||||
type PluginTree pfs.Tree
|
||||
|
||||
func (pt *PluginTree) GenerateTS(path string) (WriteDiffer, error) {
|
||||
t := (*pfs.Tree)(pt)
|
||||
|
||||
// TODO replace with cuetsy's TS AST
|
||||
f := &tsFile{}
|
||||
|
||||
pi := t.RootPlugin()
|
||||
slotimps := pi.SlotImplementations()
|
||||
if len(slotimps) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
for _, im := range pi.CUEImports() {
|
||||
if tsim := convertImport(im); tsim != nil {
|
||||
f.Imports = append(f.Imports, tsim)
|
||||
}
|
||||
}
|
||||
|
||||
for slotname, lin := range slotimps {
|
||||
v := thema.LatestVersion(lin)
|
||||
sch := thema.SchemaP(lin, v)
|
||||
// TODO need call expressions in cuetsy tsast to be able to do these
|
||||
sec := tsSection{
|
||||
V: v,
|
||||
ModelName: slotname,
|
||||
}
|
||||
|
||||
// TODO this is hardcoded for now, but should ultimately be a property of
|
||||
// whether the slot is a grouped lineage:
|
||||
// https://github.com/grafana/thema/issues/62
|
||||
switch slotname {
|
||||
case "Panel", "DSConfig":
|
||||
b, err := cuetsy.Generate(sch.UnwrapCUE(), cuetsy.Config{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: error translating %s lineage to TypeScript: %w", path, slotname, err)
|
||||
}
|
||||
sec.Body = string(b)
|
||||
case "Query":
|
||||
a, err := cuetsy.GenerateSingleAST(strings.Title(lin.Name()), sch.UnwrapCUE(), cuetsy.TypeInterface)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: error translating %s lineage to TypeScript: %w", path, slotname, err)
|
||||
}
|
||||
sec.Body = fmt.Sprint(a)
|
||||
default:
|
||||
panic("unrecognized slot name: " + slotname)
|
||||
}
|
||||
|
||||
f.Sections = append(f.Sections, sec)
|
||||
}
|
||||
|
||||
wd := NewWriteDiffer()
|
||||
var buf bytes.Buffer
|
||||
err := tsSectionTemplate.Execute(&buf, f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: error executing plugin TS generator template: %w", path, err)
|
||||
}
|
||||
wd[filepath.Join(path, "models.gen.ts")] = buf.Bytes()
|
||||
return wd, nil
|
||||
}
|
||||
|
||||
// TODO convert this to use cuetsy ts types, once import * form is supported
|
||||
func convertImport(im *ast.ImportSpec) *tsImport {
|
||||
var err error
|
||||
tsim := &tsImport{}
|
||||
tsim.Pkg, err = MapCUEImportToTS(strings.Trim(im.Path.Value, "\""))
|
||||
if err != nil {
|
||||
// should be unreachable if paths has been verified already
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if tsim.Pkg == "" {
|
||||
// Empty string mapping means skip it
|
||||
return nil
|
||||
}
|
||||
|
||||
if im.Name != nil && im.Name.String() != "" {
|
||||
tsim.Ident = im.Name.String()
|
||||
} else {
|
||||
@@ -196,145 +182,15 @@ func convertImport(im *ast.ImportSpec) *tsImport {
|
||||
return tsim
|
||||
}
|
||||
|
||||
var themamodpath string = filepath.Join("cue.mod", "pkg", "github.com", "grafana", "thema")
|
||||
|
||||
// all copied and hacked up from Thema's LoadInstancesWithThema, simply to allow setting the
|
||||
// package name
|
||||
func loadInstancesWithThema(modFS fs.FS, dir string, pkgname string) (*build.Instance, error) {
|
||||
var modname string
|
||||
err := fs.WalkDir(modFS, "cue.mod", func(path string, d fs.DirEntry, err error) error {
|
||||
// fs.FS implementations tend to not use path separators as expected. Use a
|
||||
// normalized one for comparisons, but retain the original for calls back into modFS.
|
||||
normpath := filepath.FromSlash(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if d.IsDir() {
|
||||
switch normpath {
|
||||
case filepath.Join("cue.mod", "gen"), filepath.Join("cue.mod", "usr"):
|
||||
return fs.SkipDir
|
||||
case themamodpath:
|
||||
return fmt.Errorf("path %q already exists in modFS passed to InstancesWithThema, must be absent for dynamic dependency injection", themamodpath)
|
||||
}
|
||||
return nil
|
||||
} else if normpath == filepath.Join("cue.mod", "module.cue") {
|
||||
modf, err := modFS.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer modf.Close() // nolint: errcheck
|
||||
|
||||
b, err := io.ReadAll(modf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
modname, err = cuecontext.New().CompileBytes(b).LookupPath(cue.MakePath(cue.Str("module"))).String()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if modname == "" {
|
||||
return fmt.Errorf("InstancesWithThema requires non-empty module name in modFS' cue.mod/module.cue")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if modname == "" {
|
||||
return nil, errors.New("cue.mod/module.cue did not exist")
|
||||
}
|
||||
|
||||
modroot := filepath.FromSlash(filepath.Join("/", modname))
|
||||
overlay := make(map[string]load.Source)
|
||||
if err := tload.ToOverlay(modroot, modFS, overlay); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Special case for when we're calling this loader with paths inside the thema module
|
||||
if modname == "github.com/grafana/thema" {
|
||||
if err := tload.ToOverlay(modroot, thema.CueJointFS, overlay); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if err := tload.ToOverlay(filepath.Join(modroot, themamodpath), thema.CueFS, overlay); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if dir == "" {
|
||||
dir = "."
|
||||
}
|
||||
|
||||
cfg := &load.Config{
|
||||
Overlay: overlay,
|
||||
ModuleRoot: modroot,
|
||||
Module: modname,
|
||||
Dir: filepath.Join(modroot, dir),
|
||||
Package: pkgname,
|
||||
}
|
||||
if dir == "." {
|
||||
cfg.Package = filepath.Base(modroot)
|
||||
cfg.Dir = modroot
|
||||
}
|
||||
|
||||
inst := load.Instances(nil, cfg)[0]
|
||||
if inst.Err != nil {
|
||||
return nil, inst.Err
|
||||
}
|
||||
|
||||
return inst, nil
|
||||
}
|
||||
|
||||
func toOverlay(prefix string, vfs fs.FS, overlay map[string]load.Source) error {
|
||||
if !filepath.IsAbs(prefix) {
|
||||
return fmt.Errorf("must provide absolute path prefix when generating cue overlay, got %q", prefix)
|
||||
}
|
||||
err := fs.WalkDir(vfs, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := vfs.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func(f fs.File) {
|
||||
err := f.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}(f)
|
||||
|
||||
b, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
overlay[filepath.Join(prefix, path)] = load.FromBytes(b)
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type tsFile struct {
|
||||
V thema.SyntacticVersion
|
||||
WriteModelVersion bool
|
||||
Imports []*tsImport
|
||||
Body string
|
||||
Imports []*tsImport
|
||||
Sections []tsSection
|
||||
}
|
||||
|
||||
type tsSection struct {
|
||||
V thema.SyntacticVersion
|
||||
ModelName string
|
||||
Body string
|
||||
}
|
||||
|
||||
type tsImport struct {
|
||||
@@ -342,14 +198,14 @@ type tsImport struct {
|
||||
Pkg string
|
||||
}
|
||||
|
||||
var tsTemplate = template.Must(template.New("cuetsygen").Parse(`//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
var tsSectionTemplate = template.Must(template.New("cuetsymulti").Parse(`//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
// This file is autogenerated. DO NOT EDIT.
|
||||
//
|
||||
// To regenerate, run "make gen-cue" from the repository root.
|
||||
//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
{{range .Imports}}
|
||||
import * as {{.Ident}} from '{{.Pkg}}';{{end}}
|
||||
{{if .WriteModelVersion }}
|
||||
export const modelVersion = Object.freeze([{{index .V 0}}, {{index .V 1}}]);
|
||||
{{range .Sections}}{{if ne .ModelName "" }}
|
||||
export const {{.ModelName}}ModelVersion = Object.freeze([{{index .V 0}}, {{index .V 1}}]);
|
||||
{{end}}
|
||||
{{.Body}}`))
|
||||
{{.Body}}{{end}}`))
|
||||
|
||||
Reference in New Issue
Block a user