Schema: introduce CLI command to convert all CUE files to TS (#39694)

* First pass at cuetsify command

* Update go deps

* Small tweaks to input cue files

* Correct ts import structure, whitespace

* Latest version of cuetsy

* add ordinal option

* upate cue file

* Fix merge garbage

* Remove dead code

* Revert "upate cue file"

This reverts commit e40b1df83e.

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
This commit is contained in:
sam boyer 2021-09-29 04:59:05 -04:00 committed by GitHub
parent 4263357bbc
commit c786d22705
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 362 additions and 58 deletions

5
go.mod
View File

@ -49,6 +49,7 @@ require (
github.com/google/wire v0.5.0
github.com/gorilla/websocket v1.4.2
github.com/gosimple/slug v1.9.0
github.com/grafana/cuetsy v0.0.0-20210928021233-5ddfb47f9a7d
github.com/grafana/grafana-aws-sdk v0.7.0
github.com/grafana/grafana-plugin-sdk-go v0.114.0
github.com/grafana/loki v1.6.2-0.20210520072447-15d417efe103
@ -139,7 +140,7 @@ require (
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/cockroachdb/apd/v2 v2.0.1 // indirect
github.com/cockroachdb/apd/v2 v2.0.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/deepmap/oapi-codegen v1.6.0 // indirect
github.com/dennwc/varint v1.0.0 // indirect
@ -235,7 +236,7 @@ require (
go.uber.org/goleak v1.1.10 // indirect
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34 // indirect
golang.org/x/text v0.3.6 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 // indirect

8
go.sum
View File

@ -411,8 +411,9 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed h1:OZmjad4L3H8ncOIR8rn
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/cockroachdb/apd/v2 v2.0.1 h1:y1Rh3tEU89D+7Tgbw+lp52T6p/GJLpDmNvr10UWqLTE=
github.com/cockroachdb/apd/v2 v2.0.1/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw=
github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b8034E=
github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw=
github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk=
github.com/cockroachdb/datadriven v0.0.0-20190531201743-edce55837238/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
@ -1188,6 +1189,8 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gosimple/slug v1.9.0 h1:r5vDcYrFz9BmfIAMC829un9hq7hKM4cHUrsv36LbEqs=
github.com/gosimple/slug v1.9.0/go.mod h1:AMZ+sOVe65uByN3kgEyf9WEBKBCSS+dJjMX9x4vDJbg=
github.com/grafana/cuetsy v0.0.0-20210928021233-5ddfb47f9a7d h1:vYCNM25g5aEactkMiILJvm2jW2BVGYB1QzLx7lAhMLw=
github.com/grafana/cuetsy v0.0.0-20210928021233-5ddfb47f9a7d/go.mod h1:H9Ei+Q808FCWyeEzpaW5GMfBvXCuFOfQa4x/vzKY+Fg=
github.com/grafana/go-mssqldb v0.0.0-20210326084033-d0ce3c521036 h1:GplhUk6Xes5JIhUUrggPcPBhOn+eT8+WsHiebvq7GgA=
github.com/grafana/go-mssqldb v0.0.0-20210326084033-d0ce3c521036/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
github.com/grafana/grafana-aws-sdk v0.7.0 h1:D+Lhxi3P/7vpyDHUK/fdX9bL2mRz8hLG04ucNf1E02o=
@ -2836,8 +2839,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

View File

@ -4,7 +4,7 @@ AxisPlacement: "auto" | "top" | "right" | "bottom" | "left" | "hidden" @cue
VisibilityMode: "auto" | "never" | "always" @cuetsy(kind="enum")
DrawStyle: "line" | "bars" | "points" @cuetsy(kind="enum")
LineInterpolation: "linear" | "smooth" | "stepBefore" | "stepAfter" @cuetsy(kind="enum")
ScaleDistribution: "linear" | "log" @cuetsy(kind="enum")
ScaleDistribution: "linear" | "log" | "ordinal" @cuetsy(kind="enum")
GraphGradientMode: "none" | "opacity" | "hue" | "scheme" @cuetsy(kind="enum")
StackingMode: "none" | "normal" | "percent" @cuetsy(kind="enum")
BarAlignment: -1 | 0 | 1 @cuetsy(kind="enum",memberNames="Before|Center|After")

View File

@ -3,17 +3,7 @@ package schema
// TODO -- should not be table specific!
FieldTextAlignment: "auto" | "left" | "right" | "center" @cuetsy(kind="type")
// FIXME can't write enums as structs, must use disjunctions
TableCellDisplayMode: {
Auto: "auto"
ColorText: "color-text"
ColorBackground: "color-background"
GradientGauge: "gradient-gauge"
LcdGauge: "lcd-gauge"
JSONView: "json-view"
BasicGauge: "basic"
Image: "image"
} @cuetsy(kind="enum")
TableCellDisplayMode: "auto" | "color-text" | "color-background" | "gradient-gauge" | "lcd-gauge" | "json-view" | "basic" | "image" @cuetsy(kind="enum",memberNames="Auto|ColorText|ColorBackground|GradientGauge|LcdGauge|JSONView|BasicGauge|Image")
TableFieldOptions: {
width?: number

View File

@ -169,6 +169,19 @@ so must be recompiled to validate newly-added CUE files.`,
},
},
},
{
Name: "gen-ts",
Usage: "generate TypeScript from all known CUE file types",
Description: `gen-ts generates TypeScript from all CUE files at
expected positions in the filesystem tree of a Grafana repository.`,
Action: runCueCommand(cmd.generateTypescript),
Flags: []cli.Flag{
&cli.StringFlag{
Name: "grafana-root",
Usage: "path to the root of a Grafana repository in which to generate TypeScript from CUE files",
},
},
},
}
var Commands = []*cli.Command{

View File

@ -0,0 +1,298 @@
package commands
import (
"bytes"
gerrors "errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"text/template"
"cuelang.org/go/cue"
"cuelang.org/go/cue/ast"
"cuelang.org/go/cue/cuecontext"
"cuelang.org/go/cue/errors"
cload "cuelang.org/go/cue/load"
"cuelang.org/go/cue/parser"
"github.com/grafana/cuetsy"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/schema/load"
)
// FIXME almost this whole file is a sloppy, one-off hack that just goes around actually making
// the API we need. Parts need to be factored out appropriately.
var ctx = cuecontext.New()
const allowedImport = "github.com/grafana/grafana/packages/grafana-schema/src/schema"
var importMap = map[string]string{
allowedImport: "@grafana/schema",
}
const prefix = "/"
func (cmd Command) generateTypescript(c utils.CommandLine) error {
root := c.String("grafana-root")
if root == "" {
return gerrors.New("must provide path to the root of a Grafana repository checkout")
}
var fspaths load.BaseLoadPaths
var err error
fspaths.BaseCueFS, err = populateMapFSFromRoot(paths.BaseCueFS, root, "")
if err != nil {
return err
}
fspaths.DistPluginCueFS, err = populateMapFSFromRoot(paths.DistPluginCueFS, root, "")
if err != nil {
return err
}
overlay, err := defaultOverlay(fspaths)
if err != nil {
return err
}
// Prep the cue load config
clcfg := &cload.Config{
Overlay: overlay,
// FIXME these module paths won't work for things not under our cue.mod - AKA third-party plugins
// ModuleRoot: prefix,
Module: "github.com/grafana/grafana",
}
// One-time load of the panel-plugin scuemata family def, for unifying to easily apply cuetsy attributes
clcfg.Dir = "cue/scuemata"
v := ctx.BuildInstance(cload.Instances(nil, clcfg)[0])
if v.Err() != nil {
return v.Err()
}
ppf := v.LookupPath(cue.ParsePath("#PanelSchema"))
_ = ppf
// FIXME hardcoding paths to exclude is not the way to handle this
excl := map[string]bool{
"cue.mod": true,
"cue/scuemata": true,
"packages/grafana-schema/src/scuemata/dashboard": true,
"packages/grafana-schema/src/scuemata/dashboard/dist": true,
}
outfiles := make(map[string][]byte)
cuetsify := func(in fs.FS) error {
seen := make(map[string]bool)
return fs.WalkDir(in, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
dir := filepath.Dir(path)
if d.IsDir() || filepath.Ext(d.Name()) != ".cue" || seen[dir] || excl[dir] {
return nil
}
seen[dir] = true
clcfg.Dir = dir
// FIXME Horrible hack to figure out the identifier used for
// imported packages - intercept the parser called by the loader to
// look at the ast.Files on their way in to building.
// Much better if we could work backwards from the cue.Value,
// maybe even directly in cuetsy itself, and figure out when a
// referenced object is "out of bounds".
// var imports sync.Map
var imports []*ast.ImportSpec
clcfg.ParseFile = func(name string, src interface{}) (*ast.File, error) {
f, err := parser.ParseFile(name, src, parser.ParseComments)
if err != nil {
return nil, err
}
imports = append(imports, f.Imports...)
return f, nil
}
// FIXME loading in this way causes all files in a dir to be loaded
// as a single cue.Instance or cue.Value, which makes it quite
// difficult to map them _back_ onto the original file and generate
// discrete .gen.ts files for each .cue input. However, going one
// .cue file at a time and passing it as the first arg to
// load.Instances() means that the other files are ignored
// completely, causing references between these files to be
// unresolved, and thus encounter a different kind of error.
insts := cload.Instances(nil, clcfg)
if len(insts) > 1 {
panic("extra instances")
}
bi := insts[0]
// dumpBuildInst(bi)
v := ctx.BuildInstance(bi)
if v.Err() != nil {
return v.Err()
}
var b []byte
f := &tsFile{}
seen := make(map[string]bool)
// FIXME explicitly mapping path patterns to conversion patterns
// is exactly what we want to avoid
switch {
// panel plugin models.cue files
case strings.Contains(path, "public/app/plugins"):
for _, im := range imports {
ip := strings.Trim(im.Path.Value, "\"")
if ip != allowedImport {
// TODO make a specific error type for this
return errors.Newf(im.Pos(), "import %q not allowed, panel plugins may only import from %q", ip, allowedImport)
}
// TODO this approach will silently swallow the unfixable
// error case where multiple files in the same dir import
// the same package to a different ident
if !seen[ip] {
seen[ip] = true
f.Imports = append(f.Imports, convertImport(im))
}
}
// val := v.LookupPath(cue.ParsePath("Panel.lineages[0][0]"))
// Extract the latest schema and its version number
f.V = &tsModver{}
lins := v.LookupPath(cue.ParsePath("Panel.lineages"))
f.V.Lin, _ = lins.Len().Int64()
f.V.Lin = f.V.Lin - 1
schs := lins.LookupPath(cue.MakePath(cue.Index(int(f.V.Lin))))
f.V.Sch, _ = schs.Len().Int64()
f.V.Sch = f.V.Sch - 1
latest := schs.LookupPath(cue.MakePath(cue.Index(int(f.V.Sch))))
sch := latest.UnifyAccept(ppf, latest)
b, err = cuetsy.Generate(sch, cuetsy.Config{})
default:
b, err = cuetsy.Generate(v, cuetsy.Config{})
}
if err != nil {
return err
}
f.Body = string(b)
var buf bytes.Buffer
err = tsTemplate.Execute(&buf, f)
outfiles[strings.Replace(path, ".cue", ".gen.ts", -1)] = buf.Bytes()
return err
})
}
err = cuetsify(fspaths.BaseCueFS)
if err != nil {
return gerrors.New(errors.Details(err, nil))
}
err = cuetsify(fspaths.DistPluginCueFS)
if err != nil {
return gerrors.New(errors.Details(err, nil))
}
for of, b := range outfiles {
err := os.WriteFile(filepath.Join(root, of), b, 0644)
if err != nil {
return err
}
}
return nil
}
func convertImport(im *ast.ImportSpec) *tsImport {
tsim := &tsImport{
Pkg: importMap[allowedImport],
}
if im.Name != nil && im.Name.String() != "" {
tsim.Ident = im.Name.String()
} else {
sl := strings.Split(im.Path.Value, "/")
final := sl[len(sl)-1]
if idx := strings.Index(final, ":"); idx != -1 {
tsim.Pkg = final[idx:]
} else {
tsim.Pkg = final
}
}
return tsim
}
func defaultOverlay(p load.BaseLoadPaths) (map[string]cload.Source, error) {
overlay := make(map[string]cload.Source)
if err := toOverlay(prefix, p.BaseCueFS, overlay); err != nil {
return nil, err
}
if err := toOverlay(prefix, p.DistPluginCueFS, overlay); err != nil {
return nil, err
}
return overlay, nil
}
func toOverlay(prefix string, vfs fs.FS, overlay map[string]cload.Source) error {
if !filepath.IsAbs(prefix) {
return fmt.Errorf("must provide absolute path prefix when generating cue overlay, got %q", prefix)
}
err := fs.WalkDir(vfs, ".", (func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
f, err := vfs.Open(path)
if err != nil {
return err
}
b, err := io.ReadAll(f)
if err != nil {
return err
}
overlay[filepath.Join(prefix, path)] = cload.FromBytes(b)
return nil
}))
if err != nil {
return err
}
return nil
}
type tsFile struct {
V *tsModver
Imports []*tsImport
Body string
}
type tsModver struct {
Lin, Sch int64
}
type tsImport struct {
Ident string
Pkg string
}
var tsTemplate = template.Must(template.New("cuetsygen").Parse(
`//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// This file was autogenerated by cuetsy. DO NOT EDIT!
//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
{{range .Imports}}
import * as {{.Ident}} from '{{.Pkg}}';{{end}}
{{if .V}}
export const modelVersion = Object.freeze([{{ .V.Lin }}, {{ .V.Sch }}]);
{{end}}
{{.Body}}`))

View File

@ -23,47 +23,18 @@ func (cmd Command) validateScuemata(c utils.CommandLine) error {
return gerrors.New("must provide path to the root of a Grafana repository checkout")
}
// Construct a MapFS with the same set of files as those embedded in
// Construct MapFS with the same set of files as those embedded in
// /embed.go, but sourced straight through from disk instead of relying on
// what's compiled. Not the greatest, because we're duplicating
// filesystem-loading logic with what's in /embed.go.
populate := func(in fs.FS, join string) (fs.FS, error) {
out := make(fstest.MapFS)
err := fs.WalkDir(in, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
// Ignore gosec warning G304. The input set here is necessarily
// constrained to files specified in embed.go
// nolint:gosec
b, err := os.Open(filepath.Join(root, join, path))
if err != nil {
return err
}
byt, err := io.ReadAll(b)
if err != nil {
return err
}
out[path] = &fstest.MapFile{Data: byt}
return nil
})
return out, err
}
var fspaths load.BaseLoadPaths
var err error
fspaths.BaseCueFS, err = populate(paths.BaseCueFS, "")
fspaths.BaseCueFS, err = populateMapFSFromRoot(paths.BaseCueFS, root, "")
if err != nil {
return err
}
fspaths.DistPluginCueFS, err = populate(paths.DistPluginCueFS, "")
fspaths.DistPluginCueFS, err = populateMapFSFromRoot(paths.DistPluginCueFS, root, "")
if err != nil {
return err
}
@ -75,6 +46,36 @@ func (cmd Command) validateScuemata(c utils.CommandLine) error {
return nil
}
// Helper function that populates an fs.FS by walking over a virtual filesystem,
// and reading files from disk corresponding to each file encountered.
func populateMapFSFromRoot(in fs.FS, root, join string) (fs.FS, error) {
out := make(fstest.MapFS)
err := fs.WalkDir(in, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
// Ignore gosec warning G304. The input set here is necessarily
// constrained to files specified in embed.go
// nolint:gosec
b, err := os.Open(filepath.Join(root, join, path))
if err != nil {
return err
}
byt, err := io.ReadAll(b)
if err != nil {
return err
}
out[path] = &fstest.MapFile{Data: byt}
return nil
})
return out, err
}
func (cmd Command) validateResources(c utils.CommandLine) error {
filename := c.String("dashboard")
baseonly := c.Bool("base-only")

View File

@ -28,9 +28,7 @@ Panel: {
combine?: bool
}
PanelFieldConfig: {
ui.GraphFieldConfig
}
PanelFieldConfig: ui.GraphFieldConfig
}
]
]

View File

@ -22,18 +22,18 @@ Panel: {
lineages: [
[
{
#TimelineMode: "changes" | "samples" @cuetsy(kind="enum")
#TimelineValueAlignment: "center" | "left" | "right" @cuetsy(kind="type")
TimelineMode: "changes" | "samples" @cuetsy(kind="enum")
TimelineValueAlignment: "center" | "left" | "right" @cuetsy(kind="type")
PanelOptions: {
// FIXME ts comments indicate this shouldn't be in the saved model, but currently is emitted
mode?: #TimelineMode
mode?: TimelineMode
ui.OptionsWithLegend
ui.OptionsWithTooltip
showValue: ui.VisibilityMode | *"auto"
rowHeight: number | *0.9
colWidth?: number
mergeValues?: bool | *true
alignValue?: #TimelineValueAlignment | *"left"
alignValue?: TimelineValueAlignment | *"left"
}
PanelFieldConfig: {
ui.HideableFieldConfig

View File

@ -18,10 +18,9 @@ Panel: {
lineages: [
[
{
TextMode: "html" | "markdown" @cuetsy(kind="enum",withName="TextMode")
TextMode: "html" | "markdown" @cuetsy(kind="enum",memberNames="HTML|Markdown")
PanelOptions: {
mode: TextMode | *"markdown"
mode: TextMode | *"markdown"
content: string | *"""
# Title