IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Scuemata: Add test to validate devenv resources (#35810)

Browse Source 
* Add test for devenv resources

* Refactor validation tests for grokkability

* Devenv dashboards error-tracking script

* Refactor to use cueerrors.Details()

* Further test refinement

* Close major elements of dashboard schema

* Centralize dashboard validation tests

General dashboard validation testing belongs in the load package.

* Better names for error context on glue CUE code

* Fixup validate-resource

Do only one of base or dist, and fix copied docs.

* Skip the devenv test

* Remove test for validateResources

* Fix shellcheck

* Backend linter

Co-authored-by: sam boyer <sdboyer@grafana.com>
This commit is contained in:
Dimitris Sotirakis 2021-07-16 03:08:03 +03:00 committed by GitHub
parent 8de218d5f1
commit 2e0dc835cf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 115 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cue/data/gen.cue
View File

@ -80,7 +80,6 @@ Family: scuemata.#Family & {
// synthetic Family to represent them in Go, for ease of generating
// e.g. JSON Schema.
#Panel: {
...
// The panel plugin type id.
type: !=""
@ -135,7 +134,6 @@ Family: scuemata.#Family & {
options: {...}
fieldConfig: {
defaults: {
...
// The display value for this field. This supports template variables blank is auto
displayName?: string
@ -189,7 +187,7 @@ Family: scuemata.#Family & {
// Can always exist. Valid fields within this are
// defined by the panel plugin - that's the
// PanelFieldConfig that comes from the plugin.
custom?: {...}
custom?: {}
}
overrides: [...{
matcher: {

7
pkg/cmd/grafana-cli/commands/commands.go
View File

@ -142,13 +142,18 @@ var cueCommands = []*cli.Command{
},
{
Name: "validate-resource",
Usage: "validate *.cue files in the project",
Usage: "validate resource files (e.g. dashboard JSON) against schema",
Action: runPluginCommand(cmd.validateResources),
Flags: []cli.Flag{
&cli.StringFlag{
Name: "dashboard",
Usage: "dashboard JSON file to validate",
},
&cli.BoolFlag{
Name: "base-only",
Usage: "validate using only base schema, not dist (includes plugin schema)",
Value: false,
},
},
},
}

38
pkg/cmd/grafana-cli/commands/scuemata_validation_command.go
View File

@ -1,10 +1,12 @@
package commands
import (
gerrors "errors"
"fmt"
"os"
"path/filepath"
"cuelang.org/go/cue/errors"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/schema"
"github.com/grafana/grafana/pkg/schema/load"
@ -25,37 +27,31 @@ func (cmd Command) validateScuemataBasics(c utils.CommandLine) error {
}
func (cmd Command) validateResources(c utils.CommandLine) error {
resource := c.String("dashboard")
b, err := os.Open(filepath.Clean(resource))
filename := c.String("dashboard")
baseonly := c.Bool("base-only")
if filename == "" {
return gerrors.New("must specify dashboard to validate with --dashboard")
}
b, err := os.Open(filepath.Clean(filename))
res := schema.Resource{Value: b, Name: filename}
if err != nil {
return err
}
if err := validateResources(b, paths, load.BaseDashboardFamily); err != nil {
return err
var sch schema.VersionedCueSchema
if baseonly {
sch, err = load.BaseDashboardFamily(paths)
} else {
sch, err = load.DistDashboardFamily(paths)
}
if err := validateResources(b, paths, load.DistDashboardFamily); err != nil {
return err
}
return nil
}
func validateResources(resource interface{}, p load.BaseLoadPaths, loader func(p load.BaseLoadPaths) (schema.VersionedCueSchema, error)) error {
dash, err := loader(p)
if err != nil {
return fmt.Errorf("error while loading dashboard scuemata, err: %w", err)
}
// Validate checks that the resource is correct with respect to the schema.
if resource != nil {
err = dash.Validate(schema.Resource{Value: resource})
if err != nil {
return fmt.Errorf("failed validation: %w", err)
}
err = sch.Validate(res)
if err != nil {
return gerrors.New(errors.Details(err, nil))
}
return nil
}

53
pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go
View File

@ -1,9 +1,7 @@
package commands
import (
"io/fs"
"os"
"path/filepath"
"testing"
"testing/fstest"
@ -67,55 +65,4 @@ func TestValidateScuemataBasics(t *testing.T) {
err = validateScuemata(baseLoadPaths, load.DistDashboardFamily)
assert.EqualError(t, err, "all schema should be valid with respect to basic CUE rules, Family.lineages.0.0: field #Panel not allowed")
})
t.Run("Testing validateResources against scuemata and resource inputs", func(t *testing.T) {
validPanel, err := os.ReadFile("testdata/panels/valid_resource_panel.json")
require.NoError(t, err)
invalidPanel, err := os.ReadFile("testdata/panels/invalid_resource_panel.json")
require.NoError(t, err)
filesystem := fstest.MapFS{
"valid.json": &fstest.MapFile{Data: validPanel},
"invalid.json": &fstest.MapFile{Data: invalidPanel},
}
mergedFS := mergefs.Merge(filesystem, defaultBaseLoadPaths.BaseCueFS)
var baseLoadPaths = load.BaseLoadPaths{
BaseCueFS: mergedFS,
DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
}
require.NoError(t, fs.WalkDir(mergedFS, ".", func(path string, d fs.DirEntry, err error) error {
require.NoError(t, err)
if d.IsDir() || filepath.Ext(d.Name()) != ".json" {
return nil
}
if d.Name() == "valid.json" {
t.Run(path, func(t *testing.T) {
b, err := mergedFS.Open(path)
require.NoError(t, err, "failed to open dashboard file")
err = validateResources(b, baseLoadPaths, load.BaseDashboardFamily)
require.NoError(t, err, "error while loading base dashboard scuemata")
err = validateResources(b, baseLoadPaths, load.DistDashboardFamily)
require.NoError(t, err, "error while loading base dashboard scuemata")
})
}
if d.Name() == "invalid.json" {
t.Run(path, func(t *testing.T) {
b, err := mergedFS.Open(path)
require.NoError(t, err, "failed to open dashboard file")
err = validateResources(b, baseLoadPaths, load.BaseDashboardFamily)
assert.EqualError(t, err, "failed validation: Family.lineages.0.0.panels.0.type: incomplete value !=\"\"")
})
}
return nil
}))
})
}

8
pkg/schema/load/dashboard.go
View File

@ -81,7 +81,7 @@ func DistDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
// Value.Fill() can't target definitions. Need new method based on cue.Path;
// a CL has been merged that creates FillPath and will be in the next
// release of CUE.
dummy, _ := rt.Compile("mergeStruct", `
dummy, _ := rt.Compile("glue-unifyPanelDashboard", `
obj: {}
dummy: {
#Panel: obj
@ -125,7 +125,11 @@ type compositeDashboardSchema struct {
// Validate checks that the resource is correct with respect to the schema.
func (cds *compositeDashboardSchema) Validate(r schema.Resource) error {
rv, err := rt.Compile("resource", r.Value)
name := r.Name
if name == "" {
name = "resource"
}
rv, err := rt.Compile(name, r.Value)
if err != nil {
return err
}

6
pkg/schema/load/generic.go
View File

@ -102,7 +102,11 @@ type genericVersionedSchema struct {
// Validate checks that the resource is correct with respect to the schema.
func (gvs *genericVersionedSchema) Validate(r schema.Resource) error {
rv, err := rt.Compile("resource", r.Value)
name := r.Name
if name == "" {
name = "resource"
}
rv, err := rt.Compile(name, r.Value)
if err != nil {
return err
}

76
pkg/schema/load/load_test.go
View File

@ -1,13 +1,16 @@
package load
import (
"encoding/json"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"testing"
"testing/fstest"
"cuelang.org/go/cue/errors"
"github.com/grafana/grafana/pkg/schema"
"github.com/laher/mergefs"
"github.com/stretchr/testify/require"
@ -45,8 +48,11 @@ func TestScuemataBasics(t *testing.T) {
}
}
func TestDashboardValidity(t *testing.T) {
validdir := os.DirFS(filepath.Join("testdata", "artifacts", "dashboards"))
func TestDevenvDashboardValidity(t *testing.T) {
// TODO un-skip when tests pass on all devenv dashboards
t.Skip()
// validdir := os.DirFS(filepath.Join("..", "..", "..", "devenv", "dev-dashboards"))
validdir := filepath.Join("..", "..", "..", "devenv", "dev-dashboards")
dash, err := BaseDashboardFamily(p)
require.NoError(t, err, "error while loading base dashboard scuemata")
@ -54,29 +60,55 @@ func TestDashboardValidity(t *testing.T) {
ddash, err := DistDashboardFamily(p)
require.NoError(t, err, "error while loading dist dashboard scuemata")
require.NoError(t, fs.WalkDir(validdir, ".", func(path string, d fs.DirEntry, err error) error {
require.NoError(t, err)
doTest := func(sch schema.VersionedCueSchema) func(t *testing.T) {
return func(t *testing.T) {
t.Parallel()
require.NoError(t, filepath.Walk(validdir, func(path string, d fs.FileInfo, err error) error {
require.NoError(t, err)
if d.IsDir() || filepath.Ext(d.Name()) != ".json" {
return nil
if d.IsDir() || filepath.Ext(d.Name()) != ".json" {
return nil
}
// Ignore gosec warning G304 since it's a test
// nolint:gosec
b, err := os.Open(path)
require.NoError(t, err, "failed to open dashboard file")
// Only try to validate dashboards with schemaVersion >= 30
jtree := make(map[string]interface{})
byt, err := io.ReadAll(b)
if err != nil {
t.Fatal(err)
}
require.NoError(t, json.Unmarshal(byt, &jtree))
if oldschemav, has := jtree["schemaVersion"]; !has {
t.Logf("no schemaVersion in %s", path)
return nil
} else {
if !(oldschemav.(float64) > 29) {
t.Logf("schemaVersion is %v, older than 30, skipping %s", oldschemav, path)
return nil
}
}
t.Run(filepath.Base(path), func(t *testing.T) {
err := sch.Validate(schema.Resource{Value: byt, Name: path})
if err != nil {
// Testify trims errors to short length. We want the full text
t.Fatal(errors.Details(err, nil))
}
})
return nil
}))
}
}
t.Run(path, func(t *testing.T) {
b, err := validdir.Open(path)
require.NoError(t, err, "failed to open dashboard file")
t.Run("base", func(t *testing.T) {
_, err := schema.SearchAndValidate(dash, b)
require.NoError(t, err, "dashboard failed validation")
})
t.Run("dist", func(t *testing.T) {
_, err := schema.SearchAndValidate(ddash, b)
require.NoError(t, err, "dashboard failed validation")
})
})
return nil
}))
// TODO will need to expand this appropriately when the scuemata contain
// more than one schema
t.Run("base", doTest(dash))
t.Run("dist", doTest(ddash))
}
func TestPanelValidity(t *testing.T) {

4
pkg/schema/load/panel.go
View File

@ -16,7 +16,7 @@ import (
// Returns a disjunction of structs representing each panel schema version
// (post-mapping from on-disk #PanelModel form) from each scuemata in the map.
func disjunctPanelScuemata(scuemap map[string]schema.VersionedCueSchema) (cue.Value, error) {
partsi, err := rt.Compile("panelDisjunction", `
partsi, err := rt.Compile("glue-panelDisjunction", `
allPanels: [Name=_]: {}
parts: or([for v in allPanels { v }])
`)
@ -44,7 +44,7 @@ func disjunctPanelScuemata(scuemap map[string]schema.VersionedCueSchema) (cue.Va
func mapPanelModel(id string, vcs schema.VersionedCueSchema) cue.Value {
maj, min := vcs.Version()
// Ignore err return, this can't fail to compile
inter, _ := rt.Compile("typedPanel", fmt.Sprintf(`
inter, _ := rt.Compile(fmt.Sprintf("%s-glue-panelComposition", id), fmt.Sprintf(`
in: {
type: %q
v: {

17
pkg/schema/schema.go
View File

@ -277,7 +277,11 @@ func Exact(maj, min int) SearchOption {
// that are 1) missing in the Resource AND 2) specified by the schema,
// filled with default values specified by the schema.
func ApplyDefaults(r Resource, scue cue.Value) (Resource, error) {
rv, err := rt.Compile("resource", r.Value)
name := r.Name
if name == "" {
name = "resource"
}
rv, err := rt.Compile(name, r.Value)
if err != nil {
return r, err
}
@ -306,7 +310,11 @@ func convertCUEValueToString(inputCUE cue.Value) (string, error) {
// in the where the values at those paths are the same as the default value
// given in the schema.
func TrimDefaults(r Resource, scue cue.Value) (Resource, error) {
rvInstance, err := rt.Compile("resource", r.Value)
name := r.Name
if name == "" {
name = "resource"
}
rvInstance, err := rt.Compile(name, r.Value)
if err != nil {
return r, err
}
@ -329,7 +337,7 @@ func isCueValueEqual(inputdef cue.Value, input cue.Value) bool {
func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool, error) {
// To include all optional fields, we need to use inputdef for iteration,
// since the lookuppath with optional field doesn't work very well
rvInstance, err := rt.Compile("resource", []byte{})
rvInstance, err := rt.Compile("helper", []byte{})
if err != nil {
return input, false, err
}
@ -421,6 +429,7 @@ func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool,
// TODO this is a terrible way to do this, refactor
type Resource struct {
Value interface{}
Name string
}
// WrapCUEError is a wrapper for cueErrors that occur and are not self explanatory.
@ -430,7 +439,7 @@ func WrapCUEError(err error) error {
var cErr errs.Error
m := make(map[int]string)
if ok := errors.As(err, &cErr); ok {
for _, e := range errs.Errors(err) {
for _, e := range errs.Errors(cErr) {
if e.Position().File() != nil {
line := e.Position().Line()
m[line] = fmt.Sprintf("%q: in file %s", err, e.Position().File().Name())

11
scripts/validate-devenv-dashboards.sh Executable file
View File

@ -0,0 +1,11 @@
#!/bin/bash
# Temporary - remove this script once the dashboard schema are mature
# Remove the appropriate ellipses from the schema to check for unspecified
# fields in the artifacts (validating "open")
# Run from root of grafana repo
CMD=${CLI:-bin/darwin-amd64/grafana-cli}
FILES=$(grep -rl '"schemaVersion": 30' devenv)
for DASH in ${FILES}; do echo "${DASH}"; ${CMD} cue validate-resource --dashboard "${DASH}"; done