IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

schema: Migrate from scuemata to thema (#49805)

Browse Source 
* Remove crufty scuemata bits

Buhbye to: cue/ dir with old definitions, CI steps for checking unnecessary
things, and the original dashboard scuemata file.

* Remove grafana-cli cue subcommand

* Remove old testdata

* Don't swallow errors from codegen

* Small nits and tweaks to cuectx package

* WIP - refactor pluggen to use Thema

Also consolidate the embed.FS in the repo root.

* Finish halfway rename

* Convert all panel models.cue to thema

* Rewrite pluggen to use Thema

* Remove pkg/schema, and trim command

* Remove schemaloader service and usages

Will be replaced by coremodel-centric hydrate/dehydrate system Soon™.

* Remove schemaloader from wire

* Remove hangover field on histogram models.cue

* Fix lint errors, some vestiges of trim service

* Remove unused cuetsify cli command
This commit is contained in:
sam boyer
2022-06-06 20:52:44 -04:00
committed by GitHub
parent e7d6a58037
commit 8876d56495
81 changed files with 538 additions and 32087 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
pkg/cmd/grafana-cli/commands/commands.go
View File

@@ -100,12 +100,6 @@ func runPluginCommand(command func(commandLine utils.CommandLine) error) func(co
}
}
func runCueCommand(command func(commandLine utils.CommandLine) error) func(context *cli.Context) error {
return func(context *cli.Context) error {
return command(&utils.ContextCommandLine{Context: context})
}
}
// Command contains command state.
type Command struct {
Client utils.ApiClient
@@ -197,74 +191,6 @@ var adminCommands = []*cli.Command{
},
}
var cueCommands = []*cli.Command{
{
Name: "validate-schema",
Usage: "validate known *.cue files in the Grafana project",
Action: runCueCommand(cmd.validateScuemata),
Description: `validate-schema checks that all CUE schema files are valid with respect
to basic standards - valid CUE, valid scuemata, etc. Note that this
command checks only paths that existed when grafana-cli was compiled,
so must be recompiled to validate newly-added CUE files.`,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "grafana-root",
Usage: "path to the root of a Grafana repository to validate",
},
},
},
{
Name: "validate-resource",
Usage: "validate resource files (e.g. dashboard JSON) against schema",
Action: runCueCommand(cmd.validateResources),
Flags: []cli.Flag{
&cli.StringFlag{
Name: "dashboard",
Usage: "dashboard JSON file to validate",
},
&cli.BoolFlag{
Name: "base-only",
Usage: "validate using only base schema, not dist (includes plugin schema)",
Value: false,
},
},
},
{
Name: "trim-resource",
Usage: "trim schema-specified defaults from a resource",
Action: runCueCommand(cmd.trimResource),
Flags: []cli.Flag{
&cli.StringFlag{
Name: "dashboard",
Usage: "path to file containing (valid) dashboard JSON",
},
&cli.BoolFlag{
Name: "apply",
Usage: "invert the operation: apply defaults instead of trimming them",
Value: false,
},
},
},
{
Name: "gen-ts",
Usage: "generate TypeScript from all known CUE file types",
Description: `gen-ts generates TypeScript from all CUE files at
expected positions in the filesystem tree of a Grafana repository.`,
Action: runCueCommand(cmd.generateTypescript),
Flags: []cli.Flag{
&cli.StringFlag{
Name: "grafana-root",
Usage: "path to the root of a Grafana repository in which to generate TypeScript from CUE files",
},
&cli.BoolFlag{
Name: "diff",
Usage: "diff results of codegen against files already on disk. Exits 1 if diff is non-empty",
Value: false,
},
},
},
}
var Commands = []*cli.Command{
{
Name: "plugins",
@@ -276,9 +202,4 @@ var Commands = []*cli.Command{
Usage: "Grafana admin commands",
Subcommands: adminCommands,
},
{
Name: "cue",
Usage: "Cue validation commands",
Subcommands: cueCommands,
},
}

30
pkg/cmd/grafana-cli/commands/cuetsify_command.go
View File

@@ -1,30 +0,0 @@
package commands
import (
gerrors "errors"
"cuelang.org/go/cue/cuecontext"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/codegen"
)
var ctx = cuecontext.New()
// TODO remove this whole thing
func (cmd Command) generateTypescript(c utils.CommandLine) error {
root := c.String("grafana-root")
if root == "" {
return gerrors.New("must provide path to the root of a Grafana repository checkout")
}
wd, err := codegen.CuetsifyPlugins(ctx, root)
if err != nil {
return err
}
if c.Bool("diff") {
return wd.Verify()
}
return wd.Write()
}

126
pkg/cmd/grafana-cli/commands/scuemata_validation_command.go
View File

@@ -1,126 +0,0 @@
package commands
import (
gerrors "errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"testing/fstest"
"cuelang.org/go/cue/errors"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/schema"
"github.com/grafana/grafana/pkg/schema/load"
)
var paths = load.GetDefaultLoadPaths()
func (cmd Command) validateScuemata(c utils.CommandLine) error {
root := c.String("grafana-root")
if root == "" {
return gerrors.New("must provide path to the root of a Grafana repository checkout")
}
// Construct MapFS with the same set of files as those embedded in
// /embed.go, but sourced straight through from disk instead of relying on
// what's compiled. Not the greatest, because we're duplicating
// filesystem-loading logic with what's in /embed.go.
var fspaths load.BaseLoadPaths
var err error
fspaths.BaseCueFS, err = populateMapFSFromRoot(paths.BaseCueFS, root, "")
if err != nil {
return err
}
fspaths.DistPluginCueFS, err = populateMapFSFromRoot(paths.DistPluginCueFS, root, "")
if err != nil {
return err
}
if err := validateScuemata(fspaths, load.DistDashboardFamily); err != nil {
return schema.WrapCUEError(err)
}
return nil
}
// Helper function that populates an fs.FS by walking over a virtual filesystem,
// and reading files from disk corresponding to each file encountered.
func populateMapFSFromRoot(in fs.FS, root, join string) (fs.FS, error) {
out := make(fstest.MapFS)
err := fs.WalkDir(in, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
// Ignore gosec warning G304. The input set here is necessarily
// constrained to files specified in embed.go
// nolint:gosec
b, err := os.Open(filepath.Join(root, join, path))
if err != nil {
return err
}
byt, err := io.ReadAll(b)
if err != nil {
return err
}
out[path] = &fstest.MapFile{Data: byt}
return nil
})
return out, err
}
func (cmd Command) validateResources(c utils.CommandLine) error {
filename := c.String("dashboard")
baseonly := c.Bool("base-only")
if filename == "" {
return gerrors.New("must specify dashboard to validate with --dashboard")
}
b, err := os.Open(filepath.Clean(filename))
res := schema.Resource{Value: b, Name: filename}
if err != nil {
return err
}
var sch schema.VersionedCueSchema
if baseonly {
sch, err = load.BaseDashboardFamily(paths)
} else {
sch, err = load.DistDashboardFamily(paths)
}
if err != nil {
return fmt.Errorf("error while loading dashboard scuemata, err: %w", err)
}
err = sch.Validate(res)
if err != nil {
return gerrors.New(errors.Details(err, nil))
}
return nil
}
func validateScuemata(p load.BaseLoadPaths, loader func(p load.BaseLoadPaths) (schema.VersionedCueSchema, error)) error {
dash, err := loader(p)
if err != nil {
return fmt.Errorf("error while loading dashboard scuemata, err: %w", err)
}
// Check that a CUE value exists.
cueValue := dash.CUE()
if !cueValue.Exists() {
return fmt.Errorf("cue value for schema does not exist")
}
// Check CUE validity.
if err := cueValue.Validate(); err != nil {
return fmt.Errorf("all schema should be valid with respect to basic CUE rules, %w", err)
}
return nil
}

70
pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go
View File

@@ -1,70 +0,0 @@
package commands
import (
"os"
"testing"
"testing/fstest"
"github.com/grafana/grafana/pkg/schema/load"
"github.com/laher/mergefs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var defaultBaseLoadPaths = load.GetDefaultLoadPaths()
func TestValidateScuemataBasics(t *testing.T) {
t.Run("Testing scuemata validity with valid cue schemas", func(t *testing.T) {
var baseLoadPaths = load.BaseLoadPaths{
BaseCueFS: defaultBaseLoadPaths.BaseCueFS,
DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
}
err := validateScuemata(baseLoadPaths, load.BaseDashboardFamily)
require.NoError(t, err, "error while loading base dashboard scuemata")
err = validateScuemata(baseLoadPaths, load.DistDashboardFamily)
require.NoError(t, err, "error while loading dist dashboard scuemata")
})
t.Run("Testing scuemata validity with invalid cue schemas - family missing", func(t *testing.T) {
t.Skip() // TODO debug, re-enable and move
genCue, err := os.ReadFile("testdata/missing_family.cue")
require.NoError(t, err)
filesystem := fstest.MapFS{
"packages/grafana-schema/src/scuemata/dashboard/dashboard.cue": &fstest.MapFile{Data: genCue},
}
mergedFS := mergefs.Merge(filesystem, defaultBaseLoadPaths.BaseCueFS)
var baseLoadPaths = load.BaseLoadPaths{
BaseCueFS: mergedFS,
DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
}
err = validateScuemata(baseLoadPaths, load.BaseDashboardFamily)
assert.EqualError(t, err, "error while loading dashboard scuemata, err: dashboard schema family did not exist at expected path in expected file")
})
t.Run("Testing scuemata validity with invalid cue schemas - panel missing ", func(t *testing.T) {
t.Skip() // TODO debug, re-enable and move
genCue, err := os.ReadFile("testdata/missing_panel.cue")
require.NoError(t, err)
filesystem := fstest.MapFS{
"packages/grafana-schema/src/scuemata/dashboard/dashboard.cue": &fstest.MapFile{Data: genCue},
}
mergedFS := mergefs.Merge(filesystem, defaultBaseLoadPaths.BaseCueFS)
var baseLoadPaths = load.BaseLoadPaths{
BaseCueFS: mergedFS,
DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
}
err = validateScuemata(baseLoadPaths, load.BaseDashboardFamily)
require.NoError(t, err, "error while loading base dashboard scuemata")
err = validateScuemata(baseLoadPaths, load.DistDashboardFamily)
assert.EqualError(t, err, "all schema should be valid with respect to basic CUE rules, Family.lineages.0.0: field #Panel not allowed")
})
}

78
pkg/cmd/grafana-cli/commands/testdata/missing_family.cue vendored
View File

@@ -1,78 +0,0 @@
package dashboard
import "github.com/grafana/grafana/cue/scuemata"
Dummy: scuemata.#Family & {
lineages: [
[
{ // 0.0
// Unique numeric identifier for the dashboard.
// TODO must isolate or remove identifiers local to a Grafana instance...?
id?: number
// Unique dashboard identifier that can be generated by anyone. string (8-40)
uid: string
// Title of dashboard.
title?: string
// Description of dashboard.
description?: string
gnetId?: string
// Tags associated with dashboard.
tags?: [...string]
// Theme of dashboard.
style: *"light" | "dark"
// Timezone of dashboard,
timezone?: *"browser" | "utc"
// Whether a dashboard is editable or not.
editable: bool | *true
// 0 for no shared crosshair or tooltip (default).
// 1 for shared crosshair.
// 2 for shared crosshair AND shared tooltip.
graphTooltip: >=0 & <=2 | *0
// Time range for dashboard, e.g. last 6 hours, last 7 days, etc
time?: {
from: string | *"now-6h"
to: string | *"now"
}
// Timepicker metadata.
timepicker?: {
// Whether timepicker is collapsed or not.
collapse: bool | *false
// Whether timepicker is enabled or not.
enable: bool | *true
// Whether timepicker is visible or not.
hidden: bool | *false
// Selectable intervals for auto-refresh.
refresh_intervals: [...string] | *["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"]
}
// Templating.
templating?: list: [...{...}]
// Annotations.
annotations?: list: [...{
builtIn: number | *0
// Datasource to use for annotation.
datasource: string
// Whether annotation is enabled.
enable?: bool | *true
// Whether to hide annotation.
hide?: bool | *false
// Annotation icon color.
iconColor?: string
// Name of annotation.
name?: string
type: string | *"dashboard"
// Query for annotation data.
rawQuery?: string
showIn: number | *0
}]
// Auto-refresh interval.
refresh?: string
// Version of the JSON schema, incremented each time a Grafana update brings
// changes to said schema.
schemaVersion: number | *25
// Version of the dashboard, incremented each time the dashboard is updated.
version?: number
}
]
]
}

78
pkg/cmd/grafana-cli/commands/testdata/missing_panel.cue vendored
View File

@@ -1,78 +0,0 @@
package dashboard
import "github.com/grafana/grafana/cue/scuemata"
Family: scuemata.#Family & {
lineages: [
[
{ // 0.0
// Unique numeric identifier for the dashboard.
// TODO must isolate or remove identifiers local to a Grafana instance...?
id?: number
// Unique dashboard identifier that can be generated by anyone. string (8-40)
uid: string
// Title of dashboard.
title?: string
// Description of dashboard.
description?: string
gnetId?: string
// Tags associated with dashboard.
tags?: [...string]
// Theme of dashboard.
style: *"light" | "dark"
// Timezone of dashboard,
timezone?: *"browser" | "utc"
// Whether a dashboard is editable or not.
editable: bool | *true
// 0 for no shared crosshair or tooltip (default).
// 1 for shared crosshair.
// 2 for shared crosshair AND shared tooltip.
graphTooltip: >=0 & <=2 | *0
// Time range for dashboard, e.g. last 6 hours, last 7 days, etc
time?: {
from: string | *"now-6h"
to: string | *"now"
}
// Timepicker metadata.
timepicker?: {
// Whether timepicker is collapsed or not.
collapse: bool | *false
// Whether timepicker is enabled or not.
enable: bool | *true
// Whether timepicker is visible or not.
hidden: bool | *false
// Selectable intervals for auto-refresh.
refresh_intervals: [...string] | *["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"]
}
// Templating.
templating?: list: [...{...}]
// Annotations.
annotations?: list: [...{
builtIn: number | *0
// Datasource to use for annotation.
datasource: string
// Whether annotation is enabled.
enable?: bool | *true
// Whether to hide annotation.
hide?: bool | *false
// Annotation icon color.
iconColor?: string
// Name of annotation.
name?: string
type: string | *"dashboard"
// Query for annotation data.
rawQuery?: string
showIn: number | *0
}]
// Auto-refresh interval.
refresh?: string
// Version of the JSON schema, incremented each time a Grafana update brings
// changes to said schema.
schemaVersion: number | *25
// Version of the dashboard, incremented each time the dashboard is updated.
version?: number
}
]
]
}

18
pkg/cmd/grafana-cli/commands/testdata/panels/invalid_resource_panel.json vendored
View File

@@ -1,18 +0,0 @@
{
"panels": [
{
"datasource": "${DS_GDEV-TESTDATA}",
"fieldConfig": {
"defaults": {
"custom": {
"align": "right",
"filterable": false
},
"decimals": 3,
"mappings": [],
"unit": "watt"
}
}
}
]
}

152
pkg/cmd/grafana-cli/commands/testdata/panels/valid_resource_panel.json vendored
View File

@@ -1,152 +0,0 @@
{
"__inputs": [
{
"name": "DS_GDEV-TESTDATA",
"label": "gdev-testdata",
"description": "",
"type": "datasource",
"pluginId": "testdata",
"pluginName": "TestData DB"
}
],
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "7.5.0-pre"
},
{
"type": "panel",
"id": "table",
"name": "Table",
"version": ""
},
{
"type": "datasource",
"id": "testdata",
"name": "TestData DB",
"version": "1.0.0"
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"rawQuery": "wtf",
"showIn": 0,
"type": "dashboard"
}
]
},
"editable": true,
"graphTooltip": 0,
"id": 42,
"links": [],
"panels": [
{
"datasource": "${DS_GDEV-TESTDATA}",
"fieldConfig": {
"defaults": {
"custom": {
"align": "right",
"filterable": false
},
"decimals": 3,
"unit": "watt"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Max"
},
"properties": [
{
"id": "custom.displayMode",
"value": "lcd-gauge"
}
]
},
{
"matcher": {
"id": "byName",
"options": "A"
},
"properties": [
{
"id": "custom.width",
"value": 200
}
]
}
]
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"showHeader": true,
"sortBy": []
},
"pluginVersion": "7.5.0-pre",
"targets": [
{
"alias": "",
"csvWave": {
"timeStep": 60,
"valuesCSV": "0,0,2,2,1,1"
},
"lines": 10,
"points": [],
"pulseWave": {
"offCount": 3,
"offValue": 1,
"onCount": 3,
"onValue": 2,
"timeStep": 60
},
"refId": "A",
"scenarioId": "random_walk_table",
"stream": {
"bands": 1,
"noise": 2.2,
"speed": 250,
"spread": 3.5,
"type": "signal"
},
"stringInput": ""
}
],
"title": "Panel Title",
"type": "table",
"panelSchema": {
"maj": 0,
"min": 0
}
}
],
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "browser",
"title": "with table",
"uid": "emal8gQMz",
"version": 2
}

59
pkg/cmd/grafana-cli/commands/trim_command.go
View File

@@ -1,59 +0,0 @@
package commands
import (
"bytes"
"encoding/json"
gerrors "errors"
"fmt"
"io"
"os"
"path/filepath"
"cuelang.org/go/cue/errors"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/schema"
"github.com/grafana/grafana/pkg/schema/load"
)
func (cmd Command) trimResource(c utils.CommandLine) error {
filename := c.String("dashboard")
if filename == "" {
return gerrors.New("must specify dashboard file path with --dashboard")
}
apply := c.Bool("apply")
f, err := os.Open(filepath.Clean(filename))
if err != nil {
return err
}
b, err := io.ReadAll(f)
if err != nil {
return err
}
res := schema.Resource{Value: string(b), Name: filename}
sch, err := load.DistDashboardFamily(paths)
if err != nil {
return fmt.Errorf("error while loading dashboard scuemata, err: %w", err)
}
var out schema.Resource
if apply {
out, err = schema.ApplyDefaults(res, sch.CUE())
} else {
out, err = schema.TrimDefaults(res, sch.CUE())
}
if err != nil {
return gerrors.New(errors.Details(err, nil))
}
b = []byte(out.Value.(string))
var buf bytes.Buffer
err = json.Indent(&buf, b, "", " ")
if err != nil {
return err
}
fmt.Println(buf.String())
return nil
}