Scuemata: Add grafana-cli command to validate resources against scuemata (#33852)

* Add validate-resource cli command

* Fixes according to reviewer's comments

pkg/cmd/grafana-cli/commands/commands.go

@@ -140,6 +140,17 @@ var cueCommands = []*cli.Command{
 		Usage:  "validate *.cue files in the project",
 		Action: runPluginCommand(cmd.validateScuemataBasics),
 	},
+	{
+		Name:   "validate-resource",
+		Usage:  "validate *.cue files in the project",
+		Action: runPluginCommand(cmd.validateResources),
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "dashboard",
+				Usage: "dashboard JSON file to validate",
+			},
+		},
+	},
 }
 
 var Commands = []*cli.Command{

pkg/cmd/grafana-cli/commands/scuemata_validation_command.go

@@ -2,6 +2,8 @@ package commands
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 	"github.com/grafana/grafana/pkg/schema"
@@ -11,18 +13,53 @@ import (
 var paths = load.GetDefaultLoadPaths()
 
 func (cmd Command) validateScuemataBasics(c utils.CommandLine) error {
-	if err := validate(paths, load.BaseDashboardFamily); err != nil {
+	if err := validateScuemata(paths, load.BaseDashboardFamily); err != nil {
 		return err
 	}
 
-	if err := validate(paths, load.DistDashboardFamily); err != nil {
+	if err := validateScuemata(paths, load.DistDashboardFamily); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func validate(p load.BaseLoadPaths, loader func(p load.BaseLoadPaths) (schema.VersionedCueSchema, error)) error {
+func (cmd Command) validateResources(c utils.CommandLine) error {
+	resource := c.String("dashboard")
+	b, err := os.Open(filepath.Clean(resource))
+	if err != nil {
+		return err
+	}
+
+	if err := validateResources(b, paths, load.BaseDashboardFamily); err != nil {
+		return err
+	}
+
+	if err := validateResources(b, paths, load.DistDashboardFamily); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func validateResources(resource interface{}, p load.BaseLoadPaths, loader func(p load.BaseLoadPaths) (schema.VersionedCueSchema, error)) error {
+	dash, err := loader(p)
+	if err != nil {
+		return fmt.Errorf("error while loading dashboard scuemata, err: %w", err)
+	}
+
+	// Validate checks that the resource is correct with respect to the schema.
+	if resource != nil {
+		err = dash.Validate(schema.Resource{Value: resource})
+		if err != nil {
+			return fmt.Errorf("failed validation: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func validateScuemata(p load.BaseLoadPaths, loader func(p load.BaseLoadPaths) (schema.VersionedCueSchema, error)) error {
 	dash, err := loader(p)
 	if err != nil {
 		return fmt.Errorf("error while loading dashboard scuemata, err: %w", err)

pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go

@@ -1,7 +1,9 @@
 package commands
 
 import (
+	"io/fs"
 	"os"
+	"path/filepath"
 	"testing"
 	"testing/fstest"
 
@@ -19,10 +21,10 @@ func TestValidateScuemataBasics(t *testing.T) {
 			DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
 		}
 
-		err := validate(baseLoadPaths, load.BaseDashboardFamily)
+		err := validateScuemata(baseLoadPaths, load.BaseDashboardFamily)
 		require.NoError(t, err, "error while loading base dashboard scuemata")
 
-		err = validate(baseLoadPaths, load.DistDashboardFamily)
+		err = validateScuemata(baseLoadPaths, load.DistDashboardFamily)
 		require.NoError(t, err, "error while loading dist dashboard scuemata")
 	})
 
@@ -30,17 +32,17 @@ func TestValidateScuemataBasics(t *testing.T) {
 		genCue, err := os.ReadFile("testdata/missing_family_gen.cue")
 		require.NoError(t, err)
 
-		fs := fstest.MapFS{
+		filesystem := fstest.MapFS{
 			"cue/data/gen.cue": &fstest.MapFile{Data: genCue},
 		}
-		mergedFS := Merge(fs, defaultBaseLoadPaths.BaseCueFS)
+		mergedFS := Merge(filesystem, defaultBaseLoadPaths.BaseCueFS)
 
 		var baseLoadPaths = load.BaseLoadPaths{
 			BaseCueFS:       mergedFS,
 			DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
 		}
 
-		err = validate(baseLoadPaths, load.BaseDashboardFamily)
+		err = validateScuemata(baseLoadPaths, load.BaseDashboardFamily)
 		assert.EqualError(t, err, "error while loading dashboard scuemata, err: dashboard schema family did not exist at expected path in expected file")
 	})
 
@@ -48,20 +50,71 @@ func TestValidateScuemataBasics(t *testing.T) {
 		genCue, err := os.ReadFile("testdata/missing_panel_gen.cue")
 		require.NoError(t, err)
 
-		fs := fstest.MapFS{
+		filesystem := fstest.MapFS{
 			"cue/data/gen.cue": &fstest.MapFile{Data: genCue},
 		}
-		mergedFS := Merge(fs, defaultBaseLoadPaths.BaseCueFS)
+		mergedFS := Merge(filesystem, defaultBaseLoadPaths.BaseCueFS)
 
 		var baseLoadPaths = load.BaseLoadPaths{
 			BaseCueFS:       mergedFS,
 			DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
 		}
 
-		err = validate(baseLoadPaths, load.BaseDashboardFamily)
+		err = validateScuemata(baseLoadPaths, load.BaseDashboardFamily)
 		require.NoError(t, err, "error while loading base dashboard scuemata")
 
-		err = validate(baseLoadPaths, load.DistDashboardFamily)
+		err = validateScuemata(baseLoadPaths, load.DistDashboardFamily)
 		assert.EqualError(t, err, "all schema should be valid with respect to basic CUE rules, Family.lineages.0.0: field #Panel not allowed")
 	})
+
+	t.Run("Testing validateResources against scuemata and resource inputs", func(t *testing.T) {
+		validPanel, err := os.ReadFile("testdata/panels/valid_resource_panel.json")
+		require.NoError(t, err)
+
+		invalidPanel, err := os.ReadFile("testdata/panels/invalid_resource_panel.json")
+		require.NoError(t, err)
+
+		filesystem := fstest.MapFS{
+			"valid.json":   &fstest.MapFile{Data: validPanel},
+			"invalid.json": &fstest.MapFile{Data: invalidPanel},
+		}
+		mergedFS := Merge(filesystem, defaultBaseLoadPaths.BaseCueFS)
+
+		var baseLoadPaths = load.BaseLoadPaths{
+			BaseCueFS:       mergedFS,
+			DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
+		}
+
+		require.NoError(t, fs.WalkDir(mergedFS, ".", func(path string, d fs.DirEntry, err error) error {
+			require.NoError(t, err)
+
+			if d.IsDir() || filepath.Ext(d.Name()) != ".json" {
+				return nil
+			}
+
+			if d.Name() == "valid.json" {
+				t.Run(path, func(t *testing.T) {
+					b, err := mergedFS.Open(path)
+					require.NoError(t, err, "failed to open dashboard file")
+
+					err = validateResources(b, baseLoadPaths, load.BaseDashboardFamily)
+					require.NoError(t, err, "error while loading base dashboard scuemata")
+
+					err = validateResources(b, baseLoadPaths, load.DistDashboardFamily)
+					require.NoError(t, err, "error while loading base dashboard scuemata")
+				})
+			}
+			if d.Name() == "invalid.json" {
+				t.Run(path, func(t *testing.T) {
+					b, err := mergedFS.Open(path)
+					require.NoError(t, err, "failed to open dashboard file")
+
+					err = validateResources(b, baseLoadPaths, load.BaseDashboardFamily)
+					assert.EqualError(t, err, "failed validation: Family.lineages.0.0.panels.0.fieldConfig.defaults: field mappings not allowed")
+				})
+			}
+
+			return nil
+		}))
+	})
 }

pkg/cmd/grafana-cli/commands/testdata/panels/invalid_resource_panel.json

@@ -0,0 +1,18 @@
+{
+  "panels": [
+    {
+      "datasource": "${DS_GDEV-TESTDATA}",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "right",
+            "filterable": false
+          },
+          "decimals": 3,
+          "mappings": [],
+          "unit": "watt"
+        }
+      }
+    }
+  ]
+}

pkg/cmd/grafana-cli/commands/testdata/panels/valid_resource_panel.json

@@ -0,0 +1,152 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_GDEV-TESTDATA",
+      "label": "gdev-testdata",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "testdata",
+      "pluginName": "TestData DB"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.5.0-pre"
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "testdata",
+      "name": "TestData DB",
+      "version": "1.0.0"
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "rawQuery": "wtf",
+        "showIn": 0,
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "graphTooltip": 0,
+  "id": 42,
+  "links": [],
+  "panels": [
+    {
+      "datasource": "${DS_GDEV-TESTDATA}",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "right",
+            "filterable": false
+          },
+          "decimals": 3,
+          "unit": "watt"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Max"
+            },
+            "properties": [
+              {
+                "id": "custom.displayMode",
+                "value": "lcd-gauge"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "A"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 200
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "7.5.0-pre",
+      "targets": [
+        {
+          "alias": "",
+          "csvWave": {
+            "timeStep": 60,
+            "valuesCSV": "0,0,2,2,1,1"
+          },
+          "lines": 10,
+          "points": [],
+          "pulseWave": {
+            "offCount": 3,
+            "offValue": 1,
+            "onCount": 3,
+            "onValue": 2,
+            "timeStep": 60
+          },
+          "refId": "A",
+          "scenarioId": "random_walk_table",
+          "stream": {
+            "bands": 1,
+            "noise": 2.2,
+            "speed": 250,
+            "spread": 3.5,
+            "type": "signal"
+          },
+          "stringInput": ""
+        }
+      ],
+      "title": "Panel Title",
+      "type": "table",
+      "panelSchema": {
+        "maj": 0,
+        "min": 0
+      }
+    }
+  ],
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timezone": "browser",
+  "title": "with table",
+  "uid": "emal8gQMz",
+  "version": 2
+}