schema: Migrate from scuemata to thema (#49805)

* Remove crufty scuemata bits

Buhbye to: cue/ dir with old definitions, CI steps for checking unnecessary
things, and the original dashboard scuemata file.

* Remove grafana-cli cue subcommand

* Remove old testdata

* Don't swallow errors from codegen

* Small nits and tweaks to cuectx package

* WIP - refactor pluggen to use Thema

Also consolidate the embed.FS in the repo root.

* Finish halfway rename

* Convert all panel models.cue to thema

* Rewrite pluggen to use Thema

* Remove pkg/schema, and trim command

* Remove schemaloader service and usages

Will be replaced by coremodel-centric hydrate/dehydrate system Soon™.

* Remove schemaloader from wire

* Remove hangover field on histogram models.cue

* Fix lint errors, some vestiges of trim service

* Remove unused cuetsify cli command

pkg/services/dashboardimport/api/api.go

@@ -6,7 +6,6 @@ import (
 	"github.com/grafana/grafana/pkg/api/apierrors"
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/api/routing"
-	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/middleware"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -19,17 +18,15 @@ import (
 type ImportDashboardAPI struct {
 	dashboardImportService dashboardimport.Service
 	quotaService           QuotaService
-	schemaLoaderService    SchemaLoaderService
 	pluginStore            plugins.Store
 	ac                     accesscontrol.AccessControl
 }
 
 func New(dashboardImportService dashboardimport.Service, quotaService QuotaService,
-	schemaLoaderService SchemaLoaderService, pluginStore plugins.Store, ac accesscontrol.AccessControl) *ImportDashboardAPI {
+	pluginStore plugins.Store, ac accesscontrol.AccessControl) *ImportDashboardAPI {
 	return &ImportDashboardAPI{
 		dashboardImportService: dashboardImportService,
 		quotaService:           quotaService,
-		schemaLoaderService:    schemaLoaderService,
 		pluginStore:            pluginStore,
 		ac:                     ac,
 	}
@@ -65,14 +62,6 @@ func (api *ImportDashboardAPI) ImportDashboard(c *models.ReqContext) response.Re
 		return response.Error(403, "Quota reached", nil)
 	}
 
-	trimDefaults := c.QueryBoolWithDefault("trimdefaults", true)
-	if trimDefaults && !api.schemaLoaderService.IsDisabled() {
-		req.Dashboard, err = api.schemaLoaderService.DashboardApplyDefaults(req.Dashboard)
-		if err != nil {
-			return response.Error(http.StatusInternalServerError, "Error while applying default value to the dashboard json", err)
-		}
-	}
-
 	req.User = c.SignedInUser
 	resp, err := api.dashboardImportService.ImportDashboard(c.Req.Context(), &req)
 	if err != nil {
@@ -91,8 +80,3 @@ type quotaServiceFunc func(c *models.ReqContext, target string) (bool, error)
 func (fn quotaServiceFunc) QuotaReached(c *models.ReqContext, target string) (bool, error) {
 	return fn(c, target)
 }
-
-type SchemaLoaderService interface {
-	IsDisabled() bool
-	DashboardApplyDefaults(input *simplejson.Json) (*simplejson.Json, error)
-}

pkg/services/dashboardimport/api/api_test.go

@@ -26,15 +26,7 @@ func TestImportDashboardAPI(t *testing.T) {
 			},
 		}
 
-		schemaLoaderServiceCalled := false
-		schemaLoaderService := &schemaLoaderServiceMock{
-			dashboardApplyDefaultsFunc: func(input *simplejson.Json) (*simplejson.Json, error) {
-				schemaLoaderServiceCalled = true
-				return input, nil
-			},
-		}
-
-		importDashboardAPI := New(service, quotaServiceFunc(quotaNotReached), schemaLoaderService, nil, acmock.New().WithDisabled())
+		importDashboardAPI := New(service, quotaServiceFunc(quotaNotReached), nil, acmock.New().WithDisabled())
 		routeRegister := routing.NewRouteRegister()
 		importDashboardAPI.RegisterAPIEndpoints(routeRegister)
 		s := webtest.NewServer(t, routeRegister)
@@ -98,7 +90,6 @@ func TestImportDashboardAPI(t *testing.T) {
 			require.NoError(t, err)
 			require.NoError(t, resp.Body.Close())
 			require.Equal(t, http.StatusOK, resp.StatusCode)
-			require.False(t, schemaLoaderServiceCalled)
 			require.True(t, importDashboardServiceCalled)
 		})
 	})
@@ -112,16 +103,7 @@ func TestImportDashboardAPI(t *testing.T) {
 			},
 		}
 
-		schemaLoaderServiceCalled := false
-		schemaLoaderService := &schemaLoaderServiceMock{
-			enabled: true,
-			dashboardApplyDefaultsFunc: func(input *simplejson.Json) (*simplejson.Json, error) {
-				schemaLoaderServiceCalled = true
-				return input, nil
-			},
-		}
-
-		importDashboardAPI := New(service, quotaServiceFunc(quotaNotReached), schemaLoaderService, nil, acmock.New().WithDisabled())
+		importDashboardAPI := New(service, quotaServiceFunc(quotaNotReached), nil, acmock.New().WithDisabled())
 		routeRegister := routing.NewRouteRegister()
 		importDashboardAPI.RegisterAPIEndpoints(routeRegister)
 		s := webtest.NewServer(t, routeRegister)
@@ -140,15 +122,13 @@ func TestImportDashboardAPI(t *testing.T) {
 			require.NoError(t, err)
 			require.NoError(t, resp.Body.Close())
 			require.Equal(t, http.StatusOK, resp.StatusCode)
-			require.True(t, schemaLoaderServiceCalled)
 			require.True(t, importDashboardServiceCalled)
 		})
 	})
 
 	t.Run("Quota reached", func(t *testing.T) {
 		service := &serviceMock{}
-		schemaLoaderService := &schemaLoaderServiceMock{}
-		importDashboardAPI := New(service, quotaServiceFunc(quotaReached), schemaLoaderService, nil, acmock.New().WithDisabled())
+		importDashboardAPI := New(service, quotaServiceFunc(quotaReached), nil, acmock.New().WithDisabled())
 
 		routeRegister := routing.NewRouteRegister()
 		importDashboardAPI.RegisterAPIEndpoints(routeRegister)
@@ -184,23 +164,6 @@ func (s *serviceMock) ImportDashboard(ctx context.Context, req *dashboardimport.
 	return nil, nil
 }
 
-type schemaLoaderServiceMock struct {
-	enabled                    bool
-	dashboardApplyDefaultsFunc func(input *simplejson.Json) (*simplejson.Json, error)
-}
-
-func (s *schemaLoaderServiceMock) IsDisabled() bool {
-	return !s.enabled
-}
-
-func (s *schemaLoaderServiceMock) DashboardApplyDefaults(input *simplejson.Json) (*simplejson.Json, error) {
-	if s.dashboardApplyDefaultsFunc != nil {
-		return s.dashboardApplyDefaultsFunc(input)
-	}
-
-	return input, nil
-}
-
 func quotaReached(c *models.ReqContext, target string) (bool, error) {
 	return true, nil
 }

pkg/services/dashboardimport/service/service.go

@@ -14,11 +14,10 @@ import (
 	"github.com/grafana/grafana/pkg/services/librarypanels"
 	"github.com/grafana/grafana/pkg/services/plugindashboards"
 	"github.com/grafana/grafana/pkg/services/quota"
-	"github.com/grafana/grafana/pkg/services/schemaloader"
 )
 
 func ProvideService(routeRegister routing.RouteRegister,
-	quotaService *quota.QuotaService, schemaLoaderService *schemaloader.SchemaLoaderService,
+	quotaService *quota.QuotaService,
 	pluginDashboardService plugindashboards.Service, pluginStore plugins.Store,
 	libraryPanelService librarypanels.Service, dashboardService dashboards.DashboardService,
 	ac accesscontrol.AccessControl,
@@ -29,7 +28,7 @@ func ProvideService(routeRegister routing.RouteRegister,
 		libraryPanelService:    libraryPanelService,
 	}
 
-	dashboardImportAPI := api.New(s, quotaService, schemaLoaderService, pluginStore, ac)
+	dashboardImportAPI := api.New(s, quotaService, pluginStore, ac)
 	dashboardImportAPI.RegisterAPIEndpoints(routeRegister)
 
 	return s

pkg/services/schemaloader/schemaloader.go

@@ -1,144 +0,0 @@
-package schemaloader
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"github.com/grafana/grafana"
-	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/schema"
-	"github.com/grafana/grafana/pkg/schema/load"
-	"github.com/grafana/grafana/pkg/services/featuremgmt"
-
-	"github.com/grafana/grafana/pkg/infra/log"
-)
-
-const ServiceName = "SchemaLoader"
-
-var baseLoadPath load.BaseLoadPaths = load.BaseLoadPaths{
-	BaseCueFS:       grafana.CoreSchema,
-	DistPluginCueFS: grafana.PluginSchema,
-}
-
-type RenderUser struct {
-	OrgID   int64
-	UserID  int64
-	OrgRole string
-}
-
-func ProvideService(features featuremgmt.FeatureToggles) (*SchemaLoaderService, error) {
-	dashFam, err := load.BaseDashboardFamily(baseLoadPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load dashboard cue schema from path %q: %w", baseLoadPath, err)
-	}
-	s := &SchemaLoaderService{
-		features:   features,
-		DashFamily: dashFam,
-		log:        log.New("schemaloader"),
-	}
-	return s, nil
-}
-
-type SchemaLoaderService struct {
-	log        log.Logger
-	DashFamily schema.VersionedCueSchema
-	features   featuremgmt.FeatureToggles
-}
-
-func (rs *SchemaLoaderService) IsDisabled() bool {
-	if rs.features == nil {
-		return true
-	}
-	return !rs.features.IsEnabled(featuremgmt.FlagTrimDefaults)
-}
-
-func (rs *SchemaLoaderService) DashboardApplyDefaults(input *simplejson.Json) (*simplejson.Json, error) {
-	val, _ := input.Map()
-	val = removeNils(val)
-	data, _ := json.Marshal(val)
-	dsSchema := schema.Find(rs.DashFamily, schema.Latest())
-	result, err := schema.ApplyDefaults(schema.Resource{Value: data}, dsSchema.CUE())
-	if err != nil {
-		return input, err
-	}
-	output, err := simplejson.NewJson([]byte(result.Value.(string)))
-	if err != nil {
-		return input, err
-	}
-	return output, nil
-}
-
-func (rs *SchemaLoaderService) DashboardTrimDefaults(input simplejson.Json) (simplejson.Json, error) {
-	val, _ := input.Map()
-	val = removeNils(val)
-	data, _ := json.Marshal(val)
-
-	dsSchema, err := schema.SearchAndValidate(rs.DashFamily, string(data))
-	if err != nil {
-		return input, err
-	}
-
-	result, err := schema.TrimDefaults(schema.Resource{Value: data}, dsSchema.CUE())
-	if err != nil {
-		return input, err
-	}
-	output, err := simplejson.NewJson([]byte(result.Value.(string)))
-	if err != nil {
-		return input, err
-	}
-	return *output, nil
-}
-
-func removeNils(initialMap map[string]interface{}) map[string]interface{} {
-	withoutNils := map[string]interface{}{}
-	for key, value := range initialMap {
-		_, ok := value.(map[string]interface{})
-		if ok {
-			value = removeNils(value.(map[string]interface{}))
-			withoutNils[key] = value
-			continue
-		}
-		_, ok = value.([]interface{})
-		if ok {
-			value = removeNilArray(value.([]interface{}))
-			withoutNils[key] = value
-			continue
-		}
-		if value != nil {
-			if val, ok := value.(string); ok {
-				if val == "" {
-					continue
-				}
-			}
-			withoutNils[key] = value
-		}
-	}
-	return withoutNils
-}
-
-func removeNilArray(initialArray []interface{}) []interface{} {
-	withoutNils := []interface{}{}
-	for _, value := range initialArray {
-		_, ok := value.(map[string]interface{})
-		if ok {
-			value = removeNils(value.(map[string]interface{}))
-			withoutNils = append(withoutNils, value)
-			continue
-		}
-		_, ok = value.([]interface{})
-		if ok {
-			value = removeNilArray(value.([]interface{}))
-			withoutNils = append(withoutNils, value)
-			continue
-		}
-		if value != nil {
-			if val, ok := value.(string); ok {
-				if val == "" {
-					continue
-				}
-			}
-			withoutNils = append(withoutNils, value)
-		}
-	}
-	return withoutNils
-}