Introduce coremodels framework (extracted from intent-api) (#47653)

* Copy over most of coremodel from intent-api branch

* Fix import paths

* Fix incorrect provider name

* Add root compgen file, fixup componentroot.yaml

* go mod tidy

* Remove compgen for now

* Add dashboard coremodel

* Remove datasource coremodel, for now

* Tweak comments on dashboard struct model

* devenv: add dashboard testing directly

* Fixup dashboard schema for openness, heatmap

* Update Thema to tip

* Fix wire/registry references

* Fix hclog version
This commit is contained in:
sam boyer
2022-04-14 14:54:35 -04:00
committed by GitHub
parent 758364e78b
commit 2a178bd73c
13 changed files with 890 additions and 44 deletions

View File

@@ -0,0 +1,40 @@
package dashboard
import (
"github.com/grafana/thema"
)
// Coremodel contains the foundational schema declaration for dashboards.
type Coremodel struct {
lin thema.Lineage
}
// Lineage returns the canonical dashboard Lineage.
func (c *Coremodel) Lineage() thema.Lineage {
return c.lin
}
func (c *Coremodel) CurrentSchema() thema.Schema {
sch, err := c.lin.Schema(currentVersion)
if err != nil {
// Only reachable if our own schema currentVersion does not exist, which
// can really only happen transitionally during development
panic(err)
}
return sch
}
func (c *Coremodel) GoType() interface{} {
return &model{}
}
func ProvideCoremodel(lib thema.Library) (*Coremodel, error) {
lin, err := Lineage(lib)
if err != nil {
return nil, err
}
return &Coremodel{
lin: lin,
}, nil
}

View File

@@ -0,0 +1,344 @@
package dashboard
import "github.com/grafana/thema"
thema.#Lineage
name: "dashboard"
seqs: [
{
schemas: [
{ // 0.0
// Unique numeric identifier for the dashboard.
// TODO must isolate or remove identifiers local to a Grafana instance...?
id?: int64
// Unique dashboard identifier that can be generated by anyone. string (8-40)
uid?: string
// Title of dashboard.
title?: string
// Description of dashboard.
description?: string
gnetId?: string
// Tags associated with dashboard.
tags?: [...string]
// Theme of dashboard.
style: *"light" | "dark"
// Timezone of dashboard,
timezone?: *"browser" | "utc" | ""
// Whether a dashboard is editable or not.
editable: bool | *true
// 0 for no shared crosshair or tooltip (default).
// 1 for shared crosshair.
// 2 for shared crosshair AND shared tooltip.
graphTooltip: uint8 & >=0 & <=2 | *0
// Time range for dashboard, e.g. last 6 hours, last 7 days, etc
time?: {
from: string | *"now-6h"
to: string | *"now"
}
// Timepicker metadata.
timepicker?: {
// Whether timepicker is collapsed or not.
collapse: bool | *false
// Whether timepicker is enabled or not.
enable: bool | *true
// Whether timepicker is visible or not.
hidden: bool | *false
// Selectable intervals for auto-refresh.
refresh_intervals: [...string] | *["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"]
}
// Templating.
templating?: list: [...{...}]
// Annotations.
annotations?: list: [...{
builtIn: uint8 | *0
// Datasource to use for annotation.
datasource: string
// Whether annotation is enabled.
enable: bool | *true
// Whether to hide annotation.
hide?: bool | *false
// Annotation icon color.
iconColor?: string
// Name of annotation.
name?: string
type: string | *"dashboard"
// Query for annotation data.
rawQuery?: string
showIn: uint8 | *0
}]
// Auto-refresh interval.
refresh?: string | false
// Version of the JSON schema, incremented each time a Grafana update brings
// changes to said schema.
// FIXME this is the old schema numbering system, and will be replaced by Thema's themaVersion
schemaVersion: uint16 | *33
// Version of the dashboard, incremented each time the dashboard is updated.
version?: uint32
panels?: [...(#Panel | #GraphPanel | #HeatmapPanel | #RowPanel)]
// TODO docs
#FieldColorModeId: "thresholds" | "palette-classic" | "palette-saturated" | "continuous-GrYlRd" | "fixed" @cuetsy(kind="enum")
// TODO docs
#FieldColorSeriesByMode: "min" | "max" | "last" @cuetsy(kind="type")
// TODO docs
#FieldColor: {
// The main color scheme mode
mode: #FieldColorModeId | string
// Stores the fixed color value if mode is fixed
fixedColor?: string
// Some visualizations need to know how to assign a series color from by value color schemes
seriesBy?: #FieldColorSeriesByMode
} @cuetsy(kind="interface")
// TODO docs
#Threshold: {
// TODO docs
// FIXME the corresponding typescript field is required/non-optional, but nulls currently appear here when serializing -Infinity to JSON
value?: number
// TODO docs
color: string
// TODO docs
// TODO are the values here enumerable into a disjunction?
// Some seem to be listed in typescript comment
state?: string
} @cuetsy(kind="interface")
#ThresholdsMode: "absolute" | "percentage" @cuetsy(kind="enum")
#ThresholdsConfig: {
mode: #ThresholdsMode
// Must be sorted by 'value', first value is always -Infinity
steps: [...#Threshold]
} @cuetsy(kind="interface")
// TODO docs
// FIXME this is extremely underspecfied; wasn't obvious which typescript types corresponded to it
#Transformation: {
id: string
options: {...}
}
// Schema for panel targets is specified by datasource
// plugins. We use a placeholder definition, which the Go
// schema loader either left open/as-is with the Base
// variant of the Dashboard and Panel families, or filled
// with types derived from plugins in the Instance variant.
// When working directly from CUE, importers can extend this
// type directly to achieve the same effect.
#Target: {...}
// Dashboard panels. Panels are canonically defined inline
// because they share a version timeline with the dashboard
// schema; they do not evolve independently.
#Panel: {
// The panel plugin type id.
type: !=""
// TODO docs
id?: uint32
// FIXME this almost certainly has to be changed in favor of scuemata versions
pluginVersion?: string
// TODO docs
tags?: [...string]
// Internal - the exact major and minor versions of the panel plugin
// schema. Hidden and therefore not a part of the data model, but
// expected to be filled with panel plugin schema versions so that it's
// possible to figure out which schema version matched on a successful
// unification.
// _pv: { maj: int, min: int }
// The major and minor versions of the panel plugin for this schema.
// TODO 2-tuple list instead of struct?
// panelSchema?: { maj: number, min: number }
panelSchema?: [number, number]
// TODO docs
targets?: [...#Target]
// Panel title.
title?: string
// Description.
description?: string
// Whether to display the panel without a background.
transparent: bool | *false
// The datasource used in all targets.
datasource?: {
type?: string
uid?: string
}
// Grid position.
gridPos?: {
// Panel
h: uint32 & >0 | *9
// Panel
w: uint32 & >0 & <=24 | *12
// Panel x
x: uint32 & >=0 & <24 | *0
// Panel y
y: uint32 & >=0 | *0
// true if fixed
static?: bool
}
// Panel links.
// FIXME this is temporarily specified as a closed list so
// that validation will pass when no links are present, but
// to force a failure as soon as it's checked against there
// being anything in the list so it can be fixed in
// accordance with that object
links?: []
// Name of template variable to repeat for.
repeat?: string
// Direction to repeat in if 'repeat' is set.
// "h" for horizontal, "v" for vertical.
repeatDirection: *"h" | "v"
// TODO docs
maxDataPoints?: number
// TODO docs
thresholds?: [...]
// TODO docs
timeRegions?: [...]
transformations: [...#Transformation]
// TODO docs
// TODO tighter constraint
interval?: string
// TODO docs
// TODO tighter constraint
timeFrom?: string
// TODO docs
// TODO tighter constraint
timeShift?: string
// options is specified by the PanelOptions field in panel
// plugin schemas.
options: {...}
fieldConfig: {
defaults: {
// The display value for this field. This supports template variables blank is auto
displayName?: string
// This can be used by data sources that return and explicit naming structure for values and labels
// When this property is configured, this value is used rather than the default naming strategy.
displayNameFromDS?: string
// Human readable field metadata
description?: string
// An explict path to the field in the datasource. When the frame meta includes a path,
// This will default to `${frame.meta.path}/${field.name}
//
// When defined, this value can be used as an identifier within the datasource scope, and
// may be used to update the results
path?: string
// True if data source can write a value to the path. Auth/authz are supported separately
writeable?: bool
// True if data source field supports ad-hoc filters
filterable?: bool
// Numeric Options
unit?: string
// Significant digits (for display)
decimals?: number
min?: number
max?: number
// Convert input values into a display string
//
// TODO this one corresponds to a complex type with
// generics on the typescript side. Ouch. Will
// either need special care, or we'll just need to
// accept a very loosely specified schema. It's very
// unlikely we'll be able to translate cue to
// typescript generics in the general case, though
// this particular one *may* be able to work.
mappings?: [...{...}]
// Map numeric values to states
thresholds?: #ThresholdsConfig
// // Map values to a display color
color?: #FieldColor
// // Used when reducing field values
// nullValueMode?: NullValueMode
// // The behavior when clicking on a result
links?: [...]
// Alternative to empty string
noValue?: string
// custom is specified by the PanelFieldConfig field
// in panel plugin schemas.
custom?: {...}
}
overrides: [...{
matcher: {
id: string | *""
options?: _
}
properties: [...{
id: string | *""
value?: _
}]
}]
}
}
// Row panel
#RowPanel: {
type: "row"
collapsed: bool | *false
title?: string
// Name of default datasource.
datasource?: string
gridPos?: {
// Panel
h: uint32 & >0 | *9
// Panel
w: uint32 & >0 & <=24 | *12
// Panel x
x: uint32 & >=0 & <24 | *0
// Panel y
y: uint32 & >=0 | *0
// true if fixed
static?: bool
}
id: uint32
panels: [...(#Panel | #GraphPanel | #HeatmapPanel)]
// Name of template variable to repeat for.
repeat?: string
}
// Support for legacy graph and heatmap panels.
#GraphPanel: {
...
type: "graph"
}
#HeatmapPanel: {
...
type: "heatmap"
}
}
]
}
]

View File

@@ -0,0 +1,89 @@
package dashboard
import (
"embed"
"path/filepath"
"github.com/grafana/thema"
"github.com/grafana/grafana/pkg/cuectx"
)
var (
//go:embed lineage.cue
cueFS embed.FS
// TODO: this should be generated by Thema.
currentVersion = thema.SV(0, 0)
)
// Lineage returns the Thema lineage representing Grafana dashboards. The
// lineage is the canonical specification of the current datasource schema, all
// prior schema versions, and the mappings that allow migration between schema
// versions.
//
// This is the base variant of the schema, which does not include any composed
// plugin schemas.
func Lineage(lib thema.Library, opts ...thema.BindOption) (thema.Lineage, error) {
return cuectx.LoadGrafanaInstancesWithThema(filepath.Join("pkg", "coremodel", "dashboard"), cueFS, lib, opts...)
}
// Model is a dummy struct stand-in for dashboards.
//
// It exists solely to trick compgen into accepting the dashboard coremodel as valid.
type Model struct{}
// model is a hacky Go struct representing a dashboard.
//
// This exists solely because the coremodel framework enforces that there is a Go struct to which
// all valid Thema schema instances can be assigned, per Thema's assignability checker. See
// https://github.com/grafana/thema/blob/main/docs/invariants.md#go-assignability for rules.
//
// DO NOT RELY ON THIS FOR ANYTHING REAL. It is unclear whether we will ever attempt to have a correct, complete
// Go struct representation of dashboards, let alone compress it into a single struct.
type model struct {
Title string `json:"title"`
Description string `json:"description"`
GnetId string `json:"gnetId"`
Tags []string `json:"tags"`
Style string `json:"style"`
Timezone string `json:"timezone"`
Editable bool `json:"editable"`
GraphTooltip uint8 `json:"graphTooltip"`
Time struct {
From string `json:"from"`
To string `json:"to"`
} `json:"time"`
Timepicker struct {
Collapse bool `json:"collapse"`
Enable bool `json:"enable"`
Hidden bool `json:"hidden"`
RefreshIntervals []string `json:"refresh_intervals"`
} `json:"timepicker"`
Templating struct {
List []interface{} `json:"list"`
} `json:"templating"`
Annotations struct {
List []struct {
Name string `json:"name"`
Type string `json:"type"`
BuiltIn uint8 `json:"builtIn"`
Datasource string `json:"datasource"`
Enable bool `json:"enable"`
Hide bool `json:"hide,omitempty"`
IconColor string `json:"iconColor"`
RawQuery string `json:"rawQuery,omitempty"`
ShowIn int `json:"showIn"`
} `json:"list"`
} `json:"annotations"`
Refresh interface{} `json:"refresh"` // (bool|string)
SchemaVersion int `json:"schemaVersion"`
Panels []interface{} `json:"panels"`
////
Uid string `json:"uid"`
// OrgId int64 `json:"orgId"`
Id int64 `json:"id,omitempty"`
Version int `json:"version"`
}

116
pkg/cuectx/ctx.go Normal file
View File

@@ -0,0 +1,116 @@
// Package cuectx provides a single, central CUE context (runtime) and Thema
// library that can be used uniformly across Grafana, and related helper
// functions for loading Thema lineages.
package cuectx
import (
"io"
"io/fs"
"path/filepath"
"testing/fstest"
"cuelang.org/go/cue"
"cuelang.org/go/cue/cuecontext"
"github.com/grafana/thema"
"github.com/grafana/thema/kernel"
"github.com/grafana/thema/load"
)
var ctx *cue.Context = cuecontext.New()
var lib thema.Library = thema.NewLibrary(ctx)
// ProvideCUEContext is a wire service provider of a central cue.Context.
func ProvideCUEContext() *cue.Context {
return ctx
}
// ProvideThemaLibrary is a wire service provider of a central thema.Library.
func ProvideThemaLibrary() thema.Library {
return lib
}
// JSONtoCUE attempts to decode the given []byte into a cue.Value, relying on
// the central Grafana cue.Context provided in this package.
//
// The provided path argument determines the name given to the input bytes if
// later CUE operations (e.g. Thema validation) produce errors related to the
// returned cue.Value.
//
// This is a convenience function for one-off JSON decoding. It's wasteful to
// call it repeatedly. Most use cases use cases should probably prefer making
// their own Thema/CUE decoders.
func JSONtoCUE(path string, b []byte) (cue.Value, error) {
return kernel.NewJSONDecoder(path)(ctx, b)
}
// LoadGrafanaInstancesWithThema loads CUE files containing a lineage
// representing some Grafana core model schema. It is expected to be used when
// implementing a thema.LineageFactory.
//
// This function primarily juggles paths to make CUE's loader happy. Provide the
// path from the grafana root to the directory containing the lineage.cue. The
// lineage.cue file must be the sole contents of the provided fs.FS.
//
// More details on underlying behavior can be found in the docs for github.com/grafana/thema/load.InstancesWithThema.
func LoadGrafanaInstancesWithThema(
path string,
cueFS fs.FS,
lib thema.Library,
opts ...thema.BindOption,
) (thema.Lineage, error) {
prefix := filepath.FromSlash(path)
fs, err := prefixWithGrafanaCUE(prefix, cueFS)
if err != nil {
return nil, err
}
inst, err := load.InstancesWithThema(fs, prefix)
// Need to trick loading by creating the embedded file and
// making it look like a module in the root dir.
if err != nil {
return nil, err
}
val := lib.Context().BuildInstance(inst)
lin, err := thema.BindLineage(val, lib, opts...)
if err != nil {
return nil, err
}
return lin, nil
}
func prefixWithGrafanaCUE(prefix string, inputfs fs.FS) (fs.FS, error) {
m := fstest.MapFS{
filepath.Join("cue.mod", "module.cue"): &fstest.MapFile{Data: []byte(`module: "github.com/grafana/grafana"`)},
}
prefix = filepath.FromSlash(prefix)
err := fs.WalkDir(inputfs, ".", (func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
f, err := inputfs.Open(path)
if err != nil {
return err
}
defer f.Close() // nolint: errcheck
b, err := io.ReadAll(f)
if err != nil {
return err
}
m[filepath.Join(prefix, path)] = &fstest.MapFile{Data: b}
return nil
}))
return m, err
}

View File

@@ -0,0 +1 @@
package coremodel

View File

@@ -0,0 +1,25 @@
package coremodel
import (
"github.com/grafana/thema"
)
// Interface is the primary coremodel interface that must be implemented by all
// Grafana coremodels. A coremodel is the foundational, canonical schema for
// some known-at-compile-time Grafana object.
//
// Currently, all Coremodels are expressed as Thema lineages.
type Interface interface {
// Lineage should return the canonical Thema lineage for the coremodel.
Lineage() thema.Lineage
// CurrentSchema should return the schema of the version that the Grafana backend
// is currently written against. (While Grafana can accept data from all
// older versions of the Thema schema, backend Go code is written against a
// single version for simplicity)
CurrentSchema() thema.Schema
// GoType should return a pointer to the Go struct type that corresponds to
// the Current() schema.
GoType() interface{}
}

View File

@@ -0,0 +1,78 @@
package coremodel
import (
"errors"
"fmt"
"sync"
"github.com/grafana/thema"
)
var (
// ErrModelAlreadyRegistered is returned when trying to register duplicate model to Registry.
ErrModelAlreadyRegistered = errors.New("error registering duplicate model")
)
// Registry is a registry of coremodel instances.
type Registry struct {
lock sync.RWMutex
models []Interface
modelIdx map[string]Interface
}
// NewRegistry returns a new Registry with the provided coremodel instances.
func NewRegistry(models ...Interface) (*Registry, error) {
r := &Registry{
models: make([]Interface, 0, len(models)),
modelIdx: make(map[string]Interface, len(models)),
}
if err := r.addModels(models); err != nil {
return nil, err
}
return r, nil
}
// Register adds coremodels to the Registry.
func (r *Registry) Register(models ...Interface) error {
return r.addModels(models)
}
// List returns all coremodels registered in this Registry.
func (r *Registry) List() []Interface {
r.lock.RLock()
defer r.lock.RUnlock()
return r.models
}
func (r *Registry) addModels(models []Interface) error {
r.lock.Lock()
defer r.lock.Unlock()
// Update model index and return an error if trying to register a duplicate.
for _, m := range models {
k := m.Lineage().Name()
// Ensure assignability first. TODO will this blow up for dashboards?
if err := thema.AssignableTo(m.CurrentSchema(), m.GoType()); err != nil {
return fmt.Errorf("%s schema version %v not assignable to provided Go type: %w", k, m.CurrentSchema().Version(), err)
}
if _, ok := r.modelIdx[k]; ok {
return ErrModelAlreadyRegistered
}
r.modelIdx[k] = m
}
// Remake model list.
// TODO: this can be more performant (proper resizing, maybe single loop with index building, etc.).
r.models = r.models[:0]
for _, m := range r.modelIdx {
r.models = append(r.models, m)
}
return nil
}

View File

@@ -0,0 +1,17 @@
package staticregistry
import (
"github.com/grafana/grafana/pkg/coremodel/dashboard"
"github.com/grafana/grafana/pkg/framework/coremodel"
)
// ProvideRegistry provides a simple static Registry for coremodels.
// Coremodels have to be manually added.
// TODO dynamism
func ProvideRegistry(
dashboard *dashboard.Coremodel,
) (*coremodel.Registry, error) {
return coremodel.NewRegistry(
dashboard,
)
}

View File

@@ -7,11 +7,15 @@ import (
"github.com/google/wire"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana/pkg/api"
"github.com/grafana/grafana/pkg/api/avatar"
"github.com/grafana/grafana/pkg/api/routing"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/coremodel/dashboard"
"github.com/grafana/grafana/pkg/cuectx"
"github.com/grafana/grafana/pkg/expr"
cmreg "github.com/grafana/grafana/pkg/framework/coremodel/staticregistry"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/httpclient/httpclientprovider"
"github.com/grafana/grafana/pkg/infra/kvstore"
@@ -239,6 +243,10 @@ var wireBasicSet = wire.NewSet(
avatar.ProvideAvatarCacheServer,
authproxy.ProvideAuthProxy,
statscollector.ProvideService,
dashboard.ProvideCoremodel,
cmreg.ProvideRegistry,
cuectx.ProvideCUEContext,
cuectx.ProvideThemaLibrary,
)
var wireSet = wire.NewSet(