IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Scuemata: Add MergeFS implementation to enable iteration for multiple filesystems (#33989)

Browse Source 
* Add MergeFS implementation to enable iteration to multiple filesystems

* Fix linting

* Fix misplaced close file

* Move mergefs functionality into tests

* Fix linting

* Add mergefs test

* Dummy commit - remove

* Test fixes - renaming

* Fix ReadDir

# Keeps first filesystem's overridden file

* Fixes according to reviewer's comments

* Remove dummy test

* Remove walkdir

* Small test refactoring
This commit is contained in:
Dimitris Sotirakis 2021-05-18 09:22:31 +03:00 committed by GitHub
parent 3339d13a45
commit b9eab37149
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 187 additions and 474 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
pkg/cmd/grafana-cli/commands/mergefs.go Normal file
View File

@ -0,0 +1,92 @@
package commands
import (
"errors"
"io/fs"
"os"
"sort"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
)
// MergeFS contains a slice of different filesystems that can be merged together
type MergeFS struct {
filesystems []fs.FS
}
// Merge filesystems
func Merge(filesystems ...fs.FS) fs.FS {
return MergeFS{filesystems: filesystems}
}
// Open opens the named file.
func (mfs MergeFS) Open(name string) (fs.File, error) {
for _, filesystem := range mfs.filesystems {
file, err := filesystem.Open(name)
if err == nil {
return file, nil
}
}
return nil, os.ErrNotExist
}
// ReadDir reads from the directory, and produces a DirEntry array of different
// directories.
//
// It iterates through all different filesystems that exist in the mfs MergeFS
// filesystem slice and it identifies overlapping directories that exist in different
// filesystems
func (mfs MergeFS) ReadDir(name string) ([]fs.DirEntry, error) {
dirsMap := make(map[string]fs.DirEntry)
for _, filesystem := range mfs.filesystems {
if fsys, ok := filesystem.(fs.ReadDirFS); ok {
dir, err := fsys.ReadDir(name)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
logger.Debugf("directory in filepath %s was not found in filesystem", name)
continue
}
return nil, err
}
for _, v := range dir {
if _, ok := dirsMap[v.Name()]; !ok {
dirsMap[v.Name()] = v
}
}
continue
}
file, err := filesystem.Open(name)
if err != nil {
logger.Debugf("filepath %s was not found in filesystem", name)
continue
}
dir, ok := file.(fs.ReadDirFile)
if !ok {
return nil, &fs.PathError{Op: "readdir", Path: name, Err: errors.New("not implemented")}
}
fsDirs, err := dir.ReadDir(-1)
if err != nil {
return nil, err
}
sort.Slice(fsDirs, func(i, j int) bool { return fsDirs[i].Name() < fsDirs[j].Name() })
for _, v := range fsDirs {
if _, ok := dirsMap[v.Name()]; !ok {
dirsMap[v.Name()] = v
}
}
if err := file.Close(); err != nil {
logger.Error("failed to close file", "err", err)
}
}
dirs := make([]fs.DirEntry, 0, len(dirsMap))
for _, value := range dirsMap {
dirs = append(dirs, value)
}
sort.Slice(dirs, func(i, j int) bool { return dirs[i].Name() < dirs[j].Name() })
return dirs, nil
}

68
pkg/cmd/grafana-cli/commands/mergefs_test.go Normal file
View File

@ -0,0 +1,68 @@
package commands
import (
"io/fs"
"os"
"path/filepath"
"testing"
"testing/fstest"
"github.com/stretchr/testify/require"
)
func TestMergeFS(t *testing.T) {
var filePaths = []struct {
path string
dirArrayLength int
child string
}{
// MapFS takes in account the current directory in addition to all included directories and produces a "" dir
{"a", 1, "z"},
{"a/z", 1, "bar.cue"},
{"b", 1, "z"},
{"b/z", 1, "foo.cue"},
}
tempDir := os.DirFS(filepath.Join("testdata", "mergefs"))
a := fstest.MapFS{
"a": &fstest.MapFile{Mode: fs.ModeDir},
"a/z": &fstest.MapFile{Mode: fs.ModeDir},
"a/z/bar.cue": &fstest.MapFile{Data: []byte("bar")},
}
filesystem := Merge(tempDir, a)
t.Run("testing mergefs.ReadDir", func(t *testing.T) {
for _, fp := range filePaths {
t.Run("testing path: "+fp.path, func(t *testing.T) {
dirs, err := fs.ReadDir(filesystem, fp.path)
require.NoError(t, err)
require.Len(t, dirs, fp.dirArrayLength)
for i := 0; i < len(dirs); i++ {
require.Equal(t, dirs[i].Name(), fp.child)
}
})
}
})
t.Run("testing mergefs.Open", func(t *testing.T) {
data := make([]byte, 3)
file, err := filesystem.Open("a/z/bar.cue")
require.NoError(t, err)
_, err = file.Read(data)
require.NoError(t, err)
require.Equal(t, "bar", string(data))
file, err = filesystem.Open("b/z/foo.cue")
require.NoError(t, err)
_, err = file.Read(data)
require.NoError(t, err)
require.Equal(t, "foo", string(data))
err = file.Close()
require.NoError(t, err)
})
}

40
pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go
View File

@ -2,21 +2,21 @@ package commands
import (
"os"
"path/filepath"
"testing"
"testing/fstest"
"github.com/grafana/grafana/pkg/schema/load"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var defaultBaseLoadPaths = load.GetDefaultLoadPaths()
func TestValidateScuemataBasics(t *testing.T) {
t.Run("Testing scuemata validity with valid cue schemas", func(t *testing.T) {
tempDir := os.DirFS(filepath.Join("testdata", "valid_scuemata"))
var baseLoadPaths = load.BaseLoadPaths{
BaseCueFS: tempDir,
DistPluginCueFS: load.GetDefaultLoadPaths().DistPluginCueFS,
BaseCueFS: defaultBaseLoadPaths.BaseCueFS,
DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
}
err := validate(baseLoadPaths, load.BaseDashboardFamily)
@ -27,26 +27,38 @@ func TestValidateScuemataBasics(t *testing.T) {
})
t.Run("Testing scuemata validity with invalid cue schemas - family missing", func(t *testing.T) {
tempDir := os.DirFS(filepath.Join("testdata", "invalid_scuemata_missing_family"))
genCue, err := os.ReadFile("testdata/missing_family_gen.cue")
require.NoError(t, err)
fs := fstest.MapFS{
"cue/data/gen.cue": &fstest.MapFile{Data: genCue},
}
mergedFS := Merge(fs, defaultBaseLoadPaths.BaseCueFS)
var baseLoadPaths = load.BaseLoadPaths{
BaseCueFS: tempDir,
DistPluginCueFS: load.GetDefaultLoadPaths().DistPluginCueFS,
BaseCueFS: mergedFS,
DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
}
err := validate(baseLoadPaths, load.BaseDashboardFamily)
err = validate(baseLoadPaths, load.BaseDashboardFamily)
assert.EqualError(t, err, "error while loading dashboard scuemata, err: dashboard schema family did not exist at expected path in expected file")
})
t.Run("Testing scuemata validity with invalid cue schemas - panel missing", func(t *testing.T) {
tempDir := os.DirFS(filepath.Join("testdata", "invalid_scuemata_missing_panel"))
t.Run("Testing scuemata validity with invalid cue schemas - panel missing ", func(t *testing.T) {
genCue, err := os.ReadFile("testdata/missing_panel_gen.cue")
require.NoError(t, err)
fs := fstest.MapFS{
"cue/data/gen.cue": &fstest.MapFile{Data: genCue},
}
mergedFS := Merge(fs, defaultBaseLoadPaths.BaseCueFS)
var baseLoadPaths = load.BaseLoadPaths{
BaseCueFS: tempDir,
DistPluginCueFS: load.GetDefaultLoadPaths().DistPluginCueFS,
BaseCueFS: mergedFS,
DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
}
err := validate(baseLoadPaths, load.BaseDashboardFamily)
err = validate(baseLoadPaths, load.BaseDashboardFamily)
require.NoError(t, err, "error while loading base dashboard scuemata")
err = validate(baseLoadPaths, load.DistDashboardFamily)

22
pkg/cmd/grafana-cli/commands/testdata/invalid_scuemata_missing_family/cue/scuemata/panel-plugin.cue vendored
View File

@ -1,22 +0,0 @@
package scuemata
// Definition of the shape of a panel plugin's schema declarations in its
// schema.cue file.
//
// Note that these keys do not appear directly in any real JSON artifact;
// rather, they are composed into panel structures as they are defined within
// the larger Dashboard schema.
#PanelSchema: {
PanelOptions: {...}
PanelFieldConfig: {...}
...
}
// A lineage of panel schema
#PanelLineage: [#PanelSchema, ...#PanelSchema]
// Panel plugin-specific Family
#PanelFamily: {
lineages: [#PanelLineage, ...#PanelLineage]
migrations: [...#Migration]
}

60
pkg/cmd/grafana-cli/commands/testdata/invalid_scuemata_missing_family/cue/scuemata/scuemata.cue vendored
View File

@ -1,60 +0,0 @@
package scuemata
// A family is a collection of schemas that specify a single kind of object,
// allowing evolution of the canonical schema for that kind of object over time.
//
// The schemas are organized into a list of Lineages, which are themselves ordered
// lists of schemas where each schema with its predecessor in the lineage.
//
// If it is desired to define a schema with a breaking schema relative to its
// predecessors, a new Lineage must be created, as well as a Migration that defines
// a mapping to the new schema from the latest schema in prior Lineage.
//
// The version number of a schema is not controlled by the schema itself, but by
// its position in the list of lineages - e.g., 0.0 corresponds to the first
// schema in the first lineage.
#Family: {
lineages: [#Lineage, ...#Lineage]
migrations: [...#Migration]
let lseq = lineages[len(lineages)-1]
latest: #LastSchema & {_p: lseq}
}
// A Lineage is a non-empty list containing an ordered series of schemas that
// all describe a single kind of object, where each schema is backwards
// compatible with its predecessor.
#Lineage: [{...}, ...{...}]
#LastSchema: {
_p: #Lineage
_p[len(_p)-1]
}
// A Migration defines a relation between two schemas, "_from" and "_to". The
// relation expresses any complex mappings that must be performed to
// transform an input artifact valid with respect to the _from schema, into
// an artifact valid with respect to the _to schema. This is accomplished
// in two stages:
// 1. A Migration is initially defined by passing in schemas for _from and _to,
// and mappings that translate _from to _to are defined in _rel.
// 2. A concrete object may then be unified with _to, resulting in its values
// being mapped onto "result" by way of _rel.
//
// This is the absolute simplest possible definition of a Migration. It's
// incumbent on the implementor to manually ensure the correctness and
// completeness of the mapping. The primary value in defining such a generic
// structure is to allow comparably generic logic for migrating concrete
// artifacts through schema changes.
//
// If _to isn't backwards compatible (accretion-only) with _from, then _rel must
// explicitly enumerate every field in _from and map it to a field in _to, even
// if they're identical. This is laborious for anything outside trivially tiny
// schema. We'll want to eventually add helpers for whitelisting or blacklisting
// of paths in _from, so that migrations of larger schema can focus narrowly on
// the points of actual change.
#Migration: {
from: {...}
to: {...}
rel: {...}
result: to & rel
}

22
pkg/cmd/grafana-cli/commands/testdata/invalid_scuemata_missing_panel/cue/scuemata/panel-plugin.cue vendored
View File

@ -1,22 +0,0 @@
package scuemata
// Definition of the shape of a panel plugin's schema declarations in its
// schema.cue file.
//
// Note that these keys do not appear directly in any real JSON artifact;
// rather, they are composed into panel structures as they are defined within
// the larger Dashboard schema.
#PanelSchema: {
PanelOptions: {...}
PanelFieldConfig?: {...}
...
}
// A lineage of panel schema
#PanelLineage: [#PanelSchema, ...#PanelSchema]
// Panel plugin-specific Family
#PanelFamily: {
lineages: [#PanelLineage, ...#PanelLineage]
migrations: [...#Migration]
}

60
pkg/cmd/grafana-cli/commands/testdata/invalid_scuemata_missing_panel/cue/scuemata/scuemata.cue vendored
View File

@ -1,60 +0,0 @@
package scuemata
// A family is a collection of schemas that specify a single kind of object,
// allowing evolution of the canonical schema for that kind of object over time.
//
// The schemas are organized into a list of Lineages, which are themselves ordered
// lists of schemas where each schema with its predecessor in the lineage.
//
// If it is desired to define a schema with a breaking schema relative to its
// predecessors, a new Lineage must be created, as well as a Migration that defines
// a mapping to the new schema from the latest schema in prior Lineage.
//
// The version number of a schema is not controlled by the schema itself, but by
// its position in the list of lineages - e.g., 0.0 corresponds to the first
// schema in the first lineage.
#Family: {
lineages: [#Lineage, ...#Lineage]
migrations: [...#Migration]
let lseq = lineages[len(lineages)-1]
latest: #LastSchema & {_p: lseq}
}
// A Lineage is a non-empty list containing an ordered series of schemas that
// all describe a single kind of object, where each schema is backwards
// compatible with its predecessor.
#Lineage: [{...}, ...{...}]
#LastSchema: {
_p: #Lineage
_p[len(_p)-1]
}
// A Migration defines a relation between two schemas, "_from" and "_to". The
// relation expresses any complex mappings that must be performed to
// transform an input artifact valid with respect to the _from schema, into
// an artifact valid with respect to the _to schema. This is accomplished
// in two stages:
// 1. A Migration is initially defined by passing in schemas for _from and _to,
// and mappings that translate _from to _to are defined in _rel.
// 2. A concrete object may then be unified with _to, resulting in its values
// being mapped onto "result" by way of _rel.
//
// This is the absolute simplest possible definition of a Migration. It's
// incumbent on the implementor to manually ensure the correctness and
// completeness of the mapping. The primary value in defining such a generic
// structure is to allow comparably generic logic for migrating concrete
// artifacts through schema changes.
//
// If _to isn't backwards compatible (accretion-only) with _from, then _rel must
// explicitly enumerate every field in _from and map it to a field in _to, even
// if they're identical. This is laborious for anything outside trivially tiny
// schema. We'll want to eventually add helpers for whitelisting or blacklisting
// of paths in _from, so that migrations of larger schema can focus narrowly on
// the points of actual change.
#Migration: {
from: {...}
to: {...}
rel: {...}
result: to & rel
}

1
pkg/cmd/grafana-cli/commands/testdata/mergefs/b/z/foo.cue vendored Normal file
View File

@ -0,0 +1 @@
foo

0
pkg/cmd/grafana-cli/commands/testdata/invalid_scuemata_missing_family/cue/data/gen.cue → pkg/cmd/grafana-cli/commands/testdata/missing_family_gen.cue vendored
View File

0
pkg/cmd/grafana-cli/commands/testdata/invalid_scuemata_missing_panel/cue/data/gen.cue → pkg/cmd/grafana-cli/commands/testdata/missing_panel_gen.cue vendored
View File

214
pkg/cmd/grafana-cli/commands/testdata/valid_scuemata/cue/data/gen.cue vendored
View File

@ -1,214 +0,0 @@
package grafanaschema
import "github.com/grafana/grafana/cue/scuemata"
Family: scuemata.#Family & {
lineages: [
[
{ // 0.0
// Unique numeric identifier for the dashboard.
// TODO must isolate or remove identifiers local to a Grafana instance...?
id?: number
// Unique dashboard identifier that can be generated by anyone. string (8-40)
uid: string
// Title of dashboard.
title?: string
// Description of dashboard.
description?: string
gnetId?: string
// Tags associated with dashboard.
tags?: [...string]
// Theme of dashboard.
style: *"light" | "dark"
// Timezone of dashboard,
timezone?: *"browser" | "utc"
// Whether a dashboard is editable or not.
editable: bool | *true
// 0 for no shared crosshair or tooltip (default).
// 1 for shared crosshair.
// 2 for shared crosshair AND shared tooltip.
graphTooltip: >=0 & <=2 | *0
// Time range for dashboard, e.g. last 6 hours, last 7 days, etc
time?: {
from: string | *"now-6h"
to: string | *"now"
}
// Timepicker metadata.
timepicker?: {
// Whether timepicker is collapsed or not.
collapse: bool | *false
// Whether timepicker is enabled or not.
enable: bool | *true
// Whether timepicker is visible or not.
hidden: bool | *false
// Selectable intervals for auto-refresh.
refresh_intervals: [...string] | *["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"]
}
// Templating.
templating?: list: [...{...}]
// Annotations.
annotations?: list: [...{
builtIn: number | *0
// Datasource to use for annotation.
datasource: string
// Whether annotation is enabled.
enable?: bool | *true
// Whether to hide annotation.
hide?: bool | *false
// Annotation icon color.
iconColor?: string
// Name of annotation.
name?: string
type: string | *"dashboard"
// Query for annotation data.
rawQuery?: string
showIn: number | *0
}]
// Auto-refresh interval.
refresh?: string
// Version of the JSON schema, incremented each time a Grafana update brings
// changes to said schema.
schemaVersion: number | *25
// Version of the dashboard, incremented each time the dashboard is updated.
version?: number
panels?: [...#Panel]
// Dashboard panels. Panels are canonically defined inline
// because they share a version timeline with the dashboard
// schema; they do not vary independently. We create a separate,
// synthetic Family to represent them in Go, for ease of generating
// e.g. JSON Schema.
#Panel: {
...
// The panel plugin type id.
type: !=""
// Internal - the exact major and minor versions of the panel plugin
// schema. Hidden and therefore not a part of the data model, but
// expected to be filled with panel plugin schema versions so that it's
// possible to figure out which schema version matched on a successful
// unification.
// _pv: { maj: int, min: int }
// The major and minor versions of the panel plugin for this schema.
// TODO 2-tuple list instead of struct?
panelSchema?: { maj: number, min: number }
// Panel title.
title?: string
// Description.
description?: string
// Whether to display the panel without a background.
transparent: bool | *false
// Name of default datasource.
datasource?: string
// Grid position.
gridPos?: {
// Panel
h: number & >0 | *9
// Panel
w: number & >0 & <=24 | *12
// Panel x
x: number & >=0 & <24 | *0
// Panel y
y: number & >=0 | *0
// true if fixed
static?: bool
}
// Panel links.
// links?: [..._panelLink]
// Name of template variable to repeat for.
repeat?: string
// Direction to repeat in if 'repeat' is set.
// "h" for horizontal, "v" for vertical.
repeatDirection: *"h" | "v"
// Schema for panel targets is specified by datasource
// plugins. We use a placeholder definition, which the Go
// schema loader either left open/as-is with the Base
// variant of the Dashboard and Panel families, or filled
// with types derived from plugins in the Instance variant.
// When working directly from CUE, importers can extend this
// type directly to achieve the same effect.
targets?: [...{...}]
// The values depend on panel type
options: {...}
fieldConfig: {
defaults: {
// The display value for this field. This supports template variables blank is auto
displayName?: string
// This can be used by data sources that return and explicit naming structure for values and labels
// When this property is configured, this value is used rather than the default naming strategy.
displayNameFromDS?: string
// Human readable field metadata
description?: string
// An explict path to the field in the datasource. When the frame meta includes a path,
// This will default to `${frame.meta.path}/${field.name}
//
// When defined, this value can be used as an identifier within the datasource scope, and
// may be used to update the results
path?: string
// True if data source can write a value to the path. Auth/authz are supported separately
writeable?: bool
// True if data source field supports ad-hoc filters
filterable?: bool
// Numeric Options
unit?: string
// Significant digits (for display)
decimals?: number
min?: number
max?: number
// // Convert input values into a display string
// mappings?: ValueMapping[];
// // Map numeric values to states
// thresholds?: ThresholdsConfig;
// // Map values to a display color
// color?: FieldColor;
// // Used when reducing field values
// nullValueMode?: NullValueMode;
// // The behavior when clicking on a result
// links?: DataLink[];
// Alternative to empty string
noValue?: string
// Can always exist. Valid fields within this are
// defined by the panel plugin - that's the
// PanelFieldConfig that comes from the plugin.
custom?: {...}
}
overrides: [...{
matcher: {
id: string | *""
options?: _
}
properties: [...{
id: string | *""
value?: _
}]
}]
}
}
}
]
]
}
#Latest: {
#Dashboard: Family.latest
#Panel: Family.latest._Panel
}

22
pkg/cmd/grafana-cli/commands/testdata/valid_scuemata/cue/scuemata/panel-plugin.cue vendored
View File

@ -1,22 +0,0 @@
package scuemata
// Definition of the shape of a panel plugin's schema declarations in its
// schema.cue file.
//
// Note that these keys do not appear directly in any real JSON artifact;
// rather, they are composed into panel structures as they are defined within
// the larger Dashboard schema.
#PanelSchema: {
PanelOptions: {...}
PanelFieldConfig?: {...}
...
}
// A lineage of panel schema
#PanelLineage: [#PanelSchema, ...#PanelSchema]
// Panel plugin-specific Family
#PanelFamily: {
lineages: [#PanelLineage, ...#PanelLineage]
migrations: [...#Migration]
}

60
pkg/cmd/grafana-cli/commands/testdata/valid_scuemata/cue/scuemata/scuemata.cue vendored
View File

@ -1,60 +0,0 @@
package scuemata
// A family is a collection of schemas that specify a single kind of object,
// allowing evolution of the canonical schema for that kind of object over time.
//
// The schemas are organized into a list of Lineages, which are themselves ordered
// lists of schemas where each schema with its predecessor in the lineage.
//
// If it is desired to define a schema with a breaking schema relative to its
// predecessors, a new Lineage must be created, as well as a Migration that defines
// a mapping to the new schema from the latest schema in prior Lineage.
//
// The version number of a schema is not controlled by the schema itself, but by
// its position in the list of lineages - e.g., 0.0 corresponds to the first
// schema in the first lineage.
#Family: {
lineages: [#Lineage, ...#Lineage]
migrations: [...#Migration]
let lseq = lineages[len(lineages)-1]
latest: #LastSchema & {_p: lseq}
}
// A Lineage is a non-empty list containing an ordered series of schemas that
// all describe a single kind of object, where each schema is backwards
// compatible with its predecessor.
#Lineage: [{...}, ...{...}]
#LastSchema: {
_p: #Lineage
_p[len(_p)-1]
}
// A Migration defines a relation between two schemas, "_from" and "_to". The
// relation expresses any complex mappings that must be performed to
// transform an input artifact valid with respect to the _from schema, into
// an artifact valid with respect to the _to schema. This is accomplished
// in two stages:
// 1. A Migration is initially defined by passing in schemas for _from and _to,
// and mappings that translate _from to _to are defined in _rel.
// 2. A concrete object may then be unified with _to, resulting in its values
// being mapped onto "result" by way of _rel.
//
// This is the absolute simplest possible definition of a Migration. It's
// incumbent on the implementor to manually ensure the correctness and
// completeness of the mapping. The primary value in defining such a generic
// structure is to allow comparably generic logic for migrating concrete
// artifacts through schema changes.
//
// If _to isn't backwards compatible (accretion-only) with _from, then _rel must
// explicitly enumerate every field in _from and map it to a field in _to, even
// if they're identical. This is laborious for anything outside trivially tiny
// schema. We'll want to eventually add helpers for whitelisting or blacklisting
// of paths in _from, so that migrations of larger schema can focus narrowly on
// the points of actual change.
#Migration: {
from: {...}
to: {...}
rel: {...}
result: to & rel
}