diff --git a/pkg/schema/load/applydefault_test.go b/pkg/schema/load/applydefault_test.go deleted file mode 100644 index e429843b4a6..00000000000 --- a/pkg/schema/load/applydefault_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package load - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "path/filepath" - "testing" - - "cuelang.org/go/cue" - "github.com/google/go-cmp/cmp" - "github.com/grafana/grafana/pkg/schema" - "golang.org/x/tools/txtar" -) - -var CasesDir = filepath.Join("testdata", "artifacts", "dashboards", "trimdefault") - -type Case struct { - Name string - - CUE string - Full string - Trimed string -} - -func TestGenerate(t *testing.T) { - cases, err := loadCases(CasesDir) - if err != nil { - t.Fatal(err) - } - - for _, c := range cases { - t.Run(c.Name+" apply default value", func(t *testing.T) { - var r cue.Runtime - scmInstance, err := r.Compile(c.Name+".cue", c.CUE) - if err != nil { - t.Fatal(err) - } - inputResource := schema.Resource{Value: c.Trimed} - scm := genericVersionedSchema{actual: scmInstance.Value()} - out, err := scm.ApplyDefaults(inputResource) - if err != nil { - t.Fatal(err) - } - b := []byte(out.Value.(string)) - - if s := cmp.Diff(string(b), c.Full); s != "" { - t.Fatal(s) - } - }) - } - - for _, c := range cases { - t.Run(c.Name+" trim default value", func(t *testing.T) { - var r cue.Runtime - scmInstance, err := r.Compile(c.Name+".cue", c.CUE) - if err != nil { - t.Fatal(err) - } - inputResource := schema.Resource{Value: c.Full} - scm := genericVersionedSchema{actual: scmInstance.Value()} - out, err := scm.TrimDefaults(inputResource) - if err != nil { - t.Fatal(err) - } - b := []byte(out.Value.(string)) - if s := cmp.Diff(string(b), c.Trimed); s != "" { - t.Fatal(s) - } - }) - } -} - -func loadCases(dir string) ([]Case, error) { - files, err := ioutil.ReadDir(dir) - if err != nil { - return nil, err - } - - var cases []Case - - for _, fi := range files { - file := filepath.Join(dir, fi.Name()) - a, err := txtar.ParseFile(file) - if err != nil { - return nil, err - } - - if len(a.Files) != 3 { - return nil, fmt.Errorf("Malformed test case '%s': Must contain exactly three files (CUE, Full and Trimed), but has %d", file, len(a.Files)) - } - - fullBuffer := new(bytes.Buffer) - fullJson := a.Files[1].Data - if err := json.Compact(fullBuffer, fullJson); err != nil { - return nil, err - } - - trimBuffer := new(bytes.Buffer) - trimedJson := a.Files[2].Data - if err := json.Compact(trimBuffer, trimedJson); err != nil { - return nil, err - } - - cases = append(cases, Case{ - Name: fi.Name(), - CUE: string(a.Files[0].Data), - Full: fullBuffer.String(), - Trimed: trimBuffer.String(), - }) - } - - return cases, nil -} diff --git a/pkg/schema/load/dashboard.go b/pkg/schema/load/dashboard.go index 08569c3b596..5bc98e425e5 100644 --- a/pkg/schema/load/dashboard.go +++ b/pkg/schema/load/dashboard.go @@ -129,20 +129,6 @@ func (cds *compositeDashboardSchema) Validate(r schema.Resource) error { return cds.actual.Unify(rv.Value()).Validate(cue.Concrete(true)) } -// ApplyDefaults returns a new, concrete copy of the Resource with all paths -// that are 1) missing in the Resource AND 2) specified by the schema, -// filled with default values specified by the schema. -func (cds *compositeDashboardSchema) ApplyDefaults(_ schema.Resource) (schema.Resource, error) { - panic("not implemented") // TODO: Implement -} - -// TrimDefaults returns a new, concrete copy of the Resource where all paths -// in the where the values at those paths are the same as the default value -// given in the schema. -func (cds *compositeDashboardSchema) TrimDefaults(_ schema.Resource) (schema.Resource, error) { - panic("not implemented") // TODO: Implement -} - // CUE returns the cue.Value representing the actual schema. func (cds *compositeDashboardSchema) CUE() cue.Value { return cds.actual diff --git a/pkg/schema/load/generic.go b/pkg/schema/load/generic.go index 1cb0019141e..bedded41e48 100644 --- a/pkg/schema/load/generic.go +++ b/pkg/schema/load/generic.go @@ -1,13 +1,8 @@ package load import ( - "bytes" - "fmt" - "strings" - "cuelang.org/go/cue" "cuelang.org/go/cue/load" - cuejson "cuelang.org/go/pkg/encoding/json" "github.com/grafana/grafana/pkg/schema" ) @@ -108,142 +103,6 @@ func (gvs *genericVersionedSchema) Validate(r schema.Resource) error { return gvs.actual.Unify(rv.Value()).Validate(cue.Concrete(true)) } -// ApplyDefaults returns a new, concrete copy of the Resource with all paths -// that are 1) missing in the Resource AND 2) specified by the schema, -// filled with default values specified by the schema. -func (gvs *genericVersionedSchema) ApplyDefaults(r schema.Resource) (schema.Resource, error) { - rv, err := rt.Compile("resource", r.Value) - if err != nil { - return r, err - } - rvUnified := rv.Value().Unify(gvs.CUE()) - re, err := convertCUEValueToString(rvUnified) - if err != nil { - return r, err - } - return schema.Resource{Value: re}, nil -} - -func convertCUEValueToString(inputCUE cue.Value) (string, error) { - re, err := cuejson.Marshal(inputCUE) - if err != nil { - return re, err - } - - result := []byte(re) - result = bytes.Replace(result, []byte("\\u003c"), []byte("<"), -1) - result = bytes.Replace(result, []byte("\\u003e"), []byte(">"), -1) - result = bytes.Replace(result, []byte("\\u0026"), []byte("&"), -1) - return string(result), nil -} - -// TrimDefaults returns a new, concrete copy of the Resource where all paths -// in the where the values at those paths are the same as the default value -// given in the schema. -func (gvs *genericVersionedSchema) TrimDefaults(r schema.Resource) (schema.Resource, error) { - rvInstance, err := rt.Compile("resource", r.Value) - if err != nil { - return r, err - } - rv, _, err := removeDefaultHelper(gvs.CUE(), rvInstance.Value()) - if err != nil { - return r, err - } - re, err := convertCUEValueToString(rv) - fmt.Println("the trimed fields would be: ", re) - if err != nil { - return r, err - } - return schema.Resource{Value: re}, nil -} - -func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool, error) { - // Since for now, panel definition is open validation, - // we need to loop on the input CUE for trimming - rvInstance, err := rt.Compile("resource", []byte{}) - if err != nil { - return input, false, err - } - rv := rvInstance.Value() - - switch inputdef.IncompleteKind() { - case cue.StructKind: - // Get all fields including optional fields - iter, err := inputdef.Fields(cue.Optional(true)) - if err != nil { - return rv, false, err - } - keySet := make(map[string]bool) - for iter.Next() { - lable, _ := iter.Value().Label() - keySet[lable] = true - lv := input.LookupPath(cue.MakePath(cue.Str(lable))) - if err != nil { - continue - } - if lv.Exists() { - re, isEqual, err := removeDefaultHelper(iter.Value(), lv) - if err == nil && !isEqual { - rv = rv.FillPath(cue.MakePath(cue.Str(lable)), re) - } - } - } - // Get all the fields that are not defined in schema yet for panel - iter, err = input.Fields() - if err != nil { - return rv, false, err - } - for iter.Next() { - lable, _ := iter.Value().Label() - if exists := keySet[lable]; !exists { - rv = rv.FillPath(cue.MakePath(cue.Str(lable)), iter.Value()) - } - } - return rv, false, nil - case cue.ListKind: - val, _ := inputdef.Default() - err1 := input.Subsume(val) - err2 := val.Subsume(input) - if val.IsConcrete() && err1 == nil && err2 == nil { - return rv, true, nil - } - ele := inputdef.LookupPath(cue.MakePath(cue.AnyIndex)) - if ele.IncompleteKind() == cue.BottomKind { - return rv, true, nil - } - - iter, err := input.List() - if err != nil { - return rv, true, nil - } - var iterlist []string - for iter.Next() { - re, isEqual, err := removeDefaultHelper(ele, iter.Value()) - if err == nil && !isEqual { - reString, err := convertCUEValueToString(re) - if err != nil { - return rv, true, nil - } - iterlist = append(iterlist, reString) - } - } - iterlistContent := fmt.Sprintf("[%s]", strings.Join(iterlist, ",")) - liInstance, err := rt.Compile("resource", []byte(iterlistContent)) - if err != nil { - return rv, false, err - } - return liInstance.Value(), false, nil - default: - val, _ := inputdef.Default() - err1 := input.Subsume(val) - err2 := val.Subsume(input) - if val.IsConcrete() && err1 == nil && err2 == nil { - return input, true, nil - } - return input, false, nil - } -} - // CUE returns the cue.Value representing the actual schema. func (gvs *genericVersionedSchema) CUE() cue.Value { return gvs.actual diff --git a/pkg/schema/load/load_test.go b/pkg/schema/load/load_test.go index 2f68888d98c..07d6dff602f 100644 --- a/pkg/schema/load/load_test.go +++ b/pkg/schema/load/load_test.go @@ -51,7 +51,7 @@ func TestDashboardValidity(t *testing.T) { // TODO FIXME remove this once we actually have dashboard schema filled in // enough that the tests pass, lol t.Skip() - validdir := os.DirFS(filepath.Join("testdata", "artifacts", "dashboards", "basic")) + validdir := os.DirFS(filepath.Join("testdata", "artifacts", "dashboards")) dash, err := BaseDashboardFamily(p) require.NoError(t, err, "error while loading base dashboard scuemata") diff --git a/pkg/schema/load/testdata/artifacts/dashboards/basic/basic.json b/pkg/schema/load/testdata/artifacts/dashboards/basic.json similarity index 100% rename from pkg/schema/load/testdata/artifacts/dashboards/basic/basic.json rename to pkg/schema/load/testdata/artifacts/dashboards/basic.json diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 4035918b82c..719eb34ba94 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -1,13 +1,18 @@ package schema import ( + "bytes" "errors" "fmt" "math/bits" + "strings" "cuelang.org/go/cue" + cuejson "cuelang.org/go/pkg/encoding/json" ) +var rt = &cue.Runtime{} + // CueSchema represents a single, complete CUE-based schema that can perform // operations on Resources. // @@ -23,16 +28,6 @@ type CueSchema interface { // Validate checks that the resource is correct with respect to the schema. Validate(Resource) error - // ApplyDefaults returns a new, concrete copy of the Resource with all paths - // that are 1) missing in the Resource AND 2) specified by the schema, - // filled with default values specified by the schema. - ApplyDefaults(Resource) (Resource, error) - - // TrimDefaults returns a new, concrete copy of the Resource where all paths - // in the where the values at those paths are the same as the default value - // given in the schema. - TrimDefaults(Resource) (Resource, error) - // Migrate transforms a Resource into a new Resource that is correct with // respect to its Successor schema. It returns the transformed resource, // the schema to which the resource now conforms, and any errors that @@ -257,6 +252,142 @@ func Exact(maj, min int) SearchOption { } } +// ApplyDefaults returns a new, concrete copy of the Resource with all paths +// that are 1) missing in the Resource AND 2) specified by the schema, +// filled with default values specified by the schema. +func ApplyDefaults(r Resource, scue cue.Value) (Resource, error) { + rv, err := rt.Compile("resource", r.Value) + if err != nil { + return r, err + } + rvUnified := rv.Value().Unify(scue) + re, err := convertCUEValueToString(rvUnified) + if err != nil { + return r, err + } + return Resource{Value: re}, nil +} + +func convertCUEValueToString(inputCUE cue.Value) (string, error) { + re, err := cuejson.Marshal(inputCUE) + if err != nil { + return re, err + } + + result := []byte(re) + result = bytes.Replace(result, []byte("\\u003c"), []byte("<"), -1) + result = bytes.Replace(result, []byte("\\u003e"), []byte(">"), -1) + result = bytes.Replace(result, []byte("\\u0026"), []byte("&"), -1) + return string(result), nil +} + +// TrimDefaults returns a new, concrete copy of the Resource where all paths +// in the where the values at those paths are the same as the default value +// given in the schema. +func TrimDefaults(r Resource, scue cue.Value) (Resource, error) { + rvInstance, err := rt.Compile("resource", r.Value) + if err != nil { + return r, err + } + rv, _, err := removeDefaultHelper(scue, rvInstance.Value()) + if err != nil { + return r, err + } + re, err := convertCUEValueToString(rv) + if err != nil { + return r, err + } + return Resource{Value: re}, nil +} + +func isCueValueEqual(inputdef cue.Value, input cue.Value) bool { + val, _ := inputdef.Default() + return input.Subsume(val) == nil && val.Subsume(input) == nil +} + +func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool, error) { + // To include all optional fields, we need to use inputdef for iteration, + // since the lookuppath with optional field doesn't work very well + rvInstance, err := rt.Compile("resource", []byte{}) + if err != nil { + return input, false, err + } + rv := rvInstance.Value() + + switch inputdef.IncompleteKind() { + case cue.StructKind: + // Get all fields including optional fields + iter, err := inputdef.Fields(cue.Optional(true)) + if err != nil { + return rv, false, err + } + keySet := make(map[string]bool) + for iter.Next() { + lable, _ := iter.Value().Label() + keySet[lable] = true + lv := input.LookupPath(cue.MakePath(cue.Str(lable))) + if err != nil { + continue + } + if lv.Exists() { + re, isEqual, err := removeDefaultHelper(iter.Value(), lv) + if err == nil && !isEqual { + rv = rv.FillPath(cue.MakePath(cue.Str(lable)), re) + } + } + } + // Get all the fields that are not defined in schema yet for panel + iter, err = input.Fields() + if err != nil { + return rv, false, err + } + for iter.Next() { + lable, _ := iter.Value().Label() + if exists := keySet[lable]; !exists { + rv = rv.FillPath(cue.MakePath(cue.Str(lable)), iter.Value()) + } + } + return rv, false, nil + case cue.ListKind: + if isCueValueEqual(inputdef, input) { + return rv, true, nil + } + ele := inputdef.LookupPath(cue.MakePath(cue.AnyIndex)) + if ele.IncompleteKind() == cue.BottomKind { + return rv, true, nil + } + + iter, err := input.List() + if err != nil { + return rv, true, nil + } + + // The following code is workaround since today overwrite list element doesn't work + var iterlist []string + for iter.Next() { + re, isEqual, err := removeDefaultHelper(ele, iter.Value()) + if err == nil && !isEqual { + reString, err := convertCUEValueToString(re) + if err != nil { + return rv, true, nil + } + iterlist = append(iterlist, reString) + } + } + iterlistContent := fmt.Sprintf("[%s]", strings.Join(iterlist, ",")) + liInstance, err := rt.Compile("resource", []byte(iterlistContent)) + if err != nil { + return rv, false, err + } + return liInstance.Value(), false, nil + default: + if isCueValueEqual(inputdef, input) { + return input, true, nil + } + return input, false, nil + } +} + // A Resource represents a concrete data object - e.g., JSON // representing a dashboard. // diff --git a/pkg/schema/schema_test.go b/pkg/schema/schema_test.go index fee087844dd..178bbb1e784 100644 --- a/pkg/schema/schema_test.go +++ b/pkg/schema/schema_test.go @@ -1,4 +1,108 @@ package schema -// TODO tests for this stuff! Everything in this package is totally generic, -// nothing is specific to Grafana +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + "testing" + + "cuelang.org/go/cue" + "github.com/google/go-cmp/cmp" + "golang.org/x/tools/txtar" +) + +var CasesDir = filepath.Join("testdata", "trimapplydefaults") + +type Case struct { + Name string + CUE string + Full string + Trimmed string +} + +func TestGenerate(t *testing.T) { + cases, err := loadCases(CasesDir) + if err != nil { + t.Fatal(err) + } + + for _, c := range cases { + t.Run(c.Name+" apply default value", func(t *testing.T) { + var r cue.Runtime + scmInstance, err := r.Compile(c.Name+".cue", c.CUE) + if err != nil { + t.Fatal(err) + } + inputResource := Resource{Value: c.Trimmed} + out, err := ApplyDefaults(inputResource, scmInstance.Value()) + if err != nil { + t.Fatal(err) + } + b := []byte(out.Value.(string)) + + if s := cmp.Diff(string(b), c.Full); s != "" { + t.Fatal(s) + } + }) + } + + for _, c := range cases { + t.Run(c.Name+" trim default value", func(t *testing.T) { + var r cue.Runtime + scmInstance, err := r.Compile(c.Name+".cue", c.CUE) + if err != nil { + t.Fatal(err) + } + inputResource := Resource{Value: c.Full} + out, err := TrimDefaults(inputResource, scmInstance.Value()) + if err != nil { + t.Fatal(err) + } + b := []byte(out.Value.(string)) + if s := cmp.Diff(string(b), c.Trimmed); s != "" { + t.Fatal(s) + } + }) + } +} + +func loadCases(dir string) ([]Case, error) { + files, err := ioutil.ReadDir(dir) + if err != nil { + return nil, err + } + var cases []Case + for _, fi := range files { + file := filepath.Join(dir, fi.Name()) + a, err := txtar.ParseFile(file) + if err != nil { + return nil, err + } + + if len(a.Files) != 3 { + return nil, fmt.Errorf("Malformed test case '%s': Must contain exactly three files (CUE, Full and Trimed), but has %d", file, len(a.Files)) + } + + fullBuffer := new(bytes.Buffer) + fullJson := a.Files[1].Data + if err := json.Compact(fullBuffer, fullJson); err != nil { + return nil, err + } + + trimBuffer := new(bytes.Buffer) + trimedJson := a.Files[2].Data + if err := json.Compact(trimBuffer, trimedJson); err != nil { + return nil, err + } + + cases = append(cases, Case{ + Name: fi.Name(), + CUE: string(a.Files[0].Data), + Full: fullBuffer.String(), + Trimmed: trimBuffer.String(), + }) + } + return cases, nil +} diff --git a/pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test1 b/pkg/schema/testdata/trimapplydefaults/test1 similarity index 100% rename from pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test1 rename to pkg/schema/testdata/trimapplydefaults/test1 diff --git a/pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test2 b/pkg/schema/testdata/trimapplydefaults/test2 similarity index 100% rename from pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test2 rename to pkg/schema/testdata/trimapplydefaults/test2 diff --git a/pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test3 b/pkg/schema/testdata/trimapplydefaults/test3 similarity index 100% rename from pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test3 rename to pkg/schema/testdata/trimapplydefaults/test3 diff --git a/pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test4 b/pkg/schema/testdata/trimapplydefaults/test4 similarity index 100% rename from pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test4 rename to pkg/schema/testdata/trimapplydefaults/test4 diff --git a/pkg/services/schemaloader/schemaloader.go b/pkg/services/schemaloader/schemaloader.go index 6ea729778c2..647c6b206eb 100644 --- a/pkg/services/schemaloader/schemaloader.go +++ b/pkg/services/schemaloader/schemaloader.go @@ -62,7 +62,7 @@ func (rs *SchemaLoaderService) DashboardApplyDefaults(input *simplejson.Json) (* val = removeNils(val) data, _ := json.Marshal(val) dsSchema := schema.Find(rs.DashFamily, schema.Latest()) - result, err := dsSchema.ApplyDefaults(schema.Resource{Value: data}) + result, err := schema.ApplyDefaults(schema.Resource{Value: data}, dsSchema.CUE()) if err != nil { return input, err } @@ -83,7 +83,7 @@ func (rs *SchemaLoaderService) DashboardTrimDefaults(input simplejson.Json) (sim return input, err } // spew.Dump(dsSchema) - result, err := dsSchema.TrimDefaults(schema.Resource{Value: data}) + result, err := schema.TrimDefaults(schema.Resource{Value: data}, dsSchema.CUE()) if err != nil { return input, err }