Scuemata: Add MergeFS implementation to enable iteration for multiple filesystems (#33989)

* Add MergeFS implementation to enable iteration to multiple filesystems

* Fix linting

* Fix misplaced close file

* Move mergefs functionality into tests

* Fix linting

* Add mergefs test

* Dummy commit - remove

* Test fixes - renaming

* Fix ReadDir

# Keeps first filesystem's overridden file

* Fixes according to reviewer's comments

* Remove dummy test

* Remove walkdir

* Small test refactoring

pkg/cmd/grafana-cli/commands/mergefs.go

@@ -0,0 +1,92 @@
+package commands
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+	"sort"
+
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
+)
+
+// MergeFS contains a slice of different filesystems that can be merged together
+type MergeFS struct {
+	filesystems []fs.FS
+}
+
+// Merge filesystems
+func Merge(filesystems ...fs.FS) fs.FS {
+	return MergeFS{filesystems: filesystems}
+}
+
+// Open opens the named file.
+func (mfs MergeFS) Open(name string) (fs.File, error) {
+	for _, filesystem := range mfs.filesystems {
+		file, err := filesystem.Open(name)
+		if err == nil {
+			return file, nil
+		}
+	}
+	return nil, os.ErrNotExist
+}
+
+// ReadDir reads from the directory, and produces a DirEntry array of different
+// directories.
+//
+// It iterates through all different filesystems that exist in the mfs MergeFS
+// filesystem slice and it identifies overlapping directories that exist in different
+// filesystems
+func (mfs MergeFS) ReadDir(name string) ([]fs.DirEntry, error) {
+	dirsMap := make(map[string]fs.DirEntry)
+	for _, filesystem := range mfs.filesystems {
+		if fsys, ok := filesystem.(fs.ReadDirFS); ok {
+			dir, err := fsys.ReadDir(name)
+			if err != nil {
+				if errors.Is(err, fs.ErrNotExist) {
+					logger.Debugf("directory in filepath %s was not found in filesystem", name)
+					continue
+				}
+				return nil, err
+			}
+			for _, v := range dir {
+				if _, ok := dirsMap[v.Name()]; !ok {
+					dirsMap[v.Name()] = v
+				}
+			}
+			continue
+		}
+
+		file, err := filesystem.Open(name)
+		if err != nil {
+			logger.Debugf("filepath %s was not found in filesystem", name)
+			continue
+		}
+
+		dir, ok := file.(fs.ReadDirFile)
+		if !ok {
+			return nil, &fs.PathError{Op: "readdir", Path: name, Err: errors.New("not implemented")}
+		}
+
+		fsDirs, err := dir.ReadDir(-1)
+		if err != nil {
+			return nil, err
+		}
+		sort.Slice(fsDirs, func(i, j int) bool { return fsDirs[i].Name() < fsDirs[j].Name() })
+		for _, v := range fsDirs {
+			if _, ok := dirsMap[v.Name()]; !ok {
+				dirsMap[v.Name()] = v
+			}
+		}
+		if err := file.Close(); err != nil {
+			logger.Error("failed to close file", "err", err)
+		}
+	}
+	dirs := make([]fs.DirEntry, 0, len(dirsMap))
+
+	for _, value := range dirsMap {
+		dirs = append(dirs, value)
+	}
+
+	sort.Slice(dirs, func(i, j int) bool { return dirs[i].Name() < dirs[j].Name() })
+	return dirs, nil
+}

pkg/cmd/grafana-cli/commands/mergefs_test.go

@@ -0,0 +1,68 @@
+package commands
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+	"testing/fstest"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMergeFS(t *testing.T) {
+	var filePaths = []struct {
+		path           string
+		dirArrayLength int
+		child          string
+	}{
+		// MapFS takes in account the current directory in addition to all included directories and produces a "" dir
+		{"a", 1, "z"},
+		{"a/z", 1, "bar.cue"},
+		{"b", 1, "z"},
+		{"b/z", 1, "foo.cue"},
+	}
+
+	tempDir := os.DirFS(filepath.Join("testdata", "mergefs"))
+	a := fstest.MapFS{
+		"a":           &fstest.MapFile{Mode: fs.ModeDir},
+		"a/z":         &fstest.MapFile{Mode: fs.ModeDir},
+		"a/z/bar.cue": &fstest.MapFile{Data: []byte("bar")},
+	}
+
+	filesystem := Merge(tempDir, a)
+
+	t.Run("testing mergefs.ReadDir", func(t *testing.T) {
+		for _, fp := range filePaths {
+			t.Run("testing path: "+fp.path, func(t *testing.T) {
+				dirs, err := fs.ReadDir(filesystem, fp.path)
+				require.NoError(t, err)
+				require.Len(t, dirs, fp.dirArrayLength)
+
+				for i := 0; i < len(dirs); i++ {
+					require.Equal(t, dirs[i].Name(), fp.child)
+				}
+			})
+		}
+	})
+
+	t.Run("testing mergefs.Open", func(t *testing.T) {
+		data := make([]byte, 3)
+		file, err := filesystem.Open("a/z/bar.cue")
+		require.NoError(t, err)
+
+		_, err = file.Read(data)
+		require.NoError(t, err)
+		require.Equal(t, "bar", string(data))
+
+		file, err = filesystem.Open("b/z/foo.cue")
+		require.NoError(t, err)
+
+		_, err = file.Read(data)
+		require.NoError(t, err)
+		require.Equal(t, "foo", string(data))
+
+		err = file.Close()
+		require.NoError(t, err)
+	})
+}

pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go

@@ -2,21 +2,21 @@ package commands
 
 import (
 	"os"
-	"path/filepath"
 	"testing"
+	"testing/fstest"
 
 	"github.com/grafana/grafana/pkg/schema/load"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+var defaultBaseLoadPaths = load.GetDefaultLoadPaths()
+
 func TestValidateScuemataBasics(t *testing.T) {
 	t.Run("Testing scuemata validity with valid cue schemas", func(t *testing.T) {
-		tempDir := os.DirFS(filepath.Join("testdata", "valid_scuemata"))
-
 		var baseLoadPaths = load.BaseLoadPaths{
-			BaseCueFS:       tempDir,
+			BaseCueFS:       defaultBaseLoadPaths.BaseCueFS,
-			DistPluginCueFS: load.GetDefaultLoadPaths().DistPluginCueFS,
+			DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
 		}
 
 		err := validate(baseLoadPaths, load.BaseDashboardFamily)
@@ -27,26 +27,38 @@ func TestValidateScuemataBasics(t *testing.T) {
 	})
 
 	t.Run("Testing scuemata validity with invalid cue schemas - family missing", func(t *testing.T) {
-		tempDir := os.DirFS(filepath.Join("testdata", "invalid_scuemata_missing_family"))
+		genCue, err := os.ReadFile("testdata/missing_family_gen.cue")
+		require.NoError(t, err)
+
+		fs := fstest.MapFS{
+			"cue/data/gen.cue": &fstest.MapFile{Data: genCue},
+		}
+		mergedFS := Merge(fs, defaultBaseLoadPaths.BaseCueFS)
 
 		var baseLoadPaths = load.BaseLoadPaths{
-			BaseCueFS:       tempDir,
+			BaseCueFS:       mergedFS,
-			DistPluginCueFS: load.GetDefaultLoadPaths().DistPluginCueFS,
+			DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
 		}
 
-		err := validate(baseLoadPaths, load.BaseDashboardFamily)
+		err = validate(baseLoadPaths, load.BaseDashboardFamily)
 		assert.EqualError(t, err, "error while loading dashboard scuemata, err: dashboard schema family did not exist at expected path in expected file")
 	})
 
-	t.Run("Testing scuemata validity with invalid cue schemas - panel missing", func(t *testing.T) {
+	t.Run("Testing scuemata validity with invalid cue schemas - panel missing ", func(t *testing.T) {
-		tempDir := os.DirFS(filepath.Join("testdata", "invalid_scuemata_missing_panel"))
+		genCue, err := os.ReadFile("testdata/missing_panel_gen.cue")
+		require.NoError(t, err)
+
+		fs := fstest.MapFS{
+			"cue/data/gen.cue": &fstest.MapFile{Data: genCue},
+		}
+		mergedFS := Merge(fs, defaultBaseLoadPaths.BaseCueFS)
 
 		var baseLoadPaths = load.BaseLoadPaths{
-			BaseCueFS:       tempDir,
+			BaseCueFS:       mergedFS,
-			DistPluginCueFS: load.GetDefaultLoadPaths().DistPluginCueFS,
+			DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
 		}
 
-		err := validate(baseLoadPaths, load.BaseDashboardFamily)
+		err = validate(baseLoadPaths, load.BaseDashboardFamily)
 		require.NoError(t, err, "error while loading base dashboard scuemata")
 
 		err = validate(baseLoadPaths, load.DistDashboardFamily)

pkg/cmd/grafana-cli/commands/testdata/invalid_scuemata_missing_family/cue/scuemata/panel-plugin.cue

@@ -1,22 +0,0 @@
-package scuemata
-
-// Definition of the shape of a panel plugin's schema declarations in its
-// schema.cue file.
-//
-// Note that these keys do not appear directly in any real JSON artifact;
-// rather, they are composed into panel structures as they are defined within
-// the larger Dashboard schema.
-#PanelSchema: {
-    PanelOptions: {...}
-    PanelFieldConfig: {...}
-    ...
-}
-
-// A lineage of panel schema
-#PanelLineage: [#PanelSchema, ...#PanelSchema]
-
-// Panel plugin-specific Family
-#PanelFamily: {
-    lineages: [#PanelLineage, ...#PanelLineage]
-    migrations: [...#Migration]
-}

pkg/cmd/grafana-cli/commands/testdata/invalid_scuemata_missing_family/cue/scuemata/scuemata.cue

@@ -1,60 +0,0 @@
-package scuemata
-
-// A family is a collection of schemas that specify a single kind of object,
-// allowing evolution of the canonical schema for that kind of object over time.
-//
-// The schemas are organized into a list of Lineages, which are themselves ordered
-// lists of schemas where each schema with its predecessor in the lineage.
-//
-// If it is desired to define a schema with a breaking schema relative to its
-// predecessors, a new Lineage must be created, as well as a Migration that defines
-// a mapping to the new schema from the latest schema in prior Lineage.
-//
-// The version number of a schema is not controlled by the schema itself, but by
-// its position in the list of lineages - e.g., 0.0 corresponds to the first
-// schema in the first lineage.
-#Family: {
-	lineages: [#Lineage, ...#Lineage]
-	migrations: [...#Migration]
-	let lseq = lineages[len(lineages)-1]
-	latest: #LastSchema & {_p: lseq}
-}
-
-// A Lineage is a non-empty list containing an ordered series of schemas that
-// all describe a single kind of object, where each schema is backwards
-// compatible with its predecessor.
-#Lineage: [{...}, ...{...}]
-
-#LastSchema: {
-	_p: #Lineage
-	_p[len(_p)-1]
-}
-
-// A Migration defines a relation between two schemas, "_from" and "_to". The
-// relation expresses any complex mappings that must be performed to
-// transform an input artifact valid with respect to the _from schema, into
-// an artifact valid with respect to the _to schema. This is accomplished
-// in two stages:
-//   1. A Migration is initially defined by passing in schemas for _from and _to,
-//      and mappings that translate _from to _to are defined in _rel. 
-//   2. A concrete object may then be unified with _to, resulting in its values
-//      being mapped onto "result" by way of _rel.
-//
-// This is the absolute simplest possible definition of a Migration. It's
-// incumbent on the implementor to manually ensure the correctness and
-// completeness of the mapping. The primary value in defining such a generic
-// structure is to allow comparably generic logic for migrating concrete
-// artifacts through schema changes.
-//
-// If _to isn't backwards compatible (accretion-only) with _from, then _rel must
-// explicitly enumerate every field in _from and map it to a field in _to, even
-// if they're identical. This is laborious for anything outside trivially tiny
-// schema. We'll want to eventually add helpers for whitelisting or blacklisting
-// of paths in _from, so that migrations of larger schema can focus narrowly on
-// the points of actual change.
-#Migration: {
-	from: {...}
-	to: {...}
-	rel: {...}
-	result: to & rel
-}

pkg/cmd/grafana-cli/commands/testdata/invalid_scuemata_missing_panel/cue/scuemata/panel-plugin.cue

@@ -1,22 +0,0 @@
-package scuemata
-
-// Definition of the shape of a panel plugin's schema declarations in its
-// schema.cue file.
-//
-// Note that these keys do not appear directly in any real JSON artifact;
-// rather, they are composed into panel structures as they are defined within
-// the larger Dashboard schema.
-#PanelSchema: {
-    PanelOptions: {...}
-    PanelFieldConfig?: {...}
-    ...
-}
-
-// A lineage of panel schema
-#PanelLineage: [#PanelSchema, ...#PanelSchema]
-
-// Panel plugin-specific Family
-#PanelFamily: {
-    lineages: [#PanelLineage, ...#PanelLineage]
-    migrations: [...#Migration]
-}

pkg/cmd/grafana-cli/commands/testdata/invalid_scuemata_missing_panel/cue/scuemata/scuemata.cue

@@ -1,60 +0,0 @@
-package scuemata
-
-// A family is a collection of schemas that specify a single kind of object,
-// allowing evolution of the canonical schema for that kind of object over time.
-//
-// The schemas are organized into a list of Lineages, which are themselves ordered
-// lists of schemas where each schema with its predecessor in the lineage.
-//
-// If it is desired to define a schema with a breaking schema relative to its
-// predecessors, a new Lineage must be created, as well as a Migration that defines
-// a mapping to the new schema from the latest schema in prior Lineage.
-//
-// The version number of a schema is not controlled by the schema itself, but by
-// its position in the list of lineages - e.g., 0.0 corresponds to the first
-// schema in the first lineage.
-#Family: {
-	lineages: [#Lineage, ...#Lineage]
-	migrations: [...#Migration]
-	let lseq = lineages[len(lineages)-1]
-	latest: #LastSchema & {_p: lseq}
-}
-
-// A Lineage is a non-empty list containing an ordered series of schemas that
-// all describe a single kind of object, where each schema is backwards
-// compatible with its predecessor.
-#Lineage: [{...}, ...{...}]
-
-#LastSchema: {
-	_p: #Lineage
-	_p[len(_p)-1]
-}
-
-// A Migration defines a relation between two schemas, "_from" and "_to". The
-// relation expresses any complex mappings that must be performed to
-// transform an input artifact valid with respect to the _from schema, into
-// an artifact valid with respect to the _to schema. This is accomplished
-// in two stages:
-//   1. A Migration is initially defined by passing in schemas for _from and _to,
-//      and mappings that translate _from to _to are defined in _rel. 
-//   2. A concrete object may then be unified with _to, resulting in its values
-//      being mapped onto "result" by way of _rel.
-//
-// This is the absolute simplest possible definition of a Migration. It's
-// incumbent on the implementor to manually ensure the correctness and
-// completeness of the mapping. The primary value in defining such a generic
-// structure is to allow comparably generic logic for migrating concrete
-// artifacts through schema changes.
-//
-// If _to isn't backwards compatible (accretion-only) with _from, then _rel must
-// explicitly enumerate every field in _from and map it to a field in _to, even
-// if they're identical. This is laborious for anything outside trivially tiny
-// schema. We'll want to eventually add helpers for whitelisting or blacklisting
-// of paths in _from, so that migrations of larger schema can focus narrowly on
-// the points of actual change.
-#Migration: {
-	from: {...}
-	to: {...}
-	rel: {...}
-	result: to & rel
-}

pkg/cmd/grafana-cli/commands/testdata/mergefs/b/z/foo.cue

@@ -0,0 +1 @@
+foo

pkg/cmd/grafana-cli/commands/testdata/valid_scuemata/cue/data/gen.cue

@@ -1,214 +0,0 @@
-package grafanaschema
-
-import "github.com/grafana/grafana/cue/scuemata"
-
-Family: scuemata.#Family & {
-	lineages: [
-		[
-			{ // 0.0
-                // Unique numeric identifier for the dashboard.
-                // TODO must isolate or remove identifiers local to a Grafana instance...?
-                id?: number
-                // Unique dashboard identifier that can be generated by anyone. string (8-40)
-                uid: string
-                // Title of dashboard.
-                title?: string
-                // Description of dashboard.
-                description?: string
-
-                gnetId?: string
-                // Tags associated with dashboard.
-                tags?: [...string]
-                // Theme of dashboard.
-                style: *"light" | "dark"
-                // Timezone of dashboard,
-                timezone?: *"browser" | "utc"
-                // Whether a dashboard is editable or not.
-                editable: bool | *true
-                // 0 for no shared crosshair or tooltip (default).
-                // 1 for shared crosshair.
-                // 2 for shared crosshair AND shared tooltip.
-                graphTooltip: >=0 & <=2 | *0
-                // Time range for dashboard, e.g. last 6 hours, last 7 days, etc
-                time?: {
-                    from: string | *"now-6h"
-                    to:   string | *"now"
-                }
-                // Timepicker metadata.
-                timepicker?: {
-                    // Whether timepicker is collapsed or not.
-                    collapse: bool | *false
-                    // Whether timepicker is enabled or not.
-                    enable: bool | *true
-                    // Whether timepicker is visible or not.
-                    hidden: bool | *false
-                    // Selectable intervals for auto-refresh.
-                    refresh_intervals: [...string] | *["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"]
-                }
-                // Templating.
-                templating?: list: [...{...}]
-                // Annotations.
-                annotations?: list: [...{
-                    builtIn: number | *0
-                    // Datasource to use for annotation.
-                    datasource: string
-                    // Whether annotation is enabled.
-                    enable?: bool | *true
-                    // Whether to hide annotation.
-                    hide?: bool | *false
-                    // Annotation icon color.
-                    iconColor?: string
-                    // Name of annotation.
-                    name?: string
-                    type: string | *"dashboard"
-                    // Query for annotation data.
-                    rawQuery?: string
-                    showIn:   number | *0
-                }]
-                // Auto-refresh interval.
-                refresh?: string
-                // Version of the JSON schema, incremented each time a Grafana update brings
-                // changes to said schema.
-                schemaVersion: number | *25
-                // Version of the dashboard, incremented each time the dashboard is updated.
-                version?: number
-                panels?: [...#Panel]
-
-                // Dashboard panels. Panels are canonically defined inline
-                // because they share a version timeline with the dashboard
-                // schema; they do not vary independently. We create a separate,
-                // synthetic Family to represent them in Go, for ease of generating
-                // e.g. JSON Schema.
-                #Panel: {
-                    ...
-                    // The panel plugin type id.
-                    type: !=""
-
-                    // Internal - the exact major and minor versions of the panel plugin
-                    // schema. Hidden and therefore not a part of the data model, but
-                    // expected to be filled with panel plugin schema versions so that it's
-                    // possible to figure out which schema version matched on a successful
-                    // unification.
-                    // _pv: { maj: int, min: int }
-                    // The major and minor versions of the panel plugin for this schema.
-                    // TODO 2-tuple list instead of struct?
-                    panelSchema?: { maj: number, min: number }
-
-                    // Panel title.
-                    title?: string
-                    // Description.
-                    description?: string
-                    // Whether to display the panel without a background.
-                    transparent: bool | *false
-                    // Name of default datasource.
-                    datasource?: string
-                    // Grid position.
-                    gridPos?: {
-                        // Panel
-                        h: number & >0 | *9
-                        // Panel
-                        w: number & >0 & <=24 | *12
-                        // Panel x
-                        x: number & >=0 & <24 | *0
-                        // Panel y
-                        y: number & >=0 | *0
-                        // true if fixed
-                        static?: bool
-                    }
-                    // Panel links.
-                    // links?: [..._panelLink]
-                    // Name of template variable to repeat for.
-                    repeat?: string
-                    // Direction to repeat in if 'repeat' is set.
-                    // "h" for horizontal, "v" for vertical.
-                    repeatDirection: *"h" | "v"
-                    // Schema for panel targets is specified by datasource
-                    // plugins. We use a placeholder definition, which the Go
-                    // schema loader either left open/as-is with the Base
-                    // variant of the Dashboard and Panel families, or filled
-                    // with types derived from plugins in the Instance variant.
-                    // When working directly from CUE, importers can extend this
-                    // type directly to achieve the same effect.
-                    targets?: [...{...}]
-
-                    // The values depend on panel type
-                    options: {...}
-
-                    fieldConfig: {
-                        defaults: {
-                            // The display value for this field.  This supports template variables blank is auto
-                            displayName?: string
-
-                            // This can be used by data sources that return and explicit naming structure for values and labels
-                            // When this property is configured, this value is used rather than the default naming strategy.
-                            displayNameFromDS?: string
-
-                            // Human readable field metadata
-                            description?: string
-
-                            // An explict path to the field in the datasource.  When the frame meta includes a path,
-                            // This will default to `${frame.meta.path}/${field.name}
-                            //
-                            // When defined, this value can be used as an identifier within the datasource scope, and
-                            // may be used to update the results
-                            path?: string
-
-                            // True if data source can write a value to the path.  Auth/authz are supported separately
-                            writeable?: bool
-
-                            // True if data source field supports ad-hoc filters
-                            filterable?: bool
-
-                            // Numeric Options
-                            unit?: string
-
-                            // Significant digits (for display)
-                            decimals?: number
-
-                            min?: number
-                            max?: number
-
-                            //   // Convert input values into a display string
-                            //   mappings?: ValueMapping[];
-
-                            //   // Map numeric values to states
-                            //   thresholds?: ThresholdsConfig;
-
-                            //   // Map values to a display color
-                            //   color?: FieldColor;
-
-                            //   // Used when reducing field values
-                            //   nullValueMode?: NullValueMode;
-
-                            //   // The behavior when clicking on a result
-                            //   links?: DataLink[];
-
-                            // Alternative to empty string
-                            noValue?: string
-
-                            // Can always exist. Valid fields within this are
-                            // defined by the panel plugin - that's the
-                            // PanelFieldConfig that comes from the plugin.
-                            custom?: {...}
-                        }
-                        overrides: [...{
-                            matcher: {
-                                id: string | *""
-                                options?: _
-                            }
-                            properties: [...{
-                                id: string | *""
-                                value?: _
-                            }]
-                        }]
-                    }
-                }
-            }
-		]
-	]
-}
-
-#Latest: {
-    #Dashboard: Family.latest
-    #Panel: Family.latest._Panel
-}

pkg/cmd/grafana-cli/commands/testdata/valid_scuemata/cue/scuemata/panel-plugin.cue

@@ -1,22 +0,0 @@
-package scuemata
-
-// Definition of the shape of a panel plugin's schema declarations in its
-// schema.cue file.
-//
-// Note that these keys do not appear directly in any real JSON artifact;
-// rather, they are composed into panel structures as they are defined within
-// the larger Dashboard schema.
-#PanelSchema: {
-	PanelOptions: {...}
-	PanelFieldConfig?: {...}
-	...
-}
-
-// A lineage of panel schema
-#PanelLineage: [#PanelSchema, ...#PanelSchema]
-
-// Panel plugin-specific Family
-#PanelFamily: {
-	lineages: [#PanelLineage, ...#PanelLineage]
-	migrations: [...#Migration]
-}

pkg/cmd/grafana-cli/commands/testdata/valid_scuemata/cue/scuemata/scuemata.cue

@@ -1,60 +0,0 @@
-package scuemata
-
-// A family is a collection of schemas that specify a single kind of object,
-// allowing evolution of the canonical schema for that kind of object over time.
-//
-// The schemas are organized into a list of Lineages, which are themselves ordered
-// lists of schemas where each schema with its predecessor in the lineage.
-//
-// If it is desired to define a schema with a breaking schema relative to its
-// predecessors, a new Lineage must be created, as well as a Migration that defines
-// a mapping to the new schema from the latest schema in prior Lineage.
-//
-// The version number of a schema is not controlled by the schema itself, but by
-// its position in the list of lineages - e.g., 0.0 corresponds to the first
-// schema in the first lineage.
-#Family: {
-	lineages: [#Lineage, ...#Lineage]
-	migrations: [...#Migration]
-	let lseq = lineages[len(lineages)-1]
-	latest: #LastSchema & {_p: lseq}
-}
-
-// A Lineage is a non-empty list containing an ordered series of schemas that
-// all describe a single kind of object, where each schema is backwards
-// compatible with its predecessor.
-#Lineage: [{...}, ...{...}]
-
-#LastSchema: {
-	_p: #Lineage
-	_p[len(_p)-1]
-}
-
-// A Migration defines a relation between two schemas, "_from" and "_to". The
-// relation expresses any complex mappings that must be performed to
-// transform an input artifact valid with respect to the _from schema, into
-// an artifact valid with respect to the _to schema. This is accomplished
-// in two stages:
-//   1. A Migration is initially defined by passing in schemas for _from and _to,
-//      and mappings that translate _from to _to are defined in _rel. 
-//   2. A concrete object may then be unified with _to, resulting in its values
-//      being mapped onto "result" by way of _rel.
-//
-// This is the absolute simplest possible definition of a Migration. It's
-// incumbent on the implementor to manually ensure the correctness and
-// completeness of the mapping. The primary value in defining such a generic
-// structure is to allow comparably generic logic for migrating concrete
-// artifacts through schema changes.
-//
-// If _to isn't backwards compatible (accretion-only) with _from, then _rel must
-// explicitly enumerate every field in _from and map it to a field in _to, even
-// if they're identical. This is laborious for anything outside trivially tiny
-// schema. We'll want to eventually add helpers for whitelisting or blacklisting
-// of paths in _from, so that migrations of larger schema can focus narrowly on
-// the points of actual change.
-#Migration: {
-	from: {...}
-	to: {...}
-	rel: {...}
-	result: to & rel
-}