Geomap: add spatial transformers (alpha) (#44020)

This commit is contained in:
Ryan McKinley
2022-01-21 14:27:26 -08:00
committed by GitHub
parent f53b3fb007
commit 082b1b4db7
17 changed files with 567 additions and 18 deletions

View File

@@ -0,0 +1,165 @@
import React, { useEffect } from 'react';
import { css } from '@emotion/css';
import {
DataTransformerID,
FrameGeometrySource,
FrameGeometrySourceMode,
GrafanaTheme2,
PanelOptionsEditorBuilder,
PluginState,
StandardEditorContext,
TransformerRegistryItem,
TransformerUIProps,
} from '@grafana/data';
import { isLineBuilderOption, spatialTransformer } from './spatialTransformer';
import { addLocationFields } from 'app/features/geo/editor/locationEditor';
import { getDefaultOptions, getTransformerOptionPane } from './optionsHelper';
import { SpatialCalculation, SpatialOperation, SpatialAction, SpatialTransformOptions } from './models.gen';
import { useTheme2 } from '@grafana/ui';
// Nothing defined in state
const supplier = (
builder: PanelOptionsEditorBuilder<SpatialTransformOptions>,
context: StandardEditorContext<SpatialTransformOptions>
) => {
const options = context.options ?? {};
builder.addSelect({
path: `action`,
name: 'Action',
description: '',
defaultValue: SpatialAction.Prepare,
settings: {
options: [
{
value: SpatialAction.Prepare,
label: 'Prepare spatial field',
description: 'Set a geometry field based on the results of other fields',
},
{
value: SpatialAction.Calculate,
label: 'Calculate value',
description: 'Use the geometry to define a new field (heading/distance/area)',
},
{ value: SpatialAction.Modify, label: 'Transform', description: 'Apply spatial operations to the geometry' },
],
},
});
if (options.action === SpatialAction.Calculate) {
builder.addSelect({
path: `calculate.calc`,
name: 'Function',
description: '',
defaultValue: SpatialCalculation.Heading,
settings: {
options: [
{ value: SpatialCalculation.Heading, label: 'Heading' },
{ value: SpatialCalculation.Area, label: 'Area' },
{ value: SpatialCalculation.Distance, label: 'Distance' },
],
},
});
} else if (options.action === SpatialAction.Modify) {
builder.addSelect({
path: `modify.op`,
name: 'Operation',
description: '',
defaultValue: SpatialOperation.AsLine,
settings: {
options: [
{
value: SpatialOperation.AsLine,
label: 'As line',
description: 'Create a single line feature with a vertex at each row',
},
{
value: SpatialOperation.LineBuilder,
label: 'Line builder',
description: 'Create a line between two points',
},
],
},
});
}
if (isLineBuilderOption(options)) {
builder.addNestedOptions({
category: ['Source'],
path: 'source',
build: (b, c) => {
const loc = (options.source ?? {}) as FrameGeometrySource;
if (!loc.mode) {
loc.mode = FrameGeometrySourceMode.Auto;
}
addLocationFields('Point', '', b, loc);
},
});
builder.addNestedOptions({
category: ['Target'],
path: 'modify',
build: (b, c) => {
const loc = (options.modify?.target ?? {}) as FrameGeometrySource;
if (!loc.mode) {
loc.mode = FrameGeometrySourceMode.Auto;
}
addLocationFields('Point', 'target.', b, loc);
},
});
} else {
addLocationFields('Location', 'source.', builder, options.source);
}
};
export const SetGeometryTransformerEditor: React.FC<TransformerUIProps<SpatialTransformOptions>> = (props) => {
// a new component is created with every change :(
useEffect(() => {
if (!props.options.source?.mode) {
const opts = getDefaultOptions(supplier);
props.onChange({ ...opts, ...props.options });
console.log('geometry useEffect', opts);
}
});
const styles = getStyles(useTheme2());
const pane = getTransformerOptionPane<SpatialTransformOptions>(props, supplier);
return (
<div>
<div>{pane.items.map((v) => v.render())}</div>
<div>
{pane.categories.map((c) => {
return (
<div key={c.props.id} className={styles.wrap}>
<h5>{c.props.title}</h5>
<div className={styles.item}>{c.items.map((s) => s.render())}</div>
</div>
);
})}
</div>
</div>
);
};
const getStyles = (theme: GrafanaTheme2) => {
return {
wrap: css`
margin-bottom: 20px;
`,
item: css`
border-left: 4px solid ${theme.colors.border.strong};
padding-left: 10px;
`,
};
};
export const spatialTransformRegistryItem: TransformerRegistryItem<SpatialTransformOptions> = {
id: DataTransformerID.spatial,
editor: SetGeometryTransformerEditor,
transformation: spatialTransformer,
name: spatialTransformer.name,
description: spatialTransformer.description,
state: PluginState.alpha,
};

View File

@@ -0,0 +1,45 @@
import { FrameGeometrySource, FrameGeometrySourceMode } from '@grafana/data';
// This file should be generated by cue schema
export enum SpatialAction {
Prepare = 'prepare',
Calculate = 'calculate',
Modify = 'modify',
}
export enum SpatialCalculation {
Heading = 'heading',
Distance = 'distance',
Area = 'area',
}
export enum SpatialOperation {
AsLine = 'asLine',
LineBuilder = 'lineBuilder',
}
export interface SpatialCalculationOption {
calc?: SpatialCalculation;
field?: string;
}
export interface ModifyOptions {
op: SpatialOperation;
target?: FrameGeometrySource;
}
/** The main transformer options */
export interface SpatialTransformOptions {
action?: SpatialAction;
source?: FrameGeometrySource;
calculate?: SpatialCalculationOption;
modify?: ModifyOptions;
}
export const defaultOptions: SpatialTransformOptions = {
action: SpatialAction.Prepare,
source: {
mode: FrameGeometrySourceMode.Auto,
},
};

View File

@@ -0,0 +1,60 @@
import { StandardEditorContext, TransformerUIProps, PanelOptionsEditorBuilder } from '@grafana/data';
import { OptionsPaneCategoryDescriptor } from 'app/features/dashboard/components/PanelEditor/OptionsPaneCategoryDescriptor';
import { PanelOptionsSupplier } from '@grafana/data/src/panel/PanelPlugin';
import { NestedValueAccess } from '@grafana/data/src/utils/OptionsUIBuilders';
import { set, get as lodashGet } from 'lodash';
import { setOptionImmutably } from 'app/features/dashboard/components/PanelEditor/utils';
import { fillOptionsPaneItems } from 'app/features/dashboard/components/PanelEditor/getVisualizationOptions';
import { SpatialTransformOptions } from './models.gen';
export function getTransformerOptionPane<T = any>(
props: TransformerUIProps<SpatialTransformOptions>,
supplier: PanelOptionsSupplier<T>
): OptionsPaneCategoryDescriptor {
const context: StandardEditorContext<unknown, unknown> = {
data: props.input,
options: props.options,
};
const root = new OptionsPaneCategoryDescriptor({ id: 'root', title: 'root' });
const getOptionsPaneCategory = (categoryNames?: string[]): OptionsPaneCategoryDescriptor => {
if (categoryNames?.length) {
const key = categoryNames[0];
let sub = root.categories.find((v) => v.props.id === key);
if (!sub) {
sub = new OptionsPaneCategoryDescriptor({ id: key, title: key });
root.categories.push(sub);
}
return sub;
}
return root;
};
const access: NestedValueAccess = {
getValue: (path: string) => lodashGet(props.options, path),
onChange: (path: string, value: any) => {
props.onChange(setOptionImmutably(props.options as any, path, value));
},
};
// Use the panel options loader
fillOptionsPaneItems(supplier, access, getOptionsPaneCategory, context);
return root;
}
export function getDefaultOptions<T = any>(supplier: PanelOptionsSupplier<T>): T {
const context: StandardEditorContext<T, unknown> = {
data: [],
options: {} as T,
};
const results = {};
const builder = new PanelOptionsEditorBuilder<T>();
supplier(builder, context);
for (const item of builder.getItems()) {
if (item.defaultValue != null) {
set(results, item.path, item.defaultValue);
}
}
return results as T;
}

View File

@@ -0,0 +1,40 @@
import { FieldMatcherID, fieldMatchers, FieldType } from '@grafana/data';
import { toDataFrame } from '@grafana/data/src/dataframe/processDataFrame';
import { DataTransformerID } from '@grafana/data/src/transformations/transformers/ids';
import { frameAsGazetter } from 'app/features/geo/gazetteer/gazetteer';
describe('spatial transformer', () => {
it('adds lat/lon based on string field', async () => {
const cfg = {
id: DataTransformerID.spatial,
options: {
lookupField: 'location',
gazetteer: 'public/gazetteer/usa-states.json',
},
};
const data = toDataFrame({
name: 'locations',
fields: [
{ name: 'location', type: FieldType.string, values: ['AL', 'AK', 'Arizona', 'Arkansas', 'Somewhere'] },
{ name: 'values', type: FieldType.number, values: [0, 10, 5, 1, 5] },
],
});
const matcher = fieldMatchers.get(FieldMatcherID.byName).get(cfg.options?.lookupField);
const frame = toDataFrame({
fields: [
{ name: 'id', values: ['AL', 'AK', 'AZ'] },
{ name: 'name', values: ['Alabama', 'Arkansas', 'Arizona'] },
{ name: 'lng', values: [-80.891064, -100.891064, -111.891064] },
{ name: 'lat', values: [12.448457, 24.448457, 33.448457] },
],
});
const gaz = frameAsGazetter(frame, { path: 'path/to/gaz.json' });
// TODO!!!!!
expect(gaz).toBeDefined();
expect(matcher).toBeDefined();
expect(data).toBeDefined();
});
});

View File

@@ -0,0 +1,81 @@
import { ArrayVector, DataFrame, DataTransformerID, DataTransformerInfo, FieldType } from '@grafana/data';
import { createLineBetween } from 'app/features/geo/format/utils';
import { getGeometryField, getLocationMatchers } from 'app/features/geo/utils/location';
import { mergeMap, from } from 'rxjs';
import { SpatialOperation, SpatialAction, SpatialTransformOptions } from './models.gen';
import { doGeomeryCalculation, toLineString } from './utils';
export const spatialTransformer: DataTransformerInfo<SpatialTransformOptions> = {
id: DataTransformerID.spatial,
name: 'Spatial operations',
description: 'Apply spatial operations to query results',
defaultOptions: {},
operator: (options) => (source) => source.pipe(mergeMap((data) => from(doSetGeometry(data, options)))),
};
export function isLineBuilderOption(options: SpatialTransformOptions): boolean {
return options.action === SpatialAction.Modify && options.modify?.op === SpatialOperation.LineBuilder;
}
async function doSetGeometry(frames: DataFrame[], options: SpatialTransformOptions): Promise<DataFrame[]> {
const location = await getLocationMatchers(options.source);
if (isLineBuilderOption(options)) {
const targetLocation = await getLocationMatchers(options.modify?.target);
return frames.map((frame) => {
const src = getGeometryField(frame, location);
const target = getGeometryField(frame, targetLocation);
if (src.field && target.field) {
const line = createLineBetween(src.field, target.field);
return {
...frame,
fields: [line, ...frame.fields],
};
}
return frame;
});
}
return frames.map((frame) => {
let info = getGeometryField(frame, location);
if (info.field) {
if (options.action === SpatialAction.Modify) {
switch (options.modify?.op) {
// SOON: extent, convex hull, etc
case SpatialOperation.AsLine:
let name = info.field.name;
if (!name || name === 'Point') {
name = 'Line';
}
return {
...frame,
fields: [
{
...info.field,
name,
parse: undefined,
type: FieldType.geo,
values: new ArrayVector([toLineString(info.field)]),
},
],
};
}
return frame;
}
const fields = info.derived ? [info.field, ...frame.fields] : frame.fields.slice(0);
if (options.action === SpatialAction.Calculate) {
fields.push(doGeomeryCalculation(info.field, options.calculate ?? {}));
info.derived = true;
}
if (info.derived) {
return {
...frame,
fields,
};
}
}
return frame;
});
}

View File

@@ -0,0 +1,108 @@
import { ArrayVector, Field, FieldType } from '@grafana/data';
import { getCenter } from 'ol/extent';
import { Geometry, LineString, Point } from 'ol/geom';
import { toLonLat } from 'ol/proj';
import { getArea, getLength } from 'ol/sphere';
import { SpatialCalculation, SpatialCalculationOption } from './models.gen';
/** Will return a field with a single row */
export function toLineString(field: Field<Geometry | undefined>): LineString {
const coords: number[][] = [];
for (const geo of field.values.toArray()) {
if (geo) {
coords.push(getCenterPoint(geo));
}
}
return new LineString(coords);
}
/** Will return a field with a single row */
export function calculateBearings(values: Array<Geometry | undefined>): number[] {
const bearing = new Array(values.length);
if (values.length > 1) {
let prev: number[] | undefined = getCenterPointWGS84(values[0]);
for (let i = 1; i < values.length; i++) {
let next: number[] | undefined = getCenterPointWGS84(values[i]);
if (prev && next) {
let degrees = (Math.atan2(next[0] - prev[0], next[1] - prev[1]) * 180) / Math.PI;
if (degrees < 0.0) {
degrees += 360.0;
}
bearing[i - 1] = bearing[i] = degrees;
}
}
} else {
bearing.fill(0);
}
return bearing;
}
export function getCenterPoint(geo: Geometry): number[] {
if (geo instanceof Point) {
return (geo as Point).getCoordinates();
}
return getCenter(geo.getExtent());
}
export function getCenterPointWGS84(geo?: Geometry): number[] | undefined {
if (!geo) {
return undefined;
}
return toLonLat(getCenterPoint(geo));
}
/** Will return a new field with calculated values */
export function doGeomeryCalculation(field: Field<Geometry | undefined>, options: SpatialCalculationOption): Field {
const values = field.values.toArray();
const buffer = new Array(field.values.length);
const op = options.calc ?? SpatialCalculation.Heading;
const name = options.field ?? op;
switch (op) {
case SpatialCalculation.Area: {
for (let i = 0; i < values.length; i++) {
const geo = values[i];
if (geo) {
buffer[i] = getArea(geo);
}
}
return {
name,
type: FieldType.number,
config: {
unit: 'areaM2',
},
values: new ArrayVector(buffer),
};
}
case SpatialCalculation.Distance: {
for (let i = 0; i < values.length; i++) {
const geo = values[i];
if (geo) {
buffer[i] = getLength(geo);
}
}
return {
name,
type: FieldType.number,
config: {
unit: 'lengthm',
},
values: new ArrayVector(buffer),
};
}
// Use heading as default
case SpatialCalculation.Heading:
default: {
return {
name,
type: FieldType.number,
config: {
unit: 'degree',
},
values: new ArrayVector(calculateBearings(values)),
};
}
}
}

View File

@@ -20,6 +20,7 @@ import { prepareTimeseriesTransformerRegistryItem } from '../components/Transfor
import { convertFieldTypeTransformRegistryItem } from '../components/TransformersUI/ConvertFieldTypeTransformerEditor';
import { fieldLookupTransformRegistryItem } from '../components/TransformersUI/lookupGazetteer/FieldLookupTransformerEditor';
import { extractFieldsTransformRegistryItem } from '../components/TransformersUI/extractFields/ExtractFieldsTransformerEditor';
import { spatialTransformRegistryItem } from '../components/TransformersUI/spatial/SpatialTransformerEditor';
export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> => {
return [
@@ -42,6 +43,7 @@ export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> =
configFromQueryTransformRegistryItem,
prepareTimeseriesTransformerRegistryItem,
convertFieldTypeTransformRegistryItem,
spatialTransformRegistryItem,
fieldLookupTransformRegistryItem,
extractFieldsTransformRegistryItem,
];

View File

@@ -8,13 +8,14 @@ import {
import { GazetteerPathEditor } from 'app/features/geo/editor/GazetteerPathEditor';
export function addLocationFields<TOptions>(
title: string,
prefix: string,
builder: PanelOptionsEditorBuilder<TOptions>,
source?: FrameGeometrySource
) {
builder.addRadio({
path: `${prefix}.mode`,
name: 'Location',
path: `${prefix}mode`,
name: title,
description: '',
defaultValue: FrameGeometrySourceMode.Auto,
settings: {
@@ -31,7 +32,7 @@ export function addLocationFields<TOptions>(
case FrameGeometrySourceMode.Coords:
builder
.addFieldNamePicker({
path: `${prefix}.latitude`,
path: `${prefix}latitude`,
name: 'Latitude field',
settings: {
filter: (f: Field) => f.type === FieldType.number,
@@ -39,7 +40,7 @@ export function addLocationFields<TOptions>(
},
})
.addFieldNamePicker({
path: `${prefix}.longitude`,
path: `${prefix}longitude`,
name: 'Longitude field',
settings: {
filter: (f: Field) => f.type === FieldType.number,
@@ -62,7 +63,7 @@ export function addLocationFields<TOptions>(
case FrameGeometrySourceMode.Lookup:
builder
.addFieldNamePicker({
path: `${prefix}.lookup`,
path: `${prefix}lookup`,
name: 'Lookup field',
settings: {
filter: (f: Field) => f.type === FieldType.string,
@@ -71,7 +72,7 @@ export function addLocationFields<TOptions>(
})
.addCustomEditor({
id: 'gazetteer',
path: `${prefix}.gazetteer`,
path: `${prefix}gazetteer`,
name: 'Gazetteer',
editor: GazetteerPathEditor,
});

View File

@@ -11,7 +11,7 @@ interface FieldInfo {
// http://geojson.xyz/
export function frameFromGeoJSON(body: Document | Element | Object | string): DataFrame {
const data = new GeoJSON().readFeatures(body);
const data = new GeoJSON().readFeatures(body, { featureProjection: 'EPSG:3857' });
const length = data.length;
const geo: Geometry[] = new Array(length).fill(null);
@@ -58,7 +58,7 @@ export function frameFromGeoJSON(body: Document | Element | Object | string): Da
for (const key of feature.getKeys()) {
const val = feature.get(key);
if (key === 'geometry' && val === geo[i]) {
if (val === geo[i] || val == null) {
continue;
}
const field = getField(key);

View File

@@ -1,12 +1,13 @@
import { ArrayVector, Field, FieldConfig, FieldType } from '@grafana/data';
import { Geometry, Point } from 'ol/geom';
import { getCenterPoint } from 'app/core/components/TransformersUI/spatial/utils';
import { Geometry, LineString, Point } from 'ol/geom';
import { fromLonLat } from 'ol/proj';
import { Gazetteer } from '../gazetteer/gazetteer';
import { decodeGeohash } from './geohash';
export function pointFieldFromGeohash(geohash: Field<string>): Field<Point> {
return {
name: 'point',
name: geohash.name ?? 'Point',
type: FieldType.geo,
values: new ArrayVector<any>(
geohash.values.toArray().map((v) => {
@@ -28,7 +29,7 @@ export function pointFieldFromLonLat(lon: Field, lat: Field): Field<Point> {
}
return {
name: 'point',
name: 'Point',
type: FieldType.geo,
values: new ArrayVector(buffer),
config: hiddenTooltipField,
@@ -49,6 +50,36 @@ export function getGeoFieldFromGazetteer(gaz: Gazetteer, field: Field<string>):
};
}
export function createLineBetween(
src: Field<Geometry | undefined>,
dest: Field<Geometry | undefined>
): Field<Geometry | undefined> {
const v0 = src.values.toArray();
const v1 = dest.values.toArray();
if (!v0 || !v1) {
throw 'missing src/dest';
}
if (v0.length !== v1.length) {
throw 'Source and destination field lengths do not match';
}
const count = src.values.length!;
const geo = new Array<Geometry | undefined>(count);
for (let i = 0; i < count; i++) {
const a = v0[i];
const b = v1[i];
if (a && b) {
geo[i] = new LineString([getCenterPoint(a), getCenterPoint(b)]);
}
}
return {
name: 'Geometry',
type: FieldType.geo,
values: new ArrayVector(geo),
config: hiddenTooltipField,
};
}
const hiddenTooltipField: FieldConfig = Object.freeze({
custom: {
hideFrom: { tooltip: true },

View File

@@ -1,3 +1,4 @@
import { getCenterPointWGS84 } from 'app/core/components/TransformersUI/spatial/utils';
import { getGazetteer } from './gazetteer';
let backendResults: any = { hello: 'world' };
@@ -57,7 +58,7 @@ describe('Placename lookup from geojson format', () => {
backendResults = geojsonObject;
const gaz = await getGazetteer('local');
expect(gaz.error).toBeUndefined();
expect(gaz.find('A')?.point()?.getCoordinates()).toMatchInlineSnapshot(`
expect(getCenterPointWGS84(gaz.find('A')?.geometry())).toMatchInlineSnapshot(`
Array [
0,
0,
@@ -68,7 +69,7 @@ describe('Placename lookup from geojson format', () => {
backendResults = geojsonObject;
const gaz = await getGazetteer('airports');
expect(gaz.error).toBeUndefined();
expect(gaz.find('B')?.point()?.getCoordinates()).toMatchInlineSnapshot(`
expect(getCenterPointWGS84(gaz.find('B')?.geometry())).toMatchInlineSnapshot(`
Array [
1,
1,
@@ -80,7 +81,7 @@ describe('Placename lookup from geojson format', () => {
backendResults = geojsonObject;
const gaz = await getGazetteer('airports');
expect(gaz.error).toBeUndefined();
expect(gaz.find('C')?.point()?.getCoordinates()).toMatchInlineSnapshot(`
expect(getCenterPointWGS84(gaz.find('C')?.geometry())).toMatchInlineSnapshot(`
Array [
2,
2,

View File

@@ -161,7 +161,16 @@ export function frameAsGazetter(frame: DataFrame, opts: { path: string; keys?: s
}
return undefined;
},
examples: (v) => [],
examples: (v) => {
const ex: string[] = [];
for (let k of lookup.keys()) {
ex.push(k);
if (ex.length > v) {
break;
}
}
return ex;
},
frame: () => frame,
count: frame.length,
};

View File

@@ -76,7 +76,7 @@ export function getLayerEditor(opts: LayerEditorOptions): NestedPanelOptions<Map
}
if (layer.showLocation) {
addLocationFields('location', builder, options.location);
addLocationFields('Location', 'location', builder, options.location);
}
if (handler.registerOptionsUI) {
handler.registerOptionsUI(builder);

View File

@@ -65,13 +65,15 @@ export const textMarker = (cfg: StyleConfigValues) => {
};
export const circleMarker = (cfg: StyleConfigValues) => {
const stroke = new Stroke({ color: cfg.color, width: cfg.lineWidth ?? 1 });
return new Style({
image: new Circle({
stroke: new Stroke({ color: cfg.color, width: cfg.lineWidth ?? 1 }),
stroke,
fill: getFillColor(cfg),
radius: cfg.size ?? DEFAULT_SIZE,
}),
text: textLabel(cfg),
stroke, // in case lines are sent to the markers layer
});
};