Transformations: Add gazetteer transformation (#40967)

2025-02-25 18:55:37 -06:00 · 2021-10-27 20:20:56 -07:00
parent 45e1765733
commit 963544ffe6
7 changed files with 293 additions and 5 deletions
--- a/packages/grafana-data/src/transformations/transformers/ids.ts
+++ b/packages/grafana-data/src/transformations/transformers/ids.ts
@@ -27,4 +27,5 @@ export enum DataTransformerID {
  rowsToFields = 'rowsToFields',
  prepareTimeSeries = 'prepareTimeSeries',
  convertFieldType = 'convertFieldType',
+  fieldLookup = 'fieldLookup',
 }
--- a/public/app/core/components/TransformersUI/lookupGazetteer/FieldLookupTransformerEditor.tsx
+++ b/public/app/core/components/TransformersUI/lookupGazetteer/FieldLookupTransformerEditor.tsx
@@ -0,0 +1,91 @@
+import React, { useCallback } from 'react';
+import {
+  DataTransformerID,
+  FieldNamePickerConfigSettings,
+  PluginState,
+  StandardEditorsRegistryItem,
+  TransformerRegistryItem,
+  TransformerUIProps,
+} from '@grafana/data';
+
+import { InlineField, InlineFieldRow } from '@grafana/ui';
+import { FieldNamePicker } from '@grafana/ui/src/components/MatchersUI/FieldNamePicker';
+import { GazetteerPathEditor } from 'app/plugins/panel/geomap/editor/GazetteerPathEditor';
+import { GazetteerPathEditorConfigSettings } from 'app/plugins/panel/geomap/types';
+import { FieldLookupOptions, fieldLookupTransformer } from './fieldLookup';
+import { FieldType } from '../../../../../../packages/grafana-data/src';
+
+const fieldNamePickerSettings: StandardEditorsRegistryItem<string, FieldNamePickerConfigSettings> = {
+  settings: {
+    width: 30,
+    filter: (f) => f.type === FieldType.string,
+    placeholderText: 'Select text field',
+    noFieldsMessage: 'No text fields found',
+  },
+  name: '',
+  id: '',
+  editor: () => null,
+};
+
+const fieldLookupSettings: StandardEditorsRegistryItem<string, GazetteerPathEditorConfigSettings> = {
+  settings: {},
+} as any;
+
+export const FieldLookupTransformerEditor: React.FC<TransformerUIProps<FieldLookupOptions>> = ({
+  input,
+  options,
+  onChange,
+}) => {
+  const onPickLookupField = useCallback(
+    (value: string | undefined) => {
+      onChange({
+        ...options,
+        lookupField: value,
+      });
+    },
+    [onChange, options]
+  );
+
+  const onPickGazetteer = useCallback(
+    (value: string | undefined) => {
+      onChange({
+        ...options,
+        gazetteer: value,
+      });
+    },
+    [onChange, options]
+  );
+  return (
+    <div>
+      <InlineFieldRow>
+        <InlineField label={'Field'} labelWidth={12}>
+          <FieldNamePicker
+            context={{ data: input }}
+            value={options?.lookupField ?? ''}
+            onChange={onPickLookupField}
+            item={fieldNamePickerSettings as any}
+          />
+        </InlineField>
+      </InlineFieldRow>
+      <InlineFieldRow>
+        <InlineField label={'Lookup'} labelWidth={12}>
+          <GazetteerPathEditor
+            value={options?.gazetteer ?? ''}
+            context={{ data: input }}
+            item={fieldLookupSettings}
+            onChange={onPickGazetteer}
+          />
+        </InlineField>
+      </InlineFieldRow>
+    </div>
+  );
+};
+
+export const fieldLookupTransformRegistryItem: TransformerRegistryItem<FieldLookupOptions> = {
+  id: DataTransformerID.fieldLookup,
+  editor: FieldLookupTransformerEditor,
+  transformation: fieldLookupTransformer,
+  name: 'Field lookup',
+  description: `Use a field value to lookup additional fields from an external source.  This current supports spatial data, but will eventuall support more formats`,
+  state: PluginState.alpha,
+};
--- a/public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.test.ts
+++ b/public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.test.ts
@@ -0,0 +1,115 @@
+import { FieldMatcherID, fieldMatchers, FieldType } from '@grafana/data';
+import { toDataFrame } from '@grafana/data/src/dataframe/processDataFrame';
+import { DataTransformerID } from '@grafana/data/src/transformations/transformers/ids';
+import { Gazetteer } from 'app/plugins/panel/geomap/gazetteer/gazetteer';
+import { addFieldsFromGazetteer } from './fieldLookup';
+
+describe('Lookup gazetteer', () => {
+  it('adds lat/lon based on string field', async () => {
+    const cfg = {
+      id: DataTransformerID.fieldLookup,
+      options: {
+        lookupField: 'location',
+        gazetteer: 'public/gazetteer/usa-states.json',
+      },
+    };
+    const data = toDataFrame({
+      name: 'locations',
+      fields: [
+        { name: 'location', type: FieldType.string, values: ['AL', 'AK', 'Arizona', 'Arkansas', 'Somewhere'] },
+        { name: 'values', type: FieldType.number, values: [0, 10, 5, 1, 5] },
+      ],
+    });
+
+    const matcher = fieldMatchers.get(FieldMatcherID.byName).get(cfg.options?.lookupField);
+
+    const values = new Map()
+      .set('AL', { name: 'Alabama', id: 'AL', coords: [-80.891064, 12.448457] })
+      .set('AK', { name: 'Arkansas', id: 'AK', coords: [-100.891064, 24.448457] })
+      .set('AZ', { name: 'Arizona', id: 'AZ', coords: [-111.891064, 33.448457] })
+      .set('Arizona', { name: 'Arizona', id: 'AZ', coords: [-111.891064, 33.448457] });
+
+    const gaz: Gazetteer = {
+      count: 3,
+      examples: () => ['AL', 'AK', 'AZ'],
+      find: (k) => {
+        let v = values.get(k);
+        if (!v && typeof k === 'string') {
+          v = values.get(k.toUpperCase());
+        }
+        return v;
+      },
+      path: 'public/gazetteer/usa-states.json',
+    };
+
+    expect(await addFieldsFromGazetteer([data], gaz, matcher)).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "creator": [Function],
+          "fields": Array [
+            Object {
+              "config": Object {},
+              "name": "location",
+              "type": "string",
+              "values": Array [
+                "AL",
+                "AK",
+                "Arizona",
+                "Arkansas",
+                "Somewhere",
+              ],
+            },
+            Object {
+              "config": Object {},
+              "name": "lon",
+              "type": "number",
+              "values": Array [
+                -80.891064,
+                -100.891064,
+                -111.891064,
+                undefined,
+                undefined,
+              ],
+            },
+            Object {
+              "config": Object {},
+              "name": "lat",
+              "type": "number",
+              "values": Array [
+                12.448457,
+                24.448457,
+                33.448457,
+                undefined,
+                undefined,
+              ],
+            },
+            Object {
+              "config": Object {},
+              "name": "values",
+              "state": Object {
+                "displayName": "values",
+              },
+              "type": "number",
+              "values": Array [
+                0,
+                10,
+                5,
+                1,
+                5,
+              ],
+            },
+          ],
+          "first": Array [
+            "AL",
+            "AK",
+            "Arizona",
+            "Arkansas",
+            "Somewhere",
+          ],
+          "length": 5,
+          "name": "locations",
+        },
+      ]
+    `);
+  });
+});
--- a/public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.ts
+++ b/public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.ts
@@ -0,0 +1,69 @@
+import {
+  ArrayVector,
+  DataFrame,
+  DataTransformerID,
+  Field,
+  FieldMatcher,
+  FieldMatcherID,
+  fieldMatchers,
+  FieldType,
+  DataTransformerInfo,
+} from '@grafana/data';
+import { COUNTRIES_GAZETTEER_PATH, Gazetteer, getGazetteer } from 'app/plugins/panel/geomap/gazetteer/gazetteer';
+import { mergeMap, from } from 'rxjs';
+
+export interface FieldLookupOptions {
+  lookupField?: string;
+  gazetteer?: string;
+}
+
+export const fieldLookupTransformer: DataTransformerInfo<FieldLookupOptions> = {
+  id: DataTransformerID.fieldLookup,
+  name: 'Lookup fields from resource',
+  description: 'Retrieve matching data based on specified field',
+  defaultOptions: {},
+
+  operator: (options) => (source) => source.pipe(mergeMap((data) => from(doGazetteerXform(data, options)))),
+};
+
+async function doGazetteerXform(frames: DataFrame[], options: FieldLookupOptions): Promise<DataFrame[]> {
+  const fieldMatches = fieldMatchers.get(FieldMatcherID.byName).get(options?.lookupField);
+
+  const gaz = await getGazetteer(options?.gazetteer ?? COUNTRIES_GAZETTEER_PATH);
+
+  return addFieldsFromGazetteer(frames, gaz, fieldMatches);
+}
+
+export function addFieldsFromGazetteer(frames: DataFrame[], gaz: Gazetteer, matcher: FieldMatcher): DataFrame[] {
+  return frames.map((frame) => {
+    const fields: Field[] = [];
+
+    for (const field of frame.fields) {
+      fields.push(field);
+
+      //if the field matches
+      if (matcher(field, frame, frames)) {
+        const values = field.values.toArray();
+        const lat = new Array<Number>(values.length);
+        const lon = new Array<Number>(values.length);
+
+        //for each value find the corresponding value in the gazetteer
+        for (let v = 0; v < values.length; v++) {
+          const foundMatchingValue = gaz.find(values[v]);
+
+          //for now start by adding lat and lon
+          if (foundMatchingValue && foundMatchingValue?.coords.length) {
+            lon[v] = foundMatchingValue.coords[0];
+            lat[v] = foundMatchingValue.coords[1];
+          }
+        }
+        fields.push({ name: 'lon', type: FieldType.number, values: new ArrayVector(lon), config: {} });
+        fields.push({ name: 'lat', type: FieldType.number, values: new ArrayVector(lat), config: {} });
+      }
+    }
+    return {
+      ...frame,
+      fields,
+    };
+  });
+}
--- a/public/app/core/utils/standardTransformers.ts
+++ b/public/app/core/utils/standardTransformers.ts
@@ -18,6 +18,7 @@ import { rowsToFieldsTransformRegistryItem } from '../components/TransformersUI/
 import { configFromQueryTransformRegistryItem } from '../components/TransformersUI/configFromQuery/ConfigFromQueryTransformerEditor';
 import { prepareTimeseriesTransformerRegistryItem } from '../components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor';
 import { convertFieldTypeTransformRegistryItem } from '../components/TransformersUI/ConvertFieldTypeTransformerEditor';
+import { fieldLookupTransformRegistryItem } from '../components/TransformersUI/lookupGazetteer/FieldLookupTransformerEditor';

 export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> => {
  return [
@@ -40,5 +41,6 @@ export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> =
    configFromQueryTransformRegistryItem,
    prepareTimeseriesTransformerRegistryItem,
    convertFieldTypeTransformRegistryItem,
+    fieldLookupTransformRegistryItem,
  ];
 };
--- a/public/app/plugins/panel/geomap/editor/GazetteerPathEditor.tsx
+++ b/public/app/plugins/panel/geomap/editor/GazetteerPathEditor.tsx
@@ -3,8 +3,9 @@ import { StandardEditorProps, SelectableValue, GrafanaTheme2 } from '@grafana/da
 import { Alert, Select, stylesFactory, useTheme2 } from '@grafana/ui';
 import { COUNTRIES_GAZETTEER_PATH, Gazetteer, getGazetteer } from '../gazetteer/gazetteer';
 import { css } from '@emotion/css';
+import { GazetteerPathEditorConfigSettings } from '../types';

-const paths: Array<SelectableValue<string>> = [
+const defaultPaths: Array<SelectableValue<string>> = [
  {
    label: 'Countries',
    description: 'Lookup countries by name, two letter code, or three letter code',
@@ -22,9 +23,15 @@ const paths: Array<SelectableValue<string>> = [
  },
 ];

-export const GazetteerPathEditor: FC<StandardEditorProps<string, any, any>> = ({ value, onChange, context }) => {
+export const GazetteerPathEditor: FC<StandardEditorProps<string, any, any, GazetteerPathEditorConfigSettings>> = ({
+  value,
+  onChange,
+  context,
+  item,
+}) => {
  const styles = getStyles(useTheme2());
  const [gaz, setGaz] = useState<Gazetteer>();
+  const settings = item.settings as any;

  useEffect(() => {
    async function fetchData() {
@@ -35,7 +42,7 @@ export const GazetteerPathEditor: FC<StandardEditorProps<string, any, any>> = ({
  }, [value, setGaz]);

  const { current, options } = useMemo(() => {
-    let options = [...paths];
+    let options = settings?.options ? [...settings.options] : [...defaultPaths];
    let current = options.find((f) => f.value === gaz?.path);
    if (!current && gaz) {
      current = {
@@ -45,7 +52,7 @@ export const GazetteerPathEditor: FC<StandardEditorProps<string, any, any>> = ({
      options.push(current);
    }
    return { options, current };
-  }, [gaz]);
+  }, [gaz, settings.options]);

  return (
    <>
--- a/public/app/plugins/panel/geomap/types.ts
+++ b/public/app/plugins/panel/geomap/types.ts
@@ -1,4 +1,4 @@
-import { MapLayerHandler, MapLayerOptions } from '@grafana/data';
+import { MapLayerHandler, MapLayerOptions, SelectableValue } from '@grafana/data';
 import BaseLayer from 'ol/layer/Base';
 import { Units } from 'ol/proj/Units';
 import { Style } from 'ol/style';
@@ -64,6 +64,9 @@ export enum ComparisonOperation {
  GTE = 'gte',
 }

+export interface GazetteerPathEditorConfigSettings {
+  options?: Array<SelectableValue<string>>;
+}
 //-------------------
 // Runtime model
 //-------------------