Gazetteer: reactor so the source is a DataFrame (#43783)

.github/CODEOWNERS

@@ -96,6 +96,7 @@ go.sum @grafana/backend-platform
 /public/app/core/components/Layers @grafana/grafana-edge-squad
 /public/app/features/canvas/ @grafana/grafana-edge-squad
 /public/app/features/dimensions/ @grafana/grafana-edge-squad
+/public/app/features/geo/ @grafana/grafana-edge-squad
 /public/app/features/live/ @grafana/grafana-edge-squad
 /public/app/features/explore/ @grafana/observability-squad
 /public/app/features/plugins @grafana/plugins-platform-frontend

packages/grafana-data/src/transformations/transformers/ids.ts

@@ -28,5 +28,6 @@ export enum DataTransformerID {
   prepareTimeSeries = 'prepareTimeSeries',
   convertFieldType = 'convertFieldType',
   fieldLookup = 'fieldLookup',
+  setGeometry = 'setGeometry',
   extractFields = 'extractFields',
 }

packages/grafana-data/src/types/dataFrame.ts

@@ -15,6 +15,7 @@ export enum FieldType {
   boolean = 'boolean',
   // Used to detect that the value is some kind of trace data to help with the visualisation and processing.
   trace = 'trace',
+  geo = 'geo',
   other = 'other', // Object, Array, etc
 }
 

public/app/core/components/TransformersUI/lookupGazetteer/FieldLookupTransformerEditor.tsx

@@ -6,14 +6,13 @@ import {
   StandardEditorsRegistryItem,
   TransformerRegistryItem,
   TransformerUIProps,
+  FieldType,
 } from '@grafana/data';
 
 import { InlineField, InlineFieldRow } from '@grafana/ui';
 import { FieldNamePicker } from '@grafana/ui/src/components/MatchersUI/FieldNamePicker';
-import { GazetteerPathEditor } from 'app/plugins/panel/geomap/editor/GazetteerPathEditor';
-import { GazetteerPathEditorConfigSettings } from 'app/plugins/panel/geomap/types';
 import { FieldLookupOptions, fieldLookupTransformer } from './fieldLookup';
-import { FieldType } from '../../../../../../packages/grafana-data/src';
+import { GazetteerPathEditor, GazetteerPathEditorConfigSettings } from 'app/features/geo/editor/GazetteerPathEditor';
 
 const fieldNamePickerSettings: StandardEditorsRegistryItem<string, FieldNamePickerConfigSettings> = {
   settings: {

public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.test.ts

@@ -1,7 +1,7 @@
 import { FieldMatcherID, fieldMatchers, FieldType } from '@grafana/data';
 import { toDataFrame } from '@grafana/data/src/dataframe/processDataFrame';
 import { DataTransformerID } from '@grafana/data/src/transformations/transformers/ids';
-import { Gazetteer } from 'app/plugins/panel/geomap/gazetteer/gazetteer';
+import { frameAsGazetter } from 'app/features/geo/gazetteer/gazetteer';
 import { addFieldsFromGazetteer } from './fieldLookup';
 
 describe('Lookup gazetteer', () => {
@@ -23,91 +23,93 @@ describe('Lookup gazetteer', () => {
 
     const matcher = fieldMatchers.get(FieldMatcherID.byName).get(cfg.options?.lookupField);
 
-    const values = new Map()
-      .set('AL', { name: 'Alabama', id: 'AL', coords: [-80.891064, 12.448457] })
-      .set('AK', { name: 'Arkansas', id: 'AK', coords: [-100.891064, 24.448457] })
-      .set('AZ', { name: 'Arizona', id: 'AZ', coords: [-111.891064, 33.448457] })
-      .set('Arizona', { name: 'Arizona', id: 'AZ', coords: [-111.891064, 33.448457] });
+    const frame = toDataFrame({
+      fields: [
+        { name: 'id', values: ['AL', 'AK', 'AZ'] },
+        { name: 'name', values: ['Alabama', 'Arkansas', 'Arizona'] },
+        { name: 'lng', values: [-80.891064, -100.891064, -111.891064] },
+        { name: 'lat', values: [12.448457, 24.448457, 33.448457] },
+      ],
+    });
+    const gaz = frameAsGazetter(frame, { path: 'path/to/gaz.json' });
+    const out = await addFieldsFromGazetteer([data], gaz, matcher)[0];
 
-    const gaz: Gazetteer = {
-      count: 3,
-      examples: () => ['AL', 'AK', 'AZ'],
-      find: (k) => {
-        let v = values.get(k);
-        if (!v && typeof k === 'string') {
-          v = values.get(k.toUpperCase());
-        }
-        return v;
-      },
-      path: 'public/gazetteer/usa-states.json',
-    };
-
-    expect(await addFieldsFromGazetteer([data], gaz, matcher)).toMatchInlineSnapshot(`
+    expect(out.fields).toMatchInlineSnapshot(`
       Array [
         Object {
-          "creator": [Function],
-          "fields": Array [
-            Object {
-              "config": Object {},
-              "name": "location",
-              "type": "string",
-              "values": Array [
-                "AL",
-                "AK",
-                "Arizona",
-                "Arkansas",
-                "Somewhere",
-              ],
-            },
-            Object {
-              "config": Object {},
-              "name": "lon",
-              "type": "number",
-              "values": Array [
-                -80.891064,
-                -100.891064,
-                -111.891064,
-                ,
-                ,
-              ],
-            },
-            Object {
-              "config": Object {},
-              "name": "lat",
-              "type": "number",
-              "values": Array [
-                12.448457,
-                24.448457,
-                33.448457,
-                ,
-                ,
-              ],
-            },
-            Object {
-              "config": Object {},
-              "name": "values",
-              "state": Object {
-                "displayName": "values",
-              },
-              "type": "number",
-              "values": Array [
-                0,
-                10,
-                5,
-                1,
-                5,
-              ],
-            },
-          ],
-          "first": Array [
+          "config": Object {},
+          "name": "location",
+          "type": "string",
+          "values": Array [
             "AL",
             "AK",
             "Arizona",
             "Arkansas",
             "Somewhere",
           ],
-          "length": 5,
-          "name": "locations",
+        },
+        Object {
+          "config": Object {},
+          "name": "id",
+          "type": "string",
+          "values": Array [
+            "AL",
+            "AK",
+            ,
+            ,
+            ,
+          ],
+        },
+        Object {
+          "config": Object {},
+          "name": "name",
+          "type": "string",
+          "values": Array [
+            "Alabama",
+            "Arkansas",
+            ,
+            ,
+            ,
+          ],
+        },
+        Object {
+          "config": Object {},
+          "name": "lng",
+          "type": "number",
+          "values": Array [
+            -80.891064,
+            -100.891064,
+            ,
+            ,
+            ,
+          ],
+        },
+        Object {
+          "config": Object {},
+          "name": "lat",
+          "type": "number",
+          "values": Array [
+            12.448457,
+            24.448457,
+            ,
+            ,
+            ,
+          ],
+        },
+        Object {
+          "config": Object {},
+          "name": "values",
+          "state": Object {
+            "displayName": "values",
+          },
+          "type": "number",
+          "values": Array [
+            0,
+            10,
+            5,
+            1,
+            5,
+          ],
         },
       ]
     `);

public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.ts

@@ -6,10 +6,9 @@ import {
   FieldMatcher,
   FieldMatcherID,
   fieldMatchers,
-  FieldType,
   DataTransformerInfo,
 } from '@grafana/data';
-import { COUNTRIES_GAZETTEER_PATH, Gazetteer, getGazetteer } from 'app/plugins/panel/geomap/gazetteer/gazetteer';
+import { COUNTRIES_GAZETTEER_PATH, Gazetteer, getGazetteer } from 'app/features/geo/gazetteer/gazetteer';
 import { mergeMap, from } from 'rxjs';
 
 export interface FieldLookupOptions {
@@ -31,11 +30,21 @@ async function doGazetteerXform(frames: DataFrame[], options: FieldLookupOptions
 
   const gaz = await getGazetteer(options?.gazetteer ?? COUNTRIES_GAZETTEER_PATH);
 
+  if (!gaz.frame) {
+    return Promise.reject('missing frame in gazetteer');
+  }
+
   return addFieldsFromGazetteer(frames, gaz, fieldMatches);
 }
 
 export function addFieldsFromGazetteer(frames: DataFrame[], gaz: Gazetteer, matcher: FieldMatcher): DataFrame[] {
+  const src = gaz.frame!()?.fields;
+  if (!src) {
+    return frames;
+  }
+
   return frames.map((frame) => {
+    const length = frame.length;
     const fields: Field[] = [];
 
     for (const field of frame.fields) {
@@ -44,21 +53,22 @@ export function addFieldsFromGazetteer(frames: DataFrame[], gaz: Gazetteer, matc
       //if the field matches
       if (matcher(field, frame, frames)) {
         const values = field.values.toArray();
-        const lat = new Array<Number>(values.length);
-        const lon = new Array<Number>(values.length);
+        const sub: any[][] = [];
+        for (const f of src) {
+          const buffer = new Array(length);
+          sub.push(buffer);
+          fields.push({ ...f, values: new ArrayVector(buffer) });
+        }
 
-        //for each value find the corresponding value in the gazetteer
-        for (let v = 0; v < values.length; v++) {
-          const foundMatchingValue = gaz.find(values[v]);
-
-          //for now start by adding lat and lon
-          if (foundMatchingValue && foundMatchingValue?.coords.length) {
-            lon[v] = foundMatchingValue.coords[0];
-            lat[v] = foundMatchingValue.coords[1];
+        // Add all values to the buffer
+        for (let v = 0; v < sub.length; v++) {
+          const found = gaz.find(values[v]);
+          if (found?.index != null) {
+            for (let i = 0; i < src.length; i++) {
+              sub[i][v] = src[i].values.get(found.index);
+            }
           }
         }
-        fields.push({ name: 'lon', type: FieldType.number, values: new ArrayVector(lon), config: {} });
-        fields.push({ name: 'lat', type: FieldType.number, values: new ArrayVector(lat), config: {} });
       }
     }
     return {

public/app/features/geo/editor/GazetteerPathEditor.tsx

@@ -3,7 +3,6 @@ import { StandardEditorProps, SelectableValue, GrafanaTheme2 } from '@grafana/da
 import { Alert, Select, stylesFactory, useTheme2 } from '@grafana/ui';
 import { COUNTRIES_GAZETTEER_PATH, Gazetteer, getGazetteer } from '../gazetteer/gazetteer';
 import { css } from '@emotion/css';
-import { GazetteerPathEditorConfigSettings } from '../types';
 
 const defaultPaths: Array<SelectableValue<string>> = [
   {
@@ -23,6 +22,10 @@ const defaultPaths: Array<SelectableValue<string>> = [
   },
 ];
 
+export interface GazetteerPathEditorConfigSettings {
+  options?: Array<SelectableValue<string>>;
+}
+
 export const GazetteerPathEditor: FC<StandardEditorProps<string, any, any, GazetteerPathEditorConfigSettings>> = ({
   value,
   onChange,

public/app/features/geo/editor/locationEditor.ts

@@ -0,0 +1,79 @@
+import {
+  Field,
+  FieldType,
+  FrameGeometrySource,
+  FrameGeometrySourceMode,
+  PanelOptionsEditorBuilder,
+} from '@grafana/data';
+import { GazetteerPathEditor } from 'app/features/geo/editor/GazetteerPathEditor';
+
+export function addLocationFields<TOptions>(
+  prefix: string,
+  builder: PanelOptionsEditorBuilder<TOptions>,
+  source?: FrameGeometrySource
+) {
+  builder.addRadio({
+    path: `${prefix}.mode`,
+    name: 'Location',
+    description: '',
+    defaultValue: FrameGeometrySourceMode.Auto,
+    settings: {
+      options: [
+        { value: FrameGeometrySourceMode.Auto, label: 'Auto' },
+        { value: FrameGeometrySourceMode.Coords, label: 'Coords' },
+        { value: FrameGeometrySourceMode.Geohash, label: 'Geohash' },
+        { value: FrameGeometrySourceMode.Lookup, label: 'Lookup' },
+      ],
+    },
+  });
+
+  switch (source?.mode) {
+    case FrameGeometrySourceMode.Coords:
+      builder
+        .addFieldNamePicker({
+          path: `${prefix}.latitude`,
+          name: 'Latitude field',
+          settings: {
+            filter: (f: Field) => f.type === FieldType.number,
+            noFieldsMessage: 'No numeric fields found',
+          },
+        })
+        .addFieldNamePicker({
+          path: `${prefix}.longitude`,
+          name: 'Longitude field',
+          settings: {
+            filter: (f: Field) => f.type === FieldType.number,
+            noFieldsMessage: 'No numeric fields found',
+          },
+        });
+      break;
+
+    case FrameGeometrySourceMode.Geohash:
+      builder.addFieldNamePicker({
+        path: `${prefix}.geohash`,
+        name: 'Geohash field',
+        settings: {
+          filter: (f: Field) => f.type === FieldType.string,
+          noFieldsMessage: 'No strings fields found',
+        },
+      });
+      break;
+
+    case FrameGeometrySourceMode.Lookup:
+      builder
+        .addFieldNamePicker({
+          path: `${prefix}.lookup`,
+          name: 'Lookup field',
+          settings: {
+            filter: (f: Field) => f.type === FieldType.string,
+            noFieldsMessage: 'No strings fields found',
+          },
+        })
+        .addCustomEditor({
+          id: 'gazetteer',
+          path: `${prefix}.gazetteer`,
+          name: 'Gazetteer',
+          editor: GazetteerPathEditor,
+        });
+  }
+}

public/app/features/geo/format/geohash.test.ts

@@ -0,0 +1,8 @@
+import { decodeGeohash } from './geohash';
+
+describe('Read GeoHASH', () => {
+  it('simple decode', () => {
+    expect(decodeGeohash('9q94r')).toEqual([-122.01416015625, 36.97998046875]);
+    expect(decodeGeohash('dr5rs')).toEqual([-73.98193359375, 40.71533203125]);
+  });
+});

public/app/features/geo/format/geojson.test.ts

@@ -0,0 +1,113 @@
+import { dataFrameToJSON, FieldType } from '@grafana/data';
+import { frameFromGeoJSON } from './geojson';
+
+describe('Read GeoJSON', () => {
+  it('supports simple read', () => {
+    const frame = frameFromGeoJSON({
+      type: 'FeatureCollection',
+      features: [
+        {
+          id: 'A',
+          type: 'Feature',
+          geometry: {
+            type: 'Point',
+            coordinates: [0, 0],
+          },
+          properties: {
+            hello: 'A',
+          },
+        },
+        {
+          type: 'Feature',
+          geometry: {
+            type: 'Point',
+            coordinates: [1, 1],
+          },
+          properties: {
+            number: 1.2,
+            hello: 'B',
+            mixed: 'first',
+          },
+        },
+        {
+          type: 'Feature',
+          geometry: {
+            type: 'Point',
+            coordinates: [2, 2],
+          },
+          properties: {
+            number: 2.3,
+            mixed: 2,
+          },
+        },
+      ],
+    });
+    const msg = dataFrameToJSON(frame);
+    expect(msg.schema).toMatchInlineSnapshot(`
+      Object {
+        "fields": Array [
+          Object {
+            "config": Object {},
+            "name": "id",
+            "type": "string",
+          },
+          Object {
+            "config": Object {},
+            "name": "geo",
+            "type": "geo",
+          },
+          Object {
+            "config": Object {},
+            "name": "hello",
+            "type": "string",
+          },
+          Object {
+            "config": Object {},
+            "name": "number",
+            "type": "number",
+          },
+          Object {
+            "config": Object {},
+            "name": "mixed",
+            "type": "string",
+          },
+        ],
+        "meta": undefined,
+        "name": undefined,
+        "refId": undefined,
+      }
+    `);
+
+    expect(
+      frame.fields.reduce((acc, v, idx, arr) => {
+        if (v.type !== FieldType.geo) {
+          acc[v.name] = v.values.toArray();
+        }
+        return acc;
+      }, {} as any)
+    ).toMatchInlineSnapshot(`
+      Object {
+        "hello": Array [
+          "A",
+          "B",
+          null,
+        ],
+        "id": Array [
+          "A",
+          null,
+          null,
+        ],
+        "mixed": Array [
+          null,
+          "first",
+          "2",
+        ],
+        "number": Array [
+          null,
+          1.2,
+          2.3,
+        ],
+      }
+    `);
+  });
+});

public/app/features/geo/format/geojson.ts

@@ -0,0 +1,125 @@
+import { ArrayVector, DataFrame, Field, FieldType, getFieldTypeFromValue } from '@grafana/data';
+import GeoJSON from 'ol/format/GeoJSON';
+import { Geometry } from 'ol/geom';
+
+interface FieldInfo {
+  values: any[];
+  types: Set<FieldType>;
+  count: number;
+}
+
+// http://geojson.xyz/
+
+export function frameFromGeoJSON(body: Document | Element | Object | string): DataFrame {
+  const data = new GeoJSON().readFeatures(body);
+  const length = data.length;
+
+  const geo: Geometry[] = new Array(length).fill(null);
+
+  const fieldOrder: string[] = [];
+  const lookup = new Map<string, FieldInfo>();
+  const getField = (name: string) => {
+    let f = lookup.get(name);
+    if (!f) {
+      f = {
+        types: new Set<FieldType>(),
+        values: new Array(length).fill(null),
+        count: 0,
+      };
+      fieldOrder.push(name);
+      lookup.set(name, f);
+    }
+    return f;
+  };
+  const getBestName = (...names: string[]) => {
+    for (const k of names) {
+      if (!lookup.has(k)) {
+        return k;
+      }
+    }
+    return '___' + names[0];
+  };
+
+  const idfield: FieldInfo = {
+    types: new Set<FieldType>(),
+    values: new Array(length).fill(null),
+    count: 0,
+  };
+  for (let i = 0; i < length; i++) {
+    const feature = data[i];
+    geo[i] = feature.getGeometry();
+
+    const id = feature.getId();
+    if (id != null) {
+      idfield.values[i] = id;
+      idfield.types.add(getFieldTypeFromValue(id));
+      idfield.count++;
+    }
+
+    for (const key of feature.getKeys()) {
+      const val = feature.get(key);
+      if (key === 'geometry' && val === geo[i]) {
+        continue;
+      }
+      const field = getField(key);
+      field.values[i] = val;
+      field.types.add(getFieldTypeFromValue(val));
+      field.count++;
+    }
+  }
+
+  const fields: Field[] = [];
+  if (idfield.count > 0) {
+    const type = ensureSingleType(idfield);
+    fields.push({
+      name: getBestName('id', '_id', '__id'),
+      type,
+      values: new ArrayVector(idfield.values),
+      config: {},
+    });
+  }
+
+  // Add a geometry field
+  fields.push({
+    name: getBestName('geo', 'geometry'),
+    type: FieldType.geo,
+    values: new ArrayVector(geo),
+    config: {},
+  });
+
+  for (const name of fieldOrder) {
+    const info = lookup.get(name);
+    if (!info) {
+      continue;
+    }
+    const type = ensureSingleType(info);
+    fields.push({
+      name,
+      type,
+      values: new ArrayVector(info.values),
+      config: {},
+    });
+  }
+
+  // Simple frame
+  return {
+    fields,
+    length,
+  };
+}
+
+function ensureSingleType(info: FieldInfo): FieldType {
+  if (info.count < 1) {
+    return FieldType.other;
+  }
+  if (info.types.size > 1) {
+    info.values = info.values.map((v) => {
+      if (v != null) {
+        return `${v}`;
+      }
+      return v;
+    });
+    return FieldType.string;
+  }
+  return info.types.values().next().value;
+}

public/app/features/geo/format/utils.ts

@@ -0,0 +1,56 @@
+import { ArrayVector, Field, FieldConfig, FieldType } from '@grafana/data';
+import { Geometry, Point } from 'ol/geom';
+import { fromLonLat } from 'ol/proj';
+import { Gazetteer } from '../gazetteer/gazetteer';
+import { decodeGeohash } from './geohash';
+
+export function pointFieldFromGeohash(geohash: Field<string>): Field<Point> {
+  return {
+    name: 'point',
+    type: FieldType.geo,
+    values: new ArrayVector<any>(
+      geohash.values.toArray().map((v) => {
+        const coords = decodeGeohash(v);
+        if (coords) {
+          return new Point(fromLonLat(coords));
+        }
+        return undefined;
+      })
+    ),
+    config: hiddenTooltipField,
+  };
+}
+
+export function pointFieldFromLonLat(lon: Field, lat: Field): Field<Point> {
+  const buffer = new Array<Point>(lon.values.length);
+  for (let i = 0; i < lon.values.length; i++) {
+    buffer[i] = new Point(fromLonLat([lon.values.get(i), lat.values.get(i)]));
+  }
+
+  return {
+    name: 'point',
+    type: FieldType.geo,
+    values: new ArrayVector(buffer),
+    config: hiddenTooltipField,
+  };
+}
+
+export function getGeoFieldFromGazetteer(gaz: Gazetteer, field: Field<string>): Field<Geometry | undefined> {
+  const count = field.values.length;
+  const geo = new Array<Geometry | undefined>(count);
+  for (let i = 0; i < count; i++) {
+    geo[i] = gaz.find(field.values.get(i))?.geometry();
+  }
+  return {
+    name: 'Geometry',
+    type: FieldType.geo,
+    values: new ArrayVector(geo),
+    config: hiddenTooltipField,
+  };
+}
+
+const hiddenTooltipField: FieldConfig = Object.freeze({
+  custom: {
+    hideFrom: { tooltip: true },
+  },
+});

public/app/features/geo/gazetteer/gazetteer.test.ts

@@ -57,26 +57,22 @@ describe('Placename lookup from geojson format', () => {
     backendResults = geojsonObject;
     const gaz = await getGazetteer('local');
     expect(gaz.error).toBeUndefined();
-    expect(gaz.find('A')).toMatchInlineSnapshot(`
-      Object {
-        "coords": Array [
-          0,
-          0,
-        ],
-      }
+    expect(gaz.find('A')?.point()?.getCoordinates()).toMatchInlineSnapshot(`
+      Array [
+        0,
+        0,
+      ]
     `);
   });
   it('can look up by a code', async () => {
     backendResults = geojsonObject;
     const gaz = await getGazetteer('airports');
     expect(gaz.error).toBeUndefined();
-    expect(gaz.find('B')).toMatchInlineSnapshot(`
-      Object {
-        "coords": Array [
-          1,
-          1,
-        ],
-      }
+    expect(gaz.find('B')?.point()?.getCoordinates()).toMatchInlineSnapshot(`
+      Array [
+        1,
+        1,
+      ]
     `);
   });
 
@@ -84,13 +80,11 @@ describe('Placename lookup from geojson format', () => {
     backendResults = geojsonObject;
     const gaz = await getGazetteer('airports');
     expect(gaz.error).toBeUndefined();
-    expect(gaz.find('C')).toMatchInlineSnapshot(`
-      Object {
-        "coords": Array [
-          2,
-          2,
-        ],
-      }
+    expect(gaz.find('C')?.point()?.getCoordinates()).toMatchInlineSnapshot(`
+      Array [
+        2,
+        2,
+      ]
     `);
   });
 });

public/app/features/geo/gazetteer/gazetteer.ts

@@ -0,0 +1,201 @@
+import { DataFrame, Field, FieldType, KeyValue, toDataFrame } from '@grafana/data';
+import { getBackendSrv } from '@grafana/runtime';
+import { loadWorldmapPoints } from './worldmap';
+import { Geometry, Point } from 'ol/geom';
+import { frameFromGeoJSON } from '../format/geojson';
+import { pointFieldFromLonLat, pointFieldFromGeohash } from '../format/utils';
+import { getCenter } from 'ol/extent';
+
+export interface PlacenameInfo {
+  point: () => Point | undefined; // lon, lat (WGS84)
+  geometry: () => Geometry | undefined;
+  frame?: DataFrame;
+  index?: number;
+}
+
+export interface Gazetteer {
+  path: string;
+  error?: string;
+  find: (key: string) => PlacenameInfo | undefined;
+  examples: (count: number) => string[];
+  frame?: () => DataFrame;
+  count?: number;
+}
+
+// Without knowing the datatype pick a good lookup function
+export function loadGazetteer(path: string, data: any): Gazetteer {
+  // try loading geojson
+  let frame: DataFrame | undefined = undefined;
+
+  if (Array.isArray(data)) {
+    const first = data[0] as any;
+    // Check for legacy worldmap syntax
+    if (first.latitude && first.longitude && (first.key || first.keys)) {
+      return loadWorldmapPoints(path, data);
+    }
+  } else {
+    if (Array.isArray(data?.features) && data?.type === 'FeatureCollection') {
+      frame = frameFromGeoJSON(data);
+    }
+  }
+
+  if (!frame) {
+    try {
+      frame = toDataFrame(data);
+    } catch (ex) {
+      return {
+        path,
+        error: `${ex}`,
+        find: (k) => undefined,
+        examples: (v) => [],
+      };
+    }
+  }
+
+  return frameAsGazetter(frame, { path });
+}
+
+export function frameAsGazetter(frame: DataFrame, opts: { path: string; keys?: string[] }): Gazetteer {
+  const keys: Field[] = [];
+  let geo: Field<Geometry> | undefined = undefined;
+  let lat: Field | undefined = undefined;
+  let lng: Field | undefined = undefined;
+  let geohash: Field | undefined = undefined;
+  let firstString: Field | undefined = undefined;
+  for (const f of frame.fields) {
+    if (f.type === FieldType.geo) {
+      geo = f;
+    }
+    if (!firstString && f.type === FieldType.string) {
+      firstString = f;
+    }
+    if (f.name) {
+      if (opts.keys && opts.keys.includes(f.name)) {
+        keys.push(f);
+      }
+
+      const name = f.name.toUpperCase();
+      switch (name) {
+        case 'LAT':
+        case 'LATITUTE':
+          lat = f;
+          break;
+
+        case 'LON':
+        case 'LNG':
+        case 'LONG':
+        case 'LONGITUE':
+          lng = f;
+          break;
+
+        case 'GEOHASH':
+          geohash = f;
+          break;
+
+        case 'ID':
+        case 'UID':
+        case 'KEY':
+        case 'CODE':
+          if (!opts.keys) {
+            keys.push(f);
+          }
+          break;
+
+        default: {
+          if (!opts.keys) {
+            if (name.endsWith('_ID') || name.endsWith('_CODE')) {
+              keys.push(f);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  // Use the first string field
+  if (!keys.length && firstString) {
+    keys.push(firstString);
+  }
+
+  let isPoint = false;
+
+  // Create a geo field from lat+lng
+  if (!geo) {
+    if (geohash) {
+      geo = pointFieldFromGeohash(geohash);
+      isPoint = true;
+    } else if (lat && lng) {
+      geo = pointFieldFromLonLat(lng, lat);
+      isPoint = true;
+    }
+  } else {
+    isPoint = geo.values.get(0).getType() === 'Point';
+  }
+
+  const lookup = new Map<string, number>();
+  keys.forEach((f) => {
+    f.values.toArray().forEach((k, idx) => {
+      const str = `${k}`;
+      lookup.set(str.toUpperCase(), idx);
+      lookup.set(str, idx);
+    });
+  });
+
+  return {
+    path: opts.path,
+    find: (k) => {
+      const index = lookup.get(k);
+      if (index != null) {
+        const g = geo?.values.get(index);
+        return {
+          frame,
+          index,
+          point: () => {
+            if (!g || isPoint) {
+              return g as Point;
+            }
+            return new Point(getCenter(g.getExtent()));
+          },
+          geometry: () => g,
+        };
+      }
+      return undefined;
+    },
+    examples: (v) => [],
+    frame: () => frame,
+    count: frame.length,
+  };
+}
+
+const registry: KeyValue<Gazetteer> = {};
+
+export const COUNTRIES_GAZETTEER_PATH = 'public/gazetteer/countries.json';
+
+/**
+ * Given a path to a file return a cached lookup function
+ */
+export async function getGazetteer(path?: string): Promise<Gazetteer> {
+  // When not specified, use the default path
+  if (!path) {
+    path = COUNTRIES_GAZETTEER_PATH;
+  }
+
+  let lookup = registry[path];
+  if (!lookup) {
+    try {
+      // block the async function
+      const data = await getBackendSrv().get(path!);
+      lookup = loadGazetteer(path, data);
+    } catch (err) {
+      console.warn('Error loading placename lookup', path, err);
+      lookup = {
+        path,
+        error: 'Error loading URL',
+        find: (k) => undefined,
+        examples: (v) => [],
+      };
+    }
+    registry[path] = lookup;
+  }
+  return lookup;
+}

public/app/features/geo/gazetteer/worldmap.test.ts

@@ -1,7 +1,8 @@
 import { getGazetteer } from './gazetteer';
 
 let backendResults: any = { hello: 'world' };
-import countriesJSON from '../../../../../gazetteer/countries.json';
+import countriesJSON from '../../../../gazetteer/countries.json';
+import { toLonLat } from 'ol/proj';
 
 jest.mock('@grafana/runtime', () => ({
   ...((jest.requireActual('@grafana/runtime') as unknown) as object),
@@ -19,16 +20,11 @@ describe('Placename lookup from worldmap format', () => {
     backendResults = countriesJSON;
     const gaz = await getGazetteer('countries');
     expect(gaz.error).toBeUndefined();
-    expect(gaz.find('US')).toMatchInlineSnapshot(`
-      Object {
-        "coords": Array [
-          -95.712891,
-          37.09024,
-        ],
-        "props": Object {
-          "name": "United States",
-        },
-      }
+    expect(toLonLat(gaz.find('US')?.point()?.getCoordinates()!)).toMatchInlineSnapshot(`
+      Array [
+        -95.712891,
+        37.09023999999998,
+      ]
     `);
     // Items with 'keys' should get allow looking them up
     expect(gaz.find('US')).toEqual(gaz.find('USA'));

public/app/features/geo/gazetteer/worldmap.ts

@@ -1,4 +1,6 @@
 import { PlacenameInfo, Gazetteer } from './gazetteer';
+import { Point } from 'ol/geom';
+import { fromLonLat } from 'ol/proj';
 
 // https://github.com/grafana/worldmap-panel/blob/master/src/data/countries.json
 export interface WorldmapPoint {
@@ -13,13 +15,14 @@ export function loadWorldmapPoints(path: string, data: WorldmapPoint[]): Gazette
   let count = 0;
   const values = new Map<string, PlacenameInfo>();
   for (const v of data) {
+    const point = new Point(fromLonLat([v.longitude, v.latitude]));
     const info: PlacenameInfo = {
-      coords: [v.longitude, v.latitude],
+      point: () => point,
+      geometry: () => point,
     };
     if (v.name) {
       values.set(v.name, info);
       values.set(v.name.toUpperCase(), info);
-      info.props = { name: v.name };
     }
     if (v.key) {
       values.set(v.key, info);

public/app/features/geo/utils/frameVectorSource.ts

@@ -0,0 +1,35 @@
+import { DataFrame } from '@grafana/data';
+import { Feature } from 'ol';
+import { Geometry } from 'ol/geom';
+import VectorSource from 'ol/source/Vector';
+import { getGeometryField, LocationFieldMatchers } from './location';
+
+export interface FrameVectorSourceOptions {}
+
+export class FrameVectorSource extends VectorSource<Geometry> {
+  constructor(private location: LocationFieldMatchers) {
+    super({});
+  }
+
+  update(frame: DataFrame) {
+    this.clear(true);
+    const info = getGeometryField(frame, this.location);
+    if (!info.field) {
+      this.changed();
+      return;
+    }
+
+    for (let i = 0; i < frame.length; i++) {
+      this.addFeatureInternal(
+        new Feature({
+          frame,
+          rowIndex: i,
+          geometry: info.field.values.get(i),
+        })
+      );
+    }
+
+    // only call this at the end
+    this.changed();
+  }
+}

public/app/features/geo/utils/location.test.ts

@@ -1,6 +1,7 @@
 import { toDataFrame, FieldType, FrameGeometrySourceMode } from '@grafana/data';
+import { Point } from 'ol/geom';
 import { toLonLat } from 'ol/proj';
-import { dataFrameToPoints, getLocationFields, getLocationMatchers } from './location';
+import { getGeometryField, getLocationFields, getLocationMatchers } from './location';
 
 const longitude = [0, -74.1];
 const latitude = [0, 40.7];
@@ -23,8 +24,9 @@ describe('handle location parsing', () => {
     expect(fields.geohash).toBeDefined();
     expect(fields.geohash?.name).toEqual('geohash');
 
-    const info = dataFrameToPoints(frame, matchers);
-    expect(info.points.map((p) => toLonLat(p.getCoordinates()))).toMatchInlineSnapshot(`
+    const info = getGeometryField(frame, matchers);
+    expect(info.field!.type).toBe(FieldType.geo);
+    expect(info.field!.values.toArray().map((p) => toLonLat((p as Point).getCoordinates()))).toMatchInlineSnapshot(`
         Array [
           Array [
             -122.01416015625001,
@@ -49,8 +51,8 @@ describe('handle location parsing', () => {
     });
 
     const matchers = await getLocationMatchers();
-    const info = dataFrameToPoints(frame, matchers);
-    expect(info.points.map((p) => toLonLat(p.getCoordinates()))).toMatchInlineSnapshot(`
+    const geo = getGeometryField(frame, matchers).field!;
+    expect(geo.values.toArray().map((p) => toLonLat((p as Point).getCoordinates()))).toMatchInlineSnapshot(`
       Array [
         Array [
           0,

public/app/features/geo/utils/location.ts

@@ -7,11 +7,11 @@ import {
   DataFrame,
   Field,
   getFieldDisplayName,
+  FieldType,
 } from '@grafana/data';
-import { Point } from 'ol/geom';
-import { fromLonLat } from 'ol/proj';
+import { Geometry } from 'ol/geom';
 import { getGazetteer, Gazetteer } from '../gazetteer/gazetteer';
-import { decodeGeohash } from './geohash';
+import { getGeoFieldFromGazetteer, pointFieldFromGeohash, pointFieldFromLonLat } from '../format/utils';
 
 export type FieldFinder = (frame: DataFrame) => Field | undefined;
 
@@ -51,6 +51,7 @@ export interface LocationFieldMatchers {
   h3: FieldFinder;
   wkt: FieldFinder;
   lookup: FieldFinder;
+  geo: FieldFinder;
   gazetteer?: Gazetteer;
 }
 
@@ -62,6 +63,7 @@ const defaultMatchers: LocationFieldMatchers = {
   h3: matchLowerNames(new Set(['h3'])),
   wkt: matchLowerNames(new Set(['wkt'])),
   lookup: matchLowerNames(new Set(['lookup'])),
+  geo: (frame: DataFrame) => frame.fields.find((f) => f.type === FieldType.geo),
 };
 
 export async function getLocationMatchers(src?: FrameGeometrySource): Promise<LocationFieldMatchers> {
@@ -102,6 +104,7 @@ export interface LocationFields {
   h3?: Field;
   wkt?: Field;
   lookup?: Field;
+  geo?: Field<Geometry>;
 }
 
 export function getLocationFields(frame: DataFrame, location: LocationFieldMatchers): LocationFields {
@@ -111,6 +114,11 @@ export function getLocationFields(frame: DataFrame, location: LocationFieldMatch
 
   // Find the best option
   if (fields.mode === FrameGeometrySourceMode.Auto) {
+    fields.geo = location.geo(frame);
+    if (fields.geo) {
+      return fields;
+    }
+
     fields.latitude = location.latitude(frame);
     fields.longitude = location.longitude(frame);
     if (fields.latitude && fields.longitude) {
@@ -145,84 +153,63 @@ export function getLocationFields(frame: DataFrame, location: LocationFieldMatch
   return fields;
 }
 
-export interface LocationInfo {
+export interface FrameGeometryField {
+  field?: Field<Geometry | undefined>;
   warning?: string;
-  points: Point[];
+  derived?: boolean;
 }
 
-export function dataFrameToPoints(frame: DataFrame, location: LocationFieldMatchers): LocationInfo {
-  const info: LocationInfo = {
-    points: [],
-  };
-  if (!frame?.length) {
-    return info;
-  }
+export function getGeometryField(frame: DataFrame, location: LocationFieldMatchers): FrameGeometryField {
   const fields = getLocationFields(frame, location);
   switch (fields.mode) {
+    case FrameGeometrySourceMode.Auto:
+      if (fields.geo) {
+        return {
+          field: fields.geo,
+        };
+      }
+      return {
+        warning: 'Unable to find location fields',
+      };
+
     case FrameGeometrySourceMode.Coords:
       if (fields.latitude && fields.longitude) {
-        info.points = getPointsFromLonLat(fields.longitude, fields.latitude);
-      } else {
-        info.warning = 'Missing latitude/longitude fields';
+        return {
+          field: pointFieldFromLonLat(fields.longitude, fields.latitude),
+          derived: true,
+        };
       }
-      break;
+      return {
+        warning: 'Missing latitude/longitude fields',
+      };
 
     case FrameGeometrySourceMode.Geohash:
       if (fields.geohash) {
-        info.points = getPointsFromGeohash(fields.geohash);
-      } else {
-        info.warning = 'Missing geohash field';
+        return {
+          field: pointFieldFromGeohash(fields.geohash),
+          derived: true,
+        };
       }
-      break;
+      return {
+        warning: 'Missing geohash field',
+      };
 
     case FrameGeometrySourceMode.Lookup:
       if (fields.lookup) {
         if (location.gazetteer) {
-          info.points = getPointsFromGazetteer(location.gazetteer, fields.lookup);
-        } else {
-          info.warning = 'Gazetteer not found';
+          return {
+            field: getGeoFieldFromGazetteer(location.gazetteer, fields.lookup),
+            derived: true,
+          };
         }
-      } else {
-        info.warning = 'Missing lookup field';
+        return {
+          warning: 'Gazetteer not found',
+        };
       }
-      break;
-
-    case FrameGeometrySourceMode.Auto:
-      info.warning = 'Unable to find location fields';
+      return {
+        warning: 'Missing lookup field',
+      };
   }
 
-  return info;
-}
-
-function getPointsFromLonLat(lon: Field<number>, lat: Field<number>): Point[] {
-  const count = lat.values.length;
-  const points = new Array<Point>(count);
-  for (let i = 0; i < count; i++) {
-    points[i] = new Point(fromLonLat([lon.values.get(i), lat.values.get(i)]));
-  }
-  return points;
-}
-
-function getPointsFromGeohash(field: Field<string>): Point[] {
-  const count = field.values.length;
-  const points = new Array<Point>(count);
-  for (let i = 0; i < count; i++) {
-    const coords = decodeGeohash(field.values.get(i));
-    if (coords) {
-      points[i] = new Point(fromLonLat(coords));
-    }
-  }
-  return points;
-}
-
-function getPointsFromGazetteer(gaz: Gazetteer, field: Field<string>): Point[] {
-  const count = field.values.length;
-  const points = new Array<Point>(count);
-  for (let i = 0; i < count; i++) {
-    const info = gaz.find(field.values.get(i));
-    if (info?.coords) {
-      points[i] = new Point(fromLonLat(info.coords));
-    }
-  }
-  return points;
+  return { warning: 'unable to find geometry' };
 }

public/app/plugins/panel/geomap/editor/layerEditor.tsx

@@ -1,19 +1,12 @@
-import {
-  MapLayerOptions,
-  FrameGeometrySourceMode,
-  FieldType,
-  Field,
-  MapLayerRegistryItem,
-  PluginState,
-} from '@grafana/data';
+import { MapLayerOptions, MapLayerRegistryItem, PluginState } from '@grafana/data';
 import { DEFAULT_BASEMAP_CONFIG, geomapLayerRegistry } from '../layers/registry';
-import { GazetteerPathEditor } from './GazetteerPathEditor';
 import { NestedPanelOptions, NestedValueAccess } from '@grafana/data/src/utils/OptionsUIBuilders';
 import { defaultMarkersConfig } from '../layers/data/markersLayer';
 import { hasAlphaPanels } from 'app/core/config';
 import { MapLayerState } from '../types';
 import { get as lodashGet } from 'lodash';
 import { setOptionImmutably } from 'app/features/dashboard/components/PanelEditor/utils';
+import { addLocationFields } from 'app/features/geo/editor/locationEditor';
 
 export interface LayerEditorOptions {
   state: MapLayerState;
@@ -83,66 +76,7 @@ export function getLayerEditor(opts: LayerEditorOptions): NestedPanelOptions<Map
       }
 
       if (layer.showLocation) {
-        builder
-          .addRadio({
-            path: 'location.mode',
-            name: 'Location',
-            description: '',
-            defaultValue: FrameGeometrySourceMode.Auto,
-            settings: {
-              options: [
-                { value: FrameGeometrySourceMode.Auto, label: 'Auto' },
-                { value: FrameGeometrySourceMode.Coords, label: 'Coords' },
-                { value: FrameGeometrySourceMode.Geohash, label: 'Geohash' },
-                { value: FrameGeometrySourceMode.Lookup, label: 'Lookup' },
-              ],
-            },
-          })
-          .addFieldNamePicker({
-            path: 'location.latitude',
-            name: 'Latitude field',
-            settings: {
-              filter: (f: Field) => f.type === FieldType.number,
-              noFieldsMessage: 'No numeric fields found',
-            },
-            showIf: (opts) => opts.location?.mode === FrameGeometrySourceMode.Coords,
-          })
-          .addFieldNamePicker({
-            path: 'location.longitude',
-            name: 'Longitude field',
-            settings: {
-              filter: (f: Field) => f.type === FieldType.number,
-              noFieldsMessage: 'No numeric fields found',
-            },
-            showIf: (opts) => opts.location?.mode === FrameGeometrySourceMode.Coords,
-          })
-          .addFieldNamePicker({
-            path: 'location.geohash',
-            name: 'Geohash field',
-            settings: {
-              filter: (f: Field) => f.type === FieldType.string,
-              noFieldsMessage: 'No strings fields found',
-            },
-            showIf: (opts) => opts.location?.mode === FrameGeometrySourceMode.Geohash,
-            // eslint-disable-next-line react/display-name
-            // info: (props) => <div>HELLO</div>,
-          })
-          .addFieldNamePicker({
-            path: 'location.lookup',
-            name: 'Lookup field',
-            settings: {
-              filter: (f: Field) => f.type === FieldType.string,
-              noFieldsMessage: 'No strings fields found',
-            },
-            showIf: (opts) => opts.location?.mode === FrameGeometrySourceMode.Lookup,
-          })
-          .addCustomEditor({
-            id: 'gazetteer',
-            path: 'location.gazetteer',
-            name: 'Gazetteer',
-            editor: GazetteerPathEditor,
-            showIf: (opts) => opts.location?.mode === FrameGeometrySourceMode.Lookup,
-          });
+        addLocationFields('location', builder, options.location);
       }
       if (handler.registerOptionsUI) {
         handler.registerOptionsUI(builder);
@@ -150,6 +84,7 @@ export function getLayerEditor(opts: LayerEditorOptions): NestedPanelOptions<Map
       if (layer.showOpacity) {
         // TODO -- add opacity check
       }
+
       builder.addBooleanSwitch({
         path: 'tooltip',
         name: 'Display tooltip',

public/app/plugins/panel/geomap/gazetteer/gazetteer.ts

@@ -1,76 +0,0 @@
-import { KeyValue } from '@grafana/data';
-import { getBackendSrv } from '@grafana/runtime';
-import { loadWorldmapPoints } from './worldmap';
-import { loadFromGeoJSON } from './geojson';
-
-// http://geojson.xyz/
-
-export interface PlacenameInfo {
-  coords: [number, number]; // lon, lat (WGS84)
-  props?: Record<string, any>;
-}
-
-export interface Gazetteer {
-  path: string;
-  error?: string;
-  find: (key: string) => PlacenameInfo | undefined;
-  count?: number;
-  examples: (count: number) => string[];
-}
-
-// Without knowing the datatype pick a good lookup function
-export function loadGazetteer(path: string, data: any): Gazetteer {
-  // Check for legacy worldmap syntax
-  if (Array.isArray(data)) {
-    const first = data[0] as any;
-    if (first.latitude && first.longitude && (first.key || first.keys)) {
-      return loadWorldmapPoints(path, data);
-    }
-  }
-
-  // try loading geojson
-  const features = data?.features;
-  if (Array.isArray(features) && data?.type === 'FeatureCollection') {
-    return loadFromGeoJSON(path, data);
-  }
-
-  return {
-    path,
-    error: 'Unable to parse locations',
-    find: (k) => undefined,
-    examples: (v) => [],
-  };
-}
-
-const registry: KeyValue<Gazetteer> = {};
-
-export const COUNTRIES_GAZETTEER_PATH = 'public/gazetteer/countries.json';
-
-/**
- * Given a path to a file return a cached lookup function
- */
-export async function getGazetteer(path?: string): Promise<Gazetteer> {
-  // When not specified, use the default path
-  if (!path) {
-    path = COUNTRIES_GAZETTEER_PATH;
-  }
-
-  let lookup = registry[path];
-  if (!lookup) {
-    try {
-      // block the async function
-      const data = await getBackendSrv().get(path!);
-      lookup = loadGazetteer(path, data);
-    } catch (err) {
-      console.warn('Error loading placename lookup', path, err);
-      lookup = {
-        path,
-        error: 'Error loading URL',
-        find: (k) => undefined,
-        examples: (v) => [],
-      };
-    }
-    registry[path] = lookup;
-  }
-  return lookup;
-}

public/app/plugins/panel/geomap/gazetteer/geojson.ts

@@ -1,75 +0,0 @@
-import GeoJSON from 'ol/format/GeoJSON';
-import { PlacenameInfo, Gazetteer } from './gazetteer';
-
-export interface GeoJSONPoint {
-  key?: string;
-  keys?: string[]; // new in grafana 8.1+
-  latitude: number;
-  longitude: number;
-  name?: string;
-}
-
-export function loadFromGeoJSON(path: string, body: any): Gazetteer {
-  const data = new GeoJSON().readFeatures(body);
-  let count = 0;
-  const values = new Map<string, PlacenameInfo>();
-  for (const f of data) {
-    const coords = f.getGeometry().getFlatCoordinates(); //for now point, eventually geometry
-    const info: PlacenameInfo = {
-      coords: coords,
-    };
-    const id = f.getId();
-    if (id) {
-      if (typeof id === 'number') {
-        values.set(id.toString(), info);
-      } else {
-        values.set(id, info);
-        values.set(id.toUpperCase(), info);
-      }
-    }
-    const properties = f.getProperties();
-    if (properties) {
-      for (const k of Object.keys(properties)) {
-        if (k.includes('_code') || k.includes('_id')) {
-          const value = properties[k];
-          if (value) {
-            if (typeof value === 'number') {
-              values.set(value.toString(), info);
-            } else {
-              values.set(value, info);
-              values.set(value.toUpperCase(), info);
-            }
-          }
-        }
-      }
-    }
-
-    count++;
-  }
-
-  return {
-    path,
-    find: (k) => {
-      let v = values.get(k);
-      if (!v && typeof k === 'string') {
-        v = values.get(k.toUpperCase());
-      }
-      return v;
-    },
-    count,
-    examples: (count) => {
-      const first: string[] = [];
-      if (values.size < 1) {
-        first.push('no values found');
-      } else {
-        for (const key of values.keys()) {
-          first.push(key);
-          if (first.length >= count) {
-            break;
-          }
-        }
-      }
-      return first;
-    },
-  };
-}

public/app/plugins/panel/geomap/layers/data/heatMap.tsx

@@ -7,12 +7,11 @@ import {
   PanelData,
 } from '@grafana/data';
 import Map from 'ol/Map';
-import Feature from 'ol/Feature';
 import * as layer from 'ol/layer';
-import * as source from 'ol/source';
-import { dataFrameToPoints, getLocationMatchers } from '../../utils/location';
+import { getLocationMatchers } from 'app/features/geo/utils/location';
 import { ScaleDimensionConfig, getScaledDimension } from 'app/features/dimensions';
 import { ScaleDimensionEditor } from 'app/features/dimensions/editors';
+import { FrameVectorSource } from 'app/features/geo/utils/frameVectorSource';
 
 // Configuration options for Heatmap overlays
 export interface HeatmapConfig {
@@ -47,19 +46,19 @@ export const heatmapLayer: MapLayerRegistryItem<HeatmapConfig> = {
    */
   create: async (map: Map, options: MapLayerOptions<HeatmapConfig>, theme: GrafanaTheme2) => {
     const config = { ...defaultOptions, ...options.config };
-    const matchers = await getLocationMatchers(options.location);
-
-    const vectorSource = new source.Vector();
+    
+    const location = await getLocationMatchers(options.location);
+    const source = new FrameVectorSource(location);
+    const WEIGHT_KEY = "_weight";
 
     // Create a new Heatmap layer
     // Weight function takes a feature as attribute and returns a normalized weight value
     const vectorLayer = new layer.Heatmap({
-      source: vectorSource,
+      source,
       blur: config.blur,
       radius: config.radius,
-      weight: function (feature) {
-        var weight = feature.get('value');
-        return weight;
+      weight: (feature) => {
+        return feature.get(WEIGHT_KEY);
       },
     });
 
@@ -67,34 +66,18 @@ export const heatmapLayer: MapLayerRegistryItem<HeatmapConfig> = {
       init: () => vectorLayer,
       update: (data: PanelData) => {
         const frame = data.series[0];
-
-        // Remove previous data before updating
-        const features = vectorLayer.getSource().getFeatures();
-        features.forEach((feature) => {
-          vectorLayer.getSource().removeFeature(feature);
-        });
-
         if (!frame) {
           return;
         }
-        // Get data points (latitude and longitude coordinates)
-        const info = dataFrameToPoints(frame, matchers);
-        if (info.warning) {
-          console.log('WARN', info.warning);
-          return; // ???
-        }
+        source.update(frame);
 
         const weightDim = getScaledDimension(frame, config.weight);
-
-        // Map each data value into new points
-        for (let i = 0; i < frame.length; i++) {
-          const cluster = new Feature({
-            geometry: info.points[i],
-            value: weightDim.get(i),
-          });
-          vectorSource.addFeature(cluster);
-        }
-        vectorLayer.setSource(vectorSource);
+        source.forEachFeature( (f) => {
+          const idx = f.get('rowIndex') as number;
+          if(idx != null) {
+            f.set(WEIGHT_KEY, weightDim.get(idx));
+          }
+        });
 
         // Set heatmap gradient colors
         let colors = ['#00f', '#0ff', '#0f0', '#ff0', '#f00'];

public/app/plugins/panel/geomap/layers/data/lastPointTracker.ts

@@ -4,7 +4,7 @@ import Feature from 'ol/Feature';
 import * as style from 'ol/style';
 import * as source from 'ol/source';
 import * as layer from 'ol/layer';
-import { dataFrameToPoints, getLocationMatchers } from '../../utils/location';
+import { getGeometryField, getLocationMatchers } from 'app/features/geo/utils/location';
 
 export interface LastPointConfig {
   icon?: string;
@@ -52,16 +52,11 @@ export const lastPointTracker: MapLayerRegistryItem<LastPointConfig> = {
       update: (data: PanelData) => {
         const frame = data.series[0];
         if (frame && frame.length) {
-          const info = dataFrameToPoints(frame, matchers);
-          if (info.warning) {
-            console.log('WARN', info.warning);
+          const out = getGeometryField(frame, matchers);
+          if (!out.field) {
             return; // ???
           }
-
-          if (info.points?.length) {
-            const last = info.points[info.points.length - 1];
-            point.setGeometry(last);
-          }
+          point.setGeometry(out.field.values.get(frame.length - 1));
         }
       },
     };

public/app/plugins/panel/geomap/layers/data/markersLayer.tsx

@@ -7,20 +7,18 @@ import {
   FrameGeometrySourceMode,
 } from '@grafana/data';
 import Map from 'ol/Map';
-import Feature, { FeatureLike } from 'ol/Feature';
-import { Point } from 'ol/geom';
-import * as source from 'ol/source';
-import { dataFrameToPoints, getLocationMatchers } from '../../utils/location';
+import { FeatureLike } from 'ol/Feature';
+import { getLocationMatchers } from 'app/features/geo/utils/location';
 import { getScaledDimension, getColorDimension, getTextDimension, getScalarDimension } from 'app/features/dimensions';
 import { ObservablePropsWrapper } from '../../components/ObservablePropsWrapper';
 import { MarkersLegend, MarkersLegendProps } from './MarkersLegend';
 import { ReplaySubject } from 'rxjs';
-import { getFeatures } from '../../utils/getFeatures';
 import { defaultStyleConfig, StyleConfig, StyleDimensions } from '../../style/types';
 import { StyleEditor } from './StyleEditor';
 import { getStyleConfigState } from '../../style/utils';
 import VectorLayer from 'ol/layer/Vector';
 import { isNumber } from 'lodash';
+import { FrameVectorSource } from 'app/features/geo/utils/frameVectorSource';
 
 // Configuration options for Circle overlays
 export interface MarkersConfig {
@@ -61,7 +59,6 @@ export const markersLayer: MapLayerRegistryItem<MarkersConfig> = {
    * @param options
    */
   create: async (map: Map, options: MapLayerOptions<MarkersConfig>, theme: GrafanaTheme2) => {
-    const matchers = await getLocationMatchers(options.location);
     // Assert default values
     const config = {
       ...defaultOptions,
@@ -75,9 +72,11 @@ export const markersLayer: MapLayerRegistryItem<MarkersConfig> = {
     }
 
     const style = await getStyleConfigState(config.style);
-
-    // eventually can also use resolution for dynamic style
-    const vectorLayer = new VectorLayer();
+    const location = await getLocationMatchers(options.location);
+    const source = new FrameVectorSource(location);
+    const vectorLayer = new VectorLayer({
+      source,
+    });
 
     if(!style.fields) {
       // Set a global style
@@ -105,8 +104,7 @@ export const markersLayer: MapLayerRegistryItem<MarkersConfig> = {
           values.rotation = dims.rotation.get(idx);
         }
         return style.maker(values)
-      }
-      );
+      });
     }
     
     return {
@@ -117,15 +115,7 @@ export const markersLayer: MapLayerRegistryItem<MarkersConfig> = {
           return; // ignore empty
         }
 
-        const features: Feature<Point>[] = [];
-
         for (const frame of data.series) {
-          const info = dataFrameToPoints(frame, matchers);
-          if (info.warning) {
-            console.log('Could not find locations', info.warning);
-            continue; // ???
-          }
-
           if (style.fields) {
             const dims: StyleDimensions = {};
             if (style.fields.color) {
@@ -143,12 +133,6 @@ export const markersLayer: MapLayerRegistryItem<MarkersConfig> = {
             style.dims = dims;
           }
 
-          const frameFeatures = getFeatures(frame, info);
-
-          if (frameFeatures) {
-            features.push(...frameFeatures);
-          }
-
           // Post updates to the legend component
           if (legend) {
             legendProps.next({
@@ -156,12 +140,10 @@ export const markersLayer: MapLayerRegistryItem<MarkersConfig> = {
               size: style.dims?.size,
             });
           }
+
+          source.update(frame);
           break; // Only the first frame for now!
         }
-
-        // Source reads the data and provides a set of features to visualize
-        const vectorSource = new source.Vector({ features });
-        vectorLayer.setSource(vectorSource);
       },
 
       // Marker overlay options

public/app/plugins/panel/geomap/types.ts

@@ -1,4 +1,4 @@
-import { MapLayerHandler, MapLayerOptions, SelectableValue } from '@grafana/data';
+import { MapLayerHandler, MapLayerOptions } from '@grafana/data';
 import { HideableFieldConfig } from '@grafana/schema';
 import { LayerElement } from 'app/core/components/Layers/types';
 import BaseLayer from 'ol/layer/Base';
@@ -70,9 +70,6 @@ export enum ComparisonOperation {
   GT = 'gt',
   GTE = 'gte',
 }
-export interface GazetteerPathEditorConfigSettings {
-  options?: Array<SelectableValue<string>>;
-}
 
 //-------------------
 // Runtime model

public/app/plugins/panel/geomap/utils/getFeatures.ts

@@ -1,26 +1,6 @@
-import { DataFrame, SelectableValue } from '@grafana/data';
-import { Feature } from 'ol';
+import { SelectableValue } from '@grafana/data';
 import { FeatureLike } from 'ol/Feature';
-import { Point } from 'ol/geom';
 import { GeometryTypeId } from '../style/types';
-import { LocationInfo } from './location';
-
-export const getFeatures = (frame: DataFrame, info: LocationInfo): Array<Feature<Point>> | undefined => {
-  const features: Array<Feature<Point>> = [];
-
-  // Map each data value into new points
-  for (let i = 0; i < frame.length; i++) {
-    features.push(
-      new Feature({
-        frame,
-        rowIndex: i,
-        geometry: info.points[i],
-      })
-    );
-  }
-
-  return features;
-};
 
 export interface LayerContentInfo {
   geometryType: GeometryTypeId;