Scenes: Add query variable support (#59553)

* WIP first attempt to query variable

* regex issue repro demo

* Refresh variable on time range change if refresh specified

* Instantiate variable runner when updating query variable options

* Simplify runners getTarget interface

* Fix issue with variable ot being updated correctly after other variable changed

* Add templateSrv.replace compatibility with query variable

* QueryVariable: use datasource variable as source

* use proper format

* Make sure variables set is correctly updated when query variable errors

* Do not destruct scopedVars when using sceneGraph.interpolate in templateSrv

* Add support for Legacy variables (metricFindQuery)

* Review

* Fix lint

* Test: Add unit for datasource by variable

* test: Add unit for datasource as var

* query: delegate interpolation to datasourceSrv

* Cleanup

Co-authored-by: Ivan Ortega <ivanortegaalba@gmail.com>

.betterer.results

@@ -4575,6 +4575,9 @@ exports[`better eslint`] = {
       [0, 0, 0, "Do not use any type assertions.", "3"],
       [0, 0, 0, "Do not use any type assertions.", "4"]
     ],
+    "public/app/features/scenes/variables/interpolation/sceneInterpolator.ts:5381": [
+      [0, 0, 0, "Do not use any type assertions.", "0"]
+    ],
     "public/app/features/scenes/variables/sets/SceneVariableSet.test.tsx:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
@@ -4588,6 +4591,16 @@ exports[`better eslint`] = {
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"]
     ],
+    "public/app/features/scenes/variables/variants/query/QueryVariable.tsx:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
+    ],
+    "public/app/features/scenes/variables/variants/query/createQueryVariableRunner.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
+    ],
+    "public/app/features/scenes/variables/variants/query/utils.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "1"]
+    ],
     "public/app/features/search/components/SearchCard.tsx:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
       [0, 0, 0, "Do not use any type assertions.", "1"],
@@ -4692,7 +4705,8 @@ exports[`better eslint`] = {
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "2"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "3"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "4"]
+      [0, 0, 0, "Unexpected any. Specify a different type.", "4"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "5"]
     ],
     "public/app/features/templating/template_srv.ts:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
@@ -4712,8 +4726,9 @@ exports[`better eslint`] = {
       [0, 0, 0, "Unexpected any. Specify a different type.", "14"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "15"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "16"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "17"],
-      [0, 0, 0, "Do not use any type assertions.", "18"]
+      [0, 0, 0, "Do not use any type assertions.", "17"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "18"],
+      [0, 0, 0, "Do not use any type assertions.", "19"]
     ],
     "public/app/features/transformers/FilterByValueTransformer/ValueMatchers/BasicMatcherEditor.tsx:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],

public/app/features/scenes/scenes/index.tsx

@@ -7,6 +7,7 @@ import { getMultipleGridLayoutTest } from './gridMultiple';
 import { getGridWithMultipleData } from './gridWithMultipleData';
 import { getGridWithRowLayoutTest } from './gridWithRow';
 import { getNestedScene } from './nested';
+import { getQueryVariableDemo } from './queryVariableDemo';
 import { getSceneWithRows } from './sceneWithRows';
 import { getVariablesDemo } from './variablesDemo';
 
@@ -26,6 +27,7 @@ export function getScenes(): SceneDef[] {
     { title: 'Grid with rows and different queries and time ranges', getScene: getGridWithMultipleTimeRanges },
     { title: 'Multiple grid layouts test', getScene: getMultipleGridLayoutTest },
     { title: 'Variables', getScene: getVariablesDemo },
+    { title: 'Query variable', getScene: getQueryVariableDemo },
   ];
 }
 

public/app/features/scenes/scenes/queryVariableDemo.tsx

@@ -0,0 +1,71 @@
+import { VariableRefresh } from '@grafana/data';
+
+import { Scene, EmbeddedScene } from '../components/Scene';
+import { SceneCanvasText } from '../components/SceneCanvasText';
+import { SceneSubMenu } from '../components/SceneSubMenu';
+import { SceneTimePicker } from '../components/SceneTimePicker';
+import { SceneFlexLayout } from '../components/layout/SceneFlexLayout';
+import { SceneTimeRange } from '../core/SceneTimeRange';
+import { VariableValueSelectors } from '../variables/components/VariableValueSelectors';
+import { SceneVariableSet } from '../variables/sets/SceneVariableSet';
+import { CustomVariable } from '../variables/variants/CustomVariable';
+import { DataSourceVariable } from '../variables/variants/DataSourceVariable';
+import { QueryVariable } from '../variables/variants/query/QueryVariable';
+
+export function getQueryVariableDemo(standalone: boolean): Scene {
+  const state = {
+    title: 'Query variable',
+    $variables: new SceneVariableSet({
+      variables: [
+        new CustomVariable({
+          name: 'metric',
+          query: 'job : job, instance : instance',
+        }),
+        new DataSourceVariable({
+          name: 'datasource',
+          query: 'prometheus',
+        }),
+        new QueryVariable({
+          name: 'instance (using datasource variable)',
+          refresh: VariableRefresh.onTimeRangeChanged,
+          query: { query: 'label_values(go_gc_duration_seconds, ${metric})' },
+          datasource: '${datasource}',
+        }),
+        new QueryVariable({
+          name: 'label values (on time range refresh)',
+          refresh: VariableRefresh.onTimeRangeChanged,
+          query: { query: 'label_values(go_gc_duration_seconds, ${metric})' },
+          datasource: { uid: 'gdev-prometheus', type: 'prometheus' },
+        }),
+        new QueryVariable({
+          name: 'legacy (graphite)',
+          refresh: VariableRefresh.onTimeRangeChanged,
+          query: { queryType: 'Default', target: 'stats.response.*' },
+          datasource: { uid: 'gdev-graphite', type: 'graphite' },
+        }),
+      ],
+    }),
+    layout: new SceneFlexLayout({
+      direction: 'row',
+      children: [
+        new SceneFlexLayout({
+          children: [
+            new SceneCanvasText({
+              size: { width: '40%' },
+              text: 'metric: ${metric}',
+              fontSize: 20,
+              align: 'center',
+            }),
+          ],
+        }),
+      ],
+    }),
+    $timeRange: new SceneTimeRange(),
+    actions: [new SceneTimePicker({})],
+    subMenu: new SceneSubMenu({
+      children: [new VariableValueSelectors({})],
+    }),
+  };
+
+  return standalone ? new Scene(state) : new EmbeddedScene(state);
+}

public/app/features/scenes/scenes/variablesDemo.tsx

@@ -1,5 +1,5 @@
 import { VizPanel } from '../components';
-import { EmbeddedScene, Scene } from '../components/Scene';
+import { Scene, EmbeddedScene } from '../components/Scene';
 import { SceneCanvasText } from '../components/SceneCanvasText';
 import { SceneSubMenu } from '../components/SceneSubMenu';
 import { SceneTimePicker } from '../components/SceneTimePicker';

public/app/features/scenes/variables/interpolation/ScopedVarsVariable.ts

@@ -9,10 +9,10 @@ import { FormatVariable } from './formatRegistry';
 export class ScopedVarsVariable implements FormatVariable {
   private static fieldAccessorCache: FieldAccessorCache = {};
 
-  public state: { name: string; value: ScopedVar };
+  public state: { name: string; value: ScopedVar; type: string };
 
   public constructor(name: string, value: ScopedVar) {
-    this.state = { name, value };
+    this.state = { name, value, type: 'scopedvar' };
   }
 
   public getValue(fieldPath: string): VariableValue {

public/app/features/scenes/variables/interpolation/formatRegistry.ts

@@ -1,6 +1,7 @@
 import { isArray, map, replace } from 'lodash';
 
 import { dateTime, Registry, RegistryItem, textUtil } from '@grafana/data';
+import { VariableType } from '@grafana/schema';
 import kbn from 'app/core/utils/kbn';
 import { ALL_VARIABLE_VALUE } from 'app/features/variables/constants';
 
@@ -18,6 +19,7 @@ export interface FormatRegistryItem extends RegistryItem {
 export interface FormatVariable {
   state: {
     name: string;
+    type: VariableType | string;
   };
 
   getValue(fieldPath?: string): VariableValue | undefined | null;

public/app/features/scenes/variables/interpolation/sceneInterpolator.test.ts

@@ -51,6 +51,7 @@ describe('sceneInterpolator', () => {
         $variables: new SceneVariableSet({
           variables: [
             new ObjectVariable({
+              type: 'custom',
               name: 'test',
               value: { prop1: 'prop1Value' },
             }),

public/app/features/scenes/variables/interpolation/sceneInterpolator.ts

@@ -1,5 +1,5 @@
 import { ScopedVars } from '@grafana/data';
-import { VariableModel } from '@grafana/schema';
+import { VariableModel, VariableType } from '@grafana/schema';
 import { variableRegex } from 'app/features/variables/utils';
 
 import { EmptyVariableSet, sceneGraph } from '../../core/sceneGraph';
@@ -12,7 +12,7 @@ import { formatRegistry, FormatRegistryID, FormatVariable } from './formatRegist
 export type CustomFormatterFn = (
   value: unknown,
   legacyVariableModel: VariableModel,
-  legacyDefaultFormatter: CustomFormatterFn
+  legacyDefaultFormatter?: CustomFormatterFn
 ) => string;
 
 /**
@@ -97,9 +97,10 @@ function formatValue(
   }
 
   if (typeof formatNameOrFn === 'function') {
-    // legacy custom formatter function, TODO
-    //return format(value, {}, this.formatValue);
-    throw new Error('Custom formatter function not supported');
+    return formatNameOrFn(value, {
+      name: variable.state.name,
+      type: variable.state.type as VariableType,
+    });
   }
 
   let args: string[] = [];

public/app/features/scenes/variables/sets/SceneVariableSet.ts

@@ -40,7 +40,7 @@ export class SceneVariableSet extends SceneObjectBase<SceneVariableSetState> imp
     this.variablesToUpdate.clear();
 
     for (const update of this.updating.values()) {
-      update.subscription.unsubscribe();
+      update.subscription?.unsubscribe();
     }
   }
 
@@ -70,12 +70,14 @@ export class SceneVariableSet extends SceneObjectBase<SceneVariableSetState> imp
         continue;
       }
 
-      this.updating.set(variable, {
+      const variableToUpdate: VariableUpdateInProgress = {
         variable,
-        subscription: variable.validateAndUpdate().subscribe({
-          next: () => this.validateAndUpdateCompleted(variable),
-          error: (err) => this.handleVariableError(variable, err),
-        }),
+      };
+
+      this.updating.set(variable, variableToUpdate);
+      variableToUpdate.subscription = variable.validateAndUpdate().subscribe({
+        next: () => this.validateAndUpdateCompleted(variable),
+        error: (err) => this.handleVariableError(variable, err),
       });
     }
   }
@@ -85,7 +87,7 @@ export class SceneVariableSet extends SceneObjectBase<SceneVariableSetState> imp
    */
   private validateAndUpdateCompleted(variable: SceneVariable) {
     const update = this.updating.get(variable);
-    update?.subscription.unsubscribe();
+    update?.subscription?.unsubscribe();
 
     this.updating.delete(variable);
     this.variablesToUpdate.delete(variable);
@@ -97,6 +99,11 @@ export class SceneVariableSet extends SceneObjectBase<SceneVariableSetState> imp
    * Not sure if this should be handled here on in MultiValueVariable
    */
   private handleVariableError(variable: SceneVariable, err: Error) {
+    const update = this.updating.get(variable);
+    update?.subscription?.unsubscribe();
+
+    this.updating.delete(variable);
+    this.variablesToUpdate.delete(variable);
     variable.setState({ loading: false, error: err });
   }
 
@@ -187,5 +194,5 @@ export class SceneVariableSet extends SceneObjectBase<SceneVariableSetState> imp
 
 export interface VariableUpdateInProgress {
   variable: SceneVariable;
-  subscription: Unsubscribable;
+  subscription?: Unsubscribable;
 }

public/app/features/scenes/variables/types.ts

@@ -1,11 +1,13 @@
 import { Observable } from 'rxjs';
 
 import { BusEventWithPayload } from '@grafana/data';
+import { VariableType } from '@grafana/schema';
 import { VariableHide } from 'app/features/variables/types';
 
 import { SceneObject, SceneObjectStatePlain } from '../core/types';
 
 export interface SceneVariableState extends SceneObjectStatePlain {
+  type: VariableType;
   name: string;
   label?: string;
   hide?: VariableHide;

public/app/features/scenes/variables/variants/ConstantVariable.ts

@@ -9,6 +9,14 @@ export class ConstantVariable
   extends SceneObjectBase<ConstantVariableState>
   implements SceneVariable<ConstantVariableState>
 {
+  public constructor(initialState: Partial<ConstantVariableState>) {
+    super({
+      type: 'constant',
+      value: '',
+      name: '',
+      ...initialState,
+    });
+  }
   public getValue(): VariableValue {
     return this.state.value;
   }

public/app/features/scenes/variables/variants/CustomVariable.tsx

@@ -19,6 +19,7 @@ export class CustomVariable extends MultiValueVariable<CustomVariableState> {
 
   public constructor(initialState: Partial<CustomVariableState>) {
     super({
+      type: 'custom',
       query: '',
       value: '',
       text: '',
@@ -30,7 +31,6 @@ export class CustomVariable extends MultiValueVariable<CustomVariableState> {
 
   public getValueOptions(args: VariableGetOptionsArgs): Observable<VariableValueOption[]> {
     const match = this.state.query.match(/(?:\\,|[^,])+/g) ?? [];
-
     const options = match.map((text) => {
       text = text.replace(/\\,/g, ',');
       const textMatch = /^(.+)\s:\s(.+)$/g.exec(text) ?? [];

public/app/features/scenes/variables/variants/DataSourceVariable.tsx

@@ -24,6 +24,7 @@ export class DataSourceVariable extends MultiValueVariable<DataSourceVariableSta
 
   public constructor(initialState: Partial<DataSourceVariableState>) {
     super({
+      type: 'datasource',
       value: '',
       text: '',
       options: [],

public/app/features/scenes/variables/variants/MultiValueVariable.test.ts

@@ -10,6 +10,17 @@ export interface ExampleVariableState extends MultiValueVariableState {
 }
 
 class ExampleVariable extends MultiValueVariable<ExampleVariableState> {
+  public constructor(initialState: Partial<ExampleVariableState>) {
+    super({
+      type: 'custom',
+      optionsToReturn: [],
+      value: '',
+      text: '',
+      name: '',
+      options: [],
+      ...initialState,
+    });
+  }
   public getValueOptions(args: VariableGetOptionsArgs): Observable<VariableValueOption[]> {
     return of(this.state.optionsToReturn);
   }

public/app/features/scenes/variables/variants/ObjectVariable.test.ts

@@ -5,6 +5,7 @@ describe('ObjectVariable', () => {
     it('it should return value according to fieldPath', () => {
       const variable = new ObjectVariable({
         name: 'test',
+        type: 'custom',
         value: {
           field1: 'value1',
           array: ['value1', 'value2', 'value3'],

public/app/features/scenes/variables/variants/TestVariable.tsx

@@ -31,6 +31,7 @@ export class TestVariable extends MultiValueVariable<TestVariableState> {
 
   public constructor(initialState: Partial<TestVariableState>) {
     super({
+      type: 'custom',
       name: 'Test',
       value: 'Value',
       text: 'Text',

public/app/features/scenes/variables/variants/query/QueryVariable.test.tsx

@@ -0,0 +1,310 @@
+import { lastValueFrom, of } from 'rxjs';
+
+import {
+  DataQueryRequest,
+  DataSourceApi,
+  DataSourceRef,
+  FieldType,
+  getDefaultTimeRange,
+  LoadingState,
+  PanelData,
+  PluginType,
+  StandardVariableSupport,
+  toDataFrame,
+  toUtc,
+  VariableRefresh,
+  VariableSupportType,
+} from '@grafana/data';
+import { SceneFlexLayout } from 'app/features/scenes/components';
+import { SceneTimeRange } from 'app/features/scenes/core/SceneTimeRange';
+
+import { SceneVariableSet } from '../../sets/SceneVariableSet';
+
+import { QueryVariable } from './QueryVariable';
+import { QueryRunner, RunnerArgs, setCreateQueryVariableRunnerFactory } from './createQueryVariableRunner';
+
+const runRequestMock = jest.fn().mockReturnValue(
+  of<PanelData>({
+    state: LoadingState.Done,
+    series: [
+      toDataFrame({
+        fields: [{ name: 'text', type: FieldType.string, values: ['A', 'AB', 'C'] }],
+      }),
+    ],
+    timeRange: getDefaultTimeRange(),
+  })
+);
+
+const fakeDsMock: DataSourceApi = {
+  name: 'fake-std',
+  type: 'fake-std',
+  getRef: () => ({ type: 'fake-std', uid: 'fake-std' }),
+  query: () =>
+    Promise.resolve({
+      data: [],
+    }),
+  testDatasource: () => Promise.resolve({ status: 'success' }),
+  meta: {
+    id: 'fake-std',
+    type: PluginType.datasource,
+    module: 'fake-std',
+    baseUrl: '',
+    name: 'fake-std',
+    info: {
+      author: { name: '' },
+      description: '',
+      links: [],
+      logos: { large: '', small: '' },
+      updated: '',
+      version: '',
+      screenshots: [],
+    },
+  },
+  // Standard variable support
+  variables: {
+    getType: () => VariableSupportType.Standard,
+    toDataQuery: (q) => ({ ...q, refId: 'FakeDataSource-refId' }),
+  },
+  id: 1,
+  uid: 'fake-std',
+};
+
+jest.mock('@grafana/runtime', () => ({
+  ...jest.requireActual('@grafana/runtime'),
+  getDataSourceSrv: () => ({
+    get: (ds: DataSourceRef): Promise<DataSourceApi> => {
+      return Promise.resolve(fakeDsMock);
+    },
+  }),
+}));
+
+class FakeQueryRunner implements QueryRunner {
+  public constructor(private datasource: DataSourceApi, private _runRequest: jest.Mock) {}
+
+  public getTarget(variable: QueryVariable) {
+    return (this.datasource.variables as StandardVariableSupport<DataSourceApi>).toDataQuery(variable.state.query);
+  }
+  public runRequest(args: RunnerArgs, request: DataQueryRequest) {
+    return this._runRequest(
+      this.datasource,
+      request,
+      (this.datasource.variables as StandardVariableSupport<DataSourceApi>).query
+    );
+  }
+}
+
+describe('QueryVariable', () => {
+  describe('When empty query is provided', () => {
+    it('Should default to empty options and empty value', async () => {
+      const variable = new QueryVariable({
+        name: 'test',
+        datasource: { uid: 'fake', type: 'fake' },
+        query: '',
+      });
+
+      await lastValueFrom(variable.validateAndUpdate());
+
+      expect(variable.state.value).toEqual('');
+      expect(variable.state.text).toEqual('');
+      expect(variable.state.options).toEqual([]);
+    });
+  });
+
+  describe('When no data source is provided', () => {
+    it('Should default to empty options and empty value', async () => {
+      const variable = new QueryVariable({
+        name: 'test',
+      });
+
+      await lastValueFrom(variable.validateAndUpdate());
+
+      expect(variable.state.value).toEqual('');
+      expect(variable.state.text).toEqual('');
+      expect(variable.state.options).toEqual([]);
+    });
+  });
+
+  describe('Issuing variable query', () => {
+    const originalNow = Date.now;
+    beforeEach(() => {
+      setCreateQueryVariableRunnerFactory(() => new FakeQueryRunner(fakeDsMock, runRequestMock));
+    });
+
+    beforeEach(() => {
+      Date.now = jest.fn(() => 60000);
+    });
+
+    afterEach(() => {
+      Date.now = originalNow;
+      runRequestMock.mockClear();
+    });
+
+    it('Should resolve variable options via provided runner', async () => {
+      const variable = new QueryVariable({
+        name: 'test',
+        datasource: { uid: 'fake-std', type: 'fake-std' },
+        query: 'query',
+      });
+
+      await lastValueFrom(variable.validateAndUpdate());
+
+      expect(variable.state.options).toEqual([
+        { label: 'A', value: 'A' },
+        { label: 'AB', value: 'AB' },
+        { label: 'C', value: 'C' },
+      ]);
+    });
+
+    it('Should pass variable scene object via request scoped vars', async () => {
+      const variable = new QueryVariable({
+        name: 'test',
+        datasource: { uid: 'fake-std', type: 'fake-std' },
+        query: 'query',
+      });
+
+      await lastValueFrom(variable.validateAndUpdate());
+      const call = runRequestMock.mock.calls[0];
+      expect(call[1].scopedVars.__sceneObject).toEqual({ value: variable, text: '__sceneObject' });
+    });
+
+    describe('when refresh on dashboard load set', () => {
+      it('Should issue variable query with default time range', async () => {
+        const variable = new QueryVariable({
+          name: 'test',
+          datasource: { uid: 'fake-std', type: 'fake-std' },
+          query: 'query',
+        });
+
+        await lastValueFrom(variable.validateAndUpdate());
+
+        expect(runRequestMock).toBeCalledTimes(1);
+        const call = runRequestMock.mock.calls[0];
+        expect(call[1].range).toEqual(getDefaultTimeRange());
+      });
+
+      it('Should not issue variable query when the closest time range changes if refresh on dahboard load is set', async () => {
+        const timeRange = new SceneTimeRange({ from: 'now-1h', to: 'now' });
+
+        const variable = new QueryVariable({
+          name: 'test',
+          datasource: { uid: 'fake-std', type: 'fake-std' },
+          query: 'query',
+          refresh: VariableRefresh.onDashboardLoad,
+          $timeRange: timeRange,
+        });
+
+        variable.activate();
+
+        await lastValueFrom(variable.validateAndUpdate());
+
+        expect(runRequestMock).toBeCalledTimes(1);
+        const call1 = runRequestMock.mock.calls[0];
+
+        // Uses default time range
+        expect(call1[1].range.raw).toEqual({
+          from: 'now-6h',
+          to: 'now',
+        });
+
+        timeRange.onTimeRangeChange({
+          from: toUtc('2020-01-01'),
+          to: toUtc('2020-01-02'),
+          raw: { from: toUtc('2020-01-01'), to: toUtc('2020-01-02') },
+        });
+
+        await Promise.resolve();
+
+        expect(runRequestMock).toBeCalledTimes(1);
+      });
+    });
+
+    describe('when refresh on time range change set', () => {
+      it('Should issue variable query with closes time range if refresh on time range change set', async () => {
+        const variable = new QueryVariable({
+          name: 'test',
+          datasource: { uid: 'fake-std', type: 'fake-std' },
+          query: 'query',
+          refresh: VariableRefresh.onTimeRangeChanged,
+        });
+
+        // @ts-expect-error
+        const scene = new SceneFlexLayout({
+          $timeRange: new SceneTimeRange({ from: 'now-1h', to: 'now' }),
+          $variables: new SceneVariableSet({
+            variables: [variable],
+          }),
+          children: [],
+        });
+
+        await lastValueFrom(variable.validateAndUpdate());
+
+        expect(runRequestMock).toBeCalledTimes(1);
+        const call = runRequestMock.mock.calls[0];
+
+        expect(call[1].range.raw).toEqual({
+          from: 'now-1h',
+          to: 'now',
+        });
+      });
+
+      it('Should issue variable query when time range changes if refresh on time range change is set', async () => {
+        const timeRange = new SceneTimeRange({ from: 'now-1h', to: 'now' });
+        const variable = new QueryVariable({
+          name: 'test',
+          datasource: { uid: 'fake-std', type: 'fake-std' },
+          query: 'query',
+          refresh: VariableRefresh.onTimeRangeChanged,
+          $timeRange: timeRange,
+        });
+
+        variable.activate();
+
+        await lastValueFrom(variable.validateAndUpdate());
+
+        expect(runRequestMock).toBeCalledTimes(1);
+        const call1 = runRequestMock.mock.calls[0];
+        expect(call1[1].range.raw).toEqual({
+          from: 'now-1h',
+          to: 'now',
+        });
+
+        timeRange.onTimeRangeChange({
+          from: toUtc('2020-01-01'),
+          to: toUtc('2020-01-02'),
+          raw: { from: toUtc('2020-01-01'), to: toUtc('2020-01-02') },
+        });
+
+        await new Promise((r) => setTimeout(r, 1));
+
+        expect(runRequestMock).toBeCalledTimes(2);
+        const call2 = runRequestMock.mock.calls[1];
+        expect(call2[1].range.raw).toEqual({
+          from: '2020-01-01T00:00:00.000Z',
+          to: '2020-01-02T00:00:00.000Z',
+        });
+      });
+    });
+  });
+
+  describe('When regex provided', () => {
+    beforeEach(() => {
+      setCreateQueryVariableRunnerFactory(() => new FakeQueryRunner(fakeDsMock, runRequestMock));
+    });
+
+    it('should return options that match regex', async () => {
+      const variable = new QueryVariable({
+        name: 'test',
+        datasource: { uid: 'fake-std', type: 'fake-std' },
+        query: 'query',
+        regex: '/^A/',
+      });
+
+      await lastValueFrom(variable.validateAndUpdate());
+
+      expect(variable.state.options).toEqual([
+        { label: 'A', value: 'A' },
+        { label: 'AB', value: 'AB' },
+      ]);
+    });
+  });
+});

public/app/features/scenes/variables/variants/query/QueryVariable.tsx

@@ -0,0 +1,166 @@
+import React from 'react';
+import { Observable, Subject, of, Unsubscribable, filter, take, mergeMap, catchError, throwError, from } from 'rxjs';
+import { v4 as uuidv4 } from 'uuid';
+
+import {
+  CoreApp,
+  DataQuery,
+  DataQueryRequest,
+  DataSourceApi,
+  DataSourceRef,
+  getDefaultTimeRange,
+  LoadingState,
+  PanelData,
+  ScopedVars,
+  VariableRefresh,
+  VariableSort,
+} from '@grafana/data';
+import { getDataSourceSrv } from '@grafana/runtime';
+import { toMetricFindValues } from 'app/features/variables/query/operators';
+
+import { sceneGraph } from '../../../core/sceneGraph';
+import { SceneComponentProps } from '../../../core/types';
+import { VariableDependencyConfig } from '../../VariableDependencyConfig';
+import { VariableValueSelect } from '../../components/VariableValueSelect';
+import { VariableValueOption } from '../../types';
+import { MultiValueVariable, MultiValueVariableState, VariableGetOptionsArgs } from '../MultiValueVariable';
+
+import { createQueryVariableRunner } from './createQueryVariableRunner';
+import { metricNamesToVariableValues } from './utils';
+
+export interface QueryVariableState extends MultiValueVariableState {
+  type: 'query';
+  datasource: DataSourceRef | string | null;
+  query: any;
+  regex: string;
+  refresh: VariableRefresh;
+  sort: VariableSort;
+}
+
+export class QueryVariable extends MultiValueVariable<QueryVariableState> {
+  private updateSubscription?: Unsubscribable;
+  private dataSourceSubject?: Subject<DataSourceApi>;
+
+  protected _variableDependency = new VariableDependencyConfig(this, {
+    statePaths: ['regex', 'query', 'datasource'],
+  });
+
+  public constructor(initialState: Partial<QueryVariableState>) {
+    super({
+      type: 'query',
+      name: '',
+      value: '',
+      text: '',
+      query: '',
+      options: [],
+      datasource: null,
+      regex: '',
+      refresh: VariableRefresh.onDashboardLoad,
+      sort: VariableSort.alphabeticalAsc,
+      ...initialState,
+    });
+  }
+
+  public activate(): void {
+    super.activate();
+    const timeRange = sceneGraph.getTimeRange(this);
+
+    if (this.state.refresh === VariableRefresh.onTimeRangeChanged) {
+      this._subs.add(
+        timeRange.subscribeToState({
+          next: () => {
+            this.updateSubscription = this.validateAndUpdate().subscribe();
+          },
+        })
+      );
+    }
+  }
+
+  public deactivate(): void {
+    super.deactivate();
+    if (this.updateSubscription) {
+      this.updateSubscription.unsubscribe();
+    }
+
+    if (this.dataSourceSubject) {
+      this.dataSourceSubject.unsubscribe();
+    }
+  }
+
+  public getValueOptions(args: VariableGetOptionsArgs): Observable<VariableValueOption[]> {
+    if (this.state.query === '' || !this.state.datasource) {
+      return of([]);
+    }
+
+    return from(this.getDataSource()).pipe(
+      mergeMap((ds) => {
+        const runner = createQueryVariableRunner(ds);
+        const target = runner.getTarget(this);
+        const request = this.getRequest(target);
+        return runner.runRequest({ variable: this }, request).pipe(
+          filter((data) => data.state === LoadingState.Done || data.state === LoadingState.Error), // we only care about done or error for now
+          take(1), // take the first result, using first caused a bug where it in some situations throw an uncaught error because of no results had been received yet
+          mergeMap((data: PanelData) => {
+            if (data.state === LoadingState.Error) {
+              return throwError(() => data.error);
+            }
+            return of(data);
+          }),
+          toMetricFindValues(),
+          mergeMap((values) => {
+            let regex = '';
+            if (this.state.regex) {
+              regex = sceneGraph.interpolate(this, this.state.regex, undefined, 'regex');
+            }
+            return of(metricNamesToVariableValues(regex, this.state.sort, values));
+          }),
+          catchError((error) => {
+            if (error.cancelled) {
+              return of([]);
+            }
+            return throwError(() => error);
+          })
+        );
+      })
+    );
+  }
+
+  private async getDataSource(): Promise<DataSourceApi> {
+    return getDataSourceSrv().get(this.state.datasource ?? '', {
+      __sceneObject: { text: '__sceneObject', value: this },
+    });
+  }
+
+  private getRequest(target: DataQuery) {
+    // TODO: add support for search filter
+    // const { searchFilter } = this.state.searchFilter;
+    // const searchFilterScope = { searchFilter: { text: searchFilter, value: searchFilter } };
+    // const searchFilterAsVars = searchFilter ? searchFilterScope : {};
+    const scopedVars: ScopedVars = {
+      // ...searchFilterAsVars,
+      __sceneObject: { text: '__sceneObject', value: this },
+    };
+
+    const range =
+      this.state.refresh === VariableRefresh.onTimeRangeChanged
+        ? sceneGraph.getTimeRange(this).state.value
+        : getDefaultTimeRange();
+
+    const request: DataQueryRequest = {
+      app: CoreApp.Dashboard,
+      requestId: uuidv4(),
+      timezone: '',
+      range,
+      interval: '',
+      intervalMs: 0,
+      targets: [target],
+      scopedVars,
+      startTime: Date.now(),
+    };
+    return request;
+  }
+
+  public static Component = ({ model }: SceneComponentProps<MultiValueVariable>) => {
+    return <VariableValueSelect model={model} />;
+  };
+}

public/app/features/scenes/variables/variants/query/createQueryVariableRunner.ts

@@ -0,0 +1,115 @@
+import { from, mergeMap, Observable, of } from 'rxjs';
+
+import {
+  DataQuery,
+  DataQueryRequest,
+  DataSourceApi,
+  getDefaultTimeRange,
+  LoadingState,
+  PanelData,
+} from '@grafana/data';
+import { runRequest } from 'app/features/query/state/runRequest';
+import { hasLegacyVariableSupport, hasStandardVariableSupport } from 'app/features/variables/guard';
+
+import { QueryVariable } from './QueryVariable';
+
+export interface RunnerArgs {
+  searchFilter?: string;
+  variable: QueryVariable;
+}
+
+export interface QueryRunner {
+  getTarget: (variable: QueryVariable) => DataQuery;
+  runRequest: (args: RunnerArgs, request: DataQueryRequest) => Observable<PanelData>;
+}
+
+class StandardQueryRunner implements QueryRunner {
+  public constructor(private datasource: DataSourceApi, private _runRequest = runRequest) {}
+
+  public getTarget(variable: QueryVariable) {
+    if (hasStandardVariableSupport(this.datasource)) {
+      return this.datasource.variables.toDataQuery(variable.state.query);
+    }
+
+    throw new Error("Couldn't create a target with supplied arguments.");
+  }
+
+  public runRequest(_: RunnerArgs, request: DataQueryRequest) {
+    if (!hasStandardVariableSupport(this.datasource)) {
+      return getEmptyMetricFindValueObservable();
+    }
+
+    if (!this.datasource.variables.query) {
+      return this._runRequest(this.datasource, request);
+    }
+
+    return this._runRequest(this.datasource, request, this.datasource.variables.query);
+  }
+}
+
+class LegacyQueryRunner implements QueryRunner {
+  public constructor(private datasource: DataSourceApi) {}
+
+  public getTarget(variable: QueryVariable) {
+    if (hasLegacyVariableSupport(this.datasource)) {
+      return variable.state.query;
+    }
+
+    throw new Error("Couldn't create a target with supplied arguments.");
+  }
+
+  public runRequest({ variable }: RunnerArgs, request: DataQueryRequest) {
+    if (!hasLegacyVariableSupport(this.datasource)) {
+      return getEmptyMetricFindValueObservable();
+    }
+
+    return from(
+      this.datasource.metricFindQuery(variable.state.query, {
+        ...request,
+        // variable is used by SQL common data source
+        variable: {
+          name: variable.state.name,
+          type: variable.state.type,
+        },
+        // TODO: add support for search filter
+        // searchFilter
+      })
+    ).pipe(
+      mergeMap((values) => {
+        if (!values || !values.length) {
+          return getEmptyMetricFindValueObservable();
+        }
+
+        const series: any = values;
+        return of({ series, state: LoadingState.Done, timeRange: request.range });
+      })
+    );
+  }
+}
+
+function getEmptyMetricFindValueObservable(): Observable<PanelData> {
+  return of({ state: LoadingState.Done, series: [], timeRange: getDefaultTimeRange() });
+}
+
+function createQueryVariableRunnerFactory(datasource: DataSourceApi): QueryRunner {
+  if (hasStandardVariableSupport(datasource)) {
+    return new StandardQueryRunner(datasource, runRequest);
+  }
+
+  if (hasLegacyVariableSupport(datasource)) {
+    return new LegacyQueryRunner(datasource);
+  }
+
+  // TODO: add support for legacy, cutom and datasource query runners
+
+  throw new Error(`Couldn't create a query runner for datasource ${datasource.type}`);
+}
+
+export let createQueryVariableRunner = createQueryVariableRunnerFactory;
+
+/**
+ * Use only in tests
+ */
+export function setCreateQueryVariableRunnerFactory(fn: (datasource: DataSourceApi) => QueryRunner) {
+  createQueryVariableRunner = fn;
+}

public/app/features/scenes/variables/variants/query/utils.ts

@@ -0,0 +1,111 @@
+import { isNumber, sortBy, toLower, uniqBy } from 'lodash';
+
+import { stringToJsRegex, VariableSort } from '@grafana/data';
+
+import { VariableValueOption } from '../../types';
+
+export const metricNamesToVariableValues = (variableRegEx: string, sort: VariableSort, metricNames: any[]) => {
+  let regex;
+  let options: VariableValueOption[] = [];
+
+  if (variableRegEx) {
+    regex = stringToJsRegex(variableRegEx);
+  }
+
+  for (let i = 0; i < metricNames.length; i++) {
+    const item = metricNames[i];
+    let text = item.text === undefined || item.text === null ? item.value : item.text;
+    let value = item.value === undefined || item.value === null ? item.text : item.value;
+
+    if (isNumber(value)) {
+      value = value.toString();
+    }
+
+    if (isNumber(text)) {
+      text = text.toString();
+    }
+
+    if (regex) {
+      const matches = getAllMatches(value, regex);
+      if (!matches.length) {
+        continue;
+      }
+
+      const valueGroup = matches.find((m) => m.groups && m.groups.value);
+      const textGroup = matches.find((m) => m.groups && m.groups.text);
+      const firstMatch = matches.find((m) => m.length > 1);
+      const manyMatches = matches.length > 1 && firstMatch;
+
+      if (valueGroup || textGroup) {
+        value = valueGroup?.groups?.value ?? textGroup?.groups?.text;
+        text = textGroup?.groups?.text ?? valueGroup?.groups?.value;
+      } else if (manyMatches) {
+        for (let j = 0; j < matches.length; j++) {
+          const match = matches[j];
+          options.push({ label: match[1], value: match[1] });
+        }
+        continue;
+      } else if (firstMatch) {
+        text = firstMatch[1];
+        value = firstMatch[1];
+      }
+    }
+
+    options.push({ label: text, value: value });
+  }
+
+  options = uniqBy(options, 'value');
+  return sortVariableValues(options, sort);
+};
+
+const getAllMatches = (str: string, regex: RegExp): RegExpExecArray[] => {
+  const results: RegExpExecArray[] = [];
+  let matches = null;
+
+  regex.lastIndex = 0;
+
+  do {
+    matches = regex.exec(str);
+    if (matches) {
+      results.push(matches);
+    }
+  } while (regex.global && matches && matches[0] !== '' && matches[0] !== undefined);
+
+  return results;
+};
+
+export const sortVariableValues = (options: any[], sortOrder: VariableSort) => {
+  if (sortOrder === VariableSort.disabled) {
+    return options;
+  }
+
+  const sortType = Math.ceil(sortOrder / 2);
+  const reverseSort = sortOrder % 2 === 0;
+
+  if (sortType === 1) {
+    options = sortBy(options, 'text');
+  } else if (sortType === 2) {
+    options = sortBy(options, (opt) => {
+      if (!opt.text) {
+        return -1;
+      }
+
+      const matches = opt.text.match(/.*?(\d+).*/);
+      if (!matches || matches.length < 2) {
+        return -1;
+      } else {
+        return parseInt(matches[1], 10);
+      }
+    });
+  } else if (sortType === 3) {
+    options = sortBy(options, (opt) => {
+      return toLower(opt.text);
+    });
+  }
+
+  if (reverseSort) {
+    options = options.reverse();
+  }
+
+  return options;
+};

public/app/features/templating/LegacyVariableWrapper.ts

@@ -1,12 +1,14 @@
+import { VariableModel, VariableType } from '@grafana/schema';
+
 import { FormatVariable } from '../scenes/variables/interpolation/formatRegistry';
 import { VariableValue } from '../scenes/variables/types';
 import { ALL_VARIABLE_TEXT, ALL_VARIABLE_VALUE } from '../variables/constants';
 
 export class LegacyVariableWrapper implements FormatVariable {
-  state: { name: string; value: VariableValue; text: VariableValue };
+  state: { name: string; value: VariableValue; text: VariableValue; type: VariableType };
 
-  constructor(name: string, value: VariableValue, text: VariableValue) {
-    this.state = { name, value, text };
+  constructor(variable: VariableModel, value: VariableValue, text: VariableValue) {
+    this.state = { name: variable.name, value, text, type: variable.type };
   }
 
   getValue(_fieldPath: string): VariableValue {
@@ -40,11 +42,14 @@ let legacyVariableWrapper: LegacyVariableWrapper | undefined;
 /**
  * Reuses a single instance to avoid unnecessary memory allocations
  */
-export function getVariableWrapper(name: string, value: VariableValue, text: VariableValue) {
+export function getVariableWrapper(variable: VariableModel, value: VariableValue, text: VariableValue) {
+  // TODO: provide more legacy variable properties, i.e. multi, includeAll that are used in custom interpolators,
+  // see Prometheus data source for example
   if (!legacyVariableWrapper) {
-    legacyVariableWrapper = new LegacyVariableWrapper(name, value, text);
+    legacyVariableWrapper = new LegacyVariableWrapper(variable, value, text);
   } else {
-    legacyVariableWrapper.state.name = name;
+    legacyVariableWrapper.state.name = variable.name;
+    legacyVariableWrapper.state.type = variable.type;
     legacyVariableWrapper.state.value = value;
     legacyVariableWrapper.state.text = text;
   }

public/app/features/templating/template_srv.test.ts

@@ -5,6 +5,7 @@ import { silenceConsoleOutput } from '../../../test/core/utils/silenceConsoleOut
 import { initTemplateSrv } from '../../../test/helpers/initTemplateSrv';
 import { mockDataSource, MockDataSourceSrv } from '../alerting/unified/mocks';
 import { FormatRegistryID } from '../scenes/variables/interpolation/formatRegistry';
+import { TestVariable } from '../scenes/variables/variants/TestVariable';
 import { VariableAdapter, variableAdapters } from '../variables/adapters';
 import { createAdHocVariableAdapter } from '../variables/adhoc/adapter';
 import { createQueryVariableAdapter } from '../variables/query/adapter';
@@ -17,6 +18,14 @@ variableAdapters.setInit(() => [
   createAdHocVariableAdapter() as unknown as VariableAdapter<VariableModel>,
 ]);
 
+const interpolateMock = jest.fn();
+
+jest.mock('../scenes/core/sceneGraph', () => ({
+  sceneGraph: {
+    interpolate: (...args: any[]) => interpolateMock(...args),
+  },
+}));
+
 describe('templateSrv', () => {
   silenceConsoleOutput();
   let _templateSrv: any;
@@ -810,4 +819,19 @@ describe('templateSrv', () => {
       expect(target).toBe('var-adhoc=value2');
     });
   });
+
+  describe('scenes compatibility', () => {
+    beforeEach(() => {
+      _templateSrv = initTemplateSrv(key, []);
+    });
+    it('should use scene interpolator when scoped var provided', () => {
+      const variable = new TestVariable({});
+
+      _templateSrv.replace('test ${test}', { __sceneObject: { value: variable } });
+
+      expect(interpolateMock).toHaveBeenCalledTimes(1);
+      expect(interpolateMock.mock.calls[0][0]).toEqual(variable);
+      expect(interpolateMock.mock.calls[0][1]).toEqual('test ${test}');
+    });
+  });
 });

public/app/features/templating/template_srv.ts

@@ -10,7 +10,10 @@ import {
 } from '@grafana/data';
 import { getDataSourceSrv, setTemplateSrv, TemplateSrv as BaseTemplateSrv } from '@grafana/runtime';
 
+import { SceneObjectBase } from '../scenes/core/SceneObjectBase';
+import { sceneGraph } from '../scenes/core/sceneGraph';
 import { formatRegistry, FormatRegistryID } from '../scenes/variables/interpolation/formatRegistry';
+import { CustomFormatterFn } from '../scenes/variables/interpolation/sceneInterpolator';
 import { variableAdapters } from '../variables/adapters';
 import { ALL_VARIABLE_TEXT, ALL_VARIABLE_VALUE } from '../variables/constants';
 import { isAdHoc } from '../variables/guard';
@@ -166,7 +169,7 @@ export class TemplateSrv implements BaseTemplateSrv {
       formatItem = formatRegistry.get(FormatRegistryID.glob);
     }
 
-    const formatVariable = getVariableWrapper(variable.name, value, text ?? value);
+    const formatVariable = getVariableWrapper(variable, value, text ?? value);
     return formatItem.formatter(value, args, formatVariable);
   }
 
@@ -276,6 +279,15 @@ export class TemplateSrv implements BaseTemplateSrv {
   }
 
   replace(target?: string, scopedVars?: ScopedVars, format?: string | Function): string {
+    if (scopedVars && scopedVars.__sceneObject && scopedVars.__sceneObject.value instanceof SceneObjectBase) {
+      return sceneGraph.interpolate(
+        scopedVars.__sceneObject.value,
+        target,
+        scopedVars,
+        format as string | CustomFormatterFn | undefined
+      );
+    }
+
     if (!target) {
       return target ?? '';
     }

public/test/setupTests.ts

@@ -6,7 +6,7 @@ import { initReactI18next } from 'react-i18next';
 import { matchers } from './matchers';
 
 failOnConsole({
-  shouldFailOnLog: process.env.CI ? true : false,
+  shouldFailOnLog: true,
 });
 
 expect.extend(matchers);