SQL: Fix couple of issues in SQLDatasource (#52080)

* Fix: sql plugins feature

(cherry picked from commit 035862bade)

* SQLDS: Use builtin annotation editor

Plus strict rule fixes

(cherry picked from commit fee2eb3716)

.betterer.results

@@ -5559,15 +5559,6 @@ exports[`better eslint`] = {
       [0, 0, 0, "Do not use any type assertions.", "1"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "2"]
     ],
-    "public/app/features/plugins/sql/datasource/SqlDatasource.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "2"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "3"],
-      [0, 0, 0, "Do not use any type assertions.", "4"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "5"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "6"]
-    ],
     "public/app/features/plugins/tests/datasource_srv.test.ts:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
@@ -7807,36 +7798,11 @@ exports[`better eslint`] = {
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"]
     ],
-    "public/app/plugins/datasource/mssql/query_ctrl.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "2"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "3"]
-    ],
     "public/app/plugins/datasource/mssql/response_parser.ts:5381": [
       [0, 0, 0, "Do not use any type assertions.", "0"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
       [0, 0, 0, "Do not use any type assertions.", "2"]
     ],
-    "public/app/plugins/datasource/mssql/specs/datasource.test.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "2"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "3"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "4"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "5"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "6"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "7"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "8"]
-    ],
-    "public/app/plugins/datasource/mssql/types.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "2"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "3"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "4"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "5"]
-    ],
     "public/app/plugins/datasource/mysql/datasource.ts:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"],

package.json

@@ -87,6 +87,7 @@
     "@babel/preset-env": "7.18.2",
     "@babel/preset-react": "7.18.6",
     "@babel/preset-typescript": "7.18.6",
+    "@babel/runtime": "7.18.6",
     "@betterer/betterer": "5.3.5",
     "@betterer/cli": "5.3.5",
     "@betterer/eslint": "5.3.5",
@@ -386,6 +387,7 @@
     "semver": "7.3.7",
     "slate": "0.47.8",
     "slate-plain-serializer": "0.7.10",
+    "sql-formatter-plus": "^1.3.6",
     "symbol-observable": "4.0.0",
     "test": "link:./public/test",
     "tether-drop": "https://github.com/torkelo/drop",

public/app/features/plugins/sql/constants.ts

@@ -110,3 +110,19 @@ export const OPERATORS = [
   { type: OperatorType.Logical, id: 'AND', operator: 'AND' },
   { type: OperatorType.Logical, id: 'OR', operator: 'OR' },
 ];
+
+export const MACRO_NAMES = [
+  '$__time',
+  '$__timeEpoch',
+  '$__timeFilter',
+  '$__timeFrom',
+  '$__timeTo',
+  '$__timeGroup',
+  '$__timeGroupAlias',
+  '$__unixEpochFilter',
+  '$__unixEpochNanoFilter',
+  '$__unixEpochNanoFrom',
+  '$__unixEpochNanoTo',
+  '$__unixEpochGroup',
+  '$__unixEpochGroupAlias',
+];

public/app/features/plugins/sql/datasource/SqlDatasource.ts

@@ -2,15 +2,14 @@ import { lastValueFrom, of } from 'rxjs';
 import { catchError, map } from 'rxjs/operators';
 
 import {
-  AnnotationEvent,
   DataFrame,
   DataFrameView,
-  DataQueryRequest,
+  DataQuery,
-  DataQueryResponse,
   DataSourceInstanceSettings,
   DataSourceRef,
   MetricFindValue,
   ScopedVars,
+  TimeRange,
 } from '@grafana/data';
 import {
   BackendDataSourceResponse,
@@ -20,10 +19,12 @@ import {
   getTemplateSrv,
   TemplateSrv,
 } from '@grafana/runtime';
-import { toTestingStatus } from '@grafana/runtime/src/utils/queryResponse';
+import { toDataQueryResponse, toTestingStatus } from '@grafana/runtime/src/utils/queryResponse';
+import { getTimeSrv } from 'app/features/dashboard/services/TimeSrv';
 
 import { VariableWithMultiSupport } from '../../../variables/types';
-import { getSearchFilterScopedVar } from '../../../variables/utils';
+import { getSearchFilterScopedVar, SearchFilterOptions } from '../../../variables/utils';
+import { MACRO_NAMES } from '../constants';
 import {
   DB,
   SQLQuery,
@@ -39,6 +40,7 @@ export abstract class SqlDatasource extends DataSourceWithBackend<SQLQuery, SQLO
   name: string;
   interval: string;
   db: DB;
+  annotations = {};
 
   constructor(
     instanceSettings: DataSourceInstanceSettings<SQLOptions>,
@@ -121,20 +123,33 @@ export abstract class SqlDatasource extends DataSourceWithBackend<SQLQuery, SQLO
     return value.replace(/''/g, "'");
   }
 
-  // eslint-ignore @typescript-eslint/no-explicit-any
+  async metricFindQuery(query: string, optionalOptions?: MetricFindQueryOptions): Promise<MetricFindValue[]> {
-  async annotationQuery(options: any): Promise<AnnotationEvent[]> {
+    const rawSql = this.templateSrv.replace(
-    if (!options.annotation.rawQuery) {
+      query,
-      return Promise.reject({
+      getSearchFilterScopedVar({ query, wildcardChar: '%', options: optionalOptions }),
-        message: 'Query missing in annotation definition',
+      this.interpolateVariable
-      });
+    );
+
+    const interpolatedQuery: SQLQuery = {
+      refId: 'tempvar',
+      datasource: this.getRef(),
+      rawSql,
+      format: QueryFormat.Table,
+    };
+
+    const response = await this.runMetaQuery(interpolatedQuery, optionalOptions);
+    return this.getResponseParser().transformMetricFindResponse(response);
   }
 
-    const query = {
+  async runSql<T>(query: string, options?: MetricFindQueryOptions) {
-      refId: options.annotation.name,
+    const frame = await this.runMetaQuery({ rawSql: query, format: QueryFormat.Table }, options);
-      datasource: this.getRef(),
+    return new DataFrameView<T>(frame);
-      rawSql: this.templateSrv.replace(options.annotation.rawQuery, options.scopedVars, this.interpolateVariable),
+  }
-      format: 'table',
+
-    };
+  private runMetaQuery(request: Partial<SQLQuery>, options?: MetricFindQueryOptions): Promise<DataFrame> {
+    const range = getTimeSrv().timeRange();
+    const refId = request.refId || 'meta';
+    const queries: DataQuery[] = [{ ...request, datasource: request.datasource || this.getRef(), refId }];
 
     return lastValueFrom(
       getBackendSrv()
@@ -142,57 +157,22 @@ export abstract class SqlDatasource extends DataSourceWithBackend<SQLQuery, SQLO
           url: '/api/ds/query',
           method: 'POST',
           data: {
-            from: options.range.from.valueOf().toString(),
+            from: options?.range?.from.valueOf().toString() || range.from.valueOf().toString(),
-            to: options.range.to.valueOf().toString(),
+            to: options?.range?.to.valueOf().toString() || range.to.valueOf().toString(),
-            queries: [query],
+            queries,
           },
-          requestId: options.annotation.name,
+          requestId: refId,
         })
         .pipe(
-          map(
+          map((res: FetchResponse<BackendDataSourceResponse>) => {
-            async (res: FetchResponse<BackendDataSourceResponse>) =>
+            const rsp = toDataQueryResponse(res, queries);
-              await this.getResponseParser().transformAnnotationResponse(options, res.data)
+            return rsp.data[0];
-          )
+          })
         )
     );
   }
 
-  async metricFindQuery(query: string, optionalOptions: any): Promise<MetricFindValue[]> {
+  testDatasource(): Promise<{ status: string; message: string }> {
-    const rawSql = this.templateSrv.replace(
-      query,
-      getSearchFilterScopedVar({ query, wildcardChar: '%', options: optionalOptions }),
-      this.interpolateVariable
-    );
-
-    const interpolatedQuery = {
-      datasourceId: this.id,
-      datasource: this.getRef(),
-      rawSql,
-      format: QueryFormat.Table,
-    };
-
-    const response = await this.runQuery(interpolatedQuery, optionalOptions);
-    return this.getResponseParser().transformMetricFindResponse(response);
-  }
-
-  async runSql<T = any>(query: string) {
-    const frame = await this.runQuery({ rawSql: query, format: QueryFormat.Table }, {});
-    return new DataFrameView<T>(frame);
-  }
-
-  private runQuery(request: Partial<SQLQuery>, options?: any): Promise<DataFrame> {
-    return new Promise((resolve) => {
-      const req = {
-        targets: [{ ...request, refId: String(Math.random()) }],
-        range: options?.range,
-      } as DataQueryRequest<SQLQuery>;
-      this.query(req).subscribe((res: DataQueryResponse) => {
-        resolve(res.data[0] || { fields: [] });
-      });
-    });
-  }
-
-  testDatasource(): Promise<any> {
     return lastValueFrom(
       getBackendSrv()
         .fetch({
@@ -223,7 +203,15 @@ export abstract class SqlDatasource extends DataSourceWithBackend<SQLQuery, SQLO
     );
   }
 
-  targetContainsTemplate(target: any) {
+  targetContainsTemplate(target: SQLQuery) {
-    return this.templateSrv.containsTemplate(target.rawSql);
+    let queryWithoutMacros = target.rawSql;
+    MACRO_NAMES.forEach((value) => {
+      queryWithoutMacros = queryWithoutMacros?.replace(value, '') || '';
+    });
+    return this.templateSrv.containsTemplate(queryWithoutMacros);
   }
 }
+
+interface MetricFindQueryOptions extends SearchFilterOptions {
+  range?: TimeRange;
+}

public/app/features/plugins/sql/defaults.ts

@@ -3,20 +3,21 @@ import { EditorMode } from '@grafana/experimental';
 import { QueryFormat, SQLQuery } from './types';
 import { createFunctionField, setGroupByField } from './utils/sql.utils';
 
-export function applyQueryDefaults(q: SQLQuery): SQLQuery {
+export function applyQueryDefaults(q?: SQLQuery): SQLQuery {
-  let editorMode = q.editorMode || EditorMode.Builder;
+  let editorMode = q?.editorMode || EditorMode.Builder;
 
   // Switching to code editor if the query was created before visual query builder was introduced.
-  if (q.editorMode === undefined && q.rawSql !== undefined) {
+  if (q?.editorMode === undefined && q?.rawSql !== undefined) {
     editorMode = EditorMode.Code;
   }
 
-  const result = {
+  const result: SQLQuery = {
     ...q,
-    format: q.format !== undefined ? q.format : QueryFormat.Table,
+    refId: q?.refId || 'A',
-    rawSql: q.rawSql || '',
+    format: q?.format !== undefined ? q.format : QueryFormat.Table,
+    rawSql: q?.rawSql || '',
     editorMode,
-    sql: q.sql || {
+    sql: q?.sql || {
       columns: [createFunctionField()],
       groupBy: [setGroupByField()],
       limit: 50,

public/app/features/plugins/sql/types.ts

@@ -1,7 +1,6 @@
 import { JsonTree } from 'react-awesome-query-builder';
 
 import {
-  AnnotationEvent,
   DataFrame,
   DataQuery,
   DataSourceJsonData,
@@ -11,7 +10,6 @@ import {
   toOption as toOptionFromData,
 } from '@grafana/data';
 import { CompletionItemKind, EditorMode, LanguageCompletionProvider } from '@grafana/experimental';
-import { BackendDataSourceResponse } from '@grafana/runtime';
 
 import { QueryWithDefaults } from './defaults';
 import {
@@ -23,7 +21,7 @@ import {
 export interface SqlQueryForInterpolation {
   dataset?: string;
   alias?: string;
-  format?: ResultFormat;
+  format?: QueryFormat;
   rawSql?: string;
   refId: string;
   hide?: boolean;
@@ -34,8 +32,6 @@ export interface SQLOptions extends DataSourceJsonData {
   database: string;
 }
 
-export type ResultFormat = 'time_series' | 'table';
-
 export enum QueryFormat {
   Timeseries = 'time_series',
   Table = 'table',
@@ -43,7 +39,7 @@ export enum QueryFormat {
 
 export interface SQLQuery extends DataQuery {
   alias?: string;
-  format?: ResultFormat | QueryFormat | string | undefined;
+  format?: QueryFormat;
   rawSql?: string;
   dataset?: string;
   table?: string;
@@ -119,7 +115,7 @@ export interface DB {
   tables: (dataset?: string) => Promise<string[]>;
   fields: (query: SQLQuery, order?: boolean) => Promise<SQLSelectableValue[]>;
   validateQuery: (query: SQLQuery, range?: TimeRange) => Promise<ValidationResults>;
-  dsID: () => string;
+  dsID: () => number;
   dispose?: (dsID?: string) => void;
   lookup: (path?: string) => Promise<Array<{ name: string; completion: string }>>;
   getSqlCompletionProvider: () => LanguageCompletionProvider;
@@ -135,7 +131,7 @@ export interface QueryEditorProps {
 
 export interface ValidationResults {
   query: SQLQuery;
-  rawSql: string;
+  rawSql?: string;
   error: string;
   isError: boolean;
   isValid: boolean;
@@ -150,7 +146,6 @@ export interface SqlQueryModel {
 }
 
 export interface ResponseParser {
-  transformAnnotationResponse: (options: object, data: BackendDataSourceResponse) => Promise<AnnotationEvent[]>;
   transformMetricFindResponse: (frame: DataFrame) => MetricFindValue[];
 }
 

public/app/features/plugins/sql/utils/useSqlChange.ts

@@ -14,7 +14,7 @@ export function useSqlChange({ query, onQueryChange, db }: UseSqlChange) {
   const onSqlChange = useCallback(
     (sql: SQLExpression) => {
       const toRawSql = db.toRawSql || defaultToRawSql;
-      const rawSql = toRawSql({ sql, dataset: query.dataset, table: query.table, refId: db.dsID() });
+      const rawSql = toRawSql({ sql, dataset: query.dataset, table: query.table, refId: query.refId });
       const newQuery: SQLQuery = { ...query, sql, rawSql };
       onQueryChange(newQuery);
     },

public/app/features/variables/utils.ts

@@ -33,10 +33,14 @@ export const SEARCH_FILTER_VARIABLE = '__searchFilter';
 export const containsSearchFilter = (query: string | unknown): boolean =>
   query && typeof query === 'string' ? query.indexOf(SEARCH_FILTER_VARIABLE) !== -1 : false;
 
+export interface SearchFilterOptions {
+  searchFilter?: string;
+}
+
 export const getSearchFilterScopedVar = (args: {
   query: string;
   wildcardChar: string;
-  options: { searchFilter?: string };
+  options?: SearchFilterOptions;
 }): ScopedVars => {
   const { query, wildcardChar } = args;
   if (!containsSearchFilter(query)) {

yarn.lock

@@ -3177,6 +3177,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@babel/polyfill@npm:^7.6.0":
+  version: 7.12.1
+  resolution: "@babel/polyfill@npm:7.12.1"
+  dependencies:
+    core-js: ^2.6.5
+    regenerator-runtime: ^0.13.4
+  checksum: 3f59a9d85a41b390b044a1be13e11ae6d8efbfcf4e07217964585c7cef337b828eecfc5e164083227189146d2b6efc1affae8f59c831438eb40b848ab6fe5f39
+  languageName: node
+  linkType: hard
+
 "@babel/preset-env@npm:7.18.2":
   version: 7.18.2
   resolution: "@babel/preset-env@npm:7.18.2"
@@ -3447,6 +3457,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@babel/runtime@npm:7.18.6":
+  version: 7.18.6
+  resolution: "@babel/runtime@npm:7.18.6"
+  dependencies:
+    regenerator-runtime: ^0.13.4
+  checksum: 8b707b64ae0524db617d0c49933b258b96376a38307dc0be8fb42db5697608bcc1eba459acce541e376cff5ed5c5287d24db5780bd776b7c75ba2c2e26ff8a2c
+  languageName: node
+  linkType: hard
+
 "@babel/runtime@npm:^7.0.0, @babel/runtime@npm:^7.1.2, @babel/runtime@npm:^7.10.1, @babel/runtime@npm:^7.11.1, @babel/runtime@npm:^7.11.2, @babel/runtime@npm:^7.12.0, @babel/runtime@npm:^7.12.1, @babel/runtime@npm:^7.12.13, @babel/runtime@npm:^7.12.5, @babel/runtime@npm:^7.13.10, @babel/runtime@npm:^7.15.4, @babel/runtime@npm:^7.5.5, @babel/runtime@npm:^7.6.2, @babel/runtime@npm:^7.7.2, @babel/runtime@npm:^7.8.4, @babel/runtime@npm:^7.8.7, @babel/runtime@npm:^7.9.2":
   version: 7.15.4
   resolution: "@babel/runtime@npm:7.15.4"
@@ -16295,7 +16314,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"core-js@npm:^2.0.0, core-js@npm:^2.4.0":
+"core-js@npm:^2.0.0, core-js@npm:^2.4.0, core-js@npm:^2.6.5":
   version: 2.6.12
   resolution: "core-js@npm:2.6.12"
   checksum: 44fa9934a85f8c78d61e0c8b7b22436330471ffe59ec5076fe7f324d6e8cf7f824b14b1c81ca73608b13bdb0fef035bd820989bf059767ad6fa13123bb8bd016
@@ -20937,6 +20956,7 @@ __metadata:
     "@babel/preset-env": 7.18.2
     "@babel/preset-react": 7.18.6
     "@babel/preset-typescript": 7.18.6
+    "@babel/runtime": 7.18.6
     "@betterer/betterer": 5.3.5
     "@betterer/cli": 5.3.5
     "@betterer/eslint": 5.3.5
@@ -21217,6 +21237,7 @@ __metadata:
     sinon: 14.0.0
     slate: 0.47.8
     slate-plain-serializer: 0.7.10
+    sql-formatter-plus: ^1.3.6
     style-loader: 3.3.1
     stylelint: 14.9.1
     stylelint-config-prettier: 9.0.3
@@ -33883,6 +33904,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"sql-formatter-plus@npm:^1.3.6":
+  version: 1.3.6
+  resolution: "sql-formatter-plus@npm:1.3.6"
+  dependencies:
+    "@babel/polyfill": ^7.6.0
+    lodash: ^4.17.15
+  checksum: 5c215f85936b3465d6778af868f0a8a2d3dd4b714ab95025c3b17b9748fe420ae2e128c523d50be0a43667bf8329a974f3248828971299146786e28d9714e752
+  languageName: node
+  linkType: hard
+
 "sqlstring@npm:^2.3.2":
   version: 2.3.3
   resolution: "sqlstring@npm:2.3.3"