SearchV2: improve searcher API, and include a fallback SQL based implementation (#49535)

pkg/services/searchV2/bluge.go

@@ -295,27 +295,6 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
 	response := &backend.DataResponse{}
 	header := &customMeta{}
 
-	// Folder listing structure.
-	idx := strings.Index(q.Query, ":")
-	if idx > 0 {
-		key := q.Query[0:idx]
-		val := q.Query[idx+1:]
-		if key == "list" {
-			q.Limit = 1000
-			q.Query = ""
-			q.Location = ""
-			q.Explain = false
-			q.SkipLocation = true
-			q.Facet = nil
-			if val == "root" || val == "" {
-				q.Kind = []string{string(entityKindFolder)}
-			} else {
-				q.Location = val
-				q.Kind = []string{string(entityKindDashboard)}
-			}
-		}
-	}
-
 	hasConstraints := false
 	fullQuery := bluge.NewBooleanQuery()
 	fullQuery.AddMust(newPermissionFilter(filter, logger))

public/app/features/search/components/DashboardListPage.tsx

@@ -46,8 +46,7 @@ export const DashboardListPage: FC<Props> = memo(({ navModel, match, location })
 
   return (
     <Page navModel={value?.pageNavModel ?? navModel}>
-      {/*Todo: remove the false to test, or when we feel confident with thsi approach */}
-      {Boolean(config.featureToggles.panelTitleSearch && !window.location.search?.includes('index=sql')) ? (
+      {Boolean(config.featureToggles.panelTitleSearch) ? (
         <Page.Contents
           isLoading={loading}
           className={css`

public/app/features/search/components/DashboardSearch.tsx

@@ -20,7 +20,7 @@ export interface Props {
 }
 
 export default function DashboardSearch({ onCloseSearch }: Props) {
-  if (config.featureToggles.panelTitleSearch && !window.location.search?.includes('index=sql')) {
+  if (config.featureToggles.panelTitleSearch) {
     // TODO: "folder:current" ????
     return <DashbaordSearchNEW onCloseSearch={onCloseSearch} />;
   }

public/app/features/search/page/components/ActionRow.tsx

@@ -9,8 +9,6 @@ import { TagFilter, TermCount } from 'app/core/components/TagFilter/TagFilter';
 
 import { DashboardQuery, SearchLayout } from '../../types';
 
-import { getSortOptions } from './sorting';
-
 export const layoutOptions = [
   { value: SearchLayout.Folders, icon: 'folder', ariaLabel: 'View by folders' },
   { value: SearchLayout.List, icon: 'list-ul', ariaLabel: 'View as list' },
@@ -26,6 +24,7 @@ interface Props {
   onStarredFilterChange?: (event: FormEvent<HTMLInputElement>) => void;
   onTagFilterChange: (tags: string[]) => void;
   getTagOptions: () => Promise<TermCount[]>;
+  getSortOptions: () => Promise<SelectableValue[]>;
   onDatasourceChange: (ds?: string) => void;
   query: DashboardQuery;
   showStarredFilter?: boolean;
@@ -54,6 +53,7 @@ export const ActionRow: FC<Props> = ({
   onStarredFilterChange = () => {},
   onTagFilterChange,
   getTagOptions,
+  getSortOptions,
   onDatasourceChange,
   query,
   showStarredFilter,

public/app/features/search/page/components/FolderSection.tsx

@@ -3,9 +3,7 @@ import React, { FC } from 'react';
 import { useAsync, useLocalStorage } from 'react-use';
 
 import { GrafanaTheme } from '@grafana/data';
-import { getBackendSrv } from '@grafana/runtime';
 import { Card, Checkbox, CollapsableSection, Icon, Spinner, stylesFactory, useTheme } from '@grafana/ui';
-import impressionSrv from 'app/core/services/impression_srv';
 import { getSectionStorageKey } from 'app/features/search/utils';
 import { useUniqueId } from 'app/plugins/datasource/influxdb/components/useUniqueId';
 
@@ -58,23 +56,14 @@ export const FolderSection: FC<SectionHeaderProps> = ({
       location: section.uid,
       sort: 'name_sort',
     };
-    if (section.title === 'Starred') {
+    if (section.itemsUIDs) {
       query = {
         uid: section.itemsUIDs, // array of UIDs
       };
       folderUid = undefined;
       folderTitle = undefined;
-    } else if (section.title === 'Recent') {
-      const ids = impressionSrv.getDashboardOpened();
-      const uids = await getBackendSrv().get(`/api/dashboards/ids/${ids.slice(0, 30).join(',')}`);
-      if (uids?.length) {
-        query = {
-          uid: uids,
-        };
-      }
-      folderUid = undefined;
-      folderTitle = undefined;
     }
+
     const raw = await getGrafanaSearcher().search({ ...query, tags });
     const v = raw.view.map(
       (item) =>

public/app/features/search/page/components/FolderView.tsx

@@ -6,6 +6,8 @@ import { GrafanaTheme2 } from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
 import { getBackendSrv } from '@grafana/runtime';
 import { Spinner, useStyles2 } from '@grafana/ui';
+import { contextSrv } from 'app/core/core';
+import impressionSrv from 'app/core/services/impression_srv';
 
 import { GENERAL_FOLDER_UID } from '../../constants';
 import { getGrafanaSearcher } from '../../service';
@@ -23,11 +25,20 @@ export const FolderView = ({ selection, selectionToggle, onTagSelected, tags, hi
   const results = useAsync(async () => {
     const folders: DashboardSection[] = [];
     if (!hidePseudoFolders) {
-      const stars = await getBackendSrv().get('api/user/stars');
-      if (stars.length > 0) {
-        folders.push({ title: 'Starred', icon: 'star', kind: 'query-star', uid: '__starred', itemsUIDs: stars });
+      if (contextSrv.isSignedIn) {
+        const stars = await getBackendSrv().get('api/user/stars');
+        if (stars.length > 0) {
+          folders.push({ title: 'Starred', icon: 'star', kind: 'query-star', uid: '__starred', itemsUIDs: stars });
+        }
+      }
+
+      const ids = impressionSrv.getDashboardOpened();
+      if (ids.length) {
+        const itemsUIDs = await getBackendSrv().get(`/api/dashboards/ids/${ids.slice(0, 30).join(',')}`);
+        if (itemsUIDs.length) {
+          folders.push({ title: 'Recent', icon: 'clock', kind: 'query-recent', uid: '__recent', itemsUIDs });
+        }
       }
-      folders.push({ title: 'Recent', icon: 'clock', kind: 'query-recent', uid: '__recent' });
     }
     folders.push({ title: 'General', url: '/dashboards', kind: 'folder', uid: GENERAL_FOLDER_UID });
 

public/app/features/search/page/components/SearchView.tsx

@@ -205,6 +205,7 @@ export const SearchView = ({ showManage, folderDTO, queryText, hidePseudoFolders
           onSortChange={onSortChange}
           onTagFilterChange={onTagFilterChange}
           getTagOptions={getTagOptions}
+          getSortOptions={getGrafanaSearcher().getSortOptions}
           onDatasourceChange={onDatasourceChange}
           query={query}
         />

public/app/features/search/page/components/columns.tsx

@@ -3,7 +3,7 @@ import { isNumber } from 'lodash';
 import React from 'react';
 import SVG from 'react-inlinesvg';
 
-import { Field } from '@grafana/data';
+import { Field, getFieldDisplayName } from '@grafana/data';
 import { config, getDataSourceSrv } from '@grafana/runtime';
 import { Checkbox, Icon, IconButton, IconName, TagList } from '@grafana/ui';
 
@@ -11,7 +11,6 @@ import { QueryResponse, SearchResultMeta } from '../../service';
 import { SelectionChecker, SelectionToggle } from '../selection';
 
 import { TableColumn } from './SearchResultsTable';
-import { getSortFieldDisplayName } from './sorting';
 
 const TYPE_COLUMN_WIDTH = 250;
 const DATASOURCE_COLUMN_WIDTH = 200;
@@ -172,7 +171,7 @@ export const generateColumns = (
 
   if (sortField) {
     columns.push({
-      Header: () => <div className={styles.sortedHeader}>{getSortFieldDisplayName(sortField.name)}</div>,
+      Header: () => <div className={styles.sortedHeader}>{getFieldDisplayName(sortField)}</div>,
       Cell: (p) => {
         let value = sortField.values.get(p.row.index);
         try {
@@ -182,7 +181,7 @@ export const generateColumns = (
         } catch {}
         return (
           <div {...p.cellProps} className={styles.sortedItems}>
-            {value}
+            {`${value}`}
           </div>
         );
       },

public/app/features/search/page/components/sorting.ts

@@ -1,37 +0,0 @@
-import { SelectableValue } from '@grafana/data';
-import { config } from '@grafana/runtime';
-
-// Enterprise only sort field values for dashboards
-const sortFields = [
-  { name: 'views_total', display: 'Views total' },
-  { name: 'views_last_30_days', display: 'Views 30 days' },
-  { name: 'errors_total', display: 'Errors total' },
-  { name: 'errors_last_30_days', display: 'Errors 30 days' },
-];
-
-// This should eventually be filled by an API call, but hardcoded is a good start
-export async function getSortOptions(): Promise<SelectableValue[]> {
-  const opts: SelectableValue[] = [
-    { value: 'name_sort', label: 'Alphabetically (A-Z)' },
-    { value: '-name_sort', label: 'Alphabetically (Z-A)' },
-  ];
-
-  if (config.licenseInfo.enabledFeatures.analytics) {
-    for (const sf of sortFields) {
-      opts.push({ value: `-${sf.name}`, label: `${sf.display} (most)` });
-      opts.push({ value: `${sf.name}`, label: `${sf.display} (least)` });
-    }
-  }
-
-  return opts;
-}
-
-/** Given the internal field name, this gives a reasonable display name for the table colum header */
-export function getSortFieldDisplayName(name: string) {
-  for (const sf of sortFields) {
-    if (sf.name === name) {
-      return sf.display;
-    }
-  }
-  return name;
-}

public/app/features/search/service/bluge.ts

@@ -1,6 +1,6 @@
 import { lastValueFrom } from 'rxjs';
 
-import { ArrayVector, DataFrame, DataFrameView, getDisplayProcessor } from '@grafana/data';
+import { ArrayVector, DataFrame, DataFrameView, getDisplayProcessor, SelectableValue } from '@grafana/data';
 import { config, getDataSourceSrv } from '@grafana/runtime';
 import { TermCount } from 'app/core/components/TagFilter/TagFilter';
 import { GrafanaDatasource } from 'app/plugins/datasource/grafana/datasource';
@@ -16,10 +16,6 @@ export class BlugeSearcher implements GrafanaSearcher {
     return doSearchQuery(query);
   }
 
-  async list(location: string): Promise<QueryResponse> {
-    return doSearchQuery({ query: `list:${location ?? ''}` });
-  }
-
   async tags(query: SearchQuery): Promise<TermCount[]> {
     const ds = (await getDataSourceSrv().get('-- Grafana --')) as GrafanaDatasource;
     const target = {
@@ -46,12 +42,29 @@ export class BlugeSearcher implements GrafanaSearcher {
     }
     return [];
   }
+
+  // This should eventually be filled by an API call, but hardcoded is a good start
+  getSortOptions(): Promise<SelectableValue[]> {
+    const opts: SelectableValue[] = [
+      { value: 'name_sort', label: 'Alphabetically (A-Z)' },
+      { value: '-name_sort', label: 'Alphabetically (Z-A)' },
+    ];
+
+    if (config.licenseInfo.enabledFeatures.analytics) {
+      for (const sf of sortFields) {
+        opts.push({ value: `-${sf.name}`, label: `${sf.display} (most)` });
+        opts.push({ value: `${sf.name}`, label: `${sf.display} (least)` });
+      }
+    }
+
+    return Promise.resolve(opts);
+  }
 }
 
 const firstPageSize = 50;
 const nextPageSizes = 100;
 
-export async function doSearchQuery(query: SearchQuery): Promise<QueryResponse> {
+async function doSearchQuery(query: SearchQuery): Promise<QueryResponse> {
   const ds = (await getDataSourceSrv().get('-- Grafana --')) as GrafanaDatasource;
   const target = {
     ...query,
@@ -84,8 +97,19 @@ export async function doSearchQuery(query: SearchQuery): Promise<QueryResponse>
 
   const meta = first.meta.custom as SearchResultMeta;
   if (!meta.locationInfo) {
-    meta.locationInfo = {};
+    meta.locationInfo = {}; // always set it so we can append
   }
+
+  // Set the field name to a better display name
+  if (meta.sortBy?.length) {
+    const field = first.fields.find((f) => f.name === meta.sortBy);
+    if (field) {
+      const name = getSortFieldDisplayName(field.name);
+      meta.sortBy = name;
+      field.name = name; // make it look nicer
+    }
+  }
+
   const view = new DataFrameView<DashboardQueryResult>(first);
   return {
     totalRows: meta.count ?? first.length,
@@ -146,3 +170,21 @@ function getTermCountsFrom(frame: DataFrame): TermCount[] {
   }
   return counts;
 }
+
+// Enterprise only sort field values for dashboards
+const sortFields = [
+  { name: 'views_total', display: 'Views total' },
+  { name: 'views_last_30_days', display: 'Views 30 days' },
+  { name: 'errors_total', display: 'Errors total' },
+  { name: 'errors_last_30_days', display: 'Errors 30 days' },
+];
+
+/** Given the internal field name, this gives a reasonable display name for the table colum header */
+function getSortFieldDisplayName(name: string) {
+  for (const sf of sortFields) {
+    if (sf.name === name) {
+      return sf.display;
+    }
+  }
+  return name;
+}

public/app/features/search/service/searcher.ts

@@ -1,11 +1,17 @@
+import { config } from '@grafana/runtime';
+
 import { BlugeSearcher } from './bluge';
+import { SQLSearcher } from './sql';
 import { GrafanaSearcher } from './types';
 
 let searcher: GrafanaSearcher | undefined = undefined;
 
 export function getGrafanaSearcher(): GrafanaSearcher {
   if (!searcher) {
-    searcher = new BlugeSearcher();
+    const useBluge =
+      config.featureToggles.panelTitleSearch && // set in system configs
+      window.location.search.indexOf('index=sql') < 0; // or URL override
+    searcher = useBluge ? new BlugeSearcher() : new SQLSearcher();
   }
   return searcher!;
 }

public/app/features/search/service/sql.ts

@@ -0,0 +1,198 @@
+import { ArrayVector, DataFrame, DataFrameView, FieldType, getDisplayProcessor, SelectableValue } from '@grafana/data';
+import { config } from '@grafana/runtime';
+import { TermCount } from 'app/core/components/TagFilter/TagFilter';
+import { backendSrv } from 'app/core/services/backend_srv';
+
+import { DashboardSearchHit } from '../types';
+
+import { LocationInfo } from './types';
+
+import { DashboardQueryResult, GrafanaSearcher, QueryResponse, SearchQuery } from '.';
+
+interface APIQuery {
+  query?: string;
+  tag?: string[];
+  limit?: number;
+  page?: number;
+  type?: string;
+  // DashboardIds []int64
+  folderIds?: number[];
+  sort?: string;
+
+  // NEW!!!! TODO TODO: needs backend support?
+  dashboardUIDs?: string[];
+}
+
+// Internal object to hold folderId
+interface LocationInfoEXT extends LocationInfo {
+  folderId?: number;
+}
+
+export class SQLSearcher implements GrafanaSearcher {
+  locationInfo: Record<string, LocationInfoEXT> = {
+    general: {
+      kind: 'folder',
+      name: 'General',
+      url: '/dashboards',
+      folderId: 0,
+    },
+  }; // share location info with everyone
+
+  async search(query: SearchQuery): Promise<QueryResponse> {
+    if (query.facet?.length) {
+      throw 'facets not supported!';
+    }
+    const q: APIQuery = {
+      limit: 1000, // 1k max values
+      tag: query.tags,
+      sort: query.sort,
+    };
+
+    if (query.query === '*') {
+      if (query.kind?.length === 1 && query.kind[0] === 'folder') {
+        q.type = 'dash-folder';
+      }
+    } else if (query.query?.length) {
+      q.query = query.query;
+    }
+
+    if (query.uid) {
+      q.query = query.uid.join(', '); // TODO! this will return nothing
+      q.dashboardUIDs = query.uid;
+    } else if (query.location?.length) {
+      let info = this.locationInfo[query.location];
+      if (!info) {
+        // This will load all folder folders
+        await this.doAPIQuery({ type: 'dash-folder', limit: 999 });
+        info = this.locationInfo[query.location];
+      }
+      q.folderIds = [info.folderId ?? 0];
+    }
+    return this.doAPIQuery(q);
+  }
+
+  // returns the appropriate sorting options
+  async getSortOptions(): Promise<SelectableValue[]> {
+    // {
+    //   "sortOptions": [
+    //     {
+    //       "description": "Sort results in an alphabetically ascending order",
+    //       "displayName": "Alphabetically (A–Z)",
+    //       "meta": "",
+    //       "name": "alpha-asc"
+    //     },
+    //     {
+    //       "description": "Sort results in an alphabetically descending order",
+    //       "displayName": "Alphabetically (Z–A)",
+    //       "meta": "",
+    //       "name": "alpha-desc"
+    //     }
+    //   ]
+    // }
+    const opts = await backendSrv.get('/api/search/sorting');
+    return opts.sortOptions.map((v: any) => ({
+      value: v.name,
+      label: v.displayName,
+    }));
+  }
+
+  // NOTE: the bluge query will find tags within the current results, the SQL based one does not
+  async tags(query: SearchQuery): Promise<TermCount[]> {
+    const terms = (await backendSrv.get('/api/dashboards/tags')) as TermCount[];
+    return terms.sort((a, b) => b.count - a.count);
+  }
+
+  async doAPIQuery(query: APIQuery): Promise<QueryResponse> {
+    const rsp = (await backendSrv.get('/api/search', query)) as DashboardSearchHit[];
+
+    // Field values (columnar)
+    const kind: string[] = [];
+    const name: string[] = [];
+    const uid: string[] = [];
+    const url: string[] = [];
+    const tags: string[][] = [];
+    const location: string[] = [];
+    const sortBy: number[] = [];
+    let sortMetaName: string | undefined;
+
+    for (let hit of rsp) {
+      const k = hit.type === 'dash-folder' ? 'folder' : 'dashboard';
+      kind.push(k);
+      name.push(hit.title);
+      uid.push(hit.uid!);
+      url.push(hit.url);
+      tags.push(hit.tags);
+      sortBy.push(hit.sortMeta!);
+
+      let v = hit.folderUid;
+      if (!v && k === 'dashboard') {
+        v = 'general';
+      }
+      location.push(v!);
+
+      if (hit.sortMetaName?.length) {
+        sortMetaName = hit.sortMetaName;
+      }
+
+      if (hit.folderUid && hit.folderTitle) {
+        this.locationInfo[hit.folderUid] = {
+          kind: 'folder',
+          name: hit.folderTitle,
+          url: hit.folderUrl!,
+          folderId: hit.folderId,
+        };
+      } else if (k === 'folder') {
+        this.locationInfo[hit.uid!] = {
+          kind: k,
+          name: hit.title!,
+          url: hit.url,
+          folderId: hit.id,
+        };
+      }
+    }
+
+    const data: DataFrame = {
+      fields: [
+        { name: 'kind', type: FieldType.string, config: {}, values: new ArrayVector(kind) },
+        { name: 'name', type: FieldType.string, config: {}, values: new ArrayVector(name) },
+        { name: 'uid', type: FieldType.string, config: {}, values: new ArrayVector(uid) },
+        { name: 'url', type: FieldType.string, config: {}, values: new ArrayVector(url) },
+        { name: 'tags', type: FieldType.other, config: {}, values: new ArrayVector(tags) },
+        { name: 'location', type: FieldType.string, config: {}, values: new ArrayVector(location) },
+      ],
+      length: name.length,
+      meta: {
+        custom: {
+          count: name.length,
+          max_score: 1,
+          locationInfo: this.locationInfo,
+        },
+      },
+    };
+
+    // Add enterprise sort fields as a field in the frame
+    if (sortMetaName?.length && sortBy.length) {
+      data.meta!.custom!.sortBy = sortMetaName;
+      data.fields.push({
+        name: sortMetaName, // Used in display
+        type: FieldType.number,
+        config: {},
+        values: new ArrayVector(sortBy),
+      });
+    }
+
+    for (const field of data.fields) {
+      field.display = getDisplayProcessor({ field, theme: config.theme2 });
+    }
+
+    const view = new DataFrameView<DashboardQueryResult>(data);
+    return {
+      totalRows: data.length,
+      view,
+
+      // Paging not supported with this version
+      loadMoreItems: async (startIndex: number, stopIndex: number): Promise<void> => {},
+      isItemLoaded: (index: number): boolean => true,
+    };
+  }
+}

public/app/features/search/service/types.ts

@@ -1,4 +1,4 @@
-import { DataFrameView } from '@grafana/data';
+import { DataFrameView, SelectableValue } from '@grafana/data';
 import { TermCount } from 'app/core/components/TagFilter/TagFilter';
 
 export interface FacetField {
@@ -32,7 +32,6 @@ export interface DashboardQueryResult {
   tags: string[];
   location: string; // url that can be split
   ds_uid: string[];
-  score?: number;
 }
 
 export interface LocationInfo {
@@ -45,6 +44,7 @@ export interface SearchResultMeta {
   count: number;
   max_score: number;
   locationInfo: Record<string, LocationInfo>;
+  sortBy?: string;
 }
 
 export interface QueryResponse {
@@ -62,6 +62,6 @@ export interface QueryResponse {
 
 export interface GrafanaSearcher {
   search: (query: SearchQuery) => Promise<QueryResponse>;
-  list: (location: string) => Promise<QueryResponse>;
   tags: (query: SearchQuery) => Promise<TermCount[]>;
+  getSortOptions: () => Promise<SelectableValue[]>;
 }