Search: create a separate HTTP endpoint (#55634)

* search: create a separate http endpoint

* search: extract api uri

* search: rename uri

* search: replicate the readiness check

* search: replicate the readiness check metric

* search: update mock

.betterer.results

@@ -5153,16 +5153,7 @@ exports[`better eslint`] = {
     "public/app/features/search/service/bluge.ts:5381": [
       [0, 0, 0, "Do not use any type assertions.", "0"],
       [0, 0, 0, "Do not use any type assertions.", "1"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "2"],
+      [0, 0, 0, "Do not use any type assertions.", "2"]
-      [0, 0, 0, "Do not use any type assertions.", "3"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "4"],
-      [0, 0, 0, "Do not use any type assertions.", "5"],
-      [0, 0, 0, "Do not use any type assertions.", "6"],
-      [0, 0, 0, "Do not use any type assertions.", "7"],
-      [0, 0, 0, "Do not use any type assertions.", "8"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "9"],
-      [0, 0, 0, "Do not use any type assertions.", "10"],
-      [0, 0, 0, "Do not use any type assertions.", "11"]
     ],
     "public/app/features/search/service/sql.ts:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"]

pkg/api/api.go

@@ -259,6 +259,10 @@ func (hs *HTTPServer) registerRoutes() {
 			apiRoute.Group("/storage", hs.StorageService.RegisterHTTPRoutes)
 		}
 
+		if hs.Features.IsEnabled(featuremgmt.FlagPanelTitleSearch) {
+			apiRoute.Group("/search-v2", hs.SearchV2HTTPService.RegisterHTTPRoutes)
+		}
+
 		// current org
 		apiRoute.Group("/org", func(orgRoute routing.RouteRegister) {
 			userIDScope := ac.Scope("users", "id", ac.Parameter(":userId"))

pkg/api/http_server.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/middleware/csrf"
+	"github.com/grafana/grafana/pkg/services/searchV2"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -139,6 +140,7 @@ type HTTPServer struct {
 	ThumbService                 thumbs.Service
 	ExportService                export.ExportService
 	StorageService               store.StorageService
+	SearchV2HTTPService          searchV2.SearchHTTPService
 	ContextHandler               *contexthandler.ContextHandler
 	SQLStore                     sqlstore.Store
 	AlertEngine                  *alerting.AlertEngine
@@ -237,7 +239,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 	publicDashboardsApi *publicdashboardsApi.Api, userService user.Service, tempUserService tempUser.Service,
 	loginAttemptService loginAttempt.Service, orgService org.Service, teamService team.Service,
 	accesscontrolService accesscontrol.Service, dashboardThumbsService dashboardThumbs.Service, navTreeService navtree.Service,
-	annotationRepo annotations.Repository, tagService tag.Service,
+	annotationRepo annotations.Repository, tagService tag.Service, searchv2HTTPService searchV2.SearchHTTPService,
 ) (*HTTPServer, error) {
 	web.Env = cfg.Env
 	m := web.New()
@@ -276,6 +278,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 		Login:                        loginService,
 		AccessControl:                accessControl,
 		DataProxy:                    dataSourceProxy,
+		SearchV2HTTPService:          searchv2HTTPService,
 		SearchService:                searchService,
 		ExportService:                exportService,
 		Live:                         live,

pkg/server/wire.go

@@ -235,6 +235,7 @@ var wireBasicSet = wire.NewSet(
 	datasourceproxy.ProvideService,
 	search.ProvideService,
 	searchV2.ProvideService,
+	searchV2.ProvideSearchHTTPService,
 	store.ProvideService,
 	export.ProvideService,
 	live.ProvideService,

pkg/services/searchV2/http.go

@@ -0,0 +1,76 @@
+package searchV2
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/api/routing"
+	"github.com/grafana/grafana/pkg/middleware"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type SearchHTTPService interface {
+	RegisterHTTPRoutes(storageRoute routing.RouteRegister)
+}
+
+type searchHTTPService struct {
+	search SearchService
+}
+
+func ProvideSearchHTTPService(search SearchService) SearchHTTPService {
+	return &searchHTTPService{search: search}
+}
+
+func (s *searchHTTPService) RegisterHTTPRoutes(storageRoute routing.RouteRegister) {
+	storageRoute.Post("/", middleware.ReqSignedIn, routing.Wrap(s.doQuery))
+}
+
+func (s *searchHTTPService) doQuery(c *models.ReqContext) response.Response {
+	searchReadinessCheckResp := s.search.IsReady(c.Req.Context(), c.OrgID)
+	if !searchReadinessCheckResp.IsReady {
+		dashboardSearchNotServedRequestsCounter.With(prometheus.Labels{
+			"reason": searchReadinessCheckResp.Reason,
+		}).Inc()
+
+		bytes, err := (&data.Frame{
+			Name: "Loading",
+		}).MarshalJSON()
+
+		if err != nil {
+			return response.Error(500, "error marshalling response", err)
+		}
+		return response.JSON(200, bytes)
+	}
+
+	body, err := io.ReadAll(c.Req.Body)
+	if err != nil {
+		return response.Error(500, "error reading bytes", err)
+	}
+
+	query := &DashboardQuery{}
+	err = json.Unmarshal(body, query)
+	if err != nil {
+		return response.Error(400, "error parsing body", err)
+	}
+
+	resp := s.search.doDashboardQuery(c.Req.Context(), c.SignedInUser, c.OrgID, *query)
+
+	if resp.Error != nil {
+		return response.Error(500, "error handling search request", resp.Error)
+	}
+
+	if len(resp.Frames) != 1 {
+		return response.Error(500, "invalid search response", errors.New("invalid search response"))
+	}
+
+	bytes, err := resp.Frames[0].MarshalJSON()
+	if err != nil {
+		return response.Error(500, "error marshalling response", err)
+	}
+
+	return response.JSON(200, bytes)
+}

pkg/services/searchV2/search_service_mock.go

@@ -8,6 +8,8 @@ import (
 	backend "github.com/grafana/grafana-plugin-sdk-go/backend"
 
 	mock "github.com/stretchr/testify/mock"
+
+	user "github.com/grafana/grafana/pkg/services/user"
 )
 
 // MockSearchService is an autogenerated mock type for the SearchService type
@@ -15,13 +17,13 @@ type MockSearchService struct {
 	mock.Mock
 }
 
-// DoDashboardQuery provides a mock function with given fields: ctx, user, orgId, query
+// DoDashboardQuery provides a mock function with given fields: ctx, _a1, orgId, query
-func (_m *MockSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgId int64, query DashboardQuery) *backend.DataResponse {
+func (_m *MockSearchService) DoDashboardQuery(ctx context.Context, _a1 *backend.User, orgId int64, query DashboardQuery) *backend.DataResponse {
-	ret := _m.Called(ctx, user, orgId, query)
+	ret := _m.Called(ctx, _a1, orgId, query)
 
 	var r0 *backend.DataResponse
 	if rf, ok := ret.Get(0).(func(context.Context, *backend.User, int64, DashboardQuery) *backend.DataResponse); ok {
-		r0 = rf(ctx, user, orgId, query)
+		r0 = rf(ctx, _a1, orgId, query)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).(*backend.DataResponse)
@@ -82,3 +84,19 @@ func (_m *MockSearchService) Run(ctx context.Context) error {
 func (_m *MockSearchService) TriggerReIndex() {
 	_m.Called()
 }
+
+// doDashboardQuery provides a mock function with given fields: ctx, _a1, orgId, query
+func (_m *MockSearchService) doDashboardQuery(ctx context.Context, _a1 *user.SignedInUser, orgId int64, query DashboardQuery) *backend.DataResponse {
+	ret := _m.Called(ctx, _a1, orgId, query)
+
+	var r0 *backend.DataResponse
+	if rf, ok := ret.Get(0).(func(context.Context, *user.SignedInUser, int64, DashboardQuery) *backend.DataResponse); ok {
+		r0 = rf(ctx, _a1, orgId, query)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*backend.DataResponse)
+		}
+	}
+
+	return r0
+}

pkg/services/searchV2/service.go

@@ -26,6 +26,15 @@ import (
 var (
 	namespace                               = "grafana"
 	subsystem                               = "search"
+	dashboardSearchNotServedRequestsCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "dashboard_search_requests_not_served_total",
+			Help:      "A counter for dashboard search requests that could not be served due to an ongoing search engine indexing",
+		},
+		[]string{"reason"},
+	)
 	dashboardSearchFailureRequestsCounter = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: namespace,
@@ -194,7 +203,21 @@ func (s *StandardSearchService) getUser(ctx context.Context, backendUser *backen
 
 func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgID int64, q DashboardQuery) *backend.DataResponse {
 	start := time.Now()
-	query := s.doDashboardQuery(ctx, user, orgID, q)
+
+	signedInUser, err := s.getUser(ctx, user, orgID)
+
+	if err != nil {
+		dashboardSearchFailureRequestsCounter.With(prometheus.Labels{
+			"reason": "get_user_error",
+		}).Inc()
+
+		duration := time.Since(start).Seconds()
+		dashboardSearchFailureRequestsDuration.Observe(duration)
+
+		return &backend.DataResponse{Error: err}
+	}
+
+	query := s.doDashboardQuery(ctx, signedInUser, orgID, q)
 
 	duration := time.Since(start).Seconds()
 	if query.Error != nil {
@@ -206,16 +229,8 @@ func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *back
 	return query
 }
 
-func (s *StandardSearchService) doDashboardQuery(ctx context.Context, user *backend.User, orgID int64, q DashboardQuery) *backend.DataResponse {
+func (s *StandardSearchService) doDashboardQuery(ctx context.Context, signedInUser *user.SignedInUser, orgID int64, q DashboardQuery) *backend.DataResponse {
 	rsp := &backend.DataResponse{}
-	signedInUser, err := s.getUser(ctx, user, orgID)
-	if err != nil {
-		dashboardSearchFailureRequestsCounter.With(prometheus.Labels{
-			"reason": "get_user_error",
-		}).Inc()
-		rsp.Error = err
-		return rsp
-	}
 
 	filter, err := s.auth.GetDashboardReadFilter(signedInUser)
 	if err != nil {

pkg/services/searchV2/stub.go

@@ -5,11 +5,16 @@ import (
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/services/user"
 )
 
 type stubSearchService struct {
 }
 
+func (s *stubSearchService) doDashboardQuery(ctx context.Context, user *user.SignedInUser, orgId int64, query DashboardQuery) *backend.DataResponse {
+	return s.DoDashboardQuery(ctx, nil, orgId, query)
+}
+
 func (s *stubSearchService) IsReady(ctx context.Context, orgId int64) IsSearchReadyResponse {
 	return IsSearchReadyResponse{}
 }

pkg/services/searchV2/types.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/grafana/grafana/pkg/registry"
+	"github.com/grafana/grafana/pkg/services/user"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 )
@@ -41,6 +42,7 @@ type SearchService interface {
 	registry.CanBeDisabled
 	registry.BackgroundService
 	DoDashboardQuery(ctx context.Context, user *backend.User, orgId int64, query DashboardQuery) *backend.DataResponse
+	doDashboardQuery(ctx context.Context, user *user.SignedInUser, orgId int64, query DashboardQuery) *backend.DataResponse
 	IsReady(ctx context.Context, orgId int64) IsSearchReadyResponse
 	RegisterDashboardIndexExtender(ext DashboardIndexExtender)
 	TriggerReIndex()

public/app/features/search/service/bluge.ts

@@ -1,10 +1,13 @@
-import { lastValueFrom } from 'rxjs';
+import {
-
+  ArrayVector,
-import { ArrayVector, DataFrame, DataFrameView, getDisplayProcessor, SelectableValue } from '@grafana/data';
+  DataFrame,
+  DataFrameView,
+  getDisplayProcessor,
+  SelectableValue,
+  toDataFrame,
+} from '@grafana/data';
 import { config, getBackendSrv } from '@grafana/runtime';
 import { TermCount } from 'app/core/components/TagFilter/TagFilter';
-import { getGrafanaDatasource } from 'app/plugins/datasource/grafana/datasource';
-import { GrafanaQueryType } from 'app/plugins/datasource/grafana/types';
 
 import { replaceCurrentFolderQuery } from './utils';
 
@@ -14,6 +17,8 @@ import { DashboardQueryResult, GrafanaSearcher, QueryResponse, SearchQuery, Sear
 // and that it can not serve any search requests. We are temporarily using the old SQL Search API as a fallback when that happens.
 const loadingFrameName = 'Loading';
 
+const searchURI = 'api/search-v2';
+
 export class BlugeSearcher implements GrafanaSearcher {
   constructor(private fallbackSearcher: GrafanaSearcher) {}
 
@@ -38,36 +43,23 @@ export class BlugeSearcher implements GrafanaSearcher {
   }
 
   async tags(query: SearchQuery): Promise<TermCount[]> {
-    const ds = await getGrafanaDatasource();
+    const req = {
-    const target = {
-      refId: 'TagsQuery',
-      queryType: GrafanaQueryType.Search,
-      search: {
       ...query,
       query: query.query ?? '*',
       sort: undefined, // no need to sort the initial query results (not used)
       facet: [{ field: 'tag' }],
       limit: 1, // 0 would be better, but is ignored by the backend
-      },
     };
 
-    const data = (
+    const frame = toDataFrame(await getBackendSrv().post(searchURI, req));
-      await lastValueFrom(
-        ds.query({
-          targets: [target],
-        } as any)
-      )
-    ).data as DataFrame[];
 
-    if (data?.[0]?.name === loadingFrameName) {
+    if (frame?.name === loadingFrameName) {
       return this.fallbackSearcher.tags(query);
     }
 
-    for (const frame of data) {
     if (frame.fields[0].name === 'tag') {
       return getTermCountsFrom(frame);
     }
-    }
     return [];
   }
 
@@ -94,23 +86,15 @@ export class BlugeSearcher implements GrafanaSearcher {
 
   async doSearchQuery(query: SearchQuery): Promise<QueryResponse> {
     query = await replaceCurrentFolderQuery(query);
-    const ds = await getGrafanaDatasource();
+    const req = {
-    const target = {
-      refId: 'Search',
-      queryType: GrafanaQueryType.Search,
-      search: {
       ...query,
       query: query.query ?? '*',
       limit: query.limit ?? firstPageSize,
-      },
     };
-    const rsp = await lastValueFrom(
-      ds.query({
-        targets: [target],
-      } as any)
-    );
 
-    const first = (rsp.data?.[0] as DataFrame) ?? { fields: [], length: 0 };
+    const rsp = await getBackendSrv().post(searchURI, req);
+
+    const first = rsp ? toDataFrame(rsp) : { fields: [], length: 0 };
 
     if (first.name === loadingFrameName) {
       return this.fallbackSearcher.search(query);
@@ -154,24 +138,13 @@ export class BlugeSearcher implements GrafanaSearcher {
         if (from >= meta.count) {
           return;
         }
-        const frame = (
+        const frame = toDataFrame(
-          await lastValueFrom(
+          await getBackendSrv().post(searchURI, {
-            ds.query({
+            ...(req ?? {}),
-              targets: [
-                {
-                  ...target,
-                  search: {
-                    ...(target?.search ?? {}),
             from,
             limit: nextPageSizes,
-                  },
+          })
-                  refId: 'Page',
+        );
-                  facet: undefined,
-                },
-              ],
-            } as any)
-          )
-        ).data?.[0] as DataFrame;
 
         if (!frame) {
           console.log('no results', frame);