Query library: requiresDevMode dummy backend (#56466)

* query library - dummy backend

* fix tests

* dont explicitly marshall backend dataresponse

* skip integration tests

* null check for tests

* added query library to codeowners

* null check for tests

* lint
This commit is contained in:
Artur Wierzbicki 2022-10-07 22:31:45 +04:00 committed by GitHub
parent 23e04c0f9c
commit bf264d2f76
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 1292 additions and 13 deletions

1
.github/CODEOWNERS vendored
View File

@ -80,6 +80,7 @@ go.sum @grafana/backend-platform
/pkg/services/live/ @grafana/grafana-edge-squad
/pkg/services/searchV2/ @grafana/grafana-edge-squad
/pkg/services/store/ @grafana/grafana-edge-squad
/pkg/services/querylibrary/ @grafana/grafana-edge-squad
/pkg/services/export/ @grafana/grafana-edge-squad
/pkg/infra/filestore/ @grafana/grafana-edge-squad
/pkg/tsdb/testdatasource/sims/ @grafana/grafana-edge-squad

View File

@ -72,4 +72,5 @@ export interface FeatureToggles {
redshiftAsyncQueryDataSupport?: boolean;
athenaAsyncQueryDataSupport?: boolean;
increaseInMemDatabaseQueryCache?: boolean;
queryLibrary?: boolean;
}

View File

@ -284,6 +284,10 @@ func (hs *HTTPServer) registerRoutes() {
apiRoute.Group("/search-v2", hs.SearchV2HTTPService.RegisterHTTPRoutes)
}
if hs.QueryLibraryHTTPService != nil && !hs.QueryLibraryHTTPService.IsDisabled() {
apiRoute.Group("/query-library", hs.QueryLibraryHTTPService.RegisterHTTPRoutes)
}
// current org
apiRoute.Group("/org", func(orgRoute routing.RouteRegister) {
userIDScope := ac.Scope("users", "id", ac.Parameter(":userId"))

View File

@ -220,6 +220,12 @@ func (hs *HTTPServer) GetDashboard(c *models.ReqContext) response.Response {
return response.Error(500, "Error while loading library panels", err)
}
if hs.QueryLibraryService != nil && !hs.QueryLibraryService.IsDisabled() {
if err := hs.QueryLibraryService.UpdateDashboardQueries(c.Req.Context(), c.SignedInUser, dash); err != nil {
return response.Error(500, "Error while loading saved queries", err)
}
}
dto := dtos.DashboardFullWithMeta{
Dashboard: dash.Data,
Meta: meta,

View File

@ -15,6 +15,7 @@ import (
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/middleware/csrf"
"github.com/grafana/grafana/pkg/services/querylibrary"
"github.com/grafana/grafana/pkg/services/searchV2"
"github.com/grafana/grafana/pkg/services/store/object"
"github.com/grafana/grafana/pkg/services/userauth"
@ -143,6 +144,8 @@ type HTTPServer struct {
StorageService store.StorageService
httpObjectStore object.HTTPObjectStore
SearchV2HTTPService searchV2.SearchHTTPService
QueryLibraryHTTPService querylibrary.HTTPService
QueryLibraryService querylibrary.Service
ContextHandler *contexthandler.ContextHandler
SQLStore sqlstore.Store
AlertEngine *alerting.AlertEngine
@ -243,7 +246,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
loginAttemptService loginAttempt.Service, orgService org.Service, teamService team.Service,
accesscontrolService accesscontrol.Service, dashboardThumbsService thumbs.DashboardThumbService, navTreeService navtree.Service,
annotationRepo annotations.Repository, tagService tag.Service, searchv2HTTPService searchV2.SearchHTTPService,
userAuthService userauth.Service,
userAuthService userauth.Service, queryLibraryHTTPService querylibrary.HTTPService, queryLibraryService querylibrary.Service,
) (*HTTPServer, error) {
web.Env = cfg.Env
m := web.New()
@ -346,6 +349,8 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
annotationsRepo: annotationRepo,
tagService: tagService,
userAuthService: userAuthService,
QueryLibraryHTTPService: queryLibraryHTTPService,
QueryLibraryService: queryLibraryService,
}
if hs.Listener != nil {
hs.log.Debug("Using provided listener")

View File

@ -95,7 +95,7 @@ func TestIntegrationPluginManager(t *testing.T) {
pg := postgres.ProvideService(cfg)
my := mysql.ProvideService(cfg, hcp)
ms := mssql.ProvideService(cfg)
sv2 := searchV2.ProvideService(cfg, sqlstore.InitTestDB(t), nil, nil, tracer, features, nil, nil)
sv2 := searchV2.ProvideService(cfg, sqlstore.InitTestDB(t), nil, nil, tracer, features, nil, nil, nil)
graf := grafanads.ProvideService(sv2, nil)
coreRegistry := coreplugin.ProvideCoreRegistry(am, cw, cm, es, grap, idb, lk, otsdb, pr, tmpo, td, pg, my, ms, graf)

View File

@ -104,6 +104,7 @@ import (
publicdashboardsService "github.com/grafana/grafana/pkg/services/publicdashboards/service"
"github.com/grafana/grafana/pkg/services/query"
"github.com/grafana/grafana/pkg/services/queryhistory"
"github.com/grafana/grafana/pkg/services/querylibrary/querylibraryimpl"
"github.com/grafana/grafana/pkg/services/quota/quotaimpl"
"github.com/grafana/grafana/pkg/services/rendering"
"github.com/grafana/grafana/pkg/services/search"
@ -278,6 +279,8 @@ var wireBasicSet = wire.NewSet(
secretsManager.ProvideSecretsService,
wire.Bind(new(secrets.Service), new(*secretsManager.SecretsService)),
secretsDatabase.ProvideSecretsStore,
querylibraryimpl.ProvideService,
querylibraryimpl.ProvideHTTPService,
wire.Bind(new(secrets.Store), new(*secretsDatabase.SecretsStoreImpl)),
secretsMigrator.ProvideSecretsMigrator,
wire.Bind(new(secrets.Migrator), new(*secretsMigrator.SecretsMigrator)),

View File

@ -310,5 +310,11 @@ var (
Name: "increaseInMemDatabaseQueryCache",
Description: "Enable more in memory caching for database queries",
},
{
Name: "queryLibrary",
Description: "Reusable query library",
State: FeatureStateAlpha,
RequiresDevMode: true,
},
}
)

View File

@ -230,4 +230,8 @@ const (
// FlagIncreaseInMemDatabaseQueryCache
// Enable more in memory caching for database queries
FlagIncreaseInMemDatabaseQueryCache = "increaseInMemDatabaseQueryCache"
// FlagQueryLibrary
// Reusable query library
FlagQueryLibrary = "queryLibrary"
)

View File

@ -15,6 +15,7 @@ const (
WeightSavedItems
WeightCreate
WeightDashboard
WeightQueryLibrary
WeightExplore
WeightAlerting
WeightDataConnections

View File

@ -17,6 +17,7 @@ import (
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/services/pluginsettings"
pref "github.com/grafana/grafana/pkg/services/preference"
"github.com/grafana/grafana/pkg/services/querylibrary"
"github.com/grafana/grafana/pkg/services/star"
"github.com/grafana/grafana/pkg/setting"
)
@ -33,6 +34,7 @@ type ServiceImpl struct {
accesscontrolService ac.Service
kvStore kvstore.KVStore
apiKeyService apikey.Service
queryLibraryService querylibrary.HTTPService
// Navigation
navigationAppConfig map[string]NavigationAppConfig
@ -44,7 +46,7 @@ type NavigationAppConfig struct {
SortWeight int64
}
func ProvideService(cfg *setting.Cfg, accessControl ac.AccessControl, pluginStore plugins.Store, pluginSettings pluginsettings.Service, starService star.Service, features *featuremgmt.FeatureManager, dashboardService dashboards.DashboardService, accesscontrolService ac.Service, kvStore kvstore.KVStore, apiKeyService apikey.Service) navtree.Service {
func ProvideService(cfg *setting.Cfg, accessControl ac.AccessControl, pluginStore plugins.Store, pluginSettings pluginsettings.Service, starService star.Service, features *featuremgmt.FeatureManager, dashboardService dashboards.DashboardService, accesscontrolService ac.Service, kvStore kvstore.KVStore, apiKeyService apikey.Service, queryLibraryService querylibrary.HTTPService) navtree.Service {
service := &ServiceImpl{
cfg: cfg,
log: log.New("navtree service"),
@ -57,6 +59,7 @@ func ProvideService(cfg *setting.Cfg, accessControl ac.AccessControl, pluginStor
accesscontrolService: accesscontrolService,
kvStore: kvStore,
apiKeyService: apiKeyService,
queryLibraryService: queryLibraryService,
}
service.readNavigationSettings()
@ -121,6 +124,18 @@ func (s *ServiceImpl) GetNavTree(c *models.ReqContext, hasEditPerm bool, prefs *
})
}
if !s.queryLibraryService.IsDisabled() {
treeRoot.AddSection(&navtree.NavLink{
Text: "Query Library",
Id: "query",
SubTitle: "Store, import, export and manage your team queries in an easy way.",
Icon: "file-search-alt",
SortWeight: navtree.WeightQueryLibrary,
Section: navtree.NavSectionCore,
Url: s.cfg.AppSubURL + "/query-library",
})
}
if setting.ProfileEnabled && c.IsSignedIn {
treeRoot.AddSection(s.getProfileNode(c))
}

View File

@ -0,0 +1,87 @@
package querylibraryimpl
import (
"encoding/json"
"fmt"
"io"
"strings"
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/api/routing"
"github.com/grafana/grafana/pkg/middleware"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/querylibrary"
)
type queriesServiceHTTPHandler struct {
service querylibrary.Service
}
func (s *queriesServiceHTTPHandler) IsDisabled() bool {
return s.service.IsDisabled()
}
func (s *queriesServiceHTTPHandler) delete(c *models.ReqContext) response.Response {
uid := c.Query("uid")
err := s.service.Delete(c.Req.Context(), c.SignedInUser, uid)
if err != nil {
return response.Error(500, fmt.Sprintf("error deleting query with id %s", uid), err)
}
return response.JSON(200, map[string]interface{}{
"success": true,
})
}
func (s *queriesServiceHTTPHandler) RegisterHTTPRoutes(routes routing.RouteRegister) {
reqSignedIn := middleware.ReqSignedIn
routes.Get("/", reqSignedIn, routing.Wrap(s.getBatch))
routes.Post("/", reqSignedIn, routing.Wrap(s.update))
routes.Delete("/", reqSignedIn, routing.Wrap(s.delete))
}
func (s *queriesServiceHTTPHandler) getBatch(c *models.ReqContext) response.Response {
uids := c.QueryStrings("uid")
queries, err := s.service.GetBatch(c.Req.Context(), c.SignedInUser, uids)
if err != nil {
return response.Error(500, fmt.Sprintf("error retrieving queries: [%s]", strings.Join(uids, ",")), err)
}
return response.JSON(200, queries)
}
func (s *queriesServiceHTTPHandler) update(c *models.ReqContext) response.Response {
body, err := io.ReadAll(c.Req.Body)
if err != nil {
return response.Error(500, "error reading bytes", err)
}
query := &querylibrary.Query{}
err = json.Unmarshal(body, query)
if err != nil {
return response.Error(400, "error parsing body", err)
}
if err := s.service.Update(c.Req.Context(), c.SignedInUser, query); err != nil {
var msg string
if len(query.UID) > 0 {
msg = fmt.Sprintf("error updating query with UID %s: %s", query.UID, err.Error())
} else {
msg = fmt.Sprintf("error updating query with: %s", err.Error())
}
return response.Error(500, msg, err)
}
return response.JSON(200, map[string]interface{}{
"success": true,
})
}
func ProvideHTTPService(
queriesService querylibrary.Service,
) querylibrary.HTTPService {
return &queriesServiceHTTPHandler{
service: queriesService,
}
}

View File

@ -0,0 +1,290 @@
package querylibraryimpl
import (
"context"
"fmt"
"strings"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/expr"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/x/persistentcollection"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/querylibrary"
"github.com/grafana/grafana/pkg/services/searchV2/dslookup"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
)
func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles) querylibrary.Service {
return &service{
cfg: cfg,
log: log.New("queryLibraryService"),
features: features,
collection: persistentcollection.NewLocalFSPersistentCollection[*querylibrary.Query]("query-library", cfg.DataPath, 1),
}
}
type service struct {
cfg *setting.Cfg
features featuremgmt.FeatureToggles
log log.Logger
collection persistentcollection.PersistentCollection[*querylibrary.Query]
}
type perRequestQueryLoader struct {
service querylibrary.Service
queries map[string]*querylibrary.Query
ctx context.Context
user *user.SignedInUser
}
func (q *perRequestQueryLoader) byUID(uid string) (*querylibrary.Query, error) {
if q, ok := q.queries[uid]; ok {
return q, nil
}
queries, err := q.service.GetBatch(q.ctx, q.user, []string{uid})
if err != nil {
return nil, err
}
if len(queries) != 1 {
return nil, err
}
q.queries[uid] = queries[0]
return queries[0], nil
}
func newPerRequestQueryLoader(ctx context.Context, user *user.SignedInUser, service querylibrary.Service) queryLoader {
return &perRequestQueryLoader{queries: make(map[string]*querylibrary.Query), ctx: ctx, user: user, service: service}
}
type queryLoader interface {
byUID(uid string) (*querylibrary.Query, error)
}
func (s *service) UpdateDashboardQueries(ctx context.Context, user *user.SignedInUser, dash *models.Dashboard) error {
queryLoader := newPerRequestQueryLoader(ctx, user, s)
return s.updateQueriesRecursively(queryLoader, dash.Data)
}
func (s *service) updateQueriesRecursively(loader queryLoader, parent *simplejson.Json) error {
panels := parent.Get("panels").MustArray()
for i := range panels {
panelAsJSON := simplejson.NewFromAny(panels[i])
panelType := panelAsJSON.Get("type").MustString()
if panelType == "row" {
err := s.updateQueriesRecursively(loader, panelAsJSON)
if err != nil {
return err
}
continue
}
queryUID := panelAsJSON.GetPath("savedQueryLink", "ref", "uid").MustString()
if queryUID == "" {
continue
}
query, err := loader.byUID(queryUID)
if err != nil {
return err
}
if query == nil {
// query deleted - unlink
panelAsJSON.Set("savedQueryLink", nil)
continue
}
queriesAsMap := make([]interface{}, 0)
for idx := range query.Queries {
queriesAsMap = append(queriesAsMap, query.Queries[idx].MustMap())
}
panelAsJSON.Set("targets", queriesAsMap)
isMixed, firstDsRef := isQueryWithMixedDataSource(query)
if isMixed {
panelAsJSON.Set("datasource", map[string]interface{}{
"uid": "-- Mixed --",
"type": "datasource",
})
} else {
panelAsJSON.Set("datasource", map[string]interface{}{
"uid": firstDsRef.UID,
"type": firstDsRef.Type,
})
}
}
return nil
}
func (s *service) IsDisabled() bool {
return !s.features.IsEnabled(featuremgmt.FlagQueryLibrary) || !s.features.IsEnabled(featuremgmt.FlagPanelTitleSearch)
}
func namespaceFromUser(user *user.SignedInUser) string {
return fmt.Sprintf("orgId-%d", user.OrgID)
}
func (s *service) Search(ctx context.Context, user *user.SignedInUser, options querylibrary.QuerySearchOptions) ([]querylibrary.QueryInfo, error) {
queries, err := s.collection.Find(ctx, namespaceFromUser(user), func(_ *querylibrary.Query) (bool, error) { return true, nil })
if err != nil {
return nil, err
}
queryInfo := asQueryInfo(queries)
filteredQueryInfo := make([]querylibrary.QueryInfo, 0)
for _, q := range queryInfo {
if len(options.Query) > 0 {
lowerTitle := strings.ReplaceAll(strings.ToLower(q.Title), " ", "")
lowerQuery := strings.ReplaceAll(strings.ToLower(options.Query), " ", "")
if !strings.Contains(lowerTitle, lowerQuery) {
continue
}
}
if len(options.DatasourceUID) > 0 || len(options.DatasourceType) > 0 {
dsUids := make(map[string]bool)
dsTypes := make(map[string]bool)
for _, ds := range q.Datasource {
dsUids[ds.UID] = true
dsTypes[ds.Type] = true
}
if len(options.DatasourceType) > 0 && !dsTypes[options.DatasourceType] {
continue
}
if len(options.DatasourceUID) > 0 && !dsUids[options.DatasourceUID] {
continue
}
}
filteredQueryInfo = append(filteredQueryInfo, q)
}
return filteredQueryInfo, nil
}
func asQueryInfo(queries []*querylibrary.Query) []querylibrary.QueryInfo {
res := make([]querylibrary.QueryInfo, 0)
for _, query := range queries {
res = append(res, querylibrary.QueryInfo{
UID: query.UID,
Title: query.Title,
Description: query.Description,
Tags: query.Tags,
TimeFrom: query.Time.From,
TimeTo: query.Time.To,
SchemaVersion: query.SchemaVersion,
Datasource: extractDataSources(query),
})
}
return res
}
func getDatasourceUID(q *simplejson.Json) string {
uid := q.Get("datasource").Get("uid").MustString()
if uid == "" {
uid = q.Get("datasource").MustString()
}
if expr.IsDataSource(uid) {
return expr.DatasourceUID
}
return uid
}
func isQueryWithMixedDataSource(q *querylibrary.Query) (isMixed bool, firstDsRef dslookup.DataSourceRef) {
dsRefs := extractDataSources(q)
for _, dsRef := range dsRefs {
if dsRef.Type == expr.DatasourceType {
continue
}
if firstDsRef.UID == "" {
firstDsRef = dsRef
continue
}
if firstDsRef.UID != dsRef.UID || firstDsRef.Type != dsRef.Type {
return true, firstDsRef
}
}
return false, firstDsRef
}
func extractDataSources(query *querylibrary.Query) []dslookup.DataSourceRef {
ds := make([]dslookup.DataSourceRef, 0)
for _, q := range query.Queries {
dsUid := getDatasourceUID(q)
dsType := q.Get("datasource").Get("type").MustString()
if expr.IsDataSource(dsUid) {
dsType = expr.DatasourceType
}
ds = append(ds, dslookup.DataSourceRef{
UID: dsUid,
Type: dsType,
})
}
return ds
}
func (s *service) GetBatch(ctx context.Context, user *user.SignedInUser, uids []string) ([]*querylibrary.Query, error) {
uidMap := make(map[string]bool)
for _, uid := range uids {
uidMap[uid] = true
}
return s.collection.Find(ctx, namespaceFromUser(user), func(q *querylibrary.Query) (bool, error) {
if _, ok := uidMap[q.UID]; ok {
return true, nil
}
return false, nil
})
}
func (s *service) Update(ctx context.Context, user *user.SignedInUser, query *querylibrary.Query) error {
if query.UID == "" {
query.UID = util.GenerateShortUID()
return s.collection.Insert(ctx, namespaceFromUser(user), query)
}
_, err := s.collection.Update(ctx, namespaceFromUser(user), func(q *querylibrary.Query) (updated bool, updatedItem *querylibrary.Query, err error) {
if q.UID == query.UID {
return true, query, nil
}
return false, nil, nil
})
return err
}
func (s *service) Delete(ctx context.Context, user *user.SignedInUser, uid string) error {
_, err := s.collection.Delete(ctx, namespaceFromUser(user), func(q *querylibrary.Query) (bool, error) {
if q.UID == uid {
return true, nil
}
return false, nil
})
return err
}

View File

@ -0,0 +1,284 @@
package querylibrary_tests
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/api/dtos"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/services/querylibrary"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/services/user"
)
type queryLibraryAPIClient struct {
token string
url string
user *user.SignedInUser
sqlStore *sqlstore.SQLStore
}
func newQueryLibraryAPIClient(token string, baseUrl string, user *user.SignedInUser, sqlStore *sqlstore.SQLStore) *queryLibraryAPIClient {
return &queryLibraryAPIClient{
token: token,
url: baseUrl,
user: user,
sqlStore: sqlStore,
}
}
func (q *queryLibraryAPIClient) update(ctx context.Context, query *querylibrary.Query) error {
buf := bytes.Buffer{}
enc := json.NewEncoder(&buf)
err := enc.Encode(query)
if err != nil {
return err
}
url := fmt.Sprintf("%s/query-library", q.url)
req, err := http.NewRequestWithContext(ctx, "POST", url, &buf)
if err != nil {
return err
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", q.token))
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
defer func() {
_ = resp.Body.Close()
}()
return err
}
func (q *queryLibraryAPIClient) delete(ctx context.Context, uid string) error {
url := fmt.Sprintf("%s/query-library?uid=%s", q.url, uid)
req, err := http.NewRequestWithContext(ctx, "DELETE", url, bytes.NewBuffer([]byte("")))
if err != nil {
return err
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", q.token))
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
defer func() {
_ = resp.Body.Close()
}()
return err
}
func (q *queryLibraryAPIClient) get(ctx context.Context, uid string) (*querylibrary.Query, error) {
url := fmt.Sprintf("%s/query-library?uid=%s", q.url, uid)
req, err := http.NewRequestWithContext(ctx, "GET", url, bytes.NewBuffer([]byte("")))
if err != nil {
return nil, err
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", q.token))
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer func() {
_ = resp.Body.Close()
}()
b, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
query := make([]*querylibrary.Query, 0)
err = json.Unmarshal(b, &query)
if len(query) > 0 {
return query[0], err
}
return nil, err
}
type querySearchInfo struct {
kind string
uid string
name string
dsUIDs []string
location string
}
func (q *queryLibraryAPIClient) search(ctx context.Context, options querylibrary.QuerySearchOptions) ([]*querySearchInfo, error) {
return q.searchRetry(ctx, options, 1)
}
func (q *queryLibraryAPIClient) searchRetry(ctx context.Context, options querylibrary.QuerySearchOptions, attempt int) ([]*querySearchInfo, error) {
if attempt >= 3 {
return nil, errors.New("max attempts")
}
url := fmt.Sprintf("%s/search-v2", q.url)
text := "*"
if options.Query != "" {
text = options.Query
}
searchReq := map[string]interface{}{
"query": text,
"sort": "name_sort",
"kind": []string{"query"},
"limit": 50,
}
searchReqJson, err := simplejson.NewFromAny(searchReq).MarshalJSON()
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(searchReqJson))
if err != nil {
return nil, err
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", q.token))
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer func() {
_ = resp.Body.Close()
}()
b, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
r := &backend.DataResponse{}
err = json.Unmarshal(b, r)
if len(r.Frames) != 1 {
return nil, fmt.Errorf("expected a single frame, received %s", string(b))
}
frame := r.Frames[0]
if frame.Name == "Loading" {
time.Sleep(100 * time.Millisecond)
return q.searchRetry(ctx, options, attempt+1)
}
res := make([]*querySearchInfo, 0)
frameLen, _ := frame.RowLen()
for i := 0; i < frameLen; i++ {
fKind, _ := frame.FieldByName("kind")
fUid, _ := frame.FieldByName("uid")
fName, _ := frame.FieldByName("name")
dsUID, _ := frame.FieldByName("ds_uid")
fLocation, _ := frame.FieldByName("location")
rawValue, ok := dsUID.At(i).(json.RawMessage)
if !ok || rawValue == nil {
return nil, errors.New("invalid ds_uid field")
}
jsonValue, err := rawValue.MarshalJSON()
if err != nil {
return nil, err
}
var uids []string
err = json.Unmarshal(jsonValue, &uids)
if err != nil {
return nil, err
}
res = append(res, &querySearchInfo{
kind: fKind.At(i).(string),
uid: fUid.At(i).(string),
name: fName.At(i).(string),
dsUIDs: uids,
location: fLocation.At(i).(string),
})
}
return res, err
}
func (q *queryLibraryAPIClient) getDashboard(ctx context.Context, uid string) (*dtos.DashboardFullWithMeta, error) {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/dashboards/uid/%s", q.url, uid), bytes.NewBuffer([]byte("")))
if err != nil {
return nil, err
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", q.token))
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer func() {
_ = resp.Body.Close()
}()
b, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
res := &dtos.DashboardFullWithMeta{}
err = json.Unmarshal(b, res)
if err != nil {
return nil, err
}
return res, nil
}
func (q *queryLibraryAPIClient) createDashboard(ctx context.Context, dash *simplejson.Json) (string, error) {
buf := bytes.Buffer{}
enc := json.NewEncoder(&buf)
dashMap, err := dash.Map()
if err != nil {
return "", err
}
err = enc.Encode(dashMap)
if err != nil {
return "", err
}
url := fmt.Sprintf("%s/dashboards/db", q.url)
req, err := http.NewRequestWithContext(ctx, "POST", url, &buf)
if err != nil {
return "", err
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", q.token))
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
defer func() {
_ = resp.Body.Close()
}()
if err != nil {
return "", err
}
jsonResp, err := simplejson.NewFromReader(resp.Body)
if err != nil {
return "", err
}
return jsonResp.Get("uid").MustString(), nil
}

View File

@ -0,0 +1,72 @@
package querylibrary_tests
import (
"fmt"
"testing"
apikeygenprefix "github.com/grafana/grafana/pkg/components/apikeygenprefixed"
"github.com/grafana/grafana/pkg/server"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/org"
saAPI "github.com/grafana/grafana/pkg/services/serviceaccounts/api"
saTests "github.com/grafana/grafana/pkg/services/serviceaccounts/tests"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/tests/testinfra"
"github.com/stretchr/testify/require"
)
func createServiceAccountAdminToken(t *testing.T, name string, env *server.TestEnv) (string, *user.SignedInUser) {
t.Helper()
account := saTests.SetupUserServiceAccount(t, env.SQLStore, saTests.TestUser{
Name: name,
Role: string(org.RoleAdmin),
Login: name,
IsServiceAccount: true,
OrgID: 1,
})
keyGen, err := apikeygenprefix.New(saAPI.ServiceID)
require.NoError(t, err)
_ = saTests.SetupApiKey(t, env.SQLStore, saTests.TestApiKey{
Name: name,
Role: org.RoleAdmin,
OrgId: account.OrgID,
Key: keyGen.HashedKey,
ServiceAccountID: &account.ID,
})
return keyGen.ClientSecret, &user.SignedInUser{
UserID: account.ID,
Email: account.Email,
Name: account.Name,
Login: account.Login,
OrgID: account.OrgID,
}
}
type testContext struct {
authToken string
client *queryLibraryAPIClient
user *user.SignedInUser
}
func createTestContext(t *testing.T) testContext {
t.Helper()
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
EnableFeatureToggles: []string{featuremgmt.FlagPanelTitleSearch, featuremgmt.FlagQueryLibrary},
})
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path)
authToken, serviceAccountUser := createServiceAccountAdminToken(t, "query-library", env)
client := newQueryLibraryAPIClient(authToken, fmt.Sprintf("http://%s/api", grafanaListedAddr), serviceAccountUser, env.SQLStore)
return testContext{
authToken: authToken,
client: client,
user: serviceAccountUser,
}
}

View File

@ -0,0 +1,289 @@
package querylibrary_tests
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/services/querylibrary"
"github.com/grafana/grafana/pkg/tsdb/grafanads"
)
func TestCreateAndDelete(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
testCtx := createTestContext(t)
err := testCtx.client.update(ctx, &querylibrary.Query{
UID: "",
Title: "first query",
Tags: []string{},
Description: "",
Time: querylibrary.Time{
From: "now-15m",
To: "now-30m",
},
Queries: []*simplejson.Json{
simplejson.NewFromAny(map[string]interface{}{
"datasource": map[string]string{
"uid": grafanads.DatasourceUID,
"type": "datasource",
},
"queryType": "randomWalk",
"refId": "A",
}),
simplejson.NewFromAny(map[string]interface{}{
"datasource": map[string]string{
"uid": grafanads.DatasourceUID,
"type": "datasource",
},
"queryType": "list",
"path": "img",
"refId": "B",
}),
},
Variables: []*simplejson.Json{},
})
require.NoError(t, err)
search, err := testCtx.client.search(ctx, querylibrary.QuerySearchOptions{
Query: "",
})
require.NoError(t, err)
require.Len(t, search, 1)
info := search[0]
require.Equal(t, "query", info.kind)
require.Equal(t, "first query", info.name)
require.Equal(t, "General", info.location)
require.Equal(t, []string{grafanads.DatasourceUID, grafanads.DatasourceUID}, info.dsUIDs)
err = testCtx.client.delete(ctx, info.uid)
require.NoError(t, err)
search, err = testCtx.client.search(ctx, querylibrary.QuerySearchOptions{
Query: "",
})
require.NoError(t, err)
require.Len(t, search, 0)
query, err := testCtx.client.get(ctx, info.uid)
require.NoError(t, err)
require.Nil(t, query)
}
func createQuery(t *testing.T, ctx context.Context, testCtx testContext) string {
t.Helper()
err := testCtx.client.update(ctx, &querylibrary.Query{
UID: "",
Title: "first query",
Tags: []string{},
Description: "",
Time: querylibrary.Time{
From: "now-15m",
To: "now-30m",
},
Queries: []*simplejson.Json{
simplejson.NewFromAny(map[string]interface{}{
"datasource": map[string]string{
"uid": grafanads.DatasourceUID,
"type": "datasource",
},
"queryType": "randomWalk",
"refId": "A",
}),
simplejson.NewFromAny(map[string]interface{}{
"datasource": map[string]string{
"uid": grafanads.DatasourceUID,
"type": "datasource",
},
"queryType": "list",
"path": "img",
"refId": "B",
}),
},
Variables: []*simplejson.Json{},
})
require.NoError(t, err)
search, err := testCtx.client.search(ctx, querylibrary.QuerySearchOptions{
Query: "",
})
require.NoError(t, err)
require.Len(t, search, 1)
return search[0].uid
}
func TestDashboardGetWithLatestSavedQueries(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
testCtx := createTestContext(t)
queryUID := createQuery(t, ctx, testCtx)
dashUID, err := testCtx.client.createDashboard(ctx, simplejson.NewFromAny(map[string]interface{}{
"dashboard": map[string]interface{}{
"title": "my-new-dashboard",
"panels": []interface{}{
map[string]interface{}{
"id": int64(1),
"gridPos": map[string]interface{}{
"h": 6,
"w": 6,
"x": 0,
"y": 0,
},
},
map[string]interface{}{
"id": int64(2),
"gridPos": map[string]interface{}{
"h": 6,
"w": 6,
"x": 6,
"y": 0,
},
"savedQueryLink": map[string]interface{}{
"ref": map[string]string{
"uid": queryUID,
},
},
},
},
},
"folderId": 0,
"message": "",
"overwrite": true,
}))
require.NoError(t, err)
dashboard, err := testCtx.client.getDashboard(ctx, dashUID)
require.NoError(t, err)
panelsAsArray, err := dashboard.Dashboard.Get("panels").Array()
require.NoError(t, err)
require.Len(t, panelsAsArray, 2)
secondPanel := simplejson.NewFromAny(panelsAsArray[1])
require.Equal(t, []interface{}{
map[string]interface{}{
"datasource": map[string]interface{}{
"uid": grafanads.DatasourceUID,
"type": "datasource",
},
"queryType": "randomWalk",
"refId": "A",
},
map[string]interface{}{
"datasource": map[string]interface{}{
"uid": grafanads.DatasourceUID,
"type": "datasource",
},
"queryType": "list",
"path": "img",
"refId": "B",
},
}, secondPanel.Get("targets").MustArray())
require.Equal(t, map[string]interface{}{
"uid": grafanads.DatasourceUID,
"type": "datasource",
}, secondPanel.Get("datasource").MustMap())
// update, expect changes when getting dashboards
err = testCtx.client.update(ctx, &querylibrary.Query{
UID: queryUID,
Title: "first query",
Tags: []string{},
Description: "",
Time: querylibrary.Time{
From: "now-15m",
To: "now-30m",
},
Queries: []*simplejson.Json{
simplejson.NewFromAny(map[string]interface{}{
"datasource": map[string]interface{}{
"uid": grafanads.DatasourceUID,
"type": "datasource",
},
"queryType": "randomWalk",
"refId": "A",
}),
simplejson.NewFromAny(map[string]interface{}{
"datasource": map[string]interface{}{
"uid": "different-datasource-uid",
"type": "datasource",
},
"queryType": "randomWalk",
"path": "img",
"refId": "B",
}),
simplejson.NewFromAny(map[string]interface{}{
"datasource": map[string]interface{}{
"uid": "different-datasource-uid-2",
"type": "datasource",
},
"queryType": "randomWalk",
"path": "img",
"refId": "C",
}),
},
Variables: []*simplejson.Json{},
})
require.NoError(t, err)
dashboard, err = testCtx.client.getDashboard(ctx, dashUID)
require.NoError(t, err)
panelsAsArray, err = dashboard.Dashboard.Get("panels").Array()
require.NoError(t, err)
require.Len(t, panelsAsArray, 2)
secondPanel = simplejson.NewFromAny(panelsAsArray[1])
require.Equal(t, []interface{}{
map[string]interface{}{
"datasource": map[string]interface{}{
"uid": grafanads.DatasourceUID,
"type": "datasource",
},
"queryType": "randomWalk",
"refId": "A",
},
map[string]interface{}{
"datasource": map[string]interface{}{
"uid": "different-datasource-uid",
"type": "datasource",
},
"queryType": "randomWalk",
"path": "img",
"refId": "B",
},
map[string]interface{}{
"datasource": map[string]interface{}{
"uid": "different-datasource-uid-2",
"type": "datasource",
},
"queryType": "randomWalk",
"path": "img",
"refId": "C",
},
}, secondPanel.Get("targets").MustArray())
require.Equal(t, map[string]interface{}{
"uid": "-- Mixed --",
"type": "datasource",
}, secondPanel.Get("datasource").MustMap())
}

View File

@ -0,0 +1,88 @@
package querylibrary
import (
"context"
"github.com/grafana/grafana/pkg/api/routing"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/services/searchV2/dslookup"
"github.com/grafana/grafana/pkg/services/user"
)
type Time struct {
// From Start time in epoch timestamps in milliseconds or relative using Grafana time units.
// required: true
// example: now-1h
From string `json:"from"`
// To End time in epoch timestamps in milliseconds or relative using Grafana time units.
// required: true
// example: now
To string `json:"to"`
}
type Query struct {
UID string `json:"uid"`
Title string `json:"title"`
Tags []string `json:"tags"`
Description string `json:"description"`
SchemaVersion int64 `json:"schemaVersion"`
Time Time `json:"time"`
// queries.refId Specifies an identifier of the query. Is optional and default to “A”.
// queries.datasourceId Specifies the data source to be queried. Each query in the request must have an unique datasourceId.
// queries.maxDataPoints - Species maximum amount of data points that dashboard panel can render. Is optional and default to 100.
// queries.intervalMs - Specifies the time interval in milliseconds of time series. Is optional and defaults to 1000.
// required: true
// example: [ { "refId": "A", "intervalMs": 86400000, "maxDataPoints": 1092, "datasource":{ "uid":"PD8C576611E62080A" }, "rawSql": "SELECT 1 as valueOne, 2 as valueTwo", "format": "table" } ]
Queries []*simplejson.Json `json:"queries"`
Variables []*simplejson.Json `json:"variables"`
}
type SavedQueryRef struct {
UID string `json:"uid"`
}
type SavedQueryLink struct {
Ref SavedQueryRef `json:"ref"`
}
type QueryInfo struct {
UID string `json:"uid"`
Title string `json:"title"`
Description string `json:"description"`
Tags []string `json:"tags"`
TimeFrom string `json:"timeFrom"`
TimeTo string `json:"timeTo"`
SchemaVersion int64 `json:"schemaVersion"`
Datasource []dslookup.DataSourceRef `json:"datasource,omitempty"` // UIDs
}
type QuerySearchOptions struct {
DatasourceUID string
Query string
DatasourceType string
}
type Service interface {
Search(ctx context.Context, user *user.SignedInUser, options QuerySearchOptions) ([]QueryInfo, error)
GetBatch(ctx context.Context, user *user.SignedInUser, uids []string) ([]*Query, error)
Update(ctx context.Context, user *user.SignedInUser, query *Query) error
Delete(ctx context.Context, user *user.SignedInUser, uid string) error
UpdateDashboardQueries(ctx context.Context, user *user.SignedInUser, dash *models.Dashboard) error
registry.CanBeDisabled
}
type HTTPService interface {
registry.CanBeDisabled
RegisterHTTPRoutes(routes routing.RouteRegister)
}

View File

@ -86,7 +86,7 @@ var (
func service(t *testing.T) *StandardSearchService {
service, ok := ProvideService(&setting.Cfg{Search: setting.SearchSettings{}},
nil, nil, accesscontrolmock.New(), tracing.InitializeTracerForTest(), featuremgmt.WithFeatures(),
nil, nil).(*StandardSearchService)
nil, nil, nil).(*StandardSearchService)
require.True(t, ok)
return service
}

View File

@ -23,6 +23,7 @@ const (
entityKindDashboard entityKind = object.StandardKindDashboard
entityKindFolder entityKind = object.StandardKindFolder
entityKindDatasource entityKind = object.StandardKindDataSource
entityKindQuery entityKind = object.StandardKindQuery
)
func (r entityKind) IsValid() bool {

View File

@ -5,6 +5,7 @@ import (
"errors"
"io"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/api/routing"
@ -36,14 +37,12 @@ func (s *searchHTTPService) doQuery(c *models.ReqContext) response.Response {
"reason": searchReadinessCheckResp.Reason,
}).Inc()
bytes, err := (&data.Frame{
Name: "Loading",
}).MarshalJSON()
if err != nil {
return response.Error(500, "error marshalling response", err)
}
return response.JSON(200, bytes)
return response.JSON(200, &backend.DataResponse{
Frames: []*data.Frame{{
Name: "Loading",
}},
Error: nil,
})
}
body, err := io.ReadAll(c.Req.Body)

View File

@ -0,0 +1,112 @@
package searchV2
import (
"context"
"encoding/json"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/services/querylibrary"
"github.com/grafana/grafana/pkg/services/user"
)
// TEMPORARY FILE
func (s *StandardSearchService) searchQueries(ctx context.Context, user *user.SignedInUser, q DashboardQuery) *backend.DataResponse {
queryText := q.Query
if queryText == "*" {
queryText = ""
}
queryInfo, err := s.queries.Search(ctx, user, querylibrary.QuerySearchOptions{
Query: queryText,
DatasourceUID: q.Datasource,
DatasourceType: q.DatasourceType,
})
if err != nil {
return &backend.DataResponse{Error: err}
}
header := &customMeta{
SortBy: q.Sort,
Count: uint64(len(queryInfo)),
}
fScore := data.NewFieldFromFieldType(data.FieldTypeFloat64, 0)
fUID := data.NewFieldFromFieldType(data.FieldTypeString, 0)
fKind := data.NewFieldFromFieldType(data.FieldTypeString, 0)
fPType := data.NewFieldFromFieldType(data.FieldTypeString, 0)
fName := data.NewFieldFromFieldType(data.FieldTypeString, 0)
fURL := data.NewFieldFromFieldType(data.FieldTypeString, 0)
fLocation := data.NewFieldFromFieldType(data.FieldTypeString, 0)
fTags := data.NewFieldFromFieldType(data.FieldTypeNullableJSON, 0)
fDSUIDs := data.NewFieldFromFieldType(data.FieldTypeJSON, 0)
fExplain := data.NewFieldFromFieldType(data.FieldTypeNullableJSON, 0)
fScore.Name = "score"
fUID.Name = "uid"
fKind.Name = "kind"
fName.Name = "name"
fLocation.Name = "location"
fURL.Name = "url"
fURL.Config = &data.FieldConfig{
Links: []data.DataLink{
{Title: "link", URL: "${__value.text}"},
},
}
fPType.Name = "panel_type"
fDSUIDs.Name = "ds_uid"
fTags.Name = "tags"
fExplain.Name = "explain"
frame := data.NewFrame("Query results", fKind, fUID, fName, fPType, fURL, fTags, fDSUIDs, fLocation)
if q.Explain {
frame.Fields = append(frame.Fields, fScore, fExplain)
}
frame.SetMeta(&data.FrameMeta{
Type: "search-results",
Custom: header,
})
fieldLen := 0
for _, q := range queryInfo {
fKind.Append(string(entityKindQuery))
fUID.Append(q.UID)
fPType.Append("")
fName.Append(q.Title)
fURL.Append("")
fLocation.Append("General")
tags := q.Tags
if tags == nil {
tags = make([]string, 0)
}
tagsJson := mustJsonRawMessage(tags)
fTags.Append(&tagsJson)
dsUids := make([]string, 0)
for _, dsRef := range q.Datasource {
dsUids = append(dsUids, dsRef.UID)
}
fDSUIDs.Append(mustJsonRawMessage(dsUids))
// extend fields to match the longest field
fieldLen++
for _, f := range frame.Fields {
if fieldLen > f.Len() {
f.Extend(fieldLen - f.Len())
}
}
}
return &backend.DataResponse{
Frames: data.Frames{frame},
}
}
func mustJsonRawMessage(arr []string) json.RawMessage {
js, _ := json.Marshal(arr)
return js
}

View File

@ -7,6 +7,7 @@ import (
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/services/querylibrary"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -74,6 +75,8 @@ type StandardSearchService struct {
dashboardIndex *searchIndex
extender DashboardIndexExtender
reIndexCh chan struct{}
queries querylibrary.Service
features featuremgmt.FeatureToggles
}
func (s *StandardSearchService) IsReady(ctx context.Context, orgId int64) IsSearchReadyResponse {
@ -82,7 +85,7 @@ func (s *StandardSearchService) IsReady(ctx context.Context, orgId int64) IsSear
func ProvideService(cfg *setting.Cfg, sql *sqlstore.SQLStore, entityEventStore store.EntityEventsService,
ac accesscontrol.Service, tracer tracing.Tracer, features featuremgmt.FeatureToggles, orgService org.Service,
userService user.Service) SearchService {
userService user.Service, queries querylibrary.Service) SearchService {
extender := &NoopExtender{}
s := &StandardSearchService{
cfg: cfg,
@ -106,6 +109,8 @@ func ProvideService(cfg *setting.Cfg, sql *sqlstore.SQLStore, entityEventStore s
reIndexCh: make(chan struct{}, 1),
orgService: orgService,
userService: userService,
queries: queries,
features: features,
}
return s
}
@ -234,6 +239,10 @@ func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *back
}
func (s *StandardSearchService) doDashboardQuery(ctx context.Context, signedInUser *user.SignedInUser, orgID int64, q DashboardQuery) *backend.DataResponse {
if !s.queries.IsDisabled() && len(q.Kind) == 1 && q.Kind[0] == string(entityKindQuery) {
return s.searchQueries(ctx, signedInUser, q)
}
rsp := &backend.DataResponse{}
filter, err := s.auth.GetDashboardReadFilter(signedInUser)

View File

@ -19,6 +19,7 @@ type DashboardQuery struct {
Location string `json:"location,omitempty"` // parent folder ID
Sort string `json:"sort,omitempty"` // field ASC/DESC
Datasource string `json:"ds_uid,omitempty"` // "datasource" collides with the JSON value at the same leel :()
DatasourceType string `json:"ds_type,omitempty"`
Tags []string `json:"tags,omitempty"`
Kind []string `json:"kind,omitempty"`
PanelType string `json:"panel_type,omitempty"`

View File

@ -13,6 +13,7 @@ const StandardKindFolder = "folder"
const StandardKindPanel = "panel" // types: heatmap, timeseries, table, ...
const StandardKindDataSource = "ds" // types: influx, prometheus, test, ...
const StandardKindTransform = "transform" // types: joinByField, pivot, organizeFields, ...
const StandardKindQuery = "query"
// This is a stub -- it will soon lookup in a registry of known "kinds"
// Each kind will be able to define: