IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2026-08-13 06:34:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Introduce TSDB service (#31520)

Browse Source 
* Introduce TSDB service

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

Co-authored-by: Erik Sundell <erik.sundell87@gmail.com>
Co-authored-by: Will Browne <will.browne@grafana.com>
Co-authored-by: Torkel Ödegaard <torkel@grafana.org>
Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>
Co-authored-by: Zoltán Bedi <zoltan.bedi@gmail.com>
This commit is contained in:
Arve Knudsen
2021-03-08 07:02:49 +01:00
committed by GitHub
co-authored by Erik Sundell Will Browne Torkel Ödegaard Will Browne Zoltán Bedi
parent c899bf3592
commit b79e61656a
203 changed files with 5270 additions and 4777 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/cloudmonitoring/cloudmonitoring.go
+66 -41
View File
@@ -16,14 +16,16 @@ import (
"strings"
"time"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/manager"
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
"golang.org/x/oauth2/google"
)
@@ -62,20 +64,35 @@ const (
mqlEditorMode string = "mql"
)
// CloudMonitoringExecutor executes queries for the CloudMonitoring datasource
type CloudMonitoringExecutor struct {
func init() {
registry.Register(&registry.Descriptor{
Name: "CloudMonitoringService",
InitPriority: registry.Low,
Instance: &Service{},
})
}
type Service struct {
}
func (s *Service) Init() error {
return nil
}
// Executor executes queries for the CloudMonitoring datasource.
type Executor struct {
httpClient *http.Client
dsInfo *models.DataSource
}
// NewCloudMonitoringExecutor initializes a http client
func NewCloudMonitoringExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
// NewExecutor returns an Executor.
func (s *Service) NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
httpClient, err := dsInfo.GetHttpClient()
if err != nil {
return nil, err
}
return &CloudMonitoringExecutor{
return &Executor{
httpClient: httpClient,
dsInfo: dsInfo,
}, nil
@@ -83,14 +100,14 @@ func NewCloudMonitoringExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoi
func init() {
slog = log.New("tsdb.cloudMonitoring")
tsdb.RegisterTsdbQueryEndpoint("stackdriver", NewCloudMonitoringExecutor)
}
// Query takes in the frontend queries, parses them into the CloudMonitoring query format
// executes the queries against the CloudMonitoring API and parses the response into
// the time series or table format
func (e *CloudMonitoringExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
var result *tsdb.Response
func (e *Executor) DataQuery(ctx context.Context, dsInfo *models.DataSource, tsdbQuery plugins.DataQuery) (
plugins.DataResponse, error) {
var result plugins.DataResponse
var err error
queryType := tsdbQuery.Queries[0].Model.Get("type").MustString("")
@@ -108,32 +125,34 @@ func (e *CloudMonitoringExecutor) Query(ctx context.Context, dsInfo *models.Data
return result, err
}
func (e *CloudMonitoringExecutor) getGCEDefaultProject(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
result := &tsdb.Response{
Results: make(map[string]*tsdb.QueryResult),
func (e *Executor) getGCEDefaultProject(ctx context.Context, tsdbQuery plugins.DataQuery) (plugins.DataResponse, error) {
result := plugins.DataResponse{
Results: make(map[string]plugins.DataQueryResult),
}
refId := tsdbQuery.Queries[0].RefId
queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: refId}
refID := tsdbQuery.Queries[0].RefID
queryResult := plugins.DataQueryResult{Meta: simplejson.New(), RefID: refID}
gceDefaultProject, err := e.getDefaultProject(ctx)
if err != nil {
return nil, fmt.Errorf("failed to retrieve default project from GCE metadata server, error: %w", err)
return plugins.DataResponse{}, fmt.Errorf(
"failed to retrieve default project from GCE metadata server, error: %w", err)
}
queryResult.Meta.Set("defaultProject", gceDefaultProject)
result.Results[refId] = queryResult
result.Results[refID] = queryResult
return result, nil
}
func (e *CloudMonitoringExecutor) executeTimeSeriesQuery(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
result := &tsdb.Response{
Results: make(map[string]*tsdb.QueryResult),
func (e *Executor) executeTimeSeriesQuery(ctx context.Context, tsdbQuery plugins.DataQuery) (
plugins.DataResponse, error) {
result := plugins.DataResponse{
Results: make(map[string]plugins.DataQueryResult),
}
queryExecutors, err := e.buildQueryExecutors(tsdbQuery)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
unit := e.resolvePanelUnitFromQueries(queryExecutors)
@@ -141,9 +160,9 @@ func (e *CloudMonitoringExecutor) executeTimeSeriesQuery(ctx context.Context, ts
for _, queryExecutor := range queryExecutors {
queryRes, resp, executedQueryString, err := queryExecutor.run(ctx, tsdbQuery, e)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
err = queryExecutor.parseResponse(queryRes, resp, executedQueryString)
err = queryExecutor.parseResponse(&queryRes, resp, executedQueryString)
if err != nil {
queryRes.Error = err
}
@@ -158,7 +177,7 @@ func (e *CloudMonitoringExecutor) executeTimeSeriesQuery(ctx context.Context, ts
}
frames[i].Fields[1].Config.Unit = unit
}
queryRes.Dataframes = tsdb.NewDecodedDataFrames(frames)
queryRes.Dataframes = plugins.NewDecodedDataFrames(frames)
}
result.Results[queryExecutor.getRefID()] = queryRes
}
@@ -166,7 +185,7 @@ func (e *CloudMonitoringExecutor) executeTimeSeriesQuery(ctx context.Context, ts
return result, nil
}
func (e *CloudMonitoringExecutor) resolvePanelUnitFromQueries(executors []cloudMonitoringQueryExecutor) string {
func (e *Executor) resolvePanelUnitFromQueries(executors []cloudMonitoringQueryExecutor) string {
if len(executors) == 0 {
return ""
}
@@ -186,7 +205,7 @@ func (e *CloudMonitoringExecutor) resolvePanelUnitFromQueries(executors []cloudM
return ""
}
func (e *CloudMonitoringExecutor) buildQueryExecutors(tsdbQuery *tsdb.TsdbQuery) ([]cloudMonitoringQueryExecutor, error) {
func (e *Executor) buildQueryExecutors(tsdbQuery plugins.DataQuery) ([]cloudMonitoringQueryExecutor, error) {
cloudMonitoringQueryExecutors := []cloudMonitoringQueryExecutor{}
startTime, err := tsdbQuery.TimeRange.ParseFrom()
@@ -201,10 +220,14 @@ func (e *CloudMonitoringExecutor) buildQueryExecutors(tsdbQuery *tsdb.TsdbQuery)
durationSeconds := int(endTime.Sub(startTime).Seconds())
for _, query := range tsdbQuery.Queries {
migrateLegacyQueryModel(query)
for i := range tsdbQuery.Queries {
migrateLegacyQueryModel(&tsdbQuery.Queries[i])
query := tsdbQuery.Queries[i]
q := grafanaQuery{}
model, _ := query.Model.MarshalJSON()
model, err := query.Model.MarshalJSON()
if err != nil {
return nil, err
}
if err := json.Unmarshal(model, &q); err != nil {
return nil, fmt.Errorf("could not unmarshal CloudMonitoringQuery json: %w", err)
}
@@ -215,20 +238,19 @@ func (e *CloudMonitoringExecutor) buildQueryExecutors(tsdbQuery *tsdb.TsdbQuery)
var queryInterface cloudMonitoringQueryExecutor
cmtsf := &cloudMonitoringTimeSeriesFilter{
RefID: query.RefId,
RefID: query.RefID,
GroupBys: []string{},
}
switch q.QueryType {
case metricQueryType:
if q.MetricQuery.EditorMode == mqlEditorMode {
queryInterface = &cloudMonitoringTimeSeriesQuery{
RefID: query.RefId,
RefID: query.RefID,
ProjectName: q.MetricQuery.ProjectName,
Query: q.MetricQuery.Query,
IntervalMS: query.IntervalMs,
IntervalMS: query.IntervalMS,
AliasBy: q.MetricQuery.AliasBy,
timeRange: tsdbQuery.TimeRange,
timeRange: *tsdbQuery.TimeRange,
}
} else {
cmtsf.AliasBy = q.MetricQuery.AliasBy
@@ -239,7 +261,7 @@ func (e *CloudMonitoringExecutor) buildQueryExecutors(tsdbQuery *tsdb.TsdbQuery)
}
params.Add("filter", buildFilterString(q.MetricQuery.MetricType, q.MetricQuery.Filters))
params.Add("view", q.MetricQuery.View)
setMetricAggParams(&params, &q.MetricQuery, durationSeconds, query.IntervalMs)
setMetricAggParams(&params, &q.MetricQuery, durationSeconds, query.IntervalMS)
queryInterface = cmtsf
}
case sloQueryType:
@@ -249,8 +271,10 @@ func (e *CloudMonitoringExecutor) buildQueryExecutors(tsdbQuery *tsdb.TsdbQuery)
cmtsf.Service = q.SloQuery.ServiceId
cmtsf.Slo = q.SloQuery.SloId
params.Add("filter", buildSLOFilterExpression(q.SloQuery))
setSloAggParams(&params, &q.SloQuery, durationSeconds, query.IntervalMs)
setSloAggParams(&params, &q.SloQuery, durationSeconds, query.IntervalMS)
queryInterface = cmtsf
default:
panic(fmt.Sprintf("Unrecognized query type %q", q.QueryType))
}
target = params.Encode()
@@ -268,7 +292,7 @@ func (e *CloudMonitoringExecutor) buildQueryExecutors(tsdbQuery *tsdb.TsdbQuery)
return cloudMonitoringQueryExecutors, nil
}
func migrateLegacyQueryModel(query *tsdb.Query) {
func migrateLegacyQueryModel(query *plugins.DataSubQuery) {
mq := query.Model.Get("metricQuery").MustMap()
if mq == nil {
migratedModel := simplejson.NewFromAny(map[string]interface{}{
@@ -402,7 +426,8 @@ func containsLabel(labels []string, newLabel string) bool {
return false
}
func formatLegendKeys(metricType string, defaultMetricName string, labels map[string]string, additionalLabels map[string]string, query *cloudMonitoringTimeSeriesFilter) string {
func formatLegendKeys(metricType string, defaultMetricName string, labels map[string]string,
additionalLabels map[string]string, query *cloudMonitoringTimeSeriesFilter) string {
if query.AliasBy == "" {
return defaultMetricName
}
@@ -488,7 +513,7 @@ func calcBucketBound(bucketOptions cloudMonitoringBucketOptions, n int) string {
return bucketBound
}
func (e *CloudMonitoringExecutor) createRequest(ctx context.Context, dsInfo *models.DataSource, proxyPass string, body io.Reader) (*http.Request, error) {
func (e *Executor) createRequest(ctx context.Context, dsInfo *models.DataSource, proxyPass string, body io.Reader) (*http.Request, error) {
u, err := url.Parse(dsInfo.Url)
if err != nil {
return nil, err
@@ -509,7 +534,7 @@ func (e *CloudMonitoringExecutor) createRequest(ctx context.Context, dsInfo *mod
req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
// find plugin
plugin, ok := plugins.DataSources[dsInfo.Type]
plugin, ok := manager.DataSources[dsInfo.Type]
if !ok {
return nil, errors.New("unable to find datasource plugin CloudMonitoring")
}
@@ -527,7 +552,7 @@ func (e *CloudMonitoringExecutor) createRequest(ctx context.Context, dsInfo *mod
return req, nil
}
func (e *CloudMonitoringExecutor) getDefaultProject(ctx context.Context) (string, error) {
func (e *Executor) getDefaultProject(ctx context.Context) (string, error) {
authenticationType := e.dsInfo.JsonData.Get("authenticationType").MustString(jwtAuthentication)
if authenticationType == gceAuthentication {
defaultCredentials, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/monitoring.read")