Stackdriver: Project selector (#22447)

* clean PR #17366 * udpate vendor * [WIP] Implement projects management for stackdriver * [WIP] Implement projects management for stackdriver * [WIP] Implement projects management for stackdriver * Implement projects management for stackdriver * [WIP][Tests] Fix errors * clean anonymous struct * remove await * don't store project list * Add default project on query editor * gofmt * Fix tests * Move test data source to backend * Use segment instead of dropdown. remove ensure default project since it's not being used anymore. * Fix broken annotation editor * Load gceDefaultAccount only once when in the config page * Reset error message on auth type change * Add metric find query for projects * Remove debug code * Fix broken tests * Fix typings * Fix lint error * Slightly different approach - now having a distiction between config page default project, and project that is selectable from the dropdown in the query editor. * Fix broken tests * Attempt to fix strict ts errors * Prevent state from being set multiple times * Remove noOptionsMessage since it seems to be obosolete in react select * One more attempt to solve ts strict error * Interpolate project template variable. Make sure its loaded correctly when opening variable query editor first time * Implicit any fix * fix: typescript strict null check fixes * Return empty array in case project endpoint fails * Rename project to projectName to prevent clashing with legacy query prop * Fix broken test * fix: Stackdriver - template replace on filter label should have a regex format as that escapes the dots in the label name which is not valid. Co-authored-by: Labesse Kévin <kevin@labesse.me> Co-authored-by: Elias Cédric Laouiti <elias@abtasty.com> Co-authored-by: Daniel Lee <dan.limerick@gmail.com>
2025-02-25 18:55:37 -06:00 · 2020-03-02 09:31:09 -05:00
parent 75fe3c839b
commit 934a8f08ae
156 changed files with 23461 additions and 603 deletions
--- a/pkg/tsdb/stackdriver/ensure_default_project.go
+++ b/pkg/tsdb/stackdriver/ensure_default_project.go
@@ -1,24 +0,0 @@
-package stackdriver
-
-import (
-	"context"
-
-	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/tsdb"
-)
-
-func (e *StackdriverExecutor) ensureDefaultProject(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
-	queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: tsdbQuery.Queries[0].RefId}
-	result := &tsdb.Response{
-		Results: make(map[string]*tsdb.QueryResult),
-	}
-	defaultProject, err := e.getDefaultProject(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	e.dsInfo.JsonData.Set("defaultProject", defaultProject)
-	queryResult.Meta.Set("defaultProject", defaultProject)
-	result.Results[tsdbQuery.Queries[0].RefId] = queryResult
-	return result, nil
-}
--- a/pkg/tsdb/stackdriver/stackdriver.go
+++ b/pkg/tsdb/stackdriver/stackdriver.go
@@ -15,9 +15,6 @@ import (
 	"strings"
 	"time"

-	"golang.org/x/net/context/ctxhttp"
-	"golang.org/x/oauth2/google"
-
 	"github.com/grafana/grafana/pkg/api/pluginproxy"
 	"github.com/grafana/grafana/pkg/components/null"
 	"github.com/grafana/grafana/pkg/components/simplejson"
@@ -27,6 +24,8 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb"
 	"github.com/opentracing/opentracing-go"
+	"golang.org/x/net/context/ctxhttp"
+	"golang.org/x/oauth2/google"
 )

 var (
@@ -81,8 +80,10 @@ func (e *StackdriverExecutor) Query(ctx context.Context, dsInfo *models.DataSour
 	switch queryType {
 	case "annotationQuery":
 		result, err = e.executeAnnotationQuery(ctx, tsdbQuery)
-	case "ensureDefaultProjectQuery":
-		result, err = e.ensureDefaultProject(ctx, tsdbQuery)
+	case "getProjectsListQuery":
+		result, err = e.getProjectList(ctx, tsdbQuery)
+	case "getGCEDefaultProject":
+		result, err = e.getGCEDefaultProject(ctx, tsdbQuery)
 	case "timeSeriesQuery":
 		fallthrough
 	default:
@@ -92,19 +93,29 @@ func (e *StackdriverExecutor) Query(ctx context.Context, dsInfo *models.DataSour
 	return result, err
 }

-func (e *StackdriverExecutor) executeTimeSeriesQuery(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
+func (e *StackdriverExecutor) getGCEDefaultProject(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
 	result := &tsdb.Response{
 		Results: make(map[string]*tsdb.QueryResult),
 	}
+	refId := tsdbQuery.Queries[0].RefId
+	queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: refId}

-	authenticationType := e.dsInfo.JsonData.Get("authenticationType").MustString(jwtAuthentication)
-	if authenticationType == gceAuthentication {
-		defaultProject, err := e.getDefaultProject(ctx)
-		if err != nil {
-			return nil, fmt.Errorf("Failed to retrieve default project from GCE metadata server. error: %v", err)
-		}
+	gceDefaultProject, err := e.getDefaultProject(ctx)
+	if err != nil {
+		slog.Debug("Stackdriver", "Auth", "Failed to use GCE auth: ", err)
+		return nil, fmt.Errorf("Failed to retrieve default project from GCE metadata server. error: %v", err)
+	}
+	slog.Debug("Stackdriver", "Auth", "Successfully use GCE auth: ", gceDefaultProject)

-		e.dsInfo.JsonData.Set("defaultProject", defaultProject)
+	queryResult.Meta.Set("defaultProject", gceDefaultProject)
+	result.Results[refId] = queryResult
+
+	return result, nil
+}
+
+func (e *StackdriverExecutor) executeTimeSeriesQuery(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
+	result := &tsdb.Response{
+		Results: make(map[string]*tsdb.QueryResult),
 	}

 	queries, err := e.buildQueries(tsdbQuery)
@@ -170,11 +181,12 @@ func (e *StackdriverExecutor) buildQueries(tsdbQuery *tsdb.TsdbQuery) ([]*Stackd
 		aliasBy := query.Model.Get("aliasBy").MustString()

 		stackdriverQueries = append(stackdriverQueries, &StackdriverQuery{
-			Target:   target,
-			Params:   params,
-			RefID:    query.RefId,
-			GroupBys: groupBysAsStrings,
-			AliasBy:  aliasBy,
+			Target:      target,
+			Params:      params,
+			RefID:       query.RefId,
+			GroupBys:    groupBysAsStrings,
+			AliasBy:     aliasBy,
+			ProjectName: query.Model.Get("projectName").MustString(""),
 		})
 	}

@@ -278,8 +290,7 @@ func setAggParams(params *url.Values, query *tsdb.Query, durationSeconds int) {

 func (e *StackdriverExecutor) executeQuery(ctx context.Context, query *StackdriverQuery, tsdbQuery *tsdb.TsdbQuery) (*tsdb.QueryResult, StackdriverResponse, error) {
 	queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: query.RefID}
-
-	req, err := e.createRequest(ctx, e.dsInfo)
+	req, err := e.createRequest(ctx, e.dsInfo, query, fmt.Sprintf("stackdriver%s", "v3/projects/"+query.ProjectName+"/timeSeries"))
 	if err != nil {
 		queryResult.Error = err
 		return queryResult, StackdriverResponse{}, nil
@@ -350,6 +361,28 @@ func (e *StackdriverExecutor) unmarshalResponse(res *http.Response) (Stackdriver
 	return data, nil
 }

+func (e *StackdriverExecutor) unmarshalResourceResponse(res *http.Response) (ResourceManagerProjectList, error) {
+	body, err := ioutil.ReadAll(res.Body)
+	defer res.Body.Close()
+	if err != nil {
+		return ResourceManagerProjectList{}, err
+	}
+
+	if res.StatusCode/100 != 2 {
+		slog.Error("Request failed", "status", res.Status, "body", string(body))
+		return ResourceManagerProjectList{}, fmt.Errorf(string(body))
+	}
+
+	var data ResourceManagerProjectList
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		slog.Error("Failed to unmarshal Resource manager response", "error", err, "status", res.Status, "body", string(body))
+		return ResourceManagerProjectList{}, err
+	}
+
+	return data, nil
+}
+
 func (e *StackdriverExecutor) parseResponse(queryRes *tsdb.QueryResult, data StackdriverResponse, query *StackdriverQuery) error {
 	labels := make(map[string]map[string]bool)

@@ -584,7 +617,7 @@ func calcBucketBound(bucketOptions StackdriverBucketOptions, n int) string {
 	return bucketBound
 }

-func (e *StackdriverExecutor) createRequest(ctx context.Context, dsInfo *models.DataSource) (*http.Request, error) {
+func (e *StackdriverExecutor) createRequest(ctx context.Context, dsInfo *models.DataSource, query *StackdriverQuery, proxyPass string) (*http.Request, error) {
 	u, _ := url.Parse(dsInfo.Url)
 	u.Path = path.Join(u.Path, "render")

@@ -611,14 +644,44 @@ func (e *StackdriverExecutor) createRequest(ctx context.Context, dsInfo *models.
 		}
 	}

-	projectName := dsInfo.JsonData.Get("defaultProject").MustString()
-	proxyPass := fmt.Sprintf("stackdriver%s", "v3/projects/"+projectName+"/timeSeries")
-
 	pluginproxy.ApplyRoute(ctx, req, proxyPass, stackdriverRoute, dsInfo)

 	return req, nil
 }

+func (e *StackdriverExecutor) createRequestResourceManager(ctx context.Context, dsInfo *models.DataSource) (*http.Request, error) {
+	u, _ := url.Parse(dsInfo.Url)
+	u.Path = path.Join(u.Path, "render")
+
+	req, err := http.NewRequest(http.MethodGet, "https://cloudresourcemanager.googleapis.com/", nil)
+	if err != nil {
+		slog.Error("Failed to create request", "error", err)
+		return nil, fmt.Errorf("Failed to create request. error: %v", err)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
+
+	// find plugin
+	plugin, ok := plugins.DataSources[dsInfo.Type]
+	if !ok {
+		return nil, errors.New("Unable to find datasource plugin Stackdriver")
+	}
+
+	var resourceManagerRoute *plugins.AppPluginRoute
+	for _, route := range plugin.Routes {
+		if route.Path == "cloudresourcemanager" {
+			resourceManagerRoute = route
+			break
+		}
+	}
+	proxyPass := "v1/projects"
+
+	pluginproxy.ApplyRoute(ctx, req, proxyPass, resourceManagerRoute, dsInfo)
+
+	return req, nil
+}
+
 func (e *StackdriverExecutor) getDefaultProject(ctx context.Context) (string, error) {
 	authenticationType := e.dsInfo.JsonData.Get("authenticationType").MustString(jwtAuthentication)
 	if authenticationType == gceAuthentication {
@@ -626,7 +689,67 @@ func (e *StackdriverExecutor) getDefaultProject(ctx context.Context) (string, er
 		if err != nil {
 			return "", fmt.Errorf("Failed to retrieve default project from GCE metadata server. error: %v", err)
 		}
+		token, err := defaultCredentials.TokenSource.Token()
+		if err != nil {
+			return "", fmt.Errorf("Failed to retrieve GCP credential token. error: %v", err)
+		}
+		if !token.Valid() {
+			return "", errors.New("Failed to validate GCP credentials")
+		}
+
 		return defaultCredentials.ProjectID, nil
 	}
 	return e.dsInfo.JsonData.Get("defaultProject").MustString(), nil
 }
+
+func (e *StackdriverExecutor) getProjectList(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
+	queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: tsdbQuery.Queries[0].RefId}
+	result := &tsdb.Response{
+		Results: make(map[string]*tsdb.QueryResult),
+	}
+	projectsList, err := e.getProjects(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	queryResult.Meta.Set("projectsList", projectsList)
+	result.Results[tsdbQuery.Queries[0].RefId] = queryResult
+	return result, nil
+}
+
+func (e *StackdriverExecutor) getProjects(ctx context.Context) ([]ResourceManagerProjectSelect, error) {
+	var projects []ResourceManagerProjectSelect
+
+	req, err := e.createRequestResourceManager(ctx, e.dsInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	span, ctx := opentracing.StartSpanFromContext(ctx, "resource manager query")
+	span.SetTag("datasource_id", e.dsInfo.Id)
+	span.SetTag("org_id", e.dsInfo.OrgId)
+
+	defer span.Finish()
+
+	if err := opentracing.GlobalTracer().Inject(
+		span.Context(),
+		opentracing.HTTPHeaders,
+		opentracing.HTTPHeadersCarrier(req.Header)); err != nil {
+		return nil, err
+	}
+
+	res, err := ctxhttp.Do(ctx, e.httpClient, req)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := e.unmarshalResourceResponse(res)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, project := range data.Projects {
+		projects = append(projects, ResourceManagerProjectSelect{Label: project.ProjectID, Value: project.ProjectID})
+	}
+	return projects, nil
+}
--- a/pkg/tsdb/stackdriver/types.go
+++ b/pkg/tsdb/stackdriver/types.go
@@ -5,72 +5,89 @@ import (
 	"time"
 )

-// StackdriverQuery is the query that Grafana sends from the frontend
-type StackdriverQuery struct {
-	Target   string
-	Params   url.Values
-	RefID    string
-	GroupBys []string
-	AliasBy  string
-}
+type (
+	// StackdriverQuery is the query that Grafana sends from the frontend
+	StackdriverQuery struct {
+		Target      string
+		Params      url.Values
+		RefID       string
+		GroupBys    []string
+		AliasBy     string
+		ProjectName string
+	}

-type StackdriverBucketOptions struct {
-	LinearBuckets *struct {
-		NumFiniteBuckets int64 `json:"numFiniteBuckets"`
-		Width            int64 `json:"width"`
-		Offset           int64 `json:"offset"`
-	} `json:"linearBuckets"`
-	ExponentialBuckets *struct {
-		NumFiniteBuckets int64   `json:"numFiniteBuckets"`
-		GrowthFactor     float64 `json:"growthFactor"`
-		Scale            float64 `json:"scale"`
-	} `json:"exponentialBuckets"`
-	ExplicitBuckets *struct {
-		Bounds []float64 `json:"bounds"`
-	} `json:"explicitBuckets"`
-}
+	StackdriverBucketOptions struct {
+		LinearBuckets *struct {
+			NumFiniteBuckets int64 `json:"numFiniteBuckets"`
+			Width            int64 `json:"width"`
+			Offset           int64 `json:"offset"`
+		} `json:"linearBuckets"`
+		ExponentialBuckets *struct {
+			NumFiniteBuckets int64   `json:"numFiniteBuckets"`
+			GrowthFactor     float64 `json:"growthFactor"`
+			Scale            float64 `json:"scale"`
+		} `json:"exponentialBuckets"`
+		ExplicitBuckets *struct {
+			Bounds []float64 `json:"bounds"`
+		} `json:"explicitBuckets"`
+	}

-// StackdriverResponse is the data returned from the external Google Stackdriver API
-type StackdriverResponse struct {
-	TimeSeries []struct {
-		Metric struct {
-			Labels map[string]string `json:"labels"`
-			Type   string            `json:"type"`
-		} `json:"metric"`
-		Resource struct {
-			Type   string            `json:"type"`
-			Labels map[string]string `json:"labels"`
-		} `json:"resource"`
-		MetaData   map[string]map[string]interface{} `json:"metadata"`
-		MetricKind string                            `json:"metricKind"`
-		ValueType  string                            `json:"valueType"`
-		Points     []struct {
-			Interval struct {
-				StartTime time.Time `json:"startTime"`
-				EndTime   time.Time `json:"endTime"`
-			} `json:"interval"`
-			Value struct {
-				DoubleValue       float64 `json:"doubleValue"`
-				StringValue       string  `json:"stringValue"`
-				BoolValue         bool    `json:"boolValue"`
-				IntValue          string  `json:"int64Value"`
-				DistributionValue struct {
-					Count                 string  `json:"count"`
-					Mean                  float64 `json:"mean"`
-					SumOfSquaredDeviation float64 `json:"sumOfSquaredDeviation"`
-					Range                 struct {
-						Min int `json:"min"`
-						Max int `json:"max"`
-					} `json:"range"`
-					BucketOptions StackdriverBucketOptions `json:"bucketOptions"`
-					BucketCounts  []string                 `json:"bucketCounts"`
-					Examplars     []struct {
-						Value     float64 `json:"value"`
-						Timestamp string  `json:"timestamp"`
-						// attachments
-					} `json:"examplars"`
-				} `json:"distributionValue"`
-			} `json:"value"`
-		} `json:"points"`
-	} `json:"timeSeries"`
-}
+	// StackdriverResponse is the data returned from the external Google Stackdriver API
+	StackdriverResponse struct {
+		TimeSeries []struct {
+			Metric struct {
+				Labels map[string]string `json:"labels"`
+				Type   string            `json:"type"`
+			} `json:"metric"`
+			Resource struct {
+				Type   string            `json:"type"`
+				Labels map[string]string `json:"labels"`
+			} `json:"resource"`
+			MetaData   map[string]map[string]interface{} `json:"metadata"`
+			MetricKind string                            `json:"metricKind"`
+			ValueType  string                            `json:"valueType"`
+			Points     []struct {
+				Interval struct {
+					StartTime time.Time `json:"startTime"`
+					EndTime   time.Time `json:"endTime"`
+				} `json:"interval"`
+				Value struct {
+					DoubleValue       float64 `json:"doubleValue"`
+					StringValue       string  `json:"stringValue"`
+					BoolValue         bool    `json:"boolValue"`
+					IntValue          string  `json:"int64Value"`
+					DistributionValue struct {
+						Count                 string  `json:"count"`
+						Mean                  float64 `json:"mean"`
+						SumOfSquaredDeviation float64 `json:"sumOfSquaredDeviation"`
+						Range                 struct {
+							Min int `json:"min"`
+							Max int `json:"max"`
+						} `json:"range"`
+						BucketOptions StackdriverBucketOptions `json:"bucketOptions"`
+						BucketCounts  []string                 `json:"bucketCounts"`
+						Examplars     []struct {
+							Value     float64 `json:"value"`
+							Timestamp string  `json:"timestamp"`
+							// attachments
+						} `json:"examplars"`
+					} `json:"distributionValue"`
+				} `json:"value"`
+			} `json:"points"`
+		} `json:"timeSeries"`
+	}
+
+	// ResourceManagerProjectList is the data returned from the external Google Resource Manager API
+	ResourceManagerProjectList struct {
+		Projects []ResourceManagerProject `json:"projects"`
+	}
+
+	ResourceManagerProject struct {
+		ProjectID string `json:"projectId"`
+	}
+
+	ResourceManagerProjectSelect struct {
+		Label string `json:"label"`
+		Value string `json:"value"`
+	}
+)