support GetMetricData

pkg/metrics/metrics.go

@@ -44,6 +44,7 @@ var (
 	M_Alerting_Notification_Sent         *prometheus.CounterVec
 	M_Aws_CloudWatch_GetMetricStatistics prometheus.Counter
 	M_Aws_CloudWatch_ListMetrics         prometheus.Counter
+	M_Aws_CloudWatch_GetMetricData       prometheus.Counter
 	M_DB_DataSource_QueryById            prometheus.Counter
 
 	// Timers
@@ -218,6 +219,12 @@ func init() {
 		Namespace: exporterName,
 	})
 
+	M_Aws_CloudWatch_GetMetricData = prometheus.NewCounter(prometheus.CounterOpts{
+		Name:      "aws_cloudwatch_get_metric_data_total",
+		Help:      "counter for getting metric data time series from aws",
+		Namespace: exporterName,
+	})
+
 	M_DB_DataSource_QueryById = prometheus.NewCounter(prometheus.CounterOpts{
 		Name:      "db_datasource_query_by_id_total",
 		Help:      "counter for getting datasource by id",
@@ -307,6 +314,7 @@ func initMetricVars() {
 		M_Alerting_Notification_Sent,
 		M_Aws_CloudWatch_GetMetricStatistics,
 		M_Aws_CloudWatch_ListMetrics,
+		M_Aws_CloudWatch_GetMetricData,
 		M_DB_DataSource_QueryById,
 		M_Alerting_Active_Alerts,
 		M_StatTotal_Dashboards,

pkg/tsdb/cloudwatch/cloudwatch.go

@@ -14,6 +14,7 @@ import (
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/request"
@@ -88,48 +89,63 @@ func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryCo
 		Results: make(map[string]*tsdb.QueryResult),
 	}
 
-	errCh := make(chan error, 1)
-	resCh := make(chan *tsdb.QueryResult, 1)
+	eg, ectx := errgroup.WithContext(ctx)
 
-	currentlyExecuting := 0
+	getMetricDataQueries := make(map[string]map[string]*CloudWatchQuery)
 	for i, model := range queryContext.Queries {
 		queryType := model.Model.Get("type").MustString()
 		if queryType != "timeSeriesQuery" && queryType != "" {
 			continue
 		}
-		currentlyExecuting++
-		go func(refId string, index int) {
-			queryRes, err := e.executeQuery(ctx, queryContext.Queries[index].Model, queryContext)
-			currentlyExecuting--
-			if err != nil {
-				errCh <- err
-			} else {
-				queryRes.RefId = refId
-				resCh <- queryRes
+
+		query, err := parseQuery(queryContext.Queries[i].Model)
+		if err != nil {
+			return nil, err
+		}
+		query.RefId = queryContext.Queries[i].RefId
+
+		if query.Id != "" {
+			if _, ok := getMetricDataQueries[query.Region]; !ok {
+				getMetricDataQueries[query.Region] = make(map[string]*CloudWatchQuery)
 			}
-		}(model.RefId, i)
+			getMetricDataQueries[query.Region][query.Id] = query
+			continue
+		}
+
+		eg.Go(func() error {
+			queryRes, err := e.executeQuery(ectx, query, queryContext)
+			if err != nil {
+				return err
+			}
+			result.Results[queryRes.RefId] = queryRes
+			return nil
+		})
 	}
 
-	for currentlyExecuting != 0 {
-		select {
-		case res := <-resCh:
-			result.Results[res.RefId] = res
-		case err := <-errCh:
-			return result, err
-		case <-ctx.Done():
-			return result, ctx.Err()
+	if len(getMetricDataQueries) > 0 {
+		for region, getMetricDataQuery := range getMetricDataQueries {
+			q := getMetricDataQuery
+			eg.Go(func() error {
+				queryResponses, err := e.executeGetMetricDataQuery(ectx, region, q, queryContext)
+				if err != nil {
+					return err
+				}
+				for _, queryRes := range queryResponses {
+					result.Results[queryRes.RefId] = queryRes
+				}
+				return nil
+			})
 		}
 	}
 
+	if err := eg.Wait(); err != nil {
+		return nil, err
+	}
+
 	return result, nil
 }
 
-func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) {
-	query, err := parseQuery(parameters)
-	if err != nil {
-		return nil, err
-	}
-
+func (e *CloudWatchExecutor) executeQuery(ctx context.Context, query *CloudWatchQuery, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) {
 	client, err := e.getClient(query.Region)
 	if err != nil {
 		return nil, err
@@ -201,6 +217,139 @@ func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simpl
 	return queryRes, nil
 }
 
+func (e *CloudWatchExecutor) executeGetMetricDataQuery(ctx context.Context, region string, queries map[string]*CloudWatchQuery, queryContext *tsdb.TsdbQuery) ([]*tsdb.QueryResult, error) {
+	queryResponses := make([]*tsdb.QueryResult, 0)
+
+	// validate query
+	for _, query := range queries {
+		if !(len(query.Statistics) == 1 && len(query.ExtendedStatistics) == 0) &&
+			!(len(query.Statistics) == 0 && len(query.ExtendedStatistics) == 1) {
+			return queryResponses, errors.New("Statistics count should be 1")
+		}
+	}
+
+	client, err := e.getClient(region)
+	if err != nil {
+		return queryResponses, err
+	}
+
+	startTime, err := queryContext.TimeRange.ParseFrom()
+	if err != nil {
+		return queryResponses, err
+	}
+
+	endTime, err := queryContext.TimeRange.ParseTo()
+	if err != nil {
+		return queryResponses, err
+	}
+
+	params := &cloudwatch.GetMetricDataInput{
+		StartTime: aws.Time(startTime),
+		EndTime:   aws.Time(endTime),
+		ScanBy:    aws.String("TimestampAscending"),
+	}
+	for _, query := range queries {
+		// 1 minutes resolutin metrics is stored for 15 days, 15 * 24 * 60 = 21600
+		if query.HighResolution && (((endTime.Unix() - startTime.Unix()) / int64(query.Period)) > 21600) {
+			return nil, errors.New("too long query period")
+		}
+
+		mdq := &cloudwatch.MetricDataQuery{
+			Id:         aws.String(query.Id),
+			ReturnData: aws.Bool(query.ReturnData),
+		}
+		if query.Expression != "" {
+			mdq.Expression = aws.String(query.Expression)
+		} else {
+			mdq.MetricStat = &cloudwatch.MetricStat{
+				Metric: &cloudwatch.Metric{
+					Namespace:  aws.String(query.Namespace),
+					MetricName: aws.String(query.MetricName),
+				},
+				Period: aws.Int64(int64(query.Period)),
+			}
+			for _, d := range query.Dimensions {
+				mdq.MetricStat.Metric.Dimensions = append(mdq.MetricStat.Metric.Dimensions,
+					&cloudwatch.Dimension{
+						Name:  d.Name,
+						Value: d.Value,
+					})
+			}
+			if len(query.Statistics) == 1 {
+				mdq.MetricStat.Stat = query.Statistics[0]
+			} else {
+				mdq.MetricStat.Stat = query.ExtendedStatistics[0]
+			}
+		}
+		params.MetricDataQueries = append(params.MetricDataQueries, mdq)
+	}
+
+	nextToken := ""
+	mdr := make(map[string]*cloudwatch.MetricDataResult)
+	for {
+		if nextToken != "" {
+			params.NextToken = aws.String(nextToken)
+		}
+		resp, err := client.GetMetricDataWithContext(ctx, params)
+		if err != nil {
+			return queryResponses, err
+		}
+		metrics.M_Aws_CloudWatch_GetMetricData.Add(float64(len(params.MetricDataQueries)))
+
+		for _, r := range resp.MetricDataResults {
+			if _, ok := mdr[*r.Id]; !ok {
+				mdr[*r.Id] = r
+			} else {
+				mdr[*r.Id].Timestamps = append(mdr[*r.Id].Timestamps, r.Timestamps...)
+				mdr[*r.Id].Values = append(mdr[*r.Id].Values, r.Values...)
+			}
+		}
+
+		if resp.NextToken == nil || *resp.NextToken == "" {
+			break
+		}
+		nextToken = *resp.NextToken
+	}
+
+	for i, r := range mdr {
+		if *r.StatusCode != "Complete" {
+			return queryResponses, fmt.Errorf("Part of query is failed: %s", *r.StatusCode)
+		}
+
+		queryRes := tsdb.NewQueryResult()
+		queryRes.RefId = queries[i].RefId
+		query := queries[*r.Id]
+
+		series := tsdb.TimeSeries{
+			Tags:   map[string]string{},
+			Points: make([]tsdb.TimePoint, 0),
+		}
+		for _, d := range query.Dimensions {
+			series.Tags[*d.Name] = *d.Value
+		}
+		s := ""
+		if len(query.Statistics) == 1 {
+			s = *query.Statistics[0]
+		} else {
+			s = *query.ExtendedStatistics[0]
+		}
+		series.Name = formatAlias(query, s, series.Tags)
+
+		for j, t := range r.Timestamps {
+			expectedTimestamp := r.Timestamps[j].Add(time.Duration(query.Period) * time.Second)
+			if j > 0 && expectedTimestamp.Before(*t) {
+				series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(expectedTimestamp.Unix()*1000)))
+			}
+			series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(*r.Values[j]), float64((*t).Unix())*1000))
+		}
+
+		queryRes.Series = append(queryRes.Series, &series)
+		queryResponses = append(queryResponses, queryRes)
+	}
+
+	return queryResponses, nil
+}
+
 func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) {
 	var result []*cloudwatch.Dimension
 
@@ -257,6 +406,9 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
 		return nil, err
 	}
 
+	id := model.Get("id").MustString("")
+	expression := model.Get("expression").MustString("")
+
 	dimensions, err := parseDimensions(model)
 	if err != nil {
 		return nil, err
@@ -295,6 +447,7 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
 		alias = "{{metric}}_{{stat}}"
 	}
 
+	returnData := model.Get("returnData").MustBool(false)
 	highResolution := model.Get("highResolution").MustBool(false)
 
 	return &CloudWatchQuery{
@@ -306,11 +459,18 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
 		ExtendedStatistics: aws.StringSlice(extendedStatistics),
 		Period:             period,
 		Alias:              alias,
+		Id:                 id,
+		Expression:         expression,
+		ReturnData:         returnData,
 		HighResolution:     highResolution,
 	}, nil
 }
 
 func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string) string {
+	if len(query.Id) > 0 && len(query.Expression) > 0 {
+		return query.Id
+	}
+
 	data := map[string]string{}
 	data["region"] = query.Region
 	data["namespace"] = query.Namespace
@@ -338,6 +498,7 @@ func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]stri
 func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatchQuery) (*tsdb.QueryResult, error) {
 	queryRes := tsdb.NewQueryResult()
 
+	queryRes.RefId = query.RefId
 	var value float64
 	for _, s := range append(query.Statistics, query.ExtendedStatistics...) {
 		series := tsdb.TimeSeries{

pkg/tsdb/cloudwatch/types.go

@@ -5,6 +5,7 @@ import (
 )
 
 type CloudWatchQuery struct {
+	RefId              string
 	Region             string
 	Namespace          string
 	MetricName         string
@@ -13,5 +14,8 @@ type CloudWatchQuery struct {
 	ExtendedStatistics []*string
 	Period             int
 	Alias              string
+	Id                 string
+	Expression         string
+	ReturnData         bool
 	HighResolution     bool
 }

public/app/plugins/datasource/cloudwatch/datasource.ts

@@ -30,7 +30,9 @@ export default class CloudWatchDatasource {
 
     var queries = _.filter(options.targets, item => {
       return (
-        item.hide !== true && !!item.region && !!item.namespace && !!item.metricName && !_.isEmpty(item.statistics)
+        (item.id !== '' || item.hide !== true) &&
+        ((!!item.region && !!item.namespace && !!item.metricName && !_.isEmpty(item.statistics)) ||
+          item.expression.length > 0)
       );
     }).map(item => {
       item.region = this.templateSrv.replace(this.getActualRegion(item.region), options.scopedVars);
@@ -38,6 +40,9 @@ export default class CloudWatchDatasource {
       item.metricName = this.templateSrv.replace(item.metricName, options.scopedVars);
       item.dimensions = this.convertDimensionFormat(item.dimensions, options.scopedVars);
       item.period = String(this.getPeriod(item, options)); // use string format for period in graph query, and alerting
+      item.id = this.templateSrv.replace(item.id, options.scopedVars);
+      item.expression = this.templateSrv.replace(item.expression, options.scopedVars);
+      item.returnData = typeof item.hide === 'undefined' ? true : !item.hide;
 
       return _.extend(
         {
@@ -384,11 +389,11 @@ export default class CloudWatchDatasource {
     var currentVariables = !_.isArray(variable.current.value)
       ? [variable.current]
       : variable.current.value.map(v => {
-          return {
-            text: v,
-            value: v,
-          };
-        });
+        return {
+          text: v,
+          value: v,
+        };
+      });
     let useSelectedVariables =
       selectedVariables.some(s => {
         return s.value === currentVariables[0].value;
@@ -399,6 +404,9 @@ export default class CloudWatchDatasource {
       scopedVar[variable.name] = v;
       t.refId = target.refId + '_' + v.value;
       t.dimensions[dimensionKey] = templateSrv.replace(t.dimensions[dimensionKey], scopedVar);
+      if (target.id) {
+        t.id = target.id + window.btoa(v.value).replace(/=/g, '0'); // generate unique id
+      }
       return t;
     });
   }

public/app/plugins/datasource/cloudwatch/partials/query.parameter.html

@@ -1,4 +1,4 @@
-<div class="gf-form-inline">
+<div class="gf-form-inline" ng-if="target.expression.length === 0">
 	<div class="gf-form">
 		<label class="gf-form-label query-keyword width-8">Metric</label>
 
@@ -20,7 +20,7 @@
 	</div>
 </div>
 
-<div class="gf-form-inline">
+<div class="gf-form-inline" ng-if="target.expression.length === 0">
 	<div class="gf-form">
 		<label class="gf-form-label query-keyword width-8">Dimensions</label>
 		<metric-segment ng-repeat="segment in dimSegments" segment="segment" get-options="getDimSegments(segment, $index)" on-change="dimSegmentChanged(segment, $index)"></metric-segment>
@@ -31,18 +31,31 @@
 	</div>
 </div>
 
-<div class="gf-form-inline">
+<div class="gf-form-inline" ng-if="target.statistics.length === 1">
 	<div class="gf-form">
-		<label class="gf-form-label query-keyword width-8">
-			Min period
-			<info-popover mode="right-normal">Minimum interval between points in seconds</info-popover>
-		</label>
-		<input type="text" class="gf-form-input" ng-model="target.period" spellcheck='false' placeholder="auto" ng-model-onblur ng-change="onChange()" />
+		<label class=" gf-form-label query-keyword width-8 ">Id</label>
+		<input type="text " class="gf-form-input " ng-model="target.id " spellcheck='false' ng-model-onblur ng-change="onChange() ">
 	</div>
-	<div class="gf-form max-width-30">
-		<label class="gf-form-label query-keyword width-7">Alias</label>
-		<input type="text" class="gf-form-input"  ng-model="target.alias" spellcheck='false' ng-model-onblur ng-change="onChange()">
-		<info-popover mode="right-absolute">
+	<div class="gf-form max-width-30 ">
+		<label class="gf-form-label query-keyword width-7 ">Expression</label>
+		<input type="text " class="gf-form-input " ng-model="target.expression
+	 " spellcheck='false' ng-model-onblur ng-change="onChange() ">
+	</div>
+</div>
+
+<div class="gf-form-inline ">
+	<div class="gf-form ">
+		<label class="gf-form-label query-keyword width-8 ">
+			Min period
+			<info-popover mode="right-normal ">Minimum interval between points in seconds</info-popover>
+		</label>
+		<input type="text " class="gf-form-input " ng-model="target.period " spellcheck='false' placeholder="auto
+	 " ng-model-onblur ng-change="onChange() " />
+	</div>
+	<div class="gf-form max-width-30 ">
+		<label class="gf-form-label query-keyword width-7 ">Alias</label>
+		<input type="text " class="gf-form-input " ng-model="target.alias " spellcheck='false' ng-model-onblur ng-change="onChange() ">
+		<info-popover mode="right-absolute ">
 			Alias replacement variables:
 			<ul ng-non-bindable>
 				<li>{{metric}}</li>
@@ -54,12 +67,12 @@
 			</ul>
 		</info-popover>
 	</div>
-	<div class="gf-form">
-		<gf-form-switch class="gf-form" label="HighRes" label-class="width-5" checked="target.highResolution" on-change="onChange()">
+	<div class="gf-form ">
+		<gf-form-switch class="gf-form " label="HighRes " label-class="width-5 " checked="target.highResolution " on-change="onChange() ">
 		</gf-form-switch>
 	</div>
 
-	<div class="gf-form gf-form--grow">
-		<div class="gf-form-label gf-form-label--grow"></div>
+	<div class="gf-form gf-form--grow ">
+		<div class="gf-form-label gf-form-label--grow "></div>
 	</div>
 </div>

public/app/plugins/datasource/cloudwatch/query_parameter_ctrl.ts

@@ -27,6 +27,9 @@ export class CloudWatchQueryParameterCtrl {
       target.dimensions = target.dimensions || {};
       target.period = target.period || '';
       target.region = target.region || 'default';
+      target.id = target.id || '';
+      target.expression = target.expression || '';
+      target.returnData = target.returnData || false;
       target.highResolution = target.highResolution || false;
 
       $scope.regionSegment = uiSegmentSrv.getSegmentForValue($scope.target.region, 'select region');