add CloudWatch datasource

This commit is contained in:
Mitsuhiro Tanda
2015-07-30 11:37:31 +09:00
parent 956a2bbdbe
commit 88ce05976e
17 changed files with 12212 additions and 1 deletions

View File

@@ -0,0 +1,154 @@
/* global AWS */
define([
'angular',
'lodash',
'kbn',
'moment',
'./queryCtrl',
'aws-sdk',
],
function (angular, _, kbn) {
'use strict';
var module = angular.module('grafana.services');
module.factory('CloudWatchDatasource', function($q, $http, templateSrv) {
function CloudWatchDatasource(datasource) {
this.type = 'cloudwatch';
this.name = datasource.name;
this.supportMetrics = true;
AWS.config.update({ region: datasource.jsonData.region });
this.cloudwatch = new AWS.CloudWatch({
accessKeyId: datasource.jsonData.accessKeyId,
secretAccessKey: datasource.jsonData.secretAccessKey,
});
}
// Called once per panel (graph)
CloudWatchDatasource.prototype.query = function(options) {
var start = convertToCloudWatchTime(options.range.from);
var end = convertToCloudWatchTime(options.range.to);
var queries = [];
_.each(options.targets, _.bind(function(target) {
if (!target.namespace || !target.metricName || _.isEmpty(target.dimensions) || _.isEmpty(target.statistics)) {
return;
}
var query = {};
query.namespace = templateSrv.replace(target.namespace, options.scopedVars);
query.metricName = templateSrv.replace(target.metricName, options.scopedVars);
query.dimensions = _.map(_.keys(target.dimensions), function(key) {
return {
Name: key,
Value: target.dimensions[key]
};
});
query.statistics = _.keys(target.statistics);
query.period = target.period;
var range = (end.getTime() - start.getTime()) / 1000;
// CloudWatch limit datapoints up to 1440
if (range / query.period >= 1440) {
query.period = Math.floor(range / 1440 / 60) * 60;
}
queries.push(query);
}, this));
// No valid targets, return the empty result to save a round trip.
if (_.isEmpty(queries)) {
var d = $q.defer();
d.resolve({ data: [] });
return d.promise;
}
var allQueryPromise = _.map(queries, _.bind(function(query) {
return this.performTimeSeriesQuery(query, start, end);
}, this));
return $q.all(allQueryPromise)
.then(function(allResponse) {
var result = [];
_.each(allResponse, function(response, index) {
var metrics = transformMetricData(response, options.targets[index]);
_.each(metrics, function(m) {
result.push(m);
});
});
return { data: result };
});
};
CloudWatchDatasource.prototype.performTimeSeriesQuery = function(query, start, end) {
var params = {
Namespace: query.namespace,
MetricName: query.metricName,
Dimensions: query.dimensions,
Statistics: query.statistics,
StartTime: start,
EndTime: end,
Period: query.period
};
var d = $q.defer();
this.cloudwatch.getMetricStatistics(params, function(err, data) {
if (err) {
return d.reject(err);
}
return d.resolve(data);
});
return d.promise;
};
CloudWatchDatasource.prototype.performSuggestQuery = function(params) {
var d = $q.defer();
this.cloudwatch.listMetrics(params, function(err, data) {
if (err) {
return d.reject(err);
}
return d.resolve(data);
});
return d.promise;
};
CloudWatchDatasource.prototype.testDatasource = function() {
return this.performSuggestQuery({}).then(function () {
return { status: 'success', message: 'Data source is working', title: 'Success' };
});
};
function transformMetricData(md, options) {
var result = [];
var dimensionPart = JSON.stringify(options.dimensions);
_.each(_.keys(options.statistics), function(s) {
var metricLabel = md.Label + '_' + s + dimensionPart;
var dps = _.map(md.Datapoints, function(value) {
return [value[s], new Date(value.Timestamp).getTime()];
});
dps = _.sortBy(dps, function(dp) { return dp[1]; });
result.push({ target: metricLabel, datapoints: dps });
});
return result;
}
function convertToCloudWatchTime(date) {
return kbn.parseDate(date);
}
return CloudWatchDatasource;
});
});

View File

@@ -0,0 +1,30 @@
<h5>CloudWatch details</h5>
<div class="tight-form">
<ul class="tight-form-list">
<li class="tight-form-item" style="width: 80px">
Region
</li>
<li>
<input type="text" class="tight-form-input input-large" ng-model='current.jsonData.region' placeholder="" required></input>
</li>
</ul>
<div class="clearfix"></div>
</div>
<div class="tight-form last">
<ul class="tight-form-list">
<li class="tight-form-item" style="width: 80px">
Access Key Id
</li>
<li>
<input type="text" class="tight-form-input input-large" ng-model='current.jsonData.accessKeyId' placeholder="" required></input>
</li>
<li class="tight-form-item">
Secret Access Key
</li>
<li>
<input type="password" class="tight-form-input input-large" ng-model='current.jsonData.secretAccessKey' placeholder="" required></input>
</li>
</ul>
<div class="clearfix"></div>
</div>

View File

@@ -0,0 +1,212 @@
<div class="editor-row" style="margin-top: 10px;">
<div ng-repeat="target in panel.targets"
style="margin-bottom: 10px;"
ng-class="{'tight-form-disabled': target.hide}"
ng-controller="CloudWatchQueryCtrl"
ng-init="init()">
<div class="tight-form">
<ul class="tight-form-list pull-right">
<li class="tight-form-item">
<div class="dropdown">
<a class="pointer dropdown-toggle"
data-toggle="dropdown"
tabindex="1">
<i class="fa fa-bars"></i>
</a>
<ul class="dropdown-menu pull-right" role="menu">
<li role="menuitem">
<a tabindex="1"
ng-click="duplicate()">
Duplicate
</a>
</li>
<li role="menuitem">
<a tabindex="1"
ng-click="moveMetricQuery($index, $index-1)">
Move up
</a>
</li>
<li role="menuitem">
<a tabindex="1"
ng-click="moveMetricQuery($index, $index+1)">
Move down
</a>
</li>
</ul>
</div>
</li>
<li class="tight-form-item last">
<a class="pointer" tabindex="1" ng-click="removeDataQuery(target)">
<i class="fa fa-remove"></i>
</a>
</li>
</ul>
<ul class="tight-form-list">
<li>
<a class="tight-form-item"
ng-click="target.hide = !target.hide; get_data();"
role="menuitem">
<i class="fa fa-eye"></i>
</a>
</li>
</ul>
<ul class="tight-form-list" role="menu">
<li class="tight-form-item" style="width: 100px">
Namespace
</li>
<li>
<input type="text"
class="input-medium tight-form-input"
ng-model="target.namespace"
spellcheck='false'
bs-typeahead="suggestNamespace"
placeholder="namespace"
data-min-length=0 data-items=100
ng-change="refreshMetricData()"
>
</li>
<li class="tight-form-item">
Metric
</li>
<li>
<input type="text"
class="input-medium tight-form-input"
ng-model="target.metricName"
spellcheck='false'
bs-typeahead="suggestMetrics"
placeholder="metric name"
data-min-length=0 data-items=100
ng-change="refreshMetricData()"
>
</li>
</ul>
<div class="clearfix"></div>
</div>
<div class="tight-form">
<ul class="tight-form-list" role="menu">
<li class="tight-form-item">
<i class="fa fa-eye invisible"></i>
</li>
<li class="tight-form-item" style="width: 86px">
Dimensions
</li>
<li ng-repeat="(key, value) in target.dimensions track by $index" class="tight-form-item">
{{key}}&nbsp;=&nbsp;{{value}}
<a ng-click="removeDimension(key)">
<i class="fa fa-remove"></i>
</a>
</li>
<li class="tight-form-item" ng-hide="addDimensionMode">
<a ng-click="addDimension()">
<i class="fa fa-plus"></i>
</a>
</li>
<li ng-show="addDimensionMode">
<input type="text"
class="input-small tight-form-input"
spellcheck='false'
bs-typeahead="suggestDimensionKeys"
data-min-length=0 data-items=100
ng-model="target.currentDimensionKey"
placeholder="key">
<input type="text"
class="input-small tight-form-input"
spellcheck='false'
bs-typeahead="suggestDimensionValues"
data-min-length=0 data-items=100
ng-model="target.currentDimensionValue"
placeholder="value">
<a ng-click="addDimension()">
add dimension
</a>
</li>
</ul>
<div class="clearfix"></div>
</div>
<div class="tight-form">
<ul class="tight-form-list" role="menu">
<li class="tight-form-item">
<i class="fa fa-eye invisible"></i>
</li>
<li class="tight-form-item" style="width: 86px">
Statistics
</li>
<li class="tight-form-item">
Min&nbsp;
<input class="cr1" id="target.statistics.Minimum" type="checkbox"
ng-model="target.statistics.Minimum" ng-checked="target.statistics.Minimum" ng-change="statisticsOptionChanged()">
<label for="target.statistics.Minimum" class="cr1"></label>
</li>
<li class="tight-form-item">
Max&nbsp;
<input class="cr1" id="target.statistics.Maximum" type="checkbox"
ng-model="target.statistics.Maximum" ng-checked="target.statistics.Maximum" ng-change="statisticsOptionChanged()">
<label for="target.statistics.Maximum" class="cr1"></label>
</li>
<li class="tight-form-item">
Avg&nbsp;
<input class="cr1" id="target.statistics.Average" type="checkbox"
ng-model="target.statistics.Average" ng-checked="target.statistics.Average" ng-change="statisticsOptionChanged()">
<label for="target.statistics.Average" class="cr1"></label>
</li>
<li class="tight-form-item">
Sum&nbsp;
<input class="cr1" id="target.statistics.Sum" type="checkbox"
ng-model="target.statistics.Sum" ng-checked="target.statistics.Sum" ng-change="statisticsOptionChanged()">
<label for="target.statistics.Sum" class="cr1"></label>
</li>
<li class="tight-form-item last">
SampleCount&nbsp;
<input class="cr1" id="target.statistics.SampleCount" type="checkbox"
ng-model="target.statistics.SampleCount" ng-checked="target.statistics.SampleCount" ng-change="statisticsOptionChanged()">
<label for="target.statistics.SampleCount" class="cr1"></label>
</li>
</ul>
<div class="clearfix"></div>
</div>
<div class="tight-form">
<ul class="tight-form-list" role="menu">
<li class="tight-form-item">
<i class="fa fa-eye invisible"></i>
</li>
<li class="tight-form-item" style="width: 100px">
Period
</li>
<li>
<input type="text"
class="input-mini tight-form-input"
ng-model="target.period"
data-placement="right"
spellcheck='false'
placeholder="period"
data-min-length=0 data-items=100
ng-change="refreshMetricData()"
/>
<a bs-tooltip="target.errors.period"
style="color: rgb(229, 189, 28)"
ng-show="target.errors.period">
<i class="fa fa-warning"></i>
</a>
</li>
</ul>
<div class="clearfix"></div>
</div>
</div>
</div>

View File

@@ -0,0 +1,16 @@
{
"pluginType": "datasource",
"name": "CloudWatch",
"type": "cloudwatch",
"serviceName": "CloudWatchDatasource",
"module": "plugins/datasource/cloudwatch/datasource",
"partials": {
"config": "app/plugins/datasource/cloudwatch/partials/config.html",
"query": "app/plugins/datasource/cloudwatch/partials/query.editor.html"
},
"metrics": true
}

View File

@@ -0,0 +1,271 @@
define([
'angular',
'lodash',
'kbn',
],
function (angular, _, kbn) {
'use strict';
var module = angular.module('grafana.controllers');
var supportedMetrics = {
"AWS/AutoScaling": [
"GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity", "GroupInServiceInstances", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"
],
"AWS/Billing": [
"EstimatedCharges"
],
"AWS/CloudFront": [
"Requests", "BytesDownloaded", "BytesUploaded", "TotalErrorRate", "4xxErrorRate", "5xxErrorRate"
],
"AWS/CloudSearch": [
"SuccessfulRequests", "SearchableDocuments", "IndexUtilization", "Partitions"
],
"AWS/DynamoDB": [
"ConditionalCheckFailedRequests", "ConsumedReadCapacityUnits", "ConsumedWriteCapacityUnits", "OnlineIndexConsumedWriteCapacity", "OnlineIndexPercentageProgress", "OnlineIndexThrottleEvents", "ProvisionedReadCapacityUnits", "ProvisionedWriteCapacityUnits", "ReadThrottleEvents", "ReturnedItemCount", "SuccessfulRequestLatency", "SystemErrors", "ThrottledRequests", "UserErrors", "WriteThrottleEvents"
],
"AWS/ElastiCache": [
"CPUUtilization", "SwapUsage", "FreeableMemory", "NetworkBytesIn", "NetworkBytesOut",
"BytesUsedForCacheItems", "BytesReadIntoMemcached", "BytesWrittenOutFromMemcached", "CasBadval", "CasHits", "CasMisses", "CmdFlush", "CmdGet", "CmdSet", "CurrConnections", "CurrItems", "DecrHits", "DecrMisses", "DeleteHits", "DeleteMisses", "Evictions", "GetHits", "GetMisses", "IncrHits", "IncrMisses", "Reclaimed",
"CurrConnections", "Evictions", "Reclaimed", "NewConnections", "BytesUsedForCache", "CacheHits", "CacheMisses", "ReplicationLag", "GetTypeCmds", "SetTypeCmds", "KeyBasedCmds", "StringBasedCmds", "HashBasedCmds", "ListBasedCmds", "SetBasedCmds", "SortedSetBasedCmds", "CurrItems"
],
"AWS/EBS": [
"VolumeReadBytes", "VolumeWriteBytes", "VolumeReadOps", "VolumeWriteOps", "VolumeTotalReadTime", "VolumeTotalWriteTime", "VolumeIdleTime", "VolumeQueueLength", "VolumeThroughputPercentage", "VolumeConsumedReadWriteOps",
],
"AWS/EC2": [
"CPUCreditUsage", "CPUCreditBalance", "CPUUtilization", "DiskReadOps", "DiskWriteOps", "DiskReadBytes", "DiskWriteBytes", "NetworkIn", "NetworkOut", "StatusCheckFailed", "StatusCheckFailed_Instance", "StatusCheckFailed_System"
],
"AWS/ELB": [
"HealthyHostCount", "UnHealthyHostCount", "RequestCount", "Latency", "HTTPCode_ELB_4XX", "HTTPCode_ELB_5XX", "HTTPCode_Backend_2XX", "HTTPCode_Backend_3XX", "HTTPCode_Backend_4XX", "HTTPCode_Backend_5XX", "BackendConnectionErrors", "SurgeQueueLength", "SpilloverCount"
],
"AWS/ElasticMapReduce": [
"CoreNodesPending", "CoreNodesRunning", "HBaseBackupFailed", "HBaseMostRecentBackupDuration", "HBaseTimeSinceLastSuccessfulBackup", "HDFSBytesRead", "HDFSBytesWritten", "HDFSUtilization", "IsIdle", "JobsFailed", "JobsRunning", "LiveDataNodes", "LiveTaskTrackers", "MapSlotsOpen", "MissingBlocks", "ReduceSlotsOpen", "RemainingMapTasks", "RemainingMapTasksPerSlot", "RemainingReduceTasks", "RunningMapTasks", "RunningReduceTasks", "S3BytesRead", "S3BytesWritten", "TaskNodesPending", "TaskNodesRunning", "TotalLoad"
],
"AWS/Kinesis": [
"PutRecord.Bytes", "PutRecord.Latency", "PutRecord.Success", "PutRecords.Bytes", "PutRecords.Latency", "PutRecords.Records", "PutRecords.Success", "IncomingBytes", "IncomingRecords", "GetRecords.Bytes", "GetRecords.IteratorAgeMilliseconds", "GetRecords.Latency", "GetRecords.Success"
],
"AWS/ML": [
"PredictCount", "PredictFailureCount"
],
"AWS/OpsWorks": [
"cpu_idle", "cpu_nice", "cpu_system", "cpu_user", "cpu_waitio", "load_1", "load_5", "load_15", "memory_buffers", "memory_cached", "memory_free", "memory_swap", "memory_total", "memory_used", "procs"
],
"AWS/Redshift": [
"CPUUtilization", "DatabaseConnections", "HealthStatus", "MaintenanceMode", "NetworkReceiveThroughput", "NetworkTransmitThroughput", "PercentageDiskSpaceUsed", "ReadIOPS", "ReadLatency", "ReadThroughput", "WriteIOPS", "WriteLatency", "WriteThroughput"
],
"AWS/RDS": [
"BinLogDiskUsage", "CPUUtilization", "DatabaseConnections", "DiskQueueDepth", "FreeableMemory", "FreeStorageSpace", "ReplicaLag", "SwapUsage", "ReadIOPS", "WriteIOPS", "ReadLatency", "WriteLatency", "ReadThroughput", "WriteThroughput", "NetworkReceiveThroughput", "NetworkTransmitThroughput"
],
"AWS/Route53": [
"HealthCheckStatus", "HealthCheckPercentageHealthy"
],
"AWS/SNS": [
"NumberOfMessagesPublished", "PublishSize", "NumberOfNotificationsDelivered", "NumberOfNotificationsFailed"
],
"AWS/SQS": [
"NumberOfMessagesSent", "SentMessageSize", "NumberOfMessagesReceived", "NumberOfEmptyReceives", "NumberOfMessagesDeleted", "ApproximateNumberOfMessagesDelayed", "ApproximateNumberOfMessagesVisible", "ApproximateNumberOfMessagesNotVisible"
],
"AWS/S3": [
"BucketSizeBytes", "NumberOfObjects"
],
"AWS/SWF": [
"DecisionTaskScheduleToStartTime", "DecisionTaskStartToCloseTime", "DecisionTasksCompleted", "StartedDecisionTasksTimedOutOnClose", "WorkflowStartToCloseTime", "WorkflowsCanceled", "WorkflowsCompleted", "WorkflowsContinuedAsNew", "WorkflowsFailed", "WorkflowsTerminated", "WorkflowsTimedOut"
],
"AWS/StorageGateway": [
"CacheHitPercent", "CachePercentUsed", "CachePercentDirty", "CloudBytesDownloaded", "CloudDownloadLatency", "CloudBytesUploaded", "UploadBufferFree", "UploadBufferPercentUsed", "UploadBufferUsed", "QueuedWrites", "ReadBytes", "ReadTime", "TotalCacheSize", "WriteBytes", "WriteTime", "WorkingStorageFree", "WorkingStoragePercentUsed", "WorkingStorageUsed", "CacheHitPercent", "CachePercentUsed", "CachePercentDirty", "ReadBytes", "ReadTime", "WriteBytes", "WriteTime", "QueuedWrites"
],
"AWS/WorkSpaces": [
"Available", "Unhealthy", "ConnectionAttempt", "ConnectionSuccess", "ConnectionFailure", "SessionLaunchTime", "InSessionLatency", "SessionDisconnect"
],
};
var supportedDimensions = {
"AWS/AutoScaling": [
"AutoScalingGroupName"
],
"AWS/Billing": [
"ServiceName", "LinkedAccount", "Currency"
],
"AWS/CloudFront": [
"DistributionId", "Region"
],
"AWS/CloudSearch": [
],
"AWS/DynamoDB": [
"TableName", "GlobalSecondaryIndexName", "Operation"
],
"AWS/ElastiCache": [
"CacheClusterId", "CacheNodeId"
],
"AWS/EBS": [
"VolumeId"
],
"AWS/EC2": [
"AutoScalingGroupName", "ImageId", "InstanceId", "InstanceType"
],
"AWS/ELB": [
"LoadBalancerName", "AvailabilityZone"
],
"AWS/ElasticMapReduce": [
"ClusterId", "JobId"
],
"AWS/Kinesis": [
"StreamName"
],
"AWS/ML": [
"MLModelId", "RequestMode"
],
"AWS/OpsWorks": [
"StackId", "LayerId", "InstanceId"
],
"AWS/Redshift": [
"NodeID", "ClusterIdentifier"
],
"AWS/RDS": [
"DBInstanceIdentifier", "DatabaseClass", "EngineName"
],
"AWS/Route53": [
"HealthCheckId"
],
"AWS/SNS": [
"Application", "Platform", "TopicName"
],
"AWS/SQS": [
"QueueName"
],
"AWS/S3": [
"BucketName", "StorageType"
],
"AWS/SWF": [
"Domain", "ActivityTypeName", "ActivityTypeVersion"
],
"AWS/StorageGateway": [
"GatewayId", "GatewayName", "VolumeId"
],
"AWS/WorkSpaces": [
"DirectoryId", "WorkspaceId"
],
};
module.controller('CloudWatchQueryCtrl', function($scope) {
$scope.init = function() {
$scope.target.namespace = $scope.target.namespace || '';
$scope.target.metricName = $scope.target.metricName || '';
$scope.target.dimensions = $scope.target.dimensions || {};
$scope.target.statistics = $scope.target.statistics || {};
$scope.target.period = $scope.target.period || 60;
$scope.target.errors = validateTarget();
};
$scope.refreshMetricData = function() {
$scope.target.errors = validateTarget($scope.target);
// this does not work so good
if (!_.isEqual($scope.oldTarget, $scope.target) && _.isEmpty($scope.target.errors)) {
$scope.oldTarget = angular.copy($scope.target);
$scope.get_data();
}
};
$scope.moveMetricQuery = function(fromIndex, toIndex) {
_.move($scope.panel.targets, fromIndex, toIndex);
};
$scope.duplicate = function() {
var clone = angular.copy($scope.target);
$scope.panel.targets.push(clone);
};
$scope.suggestNamespace = function(query, callback) {
return _.keys(supportedMetrics);
};
$scope.suggestMetrics = function(query, callback) {
return supportedMetrics[$scope.target.namespace] || [];
};
$scope.suggestDimensionKeys = function(query, callback) {
return supportedDimensions[$scope.target.namespace] || [];
};
$scope.suggestDimensionValues = function(query, callback) {
if (!$scope.target.namespace || !$scope.target.metricName) {
return callback([]);
}
var params = {
Namespace: $scope.target.namespace,
MetricName: $scope.target.metricName
};
if (!_.isEmpty($scope.target.dimensions)) {
params.Dimensions = $scope.target.dimensions;
}
$scope.datasource
.performSuggestQuery(params)
.then(function(result) {
var suggestData = _.chain(result.Metrics)
.map(function(metric) {
return metric.Dimensions;
})
.flatten(true)
.filter(function(dimension) {
return dimension.Name === $scope.target.currentDimensionKey;
})
.map(function(metric) {
return metric;
})
.pluck('Value')
.uniq()
.value();
callback(suggestData);
}, function() {
callback([]);
});
};
$scope.addDimension = function() {
if (!$scope.addDimensionMode) {
$scope.addDimensionMode = true;
return;
}
if (!$scope.target.dimensions) {
$scope.target.dimensions = {};
}
$scope.target.dimensions[$scope.target.currentDimensionKey] = $scope.target.currentDimensionValue;
$scope.target.currentDimensionKey = '';
$scope.target.currentDimensionValue = '';
$scope.refreshMetricData();
$scope.addDimensionMode = false;
};
$scope.removeDimension = function(key) {
delete $scope.target.dimensions[key];
$scope.refreshMetricData();
};
$scope.statisticsOptionChanged = function() {
$scope.refreshMetricData();
};
// TODO: validate target
function validateTarget() {
var errs = {};
if ($scope.target.period < 60 || ($scope.target.period % 60) !== 0) {
errs.period = 'Period must be at least 60 seconds and must be a multiple of 60';
}
return errs;
}
});
});