Merge branch 'master' into alerting_definitions

Conflicts: conf/defaults.ini pkg/api/dashboard.go
2025-02-25 18:55:37 -06:00 · 2016-06-06 08:43:01 +02:00
parent 6a49d4ed6b 7cbaf06097
commit f36a44c49a
49 changed files with 5530 additions and 223 deletions
--- a/pkg/api/api.go
+++ b/pkg/api/api.go
@@ -237,7 +237,10 @@ func Register(r *macaron.Macaron) {
 		r.Get("/search/", Search)

 		// metrics
-		r.Get("/metrics/test", GetTestMetrics)
+		r.Get("/metrics/test", wrap(GetTestMetrics))
+
+		// metrics
+		r.Get("/metrics", wrap(GetInternalMetrics))

 		r.Group("/alerts", func() {
 			r.Group("/rules", func() {
--- a/pkg/api/app_routes.go
+++ b/pkg/api/app_routes.go
@@ -30,7 +30,7 @@ func InitAppPluginRoutes(r *macaron.Macaron) {
 			}
 			handlers = append(handlers, AppPluginRoute(route, plugin.Id))
 			r.Route(url, route.Method, handlers...)
-			log.Info("Plugins: Adding proxy route %s", url)
+			log.Debug("Plugins: Adding proxy route %s", url)
 		}
 	}
 }
--- a/pkg/api/common.go
+++ b/pkg/api/common.go
@@ -12,8 +12,12 @@ import (
 )

 var (
-	NotFound    = ApiError(404, "Not found", nil)
-	ServerError = ApiError(500, "Server error", nil)
+	NotFound = func() Response {
+		return ApiError(404, "Not found", nil)
+	}
+	ServerError = func(err error) Response {
+		return ApiError(500, "Server error", err)
+	}
 )

 type Response interface {
@@ -34,7 +38,7 @@ func wrap(action interface{}) macaron.Handler {
 		if err == nil && val != nil && len(val) > 0 {
 			res = val[0].Interface().(Response)
 		} else {
-			res = ServerError
+			res = ServerError(err)
 		}

 		res.WriteTo(c.Resp)
--- a/pkg/api/dashboard.go
+++ b/pkg/api/dashboard.go
@@ -32,8 +32,6 @@ func isDashboardStarredByUser(c *middleware.Context, dashId int64) (bool, error)
 }

 func GetDashboard(c *middleware.Context) {
-	metrics.M_Api_Dashboard_Get.Inc(1)
-
 	slug := strings.ToLower(c.Params(":slug"))

 	query := m.GetDashboardQuery{Slug: slug, OrgId: c.OrgId}
@@ -77,6 +75,7 @@ func GetDashboard(c *middleware.Context) {
 		},
 	}

+	c.TimeRequest(metrics.M_Api_Dashboard_Get)
 	c.JSON(200, dto)
 }

@@ -166,8 +165,7 @@ func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) {
 		}
 	}

-	metrics.M_Api_Dashboard_Post.Inc(1)
-
+	c.TimeRequest(metrics.M_Api_Dashboard_Save)
 	c.JSON(200, util.DynMap{"status": "success", "slug": cmd.Result.Slug, "version": cmd.Result.Version})
 }

--- a/pkg/api/dataproxy.go
+++ b/pkg/api/dataproxy.go
@@ -10,6 +10,7 @@ import (

 	"github.com/grafana/grafana/pkg/api/cloudwatch"
 	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/metrics"
 	"github.com/grafana/grafana/pkg/middleware"
 	m "github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/setting"
@@ -80,7 +81,10 @@ func getDatasource(id int64, orgId int64) (*m.DataSource, error) {
 }

 func ProxyDataSourceRequest(c *middleware.Context) {
+	c.TimeRequest(metrics.M_DataSource_ProxyReq_Timer)
+
 	ds, err := getDatasource(c.ParamsInt64(":id"), c.OrgId)
+
 	if err != nil {
 		c.JsonApiErr(500, "Unable to load datasource meta data", err)
 		return
--- a/pkg/api/metrics.go
+++ b/pkg/api/metrics.go
@@ -1,10 +1,15 @@
 package api

 import (
-	"github.com/grafana/grafana/pkg/api/dtos"
-	"github.com/grafana/grafana/pkg/middleware"
+	"encoding/json"
 	"math/rand"
+	"net/http"
 	"strconv"
+
+	"github.com/grafana/grafana/pkg/api/dtos"
+	"github.com/grafana/grafana/pkg/metrics"
+	"github.com/grafana/grafana/pkg/middleware"
+	"github.com/grafana/grafana/pkg/util"
 )

 func GetTestMetrics(c *middleware.Context) {
@@ -34,3 +39,51 @@ func GetTestMetrics(c *middleware.Context) {

 	c.JSON(200, &result)
 }
+
+func GetInternalMetrics(c *middleware.Context) Response {
+	if metrics.UseNilMetrics {
+		return Json(200, util.DynMap{"message": "Metrics disabled"})
+	}
+
+	snapshots := metrics.MetricStats.GetSnapshots()
+
+	resp := make(map[string]interface{})
+
+	for _, m := range snapshots {
+		metricName := m.Name() + m.StringifyTags()
+
+		switch metric := m.(type) {
+		case metrics.Counter:
+			resp[metricName] = map[string]interface{}{
+				"count": metric.Count(),
+			}
+		case metrics.Timer:
+			percentiles := metric.Percentiles([]float64{0.25, 0.75, 0.90, 0.99})
+			resp[metricName] = map[string]interface{}{
+				"count": metric.Count(),
+				"min":   metric.Min(),
+				"max":   metric.Max(),
+				"mean":  metric.Mean(),
+				"std":   metric.StdDev(),
+				"p25":   percentiles[0],
+				"p75":   percentiles[1],
+				"p90":   percentiles[2],
+				"p99":   percentiles[3],
+			}
+		}
+	}
+
+	var b []byte
+	var err error
+	if b, err = json.MarshalIndent(resp, "", " "); err != nil {
+		return ApiError(500, "body json marshal", err)
+	}
+
+	return &NormalResponse{
+		body:   b,
+		status: 200,
+		header: http.Header{
+			"Content-Type": []string{"application/json"},
+		},
+	}
+}
--- a/pkg/api/search.go
+++ b/pkg/api/search.go
@@ -4,6 +4,7 @@ import (
 	"strconv"

 	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/metrics"
 	"github.com/grafana/grafana/pkg/middleware"
 	"github.com/grafana/grafana/pkg/services/search"
 )
@@ -42,5 +43,6 @@ func Search(c *middleware.Context) {
 		return
 	}

+	c.TimeRequest(metrics.M_Api_Dashboard_Search)
 	c.JSON(200, searchQuery.Result)
 }
--- a/pkg/cmd/grafana-server/main.go
+++ b/pkg/cmd/grafana-server/main.go
@@ -59,6 +59,7 @@ func main() {
 	flag.Parse()
 	writePIDFile()
 	initRuntime()
+	metrics.Init()

 	search.Init()
 	login.Init()
@@ -71,10 +72,6 @@ func main() {
 		log.Fatal(3, "Notification service failed to initialize", err)
 	}

-	if setting.ReportingEnabled {
-		go metrics.StartUsageReportLoop()
-	}
-
 	StartServer()
 	exitChan <- 0
 }
@@ -92,6 +89,7 @@ func initRuntime() {

 	log.Info("Starting Grafana")
 	log.Info("Version: %v, Commit: %v, Build date: %v", setting.BuildVersion, setting.BuildCommit, time.Unix(setting.BuildStamp, 0))
+
 	setting.LogConfigurationInfo()

 	sqlstore.NewEngine()
@@ -120,9 +118,7 @@ func listenToSystemSignels() {
 	signalChan := make(chan os.Signal, 1)
 	code := 0

-	signal.Notify(signalChan, os.Interrupt)
-	signal.Notify(signalChan, os.Kill)
-	signal.Notify(signalChan, syscall.SIGTERM)
+	signal.Notify(signalChan, os.Interrupt, os.Kill, syscall.SIGTERM)

 	select {
 	case sig := <-signalChan:
--- a/pkg/cmd/grafana-server/web.go
+++ b/pkg/cmd/grafana-server/web.go
@@ -31,7 +31,7 @@ func newMacaron() *macaron.Macaron {

 	for _, route := range plugins.StaticRoutes {
 		pluginRoute := path.Join("/public/plugins/", route.PluginId)
-		log.Info("Plugins: Adding route %s -> %s", pluginRoute, route.Directory)
+		log.Debug("Plugins: Adding route %s -> %s", pluginRoute, route.Directory)
 		mapStatic(m, route.Directory, "", pluginRoute)
 	}

--- a/pkg/metrics/EMWA.go
+++ b/pkg/metrics/EMWA.go
@@ -0,0 +1,122 @@
+// includes code from
+// https://raw.githubusercontent.com/rcrowley/go-metrics/master/sample.go
+// Copyright 2012 Richard Crowley. All rights reserved.
+
+package metrics
+
+import (
+	"math"
+	"sync"
+	"sync/atomic"
+)
+
+// EWMAs continuously calculate an exponentially-weighted moving average
+// based on an outside source of clock ticks.
+type EWMA interface {
+	Rate() float64
+	Snapshot() EWMA
+	Tick()
+	Update(int64)
+}
+
+// NewEWMA constructs a new EWMA with the given alpha.
+func NewEWMA(alpha float64) EWMA {
+	if UseNilMetrics {
+		return NilEWMA{}
+	}
+	return &StandardEWMA{alpha: alpha}
+}
+
+// NewEWMA1 constructs a new EWMA for a one-minute moving average.
+func NewEWMA1() EWMA {
+	return NewEWMA(1 - math.Exp(-5.0/60.0/1))
+}
+
+// NewEWMA5 constructs a new EWMA for a five-minute moving average.
+func NewEWMA5() EWMA {
+	return NewEWMA(1 - math.Exp(-5.0/60.0/5))
+}
+
+// NewEWMA15 constructs a new EWMA for a fifteen-minute moving average.
+func NewEWMA15() EWMA {
+	return NewEWMA(1 - math.Exp(-5.0/60.0/15))
+}
+
+// EWMASnapshot is a read-only copy of another EWMA.
+type EWMASnapshot float64
+
+// Rate returns the rate of events per second at the time the snapshot was
+// taken.
+func (a EWMASnapshot) Rate() float64 { return float64(a) }
+
+// Snapshot returns the snapshot.
+func (a EWMASnapshot) Snapshot() EWMA { return a }
+
+// Tick panics.
+func (EWMASnapshot) Tick() {
+	panic("Tick called on an EWMASnapshot")
+}
+
+// Update panics.
+func (EWMASnapshot) Update(int64) {
+	panic("Update called on an EWMASnapshot")
+}
+
+// NilEWMA is a no-op EWMA.
+type NilEWMA struct{}
+
+// Rate is a no-op.
+func (NilEWMA) Rate() float64 { return 0.0 }
+
+// Snapshot is a no-op.
+func (NilEWMA) Snapshot() EWMA { return NilEWMA{} }
+
+// Tick is a no-op.
+func (NilEWMA) Tick() {}
+
+// Update is a no-op.
+func (NilEWMA) Update(n int64) {}
+
+// StandardEWMA is the standard implementation of an EWMA and tracks the number
+// of uncounted events and processes them on each tick.  It uses the
+// sync/atomic package to manage uncounted events.
+type StandardEWMA struct {
+	uncounted int64 // /!\ this should be the first member to ensure 64-bit alignment
+	alpha     float64
+	rate      float64
+	init      bool
+	mutex     sync.Mutex
+}
+
+// Rate returns the moving average rate of events per second.
+func (a *StandardEWMA) Rate() float64 {
+	a.mutex.Lock()
+	defer a.mutex.Unlock()
+	return a.rate * float64(1e9)
+}
+
+// Snapshot returns a read-only copy of the EWMA.
+func (a *StandardEWMA) Snapshot() EWMA {
+	return EWMASnapshot(a.Rate())
+}
+
+// Tick ticks the clock to update the moving average.  It assumes it is called
+// every five seconds.
+func (a *StandardEWMA) Tick() {
+	count := atomic.LoadInt64(&a.uncounted)
+	atomic.AddInt64(&a.uncounted, -count)
+	instantRate := float64(count) / float64(5e9)
+	a.mutex.Lock()
+	defer a.mutex.Unlock()
+	if a.init {
+		a.rate += a.alpha * (instantRate - a.rate)
+	} else {
+		a.init = true
+		a.rate = instantRate
+	}
+}
+
+// Update adds n uncounted events.
+func (a *StandardEWMA) Update(n int64) {
+	atomic.AddInt64(&a.uncounted, n)
+}
--- a/pkg/metrics/combos.go
+++ b/pkg/metrics/combos.go
@@ -0,0 +1,46 @@
+package metrics
+
+// type comboCounterRef struct {
+// 	*MetricMeta
+// 	usageCounter  Counter
+// 	metricCounter Counter
+// }
+//
+// func RegComboCounter(name string, tagStrings ...string) Counter {
+// 	meta := NewMetricMeta(name, tagStrings)
+// 	cr := &comboCounterRef{
+// 		MetricMeta:    meta,
+// 		usageCounter:  NewCounter(meta),
+// 		metricCounter: NewCounter(meta),
+// 	}
+//
+// 	UsageStats.Register(cr.usageCounter)
+// 	MetricStats.Register(cr.metricCounter)
+//
+// 	return cr
+// }
+//
+// func (c comboCounterRef) Clear() {
+// 	c.usageCounter.Clear()
+// 	c.metricCounter.Clear()
+// }
+//
+// func (c comboCounterRef) Count() int64 {
+// 	panic("Count called on a combocounter ref")
+// }
+//
+// // Dec panics.
+// func (c comboCounterRef) Dec(i int64) {
+// 	c.usageCounter.Dec(i)
+// 	c.metricCounter.Dec(i)
+// }
+//
+// // Inc panics.
+// func (c comboCounterRef) Inc(i int64) {
+// 	c.usageCounter.Inc(i)
+// 	c.metricCounter.Inc(i)
+// }
+//
+// func (c comboCounterRef) Snapshot() Metric {
+// 	return c.metricCounter.Snapshot()
+// }
--- a/pkg/metrics/common.go
+++ b/pkg/metrics/common.go
@@ -0,0 +1,61 @@
+package metrics
+
+import "github.com/grafana/grafana/pkg/log"
+
+type MetricMeta struct {
+	tags map[string]string
+	name string
+}
+
+func NewMetricMeta(name string, tagStrings []string) *MetricMeta {
+	if len(tagStrings)%2 != 0 {
+		log.Fatal(3, "Metrics: tags array is missing value for key, %v", tagStrings)
+	}
+
+	tags := make(map[string]string)
+	for i := 0; i < len(tagStrings); i += 2 {
+		tags[tagStrings[i]] = tagStrings[i+1]
+	}
+
+	return &MetricMeta{
+		tags: tags,
+		name: name,
+	}
+}
+
+func (m *MetricMeta) Name() string {
+	return m.name
+}
+
+func (m *MetricMeta) GetTagsCopy() map[string]string {
+	if len(m.tags) == 0 {
+		return make(map[string]string)
+	}
+
+	copy := make(map[string]string)
+	for k2, v2 := range m.tags {
+		copy[k2] = v2
+	}
+
+	return copy
+}
+
+func (m *MetricMeta) StringifyTags() string {
+	if len(m.tags) == 0 {
+		return ""
+	}
+
+	str := ""
+	for key, value := range m.tags {
+		str += "." + key + "_" + value
+	}
+
+	return str
+}
+
+type Metric interface {
+	Name() string
+	GetTagsCopy() map[string]string
+	StringifyTags() string
+	Snapshot() Metric
+}
--- a/pkg/metrics/counter.go
+++ b/pkg/metrics/counter.go
@@ -4,45 +4,33 @@ import "sync/atomic"

 // Counters hold an int64 value that can be incremented and decremented.
 type Counter interface {
+	Metric
+
 	Clear()
 	Count() int64
 	Dec(int64)
 	Inc(int64)
-	Snapshot() Counter
 }

 // NewCounter constructs a new StandardCounter.
-func NewCounter() Counter {
-	return &StandardCounter{0}
+func NewCounter(meta *MetricMeta) Counter {
+	return &StandardCounter{
+		MetricMeta: meta,
+		count:      0,
+	}
 }

-// CounterSnapshot is a read-only copy of another Counter.
-type CounterSnapshot int64
-
-// Clear panics.
-func (CounterSnapshot) Clear() {
-	panic("Clear called on a CounterSnapshot")
+func RegCounter(name string, tagStrings ...string) Counter {
+	cr := NewCounter(NewMetricMeta(name, tagStrings))
+	MetricStats.Register(cr)
+	return cr
 }

-// Count returns the count at the time the snapshot was taken.
-func (c CounterSnapshot) Count() int64 { return int64(c) }
-
-// Dec panics.
-func (CounterSnapshot) Dec(int64) {
-	panic("Dec called on a CounterSnapshot")
-}
-
-// Inc panics.
-func (CounterSnapshot) Inc(int64) {
-	panic("Inc called on a CounterSnapshot")
-}
-
-// Snapshot returns the snapshot.
-func (c CounterSnapshot) Snapshot() Counter { return c }
-
 // StandardCounter is the standard implementation of a Counter and uses the
 // sync/atomic package to manage a single int64 value.
 type StandardCounter struct {
+	*MetricMeta
+
 	count int64
 }

@@ -66,7 +54,9 @@ func (c *StandardCounter) Inc(i int64) {
 	atomic.AddInt64(&c.count, i)
 }

-// Snapshot returns a read-only copy of the counter.
-func (c *StandardCounter) Snapshot() Counter {
-	return CounterSnapshot(c.Count())
+func (c *StandardCounter) Snapshot() Metric {
+	return &StandardCounter{
+		MetricMeta: c.MetricMeta,
+		count:      c.count,
+	}
 }
--- a/pkg/metrics/delta.go
+++ b/pkg/metrics/delta.go
@@ -0,0 +1,11 @@
+package metrics
+
+import "math"
+
+func calculateDelta(oldValue, newValue int64) int64 {
+	if oldValue < newValue {
+		return newValue - oldValue
+	} else {
+		return (math.MaxInt64 - oldValue) + (newValue - math.MinInt64) + 1
+	}
+}
--- a/pkg/metrics/gauge.go
+++ b/pkg/metrics/gauge.go
@@ -0,0 +1,82 @@
+// includes code from
+// https://raw.githubusercontent.com/rcrowley/go-metrics/master/sample.go
+// Copyright 2012 Richard Crowley. All rights reserved.
+
+package metrics
+
+import "sync/atomic"
+
+// Gauges hold an int64 value that can be set arbitrarily.
+type Gauge interface {
+	Metric
+
+	Update(int64)
+	Value() int64
+}
+
+func NewGauge(meta *MetricMeta) Gauge {
+	if UseNilMetrics {
+		return NilGauge{}
+	}
+	return &StandardGauge{
+		MetricMeta: meta,
+		value:      0,
+	}
+}
+
+func RegGauge(meta *MetricMeta) Gauge {
+	g := NewGauge(meta)
+	MetricStats.Register(g)
+	return g
+}
+
+// GaugeSnapshot is a read-only copy of another Gauge.
+type GaugeSnapshot struct {
+	*MetricMeta
+	value int64
+}
+
+// Snapshot returns the snapshot.
+func (g GaugeSnapshot) Snapshot() Metric { return g }
+
+// Update panics.
+func (GaugeSnapshot) Update(int64) {
+	panic("Update called on a GaugeSnapshot")
+}
+
+// Value returns the value at the time the snapshot was taken.
+func (g GaugeSnapshot) Value() int64 { return g.value }
+
+// NilGauge is a no-op Gauge.
+type NilGauge struct{ *MetricMeta }
+
+// Snapshot is a no-op.
+func (NilGauge) Snapshot() Metric { return NilGauge{} }
+
+// Update is a no-op.
+func (NilGauge) Update(v int64) {}
+
+// Value is a no-op.
+func (NilGauge) Value() int64 { return 0 }
+
+// StandardGauge is the standard implementation of a Gauge and uses the
+// sync/atomic package to manage a single int64 value.
+type StandardGauge struct {
+	*MetricMeta
+	value int64
+}
+
+// Snapshot returns a read-only copy of the gauge.
+func (g *StandardGauge) Snapshot() Metric {
+	return GaugeSnapshot{MetricMeta: g.MetricMeta, value: g.value}
+}
+
+// Update updates the gauge's value.
+func (g *StandardGauge) Update(v int64) {
+	atomic.StoreInt64(&g.value, v)
+}
+
+// Value returns the gauge's current value.
+func (g *StandardGauge) Value() int64 {
+	return atomic.LoadInt64(&g.value)
+}
--- a/pkg/metrics/graphite.go
+++ b/pkg/metrics/graphite.go
@@ -0,0 +1,91 @@
+package metrics
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+type GraphitePublisher struct {
+	address    string
+	protocol   string
+	prefix     string
+	prevCounts map[string]int64
+}
+
+func CreateGraphitePublisher() (*GraphitePublisher, error) {
+	graphiteSection, err := setting.Cfg.GetSection("metrics.graphite")
+	if err != nil {
+		return nil, nil
+	}
+
+	publisher := &GraphitePublisher{}
+	publisher.prevCounts = make(map[string]int64)
+	publisher.protocol = "tcp"
+	publisher.address = graphiteSection.Key("address").MustString("localhost:2003")
+	publisher.prefix = graphiteSection.Key("prefix").MustString("service.grafana.%(instance_name)s")
+
+	return publisher, nil
+}
+
+func (this *GraphitePublisher) Publish(metrics []Metric) {
+	conn, err := net.DialTimeout(this.protocol, this.address, time.Second*5)
+
+	if err != nil {
+		log.Error(3, "Metrics: GraphitePublisher:  Failed to connect to %s!", err)
+		return
+	}
+
+	buf := bytes.NewBufferString("")
+	now := time.Now().Unix()
+
+	for _, m := range metrics {
+		metricName := this.prefix + m.Name() + m.StringifyTags()
+
+		switch metric := m.(type) {
+		case Counter:
+			this.addCount(buf, metricName+".count", metric.Count(), now)
+		case Timer:
+			percentiles := metric.Percentiles([]float64{0.25, 0.75, 0.90, 0.99})
+			this.addCount(buf, metricName+".count", metric.Count(), now)
+			this.addInt(buf, metricName+".max", metric.Max(), now)
+			this.addInt(buf, metricName+".min", metric.Min(), now)
+			this.addFloat(buf, metricName+".mean", metric.Mean(), now)
+			this.addFloat(buf, metricName+".std", metric.StdDev(), now)
+			this.addFloat(buf, metricName+".p25", percentiles[0], now)
+			this.addFloat(buf, metricName+".p75", percentiles[1], now)
+			this.addFloat(buf, metricName+".p90", percentiles[2], now)
+			this.addFloat(buf, metricName+".p99", percentiles[3], now)
+		}
+	}
+
+	log.Trace("Metrics: GraphitePublisher.Publish() \n%s", buf)
+	_, err = conn.Write(buf.Bytes())
+
+	if err != nil {
+		log.Error(3, "Metrics: GraphitePublisher: Failed to send metrics! %s", err)
+	}
+}
+
+func (this *GraphitePublisher) addInt(buf *bytes.Buffer, metric string, value int64, now int64) {
+	buf.WriteString(fmt.Sprintf("%s %d %d\n", metric, value, now))
+}
+
+func (this *GraphitePublisher) addFloat(buf *bytes.Buffer, metric string, value float64, now int64) {
+	buf.WriteString(fmt.Sprintf("%s %f %d\n", metric, value, now))
+}
+
+func (this *GraphitePublisher) addCount(buf *bytes.Buffer, metric string, value int64, now int64) {
+	delta := value
+
+	if last, ok := this.prevCounts[metric]; ok {
+		delta = calculateDelta(last, value)
+	}
+
+	this.prevCounts[metric] = value
+	buf.WriteString(fmt.Sprintf("%s %d %d\n", metric, delta, now))
+}
--- a/pkg/metrics/histogram.go
+++ b/pkg/metrics/histogram.go
@@ -0,0 +1,189 @@
+// includes code from
+// https://raw.githubusercontent.com/rcrowley/go-metrics/master/sample.go
+// Copyright 2012 Richard Crowley. All rights reserved.
+
+package metrics
+
+// Histograms calculate distribution statistics from a series of int64 values.
+type Histogram interface {
+	Metric
+
+	Clear()
+	Count() int64
+	Max() int64
+	Mean() float64
+	Min() int64
+	Percentile(float64) float64
+	Percentiles([]float64) []float64
+	StdDev() float64
+	Sum() int64
+	Update(int64)
+	Variance() float64
+}
+
+func NewHistogram(meta *MetricMeta, s Sample) Histogram {
+	return &StandardHistogram{
+		MetricMeta: meta,
+		sample:     s,
+	}
+}
+
+// HistogramSnapshot is a read-only copy of another Histogram.
+type HistogramSnapshot struct {
+	*MetricMeta
+	sample *SampleSnapshot
+}
+
+// Clear panics.
+func (*HistogramSnapshot) Clear() {
+	panic("Clear called on a HistogramSnapshot")
+}
+
+// Count returns the number of samples recorded at the time the snapshot was
+// taken.
+func (h *HistogramSnapshot) Count() int64 { return h.sample.Count() }
+
+// Max returns the maximum value in the sample at the time the snapshot was
+// taken.
+func (h *HistogramSnapshot) Max() int64 { return h.sample.Max() }
+
+// Mean returns the mean of the values in the sample at the time the snapshot
+// was taken.
+func (h *HistogramSnapshot) Mean() float64 { return h.sample.Mean() }
+
+// Min returns the minimum value in the sample at the time the snapshot was
+// taken.
+func (h *HistogramSnapshot) Min() int64 { return h.sample.Min() }
+
+// Percentile returns an arbitrary percentile of values in the sample at the
+// time the snapshot was taken.
+func (h *HistogramSnapshot) Percentile(p float64) float64 {
+	return h.sample.Percentile(p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of values in the sample
+// at the time the snapshot was taken.
+func (h *HistogramSnapshot) Percentiles(ps []float64) []float64 {
+	return h.sample.Percentiles(ps)
+}
+
+// Sample returns the Sample underlying the histogram.
+func (h *HistogramSnapshot) Sample() Sample { return h.sample }
+
+// Snapshot returns the snapshot.
+func (h *HistogramSnapshot) Snapshot() Metric { return h }
+
+// StdDev returns the standard deviation of the values in the sample at the
+// time the snapshot was taken.
+func (h *HistogramSnapshot) StdDev() float64 { return h.sample.StdDev() }
+
+// Sum returns the sum in the sample at the time the snapshot was taken.
+func (h *HistogramSnapshot) Sum() int64 { return h.sample.Sum() }
+
+// Update panics.
+func (*HistogramSnapshot) Update(int64) {
+	panic("Update called on a HistogramSnapshot")
+}
+
+// Variance returns the variance of inputs at the time the snapshot was taken.
+func (h *HistogramSnapshot) Variance() float64 { return h.sample.Variance() }
+
+// NilHistogram is a no-op Histogram.
+type NilHistogram struct {
+	*MetricMeta
+}
+
+// Clear is a no-op.
+func (NilHistogram) Clear() {}
+
+// Count is a no-op.
+func (NilHistogram) Count() int64 { return 0 }
+
+// Max is a no-op.
+func (NilHistogram) Max() int64 { return 0 }
+
+// Mean is a no-op.
+func (NilHistogram) Mean() float64 { return 0.0 }
+
+// Min is a no-op.
+func (NilHistogram) Min() int64 { return 0 }
+
+// Percentile is a no-op.
+func (NilHistogram) Percentile(p float64) float64 { return 0.0 }
+
+// Percentiles is a no-op.
+func (NilHistogram) Percentiles(ps []float64) []float64 {
+	return make([]float64, len(ps))
+}
+
+// Sample is a no-op.
+func (NilHistogram) Sample() Sample { return NilSample{} }
+
+// Snapshot is a no-op.
+func (n NilHistogram) Snapshot() Metric { return n }
+
+// StdDev is a no-op.
+func (NilHistogram) StdDev() float64 { return 0.0 }
+
+// Sum is a no-op.
+func (NilHistogram) Sum() int64 { return 0 }
+
+// Update is a no-op.
+func (NilHistogram) Update(v int64) {}
+
+// Variance is a no-op.
+func (NilHistogram) Variance() float64 { return 0.0 }
+
+// StandardHistogram is the standard implementation of a Histogram and uses a
+// Sample to bound its memory use.
+type StandardHistogram struct {
+	*MetricMeta
+	sample Sample
+}
+
+// Clear clears the histogram and its sample.
+func (h *StandardHistogram) Clear() { h.sample.Clear() }
+
+// Count returns the number of samples recorded since the histogram was last
+// cleared.
+func (h *StandardHistogram) Count() int64 { return h.sample.Count() }
+
+// Max returns the maximum value in the sample.
+func (h *StandardHistogram) Max() int64 { return h.sample.Max() }
+
+// Mean returns the mean of the values in the sample.
+func (h *StandardHistogram) Mean() float64 { return h.sample.Mean() }
+
+// Min returns the minimum value in the sample.
+func (h *StandardHistogram) Min() int64 { return h.sample.Min() }
+
+// Percentile returns an arbitrary percentile of the values in the sample.
+func (h *StandardHistogram) Percentile(p float64) float64 {
+	return h.sample.Percentile(p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of the values in the
+// sample.
+func (h *StandardHistogram) Percentiles(ps []float64) []float64 {
+	return h.sample.Percentiles(ps)
+}
+
+// Sample returns the Sample underlying the histogram.
+func (h *StandardHistogram) Sample() Sample { return h.sample }
+
+// Snapshot returns a read-only copy of the histogram.
+func (h *StandardHistogram) Snapshot() Metric {
+	return &HistogramSnapshot{sample: h.sample.Snapshot().(*SampleSnapshot)}
+}
+
+// StdDev returns the standard deviation of the values in the sample.
+func (h *StandardHistogram) StdDev() float64 { return h.sample.StdDev() }
+
+// Sum returns the sum in the sample.
+func (h *StandardHistogram) Sum() int64 { return h.sample.Sum() }
+
+// Update samples a new value.
+func (h *StandardHistogram) Update(v int64) { h.sample.Update(v) }
+
+// Variance returns the variance of the values in the sample.
+func (h *StandardHistogram) Variance() float64 { return h.sample.Variance() }
--- a/pkg/metrics/histogram_test.go
+++ b/pkg/metrics/histogram_test.go
@@ -0,0 +1,90 @@
+// includes code from
+// https://raw.githubusercontent.com/rcrowley/go-metrics/master/sample.go
+// Copyright 2012 Richard Crowley. All rights reserved.
+
+package metrics
+
+import "testing"
+
+func BenchmarkHistogram(b *testing.B) {
+	h := NewHistogram(nil, NewUniformSample(100))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		h.Update(int64(i))
+	}
+}
+
+func TestHistogram10000(t *testing.T) {
+	h := NewHistogram(nil, NewUniformSample(100000))
+	for i := 1; i <= 10000; i++ {
+		h.Update(int64(i))
+	}
+	testHistogram10000(t, h)
+}
+
+func TestHistogramEmpty(t *testing.T) {
+	h := NewHistogram(nil, NewUniformSample(100))
+	if count := h.Count(); 0 != count {
+		t.Errorf("h.Count(): 0 != %v\n", count)
+	}
+	if min := h.Min(); 0 != min {
+		t.Errorf("h.Min(): 0 != %v\n", min)
+	}
+	if max := h.Max(); 0 != max {
+		t.Errorf("h.Max(): 0 != %v\n", max)
+	}
+	if mean := h.Mean(); 0.0 != mean {
+		t.Errorf("h.Mean(): 0.0 != %v\n", mean)
+	}
+	if stdDev := h.StdDev(); 0.0 != stdDev {
+		t.Errorf("h.StdDev(): 0.0 != %v\n", stdDev)
+	}
+	ps := h.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 0.0 != ps[0] {
+		t.Errorf("median: 0.0 != %v\n", ps[0])
+	}
+	if 0.0 != ps[1] {
+		t.Errorf("75th percentile: 0.0 != %v\n", ps[1])
+	}
+	if 0.0 != ps[2] {
+		t.Errorf("99th percentile: 0.0 != %v\n", ps[2])
+	}
+}
+
+func TestHistogramSnapshot(t *testing.T) {
+	h := NewHistogram(nil, NewUniformSample(100000))
+	for i := 1; i <= 10000; i++ {
+		h.Update(int64(i))
+	}
+	snapshot := h.Snapshot().(Histogram)
+	h.Update(0)
+	testHistogram10000(t, snapshot)
+}
+
+func testHistogram10000(t *testing.T, h Histogram) {
+	if count := h.Count(); 10000 != count {
+		t.Errorf("h.Count(): 10000 != %v\n", count)
+	}
+	if min := h.Min(); 1 != min {
+		t.Errorf("h.Min(): 1 != %v\n", min)
+	}
+	if max := h.Max(); 10000 != max {
+		t.Errorf("h.Max(): 10000 != %v\n", max)
+	}
+	if mean := h.Mean(); 5000.5 != mean {
+		t.Errorf("h.Mean(): 5000.5 != %v\n", mean)
+	}
+	if stdDev := h.StdDev(); 2886.751331514372 != stdDev {
+		t.Errorf("h.StdDev(): 2886.751331514372 != %v\n", stdDev)
+	}
+	ps := h.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 5000.5 != ps[0] {
+		t.Errorf("median: 5000.5 != %v\n", ps[0])
+	}
+	if 7500.75 != ps[1] {
+		t.Errorf("75th percentile: 7500.75 != %v\n", ps[1])
+	}
+	if 9900.99 != ps[2] {
+		t.Errorf("99th percentile: 9900.99 != %v\n", ps[2])
+	}
+}
--- a/pkg/metrics/meter.go
+++ b/pkg/metrics/meter.go
@@ -0,0 +1,221 @@
+// includes code from
+// https://raw.githubusercontent.com/rcrowley/go-metrics/master/sample.go
+// Copyright 2012 Richard Crowley. All rights reserved.
+
+package metrics
+
+import (
+	"sync"
+	"time"
+)
+
+// Meters count events to produce exponentially-weighted moving average rates
+// at one-, five-, and fifteen-minutes and a mean rate.
+type Meter interface {
+	Metric
+
+	Count() int64
+	Mark(int64)
+	Rate1() float64
+	Rate5() float64
+	Rate15() float64
+	RateMean() float64
+}
+
+// NewMeter constructs a new StandardMeter and launches a goroutine.
+func NewMeter(meta *MetricMeta) Meter {
+	if UseNilMetrics {
+		return NilMeter{}
+	}
+
+	m := newStandardMeter(meta)
+	arbiter.Lock()
+	defer arbiter.Unlock()
+	arbiter.meters = append(arbiter.meters, m)
+	if !arbiter.started {
+		arbiter.started = true
+		go arbiter.tick()
+	}
+	return m
+}
+
+type MeterSnapshot struct {
+	*MetricMeta
+	count                          int64
+	rate1, rate5, rate15, rateMean float64
+}
+
+// Count returns the count of events at the time the snapshot was taken.
+func (m *MeterSnapshot) Count() int64 { return m.count }
+
+// Mark panics.
+func (*MeterSnapshot) Mark(n int64) {
+	panic("Mark called on a MeterSnapshot")
+}
+
+// Rate1 returns the one-minute moving average rate of events per second at the
+// time the snapshot was taken.
+func (m *MeterSnapshot) Rate1() float64 { return m.rate1 }
+
+// Rate5 returns the five-minute moving average rate of events per second at
+// the time the snapshot was taken.
+func (m *MeterSnapshot) Rate5() float64 { return m.rate5 }
+
+// Rate15 returns the fifteen-minute moving average rate of events per second
+// at the time the snapshot was taken.
+func (m *MeterSnapshot) Rate15() float64 { return m.rate15 }
+
+// RateMean returns the meter's mean rate of events per second at the time the
+// snapshot was taken.
+func (m *MeterSnapshot) RateMean() float64 { return m.rateMean }
+
+// Snapshot returns the snapshot.
+func (m *MeterSnapshot) Snapshot() Metric { return m }
+
+// NilMeter is a no-op Meter.
+type NilMeter struct{ *MetricMeta }
+
+// Count is a no-op.
+func (NilMeter) Count() int64 { return 0 }
+
+// Mark is a no-op.
+func (NilMeter) Mark(n int64) {}
+
+// Rate1 is a no-op.
+func (NilMeter) Rate1() float64 { return 0.0 }
+
+// Rate5 is a no-op.
+func (NilMeter) Rate5() float64 { return 0.0 }
+
+// Rate15is a no-op.
+func (NilMeter) Rate15() float64 { return 0.0 }
+
+// RateMean is a no-op.
+func (NilMeter) RateMean() float64 { return 0.0 }
+
+// Snapshot is a no-op.
+func (NilMeter) Snapshot() Metric { return NilMeter{} }
+
+// StandardMeter is the standard implementation of a Meter.
+type StandardMeter struct {
+	*MetricMeta
+	lock        sync.RWMutex
+	snapshot    *MeterSnapshot
+	a1, a5, a15 EWMA
+	startTime   time.Time
+}
+
+func newStandardMeter(meta *MetricMeta) *StandardMeter {
+	return &StandardMeter{
+		MetricMeta: meta,
+		snapshot:   &MeterSnapshot{MetricMeta: meta},
+		a1:         NewEWMA1(),
+		a5:         NewEWMA5(),
+		a15:        NewEWMA15(),
+		startTime:  time.Now(),
+	}
+}
+
+// Count returns the number of events recorded.
+func (m *StandardMeter) Count() int64 {
+	m.lock.RLock()
+	count := m.snapshot.count
+	m.lock.RUnlock()
+	return count
+}
+
+// Mark records the occurance of n events.
+func (m *StandardMeter) Mark(n int64) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.snapshot.count += n
+	m.a1.Update(n)
+	m.a5.Update(n)
+	m.a15.Update(n)
+	m.updateSnapshot()
+}
+
+// Rate1 returns the one-minute moving average rate of events per second.
+func (m *StandardMeter) Rate1() float64 {
+	m.lock.RLock()
+	rate1 := m.snapshot.rate1
+	m.lock.RUnlock()
+	return rate1
+}
+
+// Rate5 returns the five-minute moving average rate of events per second.
+func (m *StandardMeter) Rate5() float64 {
+	m.lock.RLock()
+	rate5 := m.snapshot.rate5
+	m.lock.RUnlock()
+	return rate5
+}
+
+// Rate15 returns the fifteen-minute moving average rate of events per second.
+func (m *StandardMeter) Rate15() float64 {
+	m.lock.RLock()
+	rate15 := m.snapshot.rate15
+	m.lock.RUnlock()
+	return rate15
+}
+
+// RateMean returns the meter's mean rate of events per second.
+func (m *StandardMeter) RateMean() float64 {
+	m.lock.RLock()
+	rateMean := m.snapshot.rateMean
+	m.lock.RUnlock()
+	return rateMean
+}
+
+// Snapshot returns a read-only copy of the meter.
+func (m *StandardMeter) Snapshot() Metric {
+	m.lock.RLock()
+	snapshot := *m.snapshot
+	m.lock.RUnlock()
+	return &snapshot
+}
+
+func (m *StandardMeter) updateSnapshot() {
+	// should run with write lock held on m.lock
+	snapshot := m.snapshot
+	snapshot.rate1 = m.a1.Rate()
+	snapshot.rate5 = m.a5.Rate()
+	snapshot.rate15 = m.a15.Rate()
+	snapshot.rateMean = float64(snapshot.count) / time.Since(m.startTime).Seconds()
+}
+
+func (m *StandardMeter) tick() {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.a1.Tick()
+	m.a5.Tick()
+	m.a15.Tick()
+	m.updateSnapshot()
+}
+
+type meterArbiter struct {
+	sync.RWMutex
+	started bool
+	meters  []*StandardMeter
+	ticker  *time.Ticker
+}
+
+var arbiter = meterArbiter{ticker: time.NewTicker(5e9)}
+
+// Ticks meters on the scheduled interval
+func (ma *meterArbiter) tick() {
+	for {
+		select {
+		case <-ma.ticker.C:
+			ma.tickMeters()
+		}
+	}
+}
+
+func (ma *meterArbiter) tickMeters() {
+	ma.RLock()
+	defer ma.RUnlock()
+	for _, meter := range ma.meters {
+		meter.tick()
+	}
+}
--- a/pkg/metrics/metric_ref.go
+++ b/pkg/metrics/metric_ref.go
@@ -1,39 +0,0 @@
-package metrics
-
-type comboCounterRef struct {
-	usageCounter  Counter
-	metricCounter Counter
-}
-
-func NewComboCounterRef(name string) Counter {
-	cr := &comboCounterRef{}
-	cr.usageCounter = UsageStats.GetOrRegister(name, NewCounter).(Counter)
-	cr.metricCounter = MetricStats.GetOrRegister(name, NewCounter).(Counter)
-	return cr
-}
-
-func (c comboCounterRef) Clear() {
-	c.usageCounter.Clear()
-	c.metricCounter.Clear()
-}
-
-func (c comboCounterRef) Count() int64 {
-	panic("Count called on a combocounter ref")
-}
-
-// Dec panics.
-func (c comboCounterRef) Dec(i int64) {
-	c.usageCounter.Dec(i)
-	c.metricCounter.Dec(i)
-}
-
-// Inc panics.
-func (c comboCounterRef) Inc(i int64) {
-	c.usageCounter.Inc(i)
-	c.metricCounter.Inc(i)
-}
-
-// Snapshot returns the snapshot.
-func (c comboCounterRef) Snapshot() Counter {
-	panic("snapshot called on a combocounter ref")
-}
--- a/pkg/metrics/metrics.go
+++ b/pkg/metrics/metrics.go
@@ -1,31 +1,71 @@
 package metrics

-var UsageStats = NewRegistry()
-var MetricStats = NewRegistry()
+var MetricStats Registry
+var UseNilMetrics bool
+
+func init() {
+	// init with nil metrics
+	initMetricVars(&MetricSettings{})
+}

 var (
-	M_Instance_Start = NewComboCounterRef("instance.start")
+	M_Instance_Start                  Counter
+	M_Page_Status_200                 Counter
+	M_Page_Status_500                 Counter
+	M_Page_Status_404                 Counter
+	M_Api_Status_500                  Counter
+	M_Api_Status_404                  Counter
+	M_Api_User_SignUpStarted          Counter
+	M_Api_User_SignUpCompleted        Counter
+	M_Api_User_SignUpInvite           Counter
+	M_Api_Dashboard_Save              Timer
+	M_Api_Dashboard_Get               Timer
+	M_Api_Dashboard_Search            Timer
+	M_Api_Admin_User_Create           Counter
+	M_Api_Login_Post                  Counter
+	M_Api_Login_OAuth                 Counter
+	M_Api_Org_Create                  Counter
+	M_Api_Dashboard_Snapshot_Create   Counter
+	M_Api_Dashboard_Snapshot_External Counter
+	M_Api_Dashboard_Snapshot_Get      Counter
+	M_Models_Dashboard_Insert         Counter

-	M_Page_Status_200 = NewComboCounterRef("page.status.200")
-	M_Page_Status_500 = NewComboCounterRef("page.status.500")
-	M_Page_Status_404 = NewComboCounterRef("page.status.404")
-
-	M_Api_Status_500 = NewComboCounterRef("api.status.500")
-	M_Api_Status_404 = NewComboCounterRef("api.status.404")
-
-	M_Api_User_SignUpStarted   = NewComboCounterRef("api.user.signup_started")
-	M_Api_User_SignUpCompleted = NewComboCounterRef("api.user.signup_completed")
-	M_Api_User_SignUpInvite    = NewComboCounterRef("api.user.signup_invite")
-	M_Api_Dashboard_Get        = NewComboCounterRef("api.dashboard.get")
-	M_Api_Dashboard_Post       = NewComboCounterRef("api.dashboard.post")
-	M_Api_Admin_User_Create    = NewComboCounterRef("api.admin.user_create")
-	M_Api_Login_Post           = NewComboCounterRef("api.login.post")
-	M_Api_Login_OAuth          = NewComboCounterRef("api.login.oauth")
-	M_Api_Org_Create           = NewComboCounterRef("api.org.create")
-
-	M_Api_Dashboard_Snapshot_Create   = NewComboCounterRef("api.dashboard_snapshot.create")
-	M_Api_Dashboard_Snapshot_External = NewComboCounterRef("api.dashboard_snapshot.external")
-	M_Api_Dashboard_Snapshot_Get      = NewComboCounterRef("api.dashboard_snapshot.get")
-
-	M_Models_Dashboard_Insert = NewComboCounterRef("models.dashboard.insert")
+	// Timers
+	M_DataSource_ProxyReq_Timer Timer
 )
+
+func initMetricVars(settings *MetricSettings) {
+	UseNilMetrics = settings.Enabled == false
+	MetricStats = NewRegistry()
+
+	M_Instance_Start = RegCounter("instance_start")
+
+	M_Page_Status_200 = RegCounter("page.resp_status", "code", "200")
+	M_Page_Status_500 = RegCounter("page.resp_status", "code", "500")
+	M_Page_Status_404 = RegCounter("page.resp_status", "code", "404")
+
+	M_Api_Status_500 = RegCounter("api.resp_status", "code", "500")
+	M_Api_Status_404 = RegCounter("api.resp_status", "code", "404")
+
+	M_Api_User_SignUpStarted = RegCounter("api.user.signup_started")
+	M_Api_User_SignUpCompleted = RegCounter("api.user.signup_completed")
+	M_Api_User_SignUpInvite = RegCounter("api.user.signup_invite")
+
+	M_Api_Dashboard_Save = RegTimer("api.dashboard.save")
+	M_Api_Dashboard_Get = RegTimer("api.dashboard.get")
+	M_Api_Dashboard_Search = RegTimer("api.dashboard.search")
+
+	M_Api_Admin_User_Create = RegCounter("api.admin.user_create")
+	M_Api_Login_Post = RegCounter("api.login.post")
+	M_Api_Login_OAuth = RegCounter("api.login.oauth")
+	M_Api_Org_Create = RegCounter("api.org.create")
+
+	M_Api_Dashboard_Snapshot_Create = RegCounter("api.dashboard_snapshot.create")
+	M_Api_Dashboard_Snapshot_External = RegCounter("api.dashboard_snapshot.external")
+	M_Api_Dashboard_Snapshot_Get = RegCounter("api.dashboard_snapshot.get")
+
+	M_Models_Dashboard_Insert = RegCounter("models.dashboard.insert")
+
+	// Timers
+	M_DataSource_ProxyReq_Timer = RegTimer("api.dataproxy.request.all")
+}
--- a/pkg/metrics/report_usage.go
+++ b/pkg/metrics/report_usage.go
@@ -14,19 +14,46 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 )

-func StartUsageReportLoop() chan struct{} {
+func Init() {
+	settings := readSettings()
+	initMetricVars(settings)
+	go instrumentationLoop(settings)
+}
+
+func instrumentationLoop(settings *MetricSettings) chan struct{} {
 	M_Instance_Start.Inc(1)

-	ticker := time.NewTicker(time.Hour * 24)
+	onceEveryDayTick := time.NewTicker(time.Hour * 24)
+	secondTicker := time.NewTicker(time.Second * time.Duration(settings.IntervalSeconds))
+
 	for {
 		select {
-		case <-ticker.C:
+		case <-onceEveryDayTick.C:
 			sendUsageStats()
+		case <-secondTicker.C:
+			if settings.Enabled {
+				sendMetrics(settings)
+			}
 		}
 	}
 }

+func sendMetrics(settings *MetricSettings) {
+	if len(settings.Publishers) == 0 {
+		return
+	}
+
+	metrics := MetricStats.GetSnapshots()
+	for _, publisher := range settings.Publishers {
+		publisher.Publish(metrics)
+	}
+}
+
 func sendUsageStats() {
+	if !setting.ReportingEnabled {
+		return
+	}
+
 	log.Trace("Sending anonymous usage stats to stats.grafana.org")

 	version := strings.Replace(setting.BuildVersion, ".", "_", -1)
@@ -37,16 +64,6 @@ func sendUsageStats() {
 		"metrics": metrics,
 	}

-	UsageStats.Each(func(name string, i interface{}) {
-		switch metric := i.(type) {
-		case Counter:
-			if metric.Count() > 0 {
-				metrics[name+".count"] = metric.Count()
-				metric.Clear()
-			}
-		}
-	})
-
 	statsQuery := m.GetSystemStatsQuery{}
 	if err := bus.Dispatch(&statsQuery); err != nil {
 		log.Error(3, "Failed to get system stats", err)
--- a/pkg/metrics/registry.go
+++ b/pkg/metrics/registry.go
@@ -1,102 +1,37 @@
 package metrics

-import (
-	"fmt"
-	"reflect"
-	"sync"
-)
-
-// DuplicateMetric is the error returned by Registry.Register when a metric
-// already exists.  If you mean to Register that metric you must first
-// Unregister the existing metric.
-type DuplicateMetric string
-
-func (err DuplicateMetric) Error() string {
-	return fmt.Sprintf("duplicate metric: %s", string(err))
-}
+import "sync"

 type Registry interface {
-	// Call the given function for each registered metric.
-	Each(func(string, interface{}))
-
-	// Get the metric by the given name or nil if none is registered.
-	Get(string) interface{}
-
-	// Gets an existing metric or registers the given one.
-	// The interface can be the metric to register if not found in registry,
-	// or a function returning the metric for lazy instantiation.
-	GetOrRegister(string, interface{}) interface{}
-
-	// Register the given metric under the given name.
-	Register(string, interface{}) error
+	GetSnapshots() []Metric
+	Register(metric Metric)
 }

 // The standard implementation of a Registry is a mutex-protected map
 // of names to metrics.
 type StandardRegistry struct {
-	metrics map[string]interface{}
+	metrics []Metric
 	mutex   sync.Mutex
 }

 // Create a new registry.
 func NewRegistry() Registry {
-	return &StandardRegistry{metrics: make(map[string]interface{})}
+	return &StandardRegistry{
+		metrics: make([]Metric, 0),
+	}
+}
+
+func (r *StandardRegistry) Register(metric Metric) {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	r.metrics = append(r.metrics, metric)
 }

 // Call the given function for each registered metric.
-func (r *StandardRegistry) Each(f func(string, interface{})) {
-	for name, i := range r.registered() {
-		f(name, i)
-	}
-}
-
-// Get the metric by the given name or nil if none is registered.
-func (r *StandardRegistry) Get(name string) interface{} {
-	r.mutex.Lock()
-	defer r.mutex.Unlock()
-	return r.metrics[name]
-}
-
-// Gets an existing metric or creates and registers a new one. Threadsafe
-// alternative to calling Get and Register on failure.
-// The interface can be the metric to register if not found in registry,
-// or a function returning the metric for lazy instantiation.
-func (r *StandardRegistry) GetOrRegister(name string, i interface{}) interface{} {
-	r.mutex.Lock()
-	defer r.mutex.Unlock()
-	if metric, ok := r.metrics[name]; ok {
-		return metric
-	}
-	if v := reflect.ValueOf(i); v.Kind() == reflect.Func {
-		i = v.Call(nil)[0].Interface()
-	}
-	r.register(name, i)
-	return i
-}
-
-// Register the given metric under the given name.  Returns a DuplicateMetric
-// if a metric by the given name is already registered.
-func (r *StandardRegistry) Register(name string, i interface{}) error {
-	r.mutex.Lock()
-	defer r.mutex.Unlock()
-	return r.register(name, i)
-}
-
-func (r *StandardRegistry) register(name string, i interface{}) error {
-	if _, ok := r.metrics[name]; ok {
-		return DuplicateMetric(name)
-	}
-
-	r.metrics[name] = i
-	return nil
-}
-
-func (r *StandardRegistry) registered() map[string]interface{} {
-	metrics := make(map[string]interface{}, len(r.metrics))
-	r.mutex.Lock()
-	defer r.mutex.Unlock()
-	for name, i := range r.metrics {
-		metrics[name] = i
+func (r *StandardRegistry) GetSnapshots() []Metric {
+	metrics := make([]Metric, len(r.metrics))
+	for i, metric := range r.metrics {
+		metrics[i] = metric.Snapshot()
 	}
 	return metrics
 }
--- a/pkg/metrics/sample.go
+++ b/pkg/metrics/sample.go
@@ -0,0 +1,607 @@
+// includes code from
+// https://raw.githubusercontent.com/rcrowley/go-metrics/master/sample.go
+// Copyright 2012 Richard Crowley. All rights reserved.
+
+package metrics
+
+import (
+	"math"
+	"math/rand"
+	"sort"
+	"sync"
+	"time"
+)
+
+const rescaleThreshold = time.Hour
+
+// Samples maintain a statistically-significant selection of values from
+// a stream.
+type Sample interface {
+	Clear()
+	Count() int64
+	Max() int64
+	Mean() float64
+	Min() int64
+	Percentile(float64) float64
+	Percentiles([]float64) []float64
+	Size() int
+	Snapshot() Sample
+	StdDev() float64
+	Sum() int64
+	Update(int64)
+	Values() []int64
+	Variance() float64
+}
+
+// ExpDecaySample is an exponentially-decaying sample using a forward-decaying
+// priority reservoir.  See Cormode et al's "Forward Decay: A Practical Time
+// Decay Model for Streaming Systems".
+//
+// <http://www.research.att.com/people/Cormode_Graham/library/publications/CormodeShkapenyukSrivastavaXu09.pdf>
+type ExpDecaySample struct {
+	alpha         float64
+	count         int64
+	mutex         sync.Mutex
+	reservoirSize int
+	t0, t1        time.Time
+	values        *expDecaySampleHeap
+}
+
+// NewExpDecaySample constructs a new exponentially-decaying sample with the
+// given reservoir size and alpha.
+func NewExpDecaySample(reservoirSize int, alpha float64) Sample {
+	s := &ExpDecaySample{
+		alpha:         alpha,
+		reservoirSize: reservoirSize,
+		t0:            time.Now(),
+		values:        newExpDecaySampleHeap(reservoirSize),
+	}
+	s.t1 = s.t0.Add(rescaleThreshold)
+	return s
+}
+
+// Clear clears all samples.
+func (s *ExpDecaySample) Clear() {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	s.count = 0
+	s.t0 = time.Now()
+	s.t1 = s.t0.Add(rescaleThreshold)
+	s.values.Clear()
+}
+
+// Count returns the number of samples recorded, which may exceed the
+// reservoir size.
+func (s *ExpDecaySample) Count() int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return s.count
+}
+
+// Max returns the maximum value in the sample, which may not be the maximum
+// value ever to be part of the sample.
+func (s *ExpDecaySample) Max() int64 {
+	return SampleMax(s.Values())
+}
+
+// Mean returns the mean of the values in the sample.
+func (s *ExpDecaySample) Mean() float64 {
+	return SampleMean(s.Values())
+}
+
+// Min returns the minimum value in the sample, which may not be the minimum
+// value ever to be part of the sample.
+func (s *ExpDecaySample) Min() int64 {
+	return SampleMin(s.Values())
+}
+
+// Percentile returns an arbitrary percentile of values in the sample.
+func (s *ExpDecaySample) Percentile(p float64) float64 {
+	return SamplePercentile(s.Values(), p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of values in the
+// sample.
+func (s *ExpDecaySample) Percentiles(ps []float64) []float64 {
+	return SamplePercentiles(s.Values(), ps)
+}
+
+// Size returns the size of the sample, which is at most the reservoir size.
+func (s *ExpDecaySample) Size() int {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return s.values.Size()
+}
+
+// Snapshot returns a read-only copy of the sample.
+func (s *ExpDecaySample) Snapshot() Sample {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	vals := s.values.Values()
+	values := make([]int64, len(vals))
+	for i, v := range vals {
+		values[i] = v.v
+	}
+	return &SampleSnapshot{
+		count:  s.count,
+		values: values,
+	}
+}
+
+// StdDev returns the standard deviation of the values in the sample.
+func (s *ExpDecaySample) StdDev() float64 {
+	return SampleStdDev(s.Values())
+}
+
+// Sum returns the sum of the values in the sample.
+func (s *ExpDecaySample) Sum() int64 {
+	return SampleSum(s.Values())
+}
+
+// Update samples a new value.
+func (s *ExpDecaySample) Update(v int64) {
+	s.update(time.Now(), v)
+}
+
+// Values returns a copy of the values in the sample.
+func (s *ExpDecaySample) Values() []int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	vals := s.values.Values()
+	values := make([]int64, len(vals))
+	for i, v := range vals {
+		values[i] = v.v
+	}
+	return values
+}
+
+// Variance returns the variance of the values in the sample.
+func (s *ExpDecaySample) Variance() float64 {
+	return SampleVariance(s.Values())
+}
+
+// update samples a new value at a particular timestamp.  This is a method all
+// its own to facilitate testing.
+func (s *ExpDecaySample) update(t time.Time, v int64) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	s.count++
+	if s.values.Size() == s.reservoirSize {
+		s.values.Pop()
+	}
+	s.values.Push(expDecaySample{
+		k: math.Exp(t.Sub(s.t0).Seconds()*s.alpha) / rand.Float64(),
+		v: v,
+	})
+	if t.After(s.t1) {
+		values := s.values.Values()
+		t0 := s.t0
+		s.values.Clear()
+		s.t0 = t
+		s.t1 = s.t0.Add(rescaleThreshold)
+		for _, v := range values {
+			v.k = v.k * math.Exp(-s.alpha*s.t0.Sub(t0).Seconds())
+			s.values.Push(v)
+		}
+	}
+}
+
+// NilSample is a no-op Sample.
+type NilSample struct{}
+
+// Clear is a no-op.
+func (NilSample) Clear() {}
+
+// Count is a no-op.
+func (NilSample) Count() int64 { return 0 }
+
+// Max is a no-op.
+func (NilSample) Max() int64 { return 0 }
+
+// Mean is a no-op.
+func (NilSample) Mean() float64 { return 0.0 }
+
+// Min is a no-op.
+func (NilSample) Min() int64 { return 0 }
+
+// Percentile is a no-op.
+func (NilSample) Percentile(p float64) float64 { return 0.0 }
+
+// Percentiles is a no-op.
+func (NilSample) Percentiles(ps []float64) []float64 {
+	return make([]float64, len(ps))
+}
+
+// Size is a no-op.
+func (NilSample) Size() int { return 0 }
+
+// Sample is a no-op.
+func (NilSample) Snapshot() Sample { return NilSample{} }
+
+// StdDev is a no-op.
+func (NilSample) StdDev() float64 { return 0.0 }
+
+// Sum is a no-op.
+func (NilSample) Sum() int64 { return 0 }
+
+// Update is a no-op.
+func (NilSample) Update(v int64) {}
+
+// Values is a no-op.
+func (NilSample) Values() []int64 { return []int64{} }
+
+// Variance is a no-op.
+func (NilSample) Variance() float64 { return 0.0 }
+
+// SampleMax returns the maximum value of the slice of int64.
+func SampleMax(values []int64) int64 {
+	if 0 == len(values) {
+		return 0
+	}
+	var max int64 = math.MinInt64
+	for _, v := range values {
+		if max < v {
+			max = v
+		}
+	}
+	return max
+}
+
+// SampleMean returns the mean value of the slice of int64.
+func SampleMean(values []int64) float64 {
+	if 0 == len(values) {
+		return 0.0
+	}
+	return float64(SampleSum(values)) / float64(len(values))
+}
+
+// SampleMin returns the minimum value of the slice of int64.
+func SampleMin(values []int64) int64 {
+	if 0 == len(values) {
+		return 0
+	}
+	var min int64 = math.MaxInt64
+	for _, v := range values {
+		if min > v {
+			min = v
+		}
+	}
+	return min
+}
+
+// SamplePercentiles returns an arbitrary percentile of the slice of int64.
+func SamplePercentile(values int64Slice, p float64) float64 {
+	return SamplePercentiles(values, []float64{p})[0]
+}
+
+// SamplePercentiles returns a slice of arbitrary percentiles of the slice of
+// int64.
+func SamplePercentiles(values int64Slice, ps []float64) []float64 {
+	scores := make([]float64, len(ps))
+	size := len(values)
+	if size > 0 {
+		sort.Sort(values)
+		for i, p := range ps {
+			pos := p * float64(size+1)
+			if pos < 1.0 {
+				scores[i] = float64(values[0])
+			} else if pos >= float64(size) {
+				scores[i] = float64(values[size-1])
+			} else {
+				lower := float64(values[int(pos)-1])
+				upper := float64(values[int(pos)])
+				scores[i] = lower + (pos-math.Floor(pos))*(upper-lower)
+			}
+		}
+	}
+	return scores
+}
+
+// SampleSnapshot is a read-only copy of another Sample.
+type SampleSnapshot struct {
+	count  int64
+	values []int64
+}
+
+// Clear panics.
+func (*SampleSnapshot) Clear() {
+	panic("Clear called on a SampleSnapshot")
+}
+
+// Count returns the count of inputs at the time the snapshot was taken.
+func (s *SampleSnapshot) Count() int64 { return s.count }
+
+// Max returns the maximal value at the time the snapshot was taken.
+func (s *SampleSnapshot) Max() int64 { return SampleMax(s.values) }
+
+// Mean returns the mean value at the time the snapshot was taken.
+func (s *SampleSnapshot) Mean() float64 { return SampleMean(s.values) }
+
+// Min returns the minimal value at the time the snapshot was taken.
+func (s *SampleSnapshot) Min() int64 { return SampleMin(s.values) }
+
+// Percentile returns an arbitrary percentile of values at the time the
+// snapshot was taken.
+func (s *SampleSnapshot) Percentile(p float64) float64 {
+	return SamplePercentile(s.values, p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of values at the time
+// the snapshot was taken.
+func (s *SampleSnapshot) Percentiles(ps []float64) []float64 {
+	return SamplePercentiles(s.values, ps)
+}
+
+// Size returns the size of the sample at the time the snapshot was taken.
+func (s *SampleSnapshot) Size() int { return len(s.values) }
+
+// Snapshot returns the snapshot.
+func (s *SampleSnapshot) Snapshot() Sample { return s }
+
+// StdDev returns the standard deviation of values at the time the snapshot was
+// taken.
+func (s *SampleSnapshot) StdDev() float64 { return SampleStdDev(s.values) }
+
+// Sum returns the sum of values at the time the snapshot was taken.
+func (s *SampleSnapshot) Sum() int64 { return SampleSum(s.values) }
+
+// Update panics.
+func (*SampleSnapshot) Update(int64) {
+	panic("Update called on a SampleSnapshot")
+}
+
+// Values returns a copy of the values in the sample.
+func (s *SampleSnapshot) Values() []int64 {
+	values := make([]int64, len(s.values))
+	copy(values, s.values)
+	return values
+}
+
+// Variance returns the variance of values at the time the snapshot was taken.
+func (s *SampleSnapshot) Variance() float64 { return SampleVariance(s.values) }
+
+// SampleStdDev returns the standard deviation of the slice of int64.
+func SampleStdDev(values []int64) float64 {
+	return math.Sqrt(SampleVariance(values))
+}
+
+// SampleSum returns the sum of the slice of int64.
+func SampleSum(values []int64) int64 {
+	var sum int64
+	for _, v := range values {
+		sum += v
+	}
+	return sum
+}
+
+// SampleVariance returns the variance of the slice of int64.
+func SampleVariance(values []int64) float64 {
+	if 0 == len(values) {
+		return 0.0
+	}
+	m := SampleMean(values)
+	var sum float64
+	for _, v := range values {
+		d := float64(v) - m
+		sum += d * d
+	}
+	return sum / float64(len(values))
+}
+
+// A uniform sample using Vitter's Algorithm R.
+//
+// <http://www.cs.umd.edu/~samir/498/vitter.pdf>
+type UniformSample struct {
+	count         int64
+	mutex         sync.Mutex
+	reservoirSize int
+	values        []int64
+}
+
+// NewUniformSample constructs a new uniform sample with the given reservoir
+// size.
+func NewUniformSample(reservoirSize int) Sample {
+	return &UniformSample{
+		reservoirSize: reservoirSize,
+		values:        make([]int64, 0, reservoirSize),
+	}
+}
+
+// Clear clears all samples.
+func (s *UniformSample) Clear() {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	s.count = 0
+	s.values = make([]int64, 0, s.reservoirSize)
+}
+
+// Count returns the number of samples recorded, which may exceed the
+// reservoir size.
+func (s *UniformSample) Count() int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return s.count
+}
+
+// Max returns the maximum value in the sample, which may not be the maximum
+// value ever to be part of the sample.
+func (s *UniformSample) Max() int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SampleMax(s.values)
+}
+
+// Mean returns the mean of the values in the sample.
+func (s *UniformSample) Mean() float64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SampleMean(s.values)
+}
+
+// Min returns the minimum value in the sample, which may not be the minimum
+// value ever to be part of the sample.
+func (s *UniformSample) Min() int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SampleMin(s.values)
+}
+
+// Percentile returns an arbitrary percentile of values in the sample.
+func (s *UniformSample) Percentile(p float64) float64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SamplePercentile(s.values, p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of values in the
+// sample.
+func (s *UniformSample) Percentiles(ps []float64) []float64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SamplePercentiles(s.values, ps)
+}
+
+// Size returns the size of the sample, which is at most the reservoir size.
+func (s *UniformSample) Size() int {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return len(s.values)
+}
+
+// Snapshot returns a read-only copy of the sample.
+func (s *UniformSample) Snapshot() Sample {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	values := make([]int64, len(s.values))
+	copy(values, s.values)
+	return &SampleSnapshot{
+		count:  s.count,
+		values: values,
+	}
+}
+
+// StdDev returns the standard deviation of the values in the sample.
+func (s *UniformSample) StdDev() float64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SampleStdDev(s.values)
+}
+
+// Sum returns the sum of the values in the sample.
+func (s *UniformSample) Sum() int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SampleSum(s.values)
+}
+
+// Update samples a new value.
+func (s *UniformSample) Update(v int64) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	s.count++
+	if len(s.values) < s.reservoirSize {
+		s.values = append(s.values, v)
+	} else {
+		r := rand.Int63n(s.count)
+		if r < int64(len(s.values)) {
+			s.values[int(r)] = v
+		}
+	}
+}
+
+// Values returns a copy of the values in the sample.
+func (s *UniformSample) Values() []int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	values := make([]int64, len(s.values))
+	copy(values, s.values)
+	return values
+}
+
+// Variance returns the variance of the values in the sample.
+func (s *UniformSample) Variance() float64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SampleVariance(s.values)
+}
+
+// expDecaySample represents an individual sample in a heap.
+type expDecaySample struct {
+	k float64
+	v int64
+}
+
+func newExpDecaySampleHeap(reservoirSize int) *expDecaySampleHeap {
+	return &expDecaySampleHeap{make([]expDecaySample, 0, reservoirSize)}
+}
+
+// expDecaySampleHeap is a min-heap of expDecaySamples.
+// The internal implementation is copied from the standard library's container/heap
+type expDecaySampleHeap struct {
+	s []expDecaySample
+}
+
+func (h *expDecaySampleHeap) Clear() {
+	h.s = h.s[:0]
+}
+
+func (h *expDecaySampleHeap) Push(s expDecaySample) {
+	n := len(h.s)
+	h.s = h.s[0 : n+1]
+	h.s[n] = s
+	h.up(n)
+}
+
+func (h *expDecaySampleHeap) Pop() expDecaySample {
+	n := len(h.s) - 1
+	h.s[0], h.s[n] = h.s[n], h.s[0]
+	h.down(0, n)
+
+	n = len(h.s)
+	s := h.s[n-1]
+	h.s = h.s[0 : n-1]
+	return s
+}
+
+func (h *expDecaySampleHeap) Size() int {
+	return len(h.s)
+}
+
+func (h *expDecaySampleHeap) Values() []expDecaySample {
+	return h.s
+}
+
+func (h *expDecaySampleHeap) up(j int) {
+	for {
+		i := (j - 1) / 2 // parent
+		if i == j || !(h.s[j].k < h.s[i].k) {
+			break
+		}
+		h.s[i], h.s[j] = h.s[j], h.s[i]
+		j = i
+	}
+}
+
+func (h *expDecaySampleHeap) down(i, n int) {
+	for {
+		j1 := 2*i + 1
+		if j1 >= n || j1 < 0 { // j1 < 0 after int overflow
+			break
+		}
+		j := j1 // left child
+		if j2 := j1 + 1; j2 < n && !(h.s[j1].k < h.s[j2].k) {
+			j = j2 // = 2*i + 2  // right child
+		}
+		if !(h.s[j].k < h.s[i].k) {
+			break
+		}
+		h.s[i], h.s[j] = h.s[j], h.s[i]
+		i = j
+	}
+}
+
+type int64Slice []int64
+
+func (p int64Slice) Len() int           { return len(p) }
+func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
+func (p int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
--- a/pkg/metrics/sample_test.go
+++ b/pkg/metrics/sample_test.go
@@ -0,0 +1,367 @@
+// includes code from
+// https://raw.githubusercontent.com/rcrowley/go-metrics/master/sample.go
+// Copyright 2012 Richard Crowley. All rights reserved.
+
+package metrics
+
+import (
+	"math/rand"
+	"runtime"
+	"testing"
+	"time"
+)
+
+// Benchmark{Compute,Copy}{1000,1000000} demonstrate that, even for relatively
+// expensive computations like Variance, the cost of copying the Sample, as
+// approximated by a make and copy, is much greater than the cost of the
+// computation for small samples and only slightly less for large samples.
+func BenchmarkCompute1000(b *testing.B) {
+	s := make([]int64, 1000)
+	for i := 0; i < len(s); i++ {
+		s[i] = int64(i)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		SampleVariance(s)
+	}
+}
+func BenchmarkCompute1000000(b *testing.B) {
+	s := make([]int64, 1000000)
+	for i := 0; i < len(s); i++ {
+		s[i] = int64(i)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		SampleVariance(s)
+	}
+}
+func BenchmarkCopy1000(b *testing.B) {
+	s := make([]int64, 1000)
+	for i := 0; i < len(s); i++ {
+		s[i] = int64(i)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		sCopy := make([]int64, len(s))
+		copy(sCopy, s)
+	}
+}
+func BenchmarkCopy1000000(b *testing.B) {
+	s := make([]int64, 1000000)
+	for i := 0; i < len(s); i++ {
+		s[i] = int64(i)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		sCopy := make([]int64, len(s))
+		copy(sCopy, s)
+	}
+}
+
+func BenchmarkExpDecaySample257(b *testing.B) {
+	benchmarkSample(b, NewExpDecaySample(257, 0.015))
+}
+
+func BenchmarkExpDecaySample514(b *testing.B) {
+	benchmarkSample(b, NewExpDecaySample(514, 0.015))
+}
+
+func BenchmarkExpDecaySample1028(b *testing.B) {
+	benchmarkSample(b, NewExpDecaySample(1028, 0.015))
+}
+
+func BenchmarkUniformSample257(b *testing.B) {
+	benchmarkSample(b, NewUniformSample(257))
+}
+
+func BenchmarkUniformSample514(b *testing.B) {
+	benchmarkSample(b, NewUniformSample(514))
+}
+
+func BenchmarkUniformSample1028(b *testing.B) {
+	benchmarkSample(b, NewUniformSample(1028))
+}
+
+func TestExpDecaySample10(t *testing.T) {
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 0; i < 10; i++ {
+		s.Update(int64(i))
+	}
+	if size := s.Count(); 10 != size {
+		t.Errorf("s.Count(): 10 != %v\n", size)
+	}
+	if size := s.Size(); 10 != size {
+		t.Errorf("s.Size(): 10 != %v\n", size)
+	}
+	if l := len(s.Values()); 10 != l {
+		t.Errorf("len(s.Values()): 10 != %v\n", l)
+	}
+	for _, v := range s.Values() {
+		if v > 10 || v < 0 {
+			t.Errorf("out of range [0, 10): %v\n", v)
+		}
+	}
+}
+
+func TestExpDecaySample100(t *testing.T) {
+	rand.Seed(1)
+	s := NewExpDecaySample(1000, 0.01)
+	for i := 0; i < 100; i++ {
+		s.Update(int64(i))
+	}
+	if size := s.Count(); 100 != size {
+		t.Errorf("s.Count(): 100 != %v\n", size)
+	}
+	if size := s.Size(); 100 != size {
+		t.Errorf("s.Size(): 100 != %v\n", size)
+	}
+	if l := len(s.Values()); 100 != l {
+		t.Errorf("len(s.Values()): 100 != %v\n", l)
+	}
+	for _, v := range s.Values() {
+		if v > 100 || v < 0 {
+			t.Errorf("out of range [0, 100): %v\n", v)
+		}
+	}
+}
+
+func TestExpDecaySample1000(t *testing.T) {
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 0; i < 1000; i++ {
+		s.Update(int64(i))
+	}
+	if size := s.Count(); 1000 != size {
+		t.Errorf("s.Count(): 1000 != %v\n", size)
+	}
+	if size := s.Size(); 100 != size {
+		t.Errorf("s.Size(): 100 != %v\n", size)
+	}
+	if l := len(s.Values()); 100 != l {
+		t.Errorf("len(s.Values()): 100 != %v\n", l)
+	}
+	for _, v := range s.Values() {
+		if v > 1000 || v < 0 {
+			t.Errorf("out of range [0, 1000): %v\n", v)
+		}
+	}
+}
+
+// This test makes sure that the sample's priority is not amplified by using
+// nanosecond duration since start rather than second duration since start.
+// The priority becomes +Inf quickly after starting if this is done,
+// effectively freezing the set of samples until a rescale step happens.
+func TestExpDecaySampleNanosecondRegression(t *testing.T) {
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 0; i < 100; i++ {
+		s.Update(10)
+	}
+	time.Sleep(1 * time.Millisecond)
+	for i := 0; i < 100; i++ {
+		s.Update(20)
+	}
+	v := s.Values()
+	avg := float64(0)
+	for i := 0; i < len(v); i++ {
+		avg += float64(v[i])
+	}
+	avg /= float64(len(v))
+	if avg > 16 || avg < 14 {
+		t.Errorf("out of range [14, 16]: %v\n", avg)
+	}
+}
+
+func TestExpDecaySampleRescale(t *testing.T) {
+	s := NewExpDecaySample(2, 0.001).(*ExpDecaySample)
+	s.update(time.Now(), 1)
+	s.update(time.Now().Add(time.Hour+time.Microsecond), 1)
+	for _, v := range s.values.Values() {
+		if v.k == 0.0 {
+			t.Fatal("v.k == 0.0")
+		}
+	}
+}
+
+func TestExpDecaySampleSnapshot(t *testing.T) {
+	now := time.Now()
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 1; i <= 10000; i++ {
+		s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i))
+	}
+	snapshot := s.Snapshot()
+	s.Update(1)
+	testExpDecaySampleStatistics(t, snapshot)
+}
+
+func TestExpDecaySampleStatistics(t *testing.T) {
+	now := time.Now()
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 1; i <= 10000; i++ {
+		s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i))
+	}
+	testExpDecaySampleStatistics(t, s)
+}
+
+func TestUniformSample(t *testing.T) {
+	rand.Seed(1)
+	s := NewUniformSample(100)
+	for i := 0; i < 1000; i++ {
+		s.Update(int64(i))
+	}
+	if size := s.Count(); 1000 != size {
+		t.Errorf("s.Count(): 1000 != %v\n", size)
+	}
+	if size := s.Size(); 100 != size {
+		t.Errorf("s.Size(): 100 != %v\n", size)
+	}
+	if l := len(s.Values()); 100 != l {
+		t.Errorf("len(s.Values()): 100 != %v\n", l)
+	}
+	for _, v := range s.Values() {
+		if v > 1000 || v < 0 {
+			t.Errorf("out of range [0, 100): %v\n", v)
+		}
+	}
+}
+
+func TestUniformSampleIncludesTail(t *testing.T) {
+	rand.Seed(1)
+	s := NewUniformSample(100)
+	max := 100
+	for i := 0; i < max; i++ {
+		s.Update(int64(i))
+	}
+	v := s.Values()
+	sum := 0
+	exp := (max - 1) * max / 2
+	for i := 0; i < len(v); i++ {
+		sum += int(v[i])
+	}
+	if exp != sum {
+		t.Errorf("sum: %v != %v\n", exp, sum)
+	}
+}
+
+func TestUniformSampleSnapshot(t *testing.T) {
+	s := NewUniformSample(100)
+	for i := 1; i <= 10000; i++ {
+		s.Update(int64(i))
+	}
+	snapshot := s.Snapshot()
+	s.Update(1)
+	testUniformSampleStatistics(t, snapshot)
+}
+
+func TestUniformSampleStatistics(t *testing.T) {
+	rand.Seed(1)
+	s := NewUniformSample(100)
+	for i := 1; i <= 10000; i++ {
+		s.Update(int64(i))
+	}
+	testUniformSampleStatistics(t, s)
+}
+
+func benchmarkSample(b *testing.B, s Sample) {
+	var memStats runtime.MemStats
+	runtime.ReadMemStats(&memStats)
+	pauseTotalNs := memStats.PauseTotalNs
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		s.Update(1)
+	}
+	b.StopTimer()
+	runtime.GC()
+	runtime.ReadMemStats(&memStats)
+	b.Logf("GC cost: %d ns/op", int(memStats.PauseTotalNs-pauseTotalNs)/b.N)
+}
+
+func testExpDecaySampleStatistics(t *testing.T, s Sample) {
+	if count := s.Count(); 10000 != count {
+		t.Errorf("s.Count(): 10000 != %v\n", count)
+	}
+	if min := s.Min(); 107 != min {
+		t.Errorf("s.Min(): 107 != %v\n", min)
+	}
+	if max := s.Max(); 10000 != max {
+		t.Errorf("s.Max(): 10000 != %v\n", max)
+	}
+	if mean := s.Mean(); 4965.98 != mean {
+		t.Errorf("s.Mean(): 4965.98 != %v\n", mean)
+	}
+	if stdDev := s.StdDev(); 2959.825156930727 != stdDev {
+		t.Errorf("s.StdDev(): 2959.825156930727 != %v\n", stdDev)
+	}
+	ps := s.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 4615 != ps[0] {
+		t.Errorf("median: 4615 != %v\n", ps[0])
+	}
+	if 7672 != ps[1] {
+		t.Errorf("75th percentile: 7672 != %v\n", ps[1])
+	}
+	if 9998.99 != ps[2] {
+		t.Errorf("99th percentile: 9998.99 != %v\n", ps[2])
+	}
+}
+
+func testUniformSampleStatistics(t *testing.T, s Sample) {
+	if count := s.Count(); 10000 != count {
+		t.Errorf("s.Count(): 10000 != %v\n", count)
+	}
+	if min := s.Min(); 37 != min {
+		t.Errorf("s.Min(): 37 != %v\n", min)
+	}
+	if max := s.Max(); 9989 != max {
+		t.Errorf("s.Max(): 9989 != %v\n", max)
+	}
+	if mean := s.Mean(); 4748.14 != mean {
+		t.Errorf("s.Mean(): 4748.14 != %v\n", mean)
+	}
+	if stdDev := s.StdDev(); 2826.684117548333 != stdDev {
+		t.Errorf("s.StdDev(): 2826.684117548333 != %v\n", stdDev)
+	}
+	ps := s.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 4599 != ps[0] {
+		t.Errorf("median: 4599 != %v\n", ps[0])
+	}
+	if 7380.5 != ps[1] {
+		t.Errorf("75th percentile: 7380.5 != %v\n", ps[1])
+	}
+	if 9986.429999999998 != ps[2] {
+		t.Errorf("99th percentile: 9986.429999999998 != %v\n", ps[2])
+	}
+}
+
+// TestUniformSampleConcurrentUpdateCount would expose data race problems with
+// concurrent Update and Count calls on Sample when test is called with -race
+// argument
+func TestUniformSampleConcurrentUpdateCount(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+	s := NewUniformSample(100)
+	for i := 0; i < 100; i++ {
+		s.Update(int64(i))
+	}
+	quit := make(chan struct{})
+	go func() {
+		t := time.NewTicker(10 * time.Millisecond)
+		for {
+			select {
+			case <-t.C:
+				s.Update(rand.Int63())
+			case <-quit:
+				t.Stop()
+				return
+			}
+		}
+	}()
+	for i := 0; i < 1000; i++ {
+		s.Count()
+		time.Sleep(5 * time.Millisecond)
+	}
+	quit <- struct{}{}
+}
--- a/pkg/metrics/settings.go
+++ b/pkg/metrics/settings.go
@@ -0,0 +1,46 @@
+package metrics
+
+import (
+	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+type MetricPublisher interface {
+	Publish(metrics []Metric)
+}
+
+type MetricSettings struct {
+	Enabled         bool
+	IntervalSeconds int64
+
+	Publishers []MetricPublisher
+}
+
+func readSettings() *MetricSettings {
+	var settings = &MetricSettings{
+		Enabled:    false,
+		Publishers: make([]MetricPublisher, 0),
+	}
+
+	var section, err = setting.Cfg.GetSection("metrics")
+	if err != nil {
+		log.Fatal(3, "Unable to find metrics config section")
+		return nil
+	}
+
+	settings.Enabled = section.Key("enabled").MustBool(false)
+	settings.IntervalSeconds = section.Key("interval_seconds").MustInt64(10)
+
+	if !settings.Enabled {
+		return settings
+	}
+
+	if graphitePublisher, err := CreateGraphitePublisher(); err != nil {
+		log.Error(3, "Metrics: Failed to init Graphite metric publisher", err)
+	} else if graphitePublisher != nil {
+		log.Info("Metrics: Graphite publisher initialized")
+		settings.Publishers = append(settings.Publishers, graphitePublisher)
+	}
+
+	return settings
+}
--- a/pkg/metrics/timer.go
+++ b/pkg/metrics/timer.go
@@ -0,0 +1,309 @@
+// includes code from
+// https://raw.githubusercontent.com/rcrowley/go-metrics/master/sample.go
+// Copyright 2012 Richard Crowley. All rights reserved.
+
+package metrics
+
+import (
+	"sync"
+	"time"
+)
+
+// Timers capture the duration and rate of events.
+type Timer interface {
+	Metric
+
+	Count() int64
+	Max() int64
+	Mean() float64
+	Min() int64
+	Percentile(float64) float64
+	Percentiles([]float64) []float64
+	Rate1() float64
+	Rate5() float64
+	Rate15() float64
+	RateMean() float64
+	StdDev() float64
+	Sum() int64
+	Time(func())
+	Update(time.Duration)
+	UpdateSince(time.Time)
+	Variance() float64
+}
+
+// NewCustomTimer constructs a new StandardTimer from a Histogram and a Meter.
+func NewCustomTimer(meta *MetricMeta, h Histogram, m Meter) Timer {
+	if UseNilMetrics {
+		return NilTimer{}
+	}
+	return &StandardTimer{
+		MetricMeta: meta,
+		histogram:  h,
+		meter:      m,
+	}
+}
+
+// NewTimer constructs a new StandardTimer using an exponentially-decaying
+// sample with the same reservoir size and alpha as UNIX load averages.
+func NewTimer(meta *MetricMeta) Timer {
+	if UseNilMetrics {
+		return NilTimer{}
+	}
+	return &StandardTimer{
+		MetricMeta: meta,
+		histogram:  NewHistogram(meta, NewExpDecaySample(1028, 0.015)),
+		meter:      NewMeter(meta),
+	}
+}
+
+func RegTimer(name string, tagStrings ...string) Timer {
+	tr := NewTimer(NewMetricMeta(name, tagStrings))
+	MetricStats.Register(tr)
+	return tr
+}
+
+// NilTimer is a no-op Timer.
+type NilTimer struct {
+	*MetricMeta
+	h Histogram
+	m Meter
+}
+
+// Count is a no-op.
+func (NilTimer) Count() int64 { return 0 }
+
+// Max is a no-op.
+func (NilTimer) Max() int64 { return 0 }
+
+// Mean is a no-op.
+func (NilTimer) Mean() float64 { return 0.0 }
+
+// Min is a no-op.
+func (NilTimer) Min() int64 { return 0 }
+
+// Percentile is a no-op.
+func (NilTimer) Percentile(p float64) float64 { return 0.0 }
+
+// Percentiles is a no-op.
+func (NilTimer) Percentiles(ps []float64) []float64 {
+	return make([]float64, len(ps))
+}
+
+// Rate1 is a no-op.
+func (NilTimer) Rate1() float64 { return 0.0 }
+
+// Rate5 is a no-op.
+func (NilTimer) Rate5() float64 { return 0.0 }
+
+// Rate15 is a no-op.
+func (NilTimer) Rate15() float64 { return 0.0 }
+
+// RateMean is a no-op.
+func (NilTimer) RateMean() float64 { return 0.0 }
+
+// Snapshot is a no-op.
+func (n NilTimer) Snapshot() Metric { return n }
+
+// StdDev is a no-op.
+func (NilTimer) StdDev() float64 { return 0.0 }
+
+// Sum is a no-op.
+func (NilTimer) Sum() int64 { return 0 }
+
+// Time is a no-op.
+func (NilTimer) Time(func()) {}
+
+// Update is a no-op.
+func (NilTimer) Update(time.Duration) {}
+
+// UpdateSince is a no-op.
+func (NilTimer) UpdateSince(time.Time) {}
+
+// Variance is a no-op.
+func (NilTimer) Variance() float64 { return 0.0 }
+
+// StandardTimer is the standard implementation of a Timer and uses a Histogram
+// and Meter.
+type StandardTimer struct {
+	*MetricMeta
+	histogram Histogram
+	meter     Meter
+	mutex     sync.Mutex
+}
+
+// Count returns the number of events recorded.
+func (t *StandardTimer) Count() int64 {
+	return t.histogram.Count()
+}
+
+// Max returns the maximum value in the sample.
+func (t *StandardTimer) Max() int64 {
+	return t.histogram.Max()
+}
+
+// Mean returns the mean of the values in the sample.
+func (t *StandardTimer) Mean() float64 {
+	return t.histogram.Mean()
+}
+
+// Min returns the minimum value in the sample.
+func (t *StandardTimer) Min() int64 {
+	return t.histogram.Min()
+}
+
+// Percentile returns an arbitrary percentile of the values in the sample.
+func (t *StandardTimer) Percentile(p float64) float64 {
+	return t.histogram.Percentile(p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of the values in the
+// sample.
+func (t *StandardTimer) Percentiles(ps []float64) []float64 {
+	return t.histogram.Percentiles(ps)
+}
+
+// Rate1 returns the one-minute moving average rate of events per second.
+func (t *StandardTimer) Rate1() float64 {
+	return t.meter.Rate1()
+}
+
+// Rate5 returns the five-minute moving average rate of events per second.
+func (t *StandardTimer) Rate5() float64 {
+	return t.meter.Rate5()
+}
+
+// Rate15 returns the fifteen-minute moving average rate of events per second.
+func (t *StandardTimer) Rate15() float64 {
+	return t.meter.Rate15()
+}
+
+// RateMean returns the meter's mean rate of events per second.
+func (t *StandardTimer) RateMean() float64 {
+	return t.meter.RateMean()
+}
+
+// Snapshot returns a read-only copy of the timer.
+func (t *StandardTimer) Snapshot() Metric {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	return &TimerSnapshot{
+		MetricMeta: t.MetricMeta,
+		histogram:  t.histogram.Snapshot().(*HistogramSnapshot),
+		meter:      t.meter.Snapshot().(*MeterSnapshot),
+	}
+}
+
+// StdDev returns the standard deviation of the values in the sample.
+func (t *StandardTimer) StdDev() float64 {
+	return t.histogram.StdDev()
+}
+
+// Sum returns the sum in the sample.
+func (t *StandardTimer) Sum() int64 {
+	return t.histogram.Sum()
+}
+
+// Record the duration of the execution of the given function.
+func (t *StandardTimer) Time(f func()) {
+	ts := time.Now()
+	f()
+	t.Update(time.Since(ts))
+}
+
+// Record the duration of an event.
+func (t *StandardTimer) Update(d time.Duration) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	t.histogram.Update(int64(d))
+	t.meter.Mark(1)
+}
+
+// Record the duration of an event that started at a time and ends now.
+func (t *StandardTimer) UpdateSince(ts time.Time) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	t.histogram.Update(int64(time.Since(ts)))
+	t.meter.Mark(1)
+}
+
+// Variance returns the variance of the values in the sample.
+func (t *StandardTimer) Variance() float64 {
+	return t.histogram.Variance()
+}
+
+// TimerSnapshot is a read-only copy of another Timer.
+type TimerSnapshot struct {
+	*MetricMeta
+	histogram *HistogramSnapshot
+	meter     *MeterSnapshot
+}
+
+// Count returns the number of events recorded at the time the snapshot was
+// taken.
+func (t *TimerSnapshot) Count() int64 { return t.histogram.Count() }
+
+// Max returns the maximum value at the time the snapshot was taken.
+func (t *TimerSnapshot) Max() int64 { return t.histogram.Max() }
+
+// Mean returns the mean value at the time the snapshot was taken.
+func (t *TimerSnapshot) Mean() float64 { return t.histogram.Mean() }
+
+// Min returns the minimum value at the time the snapshot was taken.
+func (t *TimerSnapshot) Min() int64 { return t.histogram.Min() }
+
+// Percentile returns an arbitrary percentile of sampled values at the time the
+// snapshot was taken.
+func (t *TimerSnapshot) Percentile(p float64) float64 {
+	return t.histogram.Percentile(p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of sampled values at
+// the time the snapshot was taken.
+func (t *TimerSnapshot) Percentiles(ps []float64) []float64 {
+	return t.histogram.Percentiles(ps)
+}
+
+// Rate1 returns the one-minute moving average rate of events per second at the
+// time the snapshot was taken.
+func (t *TimerSnapshot) Rate1() float64 { return t.meter.Rate1() }
+
+// Rate5 returns the five-minute moving average rate of events per second at
+// the time the snapshot was taken.
+func (t *TimerSnapshot) Rate5() float64 { return t.meter.Rate5() }
+
+// Rate15 returns the fifteen-minute moving average rate of events per second
+// at the time the snapshot was taken.
+func (t *TimerSnapshot) Rate15() float64 { return t.meter.Rate15() }
+
+// RateMean returns the meter's mean rate of events per second at the time the
+// snapshot was taken.
+func (t *TimerSnapshot) RateMean() float64 { return t.meter.RateMean() }
+
+// Snapshot returns the snapshot.
+func (t *TimerSnapshot) Snapshot() Metric { return t }
+
+// StdDev returns the standard deviation of the values at the time the snapshot
+// was taken.
+func (t *TimerSnapshot) StdDev() float64 { return t.histogram.StdDev() }
+
+// Sum returns the sum at the time the snapshot was taken.
+func (t *TimerSnapshot) Sum() int64 { return t.histogram.Sum() }
+
+// Time panics.
+func (*TimerSnapshot) Time(func()) {
+	panic("Time called on a TimerSnapshot")
+}
+
+// Update panics.
+func (*TimerSnapshot) Update(time.Duration) {
+	panic("Update called on a TimerSnapshot")
+}
+
+// UpdateSince panics.
+func (*TimerSnapshot) UpdateSince(time.Time) {
+	panic("UpdateSince called on a TimerSnapshot")
+}
+
+// Variance returns the variance of the values at the time the snapshot was
+// taken.
+func (t *TimerSnapshot) Variance() float64 { return t.histogram.Variance() }
--- a/pkg/middleware/logger.go
+++ b/pkg/middleware/logger.go
@@ -21,6 +21,7 @@ import (
 	"time"

 	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/metrics"
 	"github.com/grafana/grafana/pkg/setting"
 	"gopkg.in/macaron.v1"
 )
@@ -28,6 +29,7 @@ import (
 func Logger() macaron.Handler {
 	return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) {
 		start := time.Now()
+		c.Data["perfmon.start"] = start

 		uname := c.GetCookie(setting.CookieUserName)
 		if len(uname) == 0 {
@@ -37,7 +39,13 @@ func Logger() macaron.Handler {
 		rw := res.(macaron.ResponseWriter)
 		c.Next()

-		content := fmt.Sprintf("Completed %s %s \"%s %s %s\" %v %s %d bytes in %dus", c.RemoteAddr(), uname, req.Method, req.URL.Path, req.Proto, rw.Status(), http.StatusText(rw.Status()), rw.Size(), time.Since(start)/time.Microsecond)
+		timeTakenMs := time.Since(start) / time.Millisecond
+		content := fmt.Sprintf("Completed %s %s \"%s %s %s\" %v %s %d bytes in %dms", c.RemoteAddr(), uname, req.Method, req.URL.Path, req.Proto, rw.Status(), http.StatusText(rw.Status()), rw.Size(), timeTakenMs)
+
+		if timer, ok := c.Data["perfmon.timer"]; ok {
+			timerTyped := timer.(metrics.Timer)
+			timerTyped.Update(timeTakenMs)
+		}

 		switch rw.Status() {
 		case 200, 304:
--- a/pkg/middleware/middleware.go
+++ b/pkg/middleware/middleware.go
@@ -257,3 +257,7 @@ func (ctx *Context) JsonApiErr(status int, message string, err error) {
 func (ctx *Context) HasUserRole(role m.RoleType) bool {
 	return ctx.OrgRole.Includes(role)
 }
+
+func (ctx *Context) TimeRequest(timer metrics.Timer) {
+	ctx.Data["perfmon.timer"] = timer
+}
--- a/pkg/middleware/perf.go
+++ b/pkg/middleware/perf.go
@@ -0,0 +1,12 @@
+package middleware
+
+import (
+	"net/http"
+
+	"gopkg.in/macaron.v1"
+)
+
+func MeasureRequestTime() macaron.Handler {
+	return func(res http.ResponseWriter, req *http.Request, c *Context) {
+	}
+}
--- a/pkg/setting/setting.go
+++ b/pkg/setting/setting.go
@@ -37,9 +37,10 @@ const (

 var (
 	// App settings.
-	Env       string = DEV
-	AppUrl    string
-	AppSubUrl string
+	Env          string = DEV
+	AppUrl       string
+	AppSubUrl    string
+	InstanceName string

 	// build
 	BuildVersion string
@@ -262,6 +263,12 @@ func evalEnvVarExpression(value string) string {
 		envVar = strings.TrimPrefix(envVar, "${")
 		envVar = strings.TrimSuffix(envVar, "}")
 		envValue := os.Getenv(envVar)
+
+		// if env variable is hostname and it is emtpy use os.Hostname as default
+		if envVar == "HOSTNAME" && envValue == "" {
+			envValue, _ = os.Hostname()
+		}
+
 		return envValue
 	})
 }
@@ -398,11 +405,28 @@ func validateStaticRootPath() error {
 	return fmt.Errorf("Failed to detect generated css or javascript files in static root (%s), have you executed default grunt task?", StaticRootPath)
 }

+// func readInstanceName() string {
+// 	hostname, _ := os.Hostname()
+// 	if hostname == "" {
+// 		hostname = "hostname_unknown"
+// 	}
+//
+// 	instanceName := Cfg.Section("").Key("instance_name").MustString("")
+// 	if instanceName = "" {
+// 		// set value as it might be used in other places
+// 		Cfg.Section("").Key("instance_name").SetValue(hostname)
+// 		instanceName = hostname
+// 	}
+//
+// 	return
+// }
+
 func NewConfigContext(args *CommandLineArgs) error {
 	setHomePath(args)
 	loadConfiguration(args)

 	Env = Cfg.Section("").Key("app_mode").MustString("development")
+	InstanceName = Cfg.Section("").Key("instance_name").MustString("unknown_instance_name")
 	PluginsPath = Cfg.Section("paths").Key("plugins").String()

 	server := Cfg.Section("server")
--- a/pkg/setting/setting_test.go
+++ b/pkg/setting/setting_test.go
@@ -89,5 +89,14 @@ func TestLoadingSettings(t *testing.T) {
 			So(DataPath, ShouldEqual, "/tmp/env_override")
 		})

+		Convey("instance_name default to hostname even if hostname env is emtpy", func() {
+			NewConfigContext(&CommandLineArgs{
+				HomePath: "../../",
+			})
+
+			hostname, _ := os.Hostname()
+			So(InstanceName, ShouldEqual, hostname)
+		})
+
 	})
 }
--- a/pkg/util/strings.go
+++ b/pkg/util/strings.go
@@ -1,18 +1,18 @@
 package util

 func StringsFallback2(val1 string, val2 string) string {
-	if val1 != "" {
-		return val1
-	}
-	return val2
+	return stringsFallback(val1, val2)
 }

 func StringsFallback3(val1 string, val2 string, val3 string) string {
-	if val1 != "" {
-		return val1
-	}
-	if val2 != "" {
-		return val2
-	}
-	return val3
+	return stringsFallback(val1, val2, val3)
+}
+
+func stringsFallback(vals ...string) string {
+	for _, v := range vals {
+		if v != "" {
+			return v
+		}
+	}
+	return ""
 }
--- a/pkg/util/strings_test.go
+++ b/pkg/util/strings_test.go
@@ -0,0 +1,15 @@
+package util
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestStringsUtil(t *testing.T) {
+	Convey("Falling back until none empty string", t, func() {
+		So(StringsFallback2("1", "2"), ShouldEqual, "1")
+		So(StringsFallback2("", "2"), ShouldEqual, "2")
+		So(StringsFallback3("", "", "3"), ShouldEqual, "3")
+	})
+}