From 4f9fbcc21141b9c2e5197f382511963cadc45288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 23 Aug 2017 13:31:26 +0200 Subject: [PATCH] dataproxy: added caching of datasources when doing data proxy requests, #9078 --- pkg/api/api.go | 4 +- pkg/api/dataproxy.go | 25 +++- pkg/api/http_server.go | 6 +- pkg/metrics/metrics.go | 3 + pkg/services/sqlstore/datasource.go | 3 + public/app/core/services/backend_srv.ts | 16 ++- public/app/features/plugins/ds_edit_ctrl.ts | 38 +++--- .../patrickmn/go-cache/CONTRIBUTORS | 1 + vendor/github.com/patrickmn/go-cache/LICENSE | 2 +- .../github.com/patrickmn/go-cache/README.md | 126 +++++++----------- vendor/github.com/patrickmn/go-cache/cache.go | 32 ++++- vendor/vendor.json | 6 +- 12 files changed, 155 insertions(+), 107 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 846e3328ddd..d4d23b17ef4 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -217,8 +217,8 @@ func (hs *HttpServer) registerRoutes() { }, reqOrgAdmin) r.Get("/frontend/settings/", GetFrontendSettings) - r.Any("/datasources/proxy/:id/*", reqSignedIn, ProxyDataSourceRequest) - r.Any("/datasources/proxy/:id", reqSignedIn, ProxyDataSourceRequest) + r.Any("/datasources/proxy/:id/*", reqSignedIn, hs.ProxyDataSourceRequest) + r.Any("/datasources/proxy/:id", reqSignedIn, hs.ProxyDataSourceRequest) // Dashboard r.Group("/dashboards", func() { diff --git a/pkg/api/dataproxy.go b/pkg/api/dataproxy.go index a404c790772..8a712f99804 100644 --- a/pkg/api/dataproxy.go +++ b/pkg/api/dataproxy.go @@ -1,6 +1,9 @@ package api import ( + "fmt" + "time" + "github.com/grafana/grafana/pkg/api/pluginproxy" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/metrics" @@ -9,19 +12,35 @@ import ( "github.com/grafana/grafana/pkg/plugins" ) -func getDatasource(id int64, orgId int64) (*m.DataSource, error) { +const HeaderNameNoBackendCache = "X-Grafana-NoCache" + +func (hs *HttpServer) getDatasourceById(id int64, orgId int64, nocache bool) (*m.DataSource, error) { + cacheKey := fmt.Sprintf("ds-%d", id) + + if !nocache { + if cached, found := hs.cache.Get(cacheKey); found { + ds := cached.(*m.DataSource) + if ds.OrgId == orgId { + return ds, nil + } + } + } + query := m.GetDataSourceByIdQuery{Id: id, OrgId: orgId} if err := bus.Dispatch(&query); err != nil { return nil, err } + hs.cache.Set(cacheKey, query.Result, time.Second*5) return query.Result, nil } -func ProxyDataSourceRequest(c *middleware.Context) { +func (hs *HttpServer) ProxyDataSourceRequest(c *middleware.Context) { c.TimeRequest(metrics.M_DataSource_ProxyReq_Timer) - ds, err := getDatasource(c.ParamsInt64(":id"), c.OrgId) + nocache := c.Req.Header.Get(HeaderNameNoBackendCache) == "true" + + ds, err := hs.getDatasourceById(c.ParamsInt64(":id"), c.OrgId, nocache) if err != nil { c.JsonApiErr(500, "Unable to load datasource meta data", err) diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index 4873062a933..54f08197bae 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -9,7 +9,9 @@ import ( "net/http" "os" "path" + "time" + gocache "github.com/patrickmn/go-cache" macaron "gopkg.in/macaron.v1" "github.com/grafana/grafana/pkg/api/live" @@ -29,13 +31,15 @@ type HttpServer struct { macaron *macaron.Macaron context context.Context streamManager *live.StreamManager + cache *gocache.Cache httpSrv *http.Server } func NewHttpServer() *HttpServer { return &HttpServer{ - log: log.New("http.server"), + log: log.New("http.server"), + cache: gocache.New(5*time.Minute, 10*time.Minute), } } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 00354a00d03..dc0f7e9cabe 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -55,6 +55,7 @@ var ( M_Alerting_Notification_Sent_Pushover Counter M_Aws_CloudWatch_GetMetricStatistics Counter M_Aws_CloudWatch_ListMetrics Counter + M_DB_DataSource_QueryById Counter // Timers M_DataSource_ProxyReq_Timer Timer @@ -130,6 +131,8 @@ func initMetricVars(settings *MetricSettings) { M_Aws_CloudWatch_GetMetricStatistics = RegCounter("aws.cloudwatch.get_metric_statistics") M_Aws_CloudWatch_ListMetrics = RegCounter("aws.cloudwatch.list_metrics") + M_DB_DataSource_QueryById = RegCounter("db.datasource.query_by_id") + // Timers M_DataSource_ProxyReq_Timer = RegTimer("api.dataproxy.request.all") M_Alerting_Execution_Time = RegTimer("alerting.execution_time") diff --git a/pkg/services/sqlstore/datasource.go b/pkg/services/sqlstore/datasource.go index 831bead2360..5d8e0e049ae 100644 --- a/pkg/services/sqlstore/datasource.go +++ b/pkg/services/sqlstore/datasource.go @@ -5,6 +5,7 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/securejsondata" + "github.com/grafana/grafana/pkg/metrics" m "github.com/grafana/grafana/pkg/models" ) @@ -19,6 +20,8 @@ func init() { } func GetDataSourceById(query *m.GetDataSourceByIdQuery) error { + metrics.M_DB_DataSource_QueryById.Inc(1) + datasource := m.DataSource{OrgId: query.OrgId, Id: query.Id} has, err := x.Get(&datasource) diff --git a/public/app/core/services/backend_srv.ts b/public/app/core/services/backend_srv.ts index bffdfa05914..b753ae98530 100644 --- a/public/app/core/services/backend_srv.ts +++ b/public/app/core/services/backend_srv.ts @@ -7,8 +7,9 @@ import coreModule from 'app/core/core_module'; import appEvents from 'app/core/app_events'; export class BackendSrv { - inFlightRequests = {}; - HTTP_REQUEST_CANCELLED = -1; + private inFlightRequests = {}; + private HTTP_REQUEST_CANCELLED = -1; + private noBackendCache: boolean; /** @ngInject */ constructor(private $http, private alertSrv, private $rootScope, private $q, private $timeout, private contextSrv) { @@ -34,6 +35,13 @@ export class BackendSrv { return this.request({ method: 'PUT', url: url, data: data }); } + withNoBackendCache(callback) { + this.noBackendCache = true; + return callback().finally(() => { + this.noBackendCache = false; + }); + } + requestErrorHandler(err) { if (err.isHandled) { return; @@ -149,6 +157,10 @@ export class BackendSrv { options.headers['X-DS-Authorization'] = options.headers.Authorization; delete options.headers.Authorization; } + + if (this.noBackendCache) { + options.headers['X-Grafana-NoCache'] = 'true'; + } } return this.$http(options).then(response => { diff --git a/public/app/features/plugins/ds_edit_ctrl.ts b/public/app/features/plugins/ds_edit_ctrl.ts index b577ad3131b..d2b871a0a12 100644 --- a/public/app/features/plugins/ds_edit_ctrl.ts +++ b/public/app/features/plugins/ds_edit_ctrl.ts @@ -111,31 +111,31 @@ export class DataSourceEditCtrl { } testDatasource() { - this.testing = { done: false }; - this.datasourceSrv.get(this.current.name).then(datasource => { if (!datasource.testDatasource) { - delete this.testing; return; } - return datasource.testDatasource().then(result => { - this.testing.message = result.message; - this.testing.status = result.status; - this.testing.title = result.title; - }).catch(err => { - if (err.statusText) { - this.testing.message = err.statusText; - this.testing.title = "HTTP Error"; - } else { - this.testing.message = err.message; - this.testing.title = "Unknown error"; - } - }); - }).finally(() => { - if (this.testing) { + this.testing = {done: false}; + + // make test call in no backend cache context + this.backendSrv.withNoBackendCache(() => { + return datasource.testDatasource().then(result => { + this.testing.message = result.message; + this.testing.status = result.status; + this.testing.title = result.title; + }).catch(err => { + if (err.statusText) { + this.testing.message = err.statusText; + this.testing.title = "HTTP Error"; + } else { + this.testing.message = err.message; + this.testing.title = "Unknown error"; + } + }); + }).finally(() => { this.testing.done = true; - } + }); }); } diff --git a/vendor/github.com/patrickmn/go-cache/CONTRIBUTORS b/vendor/github.com/patrickmn/go-cache/CONTRIBUTORS index 8a4da4ed294..2b16e997415 100644 --- a/vendor/github.com/patrickmn/go-cache/CONTRIBUTORS +++ b/vendor/github.com/patrickmn/go-cache/CONTRIBUTORS @@ -6,3 +6,4 @@ code was contributed.) Dustin Sallings Jason Mooberry Sergey Shepelev +Alex Edwards diff --git a/vendor/github.com/patrickmn/go-cache/LICENSE b/vendor/github.com/patrickmn/go-cache/LICENSE index f9fe27156a5..db9903c75c5 100644 --- a/vendor/github.com/patrickmn/go-cache/LICENSE +++ b/vendor/github.com/patrickmn/go-cache/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012-2016 Patrick Mylund Nielsen and the go-cache contributors +Copyright (c) 2012-2017 Patrick Mylund Nielsen and the go-cache contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/vendor/github.com/patrickmn/go-cache/README.md b/vendor/github.com/patrickmn/go-cache/README.md index 168ff7ba5fc..c5789cc66cc 100644 --- a/vendor/github.com/patrickmn/go-cache/README.md +++ b/vendor/github.com/patrickmn/go-cache/README.md @@ -20,86 +20,62 @@ one) to recover from downtime quickly. (See the docs for `NewFrom()` for caveats ### Usage ```go - import ( - "fmt" - "github.com/patrickmn/go-cache" - "time" - ) +import ( + "fmt" + "github.com/patrickmn/go-cache" + "time" +) - func main() { +func main() { + // Create a cache with a default expiration time of 5 minutes, and which + // purges expired items every 10 minutes + c := cache.New(5*time.Minute, 10*time.Minute) - // Create a cache with a default expiration time of 5 minutes, and which - // purges expired items every 30 seconds - c := cache.New(5*time.Minute, 30*time.Second) + // Set the value of the key "foo" to "bar", with the default expiration time + c.Set("foo", "bar", cache.DefaultExpiration) - // Set the value of the key "foo" to "bar", with the default expiration time - c.Set("foo", "bar", cache.DefaultExpiration) - - // Set the value of the key "baz" to 42, with no expiration time - // (the item won't be removed until it is re-set, or removed using - // c.Delete("baz") - c.Set("baz", 42, cache.NoExpiration) - - // Get the string associated with the key "foo" from the cache - foo, found := c.Get("foo") - if found { - fmt.Println(foo) - } - - // Since Go is statically typed, and cache values can be anything, type - // assertion is needed when values are being passed to functions that don't - // take arbitrary types, (i.e. interface{}). The simplest way to do this for - // values which will only be used once--e.g. for passing to another - // function--is: - foo, found := c.Get("foo") - if found { - MyFunction(foo.(string)) - } - - // This gets tedious if the value is used several times in the same function. - // You might do either of the following instead: - if x, found := c.Get("foo"); found { - foo := x.(string) - // ... - } - // or - var foo string - if x, found := c.Get("foo"); found { - foo = x.(string) - } - // ... - // foo can then be passed around freely as a string - - // Want performance? Store pointers! - c.Set("foo", &MyStruct, cache.DefaultExpiration) - if x, found := c.Get("foo"); found { - foo := x.(*MyStruct) - // ... - } - - // If you store a reference type like a pointer, slice, map or channel, you - // do not need to run Set if you modify the underlying data. The cached - // reference points to the same memory, so if you modify a struct whose - // pointer you've stored in the cache, retrieving that pointer with Get will - // point you to the same data: - foo := &MyStruct{Num: 1} - c.Set("foo", foo, cache.DefaultExpiration) - // ... - x, _ := c.Get("foo") - foo := x.(*MyStruct) - fmt.Println(foo.Num) - // ... - foo.Num++ - // ... - x, _ := c.Get("foo") - foo := x.(*MyStruct) - foo.Println(foo.Num) - - // will print: - // 1 - // 2 + // Set the value of the key "baz" to 42, with no expiration time + // (the item won't be removed until it is re-set, or removed using + // c.Delete("baz") + c.Set("baz", 42, cache.NoExpiration) + // Get the string associated with the key "foo" from the cache + foo, found := c.Get("foo") + if found { + fmt.Println(foo) } + + // Since Go is statically typed, and cache values can be anything, type + // assertion is needed when values are being passed to functions that don't + // take arbitrary types, (i.e. interface{}). The simplest way to do this for + // values which will only be used once--e.g. for passing to another + // function--is: + foo, found := c.Get("foo") + if found { + MyFunction(foo.(string)) + } + + // This gets tedious if the value is used several times in the same function. + // You might do either of the following instead: + if x, found := c.Get("foo"); found { + foo := x.(string) + // ... + } + // or + var foo string + if x, found := c.Get("foo"); found { + foo = x.(string) + } + // ... + // foo can then be passed around freely as a string + + // Want performance? Store pointers! + c.Set("foo", &MyStruct, cache.DefaultExpiration) + if x, found := c.Get("foo"); found { + foo := x.(*MyStruct) + // ... + } +} ``` ### Reference diff --git a/vendor/github.com/patrickmn/go-cache/cache.go b/vendor/github.com/patrickmn/go-cache/cache.go index 70e4dad337a..db88d2f2cb1 100644 --- a/vendor/github.com/patrickmn/go-cache/cache.go +++ b/vendor/github.com/patrickmn/go-cache/cache.go @@ -135,6 +135,36 @@ func (c *cache) Get(k string) (interface{}, bool) { return item.Object, true } +// GetWithExpiration returns an item and its expiration time from the cache. +// It returns the item or nil, the expiration time if one is set (if the item +// never expires a zero value for time.Time is returned), and a bool indicating +// whether the key was found. +func (c *cache) GetWithExpiration(k string) (interface{}, time.Time, bool) { + c.mu.RLock() + // "Inlining" of get and Expired + item, found := c.items[k] + if !found { + c.mu.RUnlock() + return nil, time.Time{}, false + } + + if item.Expiration > 0 { + if time.Now().UnixNano() > item.Expiration { + c.mu.RUnlock() + return nil, time.Time{}, false + } + + // Return the item and the expiration time + c.mu.RUnlock() + return item.Object, time.Unix(0, item.Expiration), true + } + + // If expiration <= 0 (i.e. no expiration time set) then return the item + // and a zeroed time.Time + c.mu.RUnlock() + return item.Object, time.Time{}, true +} + func (c *cache) get(k string) (interface{}, bool) { item, found := c.items[k] if !found { @@ -1044,7 +1074,6 @@ type janitor struct { } func (j *janitor) Run(c *cache) { - j.stop = make(chan bool) ticker := time.NewTicker(j.Interval) for { select { @@ -1064,6 +1093,7 @@ func stopJanitor(c *Cache) { func runJanitor(c *cache, ci time.Duration) { j := &janitor{ Interval: ci, + stop: make(chan bool), } c.janitor = j go j.Run(c) diff --git a/vendor/vendor.json b/vendor/vendor.json index 07190f3fe94..5f74f2b3359 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -455,10 +455,10 @@ "revisionTime": "2017-07-07T05:36:02Z" }, { - "checksumSHA1": "8z32QKTSDusa4QQyunKE4kyYXZ8=", + "checksumSHA1": "JVGDxPn66bpe6xEiexs1r+y6jF0=", "path": "github.com/patrickmn/go-cache", - "revision": "e7a9def80f35fe1b170b7b8b68871d59dea117e1", - "revisionTime": "2016-11-25T23:48:19Z" + "revision": "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0", + "revisionTime": "2017-07-22T04:01:10Z" }, { "checksumSHA1": "SMUvX2B8eoFd9wnPofwBKlN6btE=",