Logging: rate limit fronted logging endpoint (#29272)

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

conf/defaults.ini

@@ -564,18 +564,24 @@ facility =
 tag =
 
 [log.frontend]
-# Should Sentry be initialized
+# Should Sentry javascript agent be initialized
 enabled = false
 
-# Sentry DSN if you wanna send events to Sentry. In this case, set custom_endpoint to empty
+# Sentry DSN if you want to send events to Sentry.
 sentry_dsn =
 
-# Custom endpoint to send Sentry events to. If this is configured, DSN will be ignored and events push to this endpoint. Default endpoint will log frontend errors to stdout.
+# Custom HTTP endpoint to send events captured by the Sentry agent to. Default will log the events to stdout.
 custom_endpoint = /log
 
 # Rate of events to be reported between 0 (none) and 1 (all), float
 sample_rate = 1.0
 
+# Requests per second limit enforced per an extended period, for Grafana backend log ingestion endpoint (/log).
+log_endpoint_requests_per_second_limit = 3
+
+# Max requests accepted per short interval of time for Grafana backend log ingestion endpoint (/log)
+log_endpoint_burst_limit = 15
+
 #################################### Usage Quotas ########################
 [quota]
 enabled = false

conf/sample.ini

@@ -555,18 +555,24 @@
 ;tag =
 
 [log.frontend]
-# Should Sentry be initialized
+# Should Sentry javascript agent be initialized
 ;enabled = false
 
-# Sentry DSN if you wanna send events to Sentry. In this case, set custom_endpoint to empty
+# Sentry DSN if you want to send events to Sentry.
 ;sentry_dsn =
 
-# Custom endpoint to send Sentry events to. If this is configured, DSN will be ignored and events push to this endpoint. Default endpoint will log frontend errors to stdout.
+# Custom HTTP endpoint to send events captured by the Sentry agent to. Default will log the events to stdout.
 ;custom_endpoint = /log
 
 # Rate of events to be reported between 0 (none) and 1 (all), float
 ;sample_rate = 1.0
 
+# Requests per second limit enforced an extended period, for Grafana backend log ingestion endpoint (/log).
+;log_endpoint_requests_per_second_limit = 3
+
+# Max requests accepted per short interval of time for Grafana backend log ingestion endpoint (/log).
+;log_endpoint_burst_limit = 15
+
 #################################### Usage Quotas ########################
 [quota]
 ; enabled = false

docs/sources/administration/configuration.md

@@ -882,6 +882,8 @@ Enable daily rotation of files, valid options are `false` or `true`. Default is
 
 Maximum number of days to keep log files. Default is `7`.
 
+<hr>
+
 ## [log.syslog]
 
 Only applicable when "syslog" used in `[log]` mode.
@@ -908,6 +910,34 @@ Syslog tag. By default, the process's `argv[0]` is used.
 
 <hr>
 
+## [log.frontend]
+
+### enabled
+
+Sentry javascript agent is initialized. Default is `false`.
+
+### sentry_dsn
+
+Sentry DSN if you want to send events to Sentry
+
+### custom_endpoint
+
+Custom HTTP endpoint to send events captured by the Sentry agent to. Default, `/log`, will log the events to stdout.
+
+### sample_rate
+
+Rate of events to be reported between `0` (none) and `1` (all, default), float.
+
+### log_endpoint_requests_per_second_limit
+
+Requests per second limit enforced per an extended period, for Grafana backend log ingestion endpoint, `/log`. Default is `3`.
+
+### log_endpoint_burst_limit
+
+Maximum requests accepted per short interval of time for Grafana backend log ingestion endpoint, `/log`. Default is `15`.
+
+<hr>
+
 ## [quota]
 
 Set quotas to `-1` to make unlimited.

pkg/api/api.go

@@ -1,6 +1,8 @@
 package api
 
 import (
+	"time"
+
 	"github.com/go-macaron/binding"
 	"github.com/grafana/grafana/pkg/api/avatar"
 	"github.com/grafana/grafana/pkg/api/dtos"
@@ -440,5 +442,5 @@ func (hs *HTTPServer) registerRoutes() {
 	r.Delete("/api/snapshots/:key", reqEditorRole, Wrap(DeleteDashboardSnapshot))
 
 	// Frontend logs
-	r.Post("/log", bind(frontendSentryEvent{}), Wrap(hs.logFrontendMessage))
+	r.Post("/log", middleware.RateLimit(hs.Cfg.Sentry.EndpointRPS, hs.Cfg.Sentry.EndpointBurst, time.Now), bind(frontendSentryEvent{}), Wrap(hs.logFrontendMessage))
 }

pkg/middleware/rate_limit.go

@@ -0,0 +1,25 @@
+package middleware
+
+import (
+	"time"
+
+	"golang.org/x/time/rate"
+	"gopkg.in/macaron.v1"
+
+	"github.com/grafana/grafana/pkg/models"
+)
+
+type getTimeFn func() time.Time
+
+// RateLimit is a very basic rate limiter.
+// Will allow average of "rps" requests per second over an extended period of time, with max "burst" requests at the same time.
+// getTime should return the current time. For non-testing purposes use time.Now
+func RateLimit(rps, burst int, getTime getTimeFn) macaron.Handler {
+	l := rate.NewLimiter(rate.Limit(rps), burst)
+	return func(c *models.ReqContext) {
+		if !l.AllowN(getTime(), 1) {
+			c.JsonApiErr(429, "Rate limit reached", nil)
+			return
+		}
+	}
+}

pkg/middleware/rate_limit_test.go

@@ -0,0 +1,86 @@
+package middleware
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/grafana/grafana/pkg/models"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/macaron.v1"
+)
+
+type execFunc func() *httptest.ResponseRecorder
+type advanceTimeFunc func(deltaTime time.Duration)
+type rateLimiterScenarioFunc func(c execFunc, t advanceTimeFunc)
+
+func rateLimiterScenario(t *testing.T, desc string, rps int, burst int, fn rateLimiterScenarioFunc) {
+	t.Run(desc, func(t *testing.T) {
+		defaultHandler := func(c *models.ReqContext) {
+			resp := make(map[string]interface{})
+			resp["message"] = "OK"
+			c.JSON(200, resp)
+		}
+		currentTime := time.Now()
+
+		m := macaron.New()
+		m.Use(macaron.Renderer(macaron.RenderOptions{
+			Directory: "",
+			Delims:    macaron.Delims{Left: "[[", Right: "]]"},
+		}))
+		m.Use(GetContextHandler(nil, nil, nil))
+		m.Get("/foo", RateLimit(rps, burst, func() time.Time { return currentTime }), defaultHandler)
+
+		fn(func() *httptest.ResponseRecorder {
+			resp := httptest.NewRecorder()
+			req, err := http.NewRequest("GET", "/foo", nil)
+			require.NoError(t, err)
+			m.ServeHTTP(resp, req)
+			return resp
+		}, func(deltaTime time.Duration) {
+			currentTime = currentTime.Add(deltaTime)
+		})
+	})
+}
+
+func TestRateLimitMiddleware(t *testing.T) {
+	rateLimiterScenario(t, "rate limit calls, with burst", 10, 10, func(doReq execFunc, advanceTime advanceTimeFunc) {
+		// first 10 calls succeed
+		for i := 0; i < 10; i++ {
+			resp := doReq()
+			assert.Equal(t, 200, resp.Code)
+		}
+
+		// next one fails
+		resp := doReq()
+		assert.Equal(t, 429, resp.Code)
+
+		// check that requests are accepted again in 1 sec
+		advanceTime(1 * time.Second)
+
+		for i := 0; i < 10; i++ {
+			resp := doReq()
+			assert.Equal(t, 200, resp.Code)
+		}
+	})
+
+	rateLimiterScenario(t, "rate limit calls, no burst", 10, 1, func(doReq execFunc, advanceTime advanceTimeFunc) {
+		// first calls succeeds
+		resp := doReq()
+		assert.Equal(t, 200, resp.Code)
+
+		// immediately fired next one fails
+		resp = doReq()
+		assert.Equal(t, 429, resp.Code)
+
+		// but spacing calls out works
+		for i := 0; i < 10; i++ {
+			advanceTime(100 * time.Millisecond)
+			resp := doReq()
+			assert.Equal(t, 200, resp.Code)
+		}
+	})
+}

pkg/setting/setting_sentry.go

@@ -5,6 +5,8 @@ type Sentry struct {
 	DSN            string  `json:"dsn"`
 	CustomEndpoint string  `json:"customEndpoint"`
 	SampleRate     float64 `json:"sampleRate"`
+	EndpointRPS    int     `json:"-"`
+	EndpointBurst  int     `json:"-"`
 }
 
 func (cfg *Cfg) readSentryConfig() {
@@ -14,5 +16,7 @@ func (cfg *Cfg) readSentryConfig() {
 		DSN:            raw.Key("sentry_dsn").String(),
 		CustomEndpoint: raw.Key("custom_endpoint").String(),
 		SampleRate:     raw.Key("sample_rate").MustFloat64(),
+		EndpointRPS:    raw.Key("log_endpoint_requests_per_second_limit").MustInt(),
+		EndpointBurst:  raw.Key("log_endpoint_burst_limit").MustInt(),
 	}
 }

public/app/core/services/echo/backends/sentry/transports/CustomEndpointTransport.ts

@@ -11,8 +11,8 @@ export interface CustomEndpointTransportOptions {
 /**
  * This is a copy of sentry's FetchTransport, edited to be able to push to any custom url
  * instead of using Sentry-specific endpoint logic.
- * Also transofrms some of the payload values to be parseable by go.
- * Sends events sequanetially and implements back-off in case of rate limiting.
+ * Also transforms some of the payload values to be parseable by go.
+ * Sends events sequentially and implements back-off in case of rate limiting.
  */
 
 export class CustomEndpointTransport implements BaseTransport {