Merge pull request #13401 from grafana/render-limits

Concurrent render limits

conf/defaults.ini

@@ -474,6 +474,10 @@ error_or_timeout = alerting
 # Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok)
 nodata_or_nullvalues = no_data
 
+# Alert notifications can include images, but rendering many images at the same time can overload the server
+# This limit will protect the server from render overloading and make sure notifications are sent out quickly
+concurrent_render_limit = 5
+
 #################################### Explore #############################
 [explore]
 # Enable the Explore section

conf/sample.ini

@@ -393,6 +393,10 @@ log_queries =
 # Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok)
 ;nodata_or_nullvalues = no_data
 
+# Alert notifications can include images, but rendering many images at the same time can overload the server
+# This limit will protect the server from render overloading and make sure notifications are sent out quickly
+;concurrent_render_limit = 5
+
 #################################### Explore #############################
 [explore]
 # Enable the Explore section

docs/sources/installation/configuration.md

@@ -566,3 +566,11 @@ Default setting for new alert rules. Defaults to categorize error and timeouts a
 > Available in 5.3  and above
 
 Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok)
+
+# concurrent_render_limit
+
+> Available in 5.3  and above
+
+Alert notifications can include images, but rendering many images at the same time can overload the server.
+This limit will protect the server from render overloading and make sure notifications are sent out quickly. Default
+value is `5`.

pkg/api/render.go

@@ -41,15 +41,16 @@ func (hs *HTTPServer) RenderToPng(c *m.ReqContext) {
 	}
 
 	result, err := hs.RenderService.Render(c.Req.Context(), rendering.Opts{
-		Width:    width,
-		Height:   height,
-		Timeout:  time.Duration(timeout) * time.Second,
-		OrgId:    c.OrgId,
-		UserId:   c.UserId,
-		OrgRole:  c.OrgRole,
-		Path:     c.Params("*") + queryParams,
-		Timezone: queryReader.Get("tz", ""),
-		Encoding: queryReader.Get("encoding", ""),
+		Width:           width,
+		Height:          height,
+		Timeout:         time.Duration(timeout) * time.Second,
+		OrgId:           c.OrgId,
+		UserId:          c.UserId,
+		OrgRole:         c.OrgRole,
+		Path:            c.Params("*") + queryParams,
+		Timezone:        queryReader.Get("tz", ""),
+		Encoding:        queryReader.Get("encoding", ""),
+		ConcurrentLimit: 30,
 	})
 
 	if err != nil && err == rendering.ErrTimeout {

pkg/services/alerting/notifier.go

@@ -11,6 +11,7 @@ import (
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/metrics"
 	"github.com/grafana/grafana/pkg/services/rendering"
+	"github.com/grafana/grafana/pkg/setting"
 
 	m "github.com/grafana/grafana/pkg/models"
 )
@@ -108,11 +109,12 @@ func (n *notificationService) uploadImage(context *EvalContext) (err error) {
 	}
 
 	renderOpts := rendering.Opts{
-		Width:   1000,
-		Height:  500,
-		Timeout: alertTimeout / 2,
-		OrgId:   context.Rule.OrgId,
-		OrgRole: m.ROLE_ADMIN,
+		Width:           1000,
+		Height:          500,
+		Timeout:         alertTimeout / 2,
+		OrgId:           context.Rule.OrgId,
+		OrgRole:         m.ROLE_ADMIN,
+		ConcurrentLimit: setting.AlertingRenderLimit,
 	}
 
 	ref, err := context.GetDashboardUID()

pkg/services/alerting/result_handler.go

@@ -100,7 +100,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 			}
 		}
 	}
-	handler.notifier.SendIfNeeded(evalContext)
 
+	handler.notifier.SendIfNeeded(evalContext)
 	return nil
 }

pkg/services/rendering/interface.go

@@ -13,15 +13,16 @@ var ErrNoRenderer = errors.New("No renderer plugin found nor is an external rend
 var ErrPhantomJSNotInstalled = errors.New("PhantomJS executable not found")
 
 type Opts struct {
-	Width    int
-	Height   int
-	Timeout  time.Duration
-	OrgId    int64
-	UserId   int64
-	OrgRole  models.RoleType
-	Path     string
-	Encoding string
-	Timezone string
+	Width           int
+	Height          int
+	Timeout         time.Duration
+	OrgId           int64
+	UserId          int64
+	OrgRole         models.RoleType
+	Path            string
+	Encoding        string
+	Timezone        string
+	ConcurrentLimit int
 }
 
 type RenderResult struct {

pkg/services/rendering/rendering.go

@@ -24,12 +24,13 @@ func init() {
 }
 
 type RenderingService struct {
-	log          log.Logger
-	pluginClient *plugin.Client
-	grpcPlugin   pluginModel.RendererPlugin
-	pluginInfo   *plugins.RendererPlugin
-	renderAction renderFunc
-	domain       string
+	log             log.Logger
+	pluginClient    *plugin.Client
+	grpcPlugin      pluginModel.RendererPlugin
+	pluginInfo      *plugins.RendererPlugin
+	renderAction    renderFunc
+	domain          string
+	inProgressCount int
 
 	Cfg *setting.Cfg `inject:""`
 }
@@ -90,6 +91,18 @@ func (rs *RenderingService) Run(ctx context.Context) error {
 }
 
 func (rs *RenderingService) Render(ctx context.Context, opts Opts) (*RenderResult, error) {
+	if rs.inProgressCount > opts.ConcurrentLimit {
+		return &RenderResult{
+			FilePath: filepath.Join(setting.HomePath, "public/img/rendering_limit.png"),
+		}, nil
+	}
+
+	defer func() {
+		rs.inProgressCount -= 1
+	}()
+
+	rs.inProgressCount += 1
+
 	if rs.renderAction != nil {
 		return rs.renderAction(ctx, opts)
 	} else {

pkg/setting/setting.go

@@ -166,6 +166,7 @@ var (
 	// Alerting
 	AlertingEnabled            bool
 	ExecuteAlerts              bool
+	AlertingRenderLimit        int
 	AlertingErrorOrTimeout     string
 	AlertingNoDataOrNullValues string
 
@@ -196,10 +197,13 @@ type Cfg struct {
 	Smtp SmtpSettings
 
 	// Rendering
-	ImagesDir                        string
-	PhantomDir                       string
-	RendererUrl                      string
-	RendererCallbackUrl              string
+	ImagesDir             string
+	PhantomDir            string
+	RendererUrl           string
+	RendererCallbackUrl   string
+	RendererLimit         int
+	RendererLimitAlerting int
+
 	DisableBruteForceLoginProtection bool
 
 	TempDataLifetime time.Duration
@@ -677,6 +681,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
 	alerting := iniFile.Section("alerting")
 	AlertingEnabled = alerting.Key("enabled").MustBool(true)
 	ExecuteAlerts = alerting.Key("execute_alerts").MustBool(true)
+	AlertingRenderLimit = alerting.Key("concurrent_render_limit").MustInt(5)
 	AlertingErrorOrTimeout = alerting.Key("error_or_timeout").MustString("alerting")
 	AlertingNoDataOrNullValues = alerting.Key("nodata_or_nullvalues").MustString("no_data")