Live: max_connections option with strict default (#34634) (#34853)

this should help Live to be enabled by default but still
do not affect setups with lots of simultenious users. To
properly handle many WS connections Grafana administrators
should tune infrastructure a bit - for example increase a
number of open files for a process. Will be in more details
in documentation.

(cherry picked from commit 6d750c000e)

Co-authored-by: Alexander Emelin <frvzmb@gmail.com>

conf/defaults.ini

@@ -888,6 +888,13 @@ plugin_admin_enabled = false
 plugin_admin_external_manage_enabled = false
 plugin_catalog_url = https://grafana.com/grafana/plugins/
 
+#################################### Grafana Live ##########################################
+[live]
+# max_connections to Grafana Live WebSocket endpoint per Grafana server instance. See Grafana Live docs
+# if you are planning to make it higher than default 100 since this can require some OS and infrastructure
+# tuning. 0 disables Live, -1 means unlimited connections.
+max_connections = 100
+
 #################################### Grafana Image Renderer Plugin ##########################
 [plugin.grafana-image-renderer]
 # Instruct headless browser instance to use a default timezone when not provided by Grafana, e.g. when rendering panel image of alert.

conf/sample.ini

@@ -874,6 +874,13 @@
 ;plugin_admin_external_manage_enabled = false
 ;plugin_catalog_url = https://grafana.com/grafana/plugins/
 
+#################################### Grafana Live ##########################################
+[live]
+# max_connections to Grafana Live WebSocket endpoint per Grafana server instance. See Grafana Live docs
+# if you are planning to make it higher than default 100 since this can require some OS and infrastructure
+# tuning. 0 disables Live, -1 means unlimited connections.
+;max_connections = 100
+
 #################################### Grafana Image Renderer Plugin ##########################
 [plugin.grafana-image-renderer]
 # Instruct headless browser instance to use a default timezone when not provided by Grafana, e.g. when rendering panel image of alert.

packages/grafana-data/src/types/config.ts

@@ -122,6 +122,7 @@ export interface GrafanaConfig {
   viewersCanEdit: boolean;
   editorsCanAdmin: boolean;
   disableSanitizeHtml: boolean;
+  liveEnabled: boolean;
   theme: GrafanaTheme;
   theme2: GrafanaTheme2;
   pluginsToPreload: string[];

packages/grafana-runtime/src/config.ts

@@ -54,6 +54,7 @@ export class GrafanaBootConfig implements GrafanaConfig {
   viewersCanEdit = false;
   editorsCanAdmin = false;
   disableSanitizeHtml = false;
+  liveEnabled = true;
   theme: GrafanaTheme;
   theme2: GrafanaTheme2;
   pluginsToPreload: string[] = [];

pkg/api/frontendsettings.go

@@ -207,6 +207,7 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i
 		"alertingErrorOrTimeout":     setting.AlertingErrorOrTimeout,
 		"alertingNoDataOrNullValues": setting.AlertingNoDataOrNullValues,
 		"alertingMinInterval":        setting.AlertingMinInterval,
+		"liveEnabled":                hs.Cfg.LiveMaxConnections != 0,
 		"autoAssignOrg":              setting.AutoAssignOrg,
 		"verifyEmailEnabled":         setting.VerifyEmailEnabled,
 		"sigV4AuthEnabled":           setting.SigV4AuthEnabled,

pkg/services/live/live.go

@@ -178,6 +178,15 @@ func (g *GrafanaLive) Init() error {
 	// different goroutines (belonging to different client connections). This is also
 	// true for other event handlers.
 	node.OnConnect(func(client *centrifuge.Client) {
+		numConnections := g.node.Hub().NumClients()
+		if g.Cfg.LiveMaxConnections >= 0 && numConnections > g.Cfg.LiveMaxConnections {
+			logger.Warn(
+				"Max number of Live connections reached, increase max_connections in [live] configuration section",
+				"user", client.UserID(), "client", client.ID(), "limit", g.Cfg.LiveMaxConnections,
+			)
+			client.Disconnect(centrifuge.DisconnectConnectionLimit)
+			return
+		}
 		var semaphore chan struct{}
 		if clientConcurrency > 1 {
 			semaphore = make(chan struct{}, clientConcurrency)

pkg/setting/setting.go

@@ -380,6 +380,11 @@ type Cfg struct {
 	ExpressionsEnabled bool
 
 	ImageUploadProvider string
+
+	// LiveMaxConnections is a maximum number of WebSocket connections to
+	// Grafana Live ws endpoint (per Grafana server instance). 0 disables
+	// Live, -1 means unlimited connections.
+	LiveMaxConnections int
 }
 
 // IsLiveConfigEnabled returns true if live should be able to save configs to SQL tables
@@ -950,6 +955,10 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
 	cfg.readDateFormats()
 	cfg.readSentryConfig()
 
+	if err := cfg.readLiveSettings(iniFile); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -1420,3 +1429,12 @@ func (cfg *Cfg) readDataSourcesSettings() {
 	datasources := cfg.Raw.Section("datasources")
 	cfg.DataSourceLimit = datasources.Key("datasource_limit").MustInt(5000)
 }
+
+func (cfg *Cfg) readLiveSettings(iniFile *ini.File) error {
+	section := iniFile.Section("live")
+	cfg.LiveMaxConnections = section.Key("max_connections").MustInt(100)
+	if cfg.LiveMaxConnections < -1 {
+		return fmt.Errorf("unexpected value %d for [live] max_connections", cfg.LiveMaxConnections)
+	}
+	return nil
+}

public/app/features/live/LiveConnectionWarning.tsx

@@ -45,8 +45,8 @@ export class LiveConnectionWarning extends PureComponent<Props, State> {
   render() {
     const { show } = this.state;
     if (show) {
-      if (!contextSrv.isSignedIn) {
-        return null; // do not show the warning for anonomous users (and /login page etc)
+      if (!contextSrv.isSignedIn || !config.liveEnabled) {
+        return null; // do not show the warning for anonymous users (and /login page etc)
       }
 
       return (

public/app/features/live/live.ts

@@ -51,7 +51,7 @@ export class CentrifugeSrv implements GrafanaLiveSrv {
   readonly connectionState: BehaviorSubject<boolean>;
   readonly connectionBlocker: Promise<void>;
   readonly scopes: Record<LiveChannelScope, GrafanaLiveScope>;
-  private orgId: number;
+  private readonly orgId: number;
 
   constructor() {
     const baseURL = window.location.origin.replace('http', 'ws');
@@ -64,7 +64,9 @@ export class CentrifugeSrv implements GrafanaLiveSrv {
       sessionId,
       orgId: this.orgId,
     });
-    this.centrifuge.connect(); // do connection
+    if (config.liveEnabled) {
+      this.centrifuge.connect(); // do connection
+    }
     this.connectionState = new BehaviorSubject<boolean>(this.centrifuge.isConnected());
     this.connectionBlocker = new Promise<void>((resolve) => {
       if (this.centrifuge.isConnected()) {