MM-69104: Elide plugin_statuses_changed payload (#36966)

* MM-69104: signal plugin_statuses_changed without a payload, refetch on demand

The plugin_statuses_changed websocket event broadcast the entire cluster-wide
plugin status slice over the best-effort (UDP) cluster transport. With enough
installed plugins the serialized payload exceeded the gossip datagram size and
was silently dropped with EMSGSIZE ("message too long"), leaving admin clients
on peer nodes with stale plugin state during plugin reinstall/restart churn.

The event is admin-only and only feeds the System Console plugin views, so
rather than push the whole slice, publish it as a payload-free signal sent
reliably over the cluster. The Plugin Management and Marketplace views subscribe
via the useWebSocket hook and debounce a refetch of a full, cluster-wide status
from the existing REST endpoint, pulling the data on demand only while a relevant
view is mounted.

* MM-69104: keep plugin_statuses as an empty array for client compatibility

Some clients call array methods on plugin_statuses (e.g. the Boards/Focalboard
plugin does data.plugin_statuses.find(...)). Omitting the field entirely makes that
throw, so send a non-nil empty slice that serializes as [] rather than null. The
payload stays effectively empty, so the cluster-broadcast size fix is preserved.

* fixup! MM-69104: signal plugin_statuses_changed without a payload, refetch on demand

* Apply suggestions from code review

Co-authored-by: Alejandro García Montoro <alejandro.garciamontoro@gmail.com>

* fixup! MM-69104: signal plugin_statuses_changed without a payload, refetch on demand

use shared useDebounce hook in usePluginStatusesSync

* fixup! MM-69104: signal plugin_statuses_changed without a payload, refetch on demand

trim docstring

* fixup! MM-69104: signal plugin_statuses_changed without a payload, refetch on demand

return pluginStatuses from hook

* fixup! MM-69104: signal plugin_statuses_changed without a payload, refetch on demand

cover hook return value

---------

Co-authored-by: Alejandro García Montoro <alejandro.garciamontoro@gmail.com>

server/channels/api4/plugin_test.go

@@ -433,12 +433,17 @@ func TestNotifyClusterPluginEvent(t *testing.T) {
 	// Successful remove
 	webSocketClient := th.CreateConnectedWebSocketClientWithClient(t, th.SystemAdminClient)
 
+	var statusesPresent bool
+	var receivedStatuses any
 	done := make(chan bool)
 	go func() {
 		for {
 			select {
 			case resp := <-webSocketClient.EventChannel:
-				if resp.EventType() == model.WebsocketEventPluginStatusesChanged && len(resp.GetData()["plugin_statuses"].([]any)) == 0 {
+				// The event only signals admins to refetch statuses; it carries an empty
+				// plugin_statuses array (compatibility shim), not the full slice.
+				if resp.EventType() == model.WebsocketEventPluginStatusesChanged {
+					receivedStatuses, statusesPresent = resp.GetData()["plugin_statuses"]
 					done <- true
 					return
 				}
@@ -455,6 +460,8 @@ func TestNotifyClusterPluginEvent(t *testing.T) {
 
 	result := <-done
 	require.True(t, result, "plugin_statuses_changed websocket event was not received")
+	require.True(t, statusesPresent, "plugin_statuses field should be present")
+	assert.Empty(t, receivedStatuses, "plugin_statuses should be an empty array")
 
 	messages = testCluster.GetMessages()
 

server/channels/app/plugin.go

@@ -158,9 +158,7 @@ func (ch *Channels) syncPluginsActiveState() {
 		pluginsEnvironment.Shutdown()
 	}
 
-	if err := ch.notifyPluginStatusesChanged(); err != nil {
-		ch.srv.Log().Warn("failed to notify plugin status changed", mlog.Err(err))
-	}
+	ch.notifyPluginStatusesChanged()
 }
 
 func (a *App) NewPluginAPI(rctx request.CTX, manifest *model.Manifest) plugin.API {

server/channels/app/plugin_install.go

@@ -150,9 +150,7 @@ func (ch *Channels) installPluginFromClusterMessage(pluginID string) {
 		logger.Error("Failed notify plugin enabled", mlog.Err(err))
 	}
 
-	if err := ch.notifyPluginStatusesChanged(); err != nil {
-		logger.Error("Failed to notify plugin status changed", mlog.Err(err))
-	}
+	ch.notifyPluginStatusesChanged()
 }
 
 // removePluginFromClusterMessage is called when a peer removes a plugin, signalling all other
@@ -166,9 +164,7 @@ func (ch *Channels) removePluginFromClusterMessage(pluginID string) {
 		logger.Error("Failed to remove plugin locally", mlog.Err(err))
 	}
 
-	if err := ch.notifyPluginStatusesChanged(); err != nil {
-		logger.Error("failed to notify plugin status changed", mlog.Err(err))
-	}
+	ch.notifyPluginStatusesChanged()
 }
 
 // InstallPlugin unpacks and installs a plugin but does not enable or activate it unless the
@@ -208,9 +204,7 @@ func (ch *Channels) installPlugin(bundle, signature io.ReadSeeker, installationS
 		logger.Warn("Failed to notify plugin enabled", mlog.Err(err))
 	}
 
-	if err := ch.notifyPluginStatusesChanged(); err != nil {
-		logger.Warn("Failed to notify plugin status changed", mlog.Err(err))
-	}
+	ch.notifyPluginStatusesChanged()
 
 	return manifest, nil
 }
@@ -555,9 +549,7 @@ func (ch *Channels) RemovePlugin(id string) *model.AppError {
 		},
 	)
 
-	if err := ch.notifyPluginStatusesChanged(); err != nil {
-		logger.Warn("Failed to notify plugin status changed", mlog.Err(err))
-	}
+	ch.notifyPluginStatusesChanged()
 
 	return nil
 }

server/channels/app/plugin_statuses.go

@@ -92,17 +92,17 @@ func (ch *Channels) getClusterPluginStatuses() (model.PluginStatuses, *model.App
 	return pluginStatuses, nil
 }
 
-func (ch *Channels) notifyPluginStatusesChanged() error {
-	pluginStatuses, err := ch.getClusterPluginStatuses()
-	if err != nil {
-		return err
-	}
-
-	// Notify any system admins.
+// notifyPluginStatusesChanged signals system admins that plugin statuses have changed without
+// carrying the full status slice. That slice (status for every installed plugin, cluster-wide)
+// routinely overflows the best-effort (UDP) cluster transport, so admin clients refetch it on
+// demand instead. The signal is sent reliably (TCP) and only to system admins (ContainsSensitiveData).
+func (ch *Channels) notifyPluginStatusesChanged() {
 	message := model.NewWebSocketEvent(model.WebsocketEventPluginStatusesChanged, "", "", "", nil, "")
-	message.Add("plugin_statuses", pluginStatuses)
+	// Compatibility shim: some clients (e.g. the Boards/Focalboard plugin) call array methods on
+	// plugin_statuses, so include an empty (non-nil, to serialize as [] rather than null) slice
+	// rather than omitting the field. The payload is intentionally always empty.
+	message.Add("plugin_statuses", model.PluginStatuses{})
 	message.GetBroadcast().ContainsSensitiveData = true
+	message.GetBroadcast().ReliableClusterSend = true
 	ch.srv.platform.Publish(message)
-
-	return nil
 }

webapp/channels/src/actions/websocket_actions.ts

@@ -36,7 +36,6 @@ import {
     UserTypes,
     RoleTypes,
     GeneralTypes,
-    AdminTypes,
     IntegrationTypes,
     PreferenceTypes,
     AppsTypes,
@@ -625,10 +624,6 @@ export function handleEvent(msg: WebSocketMessage) {
         handleLicenseChanged(msg);
         break;
 
-    case WebSocketEvents.PluginStatusesChanged:
-        handlePluginStatusesChangedEvent(msg);
-        break;
-
     case WebSocketEvents.OpenDialog:
         handleOpenDialogEvent(msg);
         break;
@@ -1735,10 +1730,6 @@ function handleLicenseChanged(msg: WebSocketMessages.LicenseChanged) {
     dispatch(getServerLimits());
 }
 
-function handlePluginStatusesChangedEvent(msg: WebSocketMessages.PluginStatusesChanged) {
-    store.dispatch({type: AdminTypes.RECEIVED_PLUGIN_STATUSES, data: msg.data.plugin_statuses});
-}
-
 function handleOpenDialogEvent(msg: WebSocketMessages.OpenDialog) {
     const data = (msg.data && msg.data.dialog);
     const dialog = JSON.parse(data) as OpenDialogRequest || {};

webapp/channels/src/components/admin_console/plugin_management/index.tsx

@@ -1,6 +1,7 @@
 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
 // See LICENSE.txt for license information.
 
+import React from 'react';
 import {connect} from 'react-redux';
 import {bindActionCreators} from 'redux';
 import type {Dispatch} from 'redux';
@@ -16,6 +17,8 @@ import {
 } from 'mattermost-redux/actions/admin';
 import {appsFeatureFlagEnabled} from 'mattermost-redux/selectors/entities/apps';
 
+import usePluginStatusesSync from 'components/common/hooks/usePluginStatusesSync';
+
 import PluginManagement from './plugin_management';
 
 function mapStateToProps(state: any) {
@@ -40,4 +43,11 @@ function mapDispatchToProps(dispatch: Dispatch) {
     };
 }
 
-export default connect(mapStateToProps, mapDispatchToProps)(PluginManagement);
+const ConnectedPluginManagement = connect(mapStateToProps, mapDispatchToProps)(PluginManagement);
+
+// Wrap the legacy class-based settings component so it can subscribe to plugin status changes
+// and refetch on demand while the page is mounted.
+export default function PluginManagementWithStatusesSync(props: React.ComponentProps<typeof ConnectedPluginManagement>) {
+    usePluginStatusesSync();
+    return <ConnectedPluginManagement {...props}/>;
+}

webapp/channels/src/components/common/hooks/usePluginStatusesSync.test.ts

@@ -0,0 +1,163 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+import {act} from '@testing-library/react';
+import * as ReactRedux from 'react-redux';
+
+import type {WebSocketMessage} from '@mattermost/client';
+import {WebSocketEvents} from '@mattermost/client';
+
+import {getPluginStatuses} from 'mattermost-redux/actions/admin';
+
+import {renderHookWithContext} from 'tests/react_testing_utils';
+import * as webSocketHooks from 'utils/use_websocket/hooks';
+
+import usePluginStatusesSync from './usePluginStatusesSync';
+
+jest.mock('mattermost-redux/actions/admin', () => ({
+    ...jest.requireActual('mattermost-redux/actions/admin'),
+    getPluginStatuses: jest.fn(() => ({type: 'MOCK_GET_PLUGIN_STATUSES'})),
+}));
+
+jest.mock('utils/use_websocket/hooks', () => ({
+    useWebSocket: jest.fn(),
+    useWebSocketClient: jest.fn(),
+}));
+
+const DEBOUNCE_DELAY_MS = 500;
+
+describe('usePluginStatusesSync', () => {
+    const dispatchMock = jest.fn();
+    const addReconnectListener = jest.fn();
+    const removeReconnectListener = jest.fn();
+
+    // The handler the hook registers with useWebSocket, captured so tests can feed it messages.
+    let messageHandler: (msg: WebSocketMessage) => void;
+
+    beforeEach(() => {
+        jest.useFakeTimers();
+        jest.spyOn(ReactRedux, 'useDispatch').mockReturnValue(dispatchMock);
+
+        (webSocketHooks.useWebSocket as jest.Mock).mockImplementation(({handler}) => {
+            messageHandler = handler;
+        });
+        (webSocketHooks.useWebSocketClient as jest.Mock).mockReturnValue({
+            addReconnectListener,
+            removeReconnectListener,
+        });
+    });
+
+    afterEach(() => {
+        jest.runOnlyPendingTimers();
+        jest.useRealTimers();
+        jest.restoreAllMocks();
+        (getPluginStatuses as jest.Mock).mockClear();
+        dispatchMock.mockClear();
+        addReconnectListener.mockClear();
+        removeReconnectListener.mockClear();
+    });
+
+    const pluginStatusesChanged = {event: WebSocketEvents.PluginStatusesChanged} as WebSocketMessage;
+
+    test('dispatches getPluginStatuses after the debounce delay on a plugin_statuses_changed event', () => {
+        renderHookWithContext(usePluginStatusesSync);
+
+        act(() => {
+            messageHandler(pluginStatusesChanged);
+        });
+
+        // Nothing dispatched until the debounce window elapses.
+        expect(getPluginStatuses).not.toHaveBeenCalled();
+
+        act(() => {
+            jest.advanceTimersByTime(DEBOUNCE_DELAY_MS);
+        });
+
+        expect(getPluginStatuses).toHaveBeenCalledTimes(1);
+        expect(dispatchMock).toHaveBeenCalledWith({type: 'MOCK_GET_PLUGIN_STATUSES'});
+    });
+
+    test('collapses multiple rapid events into a single refetch', () => {
+        renderHookWithContext(usePluginStatusesSync);
+
+        act(() => {
+            messageHandler(pluginStatusesChanged);
+            jest.advanceTimersByTime(100);
+            messageHandler(pluginStatusesChanged);
+            jest.advanceTimersByTime(100);
+            messageHandler(pluginStatusesChanged);
+        });
+
+        act(() => {
+            jest.advanceTimersByTime(DEBOUNCE_DELAY_MS);
+        });
+
+        expect(getPluginStatuses).toHaveBeenCalledTimes(1);
+    });
+
+    test('refetches on websocket reconnect', () => {
+        renderHookWithContext(() => usePluginStatusesSync());
+
+        expect(addReconnectListener).toHaveBeenCalledTimes(1);
+        const reconnectListener = addReconnectListener.mock.calls[0][0];
+
+        act(() => {
+            reconnectListener();
+            jest.advanceTimersByTime(DEBOUNCE_DELAY_MS);
+        });
+
+        expect(getPluginStatuses).toHaveBeenCalledTimes(1);
+    });
+
+    test('ignores other websocket events', () => {
+        renderHookWithContext(usePluginStatusesSync);
+
+        act(() => {
+            messageHandler({event: WebSocketEvents.Posted} as WebSocketMessage);
+            jest.advanceTimersByTime(DEBOUNCE_DELAY_MS);
+        });
+
+        expect(getPluginStatuses).not.toHaveBeenCalled();
+    });
+
+    test('cleans up on unmount: removes the reconnect listener and cancels a pending refetch', () => {
+        const {unmount} = renderHookWithContext(usePluginStatusesSync);
+
+        // Start a debounce window, then unmount before it elapses.
+        act(() => {
+            messageHandler(pluginStatusesChanged);
+        });
+
+        act(unmount);
+
+        expect(removeReconnectListener).toHaveBeenCalledTimes(1);
+        expect(removeReconnectListener).toHaveBeenCalledWith(addReconnectListener.mock.calls[0][0]);
+
+        // The pending timer was cleared, so no refetch fires after unmount.
+        act(() => {
+            jest.advanceTimersByTime(DEBOUNCE_DELAY_MS);
+        });
+
+        expect(getPluginStatuses).not.toHaveBeenCalled();
+    });
+
+    test('returns the plugin statuses from the store', () => {
+        const pluginStatuses = {
+            'com.example.plugin': {
+                id: 'com.example.plugin',
+                name: 'Example',
+                description: '',
+                version: '1.0.0',
+                active: true,
+                state: 1,
+                instances: [],
+            },
+        };
+
+        const {result} = renderHookWithContext(usePluginStatusesSync, {
+            entities: {admin: {pluginStatuses}},
+        });
+
+        expect(result.current).toEqual(pluginStatuses);
+    });
+});

webapp/channels/src/components/common/hooks/usePluginStatusesSync.ts

@@ -0,0 +1,48 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+import {useCallback, useEffect} from 'react';
+import {useDispatch, useSelector} from 'react-redux';
+
+import type {WebSocketMessage} from '@mattermost/client';
+import {WebSocketEvents} from '@mattermost/client';
+
+import {getPluginStatuses} from 'mattermost-redux/actions/admin';
+
+import {useDebounce} from 'hooks/useDebounce';
+import {useWebSocket, useWebSocketClient} from 'utils/use_websocket/hooks';
+
+import type {GlobalState} from 'types/store';
+
+const DEBOUNCE_DELAY_MS = 500;
+
+/**
+ * Refetches the cluster-wide plugin statuses whenever the server signals that they changed,
+ * and returns the current statuses from the store.
+ */
+export default function usePluginStatusesSync() {
+    const dispatch = useDispatch();
+    const wsClient = useWebSocketClient();
+    const pluginStatuses = useSelector((state: GlobalState) => state.entities.admin.pluginStatuses);
+
+    const debouncedRefetch = useDebounce(() => {
+        dispatch(getPluginStatuses());
+    }, DEBOUNCE_DELAY_MS);
+
+    const handleWebSocketMessage = useCallback((msg: WebSocketMessage) => {
+        if (msg.event === WebSocketEvents.PluginStatusesChanged) {
+            debouncedRefetch();
+        }
+    }, [debouncedRefetch]);
+
+    useWebSocket({handler: handleWebSocketMessage});
+
+    useEffect(() => {
+        wsClient.addReconnectListener(debouncedRefetch);
+        return () => {
+            wsClient.removeReconnectListener(debouncedRefetch);
+        };
+    }, [wsClient, debouncedRefetch]);
+
+    return pluginStatuses;
+}

webapp/channels/src/components/plugin_marketplace/marketplace_modal.tsx

@@ -24,6 +24,7 @@ import {closeModal} from 'actions/views/modals';
 import {getListing, getInstalledListing} from 'selectors/views/marketplace';
 import {isModalOpen} from 'selectors/views/modals';
 
+import usePluginStatusesSync from 'components/common/hooks/usePluginStatusesSync';
 import LoadingScreen from 'components/loading_screen';
 import Input, {SIZE} from 'components/widgets/inputs/input/input';
 
@@ -58,7 +59,9 @@ const MarketplaceModal = () => {
     const show = useSelector((state: GlobalState) => isModalOpen(state, ModalIdentifiers.PLUGIN_MARKETPLACE));
     const listing = useSelector(getListing);
     const installedListing = useSelector(getInstalledListing);
-    const pluginStatuses = useSelector((state: GlobalState) => state.entities.admin.pluginStatuses);
+
+    // Refetch plugin statuses while the modal is open whenever the server signals a change.
+    const pluginStatuses = usePluginStatusesSync();
     const hasFirstAdminVisitedMarketplace = useSelector(getFirstAdminVisitMarketplaceStatus);
     const isStreamlinedMarketplaceEnabled = useSelector(streamlinedMarketplaceEnabled);
     const license = useSelector(getLicense);

webapp/platform/client/src/websocket_messages.ts

@@ -10,7 +10,7 @@ import type {Draft} from '@mattermost/types/drafts';
 import type {CustomEmoji} from '@mattermost/types/emojis';
 import type {Group, GroupMember as GroupMemberType} from '@mattermost/types/groups';
 import type {OpenDialogRequest} from '@mattermost/types/integrations';
-import type {PluginManifest, PluginStatus} from '@mattermost/types/plugins';
+import type {PluginManifest} from '@mattermost/types/plugins';
 import type {Post, PostAcknowledgement as PostAcknowledgementType} from '@mattermost/types/posts';
 import type {PreferenceType} from '@mattermost/types/preferences';
 import type {PropertyField, PropertyValue} from '@mattermost/types/properties';
@@ -487,8 +487,11 @@ export type Plugin = BaseWebSocketMessage<WebSocketEvents.PluginEnabled | WebSoc
     manifest: PluginManifest;
 }>;
 
+// Signals admin clients to refetch the full plugin statuses on demand. plugin_statuses is always
+// an empty array: it carries no data, but is retained (rather than omitted) so clients that call
+// array methods on it don't break.
 export type PluginStatusesChanged = BaseWebSocketMessage<WebSocketEvents.PluginStatusesChanged, {
-    plugin_statuses: PluginStatus[];
+    plugin_statuses: never[];
 }>;
 
 export type OpenDialog = BaseWebSocketMessage<WebSocketEvents.OpenDialog, {