Plugins: Report plugin utilization of Grafana runtime dependencies (#75156)

* Plugins: Report plugin utilization of Grafana runtime dependencies

* Change approach to determine pluginName too

* Fix tests

* Update tests

* remove commented code

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -136,6 +136,7 @@ Experimental features might be changed or removed without prior notice.
 | `requestInstrumentationStatusSource`        | Include a status source label for request metrics and logs                                                   |
 | `wargamesTesting`                           | Placeholder feature flag for internal testing                                                                |
 | `alertingInsights`                          | Show the new alerting insights landing page                                                                  |
+| `pluginsAPIMetrics`                         | Sends metrics of public grafana packages usage by plugins                                                    |
 
 ## Development feature toggles
 

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -127,4 +127,5 @@ export interface FeatureToggles {
   lokiRunQueriesInParallel?: boolean;
   wargamesTesting?: boolean;
   alertingInsights?: boolean;
+  pluginsAPIMetrics?: boolean;
 }

pkg/services/featuremgmt/registry.go

@@ -759,5 +759,12 @@ var (
 			Stage:        FeatureStageExperimental,
 			Owner:        grafanaAlertingSquad,
 		},
+		{
+			Name:         "pluginsAPIMetrics",
+			Description:  "Sends metrics of public grafana packages usage by plugins",
+			FrontendOnly: true,
+			Stage:        FeatureStageExperimental,
+			Owner:        grafanaPluginsPlatformSquad,
+		},
 	}
 )

pkg/services/featuremgmt/toggles_gen.csv

@@ -108,3 +108,4 @@ requestInstrumentationStatusSource,experimental,@grafana/plugins-platform-backen
 lokiRunQueriesInParallel,privatePreview,@grafana/observability-logs,false,false,false,false
 wargamesTesting,experimental,@grafana/hosted-grafana-team,false,false,false,false
 alertingInsights,experimental,@grafana/alerting-squad,false,false,false,true
+pluginsAPIMetrics,experimental,@grafana/plugins-platform-backend,false,false,false,true

pkg/services/featuremgmt/toggles_gen.go

@@ -442,4 +442,8 @@ const (
 	// FlagAlertingInsights
 	// Show the new alerting insights landing page
 	FlagAlertingInsights = "alertingInsights"
+
+	// FlagPluginsAPIMetrics
+	// Sends metrics of public grafana packages usage by plugins
+	FlagPluginsAPIMetrics = "pluginsAPIMetrics"
 )

public/app/features/plugins/loader/packageMetrics.test.ts

@@ -0,0 +1,226 @@
+import { logInfo } from '@grafana/runtime';
+
+import { trackPackageUsage } from './packageMetrics';
+
+jest.mock('@grafana/runtime', () => ({
+  logInfo: jest.fn().mockImplementation(),
+}));
+
+// notice each test object has a different key to prevent hitting the cache
+const logInfoMock = logInfo as jest.Mock;
+const mockUsage = jest.fn();
+
+describe('trackPackageUsage', () => {
+  beforeEach(() => {
+    logInfoMock.mockClear();
+  });
+
+  describe('With document.currentScript null', () => {
+    const originalCurrentScript = document.currentScript;
+
+    // set currentScript to null
+    beforeAll(() => {
+      Object.defineProperty(document, 'currentScript', {
+        value: null,
+        writable: true,
+      });
+    });
+
+    // restore original currentScript
+    afterAll(() => {
+      Object.defineProperty(document, 'currentScript', {
+        value: originalCurrentScript,
+        writable: true,
+      });
+    });
+
+    it('should log API usage and return a proxy object', () => {
+      const obj = {
+        foo: 'bar',
+      };
+      const packageName = 'your-package';
+
+      const result = trackPackageUsage(obj, packageName);
+
+      mockUsage(result.foo);
+
+      expect(logInfoMock).toHaveBeenCalledTimes(1);
+      expect(logInfoMock).toHaveBeenLastCalledWith(`Plugin using your-package.foo`, {
+        key: 'foo',
+        parent: 'your-package',
+        packageName: 'your-package',
+        guessedPluginName: '',
+      });
+      expect(result).toEqual(obj);
+    });
+
+    it('should return a proxy object for nested properties', () => {
+      const obj = {
+        foo2: {
+          bar: 'baz',
+        },
+      };
+      const packageName = 'your-package';
+
+      const result = trackPackageUsage(obj, packageName);
+      mockUsage(result.foo2.bar);
+
+      // 2 calls, one for each attribute
+      expect(logInfoMock).toHaveBeenCalledTimes(2);
+
+      expect(logInfoMock).toHaveBeenCalledWith(`Plugin using your-package.foo2`, {
+        key: 'foo2',
+        parent: 'your-package',
+        packageName: 'your-package',
+        guessedPluginName: '',
+      });
+      expect(logInfoMock).toHaveBeenCalledWith(`Plugin using your-package.foo2.bar`, {
+        key: 'bar',
+        parent: 'your-package.foo2',
+        packageName: 'your-package',
+        guessedPluginName: '',
+      });
+
+      expect(result.foo2).toEqual(obj.foo2);
+    });
+
+    it('should not log API usage for symbols or __useDefault key', () => {
+      const obj = {
+        [Symbol('key')]: 'value',
+        __useDefault: 'default',
+      };
+      const packageName = 'your-package';
+
+      const result = trackPackageUsage(obj, packageName);
+
+      expect(logInfoMock).not.toHaveBeenCalled();
+      expect(result).toEqual(obj);
+    });
+
+    it('should return the same proxy object for the same nested property', () => {
+      const obj = {
+        foo3: {
+          bar: 'baz',
+        },
+      };
+      const packageName = 'your-package';
+
+      const result1 = trackPackageUsage(obj, packageName);
+      const result2 = trackPackageUsage(obj, packageName);
+
+      mockUsage(result1.foo3);
+
+      expect(logInfoMock).toHaveBeenCalledTimes(1);
+      expect(logInfoMock).toHaveBeenCalledWith(`Plugin using your-package.foo3`, {
+        key: 'foo3',
+        parent: 'your-package',
+        packageName: 'your-package',
+        guessedPluginName: '',
+      });
+      mockUsage(result2.foo3.bar);
+      expect(logInfoMock).toHaveBeenCalledWith(`Plugin using your-package.foo3.bar`, {
+        key: 'bar',
+        parent: 'your-package.foo3',
+        packageName: 'your-package',
+        guessedPluginName: '',
+      });
+
+      expect(result1.foo3).toEqual(obj.foo3);
+      expect(result2.foo3).toEqual(obj.foo3);
+      expect(result1.foo3).toBe(result2.foo3);
+    });
+
+    it('should not report twice the same key usage', () => {
+      const obj = {
+        cacheMe: 'please',
+        zap: {
+          cacheMeInner: 'please',
+        },
+      };
+
+      const result = trackPackageUsage(obj, 'your-package');
+
+      mockUsage(result.cacheMe);
+      expect(logInfoMock).toHaveBeenCalledTimes(1);
+      mockUsage(result.cacheMe);
+      expect(logInfoMock).toHaveBeenCalledTimes(1);
+
+      mockUsage(result.zap);
+      expect(logInfoMock).toHaveBeenCalledTimes(2);
+      mockUsage(result.zap);
+      expect(logInfoMock).toHaveBeenCalledTimes(2);
+
+      mockUsage(result.zap.cacheMeInner);
+      expect(logInfoMock).toHaveBeenCalledTimes(3);
+      mockUsage(result.zap.cacheMeInner);
+      expect(logInfoMock).toHaveBeenCalledTimes(3);
+
+      expect(result).toEqual(obj);
+    });
+  });
+
+  it('should guess the plugin name from the stacktrace and urls', () => {
+    const mockErrorConstructor = jest.fn().mockImplementation(() => {
+      return {
+        stack: `Error
+    at eval (eval at get (http://localhost:3000/public/build/app.38735bee027ded74d65e.js:167859:11), <anonymous>:1:1)
+    at Object.get (http://localhost:3000/public/build/app.38735bee027ded74d65e.js:167859:11)
+    at eval (http://localhost:3000/public/plugins/alexanderzobnin-zabbix-app/panel-triggers/module.js?_cache=1695283550582:3:2582)
+    at eval (http://localhost:3000/public/plugins/alexanderzobnin-zabbix-app/panel-triggers/module.js?_cache=1695283550582:159:22081)
+    at Object.eval (http://localhost:3000/public/plugins/alexanderzobnin-zabbix-app/panel-triggers/module.js?_cache=1695283550582:159:22087)
+    at Object.execute (http://localhost:3000/public/build/app.38735bee027ded74d65e.js:529405:37)
+    at doExec (http://localhost:3000/public/build/app.38735bee027ded74d65e.js:529955:32)
+    at postOrderExec (http://localhost:3000/public/build/app.38735bee027ded74d65e.js:529951:12)
+    at http://localhost:3000/public/build/app.38735bee027ded74d65e.js:529899:14
+    at async http://localhost:3000/public/build/app.38735bee027ded74d65e.js:166261:16`,
+      };
+    });
+
+    const errorSpy = jest.spyOn(global, 'Error').mockImplementation(mockErrorConstructor);
+
+    const obj = {
+      lord: 'me',
+    };
+
+    const result = trackPackageUsage(obj, 'your-package');
+    mockUsage(result.lord);
+
+    expect(logInfoMock).toHaveBeenCalledTimes(1);
+    expect(logInfoMock).toHaveBeenLastCalledWith(`Plugin using your-package.lord`, {
+      key: 'lord',
+      parent: 'your-package',
+      packageName: 'your-package',
+      guessedPluginName: 'alexanderzobnin-zabbix-app',
+    });
+
+    errorSpy.mockRestore();
+  });
+
+  it('Should skip tracking if document.currentScript is not null', () => {
+    // Save the original value of the attribute
+    const originalCurrentScript = document.currentScript;
+
+    // Define a new property on the document object with the mock currentScript
+    Object.defineProperty(document, 'currentScript', {
+      value: {
+        src: 'mocked-script.js',
+      },
+      writable: true,
+    });
+
+    const obj = {
+      lor: 'me',
+    };
+
+    const result = trackPackageUsage(obj, 'your-package');
+
+    mockUsage(result.lor);
+    expect(logInfoMock).not.toHaveBeenCalled();
+
+    // Restore the original value of the currentScript attribute
+    Object.defineProperty(document, 'currentScript', {
+      value: originalCurrentScript,
+      writable: true,
+    });
+  });
+});

public/app/features/plugins/loader/packageMetrics.ts

@@ -0,0 +1,101 @@
+import { logInfo } from '@grafana/runtime';
+
+const cachedMetricProxies = new WeakMap<object, unknown>();
+const trackedKeys: Record<string, boolean> = {};
+
+let pluginNameFromUrlRegex = /plugins\/([^/]*)\/.*?module\.js/i;
+
+/**
+ * This function attempts to determine the plugin name by
+ * analyzing the stack trace. It achieves this by generating
+ * an error object and accessing its stack property,
+ * which typically includes the script URL.
+ *
+ * Note that when inside an async function of any kind, the
+ * stack trace is somewhat lost and the plugin name cannot
+ * be determined most of the times.
+ *
+ * It assumes that the plugin ID is part of the URL,
+ * although this cannot be guaranteed.
+ *
+ * Please note that this function is specifically designed
+ * for plugins loaded with systemjs.
+ *
+ * It is important to treat the information provided by
+ * this function as a "best guess" and not rely on it
+ * for any business logic.
+ */
+function guessPluginNameFromStack(): string | undefined {
+  try {
+    const errorStack = new Error().stack;
+    if (errorStack?.includes('systemJSPrototype')) {
+      return undefined;
+    }
+    if (errorStack && errorStack.includes('module.js')) {
+      let match = errorStack.match(pluginNameFromUrlRegex);
+      if (match && match[1]) {
+        return match[1];
+      }
+    }
+  } catch (e) {
+    return undefined;
+  }
+  return undefined;
+}
+
+function createMetricsProxy<T extends object>(obj: T, parentName: string, packageName: string): T {
+  const handler: ProxyHandler<T> = {
+    get(target, key) {
+      if (
+        // plugins are evaluated by SystemJS and not by a browser <script> tag
+        // if document.currentScript is null this is most likely called by a plugin
+        document.currentScript === null &&
+        typeof key !== 'symbol' &&
+        // __useDefault is a implementation detail of our systemjs plugins
+        // that we don't want to track
+        key.toString() !== '__useDefault'
+      ) {
+        const pluginName = guessPluginNameFromStack() ?? '';
+        const accessPath = `${parentName}.${String(key)}`;
+
+        // we want to report API usage per-plugin when possible
+        const cacheKey = `${pluginName}:${accessPath}`;
+
+        if (!trackedKeys[cacheKey]) {
+          trackedKeys[cacheKey] = true;
+          // note: intentionally not using shorthand property assignment
+          // so any future variable name changes won't affect the metrics names
+          logInfo(`Plugin using ${accessPath}`, {
+            key: String(key),
+            parent: parentName,
+            packageName: packageName,
+            guessedPluginName: pluginName,
+          });
+        }
+      }
+
+      // typescript will not trust the key is a key of target, but given this is a proxy handler
+      // it is guarantee that `key` is a key of `target` so we can type assert to make types work
+      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+      const value = target[key as keyof T];
+
+      if (value !== null && typeof value === 'object') {
+        if (!cachedMetricProxies.has(value)) {
+          cachedMetricProxies.set(value, createMetricsProxy(value, `${parentName}.${String(key)}`, packageName));
+        }
+        return cachedMetricProxies.get(value);
+      }
+      return value;
+    },
+  };
+
+  if (typeof obj === 'object' && obj !== null) {
+    return new Proxy(obj, handler);
+  }
+
+  return obj;
+}
+
+export function trackPackageUsage<T extends object>(obj: T, packageName: string): T {
+  return createMetricsProxy(obj, packageName, packageName);
+}

public/app/features/plugins/loader/utils.ts

@@ -3,15 +3,21 @@ import { SystemJS, config } from '@grafana/runtime';
 import { sandboxPluginDependencies } from '../sandbox/plugin_dependencies';
 
 import { SHARED_DEPENDENCY_PREFIX } from './constants';
+import { trackPackageUsage } from './packageMetrics';
 
 export function buildImportMap(importMap: Record<string, System.Module>) {
   return Object.keys(importMap).reduce<Record<string, string>>((acc, key) => {
     // Use the 'package:' prefix to act as a URL instead of a bare specifier
     const module_name = `${SHARED_DEPENDENCY_PREFIX}:${key}`;
+
+    // get the module to use
+    const module = config.featureToggles.pluginsAPIMetrics ? trackPackageUsage(importMap[key], key) : importMap[key];
+
     // expose dependency to SystemJS
-    SystemJS.set(module_name, importMap[key]);
+    SystemJS.set(module_name, module);
 
     // expose dependency to sandboxed plugins
+    // the sandbox handles its own way of plugins api metrics
     sandboxPluginDependencies.set(key, importMap[key]);
 
     acc[key] = module_name;