Plugins: Allow loading panel plugins from a CDN (#59096)

* POC: Plugins CDN reverse proxy

* CDN proxy POC: changed env var names

* Add authorization: false for /public path in frontend plugin loader

* Moved CDN settings to Cfg, add some comments

* Fix error 500 in asset fetch if plugin is not using CDN

* Fix EnterpriseLicensePath declared twice

* Fix linter complaining about whitespaces

* Plugins CDN: Skip signature verification for CDN plugins

* Plugins CDN: Skip manifest and signature check for cdn plugins

* Plugins: use IsValid() and IsInternal() rather than equality checks

* Plugins CDN: remove comment

* Plugins CDN: Fix seeker can't seek when serving plugins from local fs

* Plugins CDN: add back error codes in getLocalPluginAssets

* Plugins CDN: call asset.Close() rather than asset.readSeekCloser.Close()

* Plugins CDN: Fix panic in JsonApiErr when errorMessageCoder wraps a nil error

* Plugins CDN: Add error handling to proxyCDNPluginAsset

* Plugins CDN: replace errorMessageCoder with errutil

* Plugins CDN POC: expose cdn plugin paths to frontend for system.js

* Plugins CDN: Fix cdn plugins showing as unsigned in frontend

* WIP: Add support for formatted URL

* Fix missing cdnPluginsBaseURLs in GrafanaConfig

* Plugins CDN: Remove reverse proxy mode and reverse proxy references

* Plugins CDN: Simplify asset serving logic

* Plugins CDN: sanitize redirect path

* Plugins CDN: Removed unused pluginAsset type

* Plugins CDN: Removed system.js changes

* Plugins CDN: Return different system.js baseURL and module for cdn plugins

* Plugins CDN: Ensure CDN is disabled for non-external plugins

* lint

* Plugins CDN: serve images and screenshots from CDN, refactoring

* Lint

* Plugins CDN: Fix URLs for system.js (baseUrl and module)

* Plugins CDN: Add more tests for RelativeURLForSystemJS

* Plugins CDN: Iterate only on apps when preloading

* Plugins CDN: Refactoring

* Plugins CDN: Add comments to url_constructor.go

* Plugins CDN: Update defaultHGPluginsCDNBaseURL

* Plugins CDN: undo extract meta from system js config

* refactor(plugins): migrate systemjs css plugin to typescript

* feat(plugins): introduce systemjs cdn loader plugin

* feat(plugins): add systemjs load type

* Plugins CDN: Removed RelativeURLForSystemJS

* Plugins CDN: Log backend redirect hits along with plugin info

* Plugins CDN: Add pluginsCDNBasePath to getFrontendSettingsMap

* feat(plugins): introduce cdn loading for angular plugins

* refactor(plugins): move systemjs cache buster into systemjsplugins directory

* Plugins CDN: Rename pluginsCDNBasePath to pluginsCDNBaseURL

* refactor(plugins): introduce pluginsCDNBaseURL to the frontend

* Plugins CDN: Renamed "cdn base path" to "cdn url template" in backend

* Plugins CDN: lint

* merge with main

* Instrumentation: Add prometheus counter for backend hits, log from Info to Warn

* Config: Changed key from plugins_cdn.url to plugins.plugins_cdn_base_url

* CDN: Add backend tests

* Lint: goimports

* Default CDN URL to empty string,

* Do not use CDN in setImages and module if the url template is empty

* CDN: Backend: Add test for frontend settings

* CDN: Do not log missing module.js warn if plugin is being loaded from CDN

* CDN: Add backend test for CDN plugin loader

* Removed 'cdn' signature level, switch to 'valid'

* Fix pfs.TestParseTreeTestdata for cdn plugin testdata dir

* Fix TestLoader_Load

* Fix gocyclo complexity of loadPlugins

* Plugins CDN: Moved prometheus metric to api package, removed asset_path label

* Fix missing  in config

* Changes after review

* Add pluginscdn.Service

* Fix tests

* Refactoring

* Moved all remaining CDN checks inside pluginscdn.Service

* CDN url constructor: Renamed stringURLFor to stringPath

* CDN: Moved asset URL functionality to assetpath service

* CDN: Renamed HasCDN() to IsEnabled()

* CDN: Replace assert with require

* CDN: Changes after review

* Assetpath: Handle url.Parse error

* Fix plugin_resource_test

* CDN: Change fallback redirect from 302 to 307

* goimports

* Fix tests

* Switch to contextmodel.ReqContext in plugins.go

Co-authored-by: Will Browne <will.browne@grafana.com>
Co-authored-by: Jack Westbrook <jack.westbrook@gmail.com>

public/app/features/plugins/admin/state/actions.ts

@@ -5,7 +5,7 @@ import { getBackendSrv, isFetchError } from '@grafana/runtime';
 import { importPanelPlugin } from 'app/features/plugins/importPanelPlugin';
 import { StoreState, ThunkResult } from 'app/types';
 
-import { invalidatePluginInCache } from '../../pluginCacheBuster';
+import { invalidatePluginInCache } from '../../systemjsPlugins/pluginCacheBuster';
 import {
   getRemotePlugins,
   getPluginErrors,

public/app/features/plugins/plugin_loader.ts

@@ -32,7 +32,9 @@ import * as ticks from 'app/core/utils/ticks';
 import { GenericDataSourcePlugin } from '../datasources/types';
 
 import builtInPlugins from './built_in_plugins';
-import { locateWithCache, registerPluginInCache } from './pluginCacheBuster';
+import { locateFromCDN, translateForCDN } from './systemjsPlugins/pluginCDN';
+import { fetchCSS, locateCSS } from './systemjsPlugins/pluginCSS';
+import { locateWithCache, registerPluginInCache } from './systemjsPlugins/pluginCacheBuster';
 
 // Help the 6.4 to 6.5 migration
 // The base classes were moved from @grafana/ui to @grafana/data
@@ -43,7 +45,12 @@ grafanaUI.DataSourcePlugin = grafanaData.DataSourcePlugin;
 grafanaUI.AppPlugin = grafanaData.AppPlugin;
 grafanaUI.DataSourceApi = grafanaData.DataSourceApi;
 
+grafanaRuntime.SystemJS.registry.set('css', grafanaRuntime.SystemJS.newModule({ locate: locateCSS, fetch: fetchCSS }));
 grafanaRuntime.SystemJS.registry.set('plugin-loader', grafanaRuntime.SystemJS.newModule({ locate: locateWithCache }));
+grafanaRuntime.SystemJS.registry.set(
+  'cdn-loader',
+  grafanaRuntime.SystemJS.newModule({ locate: locateFromCDN, translate: translateForCDN })
+);
 
 grafanaRuntime.SystemJS.config({
   baseURL: 'public',
@@ -52,10 +59,12 @@ grafanaRuntime.SystemJS.config({
     plugins: {
       defaultExtension: 'js',
     },
+    'plugin-cdn': {
+      defaultExtension: 'js',
+    },
   },
   map: {
     text: 'vendor/plugin-text/text.js',
-    css: 'vendor/plugin-css/css.js',
   },
   meta: {
     '/*': {
@@ -63,6 +72,14 @@ grafanaRuntime.SystemJS.config({
       authorization: true,
       loader: 'plugin-loader',
     },
+    '*.css': {
+      loader: 'css',
+    },
+    'plugin-cdn/*': {
+      esModule: true,
+      authorization: false,
+      loader: 'cdn-loader',
+    },
   },
 });
 

public/app/features/plugins/systemjsPlugins/pluginCDN.test.ts

@@ -0,0 +1,105 @@
+import { config } from '@grafana/runtime';
+
+import { translateForCDN, extractPluginNameVersionFromUrl } from './pluginCDN';
+describe('Plugin CDN', () => {
+  describe('translateForCDN', () => {
+    const load = {
+      name: 'http://localhost:3000/public/plugin-cdn/grafana-worldmap-panel/0.3.3/grafana-worldmap-panel/module.js',
+      address: 'http://my-host.com/grafana-worldmap-panel/0.3.3/grafana-worldmap-panel/module.js',
+      source: 'public/plugins/grafana-worldmap-panel/template.html',
+      metadata: {
+        extension: '',
+        deps: [],
+        format: 'amd',
+        loader: 'cdn-loader',
+        encapsulateGlobal: false,
+        cjsRequireDetection: true,
+        cjsDeferDepsExecute: false,
+        esModule: true,
+        authorization: false,
+      },
+    };
+    config.pluginsCDNBaseURL = 'http://my-host.com';
+
+    it('should update the default local path to use the CDN path', () => {
+      const translatedLoad = translateForCDN({
+        ...load,
+        source: 'public/plugins/grafana-worldmap-panel/template.html',
+      });
+      expect(translatedLoad).toBe(
+        'http://my-host.com/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/template.html'
+      );
+    });
+
+    it('should replace the default path in a multi-line source code', () => {
+      const source = `
+        const a = "public/plugins/grafana-worldmap-panel/template.html";
+        const img = "<img src='public/plugins/grafana-worldmap-panel/data/myimage.jpg'>";
+      `;
+      const expectedSource = `
+        const a = "http://my-host.com/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/template.html";
+        const img = "<img src='http://my-host.com/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/data/myimage.jpg'>";
+      `;
+      const translatedLoad = translateForCDN({ ...load, source });
+      expect(translatedLoad).toBe(expectedSource);
+    });
+
+    it('should cater for local paths starting with a slash', () => {
+      const source = `
+        const a = "/public/plugins/grafana-worldmap-panel/template.html";
+        const img = "<img src='public/plugins/grafana-worldmap-panel/data/myimage.jpg'>";
+      `;
+      const expectedSource = `
+        const a = "http://my-host.com/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/template.html";
+        const img = "<img src='http://my-host.com/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/data/myimage.jpg'>";
+      `;
+      const translatedLoad = translateForCDN({ ...load, source });
+      expect(translatedLoad).toBe(expectedSource);
+    });
+
+    it('should cater for a particular path', () => {
+      const source = `
+        .getJSON(
+          "public/plugins/grafana-worldmap-panel/data/" +
+            this.panel.locationData +
+            ".json"
+        )
+      `;
+      const expectedSource = `
+        .getJSON(
+          "http://my-host.com/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/data/" +
+            this.panel.locationData +
+            ".json"
+        )
+      `;
+      const translatedLoad = translateForCDN({ ...load, source });
+      expect(translatedLoad).toBe(expectedSource);
+    });
+
+    it('should replace sourcemap locations', () => {
+      const source = `
+        Zn(t,e)},t.Rectangle=ui,t.rectangle=function(t,e){return new ui(t,e)},t.Map=He,t.map=function(t,e){return new He(t,e)}}(e)}])});
+        //# sourceMappingURL=module.js.map
+      `;
+      const expectedSource = `
+        Zn(t,e)},t.Rectangle=ui,t.rectangle=function(t,e){return new ui(t,e)},t.Map=He,t.map=function(t,e){return new He(t,e)}}(e)}])});
+        //# sourceMappingURL=http://my-host.com/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/module.js.map
+      `;
+      const translatedLoad = translateForCDN({ ...load, source });
+      expect(translatedLoad).toBe(expectedSource);
+    });
+  });
+
+  describe('extractPluginNameVersionFromUrl', () => {
+    it('should extract the plugin name and version from a path', () => {
+      const source =
+        'http://localhost:3000/public/plugin-cdn/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/module.js';
+      const expected = {
+        name: 'grafana-worldmap-panel',
+        version: '0.3.3',
+      };
+      const expectedExtractedPluginDeets = extractPluginNameVersionFromUrl(source);
+      expect(expectedExtractedPluginDeets).toEqual(expected);
+    });
+  });
+});

public/app/features/plugins/systemjsPlugins/pluginCDN.ts

@@ -0,0 +1,29 @@
+import { config } from '@grafana/runtime';
+
+import type { SystemJSLoad } from './types';
+
+export function extractPluginNameVersionFromUrl(address: string) {
+  const path = new URL(address).pathname;
+  const match = path.split('/');
+  return { name: match[3], version: match[4] };
+}
+
+export function locateFromCDN(load: SystemJSLoad) {
+  const { address } = load;
+  const pluginPath = address.split('/public/plugin-cdn/');
+  return `${config.pluginsCDNBaseURL}/${pluginPath[1]}`;
+}
+
+export function translateForCDN(load: SystemJSLoad) {
+  const { name, version } = extractPluginNameVersionFromUrl(load.name);
+  const baseAddress = `${config.pluginsCDNBaseURL}/${name}/${version}`;
+
+  load.source = load.source.replace(/(\/?)(public\/plugins)/g, `${baseAddress}/$2`);
+  load.source = load.source.replace(/(["|'])(plugins\/.+.css)(["|'])/g, `$1${baseAddress}/public/$2$3`);
+  load.source = load.source.replace(
+    /(\/\/#\ssourceMappingURL=)(.+)\.map/g,
+    `$1${baseAddress}/public/plugins/${name}/$2.map`
+  );
+
+  return load.source;
+}

public/app/features/plugins/systemjsPlugins/pluginCSS.ts

@@ -0,0 +1,75 @@
+import { noop } from 'lodash';
+
+import { config } from '@grafana/runtime';
+
+import type { SystemJSLoad } from './types';
+
+/*
+  Locate: Overrides the location of the plugin resource
+  Plugins that import css use relative paths in Systemjs.register dependency list.
+  Rather than attempt to resolve it in the pluginCDN systemjs plugin let SystemJS resolve it to origin
+  then we can replace the "baseUrl" with the "cdnHost".
+ */
+export function locateCSS(load: SystemJSLoad) {
+  if (load.metadata.loader === 'cdn-loader' && load.address.startsWith(`${location.origin}/public/plugin-cdn`)) {
+    load.address = load.address.replace(`${location.origin}/public/plugin-cdn`, config.pluginsCDNBaseURL);
+  }
+  return load.address;
+}
+
+/*
+  Fetch: Called with second argument representing default fetch function, has full control of fetch output.
+  Plugins that have external CSS will use this plugin to load their custom styles
+*/
+export function fetchCSS(load: SystemJSLoad) {
+  const links = document.getElementsByTagName('link');
+  const linkHrefs: string[] = Array.from(links).map((link) => link.href);
+
+  // dont reload styles loaded in the head
+  if (linkHrefs.includes(load.address)) {
+    return '';
+  }
+
+  return loadCSS(load.address);
+}
+
+const bust = '?_cache=' + Date.now();
+const waitSeconds = 100;
+
+function loadCSS(url: string) {
+  return new Promise(function (resolve, reject) {
+    const timeout = setTimeout(function () {
+      reject('Unable to load CSS');
+    }, waitSeconds * 1000);
+    const _callback = function (error?: string | Error) {
+      clearTimeout(timeout);
+      link.onload = link.onerror = noop;
+      setTimeout(function () {
+        if (error) {
+          reject(error);
+        } else {
+          resolve('');
+        }
+      }, 7);
+    };
+    const link = document.createElement('link');
+    link.type = 'text/css';
+    link.rel = 'stylesheet';
+    link.href = url;
+
+    // Don't cache bust plugins loaded from cdn.
+    if (!link.href.startsWith(config.pluginsCDNBaseURL)) {
+      link.href = link.href + bust;
+    }
+
+    link.onload = function () {
+      _callback();
+    };
+
+    link.onerror = function (event) {
+      _callback(event instanceof ErrorEvent ? event.message : new Error('Error loading CSS file.'));
+    };
+
+    document.head.appendChild(link);
+  });
+}

public/app/features/plugins/systemjsPlugins/pluginCacheBuster.test.ts

@@ -1,6 +1,7 @@
-import { invalidatePluginInCache, locateWithCache, registerPluginInCache } from '../pluginCacheBuster';
 import * as pluginSettings from '../pluginSettings';
 
+import { invalidatePluginInCache, locateWithCache, registerPluginInCache } from './pluginCacheBuster';
+
 describe('PluginCacheBuster', () => {
   const now = 12345;
 

public/app/features/plugins/systemjsPlugins/pluginCacheBuster.ts

@@ -1,4 +1,4 @@
-import { clearPluginSettingsCache } from './pluginSettings';
+import { clearPluginSettingsCache } from '../pluginSettings';
 
 const cache: Record<string, string> = {};
 const initializedAt: number = Date.now();

public/app/features/plugins/systemjsPlugins/types.ts

@@ -0,0 +1,16 @@
+export type SystemJSLoad = {
+  address: string;
+  metadata: {
+    authorization: boolean;
+    cjsDeferDepsExecute: boolean;
+    cjsRequireDetection: boolean;
+    crossOrigin?: boolean;
+    encapsulateGlobal: boolean;
+    esModule: boolean;
+    integrity?: string;
+    loader: string;
+    scriptLoad?: boolean;
+  };
+  name: string;
+  source: string;
+};