Plugins: Allow loading panel plugins from a CDN (#59096)

* POC: Plugins CDN reverse proxy

* CDN proxy POC: changed env var names

* Add authorization: false for /public path in frontend plugin loader

* Moved CDN settings to Cfg, add some comments

* Fix error 500 in asset fetch if plugin is not using CDN

* Fix EnterpriseLicensePath declared twice

* Fix linter complaining about whitespaces

* Plugins CDN: Skip signature verification for CDN plugins

* Plugins CDN: Skip manifest and signature check for cdn plugins

* Plugins: use IsValid() and IsInternal() rather than equality checks

* Plugins CDN: remove comment

* Plugins CDN: Fix seeker can't seek when serving plugins from local fs

* Plugins CDN: add back error codes in getLocalPluginAssets

* Plugins CDN: call asset.Close() rather than asset.readSeekCloser.Close()

* Plugins CDN: Fix panic in JsonApiErr when errorMessageCoder wraps a nil error

* Plugins CDN: Add error handling to proxyCDNPluginAsset

* Plugins CDN: replace errorMessageCoder with errutil

* Plugins CDN POC: expose cdn plugin paths to frontend for system.js

* Plugins CDN: Fix cdn plugins showing as unsigned in frontend

* WIP: Add support for formatted URL

* Fix missing cdnPluginsBaseURLs in GrafanaConfig

* Plugins CDN: Remove reverse proxy mode and reverse proxy references

* Plugins CDN: Simplify asset serving logic

* Plugins CDN: sanitize redirect path

* Plugins CDN: Removed unused pluginAsset type

* Plugins CDN: Removed system.js changes

* Plugins CDN: Return different system.js baseURL and module for cdn plugins

* Plugins CDN: Ensure CDN is disabled for non-external plugins

* lint

* Plugins CDN: serve images and screenshots from CDN, refactoring

* Lint

* Plugins CDN: Fix URLs for system.js (baseUrl and module)

* Plugins CDN: Add more tests for RelativeURLForSystemJS

* Plugins CDN: Iterate only on apps when preloading

* Plugins CDN: Refactoring

* Plugins CDN: Add comments to url_constructor.go

* Plugins CDN: Update defaultHGPluginsCDNBaseURL

* Plugins CDN: undo extract meta from system js config

* refactor(plugins): migrate systemjs css plugin to typescript

* feat(plugins): introduce systemjs cdn loader plugin

* feat(plugins): add systemjs load type

* Plugins CDN: Removed RelativeURLForSystemJS

* Plugins CDN: Log backend redirect hits along with plugin info

* Plugins CDN: Add pluginsCDNBasePath to getFrontendSettingsMap

* feat(plugins): introduce cdn loading for angular plugins

* refactor(plugins): move systemjs cache buster into systemjsplugins directory

* Plugins CDN: Rename pluginsCDNBasePath to pluginsCDNBaseURL

* refactor(plugins): introduce pluginsCDNBaseURL to the frontend

* Plugins CDN: Renamed "cdn base path" to "cdn url template" in backend

* Plugins CDN: lint

* merge with main

* Instrumentation: Add prometheus counter for backend hits, log from Info to Warn

* Config: Changed key from plugins_cdn.url to plugins.plugins_cdn_base_url

* CDN: Add backend tests

* Lint: goimports

* Default CDN URL to empty string,

* Do not use CDN in setImages and module if the url template is empty

* CDN: Backend: Add test for frontend settings

* CDN: Do not log missing module.js warn if plugin is being loaded from CDN

* CDN: Add backend test for CDN plugin loader

* Removed 'cdn' signature level, switch to 'valid'

* Fix pfs.TestParseTreeTestdata for cdn plugin testdata dir

* Fix TestLoader_Load

* Fix gocyclo complexity of loadPlugins

* Plugins CDN: Moved prometheus metric to api package, removed asset_path label

* Fix missing  in config

* Changes after review

* Add pluginscdn.Service

* Fix tests

* Refactoring

* Moved all remaining CDN checks inside pluginscdn.Service

* CDN url constructor: Renamed stringURLFor to stringPath

* CDN: Moved asset URL functionality to assetpath service

* CDN: Renamed HasCDN() to IsEnabled()

* CDN: Replace assert with require

* CDN: Changes after review

* Assetpath: Handle url.Parse error

* Fix plugin_resource_test

* CDN: Change fallback redirect from 302 to 307

* goimports

* Fix tests

* Switch to contextmodel.ReqContext in plugins.go

Co-authored-by: Will Browne <will.browne@grafana.com>
Co-authored-by: Jack Westbrook <jack.westbrook@gmail.com>
This commit is contained in:
Giuseppe Guerra
2023-01-27 15:08:17 +01:00
committed by GitHub
co-authored by Will Browne Jack Westbrook
parent c931b8031e
commit af1e2d68da
35 changed files with 1139 additions and 188 deletions
@@ -0,0 +1,70 @@
package assetpath
import (
"fmt"
"net/url"
"path"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/pluginscdn"
)
// Service provides methods for constructing asset paths for plugins.
// It supports core plugins, external plugins stored on the local filesystem, and external plugins stored
// on the plugins CDN, and it will switch to the correct implementation depending on the plugin and the config.
type Service struct {
cdn *pluginscdn.Service
}
func ProvideService(cdn *pluginscdn.Service) *Service {
return &Service{cdn: cdn}
}
// Base returns the base path for the specified plugin.
func (s *Service) Base(pluginJSON plugins.JSONData, class plugins.Class, pluginDir string) (string, error) {
if class == plugins.Core {
return path.Join("public/app/plugins", string(pluginJSON.Type), filepath.Base(pluginDir)), nil
}
if s.cdn.PluginSupported(pluginJSON.ID) {
return s.cdn.SystemJSAssetPath(pluginJSON.ID, pluginJSON.Info.Version, "")
}
return path.Join("public/plugins", pluginJSON.ID), nil
}
// Module returns the module.js path for the specified plugin.
func (s *Service) Module(pluginJSON plugins.JSONData, class plugins.Class, pluginDir string) (string, error) {
if class == plugins.Core {
return path.Join("app/plugins", string(pluginJSON.Type), filepath.Base(pluginDir), "module"), nil
}
if s.cdn.PluginSupported(pluginJSON.ID) {
return s.cdn.SystemJSAssetPath(pluginJSON.ID, pluginJSON.Info.Version, "module")
}
return path.Join("plugins", pluginJSON.ID, "module"), nil
}
// RelativeURL returns the relative URL for an arbitrary plugin asset.
// If pathStr is an empty string, defaultStr is returned.
func (s *Service) RelativeURL(p *plugins.Plugin, pathStr, defaultStr string) (string, error) {
if pathStr == "" {
return defaultStr, nil
}
if s.cdn.PluginSupported(p.ID) {
// CDN
return s.cdn.NewCDNURLConstructor(p.ID, p.Info.Version).StringPath(pathStr)
}
// Local
u, err := url.Parse(pathStr)
if err != nil {
return "", fmt.Errorf("url parse: %w", err)
}
if u.IsAbs() {
return pathStr, nil
}
// is set as default or has already been prefixed with base path
if pathStr == defaultStr || strings.HasPrefix(pathStr, p.BaseURL) {
return pathStr, nil
}
return path.Join(p.BaseURL, pathStr), nil
}
@@ -0,0 +1,88 @@
package assetpath
import (
"testing"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/config"
"github.com/grafana/grafana/pkg/plugins/pluginscdn"
"github.com/stretchr/testify/require"
)
func extPath(pluginID string) string {
return "/grafana/data/plugins/" + pluginID
}
func TestService(t *testing.T) {
svc := ProvideService(pluginscdn.ProvideService(&config.Cfg{
PluginsCDNURLTemplate: "https://cdn.example.com/{id}/{version}/public/plugins/{id}/{assetPath}",
PluginSettings: map[string]map[string]string{
"one": {"cdn": "true"},
"two": {},
},
}))
const tableOldPath = "/grafana/public/app/plugins/panel/table-old"
jsonData := map[string]plugins.JSONData{
"table-old": {ID: "table-old", Info: plugins.Info{Version: "1.0.0"}},
"one": {ID: "one", Info: plugins.Info{Version: "1.0.0"}},
"two": {ID: "two", Info: plugins.Info{Version: "2.0.0"}},
}
t.Run("Base", func(t *testing.T) {
base, err := svc.Base(jsonData["one"], plugins.External, extPath("one"))
require.NoError(t, err)
require.Equal(t, "plugin-cdn/one/1.0.0/public/plugins/one", base)
base, err = svc.Base(jsonData["two"], plugins.External, extPath("two"))
require.NoError(t, err)
require.Equal(t, "public/plugins/two", base)
base, err = svc.Base(jsonData["table-old"], plugins.Core, tableOldPath)
require.NoError(t, err)
require.Equal(t, "public/app/plugins/table-old", base)
})
t.Run("Module", func(t *testing.T) {
module, err := svc.Module(jsonData["one"], plugins.External, extPath("one"))
require.NoError(t, err)
require.Equal(t, "plugin-cdn/one/1.0.0/public/plugins/one/module", module)
module, err = svc.Module(jsonData["two"], plugins.External, extPath("two"))
require.NoError(t, err)
require.Equal(t, "plugins/two/module", module)
module, err = svc.Module(jsonData["table-old"], plugins.Core, tableOldPath)
require.NoError(t, err)
require.Equal(t, "app/plugins/table-old/module", module)
})
t.Run("RelativeURL", func(t *testing.T) {
pluginsMap := map[string]*plugins.Plugin{
"one": {
JSONData: plugins.JSONData{ID: "one", Info: plugins.Info{Version: "1.0.0"}},
BaseURL: "plugin-cdn/one/1.0.0/public/pluginsMap/one",
},
"two": {
JSONData: plugins.JSONData{ID: "two", Info: plugins.Info{Version: "2.0.0"}},
BaseURL: "public/pluginsMap/two",
},
}
u, err := svc.RelativeURL(pluginsMap["one"], "", "default")
require.NoError(t, err)
require.Equal(t, "default", u)
u, err = svc.RelativeURL(pluginsMap["one"], "path/to/file.txt", "default")
require.NoError(t, err)
require.Equal(t, "https://cdn.example.com/one/1.0.0/public/plugins/one/path/to/file.txt", u)
u, err = svc.RelativeURL(pluginsMap["two"], "path/to/file.txt", "default")
require.NoError(t, err)
require.Equal(t, "public/pluginsMap/two/path/to/file.txt", u)
u, err = svc.RelativeURL(pluginsMap["two"], "default", "default")
require.NoError(t, err)
require.Equal(t, "default", u)
})
}
+77 -73
View File
@@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"path"
"path/filepath"
@@ -19,11 +18,13 @@ import (
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/config"
"github.com/grafana/grafana/pkg/plugins/logger"
"github.com/grafana/grafana/pkg/plugins/manager/loader/assetpath"
"github.com/grafana/grafana/pkg/plugins/manager/loader/finder"
"github.com/grafana/grafana/pkg/plugins/manager/loader/initializer"
"github.com/grafana/grafana/pkg/plugins/manager/process"
"github.com/grafana/grafana/pkg/plugins/manager/registry"
"github.com/grafana/grafana/pkg/plugins/manager/signature"
"github.com/grafana/grafana/pkg/plugins/pluginscdn"
"github.com/grafana/grafana/pkg/plugins/storage"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/util"
@@ -44,21 +45,25 @@ type Loader struct {
pluginInitializer initializer.Initializer
signatureValidator signature.Validator
pluginStorage storage.Manager
pluginsCDN *pluginscdn.Service
assetPath *assetpath.Service
log log.Logger
cfg *config.Cfg
errs map[string]*plugins.SignatureError
}
func ProvideService(cfg *config.Cfg, license plugins.Licensing, authorizer plugins.PluginLoaderAuthorizer,
pluginRegistry registry.Service, backendProvider plugins.BackendFactoryProvider,
roleRegistry plugins.RoleRegistry) *Loader {
roleRegistry plugins.RoleRegistry, pluginsCDNService *pluginscdn.Service, assetPath *assetpath.Service) *Loader {
return New(cfg, license, authorizer, pluginRegistry, backendProvider, process.NewManager(pluginRegistry),
storage.FileSystem(logger.NewLogger("loader.fs"), cfg.PluginsPath), roleRegistry)
storage.FileSystem(logger.NewLogger("loader.fs"), cfg.PluginsPath), roleRegistry, pluginsCDNService, assetPath)
}
func New(cfg *config.Cfg, license plugins.Licensing, authorizer plugins.PluginLoaderAuthorizer,
pluginRegistry registry.Service, backendProvider plugins.BackendFactoryProvider,
processManager process.Service, pluginStorage storage.Manager, roleRegistry plugins.RoleRegistry) *Loader {
processManager process.Service, pluginStorage storage.Manager, roleRegistry plugins.RoleRegistry,
pluginsCDNService *pluginscdn.Service, assetPath *assetpath.Service) *Loader {
return &Loader{
pluginFinder: finder.New(),
pluginRegistry: pluginRegistry,
@@ -69,6 +74,9 @@ func New(cfg *config.Cfg, license plugins.Licensing, authorizer plugins.PluginLo
errs: make(map[string]*plugins.SignatureError),
log: log.New("plugin.loader"),
roleRegistry: roleRegistry,
cfg: cfg,
pluginsCDN: pluginsCDNService,
assetPath: assetPath,
}
}
@@ -81,6 +89,36 @@ func (l *Loader) Load(ctx context.Context, class plugins.Class, paths []string)
return l.loadPlugins(ctx, class, pluginJSONPaths)
}
func (l *Loader) createPluginsForLoading(class plugins.Class, foundPlugins foundPlugins) map[string]*plugins.Plugin {
loadedPlugins := make(map[string]*plugins.Plugin)
for pluginDir, pluginJSON := range foundPlugins {
plugin, err := l.createPluginBase(pluginJSON, class, pluginDir)
if err != nil {
l.log.Warn("Could not create plugin base", "pluginID", pluginJSON.ID, "err", err)
continue
}
// calculate initial signature state
var sig plugins.Signature
if l.pluginsCDN.PluginSupported(plugin.ID) {
// CDN plugins have no signature checks for now.
sig = plugins.Signature{Status: plugins.SignatureValid}
} else {
sig, err = signature.Calculate(l.log, plugin)
if err != nil {
l.log.Warn("Could not calculate plugin signature state", "pluginID", plugin.ID, "err", err)
continue
}
}
plugin.Signature = sig.Status
plugin.SignatureType = sig.Type
plugin.SignatureOrg = sig.SigningOrg
loadedPlugins[plugin.PluginDir] = plugin
}
return loadedPlugins
}
func (l *Loader) loadPlugins(ctx context.Context, class plugins.Class, pluginJSONPaths []string) ([]*plugins.Plugin, error) {
var foundPlugins = foundPlugins{}
@@ -113,22 +151,8 @@ func (l *Loader) loadPlugins(ctx context.Context, class plugins.Class, pluginJSO
foundPlugins.stripDuplicates(registeredPlugins, l.log)
// calculate initial signature state
loadedPlugins := make(map[string]*plugins.Plugin)
for pluginDir, pluginJSON := range foundPlugins {
plugin := createPluginBase(pluginJSON, class, pluginDir)
sig, err := signature.Calculate(l.log, plugin)
if err != nil {
l.log.Warn("Could not calculate plugin signature state", "pluginID", plugin.ID, "err", err)
continue
}
plugin.Signature = sig.Status
plugin.SignatureType = sig.Type
plugin.SignatureOrg = sig.SigningOrg
loadedPlugins[plugin.PluginDir] = plugin
}
// create plugins structs and calculate signatures
loadedPlugins := l.createPluginsForLoading(class, foundPlugins)
// wire up plugin dependencies
for _, plugin := range loadedPlugins {
@@ -165,12 +189,13 @@ func (l *Loader) loadPlugins(ctx context.Context, class plugins.Class, pluginJSO
// clear plugin error if a pre-existing error has since been resolved
delete(l.errs, plugin.ID)
// verify module.js exists for SystemJS to load
// verify module.js exists for SystemJS to load.
// CDN plugins can be loaded with plugin.json only, so do not warn for those.
if !plugin.IsRenderer() && !plugin.IsCorePlugin() {
module := filepath.Join(plugin.PluginDir, "module.js")
if exists, err := fs.Exists(module); err != nil {
return nil, err
} else if !exists {
} else if !exists && !l.pluginsCDN.PluginSupported(plugin.ID) {
l.log.Warn("Plugin missing module.js",
"pluginID", plugin.ID,
"warning", "Missing module.js, If you loaded this plugin from git, make sure to compile it.",
@@ -312,28 +337,47 @@ func (l *Loader) readPluginJSON(pluginJSONPath string) (plugins.JSONData, error)
return plugin, nil
}
func createPluginBase(pluginJSON plugins.JSONData, class plugins.Class, pluginDir string) *plugins.Plugin {
func (l *Loader) createPluginBase(pluginJSON plugins.JSONData, class plugins.Class, pluginDir string) (*plugins.Plugin, error) {
baseURL, err := l.assetPath.Base(pluginJSON, class, pluginDir)
if err != nil {
return nil, fmt.Errorf("base url: %w", err)
}
moduleURL, err := l.assetPath.Module(pluginJSON, class, pluginDir)
if err != nil {
return nil, fmt.Errorf("module url: %w", err)
}
plugin := &plugins.Plugin{
JSONData: pluginJSON,
PluginDir: pluginDir,
BaseURL: baseURL(pluginJSON, class, pluginDir),
Module: module(pluginJSON, class, pluginDir),
BaseURL: baseURL,
Module: moduleURL,
Class: class,
}
plugin.SetLogger(log.New(fmt.Sprintf("plugin.%s", plugin.ID)))
setImages(plugin)
if err := l.setImages(plugin); err != nil {
return nil, err
}
return plugin
return plugin, nil
}
func setImages(p *plugins.Plugin) {
p.Info.Logos.Small = pluginLogoURL(p.Type, p.Info.Logos.Small, p.BaseURL)
p.Info.Logos.Large = pluginLogoURL(p.Type, p.Info.Logos.Large, p.BaseURL)
for i := 0; i < len(p.Info.Screenshots); i++ {
p.Info.Screenshots[i].Path = evalRelativePluginURLPath(p.Info.Screenshots[i].Path, p.BaseURL, p.Type)
func (l *Loader) setImages(p *plugins.Plugin) error {
var err error
for _, dst := range []*string{&p.Info.Logos.Small, &p.Info.Logos.Large} {
*dst, err = l.assetPath.RelativeURL(p, *dst, defaultLogoPath(p.Type))
if err != nil {
return fmt.Errorf("logo: %w", err)
}
}
for i := 0; i < len(p.Info.Screenshots); i++ {
screenshot := &p.Info.Screenshots[i]
screenshot.Path, err = l.assetPath.RelativeURL(p, screenshot.Path, "")
if err != nil {
return fmt.Errorf("screenshot %d relative url: %w", i, err)
}
}
return nil
}
func setDefaultNavURL(p *plugins.Plugin) {
@@ -377,36 +421,10 @@ func configureAppChildPlugin(parent *plugins.Plugin, child *plugins.Plugin) {
}
}
func pluginLogoURL(pluginType plugins.Type, path, baseURL string) string {
if path == "" {
return defaultLogoPath(pluginType)
}
return evalRelativePluginURLPath(path, baseURL, pluginType)
}
func defaultLogoPath(pluginType plugins.Type) string {
return "public/img/icn-" + string(pluginType) + ".svg"
}
func evalRelativePluginURLPath(pathStr, baseURL string, pluginType plugins.Type) string {
if pathStr == "" {
return ""
}
u, _ := url.Parse(pathStr)
if u.IsAbs() {
return pathStr
}
// is set as default or has already been prefixed with base path
if pathStr == defaultLogoPath(pluginType) || strings.HasPrefix(pathStr, baseURL) {
return pathStr
}
return path.Join(baseURL, pathStr)
}
func (l *Loader) PluginErrors() []*plugins.Error {
errs := make([]*plugins.Error, 0)
for _, err := range l.errs {
@@ -419,20 +437,6 @@ func (l *Loader) PluginErrors() []*plugins.Error {
return errs
}
func baseURL(pluginJSON plugins.JSONData, class plugins.Class, pluginDir string) string {
if class == plugins.Core {
return path.Join("public/app/plugins", string(pluginJSON.Type), filepath.Base(pluginDir))
}
return path.Join("public/plugins", pluginJSON.ID)
}
func module(pluginJSON plugins.JSONData, class plugins.Class, pluginDir string) string {
if class == plugins.Core {
return path.Join("app/plugins", string(pluginJSON.Type), filepath.Base(pluginDir), "module")
}
return path.Join("plugins", pluginJSON.ID, "module")
}
func validatePluginJSON(data plugins.JSONData) error {
if data.ID == "" || !data.Type.IsValid() {
return ErrInvalidPluginJSON
+60 -1
View File
@@ -7,6 +7,9 @@ import (
"sort"
"testing"
"github.com/grafana/grafana/pkg/plugins/manager/loader/assetpath"
"github.com/grafana/grafana/pkg/plugins/pluginscdn"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/stretchr/testify/require"
@@ -404,6 +407,61 @@ func TestLoader_Load(t *testing.T) {
},
},
},
{
name: "Load CDN plugin",
class: plugins.External,
cfg: &config.Cfg{
PluginsCDNURLTemplate: "https://cdn.example.com/{id}/{version}/public/plugins/{id}/{assetPath}",
PluginSettings: setting.PluginSettings{
"grafana-worldmap-panel": {"cdn": "true"},
},
},
pluginPaths: []string{"../testdata/cdn"},
want: []*plugins.Plugin{
{
JSONData: plugins.JSONData{
ID: "grafana-worldmap-panel",
Type: "panel",
Name: "Worldmap Panel",
Info: plugins.Info{
Version: "0.3.3",
Links: []plugins.InfoLink{
{Name: "Project site", URL: "https://github.com/grafana/worldmap-panel"},
{Name: "MIT License", URL: "https://github.com/grafana/worldmap-panel/blob/master/LICENSE"},
},
Logos: plugins.Logos{
// Path substitution
Small: "https://cdn.example.com/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/images/worldmap_logo.svg",
Large: "https://cdn.example.com/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/images/worldmap_logo.svg",
},
Screenshots: []plugins.Screenshots{
{
Name: "World",
Path: "https://cdn.example.com/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/images/worldmap-world.png",
},
{
Name: "USA",
Path: "https://cdn.example.com/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/images/worldmap-usa.png",
},
{
Name: "Light Theme",
Path: "https://cdn.example.com/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/images/worldmap-light-theme.png",
},
},
},
Dependencies: plugins.Dependencies{
GrafanaVersion: "3.x.x",
Plugins: []plugins.Dependency{},
},
},
PluginDir: filepath.Join(parentDir, "testdata/cdn/plugin"),
Class: plugins.External,
Signature: plugins.SignatureValid,
BaseURL: "plugin-cdn/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel",
Module: "plugin-cdn/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel/module",
},
},
},
}
for _, tt := range tests {
reg := fakes.NewFakePluginRegistry()
@@ -1320,9 +1378,10 @@ func Test_setPathsBasedOnApp(t *testing.T) {
}
func newLoader(cfg *config.Cfg, cbs ...func(loader *Loader)) *Loader {
cdn := pluginscdn.ProvideService(cfg)
l := New(cfg, &fakes.FakeLicensingService{}, signature.NewUnsignedAuthorizer(cfg), fakes.NewFakePluginRegistry(),
fakes.NewFakeBackendProcessProvider(), fakes.NewFakeProcessManager(), fakes.NewFakePluginStorage(),
fakes.NewFakeRoleRegistry())
fakes.NewFakeRoleRegistry(), cdn, assetpath.ProvideService(cdn))
for _, cb := range cbs {
cb(l)