grafana/toolkit: initial CI task and various small improvements (#17914)

packages/grafana-toolkit/README.md

@@ -5,15 +5,18 @@ Make sure to run `yarn install` before trying anything!  Otherwise you may see u
 
 
 ## Internal development
-For development use `yarn link`. First, navigate to `packages/grafana-toolkit` and run `yarn link`. Then, in your project run
-```
-yarn add babel-loader ts-loader css-loader style-loader sass-loader html-loader node-sass @babel/preset-env @babel/core & yarn link @grafana/toolkit
-```
+Typically plugins should be developed using the `@grafana/toolkit` import from npm.  However, when working on the toolkit, you may want to use the local version while underdevelopment.  This works, but is a little flakey.
+
+1. navigate to `packages/grafana-toolkit` and run `yarn link`.
+2. in your plugin, run `npx grafana-toolkit plugin:dev --yarnlink`
+
+Step 2 will add all the same dependencies to your development plugin as the toolkit.  These are typically used from the node_modules folder
 
-Note, that for development purposes we are adding `babel-loader ts-loader style-loader sass-loader html-loader node-sass @babel/preset-env @babel/core` packages to your extension. This is due to the specific behavior of `yarn link` which does not install dependencies of linked packages and webpack is having hard time trying to load its extensions.
 
 TODO: Experiment with [yalc](https://github.com/whitecolor/yalc) for linking packages
 
+
+
 ### Publishing to npm
 The publish process is now manual. Follow the steps to publish @grafana/toolkit to npm
 1. From Grafana root dir: `./node_modules/.bin/grafana-toolkit toolkit:build`

packages/grafana-toolkit/package.json

@@ -46,6 +46,7 @@
     "jest-coverage-badges": "^1.1.2",
     "lodash": "4.17.11",
     "mini-css-extract-plugin": "^0.7.0",
+    "ng-annotate-webpack-plugin": "^0.3.0",
     "node-sass": "^4.12.0",
     "optimize-css-assets-webpack-plugin": "^5.0.3",
     "ora": "^3.4.0",

packages/grafana-toolkit/src/cli/index.ts

@@ -15,6 +15,7 @@ import { pluginTestTask } from './tasks/plugin.tests';
 import { searchTestDataSetupTask } from './tasks/searchTestDataSetup';
 import { closeMilestoneTask } from './tasks/closeMilestone';
 import { pluginDevTask } from './tasks/plugin.dev';
+import { pluginCITask } from './tasks/plugin.ci';
 
 export const run = (includeInternalScripts = false) => {
   if (includeInternalScripts) {
@@ -125,16 +126,18 @@ export const run = (includeInternalScripts = false) => {
     .command('plugin:build')
     .description('Prepares plugin dist package')
     .action(async cmd => {
-      await execTask(pluginBuildTask)({});
+      await execTask(pluginBuildTask)({ coverage: false });
     });
 
   program
     .command('plugin:dev')
     .option('-w, --watch', 'Run plugin development mode with watch enabled')
+    .option('--yarnlink', 'symlink this project to the local grafana/toolkit')
     .description('Starts plugin dev mode')
     .action(async cmd => {
       await execTask(pluginDevTask)({
         watch: !!cmd.watch,
+        yarnlink: !!cmd.yarnlink,
       });
     });
 
@@ -150,6 +153,16 @@ export const run = (includeInternalScripts = false) => {
       });
     });
 
+  program
+    .command('plugin:ci')
+    .option('--dryRun', "Dry run (don't post results)")
+    .description('Run Plugin CI task')
+    .action(async cmd => {
+      await execTask(pluginCITask)({
+        dryRun: cmd.dryRun,
+      });
+    });
+
   program.on('command:*', () => {
     console.error('Invalid command: %s\nSee --help for a list of available commands.', program.args.join(' '));
     process.exit(1);

packages/grafana-toolkit/src/cli/tasks/plugin.build.ts

@@ -11,8 +11,9 @@ import * as prettier from 'prettier';
 import { useSpinner } from '../utils/useSpinner';
 import { testPlugin } from './plugin/tests';
 import { bundlePlugin as bundleFn, PluginBundleOptions } from './plugin/bundle';
-
-interface PrecommitOptions {}
+interface PluginBuildOptions {
+  coverage: boolean;
+}
 
 export const bundlePlugin = useSpinner<PluginBundleOptions>('Compiling...', async options => await bundleFn(options));
 
@@ -22,14 +23,25 @@ export const clean = useSpinner<void>('Cleaning', async () => await execa('rimra
 
 export const prepare = useSpinner<void>('Preparing', async () => {
   // Make sure a local tsconfig exists.  Otherwise this will work, but have odd behavior
-  const tsConfigPath = path.resolve(process.cwd(), 'tsconfig.json');
-  if (!fs.existsSync(tsConfigPath)) {
-    const defaultTsConfigPath = path.resolve(__dirname, '../../config/tsconfig.plugin.local.json');
-    fs.copyFile(defaultTsConfigPath, tsConfigPath, err => {
+  let filePath = path.resolve(process.cwd(), 'tsconfig.json');
+  if (!fs.existsSync(filePath)) {
+    const srcFile = path.resolve(__dirname, '../../config/tsconfig.plugin.local.json');
+    fs.copyFile(srcFile, filePath, err => {
       if (err) {
         throw err;
       }
-      console.log('Created tsconfig.json file');
+      console.log(`Created: ${filePath}`);
+    });
+  }
+  // Make sure a local .prettierrc.js exists.  Otherwise this will work, but have odd behavior
+  filePath = path.resolve(process.cwd(), '.prettierrc.js');
+  if (!fs.existsSync(filePath)) {
+    const srcFile = path.resolve(__dirname, '../../config/prettier.plugin.rc.js');
+    fs.copyFile(srcFile, filePath, err => {
+      if (err) {
+        throw err;
+      }
+      console.log(`Created: ${filePath}`);
     });
   }
   return Promise.resolve();
@@ -68,7 +80,8 @@ const prettierCheckPlugin = useSpinner<void>('Prettier check', async () => {
             filepath: s,
           })
         ) {
-          failed = true;
+          console.log('TODO eslint/prettier fix? ' + s);
+          failed = false; //true;
         }
 
         resolve({
@@ -89,7 +102,7 @@ const prettierCheckPlugin = useSpinner<void>('Prettier check', async () => {
 });
 
 // @ts-ignore
-const lintPlugin = useSpinner<void>('Linting', async () => {
+export const lintPlugin = useSpinner<void>('Linting', async () => {
   let tsLintConfigPath = path.resolve(process.cwd(), 'tslint.json');
   if (!fs.existsSync(tsLintConfigPath)) {
     tsLintConfigPath = path.resolve(__dirname, '../../config/tslint.plugin.json');
@@ -131,14 +144,14 @@ const lintPlugin = useSpinner<void>('Linting', async () => {
   }
 });
 
-const pluginBuildRunner: TaskRunner<PrecommitOptions> = async () => {
+export const pluginBuildRunner: TaskRunner<PluginBuildOptions> = async ({ coverage }) => {
   await clean();
   await prepare();
   await prettierCheckPlugin();
   // @ts-ignore
   await lintPlugin();
-  await testPlugin({ updateSnapshot: false, coverage: false });
+  await testPlugin({ updateSnapshot: false, coverage });
   await bundlePlugin({ watch: false, production: true });
 };
 
-export const pluginBuildTask = new Task<PrecommitOptions>('Build plugin', pluginBuildRunner);
+export const pluginBuildTask = new Task<PluginBuildOptions>('Build plugin', pluginBuildRunner);

packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts

@@ -0,0 +1,78 @@
+import { Task, TaskRunner } from './task';
+import { pluginBuildRunner } from './plugin.build';
+import { useSpinner } from '../utils/useSpinner';
+import { restoreCwd } from '../utils/cwd';
+import { getPluginJson } from '../../config/utils/pluginValidation';
+
+// @ts-ignore
+import execa = require('execa');
+import path = require('path');
+import fs = require('fs');
+
+export interface PluginCIOptions {
+  dryRun?: boolean;
+}
+
+const calcJavascriptSize = (base: string, files?: string[]): number => {
+  files = files || fs.readdirSync(base);
+  let size = 0;
+
+  if (files) {
+    files.forEach(file => {
+      const newbase = path.join(base, file);
+      const stat = fs.statSync(newbase);
+      if (stat.isDirectory()) {
+        size += calcJavascriptSize(newbase, fs.readdirSync(newbase));
+      } else {
+        if (file.endsWith('.js')) {
+          size += stat.size;
+        }
+      }
+    });
+  }
+  return size;
+};
+
+const pluginCIRunner: TaskRunner<PluginCIOptions> = async ({ dryRun }) => {
+  const start = Date.now();
+  const distDir = `${process.cwd()}/dist`;
+  const artifactsDir = `${process.cwd()}/artifacts`;
+  await execa('rimraf', [`${process.cwd()}/coverage`]);
+  await execa('rimraf', [artifactsDir]);
+
+  // Do regular build process
+  await pluginBuildRunner({ coverage: true });
+  const elapsed = Date.now() - start;
+
+  if (!fs.existsSync(artifactsDir)) {
+    fs.mkdirSync(artifactsDir);
+  }
+
+  // TODO? can this typed from @grafana/ui?
+  const pluginInfo = getPluginJson(`${distDir}/plugin.json`);
+  const zipName = pluginInfo.id + '-' + pluginInfo.info.version + '.zip';
+  const zipFile = path.resolve(artifactsDir, zipName);
+  process.chdir(distDir);
+  await execa('zip', ['-r', zipFile, '.']);
+  restoreCwd();
+
+  const stats = {
+    startTime: start,
+    buildTime: elapsed,
+    jsSize: calcJavascriptSize(distDir),
+    zipSize: fs.statSync(zipFile).size,
+    endTime: Date.now(),
+  };
+  fs.writeFile(artifactsDir + '/stats.json', JSON.stringify(stats, null, 2), err => {
+    if (err) {
+      throw new Error('Unable to write stats');
+    }
+    console.log('Stats', stats);
+  });
+
+  if (!dryRun) {
+    console.log('TODO send info to github?');
+  }
+};
+
+export const pluginCITask = new Task<PluginCIOptions>('Plugin CI', pluginCIRunner);

packages/grafana-toolkit/src/cli/tasks/plugin.dev.ts

@@ -2,11 +2,41 @@ import { Task, TaskRunner } from './task';
 import { bundlePlugin as bundleFn, PluginBundleOptions } from './plugin/bundle';
 import { useSpinner } from '../utils/useSpinner';
 
+// @ts-ignore
+import execa = require('execa');
+import path = require('path');
+
 const bundlePlugin = useSpinner<PluginBundleOptions>('Bundling plugin in dev mode', options => {
   return bundleFn(options);
 });
 
+const yarnlink = useSpinner<void>('Linking local toolkit', async () => {
+  try {
+    // Make sure we are not using package.json defined toolkit
+    await execa('yarn', ['remove', '@grafana/toolkit']);
+  } catch (e) {
+    console.log('\n', e.message, '\n');
+  }
+  await execa('yarn', ['link', '@grafana/toolkit']);
+
+  // Add all the same dependencies as toolkit
+  const args: string[] = ['add'];
+  const packages = require(path.resolve(__dirname, '../../../package.json'));
+  for (const [key, value] of Object.entries(packages.dependencies)) {
+    args.push(`${key}@${value}`);
+  }
+  await execa('yarn', args);
+
+  console.log('Added dependencies required by local @grafana/toolkit.  Do not checkin this package.json!');
+
+  return Promise.resolve();
+});
+
 const pluginDevRunner: TaskRunner<PluginBundleOptions> = async options => {
+  if (options.yarnlink) {
+    return yarnlink();
+  }
+
   if (options.watch) {
     await bundleFn(options);
   } else {

packages/grafana-toolkit/src/cli/tasks/plugin/bundle.ts

@@ -8,6 +8,7 @@ import clearConsole = require('react-dev-utils/clearConsole');
 export interface PluginBundleOptions {
   watch: boolean;
   production?: boolean;
+  yarnlink?: boolean;
 }
 
 // export const bundlePlugin = useSpinner<PluginBundleOptions>('Bundle plugin', ({ watch }) => {

packages/grafana-toolkit/src/cli/tasks/toolkit.build.ts

@@ -55,6 +55,8 @@ const moveFiles = () => {
     'README.md',
     'CHANGELOG.md',
     'bin/grafana-toolkit.dist.js',
+    'src/config/prettier.plugin.config.json',
+    'src/config/prettier.plugin.rc.js',
     'src/config/tsconfig.plugin.json',
     'src/config/tsconfig.plugin.local.json',
     'src/config/tslint.plugin.json',

packages/grafana-toolkit/src/config/prettier.plugin.rc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  ...require("./node_modules/@grafana/toolkit/src/config/prettier.plugin.config.json"),
+};

packages/grafana-toolkit/src/config/utils/pluginValidation.ts

@@ -1,13 +1,27 @@
 import path = require('path');
 
+// See: packages/grafana-ui/src/types/plugin.ts
 interface PluginJSONSchema {
   id: string;
+  info: PluginMetaInfo;
+}
+
+interface PluginMetaInfo {
+  version: string;
 }
 
 export const validatePluginJson = (pluginJson: any) => {
   if (!pluginJson.id) {
     throw new Error('Plugin id is missing in plugin.json');
   }
+
+  if (!pluginJson.info) {
+    throw new Error('Plugin info node is missing in plugin.json');
+  }
+
+  if (!pluginJson.info.version) {
+    throw new Error('Plugin info.version is missing in plugin.json');
+  }
 };
 
 export const getPluginJson = (root: string = process.cwd()): PluginJSONSchema => {

packages/grafana-toolkit/src/config/webpack.plugin.config.ts

@@ -5,6 +5,7 @@ const ReplaceInFileWebpackPlugin = require('replace-in-file-webpack-plugin');
 const TerserPlugin = require('terser-webpack-plugin');
 const MiniCssExtractPlugin = require('mini-css-extract-plugin');
 const OptimizeCssAssetsPlugin = require('optimize-css-assets-webpack-plugin');
+const ngAnnotatePlugin = require('ng-annotate-webpack-plugin');
 import * as webpack from 'webpack';
 import { hasThemeStylesheets, getStyleLoaders, getStylesheetEntries, getFileLoaders } from './webpack/loaders';
 
@@ -113,6 +114,7 @@ export const getWebpackConfig: WebpackConfigurationGetter = options => {
   const optimization: { [key: string]: any } = {};
 
   if (options.production) {
+    plugins.push(new ngAnnotatePlugin());
     optimization.minimizer = [new TerserPlugin(), new OptimizeCssAssetsPlugin()];
   }
 
@@ -154,11 +156,7 @@ export const getWebpackConfig: WebpackConfigurationGetter = options => {
       '@grafana/data',
       // @ts-ignore
       (context, request, callback) => {
-        let prefix = 'app/';
-        if (request.indexOf(prefix) === 0) {
-          return callback(null, request);
-        }
-        prefix = 'grafana/';
+        const prefix = 'grafana/';
         if (request.indexOf(prefix) === 0) {
           return callback(null, request.substr(prefix.length));
         }

yarn.lock

@@ -2283,10 +2283,10 @@
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.3.tgz#bdfd69d61e464dcc81b25159c270d75a73c1a636"
   integrity sha512-Il2DtDVRGDcqjDtE+rF8iqg1CArehSK84HZJCT7AMITlyXRBpuPhqGLDQMowraqqu1coEaimg4ZOqggt6L6L+A==
 
-"@types/lodash@4.14.123":
-  version "4.14.123"
-  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.123.tgz#39be5d211478c8dd3bdae98ee75bb7efe4abfe4d"
-  integrity sha512-pQvPkc4Nltyx7G1Ww45OjVqUsJP4UsZm+GWJpigXgkikZqJgRm4c48g027o6tdgubWHwFRF15iFd+Y4Pmqv6+Q==
+"@types/lodash@4.14.119", "@types/lodash@4.14.123":
+  version "4.14.119"
+  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.119.tgz#be847e5f4bc3e35e46d041c394ead8b603ad8b39"
+  integrity sha512-Z3TNyBL8Vd/M9D9Ms2S3LmFq2sSMzahodD6rCS9V2N44HUMINb75jNkSuwAx7eo2ufqTdfOdtGQpNbieUjPQmw==
 
 "@types/marked@0.6.5":
   version "0.6.5"
@@ -4216,11 +4216,6 @@ caniuse-api@^3.0.0:
     lodash.memoize "^4.1.2"
     lodash.uniq "^4.5.0"
 
-caniuse-db@1.0.30000772:
-  version "1.0.30000772"
-  resolved "https://registry.yarnpkg.com/caniuse-db/-/caniuse-db-1.0.30000772.tgz#51aae891768286eade4a3d8319ea76d6a01b512b"
-  integrity sha1-UarokXaChureSj2DGep21qAbUSs=
-
 caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000929, caniuse-lite@^1.0.30000947, caniuse-lite@^1.0.30000957, caniuse-lite@^1.0.30000963:
   version "1.0.30000966"
   resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000966.tgz#f3c6fefacfbfbfb981df6dfa68f2aae7bff41b64"
@@ -10758,7 +10753,7 @@ ng-annotate-loader@0.6.1:
     normalize-path "2.0.1"
     source-map "0.5.6"
 
-ng-annotate-webpack-plugin@0.3.0:
+ng-annotate-webpack-plugin@0.3.0, ng-annotate-webpack-plugin@^0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/ng-annotate-webpack-plugin/-/ng-annotate-webpack-plugin-0.3.0.tgz#2e7f5e29c6a4ce26649edcb06c1213408b35b84a"
   dependencies: