grafana/toolkit: improve CircleCI integration (#18071)

* don't actually install grafana in the setup step

* updat eversion changes

* add report stage

* update versions

* don't do failing test

* upate version

* print plugins

* update versions

* copy docs

* Update package.json
This commit is contained in:
Ryan McKinley 2019-07-12 01:25:38 -07:00 committed by Dominik Prokop
parent 1cbec50866
commit ca628832ab
7 changed files with 149 additions and 138 deletions

View File

@ -2,5 +2,5 @@
"npmClient": "yarn",
"useWorkspaces": true,
"packages": ["packages/*"],
"version": "6.4.0-alpha.6"
"version": "6.4.0-alpha.12"
}

View File

@ -1,6 +1,6 @@
{
"name": "@grafana/data",
"version": "6.4.0-alpha.2",
"version": "6.4.0-alpha.12",
"description": "Grafana Data Library",
"keywords": [
"typescript"

View File

@ -1,6 +1,6 @@
{
"name": "@grafana/runtime",
"version": "6.4.0-alpha.2",
"version": "6.4.0-alpha.12",
"description": "Grafana Runtime Library",
"keywords": [
"grafana"

View File

@ -1,6 +1,6 @@
{
"name": "@grafana/toolkit",
"version": "6.4.0-alpha.6",
"version": "6.4.0-alpha.12",
"description": "Grafana Toolkit",
"keywords": [
"grafana",

View File

@ -15,10 +15,11 @@ import { closeMilestoneTask } from './tasks/closeMilestone';
import { pluginDevTask } from './tasks/plugin.dev';
import {
ciBuildPluginTask,
ciBuildPluginDocsTask,
ciBundlePluginTask,
ciTestPluginTask,
ciPluginReportTask,
ciDeployPluginTask,
ciSetupPluginTask,
} from './tasks/plugin.ci';
import { buildPackageTask } from './tasks/package.build';
@ -148,14 +149,21 @@ export const run = (includeInternalScripts = false) => {
program
.command('plugin:ci-build')
.option('--platform <platform>', 'For backend task, which backend to run')
.option('--backend <backend>', 'For backend task, which backend to run')
.description('Build the plugin, leaving artifacts in /dist')
.action(async cmd => {
await execTask(ciBuildPluginTask)({
platform: cmd.platform,
backend: cmd.backend,
});
});
program
.command('plugin:ci-docs')
.description('Build the HTML docs')
.action(async cmd => {
await execTask(ciBuildPluginDocsTask)({});
});
program
.command('plugin:ci-bundle')
.description('Create a zip artifact for the plugin')
@ -163,24 +171,23 @@ export const run = (includeInternalScripts = false) => {
await execTask(ciBundlePluginTask)({});
});
program
.command('plugin:ci-setup')
.option('--installer <installer>', 'Name of installer to download and run')
.description('Install and configure grafana')
.action(async cmd => {
await execTask(ciSetupPluginTask)({
installer: cmd.installer,
});
});
program
.command('plugin:ci-test')
.option('--full', 'run all the tests (even stuff that will break)')
.description('end-to-end test using bundle in /artifacts')
.action(async cmd => {
await execTask(ciTestPluginTask)({
platform: cmd.platform,
full: cmd.full,
});
});
program
.command('plugin:ci-report')
.description('Build a report for this whole process')
.action(async cmd => {
await execTask(ciPluginReportTask)({});
});
program
.command('plugin:ci-deploy')
.description('Publish plugin CI results')

View File

@ -9,8 +9,8 @@ import path = require('path');
import fs = require('fs');
export interface PluginCIOptions {
platform?: string;
installer?: string;
backend?: string;
full?: boolean;
}
const calcJavascriptSize = (base: string, files?: string[]): number => {
@ -48,32 +48,6 @@ const getJobFromProcessArgv = () => {
return 'unknown_job';
};
// /**
// * Like cp -rn... BUT error if an destination file exists
// */
// async function copyDirErrorIfExists(src:string,dest:string) {
// const entries = await fs.readdirSync(src,{withFileTypes:true});
// if(!fs.existsSync(dest)) {
// fs.mkdirSync(dest);
// }
// console.log( 'DIR', src );
// for(let entry of entries) {
// const srcPath = path.join(src,entry.name);
// const destPath = path.join(dest,entry.name);
// if(entry.isDirectory()) {
// await copyDirErrorIfExists(srcPath,destPath);
// } else if(fs.existsSync(destPath)) {
// console.log( 'XXXXXXXXXXXXXXX', destPath );
// console.log( 'XXXXXXXXXXXXXXX', destPath );
// throw new Error('Duplicate entry: '+destPath);
// }
// else {
// // console.log( 'COPY', destPath );
// await fs.copyFileSync(srcPath,destPath);
// }
// }
// }
const job = process.env.CIRCLE_JOB || getJobFromProcessArgv();
const getJobFolder = () => {
@ -117,17 +91,17 @@ const writeJobStats = (startTime: number, workDir: string) => {
* Anything that should be put into the final zip file should be put in:
* ~/work/build_xxx/dist
*/
const buildPluginRunner: TaskRunner<PluginCIOptions> = async ({ platform }) => {
const buildPluginRunner: TaskRunner<PluginCIOptions> = async ({ backend }) => {
const start = Date.now();
const workDir = getJobFolder();
await execa('rimraf', [workDir]);
fs.mkdirSync(workDir);
if (platform) {
if (backend) {
console.log('TODO, backend support?');
fs.mkdirSync(path.resolve(process.cwd(), 'dist'));
const file = path.resolve(process.cwd(), 'dist', `README_${platform}.txt`);
fs.writeFile(file, `TODO... build ${platform}!`, err => {
const file = path.resolve(process.cwd(), 'dist', `README_${backend}.txt`);
fs.writeFile(file, `TODO... build bakend plugin: ${backend}!`, err => {
if (err) {
throw new Error('Unable to write: ' + file);
}
@ -149,6 +123,40 @@ const buildPluginRunner: TaskRunner<PluginCIOptions> = async ({ platform }) => {
export const ciBuildPluginTask = new Task<PluginCIOptions>('Build Plugin', buildPluginRunner);
/**
* 2. Build Docs
*
* Take /docs/* and format it into /ci/docs/HTML site
*
*/
const buildPluginDocsRunner: TaskRunner<PluginCIOptions> = async () => {
const docsSrc = path.resolve(process.cwd(), 'docs');
if (!fs.existsSync(docsSrc)) {
throw new Error('Docs folder does not exist!');
}
const start = Date.now();
const workDir = getJobFolder();
await execa('rimraf', [workDir]);
fs.mkdirSync(workDir);
const docsDest = path.resolve(process.cwd(), 'ci', 'docs');
fs.mkdirSync(docsDest);
const exe = await execa('cp', ['-rv', docsSrc + '/.', docsDest]);
console.log(exe.stdout);
fs.writeFile(path.resolve(docsDest, 'index.html'), `TODO... actually build docs`, err => {
if (err) {
throw new Error('Unable to docs');
}
});
writeJobStats(start, workDir);
};
export const ciBuildPluginDocsTask = new Task<PluginCIOptions>('Build Plugin Docs', buildPluginDocsRunner);
/**
* 2. BUNDLE
*
@ -163,6 +171,7 @@ const bundlePluginRunner: TaskRunner<PluginCIOptions> = async () => {
const ciDir = getCiFolder();
const artifactsDir = path.resolve(ciDir, 'artifacts');
const distDir = path.resolve(ciDir, 'dist');
const docsDir = path.resolve(ciDir, 'docs');
const grafanaEnvDir = path.resolve(ciDir, 'grafana-test-env');
await execa('rimraf', [artifactsDir, distDir, grafanaEnvDir]);
fs.mkdirSync(artifactsDir);
@ -192,8 +201,8 @@ const bundlePluginRunner: TaskRunner<PluginCIOptions> = async () => {
console.log('Building ZIP');
const pluginInfo = getPluginJson(`${distDir}/plugin.json`);
const zipName = pluginInfo.id + '-' + pluginInfo.info.version + '.zip';
const zipFile = path.resolve(artifactsDir, zipName);
let zipName = pluginInfo.id + '-' + pluginInfo.info.version + '.zip';
let zipFile = path.resolve(artifactsDir, zipName);
process.chdir(distDir);
await execa('zip', ['-r', zipFile, '.']);
restoreCwd();
@ -202,21 +211,49 @@ const bundlePluginRunner: TaskRunner<PluginCIOptions> = async () => {
if (zipStats.size < 100) {
throw new Error('Invalid zip file: ' + zipFile);
}
let sha1 = undefined;
const zipInfo: any = {
name: zipName,
size: zipStats.size,
};
const info: any = {
plugin: zipInfo,
};
try {
const exe = await execa('shasum', [zipFile]);
const idx = exe.stdout.indexOf(' ');
sha1 = exe.stdout.substring(0, idx);
const sha1 = exe.stdout.substring(0, idx);
fs.writeFile(zipFile + '.sha1', sha1, err => {});
zipInfo.sha1 = sha1;
} catch {
console.warn('Unable to read SHA1 Checksum');
}
const info = {
// If docs exist, zip them into artifacts
if (fs.existsSync(docsDir)) {
zipName = pluginInfo.id + '-' + pluginInfo.info.version + '-docs.zip';
zipFile = path.resolve(artifactsDir, zipName);
process.chdir(docsDir);
await execa('zip', ['-r', zipFile, '.']);
restoreCwd();
const zipStats = fs.statSync(zipFile);
const zipInfo: any = {
name: zipName,
sha1,
size: zipStats.size,
};
try {
const exe = await execa('shasum', [zipFile]);
const idx = exe.stdout.indexOf(' ');
const sha1 = exe.stdout.substring(0, idx);
fs.writeFile(zipFile + '.sha1', sha1, err => {});
zipInfo.sha1 = sha1;
} catch {
console.warn('Unable to read SHA1 Checksum');
}
info.docs = zipInfo;
}
let p = path.resolve(artifactsDir, 'info.json');
fs.writeFile(p, JSON.stringify(info, null, 2), err => {
if (err) {
@ -248,66 +285,15 @@ const bundlePluginRunner: TaskRunner<PluginCIOptions> = async () => {
export const ciBundlePluginTask = new Task<PluginCIOptions>('Bundle Plugin', bundlePluginRunner);
/**
* 3. Setup (install grafana and setup provisioning)
* 3. Test (end-to-end)
*
* deploy the zip to a running grafana instance
*
*/
const setupPluginRunner: TaskRunner<PluginCIOptions> = async ({ installer }) => {
const start = Date.now();
if (!installer) {
throw new Error('Missing installer path');
}
// Download the grafana installer
const installDir = path.resolve(process.cwd(), '.installer');
const installFile = path.resolve(installDir, installer);
if (!fs.existsSync(installFile)) {
if (!fs.existsSync(installDir)) {
fs.mkdirSync(installDir);
}
console.log('download', installer);
const exe = await execa('wget', ['-O', installFile, 'https://dl.grafana.com/oss/release/' + installer]);
console.log(exe.stdout);
}
console.log('Install Grafana');
let exe = await execa('sudo', ['apt-get', 'install', '-y', 'adduser', 'libfontconfig1']);
exe = await execa('sudo', ['dpkg', '-i', installFile]);
console.log(exe.stdout);
const customIniFile = path.resolve(getCiFolder(), 'grafana-test-env', 'custom.ini');
const configDir = '/usr/share/grafana/conf/';
exe = await execa('sudo', ['cp', '-f', customIniFile, configDir]);
console.log(exe.stdout);
// sudo service grafana-server start
console.log('Starting Grafana');
exe = await execa('sudo', ['service', 'grafana-server', 'start']);
console.log(exe.stdout);
// exe = await execa('grafana-cli', ['--version', '--homepath', '/usr/share/grafana']);
// console.log(exe.stdout);
// exe = await execa('grafana-cli', ['plugins', 'ls', '--homepath', '/usr/share/grafana']);
// console.log(exe.stdout);
const dir = getJobFolder() + '_setup';
await execa('rimraf', [dir]);
fs.mkdirSync(dir);
writeJobStats(start, dir);
};
export const ciSetupPluginTask = new Task<PluginCIOptions>('Setup Grafana', setupPluginRunner);
/**
* 4. Test (end-to-end)
*
* deploy the zip to a running grafana instance
*
*/
const testPluginRunner: TaskRunner<PluginCIOptions> = async ({ platform }) => {
const testPluginRunner: TaskRunner<PluginCIOptions> = async ({ full }) => {
const start = Date.now();
const workDir = getJobFolder();
const pluginInfo = getPluginJson(`${process.cwd()}/src/plugin.json`);
const args = {
withCredentials: true,
@ -324,10 +310,22 @@ const testPluginRunner: TaskRunner<PluginCIOptions> = async ({ platform }) => {
console.log('Grafana Version: ' + JSON.stringify(frontendSettings.data.buildInfo, null, 2));
const pluginInfo = getPluginJson(`${process.cwd()}/src/plugin.json`);
const pluginSettings = await axios.get(`api/plugins/${pluginInfo.id}/settings`, args);
const allPlugins: any[] = await axios.get('api/plugins', args).data;
// for (const plugin of allPlugins) {
// if (plugin.id === pluginInfo.id) {
// console.log('------------');
// console.log(plugin);
// console.log('------------');
// } else {
// console.log('Plugin:', plugin.id, plugin.latestVersion);
// }
// }
console.log('PLUGINS:', allPlugins);
if (full) {
const pluginSettings = await axios.get(`api/plugins/${pluginInfo.id}/settings`, args);
console.log('Plugin Info: ' + JSON.stringify(pluginSettings.data, null, 2));
}
console.log('TODO puppeteer');
@ -347,33 +345,39 @@ const testPluginRunner: TaskRunner<PluginCIOptions> = async ({ platform }) => {
export const ciTestPluginTask = new Task<PluginCIOptions>('Test Plugin (e2e)', testPluginRunner);
/**
* 4. Deploy
* 4. Report
*
* Create a report from all the previous steps
*
*/
const pluginReportRunner: TaskRunner<PluginCIOptions> = async () => {
const start = Date.now();
const workDir = getJobFolder();
const reportDir = path.resolve(process.cwd(), 'ci', 'report');
await execa('rimraf', [reportDir]);
fs.mkdirSync(reportDir);
const file = path.resolve(reportDir, `report.txt`);
fs.writeFile(file, `TODO... actually make a report (csv etc)`, err => {
if (err) {
throw new Error('Unable to write: ' + file);
}
});
console.log('TODO... real report');
writeJobStats(start, workDir);
};
export const ciPluginReportTask = new Task<PluginCIOptions>('Deploy plugin', pluginReportRunner);
/**
* 5. Deploy
*
* deploy the zip to a running grafana instance
*
*/
const deployPluginRunner: TaskRunner<PluginCIOptions> = async () => {
const start = Date.now();
// TASK Time
if (process.env.CIRCLE_INTERNAL_TASK_DATA) {
const timingInfo = fs.readdirSync(`${process.env.CIRCLE_INTERNAL_TASK_DATA}`);
if (timingInfo) {
timingInfo.forEach(file => {
console.log('TIMING INFO: ', file);
});
}
}
const elapsed = Date.now() - start;
const stats = {
job,
sha1: `${process.env.CIRCLE_SHA1}`,
startTime: start,
buildTime: elapsed,
endTime: Date.now(),
};
console.log('TODO DEPLOY??', stats);
console.log('TODO DEPLOY??');
console.log(' if PR => write a comment to github with difference ');
console.log(' if master | vXYZ ==> upload artifacts to some repo ');
};

View File

@ -1,6 +1,6 @@
{
"name": "@grafana/ui",
"version": "6.4.0-alpha.2",
"version": "6.4.0-alpha.12",
"description": "Grafana Components Library",
"keywords": [
"grafana",