mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
grafana/toolkit: remove aws-sdk and upload to grafana.com API endpoint (#20372)
* remove aws-sdk and upload directly * remove unused imports * put the plugin file in the root directory
This commit is contained in:
Ryan McKinley
Dominik Prokop
parent
e9668fd251
commit
1f018adbf3
@ -42,7 +42,6 @@
|
||||
"@types/semver": "^6.0.0",
|
||||
"@types/tmp": "^0.1.0",
|
||||
"@types/webpack": "4.4.34",
|
||||
"aws-sdk": "^2.495.0",
|
||||
"axios": "0.19.0",
|
||||
"babel-jest": "24.8.0",
|
||||
"babel-loader": "8.0.6",
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
import { Task, TaskRunner } from './task';
|
||||
import { pluginBuildRunner } from './plugin.build';
|
||||
import { restoreCwd } from '../utils/cwd';
|
||||
import { S3Client } from '../../plugins/aws';
|
||||
import { getPluginJson } from '../../config/utils/pluginValidation';
|
||||
import { getPluginId } from '../../config/utils/getPluginId';
|
||||
import { PluginMeta } from '@grafana/data';
|
||||
@ -10,28 +9,18 @@ import { PluginMeta } from '@grafana/data';
|
||||
import execa = require('execa');
|
||||
import path = require('path');
|
||||
import fs from 'fs';
|
||||
import { getPackageDetails, findImagesInFolder, appendPluginHistory, getGrafanaVersions } from '../../plugins/utils';
|
||||
import { getPackageDetails, findImagesInFolder, getGrafanaVersions } from '../../plugins/utils';
|
||||
import {
|
||||
job,
|
||||
getJobFolder,
|
||||
writeJobStats,
|
||||
getCiFolder,
|
||||
getPluginBuildInfo,
|
||||
getBuildNumber,
|
||||
getPullRequestNumber,
|
||||
getCircleDownloadBaseURL,
|
||||
} from '../../plugins/env';
|
||||
import { agregateWorkflowInfo, agregateCoverageInfo, agregateTestInfo } from '../../plugins/workflow';
|
||||
import {
|
||||
PluginPackageDetails,
|
||||
PluginBuildReport,
|
||||
PluginHistory,
|
||||
defaultPluginHistory,
|
||||
TestResultsInfo,
|
||||
PluginDevInfo,
|
||||
PluginDevSummary,
|
||||
DevSummary,
|
||||
} from '../../plugins/types';
|
||||
import { PluginPackageDetails, PluginBuildReport, TestResultsInfo } from '../../plugins/types';
|
||||
import { runEndToEndTests } from '../../plugins/e2e/launcher';
|
||||
import { getEndToEndSettings } from '../../plugins/index';
|
||||
|
||||
@ -185,6 +174,9 @@ const packagePluginRunner: TaskRunner<PluginCIOptions> = async () => {
|
||||
throw new Error('Invalid zip file: ' + zipFile);
|
||||
}
|
||||
|
||||
// Make a copy so it is easy for report to read
|
||||
await execa('cp', [pluginJsonFile, distDir]);
|
||||
|
||||
const info: PluginPackageDetails = {
|
||||
plugin: await getPackageDetails(zipFile, distDir),
|
||||
};
|
||||
@ -346,88 +338,19 @@ const pluginReportRunner: TaskRunner<PluginCIOptions> = async ({ upload }) => {
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Initalizing S3 Client');
|
||||
const s3 = new S3Client();
|
||||
|
||||
const build = pluginMeta.info.build;
|
||||
if (!build) {
|
||||
throw new Error('Metadata missing build info');
|
||||
const GRAFANA_API_KEY = process.env.GRAFANA_API_KEY;
|
||||
if (!GRAFANA_API_KEY) {
|
||||
console.log('Enter a GRAFANA_API_KEY to upload the plugin report');
|
||||
return;
|
||||
}
|
||||
const url = `https://grafana.com/api/plugins/${report.plugin.id}/ci`;
|
||||
|
||||
const version = pluginMeta.info.version || 'unknown';
|
||||
const branch = build.branch || 'unknown';
|
||||
const buildNumber = getBuildNumber();
|
||||
const root = `dev/${pluginMeta.id}`;
|
||||
const dirKey = pr ? `${root}/pr/${pr}/${buildNumber}` : `${root}/branch/${branch}/${buildNumber}`;
|
||||
|
||||
const jobKey = `${dirKey}/index.json`;
|
||||
if (await s3.exists(jobKey)) {
|
||||
throw new Error('Job already registered: ' + jobKey);
|
||||
}
|
||||
|
||||
console.log('Write Job', jobKey);
|
||||
await s3.writeJSON(jobKey, report, {
|
||||
Tagging: `version=${version}&type=${pluginMeta.type}`,
|
||||
console.log('Sending report to:', url);
|
||||
const axios = require('axios');
|
||||
const info = await axios.post(url, report, {
|
||||
headers: { Authorization: 'bearer ' + GRAFANA_API_KEY },
|
||||
});
|
||||
|
||||
// Upload logo
|
||||
const logo = await s3.uploadLogo(report.plugin.info, {
|
||||
local: path.resolve(ciDir, 'dist'),
|
||||
remote: root,
|
||||
});
|
||||
|
||||
const latest: PluginDevInfo = {
|
||||
pluginId: pluginMeta.id,
|
||||
name: pluginMeta.name,
|
||||
logo,
|
||||
build: pluginMeta.info.build!,
|
||||
version,
|
||||
};
|
||||
|
||||
let base = `${root}/branch/${branch}/`;
|
||||
latest.build.number = buildNumber;
|
||||
if (pr) {
|
||||
latest.build.pr = pr;
|
||||
base = `${root}/pr/${pr}/`;
|
||||
}
|
||||
|
||||
const historyKey = base + `history.json`;
|
||||
console.log('Read', historyKey);
|
||||
const history: PluginHistory = await s3.readJSON(historyKey, defaultPluginHistory);
|
||||
appendPluginHistory(report, latest, history);
|
||||
|
||||
await s3.writeJSON(historyKey, history);
|
||||
console.log('wrote history');
|
||||
|
||||
// Private things may want to upload
|
||||
if (upload) {
|
||||
s3.uploadPackages(packageInfo, {
|
||||
local: packageDir,
|
||||
remote: dirKey + '/packages',
|
||||
});
|
||||
|
||||
s3.uploadTestFiles(report.tests, {
|
||||
local: ciDir,
|
||||
remote: dirKey,
|
||||
});
|
||||
}
|
||||
|
||||
console.log('Update Directory Indexes');
|
||||
|
||||
let indexKey = `${root}/index.json`;
|
||||
const index: PluginDevSummary = await s3.readJSON(indexKey, { branch: {}, pr: {} });
|
||||
if (pr) {
|
||||
index.pr[pr] = latest;
|
||||
} else {
|
||||
index.branch[branch] = latest;
|
||||
}
|
||||
await s3.writeJSON(indexKey, index);
|
||||
|
||||
indexKey = `dev/index.json`;
|
||||
const pluginIndex: DevSummary = await s3.readJSON(indexKey, {});
|
||||
pluginIndex[pluginMeta.id] = latest;
|
||||
await s3.writeJSON(indexKey, pluginIndex);
|
||||
console.log('wrote index');
|
||||
console.log('RESULT: ', info);
|
||||
};
|
||||
|
||||
export const ciPluginReportTask = new Task<PluginCIOptions>('Generate Plugin Report', pluginReportRunner);
|
||||
|
||||
@ -1,183 +0,0 @@
|
||||
import AWS from 'aws-sdk';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
|
||||
import { PluginPackageDetails, ZipFileInfo, TestResultsInfo } from './types';
|
||||
import defaults from 'lodash/defaults';
|
||||
import clone from 'lodash/clone';
|
||||
import { PluginMetaInfo } from '@grafana/data';
|
||||
|
||||
interface UploadArgs {
|
||||
local: string;
|
||||
remote: string;
|
||||
}
|
||||
|
||||
export class S3Client {
|
||||
readonly bucket: string;
|
||||
readonly prefix: string;
|
||||
readonly s3: AWS.S3;
|
||||
|
||||
constructor(bucket?: string) {
|
||||
this.bucket = bucket || 'grafana-experiments';
|
||||
this.prefix = 'plugins/';
|
||||
|
||||
this.s3 = new AWS.S3({ apiVersion: '2006-03-01' });
|
||||
this.s3.headBucket({ Bucket: this.bucket }, (err, data) => {
|
||||
if (err) {
|
||||
throw new Error('Unable to read: ' + this.bucket);
|
||||
} else {
|
||||
console.log('s3: ' + data);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async uploadPackage(file: ZipFileInfo, folder: UploadArgs): Promise<string> {
|
||||
const fpath = path.resolve(process.cwd(), folder.local, file.name);
|
||||
return await this.uploadFile(fpath, folder.remote + '/' + file.name, file.md5);
|
||||
}
|
||||
|
||||
async uploadPackages(packageInfo: PluginPackageDetails, folder: UploadArgs) {
|
||||
await this.uploadPackage(packageInfo.plugin, folder);
|
||||
if (packageInfo.docs) {
|
||||
await this.uploadPackage(packageInfo.docs, folder);
|
||||
}
|
||||
}
|
||||
|
||||
async uploadTestFiles(tests: TestResultsInfo[], folder: UploadArgs) {
|
||||
for (const test of tests) {
|
||||
for (const s of test.screenshots) {
|
||||
const img = path.resolve(folder.local, 'jobs', test.job, s);
|
||||
await this.uploadFile(img, folder.remote + `/jobs/${test.job}/${s}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async uploadLogo(meta: PluginMetaInfo, folder: UploadArgs): Promise<string | undefined> {
|
||||
const { logos } = meta;
|
||||
if (logos && logos.large) {
|
||||
const img = folder.local + '/' + logos.large;
|
||||
const idx = img.lastIndexOf('.');
|
||||
const name = 'logo' + img.substring(idx);
|
||||
const key = folder.remote + '/' + name;
|
||||
await this.uploadFile(img, key);
|
||||
return name;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async uploadFile(fpath: string, path: string, md5?: string): Promise<string> {
|
||||
if (!fs.existsSync(fpath)) {
|
||||
return Promise.reject('File not found: ' + fpath);
|
||||
}
|
||||
console.log('Uploading: ' + fpath);
|
||||
const stream = fs.createReadStream(fpath);
|
||||
return new Promise((resolve, reject) => {
|
||||
this.s3.putObject(
|
||||
{
|
||||
Key: this.prefix + path,
|
||||
Bucket: this.bucket,
|
||||
Body: stream,
|
||||
ContentType: getContentTypeForFile(path),
|
||||
},
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
if (md5 && md5 !== data.ETag && `"${md5}"` !== data.ETag) {
|
||||
reject(`Upload ETag does not match MD5 (${md5} !== ${data.ETag})`);
|
||||
} else {
|
||||
resolve(data.ETag);
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async exists(key: string): Promise<boolean> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.s3.getObject(
|
||||
{
|
||||
Bucket: this.bucket,
|
||||
Key: this.prefix + key,
|
||||
},
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
resolve(false);
|
||||
} else {
|
||||
resolve(true);
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async readJSON<T>(key: string, defaultValue: T): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.s3.getObject(
|
||||
{
|
||||
Bucket: this.bucket,
|
||||
Key: this.prefix + key,
|
||||
},
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
resolve(clone(defaultValue));
|
||||
} else {
|
||||
try {
|
||||
const v = JSON.parse(data.Body as string);
|
||||
resolve(defaults(v, defaultValue));
|
||||
} catch (e) {
|
||||
console.log('ERROR', e);
|
||||
reject('Error reading response');
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async writeJSON(
|
||||
key: string,
|
||||
obj: {},
|
||||
params?: Partial<AWS.S3.Types.PutObjectRequest>
|
||||
): Promise<AWS.S3.Types.PutObjectOutput> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.s3.putObject(
|
||||
{
|
||||
...params,
|
||||
Key: this.prefix + key,
|
||||
Bucket: this.bucket,
|
||||
Body: JSON.stringify(obj, null, 2), // Pretty print
|
||||
ContentType: 'application/json',
|
||||
},
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve(data);
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function getContentTypeForFile(name: string): string | undefined {
|
||||
const idx = name.lastIndexOf('.');
|
||||
if (idx > 0) {
|
||||
const ext = name.substring(idx + 1).toLowerCase();
|
||||
if (ext === 'zip') {
|
||||
return 'application/zip';
|
||||
}
|
||||
if (ext === 'json') {
|
||||
return 'application/json';
|
||||
}
|
||||
if (ext === 'svg') {
|
||||
return 'image/svg+xml';
|
||||
}
|
||||
if (ext === 'png') {
|
||||
return 'image/png';
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@ -1,4 +1,3 @@
|
||||
export * from './aws';
|
||||
export * from './env';
|
||||
export * from './utils';
|
||||
export * from './workflow';
|
||||
|
||||
62
yarn.lock
62
yarn.lock
@ -4782,21 +4782,6 @@ autoprefixer@^9.4.9:
|
||||
postcss "^7.0.18"
|
||||
postcss-value-parser "^4.0.2"
|
||||
|
||||
aws-sdk@^2.495.0:
|
||||
version "2.553.0"
|
||||
resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.553.0.tgz#a82c611015138db8f720e0079fe929a65b359a8e"
|
||||
integrity sha512-tcITF/3ijBumvP13Qrq/VB1eRWW6szKF0xVwZ/ch0MGkaEiTQ9n3zNRPtQc1drllsVEm5u5aXp3inoi5zmq0xg==
|
||||
dependencies:
|
||||
buffer "4.9.1"
|
||||
events "1.1.1"
|
||||
ieee754 "1.1.13"
|
||||
jmespath "0.15.0"
|
||||
querystring "0.2.0"
|
||||
sax "1.2.1"
|
||||
url "0.10.3"
|
||||
uuid "3.3.2"
|
||||
xml2js "0.4.19"
|
||||
|
||||
aws-sign2@~0.7.0:
|
||||
version "0.7.0"
|
||||
resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
|
||||
@ -5638,7 +5623,7 @@ buffer-xor@^1.0.3:
|
||||
resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9"
|
||||
integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=
|
||||
|
||||
buffer@4.9.1, buffer@^4.3.0:
|
||||
buffer@^4.3.0:
|
||||
version "4.9.1"
|
||||
resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298"
|
||||
integrity sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=
|
||||
@ -8886,11 +8871,6 @@ eventemitter3@^4.0.0:
|
||||
resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.0.tgz#d65176163887ee59f386d64c82610b696a4a74eb"
|
||||
integrity sha512-qerSRB0p+UDEssxTtm6EDKcE7W4OaoisfIMl4CngyEhjpYglocpNg6UEqCvemdGhosAsg4sO2dXJOdyBifPGCg==
|
||||
|
||||
events@1.1.1:
|
||||
version "1.1.1"
|
||||
resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
|
||||
integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=
|
||||
|
||||
events@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/events/-/events-3.0.0.tgz#9a0a0dfaf62893d92b875b8f2698ca4114973e88"
|
||||
@ -10943,7 +10923,7 @@ icss-utils@^4.0.0, icss-utils@^4.1.0, icss-utils@^4.1.1:
|
||||
dependencies:
|
||||
postcss "^7.0.14"
|
||||
|
||||
ieee754@1.1.13, ieee754@^1.1.4:
|
||||
ieee754@^1.1.4:
|
||||
version "1.1.13"
|
||||
resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
|
||||
integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==
|
||||
@ -12270,11 +12250,6 @@ jest@24.8.0:
|
||||
import-local "^2.0.0"
|
||||
jest-cli "^24.8.0"
|
||||
|
||||
jmespath@0.15.0:
|
||||
version "0.15.0"
|
||||
resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217"
|
||||
integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=
|
||||
|
||||
jquery@3.4.1:
|
||||
version "3.4.1"
|
||||
resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.4.1.tgz#714f1f8d9dde4bdfa55764ba37ef214630d80ef2"
|
||||
@ -18497,12 +18472,7 @@ sass-loader@7.1.0:
|
||||
pify "^3.0.0"
|
||||
semver "^5.5.0"
|
||||
|
||||
sax@1.2.1:
|
||||
version "1.2.1"
|
||||
resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
|
||||
integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o=
|
||||
|
||||
sax@>=0.6.0, sax@^1.2.4, sax@~1.2.4:
|
||||
sax@^1.2.4, sax@~1.2.4:
|
||||
version "1.2.4"
|
||||
resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
|
||||
integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
|
||||
@ -20833,14 +20803,6 @@ url-parse@^1.4.3:
|
||||
querystringify "^2.1.1"
|
||||
requires-port "^1.0.0"
|
||||
|
||||
url@0.10.3:
|
||||
version "0.10.3"
|
||||
resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64"
|
||||
integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=
|
||||
dependencies:
|
||||
punycode "1.3.2"
|
||||
querystring "0.2.0"
|
||||
|
||||
url@^0.11.0:
|
||||
version "0.11.0"
|
||||
resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1"
|
||||
@ -20928,11 +20890,6 @@ utils-merge@1.0.1:
|
||||
resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
|
||||
integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=
|
||||
|
||||
uuid@3.3.2:
|
||||
version "3.3.2"
|
||||
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"
|
||||
integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==
|
||||
|
||||
uuid@^3.0.1, uuid@^3.1.0, uuid@^3.3.2:
|
||||
version "3.3.3"
|
||||
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.3.tgz#4568f0216e78760ee1dbf3a4d2cf53e224112866"
|
||||
@ -21590,24 +21547,11 @@ xml-name-validator@^3.0.0:
|
||||
resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
|
||||
integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==
|
||||
|
||||
xml2js@0.4.19:
|
||||
version "0.4.19"
|
||||
resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
|
||||
integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
|
||||
dependencies:
|
||||
sax ">=0.6.0"
|
||||
xmlbuilder "~9.0.1"
|
||||
|
||||
xml@^1.0.1:
|
||||
version "1.0.1"
|
||||
resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
|
||||
integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=
|
||||
|
||||
xmlbuilder@~9.0.1:
|
||||
version "9.0.7"
|
||||
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
|
||||
integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=
|
||||
|
||||
xmlhttprequest@1:
|
||||
version "1.8.0"
|
||||
resolved "https://registry.yarnpkg.com/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz#67fe075c5c24fef39f9d65f5f7b7fe75171968fc"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user