From 1f018adbf31b06b1c98660430c8303d383212420 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 14 Nov 2019 00:15:36 -0800 Subject: [PATCH] grafana/toolkit: remove aws-sdk and upload to grafana.com API endpoint (#20372) * remove aws-sdk and upload directly * remove unused imports * put the plugin file in the root directory --- packages/grafana-toolkit/package.json | 1 - .../src/cli/tasks/plugin.ci.ts | 107 ++-------- packages/grafana-toolkit/src/plugins/aws.ts | 183 ------------------ packages/grafana-toolkit/src/plugins/index.ts | 1 - yarn.lock | 62 +----- 5 files changed, 18 insertions(+), 336 deletions(-) delete mode 100644 packages/grafana-toolkit/src/plugins/aws.ts diff --git a/packages/grafana-toolkit/package.json b/packages/grafana-toolkit/package.json index 66577fba211..c53e8d3a0ad 100644 --- a/packages/grafana-toolkit/package.json +++ b/packages/grafana-toolkit/package.json @@ -42,7 +42,6 @@ "@types/semver": "^6.0.0", "@types/tmp": "^0.1.0", "@types/webpack": "4.4.34", - "aws-sdk": "^2.495.0", "axios": "0.19.0", "babel-jest": "24.8.0", "babel-loader": "8.0.6", diff --git a/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts b/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts index f8de948fa46..1a2b7d6e9ed 100644 --- a/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts +++ b/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts @@ -1,7 +1,6 @@ import { Task, TaskRunner } from './task'; import { pluginBuildRunner } from './plugin.build'; import { restoreCwd } from '../utils/cwd'; -import { S3Client } from '../../plugins/aws'; import { getPluginJson } from '../../config/utils/pluginValidation'; import { getPluginId } from '../../config/utils/getPluginId'; import { PluginMeta } from '@grafana/data'; @@ -10,28 +9,18 @@ import { PluginMeta } from '@grafana/data'; import execa = require('execa'); import path = require('path'); import fs from 'fs'; -import { getPackageDetails, findImagesInFolder, appendPluginHistory, getGrafanaVersions } from '../../plugins/utils'; +import { getPackageDetails, findImagesInFolder, getGrafanaVersions } from '../../plugins/utils'; import { job, getJobFolder, writeJobStats, getCiFolder, getPluginBuildInfo, - getBuildNumber, getPullRequestNumber, getCircleDownloadBaseURL, } from '../../plugins/env'; import { agregateWorkflowInfo, agregateCoverageInfo, agregateTestInfo } from '../../plugins/workflow'; -import { - PluginPackageDetails, - PluginBuildReport, - PluginHistory, - defaultPluginHistory, - TestResultsInfo, - PluginDevInfo, - PluginDevSummary, - DevSummary, -} from '../../plugins/types'; +import { PluginPackageDetails, PluginBuildReport, TestResultsInfo } from '../../plugins/types'; import { runEndToEndTests } from '../../plugins/e2e/launcher'; import { getEndToEndSettings } from '../../plugins/index'; @@ -185,6 +174,9 @@ const packagePluginRunner: TaskRunner = async () => { throw new Error('Invalid zip file: ' + zipFile); } + // Make a copy so it is easy for report to read + await execa('cp', [pluginJsonFile, distDir]); + const info: PluginPackageDetails = { plugin: await getPackageDetails(zipFile, distDir), }; @@ -346,88 +338,19 @@ const pluginReportRunner: TaskRunner = async ({ upload }) => { } }); - console.log('Initalizing S3 Client'); - const s3 = new S3Client(); - - const build = pluginMeta.info.build; - if (!build) { - throw new Error('Metadata missing build info'); + const GRAFANA_API_KEY = process.env.GRAFANA_API_KEY; + if (!GRAFANA_API_KEY) { + console.log('Enter a GRAFANA_API_KEY to upload the plugin report'); + return; } + const url = `https://grafana.com/api/plugins/${report.plugin.id}/ci`; - const version = pluginMeta.info.version || 'unknown'; - const branch = build.branch || 'unknown'; - const buildNumber = getBuildNumber(); - const root = `dev/${pluginMeta.id}`; - const dirKey = pr ? `${root}/pr/${pr}/${buildNumber}` : `${root}/branch/${branch}/${buildNumber}`; - - const jobKey = `${dirKey}/index.json`; - if (await s3.exists(jobKey)) { - throw new Error('Job already registered: ' + jobKey); - } - - console.log('Write Job', jobKey); - await s3.writeJSON(jobKey, report, { - Tagging: `version=${version}&type=${pluginMeta.type}`, + console.log('Sending report to:', url); + const axios = require('axios'); + const info = await axios.post(url, report, { + headers: { Authorization: 'bearer ' + GRAFANA_API_KEY }, }); - - // Upload logo - const logo = await s3.uploadLogo(report.plugin.info, { - local: path.resolve(ciDir, 'dist'), - remote: root, - }); - - const latest: PluginDevInfo = { - pluginId: pluginMeta.id, - name: pluginMeta.name, - logo, - build: pluginMeta.info.build!, - version, - }; - - let base = `${root}/branch/${branch}/`; - latest.build.number = buildNumber; - if (pr) { - latest.build.pr = pr; - base = `${root}/pr/${pr}/`; - } - - const historyKey = base + `history.json`; - console.log('Read', historyKey); - const history: PluginHistory = await s3.readJSON(historyKey, defaultPluginHistory); - appendPluginHistory(report, latest, history); - - await s3.writeJSON(historyKey, history); - console.log('wrote history'); - - // Private things may want to upload - if (upload) { - s3.uploadPackages(packageInfo, { - local: packageDir, - remote: dirKey + '/packages', - }); - - s3.uploadTestFiles(report.tests, { - local: ciDir, - remote: dirKey, - }); - } - - console.log('Update Directory Indexes'); - - let indexKey = `${root}/index.json`; - const index: PluginDevSummary = await s3.readJSON(indexKey, { branch: {}, pr: {} }); - if (pr) { - index.pr[pr] = latest; - } else { - index.branch[branch] = latest; - } - await s3.writeJSON(indexKey, index); - - indexKey = `dev/index.json`; - const pluginIndex: DevSummary = await s3.readJSON(indexKey, {}); - pluginIndex[pluginMeta.id] = latest; - await s3.writeJSON(indexKey, pluginIndex); - console.log('wrote index'); + console.log('RESULT: ', info); }; export const ciPluginReportTask = new Task('Generate Plugin Report', pluginReportRunner); diff --git a/packages/grafana-toolkit/src/plugins/aws.ts b/packages/grafana-toolkit/src/plugins/aws.ts deleted file mode 100644 index 038ca011be8..00000000000 --- a/packages/grafana-toolkit/src/plugins/aws.ts +++ /dev/null @@ -1,183 +0,0 @@ -import AWS from 'aws-sdk'; -import path from 'path'; -import fs from 'fs'; - -import { PluginPackageDetails, ZipFileInfo, TestResultsInfo } from './types'; -import defaults from 'lodash/defaults'; -import clone from 'lodash/clone'; -import { PluginMetaInfo } from '@grafana/data'; - -interface UploadArgs { - local: string; - remote: string; -} - -export class S3Client { - readonly bucket: string; - readonly prefix: string; - readonly s3: AWS.S3; - - constructor(bucket?: string) { - this.bucket = bucket || 'grafana-experiments'; - this.prefix = 'plugins/'; - - this.s3 = new AWS.S3({ apiVersion: '2006-03-01' }); - this.s3.headBucket({ Bucket: this.bucket }, (err, data) => { - if (err) { - throw new Error('Unable to read: ' + this.bucket); - } else { - console.log('s3: ' + data); - } - }); - } - - private async uploadPackage(file: ZipFileInfo, folder: UploadArgs): Promise { - const fpath = path.resolve(process.cwd(), folder.local, file.name); - return await this.uploadFile(fpath, folder.remote + '/' + file.name, file.md5); - } - - async uploadPackages(packageInfo: PluginPackageDetails, folder: UploadArgs) { - await this.uploadPackage(packageInfo.plugin, folder); - if (packageInfo.docs) { - await this.uploadPackage(packageInfo.docs, folder); - } - } - - async uploadTestFiles(tests: TestResultsInfo[], folder: UploadArgs) { - for (const test of tests) { - for (const s of test.screenshots) { - const img = path.resolve(folder.local, 'jobs', test.job, s); - await this.uploadFile(img, folder.remote + `/jobs/${test.job}/${s}`); - } - } - } - - async uploadLogo(meta: PluginMetaInfo, folder: UploadArgs): Promise { - const { logos } = meta; - if (logos && logos.large) { - const img = folder.local + '/' + logos.large; - const idx = img.lastIndexOf('.'); - const name = 'logo' + img.substring(idx); - const key = folder.remote + '/' + name; - await this.uploadFile(img, key); - return name; - } - return undefined; - } - - async uploadFile(fpath: string, path: string, md5?: string): Promise { - if (!fs.existsSync(fpath)) { - return Promise.reject('File not found: ' + fpath); - } - console.log('Uploading: ' + fpath); - const stream = fs.createReadStream(fpath); - return new Promise((resolve, reject) => { - this.s3.putObject( - { - Key: this.prefix + path, - Bucket: this.bucket, - Body: stream, - ContentType: getContentTypeForFile(path), - }, - (err, data) => { - if (err) { - reject(err); - } else { - if (md5 && md5 !== data.ETag && `"${md5}"` !== data.ETag) { - reject(`Upload ETag does not match MD5 (${md5} !== ${data.ETag})`); - } else { - resolve(data.ETag); - } - } - } - ); - }); - } - - async exists(key: string): Promise { - return new Promise((resolve, reject) => { - this.s3.getObject( - { - Bucket: this.bucket, - Key: this.prefix + key, - }, - (err, data) => { - if (err) { - resolve(false); - } else { - resolve(true); - } - } - ); - }); - } - - async readJSON(key: string, defaultValue: T): Promise { - return new Promise((resolve, reject) => { - this.s3.getObject( - { - Bucket: this.bucket, - Key: this.prefix + key, - }, - (err, data) => { - if (err) { - resolve(clone(defaultValue)); - } else { - try { - const v = JSON.parse(data.Body as string); - resolve(defaults(v, defaultValue)); - } catch (e) { - console.log('ERROR', e); - reject('Error reading response'); - } - } - } - ); - }); - } - - async writeJSON( - key: string, - obj: {}, - params?: Partial - ): Promise { - return new Promise((resolve, reject) => { - this.s3.putObject( - { - ...params, - Key: this.prefix + key, - Bucket: this.bucket, - Body: JSON.stringify(obj, null, 2), // Pretty print - ContentType: 'application/json', - }, - (err, data) => { - if (err) { - reject(err); - } else { - resolve(data); - } - } - ); - }); - } -} - -function getContentTypeForFile(name: string): string | undefined { - const idx = name.lastIndexOf('.'); - if (idx > 0) { - const ext = name.substring(idx + 1).toLowerCase(); - if (ext === 'zip') { - return 'application/zip'; - } - if (ext === 'json') { - return 'application/json'; - } - if (ext === 'svg') { - return 'image/svg+xml'; - } - if (ext === 'png') { - return 'image/png'; - } - } - return undefined; -} diff --git a/packages/grafana-toolkit/src/plugins/index.ts b/packages/grafana-toolkit/src/plugins/index.ts index e3536f3e518..1ffef8d3cd2 100644 --- a/packages/grafana-toolkit/src/plugins/index.ts +++ b/packages/grafana-toolkit/src/plugins/index.ts @@ -1,4 +1,3 @@ -export * from './aws'; export * from './env'; export * from './utils'; export * from './workflow'; diff --git a/yarn.lock b/yarn.lock index e37c11e32e8..2464a222901 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4782,21 +4782,6 @@ autoprefixer@^9.4.9: postcss "^7.0.18" postcss-value-parser "^4.0.2" -aws-sdk@^2.495.0: - version "2.553.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.553.0.tgz#a82c611015138db8f720e0079fe929a65b359a8e" - integrity sha512-tcITF/3ijBumvP13Qrq/VB1eRWW6szKF0xVwZ/ch0MGkaEiTQ9n3zNRPtQc1drllsVEm5u5aXp3inoi5zmq0xg== - dependencies: - buffer "4.9.1" - events "1.1.1" - ieee754 "1.1.13" - jmespath "0.15.0" - querystring "0.2.0" - sax "1.2.1" - url "0.10.3" - uuid "3.3.2" - xml2js "0.4.19" - aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -5638,7 +5623,7 @@ buffer-xor@^1.0.3: resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk= -buffer@4.9.1, buffer@^4.3.0: +buffer@^4.3.0: version "4.9.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" integrity sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg= @@ -8886,11 +8871,6 @@ eventemitter3@^4.0.0: resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.0.tgz#d65176163887ee59f386d64c82610b696a4a74eb" integrity sha512-qerSRB0p+UDEssxTtm6EDKcE7W4OaoisfIMl4CngyEhjpYglocpNg6UEqCvemdGhosAsg4sO2dXJOdyBifPGCg== -events@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" - integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= - events@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/events/-/events-3.0.0.tgz#9a0a0dfaf62893d92b875b8f2698ca4114973e88" @@ -10943,7 +10923,7 @@ icss-utils@^4.0.0, icss-utils@^4.1.0, icss-utils@^4.1.1: dependencies: postcss "^7.0.14" -ieee754@1.1.13, ieee754@^1.1.4: +ieee754@^1.1.4: version "1.1.13" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== @@ -12270,11 +12250,6 @@ jest@24.8.0: import-local "^2.0.0" jest-cli "^24.8.0" -jmespath@0.15.0: - version "0.15.0" - resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" - integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= - jquery@3.4.1: version "3.4.1" resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.4.1.tgz#714f1f8d9dde4bdfa55764ba37ef214630d80ef2" @@ -18497,12 +18472,7 @@ sass-loader@7.1.0: pify "^3.0.0" semver "^5.5.0" -sax@1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" - integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= - -sax@>=0.6.0, sax@^1.2.4, sax@~1.2.4: +sax@^1.2.4, sax@~1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== @@ -20833,14 +20803,6 @@ url-parse@^1.4.3: querystringify "^2.1.1" requires-port "^1.0.0" -url@0.10.3: - version "0.10.3" - resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" - integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= - dependencies: - punycode "1.3.2" - querystring "0.2.0" - url@^0.11.0: version "0.11.0" resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" @@ -20928,11 +20890,6 @@ utils-merge@1.0.1: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= -uuid@3.3.2: - version "3.3.2" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" - integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== - uuid@^3.0.1, uuid@^3.1.0, uuid@^3.3.2: version "3.3.3" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.3.tgz#4568f0216e78760ee1dbf3a4d2cf53e224112866" @@ -21590,24 +21547,11 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== -xml2js@0.4.19: - version "0.4.19" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" - integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== - dependencies: - sax ">=0.6.0" - xmlbuilder "~9.0.1" - xml@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU= -xmlbuilder@~9.0.1: - version "9.0.7" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" - integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= - xmlhttprequest@1: version "1.8.0" resolved "https://registry.yarnpkg.com/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz#67fe075c5c24fef39f9d65f5f7b7fe75171968fc"