Chore: Toolkit fixes to use toolkit for release (#24951)

* Various fixes and requests for toolkit:
1) Added toolkit to the alpine docker image, so we can get latest fixes without having to re-release
2) Removed cp from e2d docker image
3) Change github client to allow to specify an owner so it's not "grafana" only
4) Toolkit refers to grafana-ui and grafana-data "next", rather than canary

* added toolkit to e2e docker image

* Added tools to the e2e image for building/deploying oracle

* added shasum

* removing toolkit from images

packages/grafana-toolkit/bin/grafana-toolkit.js

@@ -15,18 +15,20 @@ const isLinkedMode = () => {
   }
 
   try {
-    return fs.lstatSync(`${__dirname}/../../../node_modules/@grafana/toolkit`).isSymbolicLink();
+    const resolvedPath = path.resolve(`${__dirname}/../../../node_modules/@grafana/toolkit`);
+    return fs.lstatSync(resolvedPath).isSymbolicLink();
   } catch {
     return false;
   }
 };
 
 const entrypoint = () => {
-  const defaultEntryPoint = `${__dirname}/../src/cli/index.js`;
+  const resolvedJsDir = path.resolve(`${__dirname}/../dist/src/cli/index.js`);
+  const resolvedTsDir = path.resolve(`${__dirname}/../src/cli/index.ts`);
 
   // IF we have a toolkit directory AND linked grafana toolkit AND the toolkit dir is a symbolic lik
   // THEN run everything in linked mode
-  if (isLinkedMode()) {
+  if (isLinkedMode() || !fs.existsSync(resolvedJsDir)) {
     console.log('Running in local/linked mode');
     // This bin is used for cli executed internally
     var tsProjectPath = path.resolve(__dirname, '../tsconfig.json');
@@ -36,16 +38,12 @@ const entrypoint = () => {
     });
 
     includeInternalScripts = true;
-    return '../src/cli/index.ts';
+    return resolvedTsDir;
-  }
-
-  // We are using npx, and a relative path does not find index.js
-  if (!fs.existsSync(defaultEntryPoint) && fs.existsSync(`${__dirname}/../dist/src/cli/index.js`)) {
-    return `${__dirname}/../dist/src/cli/index.js`;
   }
 
   // The default entrypoint must exist, return it now.
-  return defaultEntryPoint;
+  return resolvedJsDir;
 };
 
 require(entrypoint()).run(includeInternalScripts);
+

packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/Dockerfile

@@ -1,6 +1,6 @@
 FROM alpine 
 USER root 
 ADD scripts scripts
-ADD install /usr/local/bin
+ADD install /usr/local
 WORKDIR scripts
 RUN ./deploy.sh

packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/build.sh

@@ -3,6 +3,20 @@ set -eo pipefail
 
 source ./common.sh
 
+#
+# No longer required, but useful to keep just in case we want to deploy 
+# changes in toolkit directly to the docker image
+#
+if [ -n "$INCLUDE_TOOLKIT" ]; then
+	/bin/rm -rfv install/grafana-toolkit
+	mkdir -pv install/grafana-toolkit
+	cp -rv ../../bin install/grafana-toolkit
+	cp -rv ../../src install/grafana-toolkit
+	cp -v ../../package.json install/grafana-toolkit
+	cp -v ../../tsconfig.json install/grafana-toolkit
+fi
+
+
 output=$(docker build . | tee /dev/tty)
 hash=$(echo "$output" | tail -1 | sed -ne "s/^Successfully built \(.*\)/\1/p")
 if [ ${#hash} -gt 0 ]; then
@@ -10,3 +24,4 @@ if [ ${#hash} -gt 0 ]; then
 	docker push $DOCKER_IMAGE_NAME:latest
 fi
 
+[ -n "$INCLUDE_TOOLKIT" ] && /bin/rm -rfv install/grafana-toolkit

packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/install/bin/githubRelease.js

@@ -0,0 +1,121 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+var tslib_1 = require("tslib");
+var getPluginId_1 = require("../../config/utils/getPluginId");
+var pluginValidation_1 = require("../../config/utils/pluginValidation");
+var env_1 = require("../../plugins/env");
+var path = require("path");
+var fs = require("fs");
+// @ts-ignore
+// import execa = require('execa');
+var githubClient_1 = tslib_1.__importDefault(require("./githubClient"));
+var resolveContentType = function (extension) {
+    if (extension.startsWith('.')) {
+        extension = extension.substr(1);
+    }
+    switch (extension) {
+        case 'zip':
+            return 'application/zip';
+        case 'json':
+            return 'application/json';
+        case 'sha1':
+            return 'text/plain';
+        default:
+            return 'application/octet-stream';
+    }
+};
+var GitHubRelease = /** @class */ (function () {
+    function GitHubRelease(token, username, repository, releaseNotes, commitHash) {
+        this.token = token;
+        this.username = username;
+        this.repository = repository;
+        this.releaseNotes = releaseNotes;
+        this.commitHash = commitHash;
+        this.git = new githubClient_1.default({
+            required: true,
+            repo: repository,
+        });
+    }
+    GitHubRelease.prototype.publishAssets = function (srcLocation, destUrl) {
+        var _this = this;
+        // Add the assets. Loop through files in the ci/dist folder and upload each asset.
+        var files = fs.readdirSync(srcLocation);
+        return files.map(function (file) { return tslib_1.__awaiter(_this, void 0, void 0, function () {
+            var fileStat, fileData;
+            return tslib_1.__generator(this, function (_a) {
+                fileStat = fs.statSync(srcLocation + "/" + file);
+                fileData = fs.readFileSync(srcLocation + "/" + file);
+                return [2 /*return*/, this.git.client.post(destUrl + "?name=" + file, fileData, {
+                        headers: {
+                            'Content-Type': resolveContentType(path.extname(file)),
+                            'Content-Length': fileStat.size,
+                        },
+                        maxContentLength: fileStat.size * 2 * 1024 * 1024,
+                    })];
+            });
+        }); });
+    };
+    GitHubRelease.prototype.release = function () {
+        var _a, _b, _c, _d;
+        return tslib_1.__awaiter(this, void 0, void 0, function () {
+            var ciDir, distDir, distContentDir, pluginJsonFile, pluginInfo, PUBLISH_DIR, commitHash, latestRelease, reason_1, newReleaseResponse, publishPromises, reason_2;
+            return tslib_1.__generator(this, function (_e) {
+                switch (_e.label) {
+                    case 0:
+                        ciDir = env_1.getCiFolder();
+                        distDir = path.resolve(ciDir, 'dist');
+                        distContentDir = path.resolve(distDir, getPluginId_1.getPluginId());
+                        pluginJsonFile = path.resolve(distContentDir, 'plugin.json');
+                        pluginInfo = pluginValidation_1.getPluginJson(pluginJsonFile).info;
+                        PUBLISH_DIR = path.resolve(env_1.getCiFolder(), 'packages');
+                        commitHash = this.commitHash || ((_a = pluginInfo.build) === null || _a === void 0 ? void 0 : _a.hash);
+                        _e.label = 1;
+                    case 1:
+                        _e.trys.push([1, 5, , 6]);
+                        return [4 /*yield*/, this.git.client.get("releases/tags/v" + pluginInfo.version)];
+                    case 2:
+                        latestRelease = _e.sent();
+                        if (!(latestRelease.data.tag_name === "v" + pluginInfo.version)) return [3 /*break*/, 4];
+                        return [4 /*yield*/, this.git.client.delete("releases/" + latestRelease.data.id)];
+                    case 3:
+                        _e.sent();
+                        _e.label = 4;
+                    case 4: return [3 /*break*/, 6];
+                    case 5:
+                        reason_1 = _e.sent();
+                        if (reason_1.response.status !== 404) {
+                            // 404 just means no release found. Not an error. Anything else though, re throw the error
+                            throw reason_1;
+                        }
+                        return [3 /*break*/, 6];
+                    case 6:
+                        _e.trys.push([6, 9, , 10]);
+                        return [4 /*yield*/, this.git.client.post('releases', {
+                                tag_name: "v" + pluginInfo.version,
+                                target_commitish: commitHash,
+                                name: "v" + pluginInfo.version,
+                                body: this.releaseNotes,
+                                draft: false,
+                                prerelease: false,
+                            })];
+                    case 7:
+                        newReleaseResponse = _e.sent();
+                        publishPromises = this.publishAssets(PUBLISH_DIR, "https://uploads.github.com/repos/" + this.username + "/" + this.repository + "/releases/" + newReleaseResponse.data.id + "/assets");
+                        return [4 /*yield*/, Promise.all(publishPromises)];
+                    case 8:
+                        _e.sent();
+                        return [3 /*break*/, 10];
+                    case 9:
+                        reason_2 = _e.sent();
+                        console.log(reason_2);
+                        // Rethrow the error so that we can trigger a non-zero exit code to circle-ci
+                        throw reason_2;
+                    case 10: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    return GitHubRelease;
+}());
+exports.GitHubRelease = GitHubRelease;
+//# sourceMappingURL=githubRelease.js.map7027e10521e9

packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/scripts/deploy.sh

@@ -10,7 +10,7 @@ rm /bin/cp
 mv /usr/local/bin/cp /bin/cp
 
 sed -i -e 's/v[[:digit:]]\..*\//edge\//g' /etc/apk/repositories
-apk add nodejs npm yarn build-base openssh 
+apk add nodejs npm yarn build-base openssh git-lfs perl-utils
 
 #
 # Only relevant for testing, but cypress does not work with musl/alpine.
@@ -23,12 +23,12 @@ get_file "https://dl.google.com/go/$filename" "/tmp/$filename" "08df79b46b0adf49
 untar_file "/tmp/$filename"
 
 # Install golangci-lint
-filename="golangci-lint-1.23.7-linux-amd64.tar.gz"
+filename="golangci-lint-1.26.0-linux-amd64"
-get_file "https://github.com/golangci/golangci-lint/releases/download/v1.23.7/$filename" \
+get_file "https://github.com/golangci/golangci-lint/releases/download/v1.26.0/$filename.tar.gz" \
-    "/tmp/$filename" \
+    "/tmp/$filename.tar.gz" \
-    "34df1794a2ea8e168b3c98eed3cc0f3e13ed4cba735e4e40ef141df5c41bc086"
+    "59b0e49a4578fea574648a2fd5174ed61644c667ea1a1b54b8082fde15ef94fd"
-untar_file "/tmp/$filename"
+untar_file "/tmp/$filename.tar.gz"
-ln -s /usr/local/golangci-lint-1.23.7-linux-amd64/golangci-lint /usr/local/bin/golangci-lint
+ln -s /usr/local/${filename}/golangci-lint /usr/local/bin/golangci-lint
 ln -s /usr/local/go/bin/go /usr/local/bin/go
 ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt
 chmod 755 /usr/local/bin/golangci-lint
@@ -60,6 +60,11 @@ for file in $(ls $HOME/go/bin); do
 	mv -v $HOME/go/bin/$file /usr/local/bin/$file
 done
 
+# Install grafana-toolkit deps
+current_dir=$PWD
+cd /usr/local/grafana-toolkit && yarn install && cd $current_dir
+ln -s /usr/local/grafana-toolkit/bin/grafana-toolkit.js /usr/local/bin/grafana-toolkit
+
 # Cleanup after yourself
 /bin/rm -rf /tmp/mage 
 /bin/rm -rf $HOME/go

packages/grafana-toolkit/docker/grafana-plugin-ci-e2e/Dockerfile

@@ -1,5 +1,5 @@
 FROM debian:buster-slim 
 USER root 
 ADD scripts scripts
-ADD install /usr/local/bin
+ADD install /usr/local
 RUN cd scripts && ./deploy.sh

packages/grafana-toolkit/docker/grafana-plugin-ci-e2e/build.sh

@@ -3,6 +3,20 @@ set -eo pipefail
 
 source ./common.sh
 
+#
+# No longer required, but useful to keep just in case we want to deploy 
+# changes in toolkit directly to the docker image
+#
+if [ -n "$INCLUDE_TOOLKIT" ]; then
+	/bin/rm -rfv install/grafana-toolkit
+	mkdir -pv install/grafana-toolkit
+	cp -rv ../../bin install/grafana-toolkit
+	cp -rv ../../src install/grafana-toolkit
+	cp -v ../../package.json install/grafana-toolkit
+	cp -v ../../tsconfig.json install/grafana-toolkit
+fi
+
+
 output=$(docker build . | tee /dev/tty)
 hash=$(echo "$output" | tail -1 | sed -ne "s/^Successfully built \(.*\)/\1/p")
 if [ ${#hash} -gt 0 ]; then
@@ -10,3 +24,4 @@ if [ ${#hash} -gt 0 ]; then
 	docker push $DOCKER_IMAGE_NAME:latest
 fi
 
+[ -n "$INCLUDE_TOOLKIT" ] && /bin/rm -rfv install/grafana-toolkit

packages/grafana-toolkit/docker/grafana-plugin-ci-e2e/install/cp

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-if [ "$1" == "-rn" ]; then
-  false | busybox cp -i -r "$2" "$3" 2>/dev/null
-else
-  busybox cp $*
-fi

packages/grafana-toolkit/docker/grafana-plugin-ci-e2e/scripts/deploy-slim.sh

@@ -2,5 +2,5 @@
 source "/etc/profile"
 
 apt-get --allow-insecure-repositories update
-apt-get install --allow-unauthenticated -y build-essential wget git sudo adduser libfontconfig1 locate libnss3 libnspr4 libgdk-pixbuf2.0-0 libgtk-3-0 libpangocairo-1.0-0 libpango-1.0-0 libatk1.0-0 libcairo2 libdbus-1-3 libxcomposite1 libxrender1 libxcursor1 libxi6 libxtst6 libxrandr2 libxss1 libasound2 libatk-bridge2.0-0 libatspi2.0-0 libcups2 jq xvfb net-tools
+apt-get install --allow-unauthenticated -y build-essential wget git sudo adduser libfontconfig1 locate libnss3 libnspr4 libgdk-pixbuf2.0-0 libgtk-3-0 libpangocairo-1.0-0 libpango-1.0-0 libatk1.0-0 libcairo2 libdbus-1-3 libxcomposite1 libxrender1 libxcursor1 libxi6 libxtst6 libxrandr2 libxss1 libasound2 libatk-bridge2.0-0 libatspi2.0-0 libcups2 jq xvfb net-tools git-lfs unzip pkg-config zip libaio1 libaio-dev
 

packages/grafana-toolkit/docker/grafana-plugin-ci-e2e/scripts/deploy.sh

@@ -26,14 +26,13 @@ filename="go1.14.linux-amd64.tar.gz"
 get_file "https://dl.google.com/go/$filename" "/tmp/$filename" "08df79b46b0adf498ea9f320a0f23d6ec59e9003660b4c9c1ce8e5e2c6f823ca"
 untar_file "/tmp/$filename"
 
-
 # Install golangci-lint
-filename="golangci-lint-1.23.7-linux-amd64.tar.gz"
+filename="golangci-lint-1.26.0-linux-amd64"
-get_file "https://github.com/golangci/golangci-lint/releases/download/v1.23.7/$filename" \
+get_file "https://github.com/golangci/golangci-lint/releases/download/v1.26.0/$filename.tar.gz" \
-    "/tmp/$filename" \
+    "/tmp/$filename.tar.gz" \
-    "34df1794a2ea8e168b3c98eed3cc0f3e13ed4cba735e4e40ef141df5c41bc086"
+    "59b0e49a4578fea574648a2fd5174ed61644c667ea1a1b54b8082fde15ef94fd"
-untar_file "/tmp/$filename"
+untar_file "/tmp/$filename.tar.gz"
-ln -s /usr/local/golangci-lint-1.23.7-linux-amd64/golangci-lint /usr/local/bin/golangci-lint
+ln -s /usr/local/${filename}/golangci-lint /usr/local/bin/golangci-lint
 ln -s /usr/local/go/bin/go /usr/local/bin/go
 ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt
 chmod 755 /usr/local/bin/golangci-lint
@@ -56,5 +55,11 @@ mv $HOME/go/bin/mage /usr/local/bin
 /bin/rm -rf /tmp/mage 
 /bin/rm -rf $HOME/go
 
+# Install grafana-toolkit deps
+pushd /usr/local/grafana-toolkit
+yarn install
+ln -s /usr/local/grafana-toolkit/bin/grafana-toolkit.js /usr/local/bin/grafana-toolkit
+popd
+
 # Get the size down
 /bin/rm -rf /var/lib/apt/lists

packages/grafana-toolkit/package.json

@@ -29,10 +29,10 @@
   "dependencies": {
     "@babel/core": "7.9.0",
     "@babel/preset-env": "7.9.0",
-    "@grafana/data": "7.1.0-pre.0",
+    "@grafana/data": "next",
     "@grafana/eslint-config": "^1.0.0-rc1",
     "@grafana/tsconfig": "^1.0.0-rc1",
-    "@grafana/ui": "7.1.0-pre.0",
+    "@grafana/ui": "next",
     "@types/command-exists": "^1.2.0",
     "@types/execa": "^0.9.0",
     "@types/expect-puppeteer": "3.3.1",

packages/grafana-toolkit/src/cli/utils/githubClient.ts

@@ -1,6 +1,6 @@
 import axios, { AxiosInstance, AxiosRequestConfig } from 'axios';
 
-const grafanaURL = (repo: string) => `https://api.github.com/repos/grafana/${repo}`;
+const grafanaURL = (owner: string, repo: string) => `https://api.github.com/repos/${owner}/${repo}`;
 const enterpriseURL = 'https://api.github.com/repos/grafana/grafana-enterprise';
 
 // Encapsulates the creation of a client for the Github API
@@ -14,18 +14,19 @@ const enterpriseURL = 'https://api.github.com/repos/grafana/grafana-enterprise';
 interface GithubClientProps {
   required?: boolean;
   enterprise?: boolean;
+  owner?: string;
   repo?: string;
 }
 
 class GithubClient {
   client: AxiosInstance;
 
-  constructor({ required = false, enterprise = false, repo = 'grafana' }: GithubClientProps = {}) {
+  constructor({ required = false, enterprise = false, owner = 'grafana', repo = 'grafana' }: GithubClientProps = {}) {
     const username = process.env.GITHUB_USERNAME;
     const token = process.env.GITHUB_ACCESS_TOKEN;
 
     const clientConfig: AxiosRequestConfig = {
-      baseURL: enterprise ? enterpriseURL : grafanaURL(repo),
+      baseURL: enterprise ? enterpriseURL : grafanaURL(owner, repo),
       timeout: 10000,
     };
 

packages/grafana-toolkit/src/cli/utils/githubRelease.ts

@@ -41,6 +41,7 @@ class GitHubRelease {
 
     this.git = new GithubClient({
       required: true,
+      owner: username,
       repo: repository,
     });
   }
@@ -57,6 +58,7 @@ class GitHubRelease {
           'Content-Type': resolveContentType(path.extname(file)),
           'Content-Length': fileStat.size,
         },
+        maxContentLength: fileStat.size * 2 * 1024 * 1024,
       });
     });
   }