Build: Improve NPM publishing (#65171)

* chore(packages): remove redundant npm scripts for publishing packages

* feat(packages): rewrite npm publishing script to work for manual and ci publishes

* ci(drone): update release-canary-npm-packages step to use new script

* docs(packages): update manual release instructions

* wip(packages): attempt to validate packed npm package

* fix(packages): release-canary-npm-packages should provide canary dist-tag

* ci(packages): clean up npm package validation script

* chore(devenv): add verdaccio config to allow anon publishing for easier dev npm testing

* ci(packages): clean up publishing script

* ci(drone): during build-frontend-packages, pack and validate packed tarballs

* chore(codeowners): update for publish/validate npm packages scripts

* ci(packages): fix esm loop bug matching e2e package

* ci(npm-packages): fix failing regex

* style(lib.star): run make format-drone

* style(npm-packages): shellcheck fixes for validate-npm-packages script

* docs(packages): update readme instructions for publishing locally and manually

* refactor(npm-publish): use drone when to trigger canary releases

* chore(drone): remove redundant trigger_npm_publish var

* chore(npm-publish): remove redundant echo
This commit is contained in:
Jack Westbrook 2023-04-18 10:19:37 +02:00 committed by GitHub
parent 652fd8889e
commit efa641040d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 378 additions and 54 deletions

View File

@ -571,6 +571,8 @@ steps:
name: build-frontend
- commands:
- ./bin/build build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER}
- yarn packages:pack
- ./scripts/validate-npm-packages.sh
depends_on:
- compile-build-cmd
- yarn-install
@ -1458,6 +1460,8 @@ steps:
name: build-frontend
- commands:
- ./bin/build build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER}
- yarn packages:pack
- ./scripts/validate-npm-packages.sh
depends_on:
- compile-build-cmd
- yarn-install
@ -1734,7 +1738,7 @@ steps:
repo:
- grafana/grafana
- commands:
- ./scripts/circle-release-canary-packages.sh
- ./scripts/publish-npm-packages.sh --dist-tag 'canary' --registry 'https://registry.npmjs.org/'
depends_on:
- end-to-end-tests-dashboards-suite
- end-to-end-tests-panels-suite
@ -1746,6 +1750,9 @@ steps:
image: grafana/build-container:1.7.3
name: release-canary-npm-packages
when:
paths:
include:
- packages/**
repo:
- grafana/grafana
- commands:
@ -4685,6 +4692,8 @@ steps:
name: build-frontend
- commands:
- ./bin/build build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER}
- yarn packages:pack
- ./scripts/validate-npm-packages.sh
depends_on:
- compile-build-cmd
- yarn-install
@ -5328,6 +5337,8 @@ steps:
name: build-frontend
- commands:
- ./bin/build build-frontend-packages --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER}
- yarn packages:pack
- ./scripts/validate-npm-packages.sh
depends_on:
- compile-build-cmd
- yarn-install
@ -6049,6 +6060,8 @@ steps:
name: build-frontend
- commands:
- ./bin/build build-frontend-packages --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER}
- yarn packages:pack
- ./scripts/validate-npm-packages.sh
depends_on:
- compile-build-cmd
- yarn-install
@ -6810,6 +6823,6 @@ kind: secret
name: enterprise2_security_prefix
---
kind: signature
hmac: d3567c954d0ac9f1f47d55bc85eb6594379c888fe591428f2dcd0146f2bd3ca2
hmac: 7f282aaf4f3ba496f2d954241a16afd2facbf2cf1ef04dfb75cfec1b42fd39d1
...

2
.github/CODEOWNERS vendored
View File

@ -455,6 +455,8 @@ lerna.json @grafana/frontend-ops
/scripts/check-breaking-changes.sh @grafana/plugins-platform-frontend
/scripts/ci-* @grafana/grafana-delivery
/scripts/circle-* @grafana/grafana-delivery
/scripts/publish-npm-packages.sh @grafana/grafana-delivery @grafana/plugins-platform-frontend
/scripts/validate-npm-packages.sh @grafana/grafana-delivery @grafana/plugins-platform-frontend
/scripts/ci-frontend-metrics.sh @grafana/grafana-frontend-platform @grafana/plugins-platform-frontend @grafana/grafana-bi-squad
/scripts/cli/ @grafana/grafana-frontend-platform
/scripts/clean-git-or-error.sh @grafana/grafana-as-code

2
.gitignore vendored
View File

@ -139,7 +139,7 @@ pkg/services/quota/quotaimpl/storage/storage.json
/packages/**/package.tgz
/packages/grafana-toolkit/sass
## CI places the packages in a different location
/npm-artifacts/*.tgz
/npm-artifacts
# Ignore frontend build manifest
manifest.json

View File

@ -0,0 +1,202 @@
#
# This is the default configuration file. It allows all users to do anything,
# please read carefully the documentation and best practices to
# improve security.
#
# Do not configure host and port under `listen` in this file
# as it will be ignored when using docker.
# see https://verdaccio.org/docs/en/docker#docker-and-custom-port-configuration
#
# Look here for more config file examples:
# https://github.com/verdaccio/verdaccio/tree/5.x/conf
#
# Read about the best practices
# https://verdaccio.org/docs/best
# path to a directory with all packages
storage: /verdaccio/storage/data
# path to a directory with plugins to include
plugins: /verdaccio/plugins
# https://verdaccio.org/docs/webui
web:
title: Verdaccio
# comment out to disable gravatar support
# gravatar: false
# by default packages are ordercer ascendant (asc|desc)
# sort_packages: asc
# convert your UI to the dark side
# darkMode: true
# html_cache: true
# by default all features are displayed
# login: true
# showInfo: true
# showSettings: true
# In combination with darkMode you can force specific theme
# showThemeSwitch: true
# showFooter: true
# showSearch: true
# showRaw: true
# showDownloadTarball: true
# HTML tags injected after manifest <scripts/>
# scriptsBodyAfter:
# - '<script type="text/javascript" src="https://my.company.com/customJS.min.js"></script>'
# HTML tags injected before ends </head>
# metaScripts:
# - '<script type="text/javascript" src="https://code.jquery.com/jquery-3.5.1.slim.min.js"></script>'
# - '<script type="text/javascript" src="https://browser.sentry-cdn.com/5.15.5/bundle.min.js"></script>'
# - '<meta name="robots" content="noindex" />'
# HTML tags injected first child at <body/>
# bodyBefore:
# - '<div id="myId">html before webpack scripts</div>'
# Public path for template manifest scripts (only manifest)
# publicPath: http://somedomain.org/
# https://verdaccio.org/docs/configuration#authentication
auth:
htpasswd:
file: /verdaccio/storage/htpasswd
# Maximum amount of users allowed to register, defaults to "+infinity".
# You can set this to -1 to disable registration.
# max_users: 1000
# Hash algorithm, possible options are: "bcrypt", "md5", "sha1", "crypt".
# algorithm: bcrypt # by default is crypt, but is recommended use bcrypt for new installations
# Rounds number for "bcrypt", will be ignored for other algorithms.
# rounds: 10
# https://verdaccio.org/docs/configuration#uplinks
# a list of other known repositories we can talk to
uplinks:
npmjs:
url: https://registry.npmjs.org/
# Learn how to protect your packages
# https://verdaccio.org/docs/protect-your-dependencies/
# https://verdaccio.org/docs/configuration#packages
packages:
'@*/*':
# allow all users (including non-authenticated users) to read and
# publish scoped packages
access: $anonymous
publish: $anonymous
unpublish: $anonymous
proxy: npmjs
'**':
# allow all users (including non-authenticated users) to read and
# publish all packages
#
# you can specify usernames/groupnames (depending on your auth plugin)
# and three keywords: "$all", "$anonymous", "$authenticated"
access: $anonymous
# allow all known users to publish/publish packages
# (anyone can register by default, remember?)
publish: $anonymous
unpublish: $anonymous
# if package is not available locally, proxy requests to 'npmjs' registry
proxy: npmjs
# To improve your security configuration and avoid dependency confusion
# consider removing the proxy property for private packages
# https://verdaccio.org/docs/best#remove-proxy-to-increase-security-at-private-packages
# https://verdaccio.org/docs/configuration#server
# You can specify HTTP/1.1 server keep alive timeout in seconds for incoming connections.
# A value of 0 makes the http server behave similarly to Node.js versions prior to 8.0.0, which did not have a keep-alive timeout.
# WORKAROUND: Through given configuration you can workaround following issue https://github.com/verdaccio/verdaccio/issues/301. Set to 0 in case 60 is not enough.
server:
keepAliveTimeout: 60
# Allow `req.ip` to resolve properly when Verdaccio is behind a proxy or load-balancer
# See: https://expressjs.com/en/guide/behind-proxies.html
# trustProxy: '127.0.0.1'
# https://verdaccio.org/docs/configuration#offline-publish
# publish:
# allow_offline: false
# https://verdaccio.org/docs/configuration#url-prefix
# url_prefix: /verdaccio/
# VERDACCIO_PUBLIC_URL='https://somedomain.org';
# url_prefix: '/my_prefix'
# // url -> https://somedomain.org/my_prefix/
# VERDACCIO_PUBLIC_URL='https://somedomain.org';
# url_prefix: '/'
# // url -> https://somedomain.org/
# VERDACCIO_PUBLIC_URL='https://somedomain.org/first_prefix';
# url_prefix: '/second_prefix'
# // url -> https://somedomain.org/second_prefix/'
# https://verdaccio.org/docs/configuration#security
# security:
# api:
# legacy: true
# jwt:
# sign:
# expiresIn: 29d
# verify:
# someProp: [value]
# web:
# sign:
# expiresIn: 1h # 1 hour by default
# verify:
# someProp: [value]
# https://verdaccio.org/docs/configuration#user-rate-limit
# userRateLimit:
# windowMs: 50000
# max: 1000
# https://verdaccio.org/docs/configuration#max-body-size
# max_body_size: 10mb
# https://verdaccio.org/docs/configuration#listen-port
# listen:
# - localhost:4873 # default value
# - http://localhost:4873 # same thing
# - 0.0.0.0:4873 # listen on all addresses (INADDR_ANY)
# - https://example.org:4873 # if you want to use https
# - "[::1]:4873" # ipv6
# - unix:/tmp/verdaccio.sock # unix socket
# The HTTPS configuration is useful if you do not consider use a HTTP Proxy
# https://verdaccio.org/docs/configuration#https
# https:
# key: ./path/verdaccio-key.pem
# cert: ./path/verdaccio-cert.pem
# ca: ./path/verdaccio-csr.pem
# https://verdaccio.org/docs/configuration#proxy
# http_proxy: http://something.local/
# https_proxy: https://something.local/
# https://verdaccio.org/docs/configuration#notifications
# notify:
# method: POST
# headers: [{ "Content-Type": "application/json" }]
# endpoint: https://usagge.hipchat.com/v2/room/3729485/notification?auth_token=mySecretToken
# content: '{"color":"green","message":"New package published: * {{ name }}*","notify":true,"message_format":"text"}'
middlewares:
audit:
enabled: true
# https://verdaccio.org/docs/logger
# log settings
logs: { type: stdout, format: pretty, level: http }
#experiments:
# # support for npm token command
# token: false
# # enable tarball URL redirect for hosting tarball with a different server, the tarball_url_redirect can be a template string
# tarball_url_redirect: 'https://mycdn.com/verdaccio/${packageName}/${filename}'
# # the tarball_url_redirect can be a function, takes packageName and filename and returns the url, when working with a js configuration file
# tarball_url_redirect(packageName, filename) {
# const signedUrl = // generate a signed url
# return signedUrl;
# }
# translate your registry, api i18n not available yet
# i18n:
# list of the available translations https://github.com/verdaccio/verdaccio/blob/master/packages/plugins/ui-theme/src/i18n/ABOUT_TRANSLATIONS.md
# web: en-US

View File

@ -7,7 +7,7 @@ services:
ports:
- "4873:4873"
volumes:
- verdaccio:/verdaccio
- ./conf:/verdaccio/conf
volumes:
verdaccio:

View File

@ -29,12 +29,6 @@
"packages:clean": "rimraf ./npm-artifacts && lerna run clean --parallel",
"packages:prepare": "lerna version --no-push --no-git-tag-version --force-publish --exact",
"packages:pack": "mkdir -p ./npm-artifacts && lerna exec --no-private -- yarn pack --out \"../../npm-artifacts/%s-%v.tgz\"",
"packages:publish": "lerna exec --no-private -- npm publish package.tgz",
"packages:publishCanary": "lerna exec --no-private -- npm publish package.tgz --tag canary",
"packages:publishLatest": "lerna exec --no-private -- npm publish package.tgz",
"packages:publishNext": "lerna exec --no-private -- npm publish package.tgz --tag next",
"packages:publishTest": "lerna exec --no-private -- npm publish package.tgz --tag test",
"packages:publishDev": "lerna exec --no-private -- npm publish package.tgz --tag dev --registry http://localhost:4873",
"packages:typecheck": "lerna run typecheck",
"precommit": "yarn run lint-staged",
"prettier:check": "prettier --check --list-different=false --loglevel=warn \"**/*.{ts,tsx,scss,md,mdx}\"",

View File

@ -36,20 +36,21 @@ Every commit to main that has changes within the `packages` directory is a subje
> All of the steps below must be performed on a release branch, according to Grafana Release Guide.
> Make sure you are logged in to npm in your terminal and that you are a part of Grafana org on npm.
> You must be logged in to NPM as part of Grafana NPM org before attempting to publish to the npm registery.
1. Run `yarn packages:prepare` script from the root directory. This performs tests on the packages and prompts for the version of the packages. The version should be the same as the one being released.
1. Run `yarn packages:clean` script from the root directory. This will delete any previous builds of the packages.
2. Run `yarn packages:prepare` script from the root directory. This performs tests on the packages and prompts for the version of the packages. The version should be the same as the one being released.
- Make sure you use semver convention. So, _place a dot between prerelease id and prerelease number_, i.e. 6.3.0-alpha.1
- Make sure you confirm the version bump when prompted!
2. Run `yarn packages:build` script that compiles distribution code in `packages/grafana-*/dist`.
3. Run `yarn packages:pack` script to zip each package into `.tgz`. This is required for yarn berry to replace properties in the package.json files declared in `publishConfig`.
4. Depending whether or not it's a prerelease:
3. Run `yarn packages:build` script that compiles distribution code in `packages/grafana-*/dist`.
4. Run `yarn packages:pack` script to compress each package into `npm-artifacts/*.tgz` files. This is required for yarn to replace properties in the package.json files declared in the `publishConfig` property.
5. Depending on whether or not it's a prerelease:
- When releasing a prerelease run `packages:publishNext` to publish new versions.
- When releasing a stable version run `packages:publishLatest` to publish new versions.
- When releasing a test version run `packages:publishTest` to publish test versions.
- When releasing a prerelease run `./scripts/publish-npm-packages.sh --dist-tag 'next' --registry 'https://registry.npmjs.org/'` to publish new versions.
- When releasing a stable version run `./scripts/publish-npm-packages.sh --dist-tag 'latest' --registry 'https://registry.npmjs.org/'` to publish new versions.
- When releasing a test version run `./scripts/publish-npm-packages.sh --dist-tag 'test' --registry 'https://registry.npmjs.org/'` to publish test versions.
5. Push version commit to the release branch.
6. Revert any changes made by the `packages:prepare` script.
### Building individual packages
@ -72,21 +73,21 @@ In this guide you will set up [Verdaccio](https://verdaccio.org/) registry local
From your terminal:
1. Navigate to `devenv/local-npm` directory.
2. Run `docker-compose up`. This will start your local npm registry, available at http://localhost:4873/
3. Run `npm login --registry=http://localhost:4873 --scope=@grafana` . This will allow you to publish any @grafana/\* package into the local registry.
4. Run `npm config set @grafana:registry http://localhost:4873`. This will config your npm to install @grafana scoped packages from your local registry.
2. Run `docker-compose up`. This will start your local npm registry, available at http://localhost:4873/. Note the verdaccio config allows
3. To test `@grafana` packages published to your local npm registry uncomment `npmScopes` and `unsafeHttpWhitelist` properties in the `.yarnrc` file.
#### Publishing packages to local npm registry
You need to follow [manual packages release procedure](#manual-release). The only difference is you need to run `yarn packages:publishDev` task in order to publish to you local registry.
You need to follow [manual packages release procedure](#manual-release). The only difference is the last command in order to publish to you local registry.
From your terminal:
1. Run `yarn packages:prepare`.
2. Run `yarn packages:build`.
3. Run `yarn packages:pack`.
4. Run `yarn packages:publishDev`.
5. Navigate to http://localhost:4873 and verify that version was published
1. Run `yarn packages:clean`.
2. Run `yarn packages:prepare`.
3. Run `yarn packages:build`.
4. Run `yarn packages:pack`.
5. Run `./scripts/publish-npm-packages.sh`.
6. Navigate to http://localhost:4873 and verify the version was published
Locally published packages will be published under `dev` channel, so in your plugin package.json file you can use that channel. For example:

View File

@ -1,24 +0,0 @@
#!/usr/bin/env bash
set -eo pipefail
# shellcheck source=./scripts/helpers/exit-if-fail.sh
source "$(dirname "$0")/helpers/exit-if-fail.sh"
# check if there were any changes to packages between current and previous commit
count=$(git diff HEAD~1..HEAD --name-only -- packages | awk '{c++} END {print c}')
if [ -z "$count" ]; then
echo "No changes in packages, skipping packages publishing"
else
echo "Changes detected in ${count} packages"
echo "Starting to release latest canary version"
echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> ~/.npmrc
echo $'\nPacking packages'
yarn packages:pack
echo $'\nPublishing packages'
for file in ./npm-artifacts/*.tgz; do npm publish "$file" --tag canary; done
fi

View File

@ -567,6 +567,8 @@ def build_frontend_package_step(edition, ver_mode):
cmds = [
"./bin/build build-frontend-packages --jobs 8 --edition {} ".format(edition) +
"--build-id {}".format(build_no),
"yarn packages:pack",
"./scripts/validate-npm-packages.sh",
]
return {
@ -1179,11 +1181,21 @@ def release_canary_npm_packages_step(trigger = None):
"NPM_TOKEN": from_secret("npm_token"),
},
"commands": [
"./scripts/circle-release-canary-packages.sh",
"./scripts/publish-npm-packages.sh --dist-tag 'canary' --registry 'https://registry.npmjs.org/'",
],
}
if trigger:
step = dict(step, when = trigger)
step = dict(
step,
when = dict(
trigger,
paths = {
"include": [
"packages/**",
],
},
),
)
return step
def enterprise2_suffix(edition):

44
scripts/publish-npm-packages.sh Executable file
View File

@ -0,0 +1,44 @@
#!/bin/bash
# Set default values for dist-tag and registry for local development
# to prevent running this script and accidentally publishing to npm
dist_tag="canary"
registry="http://localhost:4873"
# shellcheck source=./scripts/helpers/exit-if-fail.sh
source "$(dirname "$0")/helpers/exit-if-fail.sh"
if [ -z "$NPM_TOKEN" ]; then
echo "The NPM_TOKEN environment variable does not exist."
exit 1
fi
# Parse command line arguments
while [[ $# -gt 0 ]]; do
key="$1"
case $key in
--dist-tag)
dist_tag="$2"
shift # past argument
shift # past value
;;
--registry)
registry="$2"
shift # past argument
shift # past value
;;
*) # unknown option
echo "Unknown option: $1"
exit 1
;;
esac
done
echo "Starting to release $dist_tag version"
echo "$registry/:_authToken=${NPM_TOKEN}" >> ~/.npmrc
# Loop over .tar files in directory and publish them to npm registry
for file in ./npm-artifacts/*.tgz; do
npm publish "$file" --tag "$dist_tag" --registry "$registry"
done

View File

@ -0,0 +1,80 @@
#!/bin/bash
# This script is used to validate the npm packages that are published to npmjs.org are in the correct format.
# It won't catch things like malformed JS or Types but it will assert that the package has
# the correct files and package.json properties.
ARTIFACTS_DIR="./npm-artifacts"
for file in "$ARTIFACTS_DIR"/*.tgz; do
echo "🔍 Checking NPM package: $file"
# get filename then strip everything after package name.
dir_name=$(basename "$file" .tgz | sed 's/^@\(.*\)-[0-9]*[.]*[0-9]*[.]*[0-9]*-\([0-9]*[a-zA-Z]*\)/\1/')
mkdir -p "./npm-artifacts/$dir_name"
tar -xzf "$file" -C "./npm-artifacts/$dir_name" --strip-components=1
# Make sure the tar wasn't empty
if [ ! -d "./npm-artifacts/$dir_name" ]; then
echo -e "❌ Failed: Empty package $dir_name.\n"
exit 1
fi
# Navigate inside the new extracted directory
pushd "./npm-artifacts/$dir_name" || exit
# Check for required files
check_files=("package.json" "README.md" "CHANGELOG.md" "LICENSE_APACHE2")
for check_file in "${check_files[@]}"; do
if [ ! -f "$check_file" ]; then
echo -e "❌ Failed: Missing required file $check_file in package $dir_name.\n"
exit 1
fi
done
# @grafana/toolkit structure is different to the other packages
if [[ "$dir_name" == "grafana-toolkit" ]]; then
if [ ! -d bin ] || [ ! -f bin/grafana-toolkit.js ]; then
echo -e "❌ Failed: Missing 'bin' directory or required files in package $dir_name.\n"
exit 1
fi
echo -e "✅ Passed: package checks for $file.\n"
popd || exit
continue
fi
# Assert commonjs builds
if [ ! -d dist ] || [ ! -f dist/index.js ] || [ ! -f dist/index.d.ts ]; then
echo -e "❌ Failed: Missing 'dist' directory or required commonjs files in package $dir_name.\n"
exit 1
fi
if [ "$(jq -r '.main' package.json)" != "dist/index.js" ] || \
[ "$(jq -r '.types' package.json)" != "dist/index.d.ts" ]; then
echo -e "❌ Failed: Incorrect package.json properties in package $dir_name.\n"
exit 1
fi
# Assert esm builds
esm_packages=("grafana-data" "grafana-ui" "grafana-runtime" "grafana-e2e-selectors" "grafana-schema")
for esm_package in "${esm_packages[@]}"; do
if [[ "$dir_name" == "$esm_package" ]]; then
if [ ! -d dist/esm ] || [ ! -f dist/esm/index.js ]; then
echo -e "❌ Failed: Missing 'dist/esm' directory or required esm files in package $dir_name.\n"
exit 1
fi
if [ "$(jq -r '.module' package.json)" != "dist/esm/index.js" ]; then
echo -e "❌ Failed: Incorrect package.json properties in package $dir_name.\n"
exit 1
fi
fi
done
echo -e "✅ Passed: package checks for $file.\n"
popd || exit
done
echo "🚀 All NPM package checks passed! 🚀"
rm -rf "${ARTIFACTS_DIR:?}/"*/
exit 0