Build: Improve NPM publishing (#65171)

* chore(packages): remove redundant npm scripts for publishing packages

* feat(packages): rewrite npm publishing script to work for manual and ci publishes

* ci(drone): update release-canary-npm-packages step to use new script

* docs(packages): update manual release instructions

* wip(packages): attempt to validate packed npm package

* fix(packages): release-canary-npm-packages should provide canary dist-tag

* ci(packages): clean up npm package validation script

* chore(devenv): add verdaccio config to allow anon publishing for easier dev npm testing

* ci(packages): clean up publishing script

* ci(drone): during build-frontend-packages, pack and validate packed tarballs

* chore(codeowners): update for publish/validate npm packages scripts

* ci(packages): fix esm loop bug matching e2e package

* ci(npm-packages): fix failing regex

* style(lib.star): run make format-drone

* style(npm-packages): shellcheck fixes for validate-npm-packages script

* docs(packages): update readme instructions for publishing locally and manually

* refactor(npm-publish): use drone when to trigger canary releases

* chore(drone): remove redundant trigger_npm_publish var

* chore(npm-publish): remove redundant echo

scripts/circle-release-canary-packages.sh

@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-set -eo pipefail
-
-# shellcheck source=./scripts/helpers/exit-if-fail.sh
-source "$(dirname "$0")/helpers/exit-if-fail.sh"
-
-# check if there were any changes to packages between current and previous commit
-count=$(git diff HEAD~1..HEAD --name-only -- packages | awk '{c++} END {print c}')
-
-if [ -z "$count" ]; then
-  echo "No changes in packages, skipping packages publishing"
-else
-  echo "Changes detected in ${count} packages"
-  echo "Starting to release latest canary version"
-
-  echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> ~/.npmrc
-
-  echo $'\nPacking packages'
-  yarn packages:pack
-
-  echo $'\nPublishing packages'
-  for file in ./npm-artifacts/*.tgz; do npm publish "$file" --tag canary; done
-
-fi

scripts/drone/steps/lib.star

@@ -567,6 +567,8 @@ def build_frontend_package_step(edition, ver_mode):
         cmds = [
             "./bin/build build-frontend-packages --jobs 8 --edition {} ".format(edition) +
             "--build-id {}".format(build_no),
+            "yarn packages:pack",
+            "./scripts/validate-npm-packages.sh",
         ]
 
     return {
@@ -1179,11 +1181,21 @@ def release_canary_npm_packages_step(trigger = None):
             "NPM_TOKEN": from_secret("npm_token"),
         },
         "commands": [
-            "./scripts/circle-release-canary-packages.sh",
+            "./scripts/publish-npm-packages.sh --dist-tag 'canary' --registry 'https://registry.npmjs.org/'",
         ],
     }
     if trigger:
-        step = dict(step, when = trigger)
+        step = dict(
+            step,
+            when = dict(
+                trigger,
+                paths = {
+                    "include": [
+                        "packages/**",
+                    ],
+                },
+            ),
+        )
     return step
 
 def enterprise2_suffix(edition):

scripts/publish-npm-packages.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Set default values for dist-tag and registry for local development
+# to prevent running this script and accidentally publishing to npm
+dist_tag="canary"
+registry="http://localhost:4873"
+
+# shellcheck source=./scripts/helpers/exit-if-fail.sh
+source "$(dirname "$0")/helpers/exit-if-fail.sh"
+
+if [ -z "$NPM_TOKEN" ]; then
+  echo "The NPM_TOKEN environment variable does not exist."
+  exit 1
+fi
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+    key="$1"
+    case $key in
+        --dist-tag)
+        dist_tag="$2"
+        shift # past argument
+        shift # past value
+        ;;
+        --registry)
+        registry="$2"
+        shift # past argument
+        shift # past value
+        ;;
+        *)    # unknown option
+        echo "Unknown option: $1"
+        exit 1
+        ;;
+    esac
+done
+
+echo "Starting to release $dist_tag version"
+
+echo "$registry/:_authToken=${NPM_TOKEN}" >> ~/.npmrc
+
+# Loop over .tar files in directory and publish them to npm registry
+for file in ./npm-artifacts/*.tgz; do
+    npm publish "$file" --tag "$dist_tag" --registry "$registry"
+done

scripts/validate-npm-packages.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# This script is used to validate the npm packages that are published to npmjs.org are in the correct format.
+# It won't catch things like malformed JS or Types but it will assert that the package has
+# the correct files and package.json properties.
+ARTIFACTS_DIR="./npm-artifacts"
+
+for file in "$ARTIFACTS_DIR"/*.tgz; do
+  echo "🔍 Checking NPM package: $file"
+  # get filename then strip everything after package name.
+  dir_name=$(basename "$file" .tgz | sed 's/^@\(.*\)-[0-9]*[.]*[0-9]*[.]*[0-9]*-\([0-9]*[a-zA-Z]*\)/\1/')
+  mkdir -p "./npm-artifacts/$dir_name"
+  tar -xzf "$file" -C "./npm-artifacts/$dir_name" --strip-components=1
+
+  # Make sure the tar wasn't empty
+  if [ ! -d "./npm-artifacts/$dir_name" ]; then
+    echo -e "❌ Failed: Empty package $dir_name.\n"
+    exit 1
+  fi
+
+  # Navigate inside the new extracted directory
+  pushd "./npm-artifacts/$dir_name" || exit
+
+  # Check for required files
+	check_files=("package.json" "README.md" "CHANGELOG.md" "LICENSE_APACHE2")
+	for check_file in "${check_files[@]}"; do
+		if [ ! -f "$check_file" ]; then
+			echo -e "❌ Failed: Missing required file $check_file in package $dir_name.\n"
+			exit 1
+		fi
+	done
+
+  # @grafana/toolkit structure is different to the other packages
+  if [[ "$dir_name" == "grafana-toolkit" ]]; then
+    if [ ! -d bin ] || [ ! -f bin/grafana-toolkit.js ]; then
+      echo -e "❌ Failed: Missing 'bin' directory or required files in package $dir_name.\n"
+      exit 1
+    fi
+
+    echo -e "✅ Passed: package checks for $file.\n"
+    popd || exit
+    continue
+  fi
+
+  # Assert commonjs builds
+  if [ ! -d dist ] || [ ! -f dist/index.js ] || [ ! -f dist/index.d.ts ]; then
+    echo -e "❌ Failed: Missing 'dist' directory or required commonjs files in package $dir_name.\n"
+    exit 1
+  fi
+
+  if [ "$(jq -r '.main' package.json)" != "dist/index.js" ] || \
+     [ "$(jq -r '.types' package.json)" != "dist/index.d.ts" ]; then
+    echo -e "❌ Failed: Incorrect package.json properties in package $dir_name.\n"
+    exit 1
+  fi
+
+  # Assert esm builds
+  esm_packages=("grafana-data" "grafana-ui" "grafana-runtime" "grafana-e2e-selectors" "grafana-schema")
+  for esm_package in "${esm_packages[@]}"; do
+    if [[ "$dir_name" == "$esm_package" ]]; then
+      if [ ! -d dist/esm ] || [ ! -f dist/esm/index.js ]; then
+        echo -e "❌ Failed: Missing 'dist/esm' directory or required esm files in package $dir_name.\n"
+        exit 1
+      fi
+
+      if [ "$(jq -r '.module' package.json)" != "dist/esm/index.js" ]; then
+        echo -e "❌ Failed: Incorrect package.json properties in package $dir_name.\n"
+        exit 1
+      fi
+    fi
+  done
+
+  echo -e "✅ Passed: package checks for $file.\n"
+  popd || exit
+
+done
+
+echo "🚀 All NPM package checks passed! 🚀"
+rm -rf "${ARTIFACTS_DIR:?}/"*/
+exit 0