mirror of
https://github.com/pgadmin-org/pgadmin4.git
synced 2026-11-01 15:53:03 -06:00
Dave Page
6c70d7ccb8
The Windows installer and the pip wheel were both shipping web/yarn.lock, web/package.json and the rest of the JavaScript build configuration into the installed tree, even though none of it is used at runtime: only the webpack output under pgadmin/static/js/generated is. Because node_modules is stripped, the lockfile that remains describes packages that are not present, so vulnerability scanners run against an installation report the whole build-time dependency tree, devDependencies included, as findings that cannot be acted on and will recur on every release no matter how current the versions are. The Linux, macOS and Docker builds already removed these files, so this brings Windows and pip into line with them, and adds webpack.* to the Linux and macOS lists so that all of the packagers now strip the same set. For pip the cleanup runs before syft, so the SBOM describes what actually ships. Verified by assembling a web tree with the cleanup applied and starting pgAdmin from it: /misc/ping and /browser/ both return 200 and every generated bundle is served.
347 lines
15 KiB
Bash
347 lines
15 KiB
Bash
# shellcheck shell=bash
|
|
|
|
_setup_env() {
|
|
echo "Setting up the environment..."
|
|
WD=$(cd "$(dirname "$1")" && pwd)
|
|
SOURCEDIR=$(realpath "${WD}/../..")
|
|
BUILDROOT=$(realpath "${WD}/../../$2-build")
|
|
DESKTOPROOT=${BUILDROOT}/desktop
|
|
METAROOT=${BUILDROOT}/meta
|
|
SERVERROOT=${BUILDROOT}/server
|
|
WEBROOT=${BUILDROOT}/web
|
|
DISTROOT=$(realpath "${WD}/../../dist")
|
|
APP_RELEASE=$(grep "^APP_RELEASE" web/version.py | cut -d"=" -f2 | sed 's/ //g')
|
|
APP_REVISION=$(grep "^APP_REVISION" web/version.py | cut -d"=" -f2 | sed 's/ //g')
|
|
APP_NAME=$(grep "^APP_NAME" web/branding.py | cut -d"=" -f2 | sed "s/'//g" | sed 's/^ //' | sed 's/ //g' | tr '[:upper:]' '[:lower:]')
|
|
APP_LONG_VERSION=${APP_RELEASE}.${APP_REVISION}
|
|
APP_SUFFIX=$(grep "^APP_SUFFIX" web/version.py | cut -d"=" -f2 | sed 's/ //g' | sed "s/'//g")
|
|
if [ -n "${APP_SUFFIX}" ]; then
|
|
APP_LONG_VERSION=${APP_LONG_VERSION}-${APP_SUFFIX}
|
|
fi
|
|
|
|
# Setting up the correct Python version for Ubuntu 18 and EL-8, which have
|
|
# new Python versions installed parallel to the old ones.
|
|
OS_VERSION=$(grep "^VERSION_ID=" /etc/os-release | awk -F "=" '{ print $2 }' | sed 's/"//g' | awk -F "." '{ print $1 }')
|
|
if [ -n "$VIRTUAL_ENV" ]; then
|
|
SYSTEM_PYTHON_PATH="$VIRTUAL_ENV/bin/python3"
|
|
echo "Detected virtual environment: $VIRTUAL_ENV"
|
|
else
|
|
SYSTEM_PYTHON_PATH='/usr/bin/python3'
|
|
echo "Not in a virtual environment, using system Python path."
|
|
fi
|
|
PYTHON_BINARY=$("${SYSTEM_PYTHON_PATH}" -c "import sys; print('python%d.%.d' % (sys.version_info.major, sys.version_info.minor))")
|
|
PYTHON_BINARY_WITHOUT_DOTS='python3'
|
|
if [ "$2" == 'redhat' ] && [ "${OS_VERSION}" == "8" ]; then
|
|
SYSTEM_PYTHON_PATH='/usr/bin/python3.9'
|
|
PYTHON_BINARY='python3.9'
|
|
PYTHON_BINARY_WITHOUT_DOTS='python39'
|
|
fi
|
|
}
|
|
|
|
_cleanup() {
|
|
echo "Cleaning up the old environment and app..."
|
|
if test -f "${SOURCEDIR}/runtime/pgAdmin4"; then
|
|
rm -rf "${SOURCEDIR}/runtime/pgAdmin4"
|
|
fi
|
|
if test -d "${BUILDROOT}"; then
|
|
rm -rf "${BUILDROOT}"
|
|
fi
|
|
rm -f "${DISTROOT}/pgadmin4"*".$1"
|
|
}
|
|
|
|
_setup_dirs() {
|
|
echo "Creating output directories..."
|
|
test -d "${BUILDROOT}" || mkdir "${BUILDROOT}"
|
|
test -d "${DESKTOPROOT}" || mkdir "${DESKTOPROOT}"
|
|
test -d "${METAROOT}" || mkdir "${METAROOT}"
|
|
test -d "${SERVERROOT}" || mkdir "${SERVERROOT}"
|
|
test -d "${WEBROOT}" || mkdir "${WEBROOT}"
|
|
test -d "${DISTROOT}" || mkdir "${DISTROOT}"
|
|
}
|
|
|
|
_create_python_virtualenv() {
|
|
echo "Creating the virtual environment..."
|
|
|
|
cd "${SERVERROOT}" || exit
|
|
|
|
# Create the required directories
|
|
mkdir -p "usr/${APP_NAME}"
|
|
cd "usr/${APP_NAME}" || exit
|
|
|
|
# Capture the system site-packages dirs *before* creating the venv -
|
|
# we need OS-provided packages not available as clean pip wheels on
|
|
# every target platform (e.g. dbus-python: needs libdbus-1-dev to
|
|
# build from source, no reliable prebuilt wheel, but is a hard
|
|
# runtime dependency - see pkg/debian/build.sh's python3-dbus dep).
|
|
SYSTEM_SITE_DIRS=$("${SYSTEM_PYTHON_PATH}" -c \
|
|
"import site; print('\n'.join(site.getsitepackages()))")
|
|
|
|
# Create the blank venv WITHOUT --system-site-packages. That flag
|
|
# uses Python's site.addsitedir() under the hood, which doesn't just
|
|
# add the directory to sys.path - it also scans it for every *.pth
|
|
# file and executes any "import ..." lines found inside them. A
|
|
# stale, improperly-uninstalled package's namespace-package .pth
|
|
# hook (legacy pip/setuptools mechanism) left in the system
|
|
# site-packages runs arbitrary code at interpreter startup, before
|
|
# core stdlib is guaranteed to resolve. A leftover
|
|
# sphinxcontrib-jsmath nspkg.pth on the el-10 build node corrupted
|
|
# sys.path early enough to break pip's own subprocess bootstrap,
|
|
# failing the whole build with a misleading "No module named
|
|
# 'importlib'"/"'traceback'" error (pgadmin4-rpm-build #176/#177).
|
|
#
|
|
# We can't just delete that file - it's in a root-owned system
|
|
# directory on a build node shared by other jobs, and the build
|
|
# user isn't guaranteed to have write access there (confirmed: our
|
|
# first attempt at deleting it silently no-opped on el-10).
|
|
#
|
|
# Instead, get the same "OS-provided packages are importable" result
|
|
# via a mechanism that doesn't trigger a .pth scan: a *plain path
|
|
# line* (no "import") in a .pth file only appends that directory to
|
|
# sys.path - it does NOT recursively scan it for further .pth files.
|
|
# Write the system site-packages dirs as plain lines into a .pth
|
|
# file inside the venv's own site-packages (which we do own), added
|
|
# once pip/wheel are installed below.
|
|
"${SYSTEM_PYTHON_PATH}" -m venv venv
|
|
# shellcheck disable=SC1091
|
|
. venv/bin/activate
|
|
|
|
VENV_SITE_PACKAGES=$(python3 -c \
|
|
"import sysconfig; print(sysconfig.get_path('purelib'))")
|
|
echo "${SYSTEM_SITE_DIRS}" > \
|
|
"${VENV_SITE_PACKAGES}/system-site-packages.pth"
|
|
|
|
# Make sure we have the wheel package present, as well as the latest pip
|
|
pip3 install --upgrade pip
|
|
pip3 install wheel
|
|
|
|
# Install the requirements.
|
|
#
|
|
# psycopg is built from source against the system libpq (see
|
|
# --no-binary psycopg). Pin the C extension to the x86-64 v1
|
|
# baseline so the resulting .so runs on every x86_64 CPU we
|
|
# claim to support. Without this, gcc on a modern build host
|
|
# may emit AVX2/BMI2/FMA instructions that SIGILL on older
|
|
# CPUs (Ivy Bridge and earlier) and on default Proxmox kvm64
|
|
# VMs at psycopg_c.pq module-load time. Issue #9935.
|
|
#
|
|
# The flags are scoped to this pip invocation so they don't
|
|
# leak into any other build steps in the same shell. Other
|
|
# arches build with their distro defaults.
|
|
if [ "$(uname -m)" = "x86_64" ]; then
|
|
CFLAGS="${CFLAGS:-} -O2 -march=x86-64 -mtune=generic" \
|
|
CXXFLAGS="${CXXFLAGS:-} -O2 -march=x86-64 -mtune=generic" \
|
|
pip3 install --force-reinstall --no-cache-dir \
|
|
--no-binary psycopg -r "${SOURCEDIR}/requirements.txt"
|
|
else
|
|
pip3 install --force-reinstall --no-cache-dir \
|
|
--no-binary psycopg -r "${SOURCEDIR}/requirements.txt"
|
|
fi
|
|
|
|
# Fixup the paths in the venv activation scripts
|
|
sed -i 's/VIRTUAL_ENV=.*/VIRTUAL_ENV="\/usr\/pgadmin4\/venv"/g' venv/bin/activate
|
|
sed -i 's/setenv VIRTUAL_ENV .*/setenv VIRTUAL_ENV "\/usr\/pgadmin4\/venv"/g' venv/bin/activate.csh
|
|
sed -i 's/set -gx VIRTUAL_ENV .*/set -gx VIRTUAL_ENV "\/usr\/pgadmin4\/venv"/g' venv/bin/activate.fish
|
|
|
|
# Remove __pycache__
|
|
find . -name "__pycache__" -type d -print0 | xargs -0 rm -rf
|
|
|
|
# Fixup hash bangs
|
|
sed -i 's/#!.*\/python3/#\!\/usr\/pgadmin4\/venv\/bin\/python3/g' venv/bin/*
|
|
|
|
# Figure out some paths for use when completing the venv
|
|
# Use "python3" here as we want the venv path
|
|
DIR_PYMODULES_PATH=$(python3 -c "from sysconfig import get_path; print(get_path('platstdlib'))")
|
|
|
|
# Use {SYSTEM_PYTHON_PATH} here as we want the system path
|
|
if [ "$1" == "debian" ]; then
|
|
PYSYSLIB_PATH=$("${SYSTEM_PYTHON_PATH}" -c "import sys; print('%s/lib/python%d.%.d' % (sys.prefix, sys.version_info.major, sys.version_info.minor))")
|
|
else
|
|
PYSYSLIB_PATH=$("${SYSTEM_PYTHON_PATH}" -c "import sys; print('%s/lib64/python%d.%.d' % (sys.prefix, sys.version_info.major, sys.version_info.minor))")
|
|
fi
|
|
|
|
# Symlink in the rest of the Python libs. This is required because the runtime
|
|
# will clear PYTHONHOME for safety, which has the side-effect of preventing
|
|
# it from finding modules that are not explicitly included in the venv
|
|
cd "${DIR_PYMODULES_PATH}" || exit
|
|
|
|
# Files
|
|
for FULLPATH in "${PYSYSLIB_PATH}"/*.py; do
|
|
FILE=${FULLPATH##*/}
|
|
if [ ! -e "${FILE}" ]; then
|
|
ln -s "${FULLPATH}" "${FILE}"
|
|
fi
|
|
done
|
|
|
|
# Paths
|
|
for FULLPATH in "${PYSYSLIB_PATH}"/*/; do
|
|
FULLPATH=${FULLPATH%*/}
|
|
FILE=${FULLPATH##*/}
|
|
if [ ! -e "${FILE}" ]; then
|
|
ln -s "${FULLPATH}" "${FILE}"
|
|
fi
|
|
done
|
|
|
|
# Remove tests
|
|
cd site-packages || exit
|
|
find . -name "test" -type d -print0 | xargs -0 rm -rf
|
|
find . -name "tests" -type d -print0 | xargs -0 rm -rf
|
|
|
|
# Link the python<version> directory to python so that the private environment path is found by the application.
|
|
if test -d "${DIR_PYMODULES_PATH}"; then
|
|
ln -s "$(basename "${DIR_PYMODULES_PATH}")" "${DIR_PYMODULES_PATH}/../python"
|
|
fi
|
|
}
|
|
|
|
_build_runtime() {
|
|
echo "Assembling the desktop runtime..."
|
|
|
|
# Get a fresh copy of electron.
|
|
|
|
ELECTRON_ARCH="x64"
|
|
if [ "$(uname -m)" == "arm64" ]; then
|
|
ELECTRON_ARCH="arm64"
|
|
fi
|
|
|
|
# Resolve the electron version from runtime/package.json, NOT from
|
|
# `npm info electron version`. The latter fetches whatever currently
|
|
# carries the `latest` dist-tag on the npm registry, which means any
|
|
# newly published electron release lands in shipped binaries without
|
|
# review. Keep the build deterministic and pinned.
|
|
ELECTRON_VERSION=$(sed -nE 's/.*"electron":[[:space:]]*"\^?([0-9.]+)".*/\1/p' "${SOURCEDIR}/runtime/package.json" | head -1)
|
|
if [ -z "${ELECTRON_VERSION}" ]; then
|
|
echo "ERROR: could not resolve electron version from runtime/package.json" >&2
|
|
exit 1
|
|
fi
|
|
|
|
pushd "${BUILDROOT}" > /dev/null || exit
|
|
while true;do
|
|
wget "https://github.com/electron/electron/releases/download/v${ELECTRON_VERSION}/electron-v${ELECTRON_VERSION}-linux-${ELECTRON_ARCH}.zip" && break
|
|
rm "electron-v${ELECTRON_VERSION}-linux-${ELECTRON_ARCH}.zip"
|
|
done
|
|
unzip "electron-v${ELECTRON_VERSION}-linux-${ELECTRON_ARCH}.zip" -d "electron-v${ELECTRON_VERSION}-linux-${ELECTRON_ARCH}"
|
|
popd > /dev/null || exit
|
|
|
|
# Copy electron into the staging directory
|
|
mkdir -p "${DESKTOPROOT}/usr/${APP_NAME}/bin"
|
|
|
|
# Change the permission for others and group the same as the owner
|
|
chmod -R og=u "${BUILDROOT}/electron-v${ELECTRON_VERSION}-linux-${ELECTRON_ARCH}"/*
|
|
# Explicitly remove write permissions for others and group
|
|
chmod -R og-w "${BUILDROOT}/electron-v${ELECTRON_VERSION}-linux-${ELECTRON_ARCH}"/*
|
|
|
|
BUNDLEDIR="${DESKTOPROOT}/usr/${APP_NAME}/bin"
|
|
|
|
# WGET:
|
|
cp -r "${BUILDROOT}/electron-v${ELECTRON_VERSION}-linux-${ELECTRON_ARCH}"/* "${BUNDLEDIR}"
|
|
# WGET END
|
|
|
|
mv "${BUNDLEDIR}/electron" "${BUNDLEDIR}/${APP_NAME}"
|
|
|
|
mkdir -p "${BUNDLEDIR}/resources/app"
|
|
cp -r "${SOURCEDIR}/runtime/assets" "${BUNDLEDIR}/resources/app/assets"
|
|
cp -r "${SOURCEDIR}/runtime/src" "${BUNDLEDIR}/resources/app/src"
|
|
|
|
cp "${SOURCEDIR}/runtime/package.json" "${BUNDLEDIR}/resources/app"
|
|
cp "${SOURCEDIR}/runtime/.yarnrc.yml" "${BUNDLEDIR}/resources/app"
|
|
|
|
# Install the runtime node_modules
|
|
pushd "${BUNDLEDIR}/resources/app" > /dev/null || exit
|
|
YARN_VERSION=$(node -p "require('./package.json').packageManager.split('@')[1]")
|
|
if [ -z "${YARN_VERSION}" ]; then
|
|
echo "ERROR: Could not determine Yarn version from package.json packageManager field."
|
|
exit 1
|
|
fi
|
|
yarn set version "${YARN_VERSION}"
|
|
yarn workspaces focus --production
|
|
|
|
# remove the yarn cache
|
|
rm -rf .yarn .yarn*
|
|
popd > /dev/null || exit
|
|
|
|
# Create the icon
|
|
mkdir -p "${DESKTOPROOT}/usr/share/icons/hicolor/128x128/apps/"
|
|
cp "${SOURCEDIR}/pkg/linux/pgadmin4-128x128.png" "${DESKTOPROOT}/usr/share/icons/hicolor/128x128/apps/${APP_NAME}.png"
|
|
mkdir -p "${DESKTOPROOT}/usr/share/icons/hicolor/64x64/apps/"
|
|
cp "${SOURCEDIR}/pkg/linux/pgadmin4-64x64.png" "${DESKTOPROOT}/usr/share/icons/hicolor/64x64/apps/${APP_NAME}.png"
|
|
mkdir -p "${DESKTOPROOT}/usr/share/icons/hicolor/48x48/apps/"
|
|
cp "${SOURCEDIR}/pkg/linux/pgadmin4-48x48.png" "${DESKTOPROOT}/usr/share/icons/hicolor/48x48/apps/${APP_NAME}.png"
|
|
mkdir -p "${DESKTOPROOT}/usr/share/icons/hicolor/32x32/apps/"
|
|
cp "${SOURCEDIR}/pkg/linux/pgadmin4-32x32.png" "${DESKTOPROOT}/usr/share/icons/hicolor/32x32/apps/${APP_NAME}.png"
|
|
mkdir -p "${DESKTOPROOT}/usr/share/icons/hicolor/16x16/apps/"
|
|
cp "${SOURCEDIR}/pkg/linux/pgadmin4-16x16.png" "${DESKTOPROOT}/usr/share/icons/hicolor/16x16/apps/${APP_NAME}.png"
|
|
|
|
mkdir -p "${DESKTOPROOT}/usr/share/applications"
|
|
cp "${SOURCEDIR}/pkg/linux/pgadmin4.desktop" "${DESKTOPROOT}/usr/share/applications"
|
|
}
|
|
|
|
_build_docs() {
|
|
echo "Building the documentation..."
|
|
cd "${SERVERROOT}" && mkdir -p "usr/${APP_NAME}/share/docs/en_US/html"
|
|
cd "${SOURCEDIR}/docs/en_US" || exit
|
|
python3 build_code_snippet.py
|
|
|
|
SYS_PYTHON_PACKS=$("${SYSTEM_PYTHON_PATH}" -c "import sys; print(':'.join([p for p in sys.path if p]))")
|
|
# shellcheck disable=SC2153
|
|
PYTHONPATH=$PYTHONPATH:${SYS_PYTHON_PACKS} python3 -msphinx . "${SERVERROOT}/usr/${APP_NAME}/share/docs/en_US/html"
|
|
}
|
|
|
|
_copy_code() {
|
|
echo "Copying the server code..."
|
|
|
|
OS_VERSION=$(grep "^VERSION_ID=" /etc/os-release | awk -F "=" '{ print $2 }' | sed 's/"//g' | awk -F "." '{ print $1 }')
|
|
if [ "$1" == 'redhat' ] && [ "${OS_VERSION}" == "8" ]; then
|
|
echo "Setting file permission through umask..."
|
|
umask u+rwx,go+rx,go-w
|
|
fi
|
|
|
|
# Remove any TCL-related files that may cause us problems
|
|
find "${SERVERROOT}/usr/${APP_NAME}/venv/" -name "_tkinter*" -print0 | xargs -0 rm -rf
|
|
|
|
pushd "${SOURCEDIR}/web" > /dev/null || exit
|
|
YARN_VERSION=$(node -p "require('./package.json').packageManager.split('@')[1]")
|
|
if [ -z "${YARN_VERSION}" ]; then
|
|
echo "ERROR: Could not determine Yarn version from package.json packageManager field."
|
|
exit 1
|
|
fi
|
|
yarn set version "${YARN_VERSION}"
|
|
yarn install
|
|
yarn run bundle
|
|
popd > /dev/null || exit
|
|
|
|
# copy the web directory to the bundle as it is required by runtime
|
|
cp -r "${SOURCEDIR}/web" "${SERVERROOT}/usr/${APP_NAME}/web/"
|
|
cp "${SOURCEDIR}/pkg/linux/config_distro.py" "${SERVERROOT}/usr/${APP_NAME}/web/"
|
|
cd "${SERVERROOT}/usr/${APP_NAME}/web/" || exit
|
|
rm -f pgadmin4.db config_local.*
|
|
rm -rf jest.config.js babel.* package.json .yarn* yarn* webpack.* .editorconfig .eslint* node_modules/ regression/ tools/ pgadmin/static/js/generated/.cache
|
|
find . -name "tests" -type d -print0 | xargs -0 rm -rf
|
|
find . -name "feature_tests" -type d -print0 | xargs -0 rm -rf
|
|
find . -name "__pycache__" -type d -print0 | xargs -0 rm -rf
|
|
|
|
# License files
|
|
cp "${SOURCEDIR}/LICENSE" "${SERVERROOT}/usr/${APP_NAME}/"
|
|
|
|
# Web setup script
|
|
mkdir -p "${WEBROOT}/usr/${APP_NAME}/bin/"
|
|
cp "${SOURCEDIR}/pkg/linux/setup-web.sh" "${WEBROOT}/usr/${APP_NAME}/bin/"
|
|
|
|
# Ensure our venv will use the correct Python interpreter, even if the
|
|
# user has configured an alternative default.
|
|
# DO THIS LAST!
|
|
cd "${SERVERROOT}/usr/${APP_NAME}/venv/bin" || exit
|
|
PYTHON_INTERPRETER=$("${SYSTEM_PYTHON_PATH}" -c "import os, sys; print(os.path.realpath(sys.executable))")
|
|
PYTHON_VERSION=$("${SYSTEM_PYTHON_PATH}" -c "import sys; print('%d.%d' % (sys.version_info.major, sys.version_info.minor))")
|
|
rm python && ln -s python3 python
|
|
rm "python${PYTHON_VERSION}" && ln -s python3 "python${PYTHON_VERSION}"
|
|
rm python3 && ln -s "${PYTHON_INTERPRETER}" python3
|
|
}
|
|
|
|
|
|
_generate_sbom() {
|
|
echo "Generating SBOMs..."
|
|
# Note that we don't generate an SBOM for the Meta package as it doesn't contain any files.
|
|
syft "${SERVERROOT}/" -o cyclonedx-json > "${SERVERROOT}/usr/${APP_NAME}/sbom-server.json"
|
|
syft "${DESKTOPROOT}/" -o cyclonedx-json > "${DESKTOPROOT}/usr/${APP_NAME}/sbom-desktop.json"
|
|
syft "${WEBROOT}/" -o cyclonedx-json > "${WEBROOT}/usr/${APP_NAME}/sbom-web.json"
|
|
}
|