grafana/docs/make-docs
github-actions[bot] 6fd8898cde
Update make docs procedure (#81170)
Co-authored-by: grafanabot <bot@grafana.com>
2024-01-24 16:03:38 +00:00

801 lines
22 KiB
Bash
Executable File

#!/bin/sh
# The source of this file is https://raw.githubusercontent.com/grafana/writers-toolkit/main/docs/make-docs.
# # `make-docs` procedure changelog
#
# Updates should conform to the guidelines in https://keepachangelog.com/en/1.1.0/.
# [Semantic versioning](https://semver.org/) is used to help the reader identify the significance of changes.
# Changes are relevant to this script and the support docs.mk GNU Make interface.
#
# ## 5.2.0 (2024-01-18)
#
# ### Changed
#
# - Updated `make vale` to use latest Vale style and configuration.
# - Updated `make vale` to use platform appropriate image.
#
# ## 5.1.2 (2023-11-08)
#
# ### Added
#
# - Hide manual_mount warning messages from non-debug output.
# Set the DEBUG environment variable to see all hidden messages.
#
# ## 5.1.1 (2023-10-30)
#
# ### Added
#
# - Support for Datadog and Oracle data source plugins repositories.
#
# ## 5.1.0 (2023-10-20)
#
# ### Added
#
# - Support for the plugins monorepo.
#
# ## 5.0.0 (2023-10-18)
#
# ### Added
#
# - Improved support for website repository.
#
# Mount more content and provide some feedback to users that the build can take time.
#
# - Ability to enter the `grafana/docs-base` container with a shell using the `ENTER` environment variable.
#
# ### Fixed
#
# - Correct key combination for interrupting the process.
#
# Keyboards use capital letters so this more accurately reflects the exact key combination users are expected to press.
#
# ### Removed
#
# - Imperfect implementation of container name.
#
# Facilitates running `make vale` and `make docs` at once.
# Container names are convenient for recognition in `docker ps` but the current implementation has more downsides than upsides.
#
# - Forced platform specification now that multiple architecture images exist.
#
# Significantly speeds up build times on larger repositories.
#
# ## 4.2.2 (2023-10-05)
# - Added support for Jira data source and MongoDB data source plugins repositories.
#
# ## 4.2.1 (2023-09-13)
# ## Fixed
#
# - Improved consistency of the webserver request loop by polling the Hugo port rather than the proxy port.
#
# ## 4.2.0 (2023-09-01)
#
# ### Added
#
# - Retry the initial webserver request up to ten times to allow for the process to start.
# If it is still failing after ten seconds, an error message is logged.
#
# ## 4.1.1 (2023-07-20)
#
# ### Fixed
#
# - Replaced use of `realpath` with POSIX compatible alternative to determine default value for REPOS_PATH.
#
# ## 4.1.0 (2023-06-16)
#
# ### Added
#
# - Mounts of `layouts` and `config` directories for the `website` project.
# Ensures that local changes to mounts or shortcodes are reflected in the development server.
#
# ### Fixed
#
# - Version inference for versioned docs pages.
# Pages in versioned projects now have the `versioned: true` front matter set to ensure that "version" in $.Page.Scratch is set on builds.
#
# ## 4.0.0 (2023-06-06)
#
# ### Removed
#
# - `doc-validator/%` target.
# The behavior of the target was not as described.
# Instead, to limit `doc-validator` to only specific files, refer to https://grafana.com/docs/writers-toolkit/writing-guide/tooling-and-workflows/validate-technical-documentation/#run-on-specific-files.
#
# ## 3.0.0 (2023-05-18)
#
# ### Fixed
#
# - Compatibility with the updated Make targets in the `website` repository.
# `docs` now runs this script itself, `server-docs` builds the site with the `docs` Hugo environment.
#
# ## 2.0.0 (2023-05-18)
#
# ### Added
#
# - Support for the grafana-cloud/frontend-observability/faro-web-sdk project.
# - Use of `doc-validator` v2.0.x which includes breaking changes to command line options.
#
# ### Fixed
#
# - Source grafana-cloud project from website repository.
#
# ### Added
#
# - Support for running the Vale linter with `make vale`.
#
# ## 1.2.1 (2023-05-05)
#
# ### Fixed
#
# - Use `latest` tag of `grafana/vale` image by default instead of hardcoded older version.
# - Fix mounting multiple projects broken by the changes in 1.0.1
#
# ## 1.2.0 (2023-05-05)
#
# ### Added
#
# - Support for running the Vale linter with `make vale`.
#
# ### Fixed
#
# ## 1.1.0 (2023-05-05)
#
# ### Added
#
# - Rewrite error output so it can be followed by text editors.
#
# ### Fixed
#
# - Fix `docs-debug` container process port.
#
# ## 1.0.1 (2023-05-04)
#
# ### Fixed
#
# - Ensure complete section hierarchy so that all projects have a visible menu.
#
# ## 1.0.0 (2023-05-04)
#
# ### Added
#
# - Build multiple projects simultaneously if all projects are checked out locally.
# - Run [`doc-validator`](https://github.com/grafana/technical-documentation/tree/main/tools/cmd/doc-validator) over projects.
# - Redirect project root to mounted version.
# For example redirect `/docs/grafana/` to `/docs/grafana/latest/`.
# - Support for Podman or Docker containers with `PODMAN` environment variable.
# - Support for projects:
# - agent
# - enterprise-logs
# - enterprise-metrics
# - enterprise-traces
# - grafana
# - grafana-cloud
# - grafana-cloud/machine-learning
# - helm-charts/mimir-distributed
# - helm-charts/tempo-distributed
# - incident
# - loki
# - mimir
# - oncall
# - opentelemetry
# - phlare
# - plugins
# - slo
# - tempo
# - writers-toolkit
set -ef
readonly DOCS_HOST_PORT="${DOCS_HOST_PORT:-3002}"
readonly DOCS_IMAGE="${DOCS_IMAGE:-grafana/docs-base:latest}"
readonly DOC_VALIDATOR_INCLUDE="${DOC_VALIDATOR_INCLUDE:-.+\.md$}"
readonly DOC_VALIDATOR_SKIP_CHECKS="${DOC_VALIDATOR_SKIP_CHECKS:-^image-}"
readonly HUGO_REFLINKSERRORLEVEL="${HUGO_REFLINKSERRORLEVEL:-WARNING}"
readonly VALE_MINALERTLEVEL="${VALE_MINALERTLEVEL:-error}"
readonly WEBSITE_EXEC="${WEBSITE_EXEC:-make server-docs}"
# If set, the docs-base image will run a prebuild script that sets up Hugo mounts.
readonly WEBSITE_MOUNTS="${WEBSITE_MOUNTS:-}"
PODMAN="$(if command -v podman >/dev/null 2>&1; then echo podman; else echo docker; fi)"
if ! command -v curl >/dev/null 2>&1; then
if ! command -v wget >/dev/null 2>&1; then
errr 'either `curl` or `wget` must be installed for this script to work.'
exit 1
fi
fi
if ! command -v "${PODMAN}" >/dev/null 2>&1; then
errr 'either `podman` or `docker` must be installed for this script to work.'
exit 1
fi
about() {
cat <<EOF
Test documentation locally with multiple source repositories.
The REPOS_PATH environment variable is a colon (:) separated list of paths in which to look for project repositories.
EOF
}
usage() {
cat <<EOF
Usage:
REPOS_PATH=<PATH[:<PATH>...]> $0 [<PROJECT>[:<VERSION>[:<REPO>[:<DIR>]]]...]
Examples:
REPOS_PATH=~/ext/grafana/ $0 writers-toolkit tempo:latest helm-charts/mimir-distributed:latest:mimir:docs/sources/mimir-distributed
EOF
}
if [ $# -lt 1 ]; then
cat <<EOF >&2
ERRR: arguments required but not supplied.
$(about)
$(usage)
EOF
exit 1
fi
readonly REPOS_PATH="${REPOS_PATH:-$(cd "$(git rev-parse --show-toplevel)/.." && echo "${PWD}")}"
if [ -z "${REPOS_PATH}" ]; then
cat <<EOF >&2
ERRR: REPOS_PATH environment variable is required but has not been provided.
$(usage)
EOF
exit 1
fi
SOURCES_as_code='as-code-docs'
SOURCES_enterprise_metrics='backend-enterprise'
SOURCES_enterprise_metrics_='backend-enterprise'
SOURCES_grafana_cloud='website'
SOURCES_grafana_cloud_alerting_and_irm_machine_learning='machine-learning'
SOURCES_grafana_cloud_alerting_and_irm_slo='slo'
SOURCES_grafana_cloud_k6='k6-docs'
SOURCES_grafana_cloud_data_configuration_integrations='cloud-onboarding'
SOURCES_grafana_cloud_frontend_observability_faro_web_sdk='faro-web-sdk'
SOURCES_helm_charts_mimir_distributed='mimir'
SOURCES_helm_charts_tempo_distributed='tempo'
SOURCES_opentelemetry='opentelemetry-docs'
SOURCES_plugins_grafana_datadog_datasource='datadog-datasource'
SOURCES_plugins_grafana_jira_datasource='jira-datasource'
SOURCES_plugins_grafana_mongodb_datasource='mongodb-datasource'
SOURCES_plugins_grafana_oracle_datasource='oracle-datasource'
SOURCES_plugins_grafana_splunk_datasource='splunk-datasource'
VERSIONS_as_code='UNVERSIONED'
VERSIONS_grafana_cloud='UNVERSIONED'
VERSIONS_grafana_cloud_alerting_and_irm_machine_learning='UNVERSIONED'
VERSIONS_grafana_cloud_alerting_and_irm_slo='UNVERSIONED'
VERSIONS_grafana_cloud_k6='UNVERSIONED'
VERSIONS_grafana_cloud_data_configuration_integrations='UNVERSIONED'
VERSIONS_grafana_cloud_frontend_observability_faro_web_sdk='UNVERSIONED'
VERSIONS_opentelemetry='UNVERSIONED'
VERSIONS_plugins_grafana_datadog_datasource='latest'
VERSIONS_plugins_grafana_jira_datasource='latest'
VERSIONS_plugins_grafana_mongodb_datasource='latest'
VERSIONS_plugins_grafana_oracle_datasource='latest'
VERSIONS_plugins_grafana_splunk_datasource='latest'
VERSIONS_technical_documentation='UNVERSIONED'
VERSIONS_website='UNVERSIONED'
VERSIONS_writers_toolkit='UNVERSIONED'
PATHS_grafana_cloud='content/docs/grafana-cloud'
PATHS_helm_charts_mimir_distributed='docs/sources/helm-charts/mimir-distributed'
PATHS_helm_charts_tempo_distributed='docs/sources/helm-charts/tempo-distributed'
PATHS_mimir='docs/sources/mimir'
PATHS_plugins_grafana_datadog_datasource='docs/sources'
PATHS_plugins_grafana_jira_datasource='docs/sources'
PATHS_plugins_grafana_mongodb_datasource='docs/sources'
PATHS_plugins_grafana_oracle_datasource='docs/sources'
PATHS_plugins_grafana_splunk_datasource='docs/sources'
PATHS_tempo='docs/sources/tempo'
PATHS_website='content'
# identifier STR
# Replace characters that are not valid in an identifier with underscores.
identifier() {
echo "$1" | tr -C '[:alnum:]_\n' '_'
}
# aget ARRAY KEY
# Get the value of KEY from associative array ARRAY.
# Characters that are not valid in an identifier are replaced with underscores.
aget() {
eval echo '$'"$(identifier "$1")_$(identifier "$2")"
}
# src returns the project source repository name for a project.
src() {
_project="$1"
case "${_project}" in
plugins/*)
if [ -z "$(aget SOURCES "${_project}")" ]; then
echo plugins-private
else
aget SOURCES "${_project}"
fi
;;
*)
if [ -z "$(aget SOURCES "${_project}")" ]; then
echo "${_project}"
else
aget SOURCES "${_project}"
fi
;;
esac
unset _project
}
# path returns the relative path within the repository that contain the docs for a project.
path() {
_project="$1"
case "${_project}" in
plugins/*)
if [ -z "$(aget PATHS "${_project}")" ]; then
echo "${_project}/docs/sources"
else
aget PATHS "${_project}"
fi
;;
*)
if [ -z "$(aget PATHS "${_project}")" ]; then
echo "docs/sources"
else
aget PATHS "${_project}"
fi
esac
unset _project
}
# version returns the version for a project. Unversioned projects return the special value 'UNVERSIONED'.
version() {
_project="$1"
case "${_project}" in
plugins/*)
if [ -z "$(aget VERSIONS "${_project}")" ]; then
echo "UNVERSIONED"
else
aget VERSIONS "${_project}"
fi
;;
*)
if [ -z "$(aget VERSIONS "${_project}")" ]; then
echo latest
else
aget VERSIONS "${_project}"
fi
esac
unset _project
}
# new_proj populates a new project structure.
new_proj() {
_project="$1"
_version="$2"
_repo="$3"
_path="$4"
# If version is not set, use the script mapping of project to default versions if it exists.
# Fallback to 'latest'.
if [ -z "${_version}" ]; then
_version="$(version "${_project}")"
fi
# If repo is not set, use the script mapping of project to repo name if it exists.
# Fallback to using the project name.
if [ -z "${_repo}" ]; then
_repo="$(src "${_project}")"
fi
# If path is not set, use the script mapping of project to docs sources path if it exists.
# Fallback to using 'docs/sources'.
if [ -z "${_path}" ]; then
_path="$(path "${_project}")"
fi
echo "${_project}:${_version}:${_repo}:${_path}"
unset _project _version _repo _path
}
# proj_url returns the webserver URL for a project.
# It expects a complete project structure as input.
proj_url() {
IFS=: read -r _project _version _ _ <<POSIX_HERESTRING
$1
POSIX_HERESTRING
if [ "${_project}" = 'website' ]; then
echo "http://localhost:${DOCS_HOST_PORT}/docs/"
unset _project _version
return
fi
if [ -z "${_version}" ] || [ "${_version}" = 'UNVERSIONED' ]; then
echo "http://localhost:${DOCS_HOST_PORT}/docs/${_project}/"
else
echo "http://localhost:${DOCS_HOST_PORT}/docs/${_project}/${_version}/"
fi
unset _project _version
}
# proj_ver returns the version for a project.
# It expects a complete project structure as input.
proj_ver() {
IFS=: read -r _ _ver _ _ <<POSIX_HERESTRING
$1
POSIX_HERESTRING
echo "${_ver}"
unset _ver
}
# proj_dst returns the container path to content source for a project.
# It expects a complete project structure as input.
proj_dst() {
IFS=: read -r _project _version _ _ <<POSIX_HERESTRING
$1
POSIX_HERESTRING
if [ "${_project}" = 'website' ]; then
echo '/hugo/content'
unset _project _version
return
fi
if [ -z "${_version}" ] || [ "${_version}" = 'UNVERSIONED' ]; then
echo "/hugo/content/docs/${_project}"
else
echo "/hugo/content/docs/${_project}/${_version}"
fi
unset _project _version
}
# repo_path returns the host path to the project repository.
# It looks for the provided repository name in each of the paths specified in the REPOS_PATH environment variable.
repo_path() {
_repo="$1"
IFS=:
for lookup in ${REPOS_PATH}; do
if [ -d "${lookup}/${_repo}" ]; then
echo "${lookup}/${_repo}"
unset _path _repo
return
fi
done
unset IFS
errr "could not find project '${_repo}' in any of the paths in REPOS_PATH '${REPOS_PATH}'."
note "you must have a checkout of the project '${_repo}' at '${REPOS_PATH##:*}/${_repo}'."
note "if you have cloned the repository into a directory with a different name, consider changing it to ${_repo}."
unset _repo
exit 1
}
# proj_src returns the host path to content source for a project.
# It expects a complete project structure as input.
# It looks for the provided repository name in each of the paths specified in the REPOS_PATH environment variable.
proj_src() {
IFS=: read -r _ _ _repo _path <<POSIX_HERESTRING
$1
POSIX_HERESTRING
_repo="$(repo_path "${_repo}")"
echo "${_repo}/${_path}"
unset _path _repo
}
# proj_canonical returns the canonical absolute path partial URI for a project.
# It expects a complete project structure as input.
proj_canonical() {
IFS=: read -r _project _version _ _ <<POSIX_HERESTRING
$1
POSIX_HERESTRING
if [ "${_project}" = 'website' ]; then
echo '/docs'
unset _project _version
return
fi
if [ -z "${_version}" ] || [ "${_version}" = 'UNVERSIONED' ]; then
echo "/docs/${_project}"
else
echo "/docs/${_project}/${_version}"
fi
unset _project _version
}
proj_to_url_src_dst_ver() {
_url="$(proj_url "$1")"
_src="$(proj_src "$1")"
_dst="$(proj_dst "$1")"
_ver="$(proj_ver "$1")"
echo "${_url}^${_src}^${_dst}^${_ver}"
unset _url _src _dst _ver
}
url_src_dst_vers() {
for arg in "$@"; do
IFS=: read -r _project _version _repo _path <<POSIX_HERESTRING
$arg
POSIX_HERESTRING
case "${_project}" in
# Workaround for arbitrary mounts where the version field is expected to be the local directory
# and the repo field is expected to be the container directory.
arbitrary)
echo "${_project}^${_version}^${_repo}^" # TODO
;;
logs)
proj_to_url_src_dst_ver "$(new_proj loki "${_version}")"
proj_to_url_src_dst_ver "$(new_proj enterprise-logs "${_version}")"
;;
metrics)
proj_to_url_src_dst_ver "$(new_proj mimir "${_version}")"
proj_to_url_src_dst_ver "$(new_proj helm-charts/mimir-distributed "${_version}")"
proj_to_url_src_dst_ver "$(new_proj enterprise-metrics "${_version}")"
;;
traces)
proj_to_url_src_dst_ver "$(new_proj tempo "${_version}")"
proj_to_url_src_dst_ver "$(new_proj enterprise-traces "${_version}")"
;;
*)
proj_to_url_src_dst_ver "$(new_proj "${_project}" "${_version}" "${_repo}" "${_path}")"
;;
esac
done
unset _project _version _repo _path
}
await_build() {
url="$1"
req="$(if command -v curl >/dev/null 2>&1; then echo 'curl -s -o /dev/null'; else echo 'wget -q'; fi)"
i=1
max=10
while [ "${i}" -ne "${max}" ]
do
sleep 1
debg "Retrying request to webserver assuming the process is still starting up."
i=$((i + 1))
if ${req} "${url}"; then
echo
echo "View documentation locally:"
for x in ${url_src_dst_vers}; do
IFS='^' read -r url _ _ <<POSIX_HERESTRING
$x
POSIX_HERESTRING
if [ -n "${url}" ]; then
if [ "${_url}" != "arbitrary" ]; then
echo " ${url}"
fi
fi
done
echo
echo 'Press Ctrl+C to stop the server'
unset i max req url
return
fi
done
echo
errr 'The build was interrupted or a build error occurred, check the previous logs for possible causes.'
note 'You might need to use Ctrl+C to end the process.'
unset i max req url
}
debg() {
if [ -n "${DEBUG}" ]; then
echo "DEBG: $1" >&2
fi
}
errr() {
echo "ERRR: $1" >&2
}
note() {
echo "NOTE: $1" >&2
}
url_src_dst_vers="$(url_src_dst_vers "$@")"
volumes=""
redirects=""
for arg in "$@"; do
IFS=: read -r _project _ _repo _ <<POSIX_HERESTRING
${arg}
POSIX_HERESTRING
if [ "${_project}" = website ]; then
note "Please be patient, building the website can take some time."
_repo="$(repo_path website)"
volumes="--volume=${_repo}/config:/hugo/config"
volumes="${volumes} --volume=${_repo}/layouts:/hugo/layouts"
volumes="${volumes} --volume=${_repo}/scripts:/hugo/scripts"
fi
unset _project _repo
done
for x in ${url_src_dst_vers}; do
IFS='^' read -r _url _src _dst _ver <<POSIX_HERESTRING
$x
POSIX_HERESTRING
if [ "${_url}" != "arbitrary" ]; then
if [ ! -f "${_src}/_index.md" ]; then
errr "Index file '${_src}/_index.md' does not exist."
note "Is '${_src}' the correct source directory?"
exit 1
fi
fi
debg "Mounting '${_src}' at container path '${_dst}'"
if [ -z "${volumes}" ]; then
volumes="--volume=${_src}:${_dst}"
else
volumes="${volumes} --volume=${_src}:${_dst}"
fi
if [ -n "${_ver}" ] && [ "${_ver}" != 'UNVERSIONED' ]; then
if [ -z "${redirects}" ]; then
redirects="${_dst}^${_ver}"
else
redirects="${redirects} ${_dst}^${_ver}"
fi
fi
unset _url _src _dst _ver
done
IFS=':' read -r image _ <<POSIX_HERESTRING
${DOCS_IMAGE}
POSIX_HERESTRING
case "${image}" in
'grafana/doc-validator')
proj="$(new_proj "$1")"
echo
"${PODMAN}" run \
--init \
--interactive \
--platform linux/amd64 \
--rm \
--tty \
${volumes} \
"${DOCS_IMAGE}" \
"--include=${DOC_VALIDATOR_INCLUDE}" \
"--skip-checks=${DOC_VALIDATOR_SKIP_CHECKS}" \
/hugo/content/docs \
"$(proj_canonical "${proj}")" | sed "s#$(proj_dst "${proj}")#sources#"
;;
'grafana/vale')
proj="$(new_proj "$1")"
echo
"${PODMAN}" run \
--init \
--interactive \
--rm \
--workdir /etc/vale \
--tty \
${volumes} \
"${DOCS_IMAGE}" \
"--minAlertLevel=${VALE_MINALERTLEVEL}" \
'--glob=*.md' \
--output=/etc/vale/rdjsonl.tmpl \
/hugo/content/docs | sed "s#$(proj_dst "${proj}")#sources#"
;;
*)
tempfile="$(mktemp -t make-docs.XXX)"
cat <<EOF >"${tempfile}"
#!/usr/bin/env bash
tc() {
set \${*,,}
echo \${*^}
}
for redirect in ${redirects}; do
IFS='^' read -r path ver <<<"\${redirect}"
echo -e "---\\nredirectURL: \"\${path/\/hugo\/content/}\"\\ntype: redirect\\nversioned: true\\n---\\n" > "\${path/\${ver}/_index.md}"
done
for x in "${url_src_dst_vers}"; do
IFS='^' read -r _ _ dst _ <<<"\${x}"
title="\${dst%/*}"
title="\$(tc \${title##*/})"
while [[ -n "\${dst}" ]]; do
if [[ ! -f "\${dst}/_index.md" ]]; then
echo -e "---title: \${title}\\n---\\n\\n# \${title}\\n\\n{{< section >}}" > "\${dst}/_index.md"
fi
dst="\${dst%/*}"
done
done
if [[ -n "${WEBSITE_MOUNTS}" ]]; then
unset WEBSITE_SKIP_MOUNTS
fi
${WEBSITE_EXEC}
EOF
chmod +x "${tempfile}"
volumes="${volumes} --volume=${tempfile}:/entrypoint"
readonly volumes
IFS='' read -r cmd <<EOF
${PODMAN} run \
--env=HUGO_REFLINKSERRORLEVEL=${HUGO_REFLINKSERRORLEVEL} \
--init \
--interactive \
--publish=${DOCS_HOST_PORT}:3002 \
--publish=3003:3003 \
--rm \
--tty \
${volumes} \
${DOCS_IMAGE}
EOF
if [ -n "${ENTER}" ]; then
${cmd} /bin/bash
elif [ -n "${DEBUG}" ]; then
await_build http://localhost:3003 &
debg "${cmd} /entrypoint"
${cmd} /entrypoint
else
await_build http://localhost:3003 &
${cmd} /entrypoint 2>&1\
| sed -u \
-e '/Web Server is available at http:\/\/localhost:3003\/ (bind address 0.0.0.0)/ d' \
-e '/^hugo server/ d' \
-e '/fatal: not a git repository (or any parent up to mount point \/)/ d' \
-e '/Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set)./ d' \
-e "/Makefile:[0-9]*: warning: overriding recipe for target 'docs'/ d" \
-e "/docs.mk:[0-9]*: warning: ignoring old recipe for target 'docs'/ d" \
-e '/\/usr\/bin\/make -j 2 proxy hserver-docs HUGO_PORT=3003/ d' \
-e '/website-proxy/ d' \
-e '/rm -rf dist*/ d' \
-e '/Press Ctrl+C to stop/ d' \
-e '/make/ d' \
-e '/WARNING: The manual_mount source directory/ d'
fi
;;
esac