#!/bin/sh # The source of this file is https://raw.githubusercontent.com/grafana/writers-toolkit/main/docs/make-docs. # # `make-docs` procedure changelog # # Updates should conform to the guidelines in https://keepachangelog.com/en/1.1.0/. # [Semantic versioning](https://semver.org/) is used to help the reader identify the significance of changes. # Changes are relevant to this script and the support docs.mk GNU Make interface. # # ## 8.1.0 (2024-08-22) # # ### Added # # - Additional website mounts for projects that use the website repository. # # Mounts are required for `make docs` to work in the website repository or with the website project. # The Makefile is also mounted for convenient development of the procedure that repository. # # ## 8.0.1 (2024-07-01) # # ### Fixed # # - Update log suppression to catch new format of website /docs/ homepage REF_NOT_FOUND warnings. # # These warnings are related to missing some pages during the build that are required for the /docs/ homepage. # They were previously suppressed but the log format changed and without this change they reappear in the latest builds. # # ## 8.0.0 (2024-05-28) # # ### Changed # # - Add environment variable `OUTPUT_FORMAT` to control the output of commands. # # The default value is `human` and means the output format is human readable. # The value `json` is also supported and outputs JSON. # # Note that the `json` format isn't supported by `make docs`, only `make doc-validator` and `make vale`. # # ## 7.0.0 (2024-05-03) # # ### Changed # # - Pull images for all recipes that use containers by default. # # Use the `PULL=false` variable to disable this behavior. # # ### Removed # # - The `docs-no-pull` target as it's redundant with the new `PULL=false` variable. # # ## 6.1.0 (2024-04-22) # # ### Changed # # - Mount volumes with SELinux labels. # # https://docs.docker.com/storage/bind-mounts/#configure-the-selinux-label # # ### Added # # - Pseudo project for including only website resources and no website content. # # Facilitates testing shortcodes and layout changes with a small documentation set instead of Grafana Cloud or the entire website. # # ## 6.0.1 (2024-02-28) # # ### Added # # - Suppress new errors relating to absent content introduced in https://github.com/grafana/website/pull/17561. # # ## 6.0.0 (2024-02-16) # # ### Changed # # - Require `jq` for human readable `make doc-validator` output. # # ## 5.4.0 (2024-02-12) # # ### Changed # # - Set `WEBSITE_MOUNTS=true` when a user includes the `website` project. # # Ensures consistent behavior across repositories. # To disable website mounts, add `export WEBSITE_MOUNTS := false` to your `variables.mk` or `variables.mk.local` file. # - Use website mounts and container volumes also when a user includes the `grafana-cloud` project. # # ## 5.3.0 (2024-02-08) # # ### Changed # # - Updated support for plugins monorepo now that multiple projects have been moved into it. # - Use `printf` instead of `echo` for better portability of output. # # https://www.in-ulm.de/~mascheck/various/echo+printf/ # # ## 5.2.0 (2024-01-18) # # ### Changed # # - Updated `make vale` to use latest Vale style and configuration. # - Updated `make vale` to use platform appropriate image. # # ## 5.1.2 (2023-11-08) # # ### Added # # - Hide manual_mount warning messages from non-debug output. # Set the DEBUG environment variable to see all hidden messages. # # ## 5.1.1 (2023-10-30) # # ### Added # # - Support for Datadog and Oracle data source plugins repositories. # # ## 5.1.0 (2023-10-20) # # ### Added # # - Support for the plugins monorepo. # # ## 5.0.0 (2023-10-18) # # ### Added # # - Improved support for website repository. # # Mount more content and provide some feedback to users that the build can take time. # # - Ability to enter the `grafana/docs-base` container with a shell using the `ENTER` environment variable. # # ### Fixed # # - Correct key combination for interrupting the process. # # Keyboards use capital letters so this more accurately reflects the exact key combination users are expected to press. # # ### Removed # # - Imperfect implementation of container name. # # Facilitates running `make vale` and `make docs` at once. # Container names are convenient for recognition in `docker ps` but the current implementation has more downsides than upsides. # # - Forced platform specification now that multiple architecture images exist. # # Significantly speeds up build times on larger repositories. # # ## 4.2.2 (2023-10-05) # - Added support for Jira data source and MongoDB data source plugins repositories. # # ## 4.2.1 (2023-09-13) # ## Fixed # # - Improved consistency of the webserver request loop by polling the Hugo port rather than the proxy port. # # ## 4.2.0 (2023-09-01) # # ### Added # # - Retry the initial webserver request up to ten times to allow for the process to start. # If it is still failing after ten seconds, an error message is logged. # # ## 4.1.1 (2023-07-20) # # ### Fixed # # - Replaced use of `realpath` with POSIX compatible alternative to determine default value for REPOS_PATH. # # ## 4.1.0 (2023-06-16) # # ### Added # # - Mounts of `layouts` and `config` directories for the `website` project. # Ensures that local changes to mounts or shortcodes are reflected in the development server. # # ### Fixed # # - Version inference for versioned docs pages. # Pages in versioned projects now have the `versioned: true` front matter set to ensure that "version" in $.Page.Scratch is set on builds. # # ## 4.0.0 (2023-06-06) # # ### Removed # # - `doc-validator/%` target. # The behavior of the target was not as described. # Instead, to limit `doc-validator` to only specific files, refer to https://grafana.com/docs/writers-toolkit/writing-guide/tooling-and-workflows/validate-technical-documentation/#run-on-specific-files. # # ## 3.0.0 (2023-05-18) # # ### Fixed # # - Compatibility with the updated Make targets in the `website` repository. # `docs` now runs this script itself, `server-docs` builds the site with the `docs` Hugo environment. # # ## 2.0.0 (2023-05-18) # # ### Added # # - Support for the grafana-cloud/frontend-observability/faro-web-sdk project. # - Use of `doc-validator` v2.0.x which includes breaking changes to command line options. # # ### Fixed # # - Source grafana-cloud project from website repository. # # ### Added # # - Support for running the Vale linter with `make vale`. # # ## 1.2.1 (2023-05-05) # # ### Fixed # # - Use `latest` tag of `grafana/vale` image by default instead of hardcoded older version. # - Fix mounting multiple projects broken by the changes in 1.0.1 # # ## 1.2.0 (2023-05-05) # # ### Added # # - Support for running the Vale linter with `make vale`. # # ### Fixed # # ## 1.1.0 (2023-05-05) # # ### Added # # - Rewrite error output so it can be followed by text editors. # # ### Fixed # # - Fix `docs-debug` container process port. # # ## 1.0.1 (2023-05-04) # # ### Fixed # # - Ensure complete section hierarchy so that all projects have a visible menu. # # ## 1.0.0 (2023-05-04) # # ### Added # # - Build multiple projects simultaneously if all projects are checked out locally. # - Run [`doc-validator`](https://github.com/grafana/technical-documentation/tree/main/tools/cmd/doc-validator) over projects. # - Redirect project root to mounted version. # For example redirect `/docs/grafana/` to `/docs/grafana/latest/`. # - Support for Podman or Docker containers with `PODMAN` environment variable. # - Support for projects: # - agent # - enterprise-logs # - enterprise-metrics # - enterprise-traces # - grafana # - grafana-cloud # - grafana-cloud/machine-learning # - helm-charts/mimir-distributed # - helm-charts/tempo-distributed # - incident # - loki # - mimir # - oncall # - opentelemetry # - phlare # - plugins # - slo # - tempo # - writers-toolkit set -ef readonly DOCS_HOST_PORT="${DOCS_HOST_PORT:-3002}" readonly DOCS_IMAGE="${DOCS_IMAGE:-grafana/docs-base:latest}" readonly DOC_VALIDATOR_INCLUDE="${DOC_VALIDATOR_INCLUDE:-.+\.md$}" readonly DOC_VALIDATOR_SKIP_CHECKS="${DOC_VALIDATOR_SKIP_CHECKS:-^image-}" readonly HUGO_REFLINKSERRORLEVEL="${HUGO_REFLINKSERRORLEVEL:-WARNING}" readonly VALE_MINALERTLEVEL="${VALE_MINALERTLEVEL:-error}" readonly WEBSITE_EXEC="${WEBSITE_EXEC:-make server-docs}" readonly OUTPUT_FORMAT="${OUTPUT_FORMAT:-human}" PODMAN="$(if command -v podman >/dev/null 2>&1; then echo podman; else echo docker; fi)" if ! command -v curl >/dev/null 2>&1; then if ! command -v wget >/dev/null 2>&1; then errr 'either `curl` or `wget` must be installed for this script to work.' exit 1 fi fi if ! command -v "${PODMAN}" >/dev/null 2>&1; then errr 'either `podman` or `docker` must be installed for this script to work.' exit 1 fi about() { cat <...]> $0 [[:[:[:]]]...] Examples: REPOS_PATH=~/ext/grafana/ $0 writers-toolkit tempo:latest helm-charts/mimir-distributed:latest:mimir:docs/sources/mimir-distributed EOF } if [ $# -lt 1 ]; then cat <&2 ERRR: arguments required but not supplied. $(about) $(usage) EOF exit 1 fi readonly REPOS_PATH="${REPOS_PATH:-$(cd "$(git rev-parse --show-toplevel)/.." && echo "${PWD}")}" if [ -z "${REPOS_PATH}" ]; then cat <&2 ERRR: REPOS_PATH environment variable is required but has not been provided. $(usage) EOF exit 1 fi SOURCES_as_code='as-code-docs' SOURCES_enterprise_metrics='backend-enterprise' SOURCES_enterprise_metrics_='backend-enterprise' SOURCES_grafana_cloud='website' SOURCES_grafana_cloud_alerting_and_irm_machine_learning='machine-learning' SOURCES_grafana_cloud_alerting_and_irm_slo='slo' SOURCES_grafana_cloud_k6='k6-docs' SOURCES_grafana_cloud_data_configuration_integrations='cloud-onboarding' SOURCES_grafana_cloud_frontend_observability_faro_web_sdk='faro-web-sdk' SOURCES_helm_charts_mimir_distributed='mimir' SOURCES_helm_charts_tempo_distributed='tempo' SOURCES_opentelemetry='opentelemetry-docs' SOURCES_plugins_grafana_datadog_datasource='datadog-datasource' SOURCES_plugins_grafana_oracle_datasource='oracle-datasource' SOURCES_resources='website' VERSIONS_as_code='UNVERSIONED' VERSIONS_grafana_cloud='UNVERSIONED' VERSIONS_grafana_cloud_alerting_and_irm_machine_learning='UNVERSIONED' VERSIONS_grafana_cloud_alerting_and_irm_slo='UNVERSIONED' VERSIONS_grafana_cloud_k6='UNVERSIONED' VERSIONS_grafana_cloud_data_configuration_integrations='UNVERSIONED' VERSIONS_grafana_cloud_frontend_observability_faro_web_sdk='UNVERSIONED' VERSIONS_opentelemetry='UNVERSIONED' VERSIONS_plugins_grafana_datadog_datasource='latest' VERSIONS_plugins_grafana_oracle_datasource='latest' VERSIONS_resources='UNVERSIONED' VERSIONS_technical_documentation='UNVERSIONED' VERSIONS_website='UNVERSIONED' VERSIONS_writers_toolkit='UNVERSIONED' PATHS_grafana_cloud='content/docs/grafana-cloud' PATHS_helm_charts_mimir_distributed='docs/sources/helm-charts/mimir-distributed' PATHS_helm_charts_tempo_distributed='docs/sources/helm-charts/tempo-distributed' PATHS_mimir='docs/sources/mimir' PATHS_plugins_grafana_datadog_datasource='docs/sources' PATHS_plugins_grafana_oracle_datasource='docs/sources' PATHS_resources='content' PATHS_tempo='docs/sources/tempo' PATHS_website='content' # identifier STR # Replace characters that are not valid in an identifier with underscores. identifier() { echo "$1" | tr -C '[:alnum:]_\n' '_' } # aget ARRAY KEY # Get the value of KEY from associative array ARRAY. # Characters that are not valid in an identifier are replaced with underscores. aget() { eval echo '$'"$(identifier "$1")_$(identifier "$2")" } # src returns the project source repository name for a project. src() { _project="$1" case "${_project}" in plugins/*) if [ -z "$(aget SOURCES "${_project}")" ]; then echo plugins-private else aget SOURCES "${_project}" fi ;; *) if [ -z "$(aget SOURCES "${_project}")" ]; then echo "${_project}" else aget SOURCES "${_project}" fi ;; esac unset _project } # path returns the relative path within the repository that contain the docs for a project. path() { _project="$1" case "${_project}" in plugins/*) if [ -z "$(aget PATHS "${_project}")" ]; then echo "${_project}/docs/sources" else aget PATHS "${_project}" fi ;; *) if [ -z "$(aget PATHS "${_project}")" ]; then echo "docs/sources" else aget PATHS "${_project}" fi esac unset _project } # version returns the version for a project. Unversioned projects return the special value 'UNVERSIONED'. version() { _project="$1" case "${_project}" in plugins/*) if [ -z "$(aget VERSIONS "${_project}")" ]; then echo "UNVERSIONED" else aget VERSIONS "${_project}" fi ;; *) if [ -z "$(aget VERSIONS "${_project}")" ]; then echo latest else aget VERSIONS "${_project}" fi esac unset _project } # new_proj populates a new project structure. new_proj() { _project="$1" _version="$2" _repo="$3" _path="$4" # If version is not set, use the script mapping of project to default versions if it exists. # Fallback to 'latest'. if [ -z "${_version}" ]; then _version="$(version "${_project}")" fi # If repo is not set, use the script mapping of project to repo name if it exists. # Fallback to using the project name. if [ -z "${_repo}" ]; then _repo="$(src "${_project}")" fi # If path is not set, use the script mapping of project to docs sources path if it exists. # Fallback to using 'docs/sources'. if [ -z "${_path}" ]; then _path="$(path "${_project}")" fi echo "${_project}:${_version}:${_repo}:${_path}" unset _project _version _repo _path } # proj_url returns the webserver URL for a project. # It expects a complete project structure as input. proj_url() { IFS=: read -r _project _version _ _ <&2 fi } errr() { printf 'ERRR: %s\r\n' "$1" >&2 } note() { printf 'NOTE: %s\r\n' "$1" >&2 } url_src_dst_vers="$(url_src_dst_vers "$@")" volumes="" redirects="" for arg in "$@"; do IFS=: read -r _project _ _repo _ </dev/null 2>&1; then errr '`jq` must be installed for the `doc-validator` target to work.' note 'To install `jq`, refer to https://jqlang.github.io/jq/download/,' exit 1 fi ${cmd} \ | jq -r '"ERROR: \(.location.path):\(.location.range.start.line // 1):\(.location.range.start.column // 1): \(.message)" + if .suggestions[0].text then "\nSuggestion: \(.suggestions[0].text)" else "" end' ;; json) ${cmd} ;; *) # default errr "Invalid output format '${OUTPUT_FORMAT}'" esac ;; 'grafana/vale') proj="$(new_proj "$1")" printf '\r\n' IFS='' read -r cmd <"${tempfile}" #!/usr/bin/env bash tc() { set \${*,,} echo \${*^} } for redirect in ${redirects}; do IFS='^' read -r path ver <<<"\${redirect}" echo -e "---\\nredirectURL: \"\${path/\/hugo\/content/}\"\\ntype: redirect\\nversioned: true\\n---\\n" > "\${path/\${ver}/_index.md}" done for x in "${url_src_dst_vers}"; do IFS='^' read -r _ _ dst _ <<<"\${x}" title="\${dst%/*}" title="\$(tc \${title##*/})" while [[ -n "\${dst}" ]]; do if [[ ! -f "\${dst}/_index.md" ]]; then echo -e "---title: \${title}\\n---\\n\\n# \${title}\\n\\n{{< section >}}" > "\${dst}/_index.md" fi dst="\${dst%/*}" done done if [[ -n "${WEBSITE_MOUNTS}" ]]; then unset WEBSITE_SKIP_MOUNTS fi ${WEBSITE_EXEC} EOF chmod +x "${tempfile}" volumes="${volumes} --volume=${tempfile}:/entrypoint:z" readonly volumes IFS='' read -r cmd <&1\ | sed -u \ -e '/Web Server is available at http:\/\/localhost:3003\/ (bind address 0.0.0.0)/ d' \ -e '/^hugo server/ d' \ -e '/fatal: not a git repository (or any parent up to mount point \/)/ d' \ -e '/Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set)./ d' \ -e "/Makefile:[0-9]*: warning: overriding recipe for target 'docs'/ d" \ -e "/docs.mk:[0-9]*: warning: ignoring old recipe for target 'docs'/ d" \ -e '/\/usr\/bin\/make -j 2 proxy hserver-docs HUGO_PORT=3003/ d' \ -e '/website-proxy/ d' \ -e '/rm -rf dist*/ d' \ -e '/Press Ctrl+C to stop/ d' \ -e '/make/ d' \ -e '/WARNING: The manual_mount source directory/ d' \ -e '/"docs\/_index.md" not found/d' fi ;; esac