Update make-docs script (#66841)

Update centralized make-docs procedure

Signed-off-by: Jack Baldry <jack.baldry@grafana.com>

.gitignore

@@ -191,4 +191,3 @@ public/api-spec.json
 deployment_tools_config.json
 
 .betterer.cache
-/docs/make-docs

docs/docs.mk

@@ -14,7 +14,12 @@ MAKEFLAGS += --no-builtin-rule
 .PHONY: help
 help: ## Display this help.
 help:
-	@awk 'BEGIN {FS = ": ##"; printf "Usage:\n  make <target>\n\nTargets:\n"} /^[a-zA-Z0-9_\.\-\/%]+: ##/ { printf "  %-45s %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
+	@awk 'BEGIN { \
+		FS = ": ##"; \
+		printf "Usage:\n  make <target>\n\nTargets:\n" \
+	} \
+	/^[a-zA-Z0-9_\.\-\/%]+: ##/ { printf "  %-15s %s\n", $$1, $$2 }' \
+	$(MAKEFILE_LIST)
 
 GIT_ROOT := $(shell git rev-parse --show-toplevel)
 
@@ -25,26 +30,38 @@ $(error "PROJECTS variable must be defined in variables.mk")
 endif
 
 # First project is considered the primary one used for doc-validator.
-PRIMARY_PROJECT := $(firstword $(subst /,-,$(PROJECTS)))
+PRIMARY_PROJECT := $(subst /,-,$(firstword $(subst :, ,$(firstword $(PROJECTS)))))
 
 # Name for the container.
+ifeq ($(origin DOCS_CONTAINER), undefined)
 export DOCS_CONTAINER := $(PRIMARY_PROJECT)-docs
+endif
 
 # Host port to publish container port to.
+ifeq ($(origin DOCS_HOST_PORT), undefined)
 export DOCS_HOST_PORT := 3002
+endif
 
 # Container image used to perform Hugo build.
+ifeq ($(origin DOCS_IMAGE), undefined)
 export DOCS_IMAGE := grafana/docs-base:latest
+endif
 
 # Container image used for doc-validator linting.
+ifeq ($(origin DOC_VALIDATOR_IMAGE), undefined)
 export DOC_VALIDATOR_IMAGE := grafana/doc-validator:latest
+endif
 
 # PATH-like list of directories within which to find projects.
 # If all projects are checked out into the same directory, ~/repos/ for example, then the default should work.
+ifeq ($(origin REPOS_PATH), undefined)
 export REPOS_PATH := $(realpath $(GIT_ROOT)/..)
+endif
 
 # How to treat Hugo relref errors.
+ifeq ($(origin HUGO_REFLINKSERRORLEVEL), undefined)
 export HUGO_REFLINKSERRORLEVEL := WARNING
+endif
 
 .PHONY: docs-rm
 docs-rm: ## Remove the docs container.
@@ -56,27 +73,38 @@ docs-pull: ## Pull documentation base image.
 
 make-docs: ## Fetch the latest make-docs script.
 make-docs:
-	curl -s -LO https://raw.githubusercontent.com/grafana/writers-toolkit/main/scripts/make-docs
+	if [[ ! -f "$(PWD)/make-docs" ]]; then
-	chmod +x make-docs
+		echo 'WARN: No make-docs script found in the working directory. Run `make update` to download it.' >&2
+		exit 1
+	fi
 
 .PHONY: docs
-docs: ## Serve documentation locally.
+docs: ## Serve documentation locally, which includes pulling the latest `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image. See also `docs-no-pull`.
 docs: docs-pull make-docs
 	$(PWD)/make-docs $(PROJECTS)
 
 .PHONY: docs-no-pull
-docs-no-pull: ## Serve documentation locally without pulling the latest docs-base image.
+docs-no-pull: ## Serve documentation locally without pulling the `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image.
 docs-no-pull: make-docs
 	$(PWD)/make-docs $(PROJECTS)
 
+.PHONY: docs-debug
+docs-debug: ## Run Hugo web server with debugging enabled. TODO: support all SERVER_FLAGS defined in website Makefile.
+docs-debug: make-docs
+	WEBSITE_EXEC='hugo server --debug' $(PWD)/make-docs $(PROJECTS)
+
 .PHONY: doc-validator
 doc-validator: ## Run docs-validator on the entire docs folder.
+doc-validator: make-docs
 	DOCS_IMAGE=$(DOC_VALIDATOR_IMAGE) $(PWD)/make-docs $(PROJECTS)
 
 .PHONY: doc-validator/%
 doc-validator/%: ## Run doc-validator on a specific path. To lint the path /docs/sources/administration, run 'make doc-validator/administration'.
-doc-validator/%:
+doc-validator/%: make-docs
 	DOCS_IMAGE=$(DOC_VALIDATOR_IMAGE) DOC_VALIDATOR_INCLUDE=$(subst doc-validator/,,$@) $(PWD)/make-docs $(PROJECTS)
 
-docs.mk: ## Fetch the latest version of this Makefile from Writers' Toolkit.
+.PHONY: update
+update: ## Fetch the latest version of this Makefile and the `make-docs` script from Writers' Toolkit.
 	curl -s -LO https://raw.githubusercontent.com/grafana/writers-toolkit/main/docs/docs.mk
+	curl -s -LO https://raw.githubusercontent.com/grafana/writers-toolkit/main/docs/make-docs
+	chmod +x make-docs

docs/make-docs

@@ -0,0 +1,359 @@
+#!/bin/sh
+# The source of this file is https://raw.githubusercontent.com/grafana/writers-toolkit/main/docs/make-docs.
+
+set -ef
+
+readonly DOCS_CONTAINER="${DOCS_CONTAINER:-make-docs}"
+readonly DOCS_HOST_PORT="${DOCS_HOST_PORT:-3002}"
+readonly DOCS_IMAGE="${DOCS_IMAGE:-grafana/docs-base:latest}"
+
+readonly DOC_VALIDATOR_INCLUDE="${DOC_VALIDATOR_INCLUDE:-.+\.md$}"
+
+readonly HUGO_REFLINKSERRORLEVEL="${HUGO_REFLINKSERRORLEVEL:-WARNING}"
+readonly WEBSITE_EXEC="${WEBSITE_EXEC:-make server}"
+# If set, the docs-base image will run a prebuild script that sets up Hugo mounts.
+readonly WEBSITE_MOUNTS="${WEBSITE_MOUNTS:-}"
+
+PODMAN="$(if command -v podman >/dev/null 2>&1; then echo podman; else echo docker; fi)"
+
+about() {
+  cat <<EOF
+Test documentation locally with multiple source repositories.
+
+The REPOS_PATH environment variable is a colon (:) separated list of paths in which to look for project repositories.
+EOF
+}
+
+usage() {
+  cat <<EOF
+Usage:
+  REPOS_PATH=<PATH[:<PATH>...]> $0 [<PROJECT>[:<VERSION>[:<REPO>[:<DIR>]]]...]
+
+Examples:
+  REPOS_PATH=~/ext/grafana/ $0 writers-toolkit tempo:latest helm-charts/mimir-distributed:latest:mimir:docs/sources/mimir-distributed
+EOF
+}
+
+if [ $# -lt 1 ]; then
+  cat <<EOF >&2
+ERRR: arguments required but not supplied.
+
+$(about)
+
+$(usage)
+EOF
+  exit 1
+fi
+
+readonly REPOS_PATH="${REPOS_PATH:-$(realpath "$(git rev-parse --show-toplevel)/..")}"
+
+if [ -z "${REPOS_PATH}" ]; then
+  cat <<EOF >&2
+ERRR: REPOS_PATH environment variable is required but has not been provided.
+
+$(usage)
+EOF
+  exit 1
+fi
+
+SOURCES_as_code='as-code-docs'
+SOURCES_enterprise_metrics='backend-enterprise'
+SOURCES_enterprise_metrics_='backend-enterprise'
+SOURCES_grafana_cloud='cloud-docs'
+SOURCES_grafana_cloud_k6='k6-docs'
+SOURCES_helm_charts_mimir_distributed='mimir'
+SOURCES_helm_charts_tempo_distributed='tempo'
+SOURCES_opentelemetry='opentelemetry-docs'
+
+VERSIONS_as_code='UNVERSIONED'
+VERSIONS_grafana_cloud='UNVERSIONED'
+VERSIONS_grafana_cloud_k6='UNVERSIONED'
+VERSIONS_opentelemetry='UNVERSIONED'
+VERSIONS_technical_documentation='UNVERSIONED'
+VERSIONS_writers_toolkit='UNVERSIONED'
+
+PATHS_helm_charts_mimir_distributed='docs/sources/helm-charts/mimir-distributed'
+PATHS_helm_charts_tempo_distributed='docs/sources/helm-charts/tempo-distributed'
+PATHS_mimir='docs/sources/mimir'
+PATHS_tempo='docs/sources/tempo'
+
+# identifier STR
+# Replace characters that are not valid in an identifier with underscores.
+identifier() {
+  echo "$1" | tr -C '[:alnum:]_\n' '_'
+}
+
+# aget ARRAY KEY
+# Get the value of KEY from associative array ARRAY.
+# Characters that are not valid in an identifier are replaced with underscores.
+aget() {
+  eval echo '$'"$(identifier "$1")_$(identifier "$2")"
+}
+
+# new_proj populates a new project structure.
+new_proj() {
+  _project="$1"
+  _version="$2"
+  _repo="$3"
+  _path="$4"
+
+  # If version is not set, use the script mapping of project to default versions if it exists.
+  # Fallback to 'latest'.
+  if [ -z "${_version}" ]; then
+    if [ -z "$(aget VERSIONS "${_project}")" ]; then
+      _version=latest
+    else
+      _version="$(aget VERSIONS "${_project}")"
+    fi
+  fi
+
+  # If repo is not set, use the script mapping of project to repo name if it exists.
+  # Fallback to using the project name.
+  if [ -z "${_repo}" ]; then
+    if [ -z "$(aget SOURCES "${_project}")" ]; then
+      _repo="${_project}"
+    else
+      _repo="$(aget SOURCES "${_project}")"
+    fi
+  fi
+
+  # If path is not set, use the script mapping of project to docs sources path if it exists.
+  # Fallback to using 'docs/sources'.
+  if [ -z "${_path}" ]; then
+    if [ -z "$(aget PATHS "${_project}")" ]; then
+      _path="docs/sources"
+    else
+      _path="$(aget PATHS "${_project}")"
+    fi
+  fi
+
+  echo "${_project}:${_version}:${_repo}:${_path}"
+  unset _project _version _repo _path
+}
+
+# proj_url returns the webserver URL for a project.
+# It expects a complete project structure as input.
+proj_url() {
+  IFS=: read -r _project _version _ _ <<POSIX_HERESTRING
+$1
+POSIX_HERESTRING
+
+  if [ -z "${_version}" ] || [ "${_version}" = 'UNVERSIONED' ]; then
+    echo "http://localhost:${DOCS_HOST_PORT}/docs/${_project}/"
+  else
+    echo "http://localhost:${DOCS_HOST_PORT}/docs/${_project}/${_version}/"
+  fi
+
+  unset _project _version
+}
+
+# proj_ver returns the version for a project.
+# It expects a complete project structure as input.
+proj_ver() {
+  IFS=: read -r _ _ver _ _ <<POSIX_HERESTRING
+$1
+POSIX_HERESTRING
+
+  echo "${_ver}"
+  unset _ver
+}
+
+# proj_dst returns the container path to content source for a project.
+# It expects a complete project structure as input.
+proj_dst() {
+  IFS=: read -r _project _version _ _ <<POSIX_HERESTRING
+$1
+POSIX_HERESTRING
+
+  if [ -z "${_version}" ] || [ "${_version}" = 'UNVERSIONED' ]; then
+    echo "/hugo/content/docs/${_project}"
+  else
+    echo "/hugo/content/docs/${_project}/${_version}"
+  fi
+  unset _project _version
+}
+
+# proj_src returns the host path to content source for a project.
+# It expects a complete project structure as input.
+# It looks for the provided repository name in each of the paths specified in the REPOS_PATH environment variable.
+proj_src() {
+  IFS=: read -r _ _ _repo _path <<POSIX_HERESTRING
+$1
+POSIX_HERESTRING
+
+  IFS=:
+  for lookup in ${REPOS_PATH}; do
+    if [ -d "${lookup}/${_repo}" ]; then
+      echo "${lookup}/${_repo}/${_path}"
+      unset _path _repo
+      return
+    fi
+  done
+  unset IFS
+
+  echo "ERRR: could not find project '${_repo}' in any of the paths in REPOS_PATH '${REPOS_PATH}'." >&2
+  echo "NOTE: you must have a checkout of the project '${_repo}' at '${REPOS_PATH##:*}/${_repo}'." >&2
+  echo "NOTE: if you have cloned the repository into a directory with a different name, consider changing it to ${_repo}." >&2
+  unset _path _repo
+  exit 1
+}
+
+# proj_canonical returns the canonical absolute path partial URI for a project.
+# It expects a complete project structure as input.
+proj_canonical() {
+  IFS=: read -r _project _version _ _ <<POSIX_HERESTRING
+$1
+POSIX_HERESTRING
+
+  if [ -z "${_version}" ] || [ "${_version}" = 'UNVERSIONED' ]; then
+    echo "/docs/${_project}"
+  else
+    echo "/docs/${_project}/${_version}"
+  fi
+  unset _project _version
+}
+
+proj_to_url_src_dst_ver() {
+  _url="$(proj_url "$1")"
+  _src="$(proj_src "$1")"
+  _dst="$(proj_dst "$1")"
+  _ver="$(proj_ver "$1")"
+
+  echo "${_url}^${_src}^${_dst}^${_ver}"
+  unset _url _src _dst _ver
+}
+
+url_src_dst_vers() {
+  for arg in "$@"; do
+    IFS=: read -r _project _version _repo _path <<POSIX_HERESTRING
+$arg
+POSIX_HERESTRING
+
+    case "${_project}" in
+      # Workaround for arbitrary mounts where the version field is expected to be the local directory
+      # and the repo field is expected to be the container directory.
+      arbitrary)
+        echo "${_project}^${_version}^${_repo}^" # TODO
+        ;;
+      logs)
+        proj_to_url_src_dst_ver "$(new_proj loki "${_version}")"
+        proj_to_url_src_dst_ver "$(new_proj enterprise-logs "${_version}")"
+        ;;
+      metrics)
+        proj_to_url_src_dst_ver "$(new_proj mimir "${_version}")"
+        proj_to_url_src_dst_ver "$(new_proj helm-charts/mimir-distributed "${_version}")"
+        proj_to_url_src_dst_ver "$(new_proj enterprise-metrics "${_version}")"
+        ;;
+      traces)
+        proj_to_url_src_dst_ver "$(new_proj tempo "${_version}")"
+        proj_to_url_src_dst_ver "$(new_proj enterprise-traces "${_version}")"
+        ;;
+      *)
+        proj_to_url_src_dst_ver "$(new_proj "${_project}" "${_version}" "${_repo}" "${_path}")"
+        ;;
+    esac
+  done
+
+  unset _project _version _repo _path
+}
+
+url_src_dst_vers="$(url_src_dst_vers "$@")"
+
+volumes=""
+redirects=""
+
+for x in ${url_src_dst_vers}; do
+  IFS='^' read -r _url _src _dst _ver <<POSIX_HERESTRING
+$x
+POSIX_HERESTRING
+
+  if [ "${_url}" != "arbitrary" ]; then
+    if [ ! -f "${_src}/_index.md" ]; then
+      echo "ERRR: Index file '${_src}/_index.md' does not exist." >&2
+      echo "Is '${_src}' the correct source directory?" >&2
+      exit 1
+    fi
+  fi
+
+  echo "DEBG: Mounting '${_src}' at container path '${_dst}'" >&2
+  if [ -z "${volumes}" ]; then
+    volumes="--volume=${_src}:${_dst}"
+  else
+    volumes="${volumes} --volume=${_src}:${_dst}"
+  fi
+
+  if [ -n "${_ver}" ] && [ "${_ver}" != 'UNVERSIONED' ]; then
+    if [ -z "${redirects}" ]; then
+      redirects="${_dst}^${_ver}"
+    else
+      redirects="${redirects} ${_dst}^${_ver}"
+    fi
+  fi
+  unset _url _src _dst _ver
+done
+
+IFS=':' read -r image _ <<POSIX_HERESTRING
+${DOCS_IMAGE}
+POSIX_HERESTRING
+
+if [ "${image}" = "grafana/doc-validator" ]; then
+  echo
+  "${PODMAN}" run \
+    --init \
+    --interactive \
+    --name "${DOCS_CONTAINER}" \
+    --platform linux/amd64 \
+    --rm \
+    --tty \
+    ${volumes} \
+    "${DOCS_IMAGE}" \
+    --skip-image-validation \
+    --include="${DOC_VALIDATOR_INCLUDE}" \
+    /hugo/content/docs \
+    "$(proj_canonical "$(new_proj "$1")")"
+else
+
+  cat <<EOF >/tmp/make-docs-entrypoint
+#!/usr/bin/env bash
+for redirect in ${redirects}; do
+  IFS='^' read -r path ver <<<"\${redirect}"
+  echo -e "---\\nredirectURL: \"\${path/\/hugo\/content/}\"\\ntype: redirect\\n---\\n" > "\${path/\${ver}/_index.md}"
+
+  if [[ -n "${WEBSITE_MOUNTS}" ]]; then
+    unset WEBSITE_SKIP_MOUNTS
+  fi
+done
+
+${WEBSITE_EXEC}
+EOF
+  chmod +x /tmp/make-docs-entrypoint
+  volumes="${volumes} --volume=/tmp/make-docs-entrypoint:/entrypoint"
+  readonly volumes
+
+  echo
+  echo "Documentation will be served at the following URLs:"
+  for x in ${url_src_dst_vers}; do
+    IFS='^' read -r url _ _ <<POSIX_HERESTRING
+$x
+POSIX_HERESTRING
+
+    if [ -n "${url}" ]; then
+      echo "  ${url}"
+    fi
+  done
+
+  echo
+  "${PODMAN}" run \
+    --env "HUGO_REFLINKSERRORLEVEL=${HUGO_REFLINKSERRORLEVEL}" \
+    --init \
+    --interactive \
+    --name "${DOCS_CONTAINER}" \
+    --platform linux/amd64 \
+    --publish "${DOCS_HOST_PORT}:3002" \
+    --publish "3003:3003" \
+    --rm \
+    --tty \
+    ${volumes} \
+    "${DOCS_IMAGE}" \
+    /entrypoint
+fi