From 617053e2063df0b171209cfecf98db536d6a26b1 Mon Sep 17 00:00:00 2001 From: Jesse Hallam Date: Thu, 16 May 2024 12:16:08 -0300 Subject: [PATCH] configure Loki for local development (#26997) * configure Loki for local development Extend our docker compose setup to include Loki (along with promtail), scraping `logs/*.log` to ingest development logs as well as all logs from running containers. While we're in here, teach Prometheus to scrape metrics from Docker containers too. * tweak promtail labels * document extra services in config.mk * update home.json --- server/build/docker-compose-generator/main.go | 2 + server/build/docker-compose.common.yml | 24 +- server/build/docker-compose.yml | 8 + .../build/docker/grafana/dashboards/home.json | 378 ++++++++++++++++-- .../grafana/provisioning/datasources/loki.yml | 11 + server/build/docker/prometheus.yml | 10 + .../promtail/promtail-local-config.yaml | 46 +++ server/config.mk | 2 +- server/docker-compose.makefile.yml | 14 + server/docker-compose.yaml | 18 +- 10 files changed, 473 insertions(+), 40 deletions(-) create mode 100644 server/build/docker/grafana/provisioning/datasources/loki.yml create mode 100644 server/build/docker/promtail/promtail-local-config.yaml diff --git a/server/build/docker-compose-generator/main.go b/server/build/docker-compose-generator/main.go index 6fba5d43ee..7d46474bcc 100644 --- a/server/build/docker-compose-generator/main.go +++ b/server/build/docker-compose-generator/main.go @@ -35,6 +35,8 @@ func main() { "keycloak": 8080, "prometheus": 9090, "grafana": 3000, + "loki": 3100, + "promtail": 3180, "mysql-read-replica": 3306, // FIXME: not recognizing the successfully running service on port 3307. } command := []string{} diff --git a/server/build/docker-compose.common.yml b/server/build/docker-compose.common.yml index 1f39e852b8..0c55da9bb8 100644 --- a/server/build/docker-compose.common.yml +++ b/server/build/docker-compose.common.yml @@ -119,15 +119,31 @@ services: - "./docker/keycloak:/opt/keycloak/data/import" prometheus: image: "prom/prometheus:v2.46.0" + user: root volumes: - "./docker/prometheus${IS_LINUX}.yml:/etc/prometheus/prometheus.yml" + - "/var/run/docker.sock:/var/run/docker.sock" networks: - mm-test grafana: - image: "grafana/grafana:10.0.3" + image: "grafana/grafana:10.4.2" volumes: - - "./docker/grafana/grafana.ini:/etc/grafana/grafana.ini" - - "./docker/grafana/provisioning:/etc/grafana/provisioning" - - "./docker/grafana/dashboards:/var/lib/grafana/dashboards" + - "./docker/grafana/grafana.ini:/etc/grafana/grafana.ini" + - "./docker/grafana/provisioning:/etc/grafana/provisioning" + - "./docker/grafana/dashboards:/var/lib/grafana/dashboards" + networks: + - mm-test + loki: + image: "grafana/loki:3.0.0" + networks: + - mm-test + promtail: + image: "grafana/promtail:3.0.0" + volumes: + - "./docker/promtail/promtail-local-config.yaml:/etc/promtail/docker-config.yaml" + - "/var/lib/docker/containers:/var/lib/docker/containers:ro" + - "/var/run/docker.sock:/var/run/docker.sock" + - "../logs:/logs" + command: -config.file=/etc/promtail/docker-config.yaml networks: - mm-test diff --git a/server/build/docker-compose.yml b/server/build/docker-compose.yml index aaecb23faa..5b2aa38c5d 100644 --- a/server/build/docker-compose.yml +++ b/server/build/docker-compose.yml @@ -44,6 +44,14 @@ services: extends: file: docker-compose.common.yml service: grafana + loki: + extends: + file: docker-compose.common.yml + service: loki + promtail: + extends: + file: docker-compose.common.yml + service: promtail start_dependencies: image: mattermost/mattermost-wait-for-dep:latest diff --git a/server/build/docker/grafana/dashboards/home.json b/server/build/docker/grafana/dashboards/home.json index 70e636612e..a84987f122 100644 --- a/server/build/docker/grafana/dashboards/home.json +++ b/server/build/docker/grafana/dashboards/home.json @@ -3,7 +3,10 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -13,18 +16,15 @@ ] }, "editable": true, - "gnetId": null, + "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 5, "links": [], "panels": [ { - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, "gridPos": { "h": 3, @@ -33,60 +33,369 @@ "y": 0 }, "id": 1, - "title": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], "type": "welcome" }, { - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, "gridPos": { - "h": 15, + "h": 7, "w": 12, "x": 0, "y": 3 }, - "headings": true, "id": 3, - "limit": 30, - "links": [], - "query": "Mattermost", - "recent": false, - "search": true, - "starred": false, + "options": { + "includeVars": false, + "keepTime": false, + "maxItems": 30, + "query": "Mattermost", + "showHeadings": true, + "showRecentlyViewed": false, + "showSearch": true, + "showStarred": false, + "tags": [] + }, + "pluginVersion": "10.4.2", "tags": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], "title": "Dashboards", "type": "dashlist" }, { - "datasource": null, + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, "fieldConfig": { "defaults": { - "custom": {} + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "Offline" + } + }, + "type": "value" + }, + { + "options": { + "from": 0, + "result": { + "color": "green", + "index": 1, + "text": "Online" + }, + "to": 99999999999999 + }, + "type": "range" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "gridPos": { - "h": 15, + "h": 12, "w": 12, "x": 12, "y": 3 }, - "id": 4, - "links": [], + "id": 7, "options": { - "feedUrl": "https://grafana.com/blog/news.xml" + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time)/", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true }, - "title": "Latest from the blog", - "type": "news" + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum(count_over_time({container=\"mattermost-postgres\"}[5m])) or vector(0)", + "legendFormat": "Postgres", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum(count_over_time({container=\"mattermost-mysql\"}[5m])) or vector(0)", + "legendFormat": "MySQL", + "queryType": "range", + "refId": "B" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum(count_over_time({container=\"mattermost-inbucket\"}[5m])) or vector(0)", + "legendFormat": "Inbucket", + "queryType": "range", + "refId": "C" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum(count_over_time({container=\"mattermost-minio\"}[5m])) or vector(0)", + "legendFormat": "MinIO", + "queryType": "range", + "refId": "D" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum(count_over_time({container=\"mattermost-openldap\"}[5m])) or vector(0)", + "legendFormat": "OpenLDAP", + "queryType": "range", + "refId": "E" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum(count_over_time({container=\"mattermost-dejavu\"}[5m])) or vector(0)", + "legendFormat": "DejaVu", + "queryType": "range", + "refId": "F" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum(count_over_time({container=\"mattermost-keycloak\"}[5m])) or vector(0)", + "legendFormat": "Keycloak", + "queryType": "range", + "refId": "G" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum(count_over_time({container=\"mattermost-elasticsearch\"}[5m])) or vector(0)", + "legendFormat": "Elasticsearch", + "queryType": "range", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(up{container=\"mattermost-prometheus\"}) or vector(0)", + "hide": false, + "legendFormat": "Prometheus", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(up{container=\"mattermost-grafana\"}) or vector(0)", + "hide": false, + "legendFormat": "Grafana", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(up{container=\"mattermost-loki\"}) or vector(0)", + "hide": false, + "legendFormat": "Loki", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(up{container=\"mattermost-promtail\"}) or vector(0)", + "hide": false, + "legendFormat": "Promtail", + "range": true, + "refId": "L" + } + ], + "title": "Panel Title", + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "Links to common queries.", + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 6, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "* [Mattermost Error Logs](/explore?schemaVersion=1&panes=%7B%22z8g%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22mattermost%5C%22%7D%20%7C%20level%3D%5C%22error%5C%22%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1)\n* [Plugin Logs](/explore?schemaVersion=1&panes=%7B%22z8g%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22mattermost%5C%22%7D%20%7C%20json%20%7C%20plugin_id%20%21%3D%20%5C%22%5C%22%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1)", + "mode": "markdown" + }, + "pluginVersion": "10.4.2", + "title": "Links", + "type": "text" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "Logs from the localhost Mattermost instance.", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 4, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{app=\"mattermost\"}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Mattermost Logs", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "Logs from the Docker services running to support Mattermost.", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 5, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{job=\"docker\"}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Docker Logs", + "type": "logs" } ], - "schemaVersion": 26, - "style": "dark", + "schemaVersion": 39, "tags": [], "templating": { "list": [] @@ -125,5 +434,6 @@ "timezone": "browser", "title": "Home", "uid": "4yRNKfaGz", - "version": 1 + "version": 1, + "weekStart": "" } diff --git a/server/build/docker/grafana/provisioning/datasources/loki.yml b/server/build/docker/grafana/provisioning/datasources/loki.yml new file mode 100644 index 0000000000..d5b78ff30b --- /dev/null +++ b/server/build/docker/grafana/provisioning/datasources/loki.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: Loki + type: 'loki' + access: 'proxy' + orgId: 1 + url: 'http://loki:3100' + isDefault: false + version: 1 + editable: false diff --git a/server/build/docker/prometheus.yml b/server/build/docker/prometheus.yml index 58f8858c16..c78a465159 100644 --- a/server/build/docker/prometheus.yml +++ b/server/build/docker/prometheus.yml @@ -6,3 +6,13 @@ scrape_configs: - job_name: 'mattermost' static_configs: - targets: ['host.docker.internal:8067'] + - job_name: fmetrics_scrape + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container' + - source_labels: ['__meta_docker_container_log_stream'] + target_label: 'logstream' diff --git a/server/build/docker/promtail/promtail-local-config.yaml b/server/build/docker/promtail/promtail-local-config.yaml new file mode 100644 index 0000000000..998aa42424 --- /dev/null +++ b/server/build/docker/promtail/promtail-local-config.yaml @@ -0,0 +1,46 @@ +server: + http_listen_port: 3180 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: mattermost + pipeline_stages: + - match: + selector: '{job="mattermost"}' + stages: + - json: + expressions: + timestamp: timestamp + level: level + - labels: + level: + - timestamp: + format: '2006-01-02 15:04:05.999 -07:00' + source: timestamp + static_configs: + - targets: + - localhost + labels: + job: mattermost + app: mattermost + __path__: /logs/*.log + # https://ruanbekker.medium.com/logging-with-docker-promtail-and-grafana-loki-d920fd790ca8 + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container' + - source_labels: ['__meta_docker_container_log_stream'] + target_label: 'logstream' + pipeline_stages: + - static_labels: + job: docker diff --git a/server/config.mk b/server/config.mk index d22efdd2cf..121ce6fe5f 100644 --- a/server/config.mk +++ b/server/config.mk @@ -5,7 +5,7 @@ # Enable services to be run in docker. # # Possible options: mysql, postgres, minio, inbucket, openldap, dejavu, -# keycloak, elasticsearch, prometheus, and grafana. +# keycloak, elasticsearch, prometheus, grafana, loki and promtail. # # Must be space separated names. # diff --git a/server/docker-compose.makefile.yml b/server/docker-compose.makefile.yml index eb10dc138e..287dfd57bf 100644 --- a/server/docker-compose.makefile.yml +++ b/server/docker-compose.makefile.yml @@ -92,6 +92,20 @@ services: extends: file: build/docker-compose.common.yml service: grafana + loki: + container_name: mattermost-loki + ports: + - "3100:3100" + extends: + file: build/docker-compose.common.yml + service: loki + promtail: + container_name: mattermost-promtail + ports: + - "3180:3180" + extends: + file: build/docker-compose.common.yml + service: promtail networks: mm-test: diff --git a/server/docker-compose.yaml b/server/docker-compose.yaml index 7c7c5269af..04260b3256 100644 --- a/server/docker-compose.yaml +++ b/server/docker-compose.yaml @@ -81,6 +81,20 @@ services: extends: file: build/docker-compose.common.yml service: grafana + loki: + container_name: mattermost-loki + ports: + - "3100:3100" + extends: + file: build/docker-compose.common.yml + service: loki + promtail: + container_name: mattermost-promtail + ports: + - "3180:3180" + extends: + file: build/docker-compose.common.yml + service: promtail start_dependencies: image: mattermost/mattermost-wait-for-dep:latest @@ -95,7 +109,9 @@ services: - elasticsearch - prometheus - grafana - command: postgres:5432 mysql:3306 minio:9000 inbucket:9001 openldap:389 elasticsearch:9200 prometheus:9090 grafana:3000 + - loki + - promtail + command: postgres:5432 mysql:3306 minio:9000 inbucket:9001 openldap:389 elasticsearch:9200 prometheus:9090 grafana:3000 loki:3100 promtail:3180 leader: build: