IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

configure Loki for local development (#26997)

Browse Source 
* configure Loki for local development

Extend our docker compose setup to include Loki (along with promtail), scraping `logs/*.log` to ingest development logs as well as all logs from running containers.

While we're in here, teach Prometheus to scrape metrics from Docker containers too.

* tweak promtail labels

* document extra services in config.mk

* update home.json
This commit is contained in:
Jesse Hallam
2024-05-16 12:16:08 -03:00
committed by GitHub
parent 65325a767c
commit 617053e206
10 changed files with 473 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/build/docker-compose-generator/main.go
View File

@@ -35,6 +35,8 @@ func main() {
"keycloak": 8080,
"prometheus": 9090,
"grafana": 3000,
"loki": 3100,
"promtail": 3180,
"mysql-read-replica": 3306, // FIXME: not recognizing the successfully running service on port 3307.
}
command := []string{}

24
server/build/docker-compose.common.yml
View File

@@ -119,15 +119,31 @@ services:
- "./docker/keycloak:/opt/keycloak/data/import"
prometheus:
image: "prom/prometheus:v2.46.0"
user: root
volumes:
- "./docker/prometheus${IS_LINUX}.yml:/etc/prometheus/prometheus.yml"
- "/var/run/docker.sock:/var/run/docker.sock"
networks:
- mm-test
grafana:
image: "grafana/grafana:10.0.3"
image: "grafana/grafana:10.4.2"
volumes:
- "./docker/grafana/grafana.ini:/etc/grafana/grafana.ini"
- "./docker/grafana/provisioning:/etc/grafana/provisioning"
- "./docker/grafana/dashboards:/var/lib/grafana/dashboards"
- "./docker/grafana/grafana.ini:/etc/grafana/grafana.ini"
- "./docker/grafana/provisioning:/etc/grafana/provisioning"
- "./docker/grafana/dashboards:/var/lib/grafana/dashboards"
networks:
- mm-test
loki:
image: "grafana/loki:3.0.0"
networks:
- mm-test
promtail:
image: "grafana/promtail:3.0.0"
volumes:
- "./docker/promtail/promtail-local-config.yaml:/etc/promtail/docker-config.yaml"
- "/var/lib/docker/containers:/var/lib/docker/containers:ro"
- "/var/run/docker.sock:/var/run/docker.sock"
- "../logs:/logs"
command: -config.file=/etc/promtail/docker-config.yaml
networks:
- mm-test

8
server/build/docker-compose.yml
View File

@@ -44,6 +44,14 @@ services:
extends:
file: docker-compose.common.yml
service: grafana
loki:
extends:
file: docker-compose.common.yml
service: loki
promtail:
extends:
file: docker-compose.common.yml
service: promtail
start_dependencies:
image: mattermost/mattermost-wait-for-dep:latest

378
server/build/docker/grafana/dashboards/home.json
View File

@@ -3,7 +3,10 @@
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
@@ -13,18 +16,15 @@
]
},
"editable": true,
"gnetId": null,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 5,
"links": [],
"panels": [
{
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"gridPos": {
"h": 3,
@@ -33,60 +33,369 @@
"y": 0
},
"id": 1,
"title": "",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"refId": "A"
}
],
"type": "welcome"
},
{
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"gridPos": {
"h": 15,
"h": 7,
"w": 12,
"x": 0,
"y": 3
},
"headings": true,
"id": 3,
"limit": 30,
"links": [],
"query": "Mattermost",
"recent": false,
"search": true,
"starred": false,
"options": {
"includeVars": false,
"keepTime": false,
"maxItems": 30,
"query": "Mattermost",
"showHeadings": true,
"showRecentlyViewed": false,
"showSearch": true,
"showStarred": false,
"tags": []
},
"pluginVersion": "10.4.2",
"tags": [],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"refId": "A"
}
],
"title": "Dashboards",
"type": "dashlist"
},
{
"datasource": null,
"datasource": {
"type": "datasource",
"uid": "-- Mixed --"
},
"fieldConfig": {
"defaults": {
"custom": {}
"color": {
"mode": "thresholds"
},
"mappings": [
{
"options": {
"0": {
"color": "red",
"index": 0,
"text": "Offline"
}
},
"type": "value"
},
{
"options": {
"from": 0,
"result": {
"color": "green",
"index": 1,
"text": "Online"
},
"to": 99999999999999
},
"type": "range"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 15,
"h": 12,
"w": 12,
"x": 12,
"y": 3
},
"id": 4,
"links": [],
"id": 7,
"options": {
"feedUrl": "https://grafana.com/blog/news.xml"
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "/^(?!Time)/",
"values": false
},
"showPercentChange": false,
"textMode": "value_and_name",
"wideLayout": true
},
"title": "Latest from the blog",
"type": "news"
"pluginVersion": "10.4.2",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-postgres\"}[5m])) or vector(0)",
"legendFormat": "Postgres",
"queryType": "range",
"refId": "A"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-mysql\"}[5m])) or vector(0)",
"legendFormat": "MySQL",
"queryType": "range",
"refId": "B"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-inbucket\"}[5m])) or vector(0)",
"legendFormat": "Inbucket",
"queryType": "range",
"refId": "C"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-minio\"}[5m])) or vector(0)",
"legendFormat": "MinIO",
"queryType": "range",
"refId": "D"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-openldap\"}[5m])) or vector(0)",
"legendFormat": "OpenLDAP",
"queryType": "range",
"refId": "E"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-dejavu\"}[5m])) or vector(0)",
"legendFormat": "DejaVu",
"queryType": "range",
"refId": "F"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-keycloak\"}[5m])) or vector(0)",
"legendFormat": "Keycloak",
"queryType": "range",
"refId": "G"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-elasticsearch\"}[5m])) or vector(0)",
"legendFormat": "Elasticsearch",
"queryType": "range",
"refId": "H"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(up{container=\"mattermost-prometheus\"}) or vector(0)",
"hide": false,
"legendFormat": "Prometheus",
"range": true,
"refId": "I"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(up{container=\"mattermost-grafana\"}) or vector(0)",
"hide": false,
"legendFormat": "Grafana",
"range": true,
"refId": "J"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(up{container=\"mattermost-loki\"}) or vector(0)",
"hide": false,
"legendFormat": "Loki",
"range": true,
"refId": "K"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(up{container=\"mattermost-promtail\"}) or vector(0)",
"hide": false,
"legendFormat": "Promtail",
"range": true,
"refId": "L"
}
],
"title": "Panel Title",
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"description": "Links to common queries.",
"gridPos": {
"h": 5,
"w": 12,
"x": 0,
"y": 10
},
"id": 6,
"options": {
"code": {
"language": "plaintext",
"showLineNumbers": false,
"showMiniMap": false
},
"content": "* [Mattermost Error Logs](/explore?schemaVersion=1&panes=%7B%22z8g%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22mattermost%5C%22%7D%20%7C%20level%3D%5C%22error%5C%22%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1)\n* [Plugin Logs](/explore?schemaVersion=1&panes=%7B%22z8g%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22mattermost%5C%22%7D%20%7C%20json%20%7C%20plugin_id%20%21%3D%20%5C%22%5C%22%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1)",
"mode": "markdown"
},
"pluginVersion": "10.4.2",
"title": "Links",
"type": "text"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"description": "Logs from the localhost Mattermost instance.",
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 15
},
"id": 4,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "code",
"expr": "{app=\"mattermost\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "Mattermost Logs",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"description": "Logs from the Docker services running to support Mattermost.",
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 25
},
"id": 5,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "code",
"expr": "{job=\"docker\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "Docker Logs",
"type": "logs"
}
],
"schemaVersion": 26,
"style": "dark",
"schemaVersion": 39,
"tags": [],
"templating": {
"list": []
@@ -125,5 +434,6 @@
"timezone": "browser",
"title": "Home",
"uid": "4yRNKfaGz",
"version": 1
"version": 1,
"weekStart": ""
}

11
server/build/docker/grafana/provisioning/datasources/loki.yml Normal file
View File

@@ -0,0 +1,11 @@
apiVersion: 1
datasources:
- name: Loki
type: 'loki'
access: 'proxy'
orgId: 1
url: 'http://loki:3100'
isDefault: false
version: 1
editable: false

10
server/build/docker/prometheus.yml
View File

@@ -6,3 +6,13 @@ scrape_configs:
- job_name: 'mattermost'
static_configs:
- targets: ['host.docker.internal:8067']
- job_name: fmetrics_scrape
docker_sd_configs:
- host: unix:///var/run/docker.sock
refresh_interval: 5s
relabel_configs:
- source_labels: ['__meta_docker_container_name']
regex: '/(.*)'
target_label: 'container'
- source_labels: ['__meta_docker_container_log_stream']
target_label: 'logstream'

46
server/build/docker/promtail/promtail-local-config.yaml Normal file
View File

@@ -0,0 +1,46 @@
server:
http_listen_port: 3180
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: http://loki:3100/loki/api/v1/push
scrape_configs:
- job_name: mattermost
pipeline_stages:
- match:
selector: '{job="mattermost"}'
stages:
- json:
expressions:
timestamp: timestamp
level: level
- labels:
level:
- timestamp:
format: '2006-01-02 15:04:05.999 -07:00'
source: timestamp
static_configs:
- targets:
- localhost
labels:
job: mattermost
app: mattermost
__path__: /logs/*.log
# https://ruanbekker.medium.com/logging-with-docker-promtail-and-grafana-loki-d920fd790ca8
- job_name: docker
docker_sd_configs:
- host: unix:///var/run/docker.sock
refresh_interval: 5s
relabel_configs:
- source_labels: ['__meta_docker_container_name']
regex: '/(.*)'
target_label: 'container'
- source_labels: ['__meta_docker_container_log_stream']
target_label: 'logstream'
pipeline_stages:
- static_labels:
job: docker

2
server/config.mk
View File

@@ -5,7 +5,7 @@
# Enable services to be run in docker.
#
# Possible options: mysql, postgres, minio, inbucket, openldap, dejavu,
# keycloak, elasticsearch, prometheus, and grafana.
# keycloak, elasticsearch, prometheus, grafana, loki and promtail.
#
# Must be space separated names.
#

14
server/docker-compose.makefile.yml
View File

@@ -92,6 +92,20 @@ services:
extends:
file: build/docker-compose.common.yml
service: grafana
loki:
container_name: mattermost-loki
ports:
- "3100:3100"
extends:
file: build/docker-compose.common.yml
service: loki
promtail:
container_name: mattermost-promtail
ports:
- "3180:3180"
extends:
file: build/docker-compose.common.yml
service: promtail
networks:
mm-test:

18
server/docker-compose.yaml
View File

@@ -81,6 +81,20 @@ services:
extends:
file: build/docker-compose.common.yml
service: grafana
loki:
container_name: mattermost-loki
ports:
- "3100:3100"
extends:
file: build/docker-compose.common.yml
service: loki
promtail:
container_name: mattermost-promtail
ports:
- "3180:3180"
extends:
file: build/docker-compose.common.yml
service: promtail
start_dependencies:
image: mattermost/mattermost-wait-for-dep:latest
@@ -95,7 +109,9 @@ services:
- elasticsearch
- prometheus
- grafana
command: postgres:5432 mysql:3306 minio:9000 inbucket:9001 openldap:389 elasticsearch:9200 prometheus:9090 grafana:3000
- loki
- promtail
command: postgres:5432 mysql:3306 minio:9000 inbucket:9001 openldap:389 elasticsearch:9200 prometheus:9090 grafana:3000 loki:3100 promtail:3180
leader:
build: