IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Update the grafana-mixin to use HTTP histograms (#39155)

Browse Source 
Signed-off-by: bergquist <carl.bergquist@gmail.com>
This commit is contained in:
Carl Bergquist 2021-10-06 14:15:12 +02:00 committed by GitHub
parent 458371c8eb
commit 075256923b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 62 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
grafana-mixin/alerts/alerts.yaml
View File

@ -4,9 +4,9 @@ groups:
- alert: GrafanaRequestsFailing - alert: GrafanaRequestsFailing
for: 5m for: 5m
expr: | expr: |
100 * namespace_job_handler_statuscode:http_request_total:rate5m{handler!~"/datasources/proxy/:id.*|/ds/query|/tsdb/query", statuscode=~"5.."} 100 * namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m{handler!~"/api/datasources/proxy/:id.*|/api/ds/query|/api/tsdb/query", status_code=~"5.."}
/ /
namespace_job_handler_statuscode:http_request_total:rate5m{handler!~"/datasources/proxy/:id.*|/ds/query|/tsdb/query"} namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m{handler!~"/api/datasources/proxy/:id.*|/api/ds/query|/api/tsdb/query"}
> 0.5 > 0.5
labels: labels:
severity: 'warning' severity: 'warning'

93
grafana-mixin/dashboards/grafana-overview.json
View File

@ -8,6 +8,12 @@
"hide": true, "hide": true,
"iconColor": "rgba(0, 211, 255, 1)", "iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts", "name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard" "type": "dashboard"
} }
] ]
@ -15,15 +21,14 @@
"editable": true, "editable": true,
"gnetId": null, "gnetId": null,
"graphTooltip": 0, "graphTooltip": 0,
"id": 35, "id": 3085,
"iteration": 1602761142538, "iteration": 1631554945276,
"links": [], "links": [],
"panels": [ "panels": [
{ {
"datasource": "$datasource", "datasource": "$datasource",
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
"custom": {},
"mappings": [], "mappings": [],
"noValue": "0", "noValue": "0",
"thresholds": { "thresholds": {
@ -58,9 +63,11 @@
"calcs": ["mean"], "calcs": ["mean"],
"fields": "", "fields": "",
"values": false "values": false
} },
"text": {},
"textMode": "auto"
}, },
"pluginVersion": "7.0.4", "pluginVersion": "8.1.3",
"targets": [ "targets": [
{ {
"expr": "grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}", "expr": "grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}",
@ -79,7 +86,6 @@
"datasource": "$datasource", "datasource": "$datasource",
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
"custom": {},
"mappings": [], "mappings": [],
"thresholds": { "thresholds": {
"mode": "absolute", "mode": "absolute",
@ -113,9 +119,11 @@
"calcs": ["mean"], "calcs": ["mean"],
"fields": "", "fields": "",
"values": false "values": false
} },
"text": {},
"textMode": "auto"
}, },
"pluginVersion": "7.0.4", "pluginVersion": "8.1.3",
"targets": [ "targets": [
{ {
"expr": "sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})", "expr": "sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})",
@ -134,7 +142,8 @@
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
"custom": { "custom": {
"align": null "align": null,
"displayMode": "auto"
}, },
"mappings": [], "mappings": [],
"thresholds": { "thresholds": {
@ -163,7 +172,7 @@
"options": { "options": {
"showHeader": true "showHeader": true
}, },
"pluginVersion": "7.0.4", "pluginVersion": "8.1.3",
"targets": [ "targets": [
{ {
"expr": "grafana_build_info{job=~\"$job\", instance=~\"$instance\"}", "expr": "grafana_build_info{job=~\"$job\", instance=~\"$instance\"}",
@ -222,7 +231,7 @@
"datasource": "$datasource", "datasource": "$datasource",
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
"custom": {} "links": []
}, },
"overrides": [] "overrides": []
}, },
@ -249,9 +258,10 @@
"linewidth": 1, "linewidth": 1,
"nullPointMode": "null", "nullPointMode": "null",
"options": { "options": {
"dataLinks": [] "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "8.1.3",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
@ -261,9 +271,9 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum by (statuscode) (irate(http_request_total{job=~\"$job\", instance=~\"$instance\"}[1m])) ", "expr": "sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m])) ",
"interval": "", "interval": "",
"legendFormat": "{{statuscode}}", "legendFormat": "{{status_code}}",
"refId": "A" "refId": "A"
} }
], ],
@ -318,7 +328,7 @@
"datasource": "$datasource", "datasource": "$datasource",
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
"custom": {} "links": []
}, },
"overrides": [] "overrides": []
}, },
@ -345,9 +355,10 @@
"linewidth": 1, "linewidth": 1,
"nullPointMode": "null", "nullPointMode": "null",
"options": { "options": {
"dataLinks": [] "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "8.1.3",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
@ -357,21 +368,24 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "max(http_request_duration_milliseconds{job=~\"$job\", instance=~\"$instance\", quantile=\"0.99\"})", "exemplar": true,
"expr": "histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"interval": "", "interval": "",
"legendFormat": "max-99th", "legendFormat": "99th Percentile",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "max(http_request_duration_milliseconds{job=~\"$job\", instance=~\"$instance\", quantile=\"0.9\"})", "exemplar": true,
"expr": "histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"interval": "", "interval": "",
"legendFormat": "max-90th", "legendFormat": "50th Percentile",
"refId": "B" "refId": "B"
}, },
{ {
"expr": "sum(irate(http_request_duration_milliseconds_sum{job=~\"$job\", instance=~\"$instance\"}[$__interval])) / sum(irate(http_request_duration_milliseconds_count{job=~\"$job\", instance=~\"$instance\"}[$__interval])) ", "exemplar": true,
"expr": "sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))",
"interval": "", "interval": "",
"legendFormat": "avg", "legendFormat": "Average",
"refId": "C" "refId": "C"
} }
], ],
@ -419,17 +433,19 @@
} }
} }
], ],
"schemaVersion": 25, "schemaVersion": 30,
"style": "dark", "style": "dark",
"tags": [], "tags": [],
"templating": { "templating": {
"list": [ "list": [
{ {
"current": { "current": {
"selected": false, "selected": true,
"text": "prometheus", "text": "dev-cortex",
"value": "prometheus" "value": "dev-cortex"
}, },
"description": null,
"error": null,
"hide": 0, "hide": 0,
"includeAll": false, "includeAll": false,
"label": null, "label": null,
@ -446,26 +462,29 @@
{ {
"allValue": ".*", "allValue": ".*",
"current": { "current": {
"selected": true, "selected": false,
"tags": [], "text": ["default/grafana"],
"text": "All", "value": ["default/grafana"]
"value": ["$__all"]
}, },
"datasource": "$datasource", "datasource": "$datasource",
"definition": "label_values(grafana_build_info, job)", "definition": "label_values(grafana_build_info, job)",
"description": null,
"error": null,
"hide": 0, "hide": 0,
"includeAll": true, "includeAll": true,
"label": null, "label": null,
"multi": true, "multi": true,
"name": "job", "name": "job",
"options": [], "options": [],
"query": "label_values(grafana_build_info, job)", "query": {
"query": "label_values(grafana_build_info, job)",
"refId": "Billing Admin-job-Variable-Query"
},
"refresh": 1, "refresh": 1,
"regex": "", "regex": "",
"skipUrlSync": false, "skipUrlSync": false,
"sort": 0, "sort": 0,
"tagValuesQuery": "", "tagValuesQuery": "",
"tags": [],
"tagsQuery": "", "tagsQuery": "",
"type": "query", "type": "query",
"useTags": false "useTags": false
@ -479,19 +498,23 @@
}, },
"datasource": "$datasource", "datasource": "$datasource",
"definition": "label_values(grafana_build_info, instance)", "definition": "label_values(grafana_build_info, instance)",
"description": null,
"error": null,
"hide": 0, "hide": 0,
"includeAll": true, "includeAll": true,
"label": null, "label": null,
"multi": true, "multi": true,
"name": "instance", "name": "instance",
"options": [], "options": [],
"query": "label_values(grafana_build_info, instance)", "query": {
"query": "label_values(grafana_build_info, instance)",
"refId": "Billing Admin-instance-Variable-Query"
},
"refresh": 1, "refresh": 1,
"regex": "", "regex": "",
"skipUrlSync": false, "skipUrlSync": false,
"sort": 0, "sort": 0,
"tagValuesQuery": "", "tagValuesQuery": "",
"tags": [],
"tagsQuery": "", "tagsQuery": "",
"type": "query", "type": "query",
"useTags": false "useTags": false
@ -508,5 +531,5 @@
"timezone": "", "timezone": "",
"title": "Grafana Overview", "title": "Grafana Overview",
"uid": "6be0s85Mk", "uid": "6be0s85Mk",
"version": 4 "version": 2
} }

4
grafana-mixin/rules/rules.yaml
View File

@ -2,6 +2,6 @@ groups:
- name: grafana_rules - name: grafana_rules
rules: rules:
# Record error rate of http requests excluding dataproxy, /ds/query and /tsdb/query requests # Record error rate of http requests excluding dataproxy, /ds/query and /tsdb/query requests
- record: namespace_job_handler_statuscode:http_request_total:rate5m - record: namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m
expr: | expr: |
sum by (namespace, job, handler, statuscode) (rate(http_request_total[5m])) sum by (namespace, job, handler, status_code) (rate(grafana_http_request_duration_seconds_count[5m]))