mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
elasticsearch: add 7.x version support (#16646)
Adds a new version option 7.0+ (70 internally). Version 7.0+ doesn't include document types in index mappings so support for handling this have been added. Version 7.0+ returns number of hits in a different way so support for handling this have been added. Version 7.0+ doesn't support sending max_concurrent_shard_requests in multisearch header so support for sending this in query string have been added. Update elastic6 docker block and dashboards (devenv) to use 6.7.1 images, filebeat index name is now filebeat-YYYY.MM.DD and dashboard include correct tags and links. Add elastic7 docker block and provisioning (devenv). Updates documentation regarding new version. Closes #15622
This commit is contained in:
Alcides Viamontes E
Marcus Efraimsson
parent
42b745a098
commit
cff2be0d66
@ -99,13 +99,44 @@ datasources:
|
||||
- name: gdev-elasticsearch-v6-filebeat
|
||||
type: elasticsearch
|
||||
access: proxy
|
||||
database: "[filebeat-6.2.4-]YYYY.MM.DD"
|
||||
database: "[filebeat-]YYYY.MM.DD"
|
||||
url: http://localhost:11200
|
||||
jsonData:
|
||||
interval: Daily
|
||||
timeField: "@timestamp"
|
||||
esVersion: 60
|
||||
|
||||
- name: gdev-elasticsearch-v7-metrics
|
||||
type: elasticsearch
|
||||
access: proxy
|
||||
database: "[metrics-]YYYY.MM.DD"
|
||||
url: http://localhost:12200
|
||||
jsonData:
|
||||
timeInterval: 10s
|
||||
interval: Daily
|
||||
timeField: "@timestamp"
|
||||
esVersion: 70
|
||||
|
||||
- name: gdev-elasticsearch-v7-logs
|
||||
type: elasticsearch
|
||||
access: proxy
|
||||
database: "[logs-]YYYY.MM.DD"
|
||||
url: http://localhost:12200
|
||||
jsonData:
|
||||
interval: Daily
|
||||
timeField: "@timestamp"
|
||||
esVersion: 70
|
||||
|
||||
- name: gdev-elasticsearch-v7-filebeat
|
||||
type: elasticsearch
|
||||
access: proxy
|
||||
database: "[filebeat-]YYYY.MM.DD"
|
||||
url: http://localhost:12200
|
||||
jsonData:
|
||||
interval: Daily
|
||||
timeField: "@timestamp"
|
||||
esVersion: 70
|
||||
|
||||
- name: gdev-mysql
|
||||
type: mysql
|
||||
url: localhost:3306
|
||||
|
||||
@ -17,11 +17,13 @@
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"iteration": 1545263815779,
|
||||
"iteration": 1555595032099,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"tags": ["gdev", "elasticsearch", "datasource-test"],
|
||||
"title": "Dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
@ -5841,9 +5843,9 @@
|
||||
}
|
||||
],
|
||||
"refresh": false,
|
||||
"schemaVersion": 16,
|
||||
"schemaVersion": 18,
|
||||
"style": "dark",
|
||||
"tags": ["gdev", "elasticsearch"],
|
||||
"tags": ["elasticsearch", "gdev", "datasource-test"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
@ -5852,7 +5854,9 @@
|
||||
"value": "gdev-elasticsearch-v5-metrics"
|
||||
},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Version One",
|
||||
"multi": false,
|
||||
"name": "version_one",
|
||||
"options": [],
|
||||
"query": "elasticsearch",
|
||||
@ -5867,7 +5871,9 @@
|
||||
"value": "gdev-elasticsearch-v6-metrics"
|
||||
},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Version Two",
|
||||
"multi": false,
|
||||
"name": "version_two",
|
||||
"options": [],
|
||||
"query": "elasticsearch",
|
||||
@ -5889,5 +5895,5 @@
|
||||
"timezone": "",
|
||||
"title": "Datasource tests - Elasticsearch comparison",
|
||||
"uid": "fuFWehBmk",
|
||||
"version": 4
|
||||
"version": 2
|
||||
}
|
||||
|
||||
@ -31,8 +31,10 @@
|
||||
"iteration": 1554310942895,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"tags": ["gdev", "elasticsearch"],
|
||||
"tags": ["gdev", "elasticsearch", "datasource-test"],
|
||||
"title": "Dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
|
||||
@ -31,10 +31,10 @@
|
||||
"iteration": 1554310560048,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": false,
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"tags": ["gdev", "elasticsearch"],
|
||||
"title": "Dashboard",
|
||||
"tags": ["gdev", "elasticsearch", "datasource-test"],
|
||||
"title": "Dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
|
||||
@ -31,8 +31,10 @@
|
||||
"iteration": 1554310839317,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"tags": ["gdev", "elasticsearch"],
|
||||
"tags": ["gdev", "elasticsearch", "datasource-test"],
|
||||
"title": "Dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
|
||||
@ -16,7 +16,15 @@
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"iteration": 1554902936982,
|
||||
"links": [],
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"tags": ["gdev", "elasticsearch", "datasource-test"],
|
||||
"title": "Dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
"panels": [
|
||||
{
|
||||
"aliasColors": {
|
||||
|
||||
@ -0,0 +1,683 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": false,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"limit": 100,
|
||||
"name": "Annotations & Alerts",
|
||||
"showIn": 0,
|
||||
"type": "dashboard"
|
||||
},
|
||||
{
|
||||
"datasource": "gdev-elasticsearch-v7-logs",
|
||||
"enable": false,
|
||||
"iconColor": "rgba(255, 96, 96, 1)",
|
||||
"limit": 100,
|
||||
"name": "test",
|
||||
"query": "",
|
||||
"showIn": 0,
|
||||
"textField": "description",
|
||||
"type": "alert"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"id": 2342,
|
||||
"iteration": 1555593977614,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"tags": ["gdev", "elasticsearch", "datasource-test"],
|
||||
"title": "Dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
"panels": [
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "gdev-elasticsearch-v7-metrics",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
"grid": {},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 2,
|
||||
"links": [],
|
||||
"nullPointMode": "connected",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"bucketAggs": [
|
||||
{
|
||||
"field": "@hostname",
|
||||
"id": "3",
|
||||
"settings": {
|
||||
"min_doc_count": 1,
|
||||
"order": "asc",
|
||||
"orderBy": "1",
|
||||
"size": "5"
|
||||
},
|
||||
"type": "terms"
|
||||
},
|
||||
{
|
||||
"field": "@timestamp",
|
||||
"id": "2",
|
||||
"settings": {
|
||||
"interval": "auto",
|
||||
"min_doc_count": 0,
|
||||
"trimEdges": 0
|
||||
},
|
||||
"type": "date_histogram"
|
||||
}
|
||||
],
|
||||
"dsType": "elasticsearch",
|
||||
"metrics": [
|
||||
{
|
||||
"field": "@value",
|
||||
"id": "1",
|
||||
"meta": {},
|
||||
"settings": {},
|
||||
"type": "max"
|
||||
}
|
||||
],
|
||||
"query": "*",
|
||||
"refId": "A",
|
||||
"target": "",
|
||||
"timeField": "@timestamp"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Top 5 servers",
|
||||
"tooltip": {
|
||||
"msResolution": true,
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "cumulative"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {
|
||||
"Count": "#6ED0E0"
|
||||
},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "gdev-elasticsearch-v7-metrics",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
"grid": {},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 7
|
||||
},
|
||||
"id": 2,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 2,
|
||||
"links": [],
|
||||
"nullPointMode": "connected",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [
|
||||
{
|
||||
"alias": "Count",
|
||||
"lines": false,
|
||||
"yaxis": 2,
|
||||
"zindex": -1
|
||||
}
|
||||
],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"alias": "{{metric}}",
|
||||
"bucketAggs": [
|
||||
{
|
||||
"field": "@timestamp",
|
||||
"id": "2",
|
||||
"settings": {
|
||||
"interval": "5m",
|
||||
"min_doc_count": 0,
|
||||
"trimEdges": 0
|
||||
},
|
||||
"type": "date_histogram"
|
||||
}
|
||||
],
|
||||
"dsType": "elasticsearch",
|
||||
"metrics": [
|
||||
{
|
||||
"field": "@value",
|
||||
"id": "1",
|
||||
"meta": {},
|
||||
"settings": {
|
||||
"percents": [25, 50, 75, 95, 99]
|
||||
},
|
||||
"type": "percentiles"
|
||||
}
|
||||
],
|
||||
"query": "@metric:cpu",
|
||||
"refId": "A",
|
||||
"target": "",
|
||||
"timeField": "@timestamp"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Percentiles & Metric filter",
|
||||
"tooltip": {
|
||||
"msResolution": false,
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "cumulative"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {
|
||||
"Count": "#6ED0E0"
|
||||
},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "gdev-elasticsearch-v7-metrics",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
"grid": {},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 7
|
||||
},
|
||||
"id": 3,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 2,
|
||||
"links": [],
|
||||
"nullPointMode": "connected",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [
|
||||
{
|
||||
"alias": "Count",
|
||||
"lines": false,
|
||||
"yaxis": 2,
|
||||
"zindex": -1
|
||||
}
|
||||
],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"alias": "{{metric}}",
|
||||
"bucketAggs": [
|
||||
{
|
||||
"field": "@timestamp",
|
||||
"id": "2",
|
||||
"settings": {
|
||||
"interval": "auto",
|
||||
"min_doc_count": 0,
|
||||
"trimEdges": 0
|
||||
},
|
||||
"type": "date_histogram"
|
||||
}
|
||||
],
|
||||
"dsType": "elasticsearch",
|
||||
"metrics": [
|
||||
{
|
||||
"field": "@value",
|
||||
"id": "1",
|
||||
"meta": {
|
||||
"std_deviation_bounds_lower": true,
|
||||
"std_deviation_bounds_upper": true
|
||||
},
|
||||
"settings": {},
|
||||
"type": "extended_stats"
|
||||
}
|
||||
],
|
||||
"query": "@metric:cpu",
|
||||
"refId": "A",
|
||||
"target": "",
|
||||
"timeField": "@timestamp"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Standard dev",
|
||||
"tooltip": {
|
||||
"msResolution": true,
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "cumulative"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"columns": [
|
||||
{
|
||||
"text": "@hostname",
|
||||
"value": "@hostname"
|
||||
},
|
||||
{
|
||||
"text": "Average",
|
||||
"value": "Average"
|
||||
},
|
||||
{
|
||||
"text": "Max",
|
||||
"value": "Max"
|
||||
},
|
||||
{
|
||||
"text": "Sum",
|
||||
"value": "Sum"
|
||||
}
|
||||
],
|
||||
"datasource": "gdev-elasticsearch-v7-metrics",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fontSize": "100%",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 13
|
||||
},
|
||||
"id": 6,
|
||||
"links": [],
|
||||
"pageSize": null,
|
||||
"scroll": true,
|
||||
"showHeader": true,
|
||||
"sort": {
|
||||
"col": 0,
|
||||
"desc": true
|
||||
},
|
||||
"styles": [
|
||||
{
|
||||
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
||||
"pattern": "@timestamp",
|
||||
"type": "date"
|
||||
},
|
||||
{
|
||||
"colorMode": null,
|
||||
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
|
||||
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
||||
"decimals": 2,
|
||||
"pattern": "/.*/",
|
||||
"thresholds": [],
|
||||
"type": "number",
|
||||
"unit": "short"
|
||||
}
|
||||
],
|
||||
"targets": [
|
||||
{
|
||||
"bucketAggs": [
|
||||
{
|
||||
"field": "@hostname",
|
||||
"id": "2",
|
||||
"settings": {
|
||||
"min_doc_count": 1,
|
||||
"order": "asc",
|
||||
"orderBy": "_term",
|
||||
"size": "0"
|
||||
},
|
||||
"type": "terms"
|
||||
}
|
||||
],
|
||||
"dsType": "elasticsearch",
|
||||
"metrics": [
|
||||
{
|
||||
"field": "@value",
|
||||
"id": "1",
|
||||
"meta": {},
|
||||
"settings": {},
|
||||
"type": "avg"
|
||||
},
|
||||
{
|
||||
"field": "@value",
|
||||
"id": "3",
|
||||
"meta": {},
|
||||
"settings": {},
|
||||
"type": "max"
|
||||
},
|
||||
{
|
||||
"field": "@value",
|
||||
"id": "4",
|
||||
"meta": {},
|
||||
"settings": {},
|
||||
"type": "sum"
|
||||
}
|
||||
],
|
||||
"refId": "B",
|
||||
"timeField": "@timestamp"
|
||||
}
|
||||
],
|
||||
"title": "ES Metrics",
|
||||
"transform": "table",
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"columns": [
|
||||
{
|
||||
"text": "@timestamp",
|
||||
"value": "@timestamp"
|
||||
},
|
||||
{
|
||||
"text": "@message",
|
||||
"value": "@message"
|
||||
},
|
||||
{
|
||||
"text": "tags",
|
||||
"value": "tags"
|
||||
},
|
||||
{
|
||||
"text": "description",
|
||||
"value": "description"
|
||||
}
|
||||
],
|
||||
"datasource": "gdev-elasticsearch-v7-logs",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fontSize": "100%",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 20
|
||||
},
|
||||
"id": 5,
|
||||
"links": [],
|
||||
"pageSize": null,
|
||||
"scroll": true,
|
||||
"showHeader": true,
|
||||
"sort": {
|
||||
"col": 0,
|
||||
"desc": true
|
||||
},
|
||||
"styles": [
|
||||
{
|
||||
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
||||
"pattern": "@timestamp",
|
||||
"type": "date"
|
||||
}
|
||||
],
|
||||
"targets": [
|
||||
{
|
||||
"bucketAggs": [],
|
||||
"dsType": "elasticsearch",
|
||||
"metrics": [
|
||||
{
|
||||
"field": "select field",
|
||||
"id": "1",
|
||||
"meta": {},
|
||||
"settings": {
|
||||
"size": 500
|
||||
},
|
||||
"type": "raw_document"
|
||||
}
|
||||
],
|
||||
"refId": "A",
|
||||
"target": "",
|
||||
"timeField": "@timestamp"
|
||||
}
|
||||
],
|
||||
"title": "ES Log query",
|
||||
"transform": "json",
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"circleMaxSize": 30,
|
||||
"circleMinSize": 2,
|
||||
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
|
||||
"datasource": "gdev-elasticsearch-v7-metrics",
|
||||
"decimals": 0,
|
||||
"esGeoPoint": "@location",
|
||||
"esMetric": "Average",
|
||||
"gridPos": {
|
||||
"h": 12,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 27
|
||||
},
|
||||
"hideEmpty": false,
|
||||
"hideZero": false,
|
||||
"id": 8,
|
||||
"initialZoom": "1",
|
||||
"links": [],
|
||||
"locationData": "geohash",
|
||||
"mapCenter": "(0°, 0°)",
|
||||
"mapCenterLatitude": 0,
|
||||
"mapCenterLongitude": 0,
|
||||
"maxDataPoints": 1,
|
||||
"mouseWheelZoom": false,
|
||||
"showLegend": true,
|
||||
"stickyLabels": false,
|
||||
"tableQueryOptions": {
|
||||
"geohashField": "geohash",
|
||||
"latitudeField": "latitude",
|
||||
"longitudeField": "longitude",
|
||||
"metricField": "metric",
|
||||
"queryType": "geohash"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"bucketAggs": [
|
||||
{
|
||||
"fake": true,
|
||||
"field": "@location",
|
||||
"id": "3",
|
||||
"settings": {
|
||||
"precision": 2
|
||||
},
|
||||
"type": "geohash_grid"
|
||||
}
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"field": "@value",
|
||||
"id": "1",
|
||||
"meta": {},
|
||||
"settings": {},
|
||||
"type": "avg"
|
||||
}
|
||||
],
|
||||
"refId": "A",
|
||||
"target": "",
|
||||
"timeField": "@timestamp"
|
||||
}
|
||||
],
|
||||
"thresholds": "0,10",
|
||||
"title": "World map panel",
|
||||
"type": "grafana-worldmap-panel",
|
||||
"unitPlural": "",
|
||||
"unitSingle": "",
|
||||
"valueName": "total"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 18,
|
||||
"style": "dark",
|
||||
"tags": ["elasticsearch", "gdev", "datasource-test"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"datasource": "gdev-elasticsearch-v7-metrics",
|
||||
"filters": [],
|
||||
"hide": 0,
|
||||
"label": "",
|
||||
"name": "Filters",
|
||||
"skipUrlSync": false,
|
||||
"type": "adhoc"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-30m",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"collapse": false,
|
||||
"enable": true,
|
||||
"notice": false,
|
||||
"now": true,
|
||||
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"],
|
||||
"status": "Stable",
|
||||
"time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"],
|
||||
"type": "timepicker"
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "Datasource tests - Elasticsearch v7",
|
||||
"uid": "Y-RvmuRWk",
|
||||
"version": 1
|
||||
}
|
||||
@ -0,0 +1,264 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"id": 2341,
|
||||
"iteration": 1555591591930,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"tags": ["gdev", "elasticsearch", "datasource-test"],
|
||||
"title": "Dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
"panels": [
|
||||
{
|
||||
"aliasColors": {
|
||||
"error": "red"
|
||||
},
|
||||
"bars": true,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "gdev-elasticsearch-v7-filebeat",
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 4,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": false,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": true,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"bucketAggs": [
|
||||
{
|
||||
"fake": true,
|
||||
"field": "fields.level",
|
||||
"id": "3",
|
||||
"settings": {
|
||||
"min_doc_count": 1,
|
||||
"order": "desc",
|
||||
"orderBy": "_term",
|
||||
"size": "10"
|
||||
},
|
||||
"type": "terms"
|
||||
},
|
||||
{
|
||||
"field": "@timestamp",
|
||||
"id": "2",
|
||||
"settings": {
|
||||
"interval": "5m",
|
||||
"min_doc_count": 1,
|
||||
"trimEdges": 0
|
||||
},
|
||||
"type": "date_histogram"
|
||||
}
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"field": "select field",
|
||||
"id": "1",
|
||||
"type": "count"
|
||||
}
|
||||
],
|
||||
"query": "fields.app:grafana",
|
||||
"refId": "A",
|
||||
"timeField": "@timestamp"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Panel Title",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"columns": [
|
||||
{
|
||||
"text": "@timestamp",
|
||||
"value": "@timestamp"
|
||||
},
|
||||
{
|
||||
"text": "fields.level",
|
||||
"value": "fields.level"
|
||||
},
|
||||
{
|
||||
"text": "message",
|
||||
"value": "message"
|
||||
}
|
||||
],
|
||||
"datasource": "gdev-elasticsearch-v7-filebeat",
|
||||
"fontSize": "100%",
|
||||
"gridPos": {
|
||||
"h": 22,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 5
|
||||
},
|
||||
"id": 2,
|
||||
"links": [],
|
||||
"pageSize": null,
|
||||
"scroll": true,
|
||||
"showHeader": true,
|
||||
"sort": {
|
||||
"col": 0,
|
||||
"desc": true
|
||||
},
|
||||
"styles": [
|
||||
{
|
||||
"alias": "Time",
|
||||
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
||||
"pattern": "@timestamp",
|
||||
"type": "date"
|
||||
},
|
||||
{
|
||||
"alias": "Level",
|
||||
"colorMode": null,
|
||||
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
|
||||
"dateFormat": "YYYY-MM-DD HH:mm:ss",
|
||||
"decimals": 2,
|
||||
"mappingType": 1,
|
||||
"pattern": "fields.level",
|
||||
"thresholds": [""],
|
||||
"type": "string",
|
||||
"unit": "short",
|
||||
"valueMaps": []
|
||||
},
|
||||
{
|
||||
"alias": "Message",
|
||||
"colorMode": null,
|
||||
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
|
||||
"decimals": 2,
|
||||
"pattern": "message",
|
||||
"preserveFormat": false,
|
||||
"sanitize": false,
|
||||
"thresholds": [],
|
||||
"type": "string",
|
||||
"unit": "short"
|
||||
}
|
||||
],
|
||||
"targets": [
|
||||
{
|
||||
"bucketAggs": [],
|
||||
"metrics": [
|
||||
{
|
||||
"field": "select field",
|
||||
"id": "1",
|
||||
"meta": {},
|
||||
"settings": {
|
||||
"size": 500
|
||||
},
|
||||
"type": "raw_document"
|
||||
}
|
||||
],
|
||||
"query": "fields.app:grafana",
|
||||
"refId": "A",
|
||||
"timeField": "@timestamp"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Panel Title",
|
||||
"transform": "json",
|
||||
"type": "table"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 18,
|
||||
"style": "dark",
|
||||
"tags": ["elasticsearch", "gdev", "datasource-test"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"datasource": "gdev-elasticsearch-v7-filebeat",
|
||||
"filters": [],
|
||||
"hide": 0,
|
||||
"label": "",
|
||||
"name": "Filters",
|
||||
"skipUrlSync": false,
|
||||
"type": "adhoc"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"],
|
||||
"time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"]
|
||||
},
|
||||
"timezone": "",
|
||||
"title": "Datasource tests - Elasticsearch v7 Filebeat",
|
||||
"uid": "M94gguRWz",
|
||||
"version": 1
|
||||
}
|
||||
@ -1,7 +1,7 @@
|
||||
# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
|
||||
|
||||
elasticsearch6:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.7.1
|
||||
command: elasticsearch
|
||||
ports:
|
||||
- "11200:9200"
|
||||
@ -15,7 +15,7 @@
|
||||
FD_PORT: 11200
|
||||
|
||||
filebeat6:
|
||||
image: docker.elastic.co/beats/filebeat:6.2.4
|
||||
image: docker.elastic.co/beats/filebeat-oss:6.7.1
|
||||
command: filebeat -e -strict.perms=false
|
||||
volumes:
|
||||
- ./docker/blocks/elastic6/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
23
devenv/docker/blocks/elastic7/docker-compose.yaml
Normal file
23
devenv/docker/blocks/elastic7/docker-compose.yaml
Normal file
@ -0,0 +1,23 @@
|
||||
# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
|
||||
|
||||
elasticsearch7:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0
|
||||
command: elasticsearch -E "discovery.type=single-node"
|
||||
ports:
|
||||
- "12200:9200"
|
||||
- "12300:9300"
|
||||
|
||||
fake-elastic7-data:
|
||||
image: grafana/fake-data-gen
|
||||
network_mode: bridge
|
||||
environment:
|
||||
FD_DATASOURCE: elasticsearch7
|
||||
FD_PORT: 12200
|
||||
|
||||
filebeat7:
|
||||
image: docker.elastic.co/beats/filebeat-oss:7.0.0
|
||||
command: filebeat -e -strict.perms=false
|
||||
volumes:
|
||||
- ./docker/blocks/elastic7/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
|
||||
- /var/log:/var/log:ro
|
||||
- ../data/log:/var/log/grafana:ro
|
||||
2
devenv/docker/blocks/elastic7/elasticsearch.yml
Normal file
2
devenv/docker/blocks/elastic7/elasticsearch.yml
Normal file
@ -0,0 +1,2 @@
|
||||
script.inline: on
|
||||
script.indexed: on
|
||||
1978
devenv/docker/blocks/elastic7/filebeat.yml
Normal file
1978
devenv/docker/blocks/elastic7/filebeat.yml
Normal file
File diff suppressed because it is too large
Load Diff
@ -146,7 +146,7 @@ Since not all datasources have the same configuration settings we only have the
|
||||
| tlsSkipVerify | boolean | *All* | Controls whether a client verifies the server's certificate chain and host name. |
|
||||
| graphiteVersion | string | Graphite | Graphite version |
|
||||
| timeInterval | string | Prometheus, Elasticsearch, InfluxDB, MySQL, PostgreSQL & MSSQL | Lowest interval/step value that should be used for this data source |
|
||||
| esVersion | number | Elasticsearch | Elasticsearch version as a number (2/5/56/60) |
|
||||
| esVersion | number | Elasticsearch | Elasticsearch version as a number (2/5/56/60/70) |
|
||||
| timeField | string | Elasticsearch | Which field that should be used as timestamp |
|
||||
| interval | string | Elasticsearch | Index date time format. nil(No Pattern), 'Hourly', 'Daily', 'Weekly', 'Monthly' or 'Yearly' |
|
||||
| authType | string | Cloudwatch | Auth provider. keys/credentials/arn |
|
||||
|
||||
@ -58,10 +58,12 @@ a time pattern for the index name or a wildcard.
|
||||
|
||||
### Elasticsearch version
|
||||
|
||||
Be sure to specify your Elasticsearch version in the version selection dropdown. This is very important as there are differences how queries are composed.
|
||||
Currently the versions available is 2.x, 5.x, 5.6+ or 6.0+. 5.6+ means a version of 5.6 or less than 6.0. 6.0+ means a version of 6.0 or higher, 6.3.2 for example.
|
||||
Be sure to specify your Elasticsearch version in the version selection dropdown. This is very important as there are differences on how queries are composed.
|
||||
Currently the versions available are `2.x`, `5.x`, `5.6+`, `6.0+` or `7.0+`. The value `5.6+` means version 5.6 or higher, but lower than 6.0. The value `6.0+` means
|
||||
version 6.0 or higher, but lower than 7.0. Finally, `7.0+` means version 7.0 or higher, but lower than 8.0.
|
||||
|
||||
### Min time interval
|
||||
|
||||
A lower limit for the auto group by time interval. Recommended to be set to write frequency, for example `1m` if your data is written every minute.
|
||||
This option can also be overridden/configured in a dashboard panel under data source options. It's important to note that this value **needs** to be formatted as a
|
||||
number followed by a valid time identifier, e.g. `1m` (1 minute) or `30s` (30 seconds). The following time identifiers are supported:
|
||||
|
||||
@ -65,7 +65,7 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb
|
||||
clientLog.Debug("Creating new client", "version", version, "timeField", timeField, "indices", strings.Join(indices, ", "))
|
||||
|
||||
switch version {
|
||||
case 2, 5, 56, 60:
|
||||
case 2, 5, 56, 60, 70:
|
||||
return &baseClientImpl{
|
||||
ctx: ctx,
|
||||
ds: ds,
|
||||
@ -112,12 +112,12 @@ type multiRequest struct {
|
||||
interval tsdb.Interval
|
||||
}
|
||||
|
||||
func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) {
|
||||
func (c *baseClientImpl) executeBatchRequest(uriPath, uriQuery string, requests []*multiRequest) (*http.Response, error) {
|
||||
bytes, err := c.encodeBatchRequests(requests)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.executeRequest(http.MethodPost, uriPath, bytes)
|
||||
return c.executeRequest(http.MethodPost, uriPath, uriQuery, bytes)
|
||||
}
|
||||
|
||||
func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte, error) {
|
||||
@ -150,9 +150,10 @@ func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte,
|
||||
return payload.Bytes(), nil
|
||||
}
|
||||
|
||||
func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*http.Response, error) {
|
||||
func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body []byte) (*http.Response, error) {
|
||||
u, _ := url.Parse(c.ds.Url)
|
||||
u.Path = path.Join(u.Path, uriPath)
|
||||
u.RawQuery = uriQuery
|
||||
|
||||
var req *http.Request
|
||||
var err error
|
||||
@ -197,7 +198,8 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch
|
||||
clientLog.Debug("Executing multisearch", "search requests", len(r.Requests))
|
||||
|
||||
multiRequests := c.createMultiSearchRequests(r.Requests)
|
||||
res, err := c.executeBatchRequest("_msearch", multiRequests)
|
||||
queryParams := c.getMultiSearchQueryParameters()
|
||||
res, err := c.executeBatchRequest("_msearch", queryParams, multiRequests)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -241,7 +243,7 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
|
||||
mr.header["search_type"] = "count"
|
||||
}
|
||||
|
||||
if c.version >= 56 {
|
||||
if c.version >= 56 && c.version < 70 {
|
||||
maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(256)
|
||||
mr.header["max_concurrent_shard_requests"] = maxConcurrentShardRequests
|
||||
}
|
||||
@ -252,6 +254,15 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
|
||||
return multiRequests
|
||||
}
|
||||
|
||||
func (c *baseClientImpl) getMultiSearchQueryParameters() string {
|
||||
if c.version >= 70 {
|
||||
maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(5)
|
||||
return fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests)
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder {
|
||||
return NewMultiSearchRequestBuilder(c.GetVersion())
|
||||
}
|
||||
|
||||
@ -103,6 +103,19 @@ func TestClient(t *testing.T) {
|
||||
So(err, ShouldBeNil)
|
||||
So(c.GetVersion(), ShouldEqual, 60)
|
||||
})
|
||||
|
||||
Convey("When version 70 should return v7.0 client", func() {
|
||||
ds := &models.DataSource{
|
||||
JsonData: simplejson.NewFromAny(map[string]interface{}{
|
||||
"esVersion": 70,
|
||||
"timeField": "@timestamp",
|
||||
}),
|
||||
}
|
||||
|
||||
c, err := NewClient(context.Background(), ds, nil)
|
||||
So(err, ShouldBeNil)
|
||||
So(c.GetVersion(), ShouldEqual, 70)
|
||||
})
|
||||
})
|
||||
|
||||
Convey("Given a fake http client", func() {
|
||||
@ -290,6 +303,60 @@ func TestClient(t *testing.T) {
|
||||
})
|
||||
})
|
||||
|
||||
Convey("and a v7.0 client", func() {
|
||||
ds := models.DataSource{
|
||||
Database: "[metrics-]YYYY.MM.DD",
|
||||
Url: ts.URL,
|
||||
JsonData: simplejson.NewFromAny(map[string]interface{}{
|
||||
"esVersion": 70,
|
||||
"maxConcurrentShardRequests": 6,
|
||||
"timeField": "@timestamp",
|
||||
"interval": "Daily",
|
||||
}),
|
||||
}
|
||||
|
||||
c, err := NewClient(context.Background(), &ds, timeRange)
|
||||
So(err, ShouldBeNil)
|
||||
So(c, ShouldNotBeNil)
|
||||
|
||||
Convey("When executing multi search", func() {
|
||||
ms, err := createMultisearchForTest(c)
|
||||
So(err, ShouldBeNil)
|
||||
c.ExecuteMultisearch(ms)
|
||||
|
||||
Convey("Should send correct request and payload", func() {
|
||||
So(req, ShouldNotBeNil)
|
||||
So(req.Method, ShouldEqual, http.MethodPost)
|
||||
So(req.URL.Path, ShouldEqual, "/_msearch")
|
||||
So(req.URL.RawQuery, ShouldEqual, "max_concurrent_shard_requests=6")
|
||||
|
||||
So(responseBuffer, ShouldNotBeNil)
|
||||
|
||||
headerBytes, err := responseBuffer.ReadBytes('\n')
|
||||
So(err, ShouldBeNil)
|
||||
bodyBytes := responseBuffer.Bytes()
|
||||
|
||||
jHeader, err := simplejson.NewJson(headerBytes)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
jBody, err := simplejson.NewJson(bodyBytes)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15")
|
||||
So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true)
|
||||
So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch")
|
||||
|
||||
Convey("and replace $__interval variable", func() {
|
||||
So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname")
|
||||
})
|
||||
|
||||
Convey("and replace $__interval_ms variable", func() {
|
||||
So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s")
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Reset(func() {
|
||||
newDatasourceHttpClient = currentNewDatasourceHttpClient
|
||||
})
|
||||
|
||||
@ -42,7 +42,7 @@ func (r *SearchRequest) MarshalJSON() ([]byte, error) {
|
||||
// SearchResponseHits represents search response hits
|
||||
type SearchResponseHits struct {
|
||||
Hits []map[string]interface{}
|
||||
Total int64
|
||||
Total map[string]interface{}
|
||||
}
|
||||
|
||||
// SearchResponse represents a search response
|
||||
|
||||
@ -8,7 +8,9 @@ export class ElasticConfigCtrl {
|
||||
constructor($scope) {
|
||||
this.current.jsonData.timeField = this.current.jsonData.timeField || '@timestamp';
|
||||
this.current.jsonData.esVersion = this.current.jsonData.esVersion || 5;
|
||||
this.current.jsonData.maxConcurrentShardRequests = this.current.jsonData.maxConcurrentShardRequests || 256;
|
||||
const defaultMaxConcurrentShardRequests = this.current.jsonData.esVersion >= 70 ? 5 : 256;
|
||||
this.current.jsonData.maxConcurrentShardRequests =
|
||||
this.current.jsonData.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests;
|
||||
}
|
||||
|
||||
indexPatternTypes = [
|
||||
@ -25,6 +27,7 @@ export class ElasticConfigCtrl {
|
||||
{ name: '5.x', value: 5 },
|
||||
{ name: '5.6+', value: 56 },
|
||||
{ name: '6.0+', value: 60 },
|
||||
{ name: '7.0+', value: 70 },
|
||||
];
|
||||
|
||||
indexPatternTypeChanged() {
|
||||
|
||||
@ -234,7 +234,7 @@ export class ElasticDatasource {
|
||||
ignore_unavailable: true,
|
||||
index: this.indexPattern.getIndexList(timeFrom, timeTo),
|
||||
};
|
||||
if (this.esVersion >= 56) {
|
||||
if (this.esVersion >= 56 && this.esVersion < 70) {
|
||||
queryHeader['max_concurrent_shard_requests'] = this.maxConcurrentShardRequests;
|
||||
}
|
||||
return angular.toJson(queryHeader);
|
||||
@ -277,12 +277,15 @@ export class ElasticDatasource {
|
||||
payload = payload.replace(/\$timeTo/g, options.range.to.valueOf());
|
||||
payload = this.templateSrv.replace(payload, options.scopedVars);
|
||||
|
||||
return this.post('_msearch', payload).then(res => {
|
||||
const url = this.getMultiSearchUrl();
|
||||
|
||||
return this.post(url, payload).then(res => {
|
||||
return new ElasticResponse(sentTargets, res).getTimeSeries();
|
||||
});
|
||||
}
|
||||
|
||||
getFields(query) {
|
||||
const configuredEsVersion = this.esVersion;
|
||||
return this.get('/_mapping').then(result => {
|
||||
const typeMap = {
|
||||
float: 'number',
|
||||
@ -347,8 +350,14 @@ export class ElasticDatasource {
|
||||
const index = result[indexName];
|
||||
if (index && index.mappings) {
|
||||
const mappings = index.mappings;
|
||||
for (const typeName in mappings) {
|
||||
const properties = mappings[typeName].properties;
|
||||
|
||||
if (configuredEsVersion < 70) {
|
||||
for (const typeName in mappings) {
|
||||
const properties = mappings[typeName].properties;
|
||||
getFieldsRecursively(properties);
|
||||
}
|
||||
} else {
|
||||
const properties = mappings.properties;
|
||||
getFieldsRecursively(properties);
|
||||
}
|
||||
}
|
||||
@ -371,7 +380,9 @@ export class ElasticDatasource {
|
||||
esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf());
|
||||
esQuery = header + '\n' + esQuery + '\n';
|
||||
|
||||
return this.post('_msearch?search_type=' + searchType, esQuery).then(res => {
|
||||
const url = this.getMultiSearchUrl();
|
||||
|
||||
return this.post(url, esQuery).then(res => {
|
||||
if (!res.responses[0].aggregations) {
|
||||
return [];
|
||||
}
|
||||
@ -386,6 +397,14 @@ export class ElasticDatasource {
|
||||
});
|
||||
}
|
||||
|
||||
getMultiSearchUrl() {
|
||||
if (this.esVersion >= 70 && this.maxConcurrentShardRequests) {
|
||||
return `_msearch?max_concurrent_shard_requests=${this.maxConcurrentShardRequests}`;
|
||||
}
|
||||
|
||||
return '_msearch';
|
||||
}
|
||||
|
||||
metricFindQuery(query) {
|
||||
query = angular.fromJson(query);
|
||||
if (!query) {
|
||||
|
||||
@ -184,7 +184,7 @@ export class ElasticResponse {
|
||||
}
|
||||
|
||||
// This is quite complex
|
||||
// need to recurise down the nested buckets to build series
|
||||
// need to recurse down the nested buckets to build series
|
||||
processBuckets(aggs, target, seriesList, table, props, depth) {
|
||||
let bucket, aggDef, esAgg, aggId;
|
||||
const maxDepth = target.bucketAggs.length - 1;
|
||||
@ -310,11 +310,13 @@ export class ElasticResponse {
|
||||
}
|
||||
|
||||
processHits(hits, seriesList) {
|
||||
const hitsTotal = typeof hits.total === 'number' ? hits.total : hits.total.value; // <- Works with Elasticsearch 7.0+
|
||||
|
||||
const series = {
|
||||
target: 'docs',
|
||||
type: 'docs',
|
||||
datapoints: [],
|
||||
total: hits.total,
|
||||
total: hitsTotal,
|
||||
filterable: true,
|
||||
};
|
||||
let propName, hit, doc, i;
|
||||
|
||||
@ -187,7 +187,7 @@ describe('ElasticDatasource', function(this: any) {
|
||||
|
||||
describe('When getting fields', () => {
|
||||
beforeEach(() => {
|
||||
createDatasource({ url: 'http://es.com', index: 'metricbeat' });
|
||||
createDatasource({ url: 'http://es.com', index: 'metricbeat', jsonData: { esVersion: 50 } });
|
||||
|
||||
ctx.backendSrv.datasourceRequest = jest.fn(options => {
|
||||
return Promise.resolve({
|
||||
@ -236,49 +236,191 @@ describe('ElasticDatasource', function(this: any) {
|
||||
});
|
||||
});
|
||||
|
||||
it('should return nested fields', () => {
|
||||
ctx.ds
|
||||
.getFields({
|
||||
find: 'fields',
|
||||
query: '*',
|
||||
})
|
||||
.then(fieldObjects => {
|
||||
const fields = _.map(fieldObjects, 'text');
|
||||
expect(fields).toEqual([
|
||||
'@timestamp',
|
||||
'beat.name.raw',
|
||||
'beat.name',
|
||||
'beat.hostname',
|
||||
'system.cpu.system',
|
||||
'system.cpu.user',
|
||||
'system.process.cpu.total',
|
||||
'system.process.name',
|
||||
]);
|
||||
});
|
||||
it('should return nested fields', async () => {
|
||||
const fieldObjects = await ctx.ds.getFields({
|
||||
find: 'fields',
|
||||
query: '*',
|
||||
});
|
||||
const fields = _.map(fieldObjects, 'text');
|
||||
expect(fields).toEqual([
|
||||
'@timestamp',
|
||||
'beat.name.raw',
|
||||
'beat.name',
|
||||
'beat.hostname',
|
||||
'system.cpu.system',
|
||||
'system.cpu.user',
|
||||
'system.process.cpu.total',
|
||||
'system.process.name',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should return fields related to query type', () => {
|
||||
ctx.ds
|
||||
.getFields({
|
||||
find: 'fields',
|
||||
query: '*',
|
||||
type: 'number',
|
||||
})
|
||||
.then(fieldObjects => {
|
||||
const fields = _.map(fieldObjects, 'text');
|
||||
expect(fields).toEqual(['system.cpu.system', 'system.cpu.user', 'system.process.cpu.total']);
|
||||
});
|
||||
it('should return number fields', async () => {
|
||||
const fieldObjects = await ctx.ds.getFields({
|
||||
find: 'fields',
|
||||
query: '*',
|
||||
type: 'number',
|
||||
});
|
||||
|
||||
ctx.ds
|
||||
.getFields({
|
||||
find: 'fields',
|
||||
query: '*',
|
||||
type: 'date',
|
||||
})
|
||||
.then(fieldObjects => {
|
||||
const fields = _.map(fieldObjects, 'text');
|
||||
expect(fields).toEqual(['@timestamp']);
|
||||
const fields = _.map(fieldObjects, 'text');
|
||||
expect(fields).toEqual(['system.cpu.system', 'system.cpu.user', 'system.process.cpu.total']);
|
||||
});
|
||||
|
||||
it('should return date fields', async () => {
|
||||
const fieldObjects = await ctx.ds.getFields({
|
||||
find: 'fields',
|
||||
query: '*',
|
||||
type: 'date',
|
||||
});
|
||||
|
||||
const fields = _.map(fieldObjects, 'text');
|
||||
expect(fields).toEqual(['@timestamp']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('When getting fields from ES 7.0', () => {
|
||||
beforeEach(() => {
|
||||
createDatasource({ url: 'http://es.com', index: 'genuine.es7._mapping.response', jsonData: { esVersion: 70 } });
|
||||
|
||||
ctx.backendSrv.datasourceRequest = jest.fn(options => {
|
||||
return Promise.resolve({
|
||||
data: {
|
||||
'genuine.es7._mapping.response': {
|
||||
mappings: {
|
||||
properties: {
|
||||
'@timestamp_millis': {
|
||||
type: 'date',
|
||||
format: 'epoch_millis',
|
||||
},
|
||||
classification_terms: {
|
||||
type: 'keyword',
|
||||
},
|
||||
domains: {
|
||||
type: 'keyword',
|
||||
},
|
||||
ip_address: {
|
||||
type: 'ip',
|
||||
},
|
||||
justification_blob: {
|
||||
properties: {
|
||||
criterion: {
|
||||
type: 'text',
|
||||
fields: {
|
||||
keyword: {
|
||||
type: 'keyword',
|
||||
ignore_above: 256,
|
||||
},
|
||||
},
|
||||
},
|
||||
overall_vote_score: {
|
||||
type: 'float',
|
||||
},
|
||||
shallow: {
|
||||
properties: {
|
||||
jsi: {
|
||||
properties: {
|
||||
sdb: {
|
||||
properties: {
|
||||
dsel2: {
|
||||
properties: {
|
||||
'bootlegged-gille': {
|
||||
properties: {
|
||||
botness: {
|
||||
type: 'float',
|
||||
},
|
||||
general_algorithm_score: {
|
||||
type: 'float',
|
||||
},
|
||||
},
|
||||
},
|
||||
'uncombed-boris': {
|
||||
properties: {
|
||||
botness: {
|
||||
type: 'float',
|
||||
},
|
||||
general_algorithm_score: {
|
||||
type: 'float',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
overall_vote_score: {
|
||||
type: 'float',
|
||||
},
|
||||
ua_terms_long: {
|
||||
type: 'keyword',
|
||||
},
|
||||
ua_terms_short: {
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should return nested fields', async () => {
|
||||
const fieldObjects = await ctx.ds.getFields({
|
||||
find: 'fields',
|
||||
query: '*',
|
||||
});
|
||||
|
||||
const fields = _.map(fieldObjects, 'text');
|
||||
expect(fields).toEqual([
|
||||
'@timestamp_millis',
|
||||
'classification_terms',
|
||||
'domains',
|
||||
'ip_address',
|
||||
'justification_blob.criterion.keyword',
|
||||
'justification_blob.criterion',
|
||||
'justification_blob.overall_vote_score',
|
||||
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
|
||||
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
|
||||
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
|
||||
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
|
||||
'overall_vote_score',
|
||||
'ua_terms_long',
|
||||
'ua_terms_short',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should return number fields', async () => {
|
||||
const fieldObjects = await ctx.ds.getFields({
|
||||
find: 'fields',
|
||||
query: '*',
|
||||
type: 'number',
|
||||
});
|
||||
|
||||
const fields = _.map(fieldObjects, 'text');
|
||||
expect(fields).toEqual([
|
||||
'justification_blob.overall_vote_score',
|
||||
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
|
||||
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
|
||||
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
|
||||
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
|
||||
'overall_vote_score',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should return date fields', async () => {
|
||||
const fieldObjects = await ctx.ds.getFields({
|
||||
find: 'fields',
|
||||
query: '*',
|
||||
type: 'date',
|
||||
});
|
||||
|
||||
const fields = _.map(fieldObjects, 'text');
|
||||
expect(fields).toEqual(['@timestamp_millis']);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user