IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-01-09 23:53:25 -06:00
Code Issues Packages Projects Releases Wiki Activity

loki: backend mode: support all query types (#45619)

Browse Source 
* loki: backend mode: support all query types

* loki: backend: adjust vector-parsing field-names

* loki: backend: no interval for streams-dataframes

* loki: backend: enable more query types

* better variable name

* removed unnecessary code

* improve frame-processing

* more unit tests

* improved code

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* remove unused code

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* simplify code

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* lint fix

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>
This commit is contained in:
Gábor Farkas 2022-02-25 09:14:17 +01:00 committed by GitHub
parent a578cf0f7c
commit 1cad35ea67
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 966 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
pkg/tsdb/loki/api.go
View File

@ -28,16 +28,41 @@ func newLokiAPI(client *http.Client, url string, log log.Logger) *LokiAPI {
func makeRequest(ctx context.Context, lokiDsUrl string, query lokiQuery) (*http.Request, error) {
qs := url.Values{}
qs.Set("query", query.Expr)
qs.Set("step", query.Step.String())
qs.Set("start", strconv.FormatInt(query.Start.UnixNano(), 10))
qs.Set("end", strconv.FormatInt(query.End.UnixNano(), 10))
// MaxLines defaults to zero when not received,
// and Loki does not like limit=0, even when it is not needed
// (for example for metric queries), so we
// only send it when it's set
if query.MaxLines > 0 {
qs.Set("limit", fmt.Sprintf("%d", query.MaxLines))
}
lokiUrl, err := url.Parse(lokiDsUrl)
if err != nil {
return nil, err
}
lokiUrl.Path = "/loki/api/v1/query_range"
switch query.QueryType {
case QueryTypeRange:
{
qs.Set("start", strconv.FormatInt(query.Start.UnixNano(), 10))
qs.Set("end", strconv.FormatInt(query.End.UnixNano(), 10))
// NOTE: technically for streams-producing queries `step`
// is ignored, so it would be nicer to not send it in such cases,
// but we cannot detect that situation, so we always send it.
// it should not break anything.
qs.Set("step", query.Step.String())
lokiUrl.Path = "/loki/api/v1/query_range"
}
case QueryTypeInstant:
{
qs.Set("time", strconv.FormatInt(query.End.UnixNano(), 10))
lokiUrl.Path = "/loki/api/v1/query"
}
default:
return nil, fmt.Errorf("invalid QueryType: %v", query.QueryType)
}
lokiUrl.RawQuery = qs.Encode()
req, err := http.NewRequestWithContext(ctx, "GET", lokiUrl.String(), nil)
@ -51,11 +76,10 @@ func makeRequest(ctx context.Context, lokiDsUrl string, query lokiQuery) (*http.
// so it is not a regression.
// twe need to have that when we migrate to backend-queries.
//
// 2. we will have to send a custom http header based on the VolumeQuery prop
// (again, not needed for the alerting scenario)
// if query.VolumeQuery {
// req.Header.Set("X-Query-Tags", "Source=logvolhist")
// }
if query.VolumeQuery {
req.Header.Set("X-Query-Tags", "Source=logvolhist")
}
return req, nil
}
@ -103,7 +127,7 @@ func makeLokiError(body io.ReadCloser) error {
return fmt.Errorf("%v", errorMessage)
}
func (api *LokiAPI) QueryRange(ctx context.Context, query lokiQuery) (*loghttp.QueryResponse, error) {
func (api *LokiAPI) Query(ctx context.Context, query lokiQuery) (*loghttp.QueryResponse, error) {
req, err := makeRequest(ctx, api.url, query)
if err != nil {
return nil, err

113
pkg/tsdb/loki/frame.go Normal file
View File

@ -0,0 +1,113 @@
package loki
import (
"fmt"
"sort"
"strings"
"github.com/grafana/grafana-plugin-sdk-go/data"
)
// we adjust the dataframes to be the way frontend & alerting
// wants them.
func adjustFrame(frame *data.Frame, query *lokiQuery) *data.Frame {
labels := getFrameLabels(frame)
timeFields, nonTimeFields := partitionFields(frame)
isMetricFrame := nonTimeFields[0].Type() != data.FieldTypeString
isMetricRange := isMetricFrame && query.QueryType == QueryTypeRange
name := formatName(labels, query)
frame.Name = name
if frame.Meta == nil {
frame.Meta = &data.FrameMeta{}
}
if isMetricRange {
frame.Meta.ExecutedQueryString = "Expr: " + query.Expr + "\n" + "Step: " + query.Step.String()
} else {
frame.Meta.ExecutedQueryString = "Expr: " + query.Expr
}
for _, field := range timeFields {
field.Name = "time"
if isMetricRange {
if field.Config == nil {
field.Config = &data.FieldConfig{}
}
field.Config.Interval = float64(query.Step.Milliseconds())
}
}
for _, field := range nonTimeFields {
field.Name = "value"
if field.Config == nil {
field.Config = &data.FieldConfig{}
}
field.Config.DisplayNameFromDS = name
}
return frame
}
func formatNamePrometheusStyle(labels map[string]string) string {
var parts []string
for k, v := range labels {
parts = append(parts, fmt.Sprintf("%s=%q", k, v))
}
sort.Strings(parts)
return fmt.Sprintf("{%s}", strings.Join(parts, ", "))
}
//If legend (using of name or pattern instead of time series name) is used, use that name/pattern for formatting
func formatName(labels map[string]string, query *lokiQuery) string {
if query.LegendFormat == "" {
return formatNamePrometheusStyle(labels)
}
result := legendFormat.ReplaceAllFunc([]byte(query.LegendFormat), func(in []byte) []byte {
labelName := strings.Replace(string(in), "{{", "", 1)
labelName = strings.Replace(labelName, "}}", "", 1)
labelName = strings.TrimSpace(labelName)
if val, exists := labels[labelName]; exists {
return []byte(val)
}
return []byte{}
})
return string(result)
}
func getFrameLabels(frame *data.Frame) map[string]string {
labels := make(map[string]string)
for _, field := range frame.Fields {
for k, v := range field.Labels {
labels[k] = v
}
}
return labels
}
func partitionFields(frame *data.Frame) ([]*data.Field, []*data.Field) {
var timeFields []*data.Field
var nonTimeFields []*data.Field
for _, field := range frame.Fields {
if field.Type() == data.FieldTypeTime {
timeFields = append(timeFields, field)
} else {
nonTimeFields = append(nonTimeFields, field)
}
}
return timeFields, nonTimeFields
}

87
pkg/tsdb/loki/frame_test.go Normal file
View File

@ -0,0 +1,87 @@
package loki
import (
"testing"
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/stretchr/testify/require"
)
func TestFormatName(t *testing.T) {
t.Run("converting metric name", func(t *testing.T) {
metric := map[string]string{
"app": "backend",
"device": "mobile",
}
query := &lokiQuery{
LegendFormat: "legend {{app}} {{ device }} {{broken}}",
}
require.Equal(t, "legend backend mobile ", formatName(metric, query))
})
t.Run("build full series name", func(t *testing.T) {
metric := map[string]string{
"app": "backend",
"device": "mobile",
}
query := &lokiQuery{
LegendFormat: "",
}
require.Equal(t, `{app="backend", device="mobile"}`, formatName(metric, query))
})
}
func TestAdjustFrame(t *testing.T) {
t.Run("response should be parsed normally", func(t *testing.T) {
field1 := data.NewField("", nil, make([]time.Time, 0))
field2 := data.NewField("", nil, make([]float64, 0))
field2.Labels = data.Labels{"app": "Application", "tag2": "tag2"}
frame := data.NewFrame("test", field1, field2)
frame.SetMeta(&data.FrameMeta{Type: data.FrameTypeTimeSeriesMany})
query := &lokiQuery{
Expr: "up(ALERTS)",
QueryType: QueryTypeRange,
LegendFormat: "legend {{app}}",
Step: time.Second * 42,
}
adjustFrame(frame, query)
require.Equal(t, frame.Name, "legend Application")
require.Equal(t, frame.Meta.ExecutedQueryString, "Expr: up(ALERTS)\nStep: 42s")
require.Equal(t, frame.Fields[0].Config.Interval, float64(42000))
require.Equal(t, frame.Fields[1].Config.DisplayNameFromDS, "legend Application")
})
t.Run("should set interval-attribute in response", func(t *testing.T) {
query := &lokiQuery{
Step: time.Second * 42,
QueryType: QueryTypeRange,
}
field1 := data.NewField("", nil, make([]time.Time, 0))
field2 := data.NewField("", nil, make([]float64, 0))
frame := data.NewFrame("test", field1, field2)
frame.SetMeta(&data.FrameMeta{Type: data.FrameTypeTimeSeriesMany})
adjustFrame(frame, query)
// to keep the test simple, we assume the
// first field is the time-field
timeField := frame.Fields[0]
require.NotNil(t, timeField)
require.Equal(t, data.FieldTypeTime, timeField.Type())
timeFieldConfig := timeField.Config
require.NotNil(t, timeFieldConfig)
require.Equal(t, float64(42000), timeFieldConfig.Interval)
})
}

24
pkg/tsdb/loki/framing_test.go
View File

@ -22,17 +22,27 @@ import (
// but i wanted to test for all of them, to be sure.
func TestSuccessResponse(t *testing.T) {
matrixQuery := lokiQuery{Expr: "up(ALERTS)", Step: time.Second * 42, QueryType: QueryTypeRange}
vectorQuery := lokiQuery{Expr: "query1", QueryType: QueryTypeInstant}
streamsQuery := lokiQuery{Expr: "query1", QueryType: QueryTypeRange}
tt := []struct {
name string
filepath string
query lokiQuery
}{
{name: "parse a simple matrix response", filepath: "matrix_simple"},
{name: "parse a matrix response with a time-gap in the middle", filepath: "matrix_gap"},
{name: "parse a simple matrix response", filepath: "matrix_simple", query: matrixQuery},
{name: "parse a matrix response with a time-gap in the middle", filepath: "matrix_gap", query: matrixQuery},
// you can produce NaN by having a metric query and add ` / 0` to the end
{name: "parse a matrix response with NaN", filepath: "matrix_nan"},
{name: "parse a matrix response with NaN", filepath: "matrix_nan", query: matrixQuery},
// you can produce Infinity by using `quantile_over_time(42,` (value larger than 1)
{name: "parse a matrix response with Infinity", filepath: "matrix_inf"},
{name: "parse a matrix response with very small step value", filepath: "matrix_small_step"},
{name: "parse a matrix response with Infinity", filepath: "matrix_inf", query: matrixQuery},
{name: "parse a matrix response with very small step value", filepath: "matrix_small_step", query: matrixQuery},
{name: "parse a simple vector response", filepath: "vector_simple", query: vectorQuery},
{name: "parse a vector response with special values", filepath: "vector_special_values", query: vectorQuery},
{name: "parse a simple streams response", filepath: "streams_simple", query: streamsQuery},
}
for _, test := range tt {
@ -43,7 +53,7 @@ func TestSuccessResponse(t *testing.T) {
bytes, err := os.ReadFile(responseFileName)
require.NoError(t, err)
frames, err := runQuery(context.Background(), makeMockedAPI(200, "application/json", bytes), &lokiQuery{Expr: "up(ALERTS)", Step: time.Second * 42})
frames, err := runQuery(context.Background(), makeMockedAPI(200, "application/json", bytes), &test.query)
require.NoError(t, err)
dr := &backend.DataResponse{
@ -103,7 +113,7 @@ func TestErrorResponse(t *testing.T) {
for _, test := range tt {
t.Run(test.name, func(t *testing.T) {
frames, err := runQuery(context.Background(), makeMockedAPI(400, test.contentType, test.body), &lokiQuery{})
frames, err := runQuery(context.Background(), makeMockedAPI(400, test.contentType, test.body), &lokiQuery{QueryType: QueryTypeRange})
require.Len(t, frames, 0)
require.Error(t, err)

69
pkg/tsdb/loki/loki.go
View File

@ -5,8 +5,6 @@ import (
"fmt"
"net/http"
"regexp"
"strings"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
@ -15,10 +13,7 @@ import (
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/loki/pkg/loghttp"
"go.opentelemetry.io/otel/attribute"
"github.com/prometheus/common/model"
)
type Service struct {
@ -44,13 +39,15 @@ type datasourceInfo struct {
URL string
}
type QueryModel struct {
type QueryJSONModel struct {
QueryType string `json:"queryType"`
Expr string `json:"expr"`
LegendFormat string `json:"legendFormat"`
Interval string `json:"interval"`
IntervalMS int `json:"intervalMS"`
Resolution int64 `json:"resolution"`
MaxLines int `json:"maxLines"`
VolumeQuery bool `json:"volumeQuery"`
}
func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc {
@ -111,67 +108,9 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
return result, nil
}
//If legend (using of name or pattern instead of time series name) is used, use that name/pattern for formatting
func formatLegend(metric model.Metric, query *lokiQuery) string {
if query.LegendFormat == "" {
return metric.String()
}
result := legendFormat.ReplaceAllFunc([]byte(query.LegendFormat), func(in []byte) []byte {
labelName := strings.Replace(string(in), "{{", "", 1)
labelName = strings.Replace(labelName, "}}", "", 1)
labelName = strings.TrimSpace(labelName)
if val, exists := metric[model.LabelName(labelName)]; exists {
return []byte(val)
}
return []byte{}
})
return string(result)
}
func parseResponse(value *loghttp.QueryResponse, query *lokiQuery) (data.Frames, error) {
frames := data.Frames{}
//We are currently processing only matrix results (for alerting)
matrix, ok := value.Data.Result.(loghttp.Matrix)
if !ok {
return frames, fmt.Errorf("unsupported result format: %q", value.Data.ResultType)
}
for _, v := range matrix {
name := formatLegend(v.Metric, query)
tags := make(map[string]string, len(v.Metric))
timeVector := make([]time.Time, 0, len(v.Values))
values := make([]float64, 0, len(v.Values))
for k, v := range v.Metric {
tags[string(k)] = string(v)
}
for _, k := range v.Values {
timeVector = append(timeVector, k.Timestamp.Time().UTC())
values = append(values, float64(k.Value))
}
timeField := data.NewField("time", nil, timeVector)
timeField.Config = &data.FieldConfig{Interval: float64(query.Step.Milliseconds())}
valueField := data.NewField("value", tags, values).SetConfig(&data.FieldConfig{DisplayNameFromDS: name})
frame := data.NewFrame(name, timeField, valueField)
frame.SetMeta(&data.FrameMeta{
ExecutedQueryString: "Expr: " + query.Expr + "\n" + "Step: " + query.Step.String(),
})
frames = append(frames, frame)
}
return frames, nil
}
// we extracted this part of the functionality to make it easy to unit-test it
func runQuery(ctx context.Context, api *LokiAPI, query *lokiQuery) (data.Frames, error) {
value, err := api.QueryRange(ctx, *query)
value, err := api.Query(ctx, *query)
if err != nil {
return data.Frames{}, err
}

26
pkg/tsdb/loki/parse_query.go
View File

@ -2,6 +2,7 @@ package loki
import (
"encoding/json"
"fmt"
"math"
"strconv"
"strings"
@ -52,10 +53,25 @@ func interpolateVariables(expr string, interval time.Duration, timeRange time.Du
return expr
}
func parseQueryType(jsonValue string) (QueryType, error) {
switch jsonValue {
case "instant":
return QueryTypeInstant, nil
case "range":
return QueryTypeRange, nil
case "":
// there are older queries stored in alerting that did not have queryType,
// those were range-queries
return QueryTypeRange, nil
default:
return QueryTypeRange, fmt.Errorf("invalid queryType: %s", jsonValue)
}
}
func parseQuery(queryContext *backend.QueryDataRequest) ([]*lokiQuery, error) {
qs := []*lokiQuery{}
for _, query := range queryContext.Queries {
model := &QueryModel{}
model := &QueryJSONModel{}
err := json.Unmarshal(query.JSON, model)
if err != nil {
return nil, err
@ -76,13 +92,21 @@ func parseQuery(queryContext *backend.QueryDataRequest) ([]*lokiQuery, error) {
expr := interpolateVariables(model.Expr, interval, timeRange)
queryType, err := parseQueryType(model.QueryType)
if err != nil {
return nil, err
}
qs = append(qs, &lokiQuery{
Expr: expr,
QueryType: queryType,
Step: step,
MaxLines: model.MaxLines,
LegendFormat: model.LegendFormat,
Start: start,
End: end,
RefID: query.RefID,
VolumeQuery: model.VolumeQuery,
})
}

114
pkg/tsdb/loki/parse_response.go Normal file
View File

@ -0,0 +1,114 @@
package loki
import (
"fmt"
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/loki/pkg/loghttp"
)
func parseResponse(value *loghttp.QueryResponse, query *lokiQuery) (data.Frames, error) {
frames, err := lokiResponseToDataFrames(value, query)
if err != nil {
return nil, err
}
for _, frame := range frames {
adjustFrame(frame, query)
}
return frames, nil
}
func lokiResponseToDataFrames(value *loghttp.QueryResponse, query *lokiQuery) (data.Frames, error) {
switch res := value.Data.Result.(type) {
case loghttp.Matrix:
return lokiMatrixToDataFrames(res, query), nil
case loghttp.Vector:
return lokiVectorToDataFrames(res, query), nil
case loghttp.Streams:
return lokiStreamsToDataFrames(res, query), nil
default:
return nil, fmt.Errorf("resultType %T not supported{", res)
}
}
func lokiMatrixToDataFrames(matrix loghttp.Matrix, query *lokiQuery) data.Frames {
frames := data.Frames{}
for _, v := range matrix {
tags := make(map[string]string, len(v.Metric))
timeVector := make([]time.Time, 0, len(v.Values))
values := make([]float64, 0, len(v.Values))
for k, v := range v.Metric {
tags[string(k)] = string(v)
}
for _, k := range v.Values {
timeVector = append(timeVector, k.Timestamp.Time().UTC())
values = append(values, float64(k.Value))
}
timeField := data.NewField("", nil, timeVector)
valueField := data.NewField("", tags, values)
frame := data.NewFrame("", timeField, valueField)
frames = append(frames, frame)
}
return frames
}
func lokiVectorToDataFrames(vector loghttp.Vector, query *lokiQuery) data.Frames {
frames := data.Frames{}
for _, v := range vector {
tags := make(map[string]string, len(v.Metric))
timeVector := []time.Time{v.Timestamp.Time().UTC()}
values := []float64{float64(v.Value)}
for k, v := range v.Metric {
tags[string(k)] = string(v)
}
timeField := data.NewField("", nil, timeVector)
valueField := data.NewField("", tags, values)
frame := data.NewFrame("", timeField, valueField)
frames = append(frames, frame)
}
return frames
}
func lokiStreamsToDataFrames(streams loghttp.Streams, query *lokiQuery) data.Frames {
frames := data.Frames{}
for _, v := range streams {
tags := make(map[string]string, len(v.Labels))
timeVector := make([]time.Time, 0, len(v.Entries))
values := make([]string, 0, len(v.Entries))
for k, v := range v.Labels {
tags[k] = v
}
for _, k := range v.Entries {
timeVector = append(timeVector, k.Timestamp.UTC())
values = append(values, k.Line)
}
timeField := data.NewField("", nil, timeVector)
valueField := data.NewField("", tags, values)
frame := data.NewFrame("", timeField, valueField)
frames = append(frames, frame)
}
return frames
}

40
pkg/tsdb/loki/loki_test.go → pkg/tsdb/loki/parse_response_test.go
View File

@ -11,45 +11,15 @@ import (
"github.com/stretchr/testify/require"
)
func TestLoki(t *testing.T) {
t.Run("converting metric name", func(t *testing.T) {
metric := map[p.LabelName]p.LabelValue{
p.LabelName("app"): p.LabelValue("backend"),
p.LabelName("device"): p.LabelValue("mobile"),
}
query := &lokiQuery{
LegendFormat: "legend {{app}} {{ device }} {{broken}}",
}
require.Equal(t, "legend backend mobile ", formatLegend(metric, query))
})
t.Run("build full series name", func(t *testing.T) {
metric := map[p.LabelName]p.LabelValue{
p.LabelName(p.MetricNameLabel): p.LabelValue("http_request_total"),
p.LabelName("app"): p.LabelValue("backend"),
p.LabelName("device"): p.LabelValue("mobile"),
}
query := &lokiQuery{
LegendFormat: "",
}
require.Equal(t, `http_request_total{app="backend", device="mobile"}`, formatLegend(metric, query))
})
}
func TestParseResponse(t *testing.T) {
t.Run("value is not of type matrix", func(t *testing.T) {
queryRes := data.Frames{}
t.Run("value is not of supported type", func(t *testing.T) {
value := loghttp.QueryResponse{
Data: loghttp.QueryResponseData{
Result: loghttp.Vector{},
Result: loghttp.Scalar{},
},
}
res, err := parseResponse(&value, nil)
require.Equal(t, queryRes, res)
require.Equal(t, len(res), 0)
require.Error(t, err)
})
@ -74,6 +44,7 @@ func TestParseResponse(t *testing.T) {
query := &lokiQuery{
Expr: "up(ALERTS)",
QueryType: QueryTypeRange,
LegendFormat: "legend {{app}}",
Step: time.Second * 42,
}
@ -117,7 +88,8 @@ func TestParseResponse(t *testing.T) {
}
query := &lokiQuery{
Step: time.Second * 42,
Step: time.Second * 42,
QueryType: QueryTypeRange,
}
frames, err := parseResponse(&value, query)

37
pkg/tsdb/loki/testdata/streams_simple.golden.txt vendored Normal file
View File

@ -0,0 +1,37 @@
🌟 This was machine generated. Do not edit. 🌟
Frame[0] {
"executedQueryString": "Expr: query1"
}
Name: {level="error", location="moon"}
Dimensions: 2 Fields by 1 Rows
+----------------------------------------+------------------------------------+
| Name: time | Name: value |
| Labels: | Labels: level=error, location=moon |
| Type: []time.Time | Type: []string |
+----------------------------------------+------------------------------------+
| 2022-02-16 16:50:44.81075712 +0000 UTC | log line error 1 |
+----------------------------------------+------------------------------------+
Frame[1] {
"executedQueryString": "Expr: query1"
}
Name: {level="info", location="moon"}
Dimensions: 2 Fields by 4 Rows
+-----------------------------------------+-----------------------------------+
| Name: time | Name: value |
| Labels: | Labels: level=info, location=moon |
| Type: []time.Time | Type: []string |
+-----------------------------------------+-----------------------------------+
| 2022-02-16 16:50:47.02773504 +0000 UTC | log line info 1 |
| 2022-02-16 16:50:46.277587968 +0000 UTC | log line info 2 |
| 2022-02-16 16:50:45.539423744 +0000 UTC | log line info 3 |
| 2022-02-16 16:50:44.091700992 +0000 UTC | log line info 4 |
+-----------------------------------------+-----------------------------------+
====== TEST DATA RESPONSE (arrow base64) ======
FRAME=QVJST1cxAAD/////cAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAAbAAAACgAAAAEAAAAIP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABA/v//CAAAACwAAAAgAAAAe2xldmVsPSJlcnJvciIsIGxvY2F0aW9uPSJtb29uIn0AAAAABAAAAG5hbWUAAAAAgP7//wgAAAAwAAAAJgAAAHsiZXhlY3V0ZWRRdWVyeVN0cmluZyI6IkV4cHI6IHF1ZXJ5MSJ9AAAEAAAAbWV0YQAAAAACAAAAGAEAAAQAAAAC////FAAAAOAAAADkAAAAAAAABeAAAAADAAAAcAAAACwAAAAEAAAA+P7//wgAAAAQAAAABQAAAHZhbHVlAAAABAAAAG5hbWUAAAAAHP///wgAAAAsAAAAIwAAAHsibGV2ZWwiOiJlcnJvciIsImxvY2F0aW9uIjoibW9vbiJ9AAYAAABsYWJlbHMAAFz///8IAAAASAAAADwAAAB7ImRpc3BsYXlOYW1lRnJvbURTIjoie2xldmVsPVwiZXJyb3JcIiwgbG9jYXRpb249XCJtb29uXCJ9In0AAAAABgAAAGNvbmZpZwAAAAAAAAQABAAEAAAABQAAAHZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEwAAAAAAAAKTAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAdGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAAB0aW1lAAAAAP/////IAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAIAAAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAaAAAAAEAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAAAAAACAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAFLi6SlLUFgAAAAAQAAAAbG9nIGxpbmUgZXJyb3IgMRAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA8AAAAAAAEAAEAAACAAgAAAAAAANAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAAbAAAACgAAAAEAAAAIP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABA/v//CAAAACwAAAAgAAAAe2xldmVsPSJlcnJvciIsIGxvY2F0aW9uPSJtb29uIn0AAAAABAAAAG5hbWUAAAAAgP7//wgAAAAwAAAAJgAAAHsiZXhlY3V0ZWRRdWVyeVN0cmluZyI6IkV4cHI6IHF1ZXJ5MSJ9AAAEAAAAbWV0YQAAAAACAAAAGAEAAAQAAAAC////FAAAAOAAAADkAAAAAAAABeAAAAADAAAAcAAAACwAAAAEAAAA+P7//wgAAAAQAAAABQAAAHZhbHVlAAAABAAAAG5hbWUAAAAAHP///wgAAAAsAAAAIwAAAHsibGV2ZWwiOiJlcnJvciIsImxvY2F0aW9uIjoibW9vbiJ9AAYAAABsYWJlbHMAAFz///8IAAAASAAAADwAAAB7ImRpc3BsYXlOYW1lRnJvbURTIjoie2xldmVsPVwiZXJyb3JcIiwgbG9jYXRpb249XCJtb29uXCJ9In0AAAAABgAAAGNvbmZpZwAAAAAAAAQABAAEAAAABQAAAHZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEwAAAAAAAAKTAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAdGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAAB0aW1lAAAAAKACAABBUlJPVzE=
FRAME=QVJST1cxAAD/////aAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALQAAAADAAAAaAAAACgAAAAEAAAAKP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABI/v//CAAAACgAAAAfAAAAe2xldmVsPSJpbmZvIiwgbG9jYXRpb249Im1vb24ifQAEAAAAbmFtZQAAAACE/v//CAAAADAAAAAmAAAAeyJleGVjdXRlZFF1ZXJ5U3RyaW5nIjoiRXhwcjogcXVlcnkxIn0AAAQAAABtZXRhAAAAAAIAAAAUAQAABAAAAAb///8UAAAA3AAAAOAAAAAAAAAF3AAAAAMAAABwAAAALAAAAAQAAAD8/v//CAAAABAAAAAFAAAAdmFsdWUAAAAEAAAAbmFtZQAAAAAg////CAAAACwAAAAiAAAAeyJsZXZlbCI6ImluZm8iLCJsb2NhdGlvbiI6Im1vb24ifQAABgAAAGxhYmVscwAAYP///wgAAABEAAAAOwAAAHsiZGlzcGxheU5hbWVGcm9tRFMiOiJ7bGV2ZWw9XCJpbmZvXCIsIGxvY2F0aW9uPVwibW9vblwifSJ9AAYAAABjb25maWcAAAAAAAAEAAQABAAAAAUAAAB2YWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAAD/////yAAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAAHgAAAAAAAAAFAAAAAAAAAMEAAoAGAAMAAgABAAKAAAAFAAAAGgAAAAEAAAAAAAAAAAAAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAUAAAAAAAAADgAAAAAAAAAPAAAAAAAAAAAAAAAAgAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAHrcPktS1BYAJCYSS1LUFgCmJuZKUtQWACfcj0pS1BYAAAAADwAAAB4AAAAtAAAAPAAAAAAAAABsb2cgbGluZSBpbmZvIDFsb2cgbGluZSBpbmZvIDJsb2cgbGluZSBpbmZvIDNsb2cgbGluZSBpbmZvIDQAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAQAAQAAAHgCAAAAAAAA0AAAAAAAAAB4AAAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAtAAAAAMAAABoAAAAKAAAAAQAAAAo/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAEj+//8IAAAAKAAAAB8AAAB7bGV2ZWw9ImluZm8iLCBsb2NhdGlvbj0ibW9vbiJ9AAQAAABuYW1lAAAAAIT+//8IAAAAMAAAACYAAAB7ImV4ZWN1dGVkUXVlcnlTdHJpbmciOiJFeHByOiBxdWVyeTEifQAABAAAAG1ldGEAAAAAAgAAABQBAAAEAAAABv///xQAAADcAAAA4AAAAAAAAAXcAAAAAwAAAHAAAAAsAAAABAAAAPz+//8IAAAAEAAAAAUAAAB2YWx1ZQAAAAQAAABuYW1lAAAAACD///8IAAAALAAAACIAAAB7ImxldmVsIjoiaW5mbyIsImxvY2F0aW9uIjoibW9vbiJ9AAAGAAAAbGFiZWxzAABg////CAAAAEQAAAA7AAAAeyJkaXNwbGF5TmFtZUZyb21EUyI6IntsZXZlbD1cImluZm9cIiwgbG9jYXRpb249XCJtb29uXCJ9In0ABgAAAGNvbmZpZwAAAAAAAAQABAAEAAAABQAAAHZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEwAAAAAAAAKTAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAdGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAAB0aW1lAAAAAJgCAABBUlJPVzE=

76
pkg/tsdb/loki/testdata/streams_simple.json vendored Normal file
View File

@ -0,0 +1,76 @@
{
"status": "success",
"data": {
"resultType": "streams",
"result": [
{
"stream": {
"level": "error",
"location": "moon"
},
"values": [
[
"1645030244810757120",
"log line error 1"
]
]
},
{
"stream": {
"level": "info",
"location": "moon"
},
"values": [
[
"1645030247027735040",
"log line info 1"
],
[
"1645030246277587968",
"log line info 2"
],
[
"1645030245539423744",
"log line info 3"
],
[
"1645030244091700992",
"log line info 4"
]
]
}
],
"stats": {
"summary": {
"bytesProcessedPerSecond": 3507022,
"linesProcessedPerSecond": 24818,
"totalBytesProcessed": 7772,
"totalLinesProcessed": 55,
"execTime": 0.002216125
},
"store": {
"totalChunksRef": 2,
"totalChunksDownloaded": 2,
"chunksDownloadTime": 0.000390958,
"headChunkBytes": 0,
"headChunkLines": 0,
"decompressedBytes": 7772,
"decompressedLines": 55,
"compressedBytes": 31432,
"totalDuplicates": 0
},
"ingester": {
"totalReached": 0,
"totalChunksMatched": 0,
"totalBatches": 0,
"totalLinesSent": 0,
"headChunkBytes": 0,
"headChunkLines": 0,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0
}
}
}
}

34
pkg/tsdb/loki/testdata/vector_simple.golden.txt vendored Normal file
View File

@ -0,0 +1,34 @@
🌟 This was machine generated. Do not edit. 🌟
Frame[0] {
"executedQueryString": "Expr: query1"
}
Name: {level="error", location="moon"}
Dimensions: 2 Fields by 1 Rows
+-------------------------------+------------------------------------+
| Name: time | Name: value |
| Labels: | Labels: level=error, location=moon |
| Type: []time.Time | Type: []float64 |
+-------------------------------+------------------------------------+
| 2022-02-16 16:41:39 +0000 UTC | 23 |
+-------------------------------+------------------------------------+
Frame[1] {
"executedQueryString": "Expr: query1"
}
Name: {level="info", location="moon"}
Dimensions: 2 Fields by 1 Rows
+-------------------------------+-----------------------------------+
| Name: time | Name: value |
| Labels: | Labels: level=info, location=moon |
| Type: []time.Time | Type: []float64 |
+-------------------------------+-----------------------------------+
| 2022-02-16 16:41:39 +0000 UTC | 47 |
+-------------------------------+-----------------------------------+
====== TEST DATA RESPONSE (arrow base64) ======
FRAME=QVJST1cxAAD/////cAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAAbAAAACgAAAAEAAAAIP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABA/v//CAAAACwAAAAgAAAAe2xldmVsPSJlcnJvciIsIGxvY2F0aW9uPSJtb29uIn0AAAAABAAAAG5hbWUAAAAAgP7//wgAAAAwAAAAJgAAAHsiZXhlY3V0ZWRRdWVyeVN0cmluZyI6IkV4cHI6IHF1ZXJ5MSJ9AAAEAAAAbWV0YQAAAAACAAAAGAEAAAQAAAAC////FAAAAOAAAADgAAAAAAAAA+AAAAADAAAAcAAAACwAAAAEAAAA+P7//wgAAAAQAAAABQAAAHZhbHVlAAAABAAAAG5hbWUAAAAAHP///wgAAAAsAAAAIwAAAHsibGV2ZWwiOiJlcnJvciIsImxvY2F0aW9uIjoibW9vbiJ9AAYAAABsYWJlbHMAAFz///8IAAAASAAAADwAAAB7ImRpc3BsYXlOYW1lRnJvbURTIjoie2xldmVsPVwiZXJyb3JcIiwgbG9jYXRpb249XCJtb29uXCJ9In0AAAAABgAAAGNvbmZpZwAAAAAAAIr///8AAAIABQAAAHZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEwAAAAAAAAKTAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAdGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAAB0aW1lAAAAAP////+4AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAEAAAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAWAAAAAEAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAADe3KXLUdQWAAAAAAAAN0AQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAABAABAAAAgAIAAAAAAADAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAAC4AAAAAwAAAGwAAAAoAAAABAAAACD+//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAAQP7//wgAAAAsAAAAIAAAAHtsZXZlbD0iZXJyb3IiLCBsb2NhdGlvbj0ibW9vbiJ9AAAAAAQAAABuYW1lAAAAAID+//8IAAAAMAAAACYAAAB7ImV4ZWN1dGVkUXVlcnlTdHJpbmciOiJFeHByOiBxdWVyeTEifQAABAAAAG1ldGEAAAAAAgAAABgBAAAEAAAAAv///xQAAADgAAAA4AAAAAAAAAPgAAAAAwAAAHAAAAAsAAAABAAAAPj+//8IAAAAEAAAAAUAAAB2YWx1ZQAAAAQAAABuYW1lAAAAABz///8IAAAALAAAACMAAAB7ImxldmVsIjoiZXJyb3IiLCJsb2NhdGlvbiI6Im1vb24ifQAGAAAAbGFiZWxzAABc////CAAAAEgAAAA8AAAAeyJkaXNwbGF5TmFtZUZyb21EUyI6IntsZXZlbD1cImVycm9yXCIsIGxvY2F0aW9uPVwibW9vblwifSJ9AAAAAAYAAABjb25maWcAAAAAAACK////AAACAAUAAAB2YWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAACgAgAAQVJST1cx
FRAME=QVJST1cxAAD/////aAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALQAAAADAAAAaAAAACgAAAAEAAAAKP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABI/v//CAAAACgAAAAfAAAAe2xldmVsPSJpbmZvIiwgbG9jYXRpb249Im1vb24ifQAEAAAAbmFtZQAAAACE/v//CAAAADAAAAAmAAAAeyJleGVjdXRlZFF1ZXJ5U3RyaW5nIjoiRXhwcjogcXVlcnkxIn0AAAQAAABtZXRhAAAAAAIAAAAUAQAABAAAAAb///8UAAAA3AAAANwAAAAAAAAD3AAAAAMAAABwAAAALAAAAAQAAAD8/v//CAAAABAAAAAFAAAAdmFsdWUAAAAEAAAAbmFtZQAAAAAg////CAAAACwAAAAiAAAAeyJsZXZlbCI6ImluZm8iLCJsb2NhdGlvbiI6Im1vb24ifQAABgAAAGxhYmVscwAAYP///wgAAABEAAAAOwAAAHsiZGlzcGxheU5hbWVGcm9tRFMiOiJ7bGV2ZWw9XCJpbmZvXCIsIGxvY2F0aW9uPVwibW9vblwifSJ9AAYAAABjb25maWcAAAAAAACK////AAACAAUAAAB2YWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAAD/////uAAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAABAAAAAAAAAAFAAAAAAAAAMEAAoAGAAMAAgABAAKAAAAFAAAAFgAAAABAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAACAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAA3tyly1HUFgAAAAAAgEdAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAQAAQAAAHgCAAAAAAAAwAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAtAAAAAMAAABoAAAAKAAAAAQAAAAo/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAEj+//8IAAAAKAAAAB8AAAB7bGV2ZWw9ImluZm8iLCBsb2NhdGlvbj0ibW9vbiJ9AAQAAABuYW1lAAAAAIT+//8IAAAAMAAAACYAAAB7ImV4ZWN1dGVkUXVlcnlTdHJpbmciOiJFeHByOiBxdWVyeTEifQAABAAAAG1ldGEAAAAAAgAAABQBAAAEAAAABv///xQAAADcAAAA3AAAAAAAAAPcAAAAAwAAAHAAAAAsAAAABAAAAPz+//8IAAAAEAAAAAUAAAB2YWx1ZQAAAAQAAABuYW1lAAAAACD///8IAAAALAAAACIAAAB7ImxldmVsIjoiaW5mbyIsImxvY2F0aW9uIjoibW9vbiJ9AAAGAAAAbGFiZWxzAABg////CAAAAEQAAAA7AAAAeyJkaXNwbGF5TmFtZUZyb21EUyI6IntsZXZlbD1cImluZm9cIiwgbG9jYXRpb249XCJtb29uXCJ9In0ABgAAAGNvbmZpZwAAAAAAAIr///8AAAIABQAAAHZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEwAAAAAAAAKTAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAdGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAAB0aW1lAAAAAJgCAABBUlJPVzE=

16
pkg/tsdb/loki/testdata/vector_simple.json vendored Normal file
View File

@ -0,0 +1,16 @@
{
"status": "success",
"data": {
"resultType": "vector",
"result": [
{
"metric": { "level": "error", "location": "moon"},
"value": [1645029699, "23"]
},
{
"metric": { "level": "info", "location": "moon" },
"value": [1645029699, "47"]
}
]
}
}

50
pkg/tsdb/loki/testdata/vector_special_values.golden.txt vendored Normal file
View File

@ -0,0 +1,50 @@
🌟 This was machine generated. Do not edit. 🌟
Frame[0] {
"executedQueryString": "Expr: query1"
}
Name: {level="error", location="moon"}
Dimensions: 2 Fields by 1 Rows
+-------------------------------+------------------------------------+
| Name: time | Name: value |
| Labels: | Labels: level=error, location=moon |
| Type: []time.Time | Type: []float64 |
+-------------------------------+------------------------------------+
| 2022-02-16 16:41:39 +0000 UTC | +Inf |
+-------------------------------+------------------------------------+
Frame[1] {
"executedQueryString": "Expr: query1"
}
Name: {level="info", location="moon"}
Dimensions: 2 Fields by 1 Rows
+-------------------------------+-----------------------------------+
| Name: time | Name: value |
| Labels: | Labels: level=info, location=moon |
| Type: []time.Time | Type: []float64 |
+-------------------------------+-----------------------------------+
| 2022-02-16 16:41:39 +0000 UTC | -Inf |
+-------------------------------+-----------------------------------+
Frame[2] {
"executedQueryString": "Expr: query1"
}
Name: {level="debug", location="moon"}
Dimensions: 2 Fields by 1 Rows
+-------------------------------+------------------------------------+
| Name: time | Name: value |
| Labels: | Labels: level=debug, location=moon |
| Type: []time.Time | Type: []float64 |
+-------------------------------+------------------------------------+
| 2022-02-16 16:41:39 +0000 UTC | NaN |
+-------------------------------+------------------------------------+
====== TEST DATA RESPONSE (arrow base64) ======
FRAME=QVJST1cxAAD/////cAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAAbAAAACgAAAAEAAAAIP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABA/v//CAAAACwAAAAgAAAAe2xldmVsPSJlcnJvciIsIGxvY2F0aW9uPSJtb29uIn0AAAAABAAAAG5hbWUAAAAAgP7//wgAAAAwAAAAJgAAAHsiZXhlY3V0ZWRRdWVyeVN0cmluZyI6IkV4cHI6IHF1ZXJ5MSJ9AAAEAAAAbWV0YQAAAAACAAAAGAEAAAQAAAAC////FAAAAOAAAADgAAAAAAAAA+AAAAADAAAAcAAAACwAAAAEAAAA+P7//wgAAAAQAAAABQAAAHZhbHVlAAAABAAAAG5hbWUAAAAAHP///wgAAAAsAAAAIwAAAHsibGV2ZWwiOiJlcnJvciIsImxvY2F0aW9uIjoibW9vbiJ9AAYAAABsYWJlbHMAAFz///8IAAAASAAAADwAAAB7ImRpc3BsYXlOYW1lRnJvbURTIjoie2xldmVsPVwiZXJyb3JcIiwgbG9jYXRpb249XCJtb29uXCJ9In0AAAAABgAAAGNvbmZpZwAAAAAAAIr///8AAAIABQAAAHZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEwAAAAAAAAKTAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAdGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAAB0aW1lAAAAAP////+4AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAEAAAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAWAAAAAEAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAADe3KXLUdQWAAAAAAAA8H8QAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAABAABAAAAgAIAAAAAAADAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAAC4AAAAAwAAAGwAAAAoAAAABAAAACD+//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAAQP7//wgAAAAsAAAAIAAAAHtsZXZlbD0iZXJyb3IiLCBsb2NhdGlvbj0ibW9vbiJ9AAAAAAQAAABuYW1lAAAAAID+//8IAAAAMAAAACYAAAB7ImV4ZWN1dGVkUXVlcnlTdHJpbmciOiJFeHByOiBxdWVyeTEifQAABAAAAG1ldGEAAAAAAgAAABgBAAAEAAAAAv///xQAAADgAAAA4AAAAAAAAAPgAAAAAwAAAHAAAAAsAAAABAAAAPj+//8IAAAAEAAAAAUAAAB2YWx1ZQAAAAQAAABuYW1lAAAAABz///8IAAAALAAAACMAAAB7ImxldmVsIjoiZXJyb3IiLCJsb2NhdGlvbiI6Im1vb24ifQAGAAAAbGFiZWxzAABc////CAAAAEgAAAA8AAAAeyJkaXNwbGF5TmFtZUZyb21EUyI6IntsZXZlbD1cImVycm9yXCIsIGxvY2F0aW9uPVwibW9vblwifSJ9AAAAAAYAAABjb25maWcAAAAAAACK////AAACAAUAAAB2YWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAACgAgAAQVJST1cx
FRAME=QVJST1cxAAD/////aAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALQAAAADAAAAaAAAACgAAAAEAAAAKP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABI/v//CAAAACgAAAAfAAAAe2xldmVsPSJpbmZvIiwgbG9jYXRpb249Im1vb24ifQAEAAAAbmFtZQAAAACE/v//CAAAADAAAAAmAAAAeyJleGVjdXRlZFF1ZXJ5U3RyaW5nIjoiRXhwcjogcXVlcnkxIn0AAAQAAABtZXRhAAAAAAIAAAAUAQAABAAAAAb///8UAAAA3AAAANwAAAAAAAAD3AAAAAMAAABwAAAALAAAAAQAAAD8/v//CAAAABAAAAAFAAAAdmFsdWUAAAAEAAAAbmFtZQAAAAAg////CAAAACwAAAAiAAAAeyJsZXZlbCI6ImluZm8iLCJsb2NhdGlvbiI6Im1vb24ifQAABgAAAGxhYmVscwAAYP///wgAAABEAAAAOwAAAHsiZGlzcGxheU5hbWVGcm9tRFMiOiJ7bGV2ZWw9XCJpbmZvXCIsIGxvY2F0aW9uPVwibW9vblwifSJ9AAYAAABjb25maWcAAAAAAACK////AAACAAUAAAB2YWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAAD/////uAAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAABAAAAAAAAAAFAAAAAAAAAMEAAoAGAAMAAgABAAKAAAAFAAAAFgAAAABAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAACAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAA3tyly1HUFgAAAAAAAPD/EAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAQAAQAAAHgCAAAAAAAAwAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAtAAAAAMAAABoAAAAKAAAAAQAAAAo/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAEj+//8IAAAAKAAAAB8AAAB7bGV2ZWw9ImluZm8iLCBsb2NhdGlvbj0ibW9vbiJ9AAQAAABuYW1lAAAAAIT+//8IAAAAMAAAACYAAAB7ImV4ZWN1dGVkUXVlcnlTdHJpbmciOiJFeHByOiBxdWVyeTEifQAABAAAAG1ldGEAAAAAAgAAABQBAAAEAAAABv///xQAAADcAAAA3AAAAAAAAAPcAAAAAwAAAHAAAAAsAAAABAAAAPz+//8IAAAAEAAAAAUAAAB2YWx1ZQAAAAQAAABuYW1lAAAAACD///8IAAAALAAAACIAAAB7ImxldmVsIjoiaW5mbyIsImxvY2F0aW9uIjoibW9vbiJ9AAAGAAAAbGFiZWxzAABg////CAAAAEQAAAA7AAAAeyJkaXNwbGF5TmFtZUZyb21EUyI6IntsZXZlbD1cImluZm9cIiwgbG9jYXRpb249XCJtb29uXCJ9In0ABgAAAGNvbmZpZwAAAAAAAIr///8AAAIABQAAAHZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEwAAAAAAAAKTAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAdGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAAB0aW1lAAAAAJgCAABBUlJPVzE=
FRAME=QVJST1cxAAD/////cAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAAbAAAACgAAAAEAAAAIP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABA/v//CAAAACwAAAAgAAAAe2xldmVsPSJkZWJ1ZyIsIGxvY2F0aW9uPSJtb29uIn0AAAAABAAAAG5hbWUAAAAAgP7//wgAAAAwAAAAJgAAAHsiZXhlY3V0ZWRRdWVyeVN0cmluZyI6IkV4cHI6IHF1ZXJ5MSJ9AAAEAAAAbWV0YQAAAAACAAAAGAEAAAQAAAAC////FAAAAOAAAADgAAAAAAAAA+AAAAADAAAAcAAAACwAAAAEAAAA+P7//wgAAAAQAAAABQAAAHZhbHVlAAAABAAAAG5hbWUAAAAAHP///wgAAAAsAAAAIwAAAHsibGV2ZWwiOiJkZWJ1ZyIsImxvY2F0aW9uIjoibW9vbiJ9AAYAAABsYWJlbHMAAFz///8IAAAASAAAADwAAAB7ImRpc3BsYXlOYW1lRnJvbURTIjoie2xldmVsPVwiZGVidWdcIiwgbG9jYXRpb249XCJtb29uXCJ9In0AAAAABgAAAGNvbmZpZwAAAAAAAIr///8AAAIABQAAAHZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEwAAAAAAAAKTAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAdGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAAB0aW1lAAAAAP////+4AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAEAAAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAWAAAAAEAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAADe3KXLUdQWAQAAAAAA+H8QAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAABAABAAAAgAIAAAAAAADAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAAC4AAAAAwAAAGwAAAAoAAAABAAAACD+//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAAQP7//wgAAAAsAAAAIAAAAHtsZXZlbD0iZGVidWciLCBsb2NhdGlvbj0ibW9vbiJ9AAAAAAQAAABuYW1lAAAAAID+//8IAAAAMAAAACYAAAB7ImV4ZWN1dGVkUXVlcnlTdHJpbmciOiJFeHByOiBxdWVyeTEifQAABAAAAG1ldGEAAAAAAgAAABgBAAAEAAAAAv///xQAAADgAAAA4AAAAAAAAAPgAAAAAwAAAHAAAAAsAAAABAAAAPj+//8IAAAAEAAAAAUAAAB2YWx1ZQAAAAQAAABuYW1lAAAAABz///8IAAAALAAAACMAAAB7ImxldmVsIjoiZGVidWciLCJsb2NhdGlvbiI6Im1vb24ifQAGAAAAbGFiZWxzAABc////CAAAAEgAAAA8AAAAeyJkaXNwbGF5TmFtZUZyb21EUyI6IntsZXZlbD1cImRlYnVnXCIsIGxvY2F0aW9uPVwibW9vblwifSJ9AAAAAAYAAABjb25maWcAAAAAAACK////AAACAAUAAAB2YWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAACgAgAAQVJST1cx

20
pkg/tsdb/loki/testdata/vector_special_values.json vendored Normal file
View File

@ -0,0 +1,20 @@
{
"status": "success",
"data": {
"resultType": "vector",
"result": [
{
"metric": { "level": "error", "location": "moon"},
"value": [1645029699, "+Inf"]
},
{
"metric": { "level": "info", "location": "moon" },
"value": [1645029699, "-Inf"]
},
{
"metric": { "level": "debug", "location": "moon" },
"value": [1645029699, "NaN"]
}
]
}
}

10
pkg/tsdb/loki/types.go
View File

@ -2,11 +2,21 @@ package loki
import "time"
type QueryType string
const (
QueryTypeRange QueryType = "range"
QueryTypeInstant QueryType = "instant"
)
type lokiQuery struct {
Expr string
QueryType QueryType
Step time.Duration
MaxLines int
LegendFormat string
Start time.Time
End time.Time
RefID string
VolumeQuery bool
}

99
public/app/plugins/datasource/loki/backendResultTransformer.ts Normal file
View File

@ -0,0 +1,99 @@
import { DataQueryRequest, DataQueryResponse, DataFrame, isDataFrame, FieldType, QueryResultMeta } from '@grafana/data';
import { LokiQuery, LokiQueryType } from './types';
import { makeTableFrames } from './makeTableFrames';
import { formatQuery, getHighlighterExpressionsFromQuery } from './query_utils';
function isMetricFrame(frame: DataFrame): boolean {
return frame.fields.every((field) => field.type === FieldType.time || field.type === FieldType.number);
}
// returns a new frame, with meta merged with it's original meta
function setFrameMeta(frame: DataFrame, meta: QueryResultMeta): DataFrame {
const { meta: oldMeta, ...rest } = frame;
// meta maybe be undefined, we need to handle that
const newMeta = { ...oldMeta, ...meta };
return {
...rest,
meta: newMeta,
};
}
function processStreamsFrames(frames: DataFrame[], queryMap: Map<string, LokiQuery>): DataFrame[] {
return frames.map((frame) => {
const query = frame.refId !== undefined ? queryMap.get(frame.refId) : undefined;
const meta: QueryResultMeta = {
preferredVisualisationType: 'logs',
searchWords: query !== undefined ? getHighlighterExpressionsFromQuery(formatQuery(query.expr)) : undefined,
};
return setFrameMeta(frame, meta);
});
}
function processMetricInstantFrames(frames: DataFrame[]): DataFrame[] {
return frames.length > 0 ? makeTableFrames(frames) : [];
}
function processMetricRangeFrames(frames: DataFrame[]): DataFrame[] {
const meta: QueryResultMeta = { preferredVisualisationType: 'graph' };
return frames.map((frame) => setFrameMeta(frame, meta));
}
// we split the frames into 3 groups, because we will handle
// each group slightly differently
function groupFrames(
frames: DataFrame[],
queryMap: Map<string, LokiQuery>
): {
streamsFrames: DataFrame[];
metricInstantFrames: DataFrame[];
metricRangeFrames: DataFrame[];
} {
const streamsFrames: DataFrame[] = [];
const metricInstantFrames: DataFrame[] = [];
const metricRangeFrames: DataFrame[] = [];
frames.forEach((frame) => {
if (!isMetricFrame(frame)) {
streamsFrames.push(frame);
} else {
const isInstantFrame = frame.refId != null && queryMap.get(frame.refId)?.queryType === LokiQueryType.Instant;
if (isInstantFrame) {
metricInstantFrames.push(frame);
} else {
metricRangeFrames.push(frame);
}
}
});
return { streamsFrames, metricInstantFrames, metricRangeFrames };
}
export function transformBackendResult(
response: DataQueryResponse,
request: DataQueryRequest<LokiQuery>
): DataQueryResponse {
const { data, ...rest } = response;
// in the typescript type, data is an array of basically anything.
// we do know that they have to be dataframes, so we make a quick check,
// this way we can be sure, and also typescript is happy.
const dataFrames = data.map((d) => {
if (!isDataFrame(d)) {
throw new Error('transformation only supports dataframe responses');
}
return d;
});
const queryMap = new Map(request.targets.map((query) => [query.refId, query]));
const { streamsFrames, metricInstantFrames, metricRangeFrames } = groupFrames(dataFrames, queryMap);
return {
...rest,
data: [
...processMetricRangeFrames(metricRangeFrames),
...processMetricInstantFrames(metricInstantFrames),
...processStreamsFrames(streamsFrames, queryMap),
],
};
}

17
public/app/plugins/datasource/loki/datasource.ts
View File

@ -44,6 +44,7 @@ import {
lokiStreamsToDataFrames,
processRangeQueryResponse,
} from './result_transformer';
import { transformBackendResult } from './backendResultTransformer';
import { addParsedLabelToQuery, getNormalizedLokiQuery, queryHasPipeParser } from './query_utils';
import {
@ -153,19 +154,7 @@ export class LokiDatasource
...this.getRangeScopedVars(request.range),
};
// if all these are true, run query through backend:
// - feature-flag is enabled
// - we are in explore-mode
// - for every query it is true that:
// - query is range query
// - and query is metric query
// - and query is not a log-volume-query (those need a custom http header)
const shouldRunBackendQuery =
config.featureToggles.lokiBackendMode &&
request.app === CoreApp.Explore &&
request.targets.every(
(query) => query.queryType === LokiQueryType.Range && isMetricsQuery(query.expr) && !query.volumeQuery
);
const shouldRunBackendQuery = config.featureToggles.lokiBackendMode && request.app === CoreApp.Explore;
if (shouldRunBackendQuery) {
// we "fix" the loki queries to have `.queryType` and not have `.instant` and `.range`
@ -173,7 +162,7 @@ export class LokiDatasource
...request,
targets: request.targets.map(getNormalizedLokiQuery),
};
return super.query(fixedRequest);
return super.query(fixedRequest).pipe(map((response) => transformBackendResult(response, fixedRequest)));
}
const filteredTargets = request.targets

146
public/app/plugins/datasource/loki/makeTableFrames.test.ts Normal file
View File

@ -0,0 +1,146 @@
import { ArrayVector, DataFrame, FieldType } from '@grafana/data';
import { makeTableFrames } from './makeTableFrames';
const frame1: DataFrame = {
name: 'frame1',
refId: 'A',
meta: {
executedQueryString: 'something1',
},
fields: [
{
name: 'Time',
type: FieldType.time,
config: {},
values: new ArrayVector([1645029699311]),
},
{
name: 'Value',
type: FieldType.number,
labels: {
level: 'error',
location: 'moon',
protocol: 'http',
},
config: {
displayNameFromDS: '{level="error", location="moon", protocol="http"}',
},
values: new ArrayVector([23]),
},
],
length: 1,
};
const frame2: DataFrame = {
name: 'frame1',
refId: 'A',
meta: {
executedQueryString: 'something1',
},
fields: [
{
name: 'Time',
type: FieldType.time,
config: {},
values: new ArrayVector([1645029699311]),
},
{
name: 'Value',
type: FieldType.number,
labels: {
level: 'info',
location: 'moon',
protocol: 'http',
},
config: {
displayNameFromDS: '{level="info", location="moon", protocol="http"}',
},
values: new ArrayVector([45]),
},
],
length: 1,
};
const frame3: DataFrame = {
name: 'frame1',
refId: 'B',
meta: {
executedQueryString: 'something1',
},
fields: [
{
name: 'Time',
type: FieldType.time,
config: {},
values: new ArrayVector([1645029699311]),
},
{
name: 'Value',
type: FieldType.number,
labels: {
level: 'error',
location: 'moon',
protocol: 'http',
},
config: {
displayNameFromDS: '{level="error", location="moon", protocol="http"}',
},
values: new ArrayVector([72]),
},
],
length: 1,
};
const outputSingle = [
{
fields: [
{ config: {}, name: 'Time', type: 'time', values: new ArrayVector([1645029699311]) },
{ config: { filterable: true }, name: 'level', type: 'string', values: new ArrayVector(['error']) },
{ config: { filterable: true }, name: 'location', type: 'string', values: new ArrayVector(['moon']) },
{ config: { filterable: true }, name: 'protocol', type: 'string', values: new ArrayVector(['http']) },
{ config: {}, name: 'Value #A', type: 'number', values: new ArrayVector([23]) },
],
length: 1,
meta: { preferredVisualisationType: 'table' },
refId: 'A',
},
];
const outputMulti = [
{
fields: [
{ config: {}, name: 'Time', type: 'time', values: new ArrayVector([1645029699311, 1645029699311]) },
{ config: { filterable: true }, name: 'level', type: 'string', values: new ArrayVector(['error', 'info']) },
{ config: { filterable: true }, name: 'location', type: 'string', values: new ArrayVector(['moon', 'moon']) },
{ config: { filterable: true }, name: 'protocol', type: 'string', values: new ArrayVector(['http', 'http']) },
{ config: {}, name: 'Value #A', type: 'number', values: new ArrayVector([23, 45]) },
],
length: 2,
meta: { preferredVisualisationType: 'table' },
refId: 'A',
},
{
fields: [
{ config: {}, name: 'Time', type: 'time', values: new ArrayVector([1645029699311]) },
{ config: { filterable: true }, name: 'level', type: 'string', values: new ArrayVector(['error']) },
{ config: { filterable: true }, name: 'location', type: 'string', values: new ArrayVector(['moon']) },
{ config: { filterable: true }, name: 'protocol', type: 'string', values: new ArrayVector(['http']) },
{ config: {}, name: 'Value #B', type: 'number', values: new ArrayVector([72]) },
],
length: 1,
meta: { preferredVisualisationType: 'table' },
refId: 'B',
},
];
describe('loki backendResultTransformer', () => {
it('converts a single instant metric dataframe to table dataframe', () => {
const result = makeTableFrames([frame1]);
expect(result).toEqual(outputSingle);
});
it('converts 3 instant metric dataframes into 2 tables', () => {
const result = makeTableFrames([frame1, frame2, frame3]);
expect(result).toEqual(outputMulti);
});
});

75
public/app/plugins/datasource/loki/makeTableFrames.ts Normal file
View File

@ -0,0 +1,75 @@
import { DataFrame, Field, FieldType, ArrayVector } from '@grafana/data';
import { groupBy } from 'lodash';
export function makeTableFrames(instantMetricFrames: DataFrame[]): DataFrame[] {
// first we remove frames that have no refId
// (we will group them by refId, so we need it to be set)
const framesWithRefId = instantMetricFrames.filter((f) => f.refId !== undefined);
const framesByRefId = groupBy(framesWithRefId, (frame) => frame.refId);
return Object.entries(framesByRefId).map(([refId, frames]) => makeTableFrame(frames, refId));
}
type NumberField = Field<number, ArrayVector<number>>;
type StringField = Field<string, ArrayVector<string>>;
function makeTableFrame(instantMetricFrames: DataFrame[], refId: string): DataFrame {
const tableTimeField: NumberField = { name: 'Time', config: {}, values: new ArrayVector(), type: FieldType.time };
const tableValueField: NumberField = {
name: `Value #${refId}`,
config: {},
values: new ArrayVector(),
type: FieldType.number,
};
// Sort metric labels, create columns for them and record their index
const allLabelNames = new Set(
instantMetricFrames.map((frame) => frame.fields.map((field) => Object.keys(field.labels ?? {})).flat()).flat()
);
const sortedLabelNames = Array.from(allLabelNames).sort();
const labelFields: StringField[] = sortedLabelNames.map((labelName) => ({
name: labelName,
config: { filterable: true },
values: new ArrayVector(),
type: FieldType.string,
}));
instantMetricFrames.forEach((frame) => {
const timeField = frame.fields.find((field) => field.type === FieldType.time);
const valueField = frame.fields.find((field) => field.type === FieldType.number);
if (timeField == null || valueField == null) {
return;
}
const timeArray = timeField.values.toArray();
const valueArray = valueField.values.toArray();
for (let x of timeArray) {
tableTimeField.values.add(x);
}
for (let x of valueArray) {
tableValueField.values.add(x);
}
const labels = valueField.labels ?? {};
for (let f of labelFields) {
const text = labels[f.name] ?? '';
// we insert the labels as many times as we have values
for (let i = 0; i < valueArray.length; i++) {
f.values.add(text);
}
}
});
return {
fields: [tableTimeField, ...labelFields, tableValueField],
refId,
meta: { preferredVisualisationType: 'table' },
length: tableTimeField.values.length,
};
}