IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2026-08-26 21:37:31 -05:00
Code Issues Packages Projects Releases Wiki Activity

Azure Monitor: Limit the log rows to 30000 (#93901)

Browse Source 
* Limit the amount of rows to 30000

* Lint

* Remove unneeded goldenframe check

* Add feature toggle

* Update toggles

* Use rowLimit variable
This commit is contained in:
Andreas Christou
2024-10-24 16:32:09 +03:00
committed by GitHub
parent ef253c99d8
commit 2a65de8aaa
10 changed files with 128 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/azuremonitor/loganalytics/azure-log-analytics-datasource.go
+2 -1
View File
@@ -320,7 +320,8 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
return nil, err
}
frame, err := ResponseTableToFrame(t, query.RefID, query.Query, query.QueryType, query.ResultFormat)
logLimitDisabled := backend.GrafanaConfigFromContext(ctx).FeatureToggles().IsEnabled("azureMonitorDisableLogLimit")
frame, err := ResponseTableToFrame(t, query.RefID, query.Query, query.QueryType, query.ResultFormat, logLimitDisabled)
if err != nil {
return nil, err
}
pkg/tsdb/azuremonitor/loganalytics/azure-response-table-frame.go
+18 -3
View File
@@ -45,12 +45,12 @@ func apiErrorToNotice(err *AzureLogAnalyticsAPIError) data.Notice {
}
// ResponseTableToFrame converts an AzureResponseTable to a data.Frame.
func ResponseTableToFrame(table *types.AzureResponseTable, refID string, executedQuery string, queryType dataquery.AzureQueryType, resultFormat dataquery.ResultFormat) (*data.Frame, error) {
func ResponseTableToFrame(table *types.AzureResponseTable, refID string, executedQuery string, queryType dataquery.AzureQueryType, resultFormat dataquery.ResultFormat, logLimitDisabled bool) (*data.Frame, error) {
if len(table.Rows) == 0 {
return nil, nil
}
converterFrame, err := converterFrameForTable(table, queryType, resultFormat)
converterFrame, err := converterFrameForTable(table, queryType, resultFormat, logLimitDisabled)
if err != nil {
return nil, err
}
@@ -66,7 +66,7 @@ func ResponseTableToFrame(table *types.AzureResponseTable, refID string, execute
return converterFrame.Frame, nil
}
func converterFrameForTable(t *types.AzureResponseTable, queryType dataquery.AzureQueryType, resultFormat dataquery.ResultFormat) (*data.FrameInputConverter, error) {
func converterFrameForTable(t *types.AzureResponseTable, queryType dataquery.AzureQueryType, resultFormat dataquery.ResultFormat, logLimitDisabled bool) (*data.FrameInputConverter, error) {
converters := []data.FieldConverter{}
colNames := make([]string, len(t.Columns))
colTypes := make([]string, len(t.Columns)) // for metadata
@@ -84,6 +84,14 @@ func converterFrameForTable(t *types.AzureResponseTable, queryType dataquery.Azu
converters = append(converters, converter)
}
rowLimit := 30000
limitExceeded := false
if len(t.Rows) > rowLimit && resultFormat == dataquery.ResultFormatLogs && !logLimitDisabled {
// We limit the number of rows to 30k to prevent crashing the browser tab as the logs viz is not virtualised.
t.Rows = t.Rows[:rowLimit]
limitExceeded = true
}
fic, err := data.NewFrameInputConverter(converters, len(t.Rows))
if err != nil {
return nil, err
@@ -98,6 +106,13 @@ func converterFrameForTable(t *types.AzureResponseTable, queryType dataquery.Azu
Custom: &LogAnalyticsMeta{ColumnTypes: colTypes},
}
if limitExceeded {
fic.Frame.AppendNotices(data.Notice{
Severity: data.NoticeSeverityWarning,
Text: "The number of results in the result set has been limited to 30,000.",
})
}
return fic, nil
}
pkg/tsdb/azuremonitor/loganalytics/azure-response-table-frame_test.go
+60 -2
View File
@@ -5,12 +5,14 @@ import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/kinds/dataquery"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/testdata"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -54,7 +56,7 @@ func TestLogTableToFrame(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
res := loadTestFileWithNumber(t, tt.testFile)
frame, err := ResponseTableToFrame(&res.Tables[0], "A", "query", dataquery.AzureQueryTypeAzureLogAnalytics, dataquery.ResultFormatTable)
frame, err := ResponseTableToFrame(&res.Tables[0], "A", "query", dataquery.AzureQueryTypeAzureLogAnalytics, dataquery.ResultFormatTable, false)
appendErrorNotice(frame, res.Error)
require.NoError(t, err)
@@ -112,7 +114,7 @@ func TestTraceTableToFrame(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
res := loadTestFileWithNumber(t, tt.testFile)
frame, err := ResponseTableToFrame(&res.Tables[0], "A", "query", tt.queryType, tt.resultFormat)
frame, err := ResponseTableToFrame(&res.Tables[0], "A", "query", tt.queryType, tt.resultFormat, false)
appendErrorNotice(frame, res.Error)
require.NoError(t, err)
@@ -121,6 +123,62 @@ func TestTraceTableToFrame(t *testing.T) {
}
}
func TestLargeLogsResponse(t *testing.T) {
t.Run("large logs response with limit enabled", func(t *testing.T) {
res := AzureLogAnalyticsResponse{
Tables: []types.AzureResponseTable{
{Name: "PrimaryResult",
Columns: []struct {
Name string `json:"name"`
Type string `json:"type"`
}{
{Name: "value", Type: "int"},
}},
},
}
rows := [][]any{}
for i := 0; i <= 30000; i++ {
rows = append(rows, []any{json.Number(strconv.Itoa(i))})
}
res.Tables[0].Rows = rows
resultFormat := dataquery.ResultFormatLogs
frame, err := ResponseTableToFrame(&res.Tables[0], "A", "query", dataquery.AzureQueryTypeAzureLogAnalytics, resultFormat, false)
appendErrorNotice(frame, res.Error)
require.NoError(t, err)
require.Equal(t, frame.Rows(), 30000)
require.Len(t, frame.Meta.Notices, 1)
require.Equal(t, frame.Meta.Notices[0], data.Notice{
Severity: data.NoticeSeverityWarning,
Text: "The number of results in the result set has been limited to 30,000.",
})
})
t.Run("large logs response with limit disabled", func(t *testing.T) {
res := AzureLogAnalyticsResponse{
Tables: []types.AzureResponseTable{
{Name: "PrimaryResult",
Columns: []struct {
Name string `json:"name"`
Type string `json:"type"`
}{
{Name: "value", Type: "int"},
}},
},
}
rows := [][]any{}
for i := 0; i < 40000; i++ {
rows = append(rows, []any{json.Number(strconv.Itoa(i))})
}
res.Tables[0].Rows = rows
resultFormat := dataquery.ResultFormatLogs
frame, err := ResponseTableToFrame(&res.Tables[0], "A", "query", dataquery.AzureQueryTypeAzureLogAnalytics, resultFormat, true)
appendErrorNotice(frame, res.Error)
require.NoError(t, err)
require.Equal(t, frame.Rows(), 40000)
require.Nil(t, frame.Meta.Notices)
})
}
func loadTestFileWithNumber(t *testing.T, name string) AzureLogAnalyticsResponse {
t.Helper()
path := filepath.Join("../testdata", name)