IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Postgres/MySQL/MSSQL: Add setting to limit maximum amount of rows processed (#38986)

Browse Source 
Adds a new setting dataproxy.row_limit that allows an operator to limit the 
amount of rows being processed/accepted in response to database queries 
originating from SQL data sources.

Closes #38975
Ref #39095

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>
This commit is contained in:
Marcus Efraimsson
2021-09-13 15:27:51 +02:00
committed by GitHub
parent 5b7dc16c06
commit 1c892a2fc4
12 changed files with 279 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
pkg/tsdb/mysql/mysql.go
View File

@@ -46,8 +46,7 @@ func characterEscape(s string, escapeChar string) string {
func ProvideService(cfg *setting.Cfg, manager backendplugin.Manager, httpClientProvider httpclient.Provider) (*Service, error) {
s := &Service{
Cfg: cfg,
im: datasource.NewInstanceManager(newInstanceSettings(httpClientProvider)),
im: datasource.NewInstanceManager(newInstanceSettings(cfg, httpClientProvider)),
}
factory := coreplugin.New(backend.ServeOpts{
QueryDataHandler: s,
@@ -59,7 +58,7 @@ func ProvideService(cfg *setting.Cfg, manager backendplugin.Manager, httpClientP
return s, nil
}
func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc {
func newInstanceSettings(cfg *setting.Cfg, httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc {
return func(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
jsonData := sqleng.JsonData{
MaxOpenConns: 0,
@@ -117,7 +116,7 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst
cnnstr += fmt.Sprintf("&time_zone='%s'", url.QueryEscape(dsInfo.JsonData.Timezone))
}
if setting.Env == setting.Dev {
if cfg.Env == setting.Dev {
logger.Debug("getEngine", "connection", cnnstr)
}
@@ -127,6 +126,7 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst
DSInfo: dsInfo,
TimeColumnNames: []string{"time", "time_sec"},
MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"},
RowLimit: cfg.DataProxyRowLimit,
}
rowTransformer := mysqlQueryResultTransformer{

80
pkg/tsdb/mysql/mysql_test.go
View File

@@ -68,6 +68,7 @@ func TestMySQL(t *testing.T) {
DSInfo: dsInfo,
TimeColumnNames: []string{"time", "time_sec"},
MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"},
RowLimit: 1000000,
}
rowTransformer := mysqlQueryResultTransformer{
@@ -1151,6 +1152,85 @@ func TestMySQL(t *testing.T) {
require.Equal(t, data.FieldTypeNullableTime, frames[0].Fields[0].Type())
require.Equal(t, data.FieldTypeNullableTime, frames[0].Fields[1].Type())
})
t.Run("When row limit set to 1", func(t *testing.T) {
dsInfo := sqleng.DataSourceInfo{}
config := sqleng.DataPluginConfiguration{
DriverName: "mysql",
ConnectionString: "",
DSInfo: dsInfo,
TimeColumnNames: []string{"time", "time_sec"},
MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"},
RowLimit: 1,
}
queryResultTransformer := mysqlQueryResultTransformer{
log: logger,
}
handler, err := sqleng.NewQueryDataHandler(config, &queryResultTransformer, newMysqlMacroEngine(logger), logger)
require.NoError(t, err)
t.Run("When doing a table query that returns 2 rows should limit the result to 1 row", func(t *testing.T) {
query := &backend.QueryDataRequest{
Queries: []backend.DataQuery{
{
JSON: []byte(`{
"rawSql": "SELECT 1 as value UNION ALL select 2 as value",
"format": "table"
}`),
RefID: "A",
TimeRange: backend.TimeRange{
From: time.Now(),
To: time.Now(),
},
},
},
}
resp, err := handler.QueryData(context.Background(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
frames := queryResult.Frames
require.NoError(t, err)
require.Equal(t, 1, len(frames))
require.Equal(t, 1, len(frames[0].Fields))
require.Equal(t, 1, frames[0].Rows())
require.Len(t, frames[0].Meta.Notices, 1)
require.Equal(t, data.NoticeSeverityWarning, frames[0].Meta.Notices[0].Severity)
})
t.Run("When doing a time series that returns 2 rows should limit the result to 1 row", func(t *testing.T) {
query := &backend.QueryDataRequest{
Queries: []backend.DataQuery{
{
JSON: []byte(`{
"rawSql": "SELECT 1 as time, 1 as value UNION ALL select 2 as time, 2 as value",
"format": "time_series"
}`),
RefID: "A",
TimeRange: backend.TimeRange{
From: time.Now(),
To: time.Now(),
},
},
},
}
resp, err := handler.QueryData(context.Background(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
frames := queryResult.Frames
require.NoError(t, err)
require.Equal(t, 1, len(frames))
require.Equal(t, 2, len(frames[0].Fields))
require.Equal(t, 1, frames[0].Rows())
require.Len(t, frames[0].Meta.Notices, 1)
require.Equal(t, data.NoticeSeverityWarning, frames[0].Meta.Notices[0].Severity)
})
})
})
}