TestData: Drop some percentage of CSV values from a request (#70404)

docs/sources/developers/kinds/composable/testdata/dataquery/schema-reference.md

@@ -22,6 +22,7 @@ title: TestDataDataQuery kind
 | `csvFileName`     | string                              | No       |         |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | `csvWave`         | [CSVWave](#csvwave)[]               | No       |         |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | `datasource`      |                                     | No       |         | For mixed data sources the selected datasource is on the query level.<br/>For non mixed scenarios this is undefined.<br/>TODO find a better way to do this ^ that's friendly to schema<br/>TODO this shouldn't be unknown but DataSourceRef &#124; null                                                                                                                                                                                                                                                                                |
+| `dropPercent`     | number                              | No       |         | Drop percentage (the chance we will lose a point 0-100)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
 | `errorType`       | string                              | No       |         | Possible values are: `server_panic`, `frontend_exception`, `frontend_observable`.                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
 | `hide`            | boolean                             | No       |         | true if query is disabled (ie should not be returned to the dashboard)<br/>Note this does not always imply that the query should not be executed since<br/>the results from a hidden query may be used as the input to other queries (SSE etc)                                                                                                                                                                                                                                                                                         |
 | `labels`          | string                              | No       |         |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |

packages/grafana-schema/src/raw/composable/testdata/dataquery/x/TestDataDataQuery_types.gen.ts

@@ -112,6 +112,10 @@ export interface TestDataDataQuery extends common.DataQuery {
   csvContent?: string;
   csvFileName?: string;
   csvWave?: Array<CSVWave>; // TODO can we prevent partial from being generated
+  /**
+   * Drop percentage (the chance we will lose a point 0-100)
+   */
+  dropPercent?: number;
   errorType?: ('server_panic' | 'frontend_exception' | 'frontend_observable');
   labels?: string;
   levelColumn?: boolean;

pkg/tsdb/testdatasource/csv_data.go

@@ -39,6 +39,14 @@ func (s *Service) handleCsvContentScenario(ctx context.Context, req *backend.Que
 			return nil, err
 		}
 
+		dropPercent := model.Get("dropPercent").MustFloat64(0)
+		if dropPercent > 0 {
+			frame, err = dropValues(frame, dropPercent)
+			if err != nil {
+				return nil, err
+			}
+		}
+
 		respD := resp.Responses[q.RefID]
 		respD.Frames = append(respD.Frames, frame)
 		resp.Responses[q.RefID] = respD
@@ -68,6 +76,14 @@ func (s *Service) handleCsvFileScenario(ctx context.Context, req *backend.QueryD
 			return nil, err
 		}
 
+		dropPercent := model.Get("dropPercent").MustFloat64(0)
+		if dropPercent > 0 {
+			frame, err = dropValues(frame, dropPercent)
+			if err != nil {
+				return nil, err
+			}
+		}
+
 		respD := resp.Responses[q.RefID]
 		respD.Frames = append(respD.Frames, frame)
 		resp.Responses[q.RefID] = respD

pkg/tsdb/testdatasource/kinds/dataquery/types_dataquery_gen.go

@@ -159,6 +159,9 @@ type TestDataDataQuery struct {
 	CsvContent  *string   `json:"csvContent,omitempty"`
 	CsvFileName *string   `json:"csvFileName,omitempty"`
 	CsvWave     []CSVWave `json:"csvWave,omitempty"`
+
+	// Drop percentage (the chance we will lose a point 0-100)
+	DropPercent     *float64           `json:"dropPercent,omitempty"`
 	ErrorType       *ErrorType         `json:"errorType,omitempty"`
 	Labels          *string            `json:"labels,omitempty"`
 	LevelColumn     *bool              `json:"levelColumn,omitempty"`

pkg/tsdb/testdatasource/utils.go

@@ -3,6 +3,8 @@ package testdatasource
 import (
 	"math/rand"
 	"time"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
 )
 
 type randomStringProvider struct {
@@ -20,3 +22,27 @@ func newRandomStringProvider(data []string) *randomStringProvider {
 func (p *randomStringProvider) Next() string {
 	return p.data[p.r.Int31n(int32(len(p.data)))]
 }
+
+func dropValues(frame *data.Frame, percent float64) (*data.Frame, error) {
+	if frame == nil || percent <= 0 || percent >= 100 {
+		return frame, nil
+	}
+	rows, err := frame.RowLen()
+	copy := frame.EmptyCopy()
+
+	percentage := percent / 100.0
+	seed := time.Now().UnixMilli()
+	r := rand.New(rand.NewSource(seed))
+	for i := 0; i < rows; i++ {
+		if r.Float64() < percentage { // .2 == 20
+			continue
+		}
+
+		// copy the row
+		for fidx, f := range copy.Fields {
+			f.Append(frame.Fields[fidx].At(i))
+		}
+	}
+
+	return copy, err
+}

public/app/core/components/OptionsUI/NumberInput.tsx

@@ -13,6 +13,7 @@ interface Props {
   step?: number;
   width?: number;
   fieldDisabled?: boolean;
+  suffix?: React.ReactNode;
 }
 
 interface State {
@@ -111,6 +112,7 @@ export class NumberInput extends PureComponent<Props, State> {
         placeholder={this.props.placeholder}
         disabled={this.props.fieldDisabled}
         width={this.props.width}
+        suffix={this.props.suffix}
       />
     );
   }

public/app/plugins/datasource/testdata/QueryEditor.tsx

@@ -3,7 +3,8 @@ import { useAsync } from 'react-use';
 
 import { QueryEditorProps, SelectableValue } from '@grafana/data';
 import { selectors as editorSelectors } from '@grafana/e2e-selectors';
-import { InlineField, InlineFieldRow, InlineSwitch, Input, Select, TextArea } from '@grafana/ui';
+import { InlineField, InlineFieldRow, InlineSwitch, Input, Select, Icon, TextArea } from '@grafana/ui';
+import { NumberInput } from 'app/core/components/OptionsUI/NumberInput';
 
 import { RandomWalkEditor, StreamingClientEditor } from './components';
 import { CSVContentEditor } from './components/CSVContentEditor';
@@ -21,7 +22,6 @@ import { CSVWave, NodesQuery, TestData, TestDataQueryType, USAQuery } from './da
 import { TestDataDataSource } from './datasource';
 import { defaultStreamQuery } from './runStreams';
 
-const showLabelsFor = ['random_walk', 'predictable_pulse'];
 const endpoints = [
   { value: 'datasources', label: 'Data Sources' },
   { value: 'search', label: 'Search' },
@@ -166,6 +166,13 @@ export const QueryEditor = ({ query, datasource, onChange, onRunQuery }: Props)
     onUpdate({ ...query, csvWave });
   };
 
+  const onDropPercentChanged = (dropPercent: number | undefined) => {
+    if (!dropPercent) {
+      dropPercent = undefined;
+    }
+    onChange({ ...query, dropPercent });
+  };
+
   const options = useMemo(
     () =>
       (scenarioList || [])
@@ -173,7 +180,15 @@ export const QueryEditor = ({ query, datasource, onChange, onRunQuery }: Props)
         .sort((a, b) => a.label.localeCompare(b.label)),
     [scenarioList]
   );
-  const showLabels = useMemo(() => showLabelsFor.includes(query.scenarioId ?? ''), [query]);
+
+  // Common options that can be added to various scenarios
+  const show = useMemo(() => {
+    const scenarioId = query.scenarioId ?? '';
+    return {
+      labels: ['random_walk', 'predictable_pulse'].includes(scenarioId),
+      dropPercent: ['csv_content', 'csv_file'].includes(scenarioId),
+    };
+  }, [query?.scenarioId]);
 
   if (loading) {
     return null;
@@ -217,7 +232,21 @@ export const QueryEditor = ({ query, datasource, onChange, onRunQuery }: Props)
             />
           </InlineField>
         )}
-        {showLabels && (
+        {show.dropPercent && (
+          <InlineField label="Drop" tooltip={'Drop a random set of points'}>
+            <NumberInput
+              min={0}
+              max={100}
+              step={5}
+              width={8}
+              onChange={onDropPercentChanged}
+              placeholder="0"
+              value={query.dropPercent}
+              suffix={<Icon name="percentage" />}
+            />
+          </InlineField>
+        )}
+        {show.labels && (
           <InlineField
             label="Labels"
             labelWidth={14}

public/app/plugins/datasource/testdata/dataquery.cue

@@ -51,6 +51,9 @@ composableKinds: DataQuery: {
 				spanCount?:             int32
 				points?: [...[...string | int64]]
 
+				// Drop percentage (the chance we will lose a point 0-100)
+				dropPercent?: float64
+
 				#TestDataQueryType: "random_walk" | "slow_query" | "random_walk_with_error" | "random_walk_table" | "exponential_heatmap_bucket_data" | "linear_heatmap_bucket_data" | "no_data_points" | "datapoints_outside_range" | "csv_metric_values" | "predictable_pulse" | "predictable_csv_wave" | "streaming_client" | "simulation" | "usa" | "live" | "grafana_api" | "arrow" | "annotations" | "table_static" | "server_error_500" | "logs" | "node_graph" | "flame_graph" | "raw_frame" | "csv_file" | "csv_content" | "trace" | "manual_entry" | "variables-query" @cuetsy(kind="enum", memberNames="RandomWalk|SlowQuery|RandomWalkWithError|RandomWalkTable|ExponentialHeatmapBucketData|LinearHeatmapBucketData|NoDataPoints|DataPointsOutsideRange|CSVMetricValues|PredictablePulse|PredictableCSVWave|StreamingClient|Simulation|USA|Live|GrafanaAPI|Arrow|Annotations|TableStatic|ServerError500|Logs|NodeGraph|FlameGraph|RawFrame|CSVFile|CSVContent|Trace|ManualEntry|VariablesQuery")
 
 				#StreamingQuery: {

public/app/plugins/datasource/testdata/dataquery.gen.ts

@@ -113,6 +113,10 @@ export interface TestData extends common.DataQuery {
   csvContent?: string;
   csvFileName?: string;
   csvWave?: Array<CSVWave>; // TODO can we prevent partial from being generated
+  /**
+   * Drop percentage (the chance we will lose a point 0-100)
+   */
+  dropPercent?: number;
   errorType?: ('server_panic' | 'frontend_exception' | 'frontend_observable');
   labels?: string;
   levelColumn?: boolean;