Loki: Added support to split queries by stream shard (#94245)

* Add shard query splitting implementation

* Shard query splitting: reuse function from query splitting

* Shard query splitting: remove max line limit

* Shard query splitting: update test

* Shard query splitting: fix types and non-sharded queries

* Merge responses: fix log merging

* Merge responses: remove legacy code

* Query splitting: add support to retry failed requests

* Query splitting: unit test request retrying

* Query splitting: add unsubscriptions

* Shard query splitting: fix retrying

* Shard query splitting: switch to dynamic grouping

* Shard query splitting: update group size thresholds and fix -1 query

* Shard query splitting: update initial group size + don't retry parse errors

* Shard query splitting: update unit test

* chore: update mock value

* Shard query splitting: add support for multiple targets

* chore: update description

* Shard query splitting: use group targets

* chore: filter hidden queries

* Shard query splitting: issue initial log query without sharding

* Splitting: fix retrying in both methods

* Merge responses: keep execution time

* Shard query splitting: remove no-shard attempt

* Shard query splitting: adjust groups based on rate of change

* chore: clean up experiments

* Shard query splittng: remove log query restrictions

* Shard query splitting: remove fallback to time splitting

* Loki: add new query direction

* Missing generated file

* LokiOptionField: integrate new query direction

* Shard query splitting: delegate non-scan queries to time splitting

* Query splitting: do not retry queries with parse errors

* Loki datasource: add placeholder for feature flag

* Shard query splitting: add function with support criteria

* Shard query splitting: refactor query modification and shard logs volume

* Shard query splitting: update unit tests

* chore: Update scan direction tooltip

* chore: formatting

* LogsVolumePanel: fix missing state in logs volume panel data

* Merge responses: better handle missing nanoseconds

* LokiQueryOptionFields: display query direction for log queries

* loki: process scan direction as backward

* Loki datasource: restrict sharding to Explore

* Retrying: invert criteria and move to response utils

* Formatting

* Use log volume refId constant

* Fix import order

* Create feature flag

* Use feature toggle

* LogsVolumePanel: prevent flashing no data while streaming

pkg/services/featuremgmt/registry.go

@@ -265,6 +265,13 @@ var (
 			Expression:     "true", // turned on by default
 			AllowSelfServe: true,
 		},
+		{
+			Name:         "lokiShardSplitting",
+			Description:  "Use stream shards to split queries into smaller subqueries",
+			Stage:        FeatureStageExperimental,
+			FrontendOnly: true,
+			Owner:        grafanaObservabilityLogsSquad,
+		},
 		{
 			Name:           "lokiQuerySplitting",
 			Description:    "Split large interval queries into subqueries with smaller time intervals",

pkg/services/featuremgmt/toggles_gen.csv

@@ -33,6 +33,7 @@ alertingBacktesting,experimental,@grafana/alerting-squad,false,false,false
 editPanelCSVDragAndDrop,experimental,@grafana/dataviz-squad,false,false,true
 alertingNoNormalState,preview,@grafana/alerting-squad,false,false,false
 logsContextDatasourceUi,GA,@grafana/observability-logs,false,false,true
+lokiShardSplitting,experimental,@grafana/observability-logs,false,false,true
 lokiQuerySplitting,GA,@grafana/observability-logs,false,false,true
 lokiQuerySplittingConfig,experimental,@grafana/observability-logs,false,false,true
 individualCookiePreferences,experimental,@grafana/grafana-backend-group,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -143,6 +143,10 @@ const (
 	// Allow datasource to provide custom UI for context view
 	FlagLogsContextDatasourceUi = "logsContextDatasourceUi"
 
+	// FlagLokiShardSplitting
+	// Use stream shards to split queries into smaller subqueries
+	FlagLokiShardSplitting = "lokiShardSplitting"
+
 	// FlagLokiQuerySplitting
 	// Split large interval queries into subqueries with smaller time intervals
 	FlagLokiQuerySplitting = "lokiQuerySplitting"

pkg/services/featuremgmt/toggles_gen.json

@@ -1986,6 +1986,22 @@
         "codeowner": "@grafana/observability-logs"
       }
     },
+    {
+      "metadata": {
+        "name": "lokiShardSplitting",
+        "resourceVersion": "1729678036788",
+        "creationTimestamp": "2024-10-23T10:06:42Z",
+        "annotations": {
+          "grafana.app/updatedTimestamp": "2024-10-23 10:07:16.788828 +0000 UTC"
+        }
+      },
+      "spec": {
+        "description": "Use stream shards to split queries into smaller subqueries",
+        "stage": "experimental",
+        "codeowner": "@grafana/observability-logs",
+        "frontend": true
+      }
+    },
     {
       "metadata": {
         "name": "lokiStructuredMetadata",

pkg/tsdb/loki/kinds/dataquery/types_dataquery_gen.go

@@ -13,6 +13,7 @@ package dataquery
 const (
 	LokiQueryDirectionBackward LokiQueryDirection = "backward"
 	LokiQueryDirectionForward  LokiQueryDirection = "forward"
+	LokiQueryDirectionScan     LokiQueryDirection = "scan"
 )
 
 // Defines values for LokiQueryType.

pkg/tsdb/loki/parse_query.go

@@ -94,6 +94,8 @@ func parseDirection(jsonPointerValue *string) (Direction, error) {
 			return DirectionBackward, nil
 		case "forward":
 			return DirectionForward, nil
+		case "scan":
+			return DirectionBackward, nil
 		default:
 			return DirectionBackward, fmt.Errorf("invalid queryDirection: %s", jsonValue)
 		}