Alerting: Clamp Loki ASH range query to configured max_query_length (#83986)

* Clamp range in loki http client to configured max_query_length

Defaults to 721h to match Loki default

pkg/services/ngalert/state/historian/loki_http.go

@@ -40,6 +40,7 @@ type LokiConfig struct {
 	TenantID          string
 	ExternalLabels    map[string]string
 	Encoder           encoder
+	MaxQueryLength    time.Duration
 }
 
 func NewLokiConfig(cfg setting.UnifiedAlertingStateHistorySettings) (LokiConfig, error) {
@@ -74,6 +75,7 @@ func NewLokiConfig(cfg setting.UnifiedAlertingStateHistorySettings) (LokiConfig,
 		BasicAuthPassword: cfg.LokiBasicAuthPassword,
 		TenantID:          cfg.LokiTenantID,
 		ExternalLabels:    cfg.ExternalLabels,
+		MaxQueryLength:    cfg.LokiMaxQueryLength,
 		// Snappy-compressed protobuf is the default, same goes for Promtail.
 		Encoder: SnappyProtoEncoder{},
 	}, nil
@@ -193,26 +195,20 @@ func (c *HttpLokiClient) Push(ctx context.Context, s []Stream) error {
 	c.metrics.BytesWritten.Add(float64(len(enc)))
 	req = req.WithContext(ctx)
 	resp, err := c.client.Do(req)
-	if resp != nil {
-		defer func() {
-			if err := resp.Body.Close(); err != nil {
-				c.log.Warn("Failed to close response body", "err", err)
-			}
-		}()
-	}
 	if err != nil {
 		return fmt.Errorf("failed to send request: %w", err)
 	}
-
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		byt, _ := io.ReadAll(resp.Body)
-		if len(byt) > 0 {
-			c.log.Error("Error response from Loki", "response", string(byt), "status", resp.StatusCode)
-		} else {
-			c.log.Error("Error response from Loki with an empty body", "status", resp.StatusCode)
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			c.log.Warn("Failed to close response body", "err", err)
 		}
-		return fmt.Errorf("received a non-200 response from loki, status: %d", resp.StatusCode)
+	}()
+
+	_, err = c.handleLokiResponse(resp)
+	if err != nil {
+		return err
 	}
+
 	return nil
 }
 
@@ -231,6 +227,7 @@ func (c *HttpLokiClient) RangeQuery(ctx context.Context, logQL string, start, en
 	if start > end {
 		return QueryRes{}, fmt.Errorf("start time cannot be after end time")
 	}
+	start, end = ClampRange(start, end, c.cfg.MaxQueryLength.Nanoseconds())
 	if limit < 1 {
 		limit = defaultPageSize
 	}
@@ -261,23 +258,15 @@ func (c *HttpLokiClient) RangeQuery(ctx context.Context, logQL string, start, en
 	if err != nil {
 		return QueryRes{}, fmt.Errorf("error executing request: %w", err)
 	}
-
 	defer func() {
-		_ = res.Body.Close()
+		if err := res.Body.Close(); err != nil {
+			c.log.Warn("Failed to close response body", "err", err)
+		}
 	}()
 
-	data, err := io.ReadAll(res.Body)
+	data, err := c.handleLokiResponse(res)
 	if err != nil {
-		return QueryRes{}, fmt.Errorf("error reading request response: %w", err)
-	}
-
-	if res.StatusCode < 200 || res.StatusCode >= 300 {
-		if len(data) > 0 {
-			c.log.Error("Error response from Loki", "response", string(data), "status", res.StatusCode)
-		} else {
-			c.log.Error("Error response from Loki with an empty body", "status", res.StatusCode)
-		}
-		return QueryRes{}, fmt.Errorf("received a non-200 response from loki, status: %d", res.StatusCode)
+		return QueryRes{}, err
 	}
 
 	result := QueryRes{}
@@ -297,3 +286,36 @@ type QueryRes struct {
 type QueryData struct {
 	Result []Stream `json:"result"`
 }
+
+func (c *HttpLokiClient) handleLokiResponse(res *http.Response) ([]byte, error) {
+	if res == nil {
+		return nil, fmt.Errorf("response is nil")
+	}
+
+	data, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("error reading request response: %w", err)
+	}
+
+	if res.StatusCode < 200 || res.StatusCode >= 300 {
+		if len(data) > 0 {
+			c.log.Error("Error response from Loki", "response", string(data), "status", res.StatusCode)
+		} else {
+			c.log.Error("Error response from Loki with an empty body", "status", res.StatusCode)
+		}
+		return nil, fmt.Errorf("received a non-200 response from loki, status: %d", res.StatusCode)
+	}
+
+	return data, nil
+}
+
+// ClampRange ensures that the time range is within the configured maximum query length.
+func ClampRange(start, end, maxTimeRange int64) (newStart int64, newEnd int64) {
+	newStart, newEnd = start, end
+
+	if maxTimeRange != 0 && end-start > maxTimeRange {
+		newStart = end - maxTimeRange
+	}
+
+	return newStart, newEnd
+}

pkg/services/ngalert/state/historian/loki_http_test.go

@@ -338,6 +338,49 @@ func TestStream(t *testing.T) {
 	})
 }
 
+func TestClampRange(t *testing.T) {
+	tc := []struct {
+		name     string
+		oldRange []int64
+		max      int64
+		newRange []int64
+	}{
+		{
+			name:     "clamps start value if max is smaller than range",
+			oldRange: []int64{5, 10},
+			max:      1,
+			newRange: []int64{9, 10},
+		},
+		{
+			name:     "returns same values if max is greater than range",
+			oldRange: []int64{5, 10},
+			max:      20,
+			newRange: []int64{5, 10},
+		},
+		{
+			name:     "returns same values if max is equal to range",
+			oldRange: []int64{5, 10},
+			max:      5,
+			newRange: []int64{5, 10},
+		},
+		{
+			name:     "returns same values if max is zero",
+			oldRange: []int64{5, 10},
+			max:      0,
+			newRange: []int64{5, 10},
+		},
+	}
+
+	for _, c := range tc {
+		t.Run(c.name, func(t *testing.T) {
+			start, end := ClampRange(c.oldRange[0], c.oldRange[1], c.max)
+
+			require.Equal(t, c.newRange[0], start)
+			require.Equal(t, c.newRange[1], end)
+		})
+	}
+}
+
 func createTestLokiClient(req client.Requester) *HttpLokiClient {
 	url, _ := url.Parse("http://some.url")
 	cfg := LokiConfig{