diff --git a/public/app/features/trails/Integrations/logs/base.ts b/public/app/features/trails/Integrations/logs/base.ts new file mode 100644 index 00000000000..629e148b889 --- /dev/null +++ b/public/app/features/trails/Integrations/logs/base.ts @@ -0,0 +1,28 @@ +import { DataSourceSettings } from '@grafana/data'; + +export type FoundLokiDataSource = Pick; + +/** + * Defines the interface for connecting metrics and their related logs. + * Implementations should provide methods for retrieving Loki data sources associated + * with a metric, and creating a Loki query expression for a given metric and data source. + * + * By using this interface, the `RelatedLogsScene` can orchestrate + * the retrieval of logs without needing to know the specifics of how we're + * associating logs with a given metric. + */ +export interface MetricsLogsConnector { + /** + * Retrieves the Loki data sources associated with the specified metric. + */ + getDataSources(selectedMetric: string): Promise; + + /** + * Constructs a Loki query expression for the specified metric and data source. + */ + getLokiQueryExpr(selectedMetric: string, datasourceUid: string): string; +} + +export function createMetricsLogsConnector(connector: T): T { + return connector; +} diff --git a/public/app/features/trails/Integrations/logs/lokiRecordingRules.test.ts b/public/app/features/trails/Integrations/logs/lokiRecordingRules.test.ts new file mode 100644 index 00000000000..1bcae3e228c --- /dev/null +++ b/public/app/features/trails/Integrations/logs/lokiRecordingRules.test.ts @@ -0,0 +1,213 @@ +import { of } from 'rxjs'; + +import type { DataSourceInstanceSettings, DataSourceJsonData } from '@grafana/data'; +import { getMockPlugin } from '@grafana/data/test/__mocks__/pluginMocks'; +import * as runtime from '@grafana/runtime'; + +import { MetricsLogsConnector } from './base'; +import { lokiRecordingRulesConnector, type RecordingRuleGroup } from './lokiRecordingRules'; + +const mockLokiDS1: DataSourceInstanceSettings = { + access: 'proxy', + id: 1, + uid: 'loki1', + name: 'Loki Main', + type: 'loki', + url: '', + jsonData: {}, + meta: { + ...getMockPlugin(), + id: 'loki', + }, + readOnly: false, + isDefault: false, + database: '', + withCredentials: false, +}; + +const mockLokiDS2: DataSourceInstanceSettings = { + ...mockLokiDS1, + id: 2, + uid: 'loki2', + name: 'Loki Secondary', +}; + +const mockRuleGroups1: RecordingRuleGroup[] = [ + { + name: 'group1', + rules: [ + { + name: 'metric_a_total', + query: 'sum(rate({app="app-A"} |= "error" [5m]))', + type: 'recording', + }, + { + name: 'metric_b_total', + query: 'sum(rate({app="app-B"} |= "warn" [5m]))', + type: 'recording', + }, + ], + }, +]; + +const mockRuleGroups2: RecordingRuleGroup[] = [ + { + name: 'group2', + rules: [ + { + name: 'metric_a_total', // Intentionally same name as in DS1 + query: 'sum(rate({app="app-C"} |= "error" [5m]))', + type: 'recording', + }, + ], + }, +]; + +// Create spy functions +const getListSpy = jest.fn().mockReturnValue([mockLokiDS1, mockLokiDS2]); +const defaultFetchImpl = (req: Request) => { + if (req.url.includes('loki1')) { + return of({ + data: { data: { groups: mockRuleGroups1 } }, + ok: true, + status: 200, + statusText: 'OK', + headers: new Headers(), + redirected: false, + type: 'basic', + url: req.url, + config: { url: req.url }, + } as runtime.FetchResponse); + } + return of({ + data: { data: { groups: mockRuleGroups2 } }, + ok: true, + status: 200, + statusText: 'OK', + headers: new Headers(), + redirected: false, + type: 'basic', + url: req.url, + config: { url: req.url }, + } as runtime.FetchResponse); +}; +const fetchSpy = jest.fn().mockImplementation(defaultFetchImpl); + +// Mock the entire @grafana/runtime module +jest.mock('@grafana/runtime', () => ({ + ...jest.requireActual('@grafana/runtime'), + getDataSourceSrv: () => ({ + getList: getListSpy, + get: jest.fn(), + getInstanceSettings: jest.fn(), + reload: jest.fn(), + }), + getBackendSrv: () => ({ + fetch: fetchSpy, + delete: jest.fn(), + get: jest.fn(), + patch: jest.fn(), + post: jest.fn(), + put: jest.fn(), + request: jest.fn(), + datasourceRequest: jest.fn(), + }), +})); + +describe('LokiRecordingRulesConnector', () => { + let consoleSpy: jest.SpyInstance; + + beforeEach(() => { + getListSpy.mockClear(); + fetchSpy.mockClear(); + fetchSpy.mockImplementation(defaultFetchImpl); + consoleSpy = jest.spyOn(console, 'error').mockImplementation(); + }); + + afterEach(() => { + consoleSpy.mockRestore(); + }); + + describe('getDataSources', () => { + it('should find all data sources containing the metric', async () => { + const connector = lokiRecordingRulesConnector; + const result = await connector.getDataSources('metric_a_total'); + + expect(result).toHaveLength(2); + expect(result).toContainEqual({ name: 'Loki Main', uid: 'loki1' }); + expect(result).toContainEqual({ name: 'Loki Secondary', uid: 'loki2' }); + + // Verify underlying calls + expect(getListSpy).toHaveBeenCalledWith({ logs: true }); + expect(fetchSpy).toHaveBeenCalledTimes(2); + }); + + it('should handle non-existent metrics', async () => { + const connector = lokiRecordingRulesConnector; + const result = await connector.getDataSources('non_existent_metric'); + + expect(result).toHaveLength(0); + }); + + it('should handle datasource fetch errors gracefully', async () => { + // Make the second datasource fail + fetchSpy.mockImplementation((req) => { + if (req.url.includes('loki1')) { + return of({ + data: { data: { groups: mockRuleGroups1 } }, + ok: true, + status: 200, + statusText: 'OK', + headers: new Headers(), + redirected: false, + type: 'basic', + url: req.url, + config: { url: req.url }, + } as runtime.FetchResponse); + } + throw new Error('Failed to fetch'); + }); + + const connector = lokiRecordingRulesConnector; + const result = await connector.getDataSources('metric_a_total'); + + // Should still get results from the working datasource + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ name: 'Loki Main', uid: 'loki1' }); + expect(consoleSpy).toHaveBeenCalled(); + }); + }); + + describe('getLokiQueryExpr', () => { + let connector: MetricsLogsConnector; + + beforeEach(async () => { + connector = lokiRecordingRulesConnector; + // Populate the rules first + await connector.getDataSources('metric_a_total'); + }); + + it('should return correct Loki query for existing metric', () => { + const result = connector.getLokiQueryExpr('metric_a_total', 'loki1'); + expect(result).toBe('{app="app-A"} |= "error"'); + }); + + it('should return empty string for non-existent metric', () => { + const result = connector.getLokiQueryExpr('non_existent_metric', 'loki1'); + expect(result).toBe(''); + }); + + it('should handle multiple occurrences of the same metric name', () => { + const query1 = connector.getLokiQueryExpr('metric_a_total', 'loki1'); + const query2 = connector.getLokiQueryExpr('metric_a_total', 'loki2'); + + expect(query1).toBe('{app="app-A"} |= "error"'); + expect(query2).toBe('{app="app-C"} |= "error"'); + }); + + it('should handle rules with hasMultipleOccurrences flag', () => { + const query = connector.getLokiQueryExpr('metric_a_total', 'loki1'); + expect(query).toBe('{app="app-A"} |= "error"'); + }); + }); +}); diff --git a/public/app/features/trails/Integrations/logsIntegration.ts b/public/app/features/trails/Integrations/logs/lokiRecordingRules.ts similarity index 84% rename from public/app/features/trails/Integrations/logsIntegration.ts rename to public/app/features/trails/Integrations/logs/lokiRecordingRules.ts index 59f19e22b1c..bb741b5507d 100644 --- a/public/app/features/trails/Integrations/logsIntegration.ts +++ b/public/app/features/trails/Integrations/logs/lokiRecordingRules.ts @@ -1,29 +1,31 @@ import { lastValueFrom } from 'rxjs'; -import type { DataSourceInstanceSettings, DataSourceJsonData, DataSourceSettings } from '@grafana/data'; +import type { DataSourceInstanceSettings, DataSourceJsonData } from '@grafana/data'; import { getBackendSrv, getDataSourceSrv, type BackendSrvRequest, type FetchResponse } from '@grafana/runtime'; import { getLogQueryFromMetricsQuery } from 'app/plugins/datasource/loki/queryUtils'; -export type RecordingRuleGroup = { +import { createMetricsLogsConnector, type FoundLokiDataSource } from './base'; + +export interface RecordingRuleGroup { name: string; rules: RecordingRule[]; -}; +} -export type RecordingRule = { +export interface RecordingRule { name: string; query: string; type: 'recording' | 'alerting' | string; labels?: Record; -}; +} -export type FoundLokiDataSource = Pick; -export type ExtractedRecordingRule = RecordingRule & { +export interface ExtractedRecordingRule extends RecordingRule { datasource: FoundLokiDataSource; hasMultipleOccurrences?: boolean; -}; -export type ExtractedRecordingRules = { +} + +export interface ExtractedRecordingRules { [dataSourceUID: string]: ExtractedRecordingRule[]; -}; +} /** * Fetch Loki recording rule groups from the specified datasource. @@ -60,7 +62,6 @@ export function extractRecordingRulesFromRuleGroups( // We only want to return the first matching rule when there are multiple rules with same name const extractedRules = new Map(); - // const extractedRules: [] = []; ruleGroups.forEach((rg) => { rg.rules .filter((r) => r.type === 'recording') @@ -167,3 +168,22 @@ export async function fetchAndExtractLokiRecordingRules() { return extractedRecordingRules; } + +const createLokiRecordingRulesConnector = () => { + let lokiRecordingRules: ExtractedRecordingRules = {}; + + return createMetricsLogsConnector({ + async getDataSources(selectedMetric: string): Promise { + lokiRecordingRules = await fetchAndExtractLokiRecordingRules(); + const lokiDataSources = getDataSourcesWithRecordingRulesContainingMetric(selectedMetric, lokiRecordingRules); + + return lokiDataSources; + }, + + getLokiQueryExpr(selectedMetric: string, datasourceUid: string): string { + return getLokiQueryForRelatedMetric(selectedMetric, datasourceUid, lokiRecordingRules); + }, + }); +}; + +export const lokiRecordingRulesConnector = createLokiRecordingRulesConnector(); diff --git a/public/app/features/trails/Integrations/logsIntegration.test.ts b/public/app/features/trails/Integrations/logsIntegration.test.ts deleted file mode 100644 index 3ec65da4099..00000000000 --- a/public/app/features/trails/Integrations/logsIntegration.test.ts +++ /dev/null @@ -1,311 +0,0 @@ -import { of } from 'rxjs'; - -import type { DataSourceInstanceSettings, DataSourceJsonData } from '@grafana/data'; -import { getMockPlugin } from '@grafana/data/test/__mocks__/pluginMocks'; -import * as runtime from '@grafana/runtime'; - -import { - extractRecordingRulesFromRuleGroups, - fetchAndExtractLokiRecordingRules, - getLokiQueryForRelatedMetric, - getDataSourcesWithRecordingRulesContainingMetric, - type ExtractedRecordingRules, - type RecordingRuleGroup, -} from './logsIntegration'; - -const mockLokiDS1: DataSourceInstanceSettings = { - access: 'proxy', - id: 1, - uid: 'loki1', - name: 'Loki Main', - type: 'loki', - url: '', - jsonData: {}, - meta: { - ...getMockPlugin(), - id: 'loki', - }, - readOnly: false, - isDefault: false, - database: '', - withCredentials: false, -}; - -const mockLokiDS2: DataSourceInstanceSettings = { - ...mockLokiDS1, - id: 2, - uid: 'loki2', - name: 'Loki Secondary', -}; - -const mockLokiDS3: DataSourceInstanceSettings = { - ...mockLokiDS1, - id: 3, - uid: 'loki3', - name: 'Loki the Third with same rules', -}; - -const mockLokiDSs = [mockLokiDS1, mockLokiDS2, mockLokiDS3]; - -const mockRuleGroups1: RecordingRuleGroup[] = [ - { - name: 'group1', - rules: [ - { - name: 'metric_a_total', - query: 'sum(rate({app="app-A"} |= "error" [5m]))', - type: 'recording', - }, - { - name: 'metric_b_total', - query: 'sum(rate({app="app-B"} |= "warn" [5m]))', - type: 'recording', - }, - ], - }, -]; - -const mockRuleGroups2: RecordingRuleGroup[] = [ - { - name: 'group2', - rules: [ - { - name: 'metric_a_total', // Intentionally same name as in DS1 - query: 'sum(rate({app="app-C"} |= "error" [5m]))', - type: 'recording', - }, - ], - }, -]; - -const mockRuleGroupsWithSameRuleName: RecordingRuleGroup[] = [ - { - name: 'group_with_same_rule_names', - rules: [ - { - name: 'metric_xx_total', - query: 'sum(rate({app="app-XX"} |= "error" [5m]))', - type: 'recording', - labels: { - customLabel: 'label value 5m', - }, - }, - { - name: 'metric_xx_total', - query: 'sum(rate({app="app-YY"} |= "warn" [10m]))', - type: 'recording', - labels: { - customLabel: 'label value 10m', - }, - }, - ], - }, -]; - -const mockExtractedRules: ExtractedRecordingRules = { - loki1: [ - { - name: 'metric_a_total', - query: 'sum(rate({app="app-A"} |= "error" [5m]))', - type: 'recording', - datasource: { name: 'Loki Main', uid: 'loki1' }, - hasMultipleOccurrences: false, - }, - { - name: 'metric_b_total', - query: 'sum(rate({app="app-B"} |= "warn" [5m]))', - type: 'recording', - datasource: { name: 'Loki Main', uid: 'loki1' }, - hasMultipleOccurrences: false, - }, - ], - loki2: [ - { - name: 'metric_a_total', - query: 'sum(rate({app="app-C"} |= "error" [5m]))', - type: 'recording', - datasource: { name: 'Loki Secondary', uid: 'loki2' }, - hasMultipleOccurrences: false, - }, - ], - loki3: [ - { - name: 'metric_xx_total', - query: 'sum(rate({app="app-XX"} |= "error" [5m]))', - type: 'recording', - datasource: { name: 'Loki the Third with same rules', uid: 'loki3' }, - hasMultipleOccurrences: true, - }, - ], -}; - -// Create spy functions -const getListSpy = jest.fn().mockReturnValue(mockLokiDSs); -const fetchSpy = jest.fn().mockImplementation((req) => { - if (req.url.includes('loki1')) { - return of({ - data: { data: { groups: mockRuleGroups1 } }, - ok: true, - status: 200, - statusText: 'OK', - headers: new Headers(), - redirected: false, - type: 'basic', - url: req.url, - config: { url: req.url }, - } as runtime.FetchResponse); - } - if (req.url.includes('loki2')) { - return of({ - data: { data: { groups: mockRuleGroups2 } }, - ok: true, - status: 200, - statusText: 'OK', - headers: new Headers(), - redirected: false, - type: 'basic', - url: req.url, - config: { url: req.url }, - } as runtime.FetchResponse); - } - if (req.url.includes('loki3')) { - return of({ - data: { data: { groups: mockRuleGroupsWithSameRuleName } }, - ok: true, - status: 200, - statusText: 'OK', - headers: new Headers(), - redirected: false, - type: 'basic', - url: req.url, - config: { url: req.url }, - } as runtime.FetchResponse); - } - return of({ - data: { data: { groups: [] } }, - ok: true, - status: 200, - statusText: 'OK', - headers: new Headers(), - redirected: false, - type: 'basic', - url: req.url, - config: { url: req.url }, - } as runtime.FetchResponse); -}); - -// Mock the entire @grafana/runtime module -jest.mock('@grafana/runtime', () => ({ - ...jest.requireActual('@grafana/runtime'), - getDataSourceSrv: () => ({ - getList: getListSpy, - get: jest.fn(), - getInstanceSettings: jest.fn(), - reload: jest.fn(), - }), - getBackendSrv: () => ({ - fetch: fetchSpy, - delete: jest.fn(), - get: jest.fn(), - patch: jest.fn(), - post: jest.fn(), - put: jest.fn(), - request: jest.fn(), - datasourceRequest: jest.fn(), - }), -})); - -describe('Logs Integration', () => { - describe('fetchAndExtractLokiRecordingRules', () => { - beforeEach(() => { - getListSpy.mockClear(); - fetchSpy.mockClear(); - }); - - it('should fetch and extract rules from all Loki data sources', async () => { - const result = await fetchAndExtractLokiRecordingRules(); - - expect(result).toEqual(mockExtractedRules); - expect(getListSpy).toHaveBeenCalledWith({ logs: true }); - expect(fetchSpy).toHaveBeenCalledTimes(mockLokiDSs.length); - }); - - it('should handle errors from individual data sources gracefully', async () => { - // Mock console.error to avoid test output pollution - const consoleSpy = jest.spyOn(console, 'error').mockImplementation(); - - fetchSpy.mockImplementation((req) => { - if (req.url.includes('loki1')) { - return of({ - data: { data: { groups: mockRuleGroups1 } }, - ok: true, - status: 200, - statusText: 'OK', - headers: new Headers(), - redirected: false, - type: 'basic', - url: req.url, - config: { url: req.url }, - } as runtime.FetchResponse); - } - throw new Error('Failed to fetch'); - }); - - const result = await fetchAndExtractLokiRecordingRules(); - - // Should still have results from the first datasource - expect(result).toHaveProperty('loki1'); - expect(result.loki1).toHaveLength(2); - expect(result.loki2).toBeUndefined(); - - expect(consoleSpy).toHaveBeenCalled(); - consoleSpy.mockRestore(); - }); - }); - - describe('getLokiQueryForRelatedMetric', () => { - it('should return the expected Loki query for a given metric', () => { - const result = getLokiQueryForRelatedMetric('metric_a_total', 'loki1', mockExtractedRules); - expect(result).toBe('{app="app-A"} |= "error"'); - }); - - it('should return empty string for non-existent data source', () => { - const result = getLokiQueryForRelatedMetric('metric_a_total', 'non-existent', mockExtractedRules); - expect(result).toBe(''); - }); - - it('should return empty string for non-existent metric', () => { - const result = getLokiQueryForRelatedMetric('non_existent_metric', 'loki1', mockExtractedRules); - expect(result).toBe(''); - }); - }); - - describe('getDataSourcesWithRecordingRulesContainingMetric', () => { - it('should find all data sources containing recording rules that define the metric of interest', () => { - const result = getDataSourcesWithRecordingRulesContainingMetric('metric_a_total', mockExtractedRules); - expect(result).toHaveLength(2); - expect(result).toContainEqual({ name: 'Loki Main', uid: 'loki1' }); - expect(result).toContainEqual({ name: 'Loki Secondary', uid: 'loki2' }); - }); - - it('should return empty array for non-existent metric', () => { - const result = getDataSourcesWithRecordingRulesContainingMetric('non_existent_metric', mockExtractedRules); - expect(result).toHaveLength(0); - }); - - it('should find single data source for unique metric', () => { - const result = getDataSourcesWithRecordingRulesContainingMetric('metric_b_total', mockExtractedRules); - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ name: 'Loki Main', uid: 'loki1' }); - }); - }); -}); - -describe('extractRecordingRulesFromRuleGroups', () => { - it('should extract only the first rule from a rule group with same rule names', () => { - const result = extractRecordingRulesFromRuleGroups(mockRuleGroupsWithSameRuleName, mockLokiDS3); - - expect(result.length).toEqual(1); - expect(result).toEqual(mockExtractedRules.loki3); - }); -}); diff --git a/public/app/features/trails/RelatedLogs/RelatedLogsScene.tsx b/public/app/features/trails/RelatedLogs/RelatedLogsScene.tsx index 1b771b40ab0..376f1c52ccb 100644 --- a/public/app/features/trails/RelatedLogs/RelatedLogsScene.tsx +++ b/public/app/features/trails/RelatedLogs/RelatedLogsScene.tsx @@ -18,12 +18,8 @@ import { import { Stack, LinkButton } from '@grafana/ui'; import { Trans } from 'app/core/internationalization'; -import { - fetchAndExtractLokiRecordingRules, - getLokiQueryForRelatedMetric, - getDataSourcesWithRecordingRulesContainingMetric, - type ExtractedRecordingRules, -} from '../Integrations/logsIntegration'; +import { MetricsLogsConnector } from '../Integrations/logs/base'; +import { lokiRecordingRulesConnector } from '../Integrations/logs/lokiRecordingRules'; import { reportExploreMetrics } from '../interactions'; import { VAR_LOGS_DATASOURCE, VAR_LOGS_DATASOURCE_EXPR, VAR_METRIC_EXPR } from '../shared'; @@ -32,7 +28,7 @@ import { NoRelatedLogsScene } from './NoRelatedLogsFoundScene'; export interface RelatedLogsSceneState extends SceneObjectState { controls: SceneObject[]; body: SceneFlexLayout; - lokiRecordingRules: ExtractedRecordingRules; + connectors: MetricsLogsConnector[]; } const LOGS_PANEL_CONTAINER_KEY = 'related_logs/logs_panel_container'; @@ -52,7 +48,7 @@ export class RelatedLogsScene extends SceneObjectBase { }), ], }), - lokiRecordingRules: {}, + connectors: [lokiRecordingRulesConnector], ...state, }); @@ -60,12 +56,12 @@ export class RelatedLogsScene extends SceneObjectBase { } private onActivate() { - fetchAndExtractLokiRecordingRules().then((lokiRecordingRules) => { - const selectedMetric = sceneGraph.interpolate(this, VAR_METRIC_EXPR); - const lokiDatasources = getDataSourcesWithRecordingRulesContainingMetric(selectedMetric, lokiRecordingRules); + const selectedMetric = sceneGraph.interpolate(this, VAR_METRIC_EXPR); + Promise.all(this.state.connectors.map((connector) => connector.getDataSources(selectedMetric))).then((results) => { + const lokiDataSources = results.flat(); const logsPanelContainer = sceneGraph.findByKeyAndType(this, LOGS_PANEL_CONTAINER_KEY, SceneFlexItem); - if (!lokiDatasources?.length) { + if (!lokiDataSources?.length) { logsPanelContainer.setState({ body: new NoRelatedLogsScene({}), }); @@ -88,12 +84,11 @@ export class RelatedLogsScene extends SceneObjectBase { new CustomVariable({ name: VAR_LOGS_DATASOURCE, label: 'Logs data source', - query: lokiDatasources?.map((ds) => `${ds.name} : ${ds.uid}`).join(','), + query: lokiDataSources?.map((ds) => `${ds.name} : ${ds.uid}`).join(','), }), ], }), controls: [new VariableValueSelectors({ layout: 'vertical' })], - lokiRecordingRules, }); } }); @@ -107,11 +102,9 @@ export class RelatedLogsScene extends SceneObjectBase { if (name === VAR_LOGS_DATASOURCE) { const selectedMetric = sceneGraph.interpolate(this, VAR_METRIC_EXPR); const selectedDatasourceUid = sceneGraph.interpolate(this, VAR_LOGS_DATASOURCE_EXPR); - const lokiQuery = getLokiQueryForRelatedMetric( - selectedMetric, - selectedDatasourceUid, - this.state.lokiRecordingRules - ); + const lokiQuery = this.state.connectors + .map((connector) => `(${connector.getLokiQueryExpr(selectedMetric, selectedDatasourceUid)})`) + .join(' and '); if (lokiQuery) { const relatedLogsQuery = sceneGraph.findByKeyAndType(this, RELATED_LOGS_QUERY_KEY, SceneQueryRunner);