ExploreMetrics: Add label cross-reference technique in Related Logs tab (#98775)

* feat: fetch logs containing active labels

* refactor: simplify & handle unhappy path

* refactor: avoid local plugins

* refactor: simplify, clarify, and add operator flexibility

* fix: messaging for no related logs

* refactor: prefer early return

* refactor: avoid problematic `instanceof` checks

* test: `getDataSources` + `getLokiQueryExpr` functionality

* refactor: add clarity

* refactor: clean up

* refactor: add clarity

* feat: account for known label name differences

* refactor: prefer shared type

* test: label name conversions

* test: update to match refactor

* fix: multi-connector query combination

* perf: prefer labels queries to full Loki queries

* fix: limit number of Loki data sources

* docs: explain purpose of parsing solution

* refactor: simplify logs queries

* docs: add clarity

* fix: handle unhappy path w/variable updates

* fix: handle unhealthy Loki data sources

public/app/features/trails/Integrations/logs/base.ts

@@ -12,6 +12,11 @@ export type FoundLokiDataSource = Pick<DataSourceSettings, 'name' | 'uid'>;
  * associating logs with a given metric.
  */
 export interface MetricsLogsConnector {
+  /**
+   * The name of the connector
+   */
+  name: string;
+
   /**
    * Retrieves the Loki data sources associated with the specified metric.
    */

public/app/features/trails/Integrations/logs/labelsCrossReference.test.ts

@@ -0,0 +1,263 @@
+import { type AdHocVariableFilter } from '@grafana/data';
+import { AdHocFiltersVariable, sceneGraph } from '@grafana/scenes';
+
+import { DataTrail } from '../../DataTrail';
+import { RelatedLogsScene } from '../../RelatedLogs/RelatedLogsScene';
+import { VAR_FILTERS } from '../../shared';
+import * as utils from '../../utils';
+
+import { createLabelsCrossReferenceConnector } from './labelsCrossReference';
+
+// Create multiple mock Loki datasources with different behaviors
+const mockLokiDS1 = {
+  uid: 'loki1',
+  name: 'Loki Production',
+  getTagKeys: jest.fn(),
+  getTagValues: jest.fn(),
+};
+
+const mockLokiDS2 = {
+  uid: 'loki2',
+  name: 'Loki Staging',
+  getTagKeys: jest.fn(),
+  getTagValues: jest.fn(),
+};
+
+const mockLokiDS3 = {
+  uid: 'loki3',
+  name: 'Loki Development',
+  getTagKeys: jest.fn(),
+  getTagValues: jest.fn(),
+};
+
+function setVariables(variables: AdHocVariableFilter[] | null) {
+  sceneGraphSpy.mockReturnValue(variables ? createAdHocVariableStub(variables) : null);
+}
+
+const createAdHocVariableStub = (filters: AdHocVariableFilter[]) => {
+  return {
+    __typename: 'AdHocFiltersVariable',
+    state: {
+      name: VAR_FILTERS,
+      type: 'adhoc',
+      filters,
+    },
+  } as unknown as AdHocFiltersVariable;
+};
+
+const filtersStub: AdHocVariableFilter[] = [
+  { key: 'environment', operator: '=', value: 'production' },
+  { key: 'app', operator: '=', value: 'frontend' },
+];
+
+const mockDatasources = [mockLokiDS1, mockLokiDS2, mockLokiDS3];
+const getListSpy = jest.fn().mockReturnValue(mockDatasources);
+const getSpy = jest.fn().mockImplementation(async (uid: string) => {
+  const ds = mockDatasources.find((ds) => ds.uid === uid);
+  if (!ds) {
+    throw new Error(`Datasource with uid ${uid} not found`);
+  }
+  return ds;
+});
+
+jest.mock('@grafana/runtime', () => ({
+  ...jest.requireActual('@grafana/runtime'),
+  getDataSourceSrv: () => ({
+    getList: getListSpy,
+    get: getSpy,
+  }),
+  getTemplateSrv: () => ({
+    getAdhocFilters: jest.fn(),
+  }),
+  getBackendSrv: () => ({
+    get: jest.fn().mockResolvedValue({ status: 'OK' }), // Mock successful health checks
+  }),
+}));
+
+const getTrailForSpy = jest.spyOn(utils, 'getTrailFor');
+const sceneGraphSpy = jest.spyOn(sceneGraph, 'lookupVariable');
+
+const mockScene = {
+  state: {},
+  useState: jest.fn(),
+} as unknown as RelatedLogsScene;
+
+describe('LabelsCrossReferenceConnector', () => {
+  beforeEach(() => {
+    getListSpy.mockClear();
+    sceneGraphSpy.mockClear();
+    getTrailForSpy.mockReturnValue(new DataTrail({}));
+    [mockLokiDS1, mockLokiDS2, mockLokiDS3].forEach((mockLokiDs) => {
+      mockLokiDs.getTagKeys.mockClear();
+      mockLokiDs.getTagValues.mockClear();
+    });
+  });
+
+  describe('getDataSources', () => {
+    it('should find multiple Loki data sources with matching labels', async () => {
+      // DS1: Has all required labels and values
+      mockLokiDS1.getTagKeys.mockResolvedValue([{ text: 'environment' }, { text: 'app' }]);
+      mockLokiDS1.getTagValues.mockResolvedValue([{ text: 'production' }, { text: 'frontend' }]);
+
+      // DS2: Has labels but missing values
+      mockLokiDS2.getTagKeys.mockResolvedValue([{ text: 'environment' }, { text: 'app' }]);
+      mockLokiDS2.getTagValues.mockResolvedValue([
+        { text: 'staging' }, // Different value
+        { text: 'frontend' },
+      ]);
+
+      // DS3: Has all required labels and values
+      mockLokiDS3.getTagKeys.mockResolvedValue([{ text: 'environment' }, { text: 'app' }]);
+      mockLokiDS3.getTagValues.mockResolvedValue([{ text: 'production' }, { text: 'frontend' }]);
+
+      setVariables(filtersStub);
+
+      const connector = createLabelsCrossReferenceConnector(mockScene);
+      const result = await connector.getDataSources();
+
+      expect(result).toHaveLength(2);
+      expect(result).toEqual([
+        { uid: 'loki1', name: 'Loki Production' },
+        { uid: 'loki3', name: 'Loki Development' },
+      ]);
+
+      // Verify that getTagKeys was called for all datasources
+      expect(mockLokiDS1.getTagKeys).toHaveBeenCalled();
+      expect(mockLokiDS2.getTagKeys).toHaveBeenCalled();
+      expect(mockLokiDS3.getTagKeys).toHaveBeenCalled();
+
+      // Verify filters were passed correctly
+      const expectedFilters = [
+        { key: 'environment', operator: '=', value: 'production' },
+        { key: 'app', operator: '=', value: 'frontend' },
+      ];
+
+      expect(mockLokiDS1.getTagKeys).toHaveBeenCalledWith(
+        expect.objectContaining({
+          filters: expect.arrayContaining(expectedFilters),
+        })
+      );
+    });
+
+    it('should handle mixed availability of label keys across datasources', async () => {
+      // DS1: Has all required labels
+      mockLokiDS1.getTagKeys.mockResolvedValue([{ text: 'environment' }, { text: 'app' }]);
+      mockLokiDS1.getTagValues.mockResolvedValue([{ text: 'production' }, { text: 'frontend' }]);
+
+      // DS2: Missing some required labels
+      mockLokiDS2.getTagKeys.mockResolvedValue([
+        { text: 'environment' }, // missing 'app'
+      ]);
+
+      // DS3: Has different set of labels
+      mockLokiDS3.getTagKeys.mockResolvedValue([{ text: 'region' }, { text: 'cluster' }]);
+
+      setVariables(filtersStub);
+
+      const connector = createLabelsCrossReferenceConnector(mockScene);
+      const result = await connector.getDataSources();
+
+      expect(result).toHaveLength(1);
+      expect(result).toEqual([{ uid: 'loki1', name: 'Loki Production' }]);
+
+      // DS2 and DS3 should not have getTagValues called since they don't have all required labels
+      expect(mockLokiDS1.getTagValues).toHaveBeenCalled();
+      expect(mockLokiDS2.getTagValues).not.toHaveBeenCalled();
+      expect(mockLokiDS3.getTagValues).not.toHaveBeenCalled();
+    });
+
+    it('should handle known label name discrepancies across multiple datasources', async () => {
+      const filtersWithKnownLabels: AdHocVariableFilter[] = [
+        { key: 'job', operator: '=', value: 'grafana' },
+        { key: 'instance', operator: '=', value: 'instance1' },
+      ];
+
+      // DS1: Has matching labels with known discrepancies
+      mockLokiDS1.getTagKeys.mockResolvedValue([{ text: 'service_name' }, { text: 'service_instance_id' }]);
+      mockLokiDS1.getTagValues.mockResolvedValue([{ text: 'grafana' }, { text: 'instance1' }]);
+
+      // DS2: Also has transformed label names
+      mockLokiDS2.getTagKeys.mockResolvedValue([{ text: 'service_name' }, { text: 'service_instance_id' }]);
+      mockLokiDS2.getTagValues.mockResolvedValue([{ text: 'grafana' }, { text: 'instance1' }]);
+
+      // DS3: Missing required labels
+      mockLokiDS3.getTagKeys.mockResolvedValue([
+        { text: 'service_name' }, // missing service_instance_id
+      ]);
+
+      setVariables(filtersWithKnownLabels);
+
+      const connector = createLabelsCrossReferenceConnector(mockScene);
+      const result = await connector.getDataSources();
+
+      expect(result).toHaveLength(2);
+      expect(result).toEqual([
+        { uid: 'loki1', name: 'Loki Production' },
+        { uid: 'loki2', name: 'Loki Staging' },
+      ]);
+
+      // Verify that label name mapping was applied correctly
+      expect(mockLokiDS1.getTagKeys).toHaveBeenCalledWith(
+        expect.objectContaining({
+          filters: expect.arrayContaining([
+            expect.objectContaining({ key: 'service_name' }),
+            expect.objectContaining({ key: 'service_instance_id' }),
+          ]),
+        })
+      );
+    });
+  });
+
+  // Rest of the tests remain the same...
+  describe('getLokiQueryExpr', () => {
+    it('should generate correct Loki query expression from filters', () => {
+      setVariables(filtersStub);
+      const connector = createLabelsCrossReferenceConnector(mockScene);
+      const result = connector.getLokiQueryExpr();
+
+      expect(result).toBe('{environment="production",app="frontend"}');
+    });
+
+    it('should handle conversion of known label names', () => {
+      const filtersWithKnownLabels: AdHocVariableFilter[] = [
+        { key: 'job', operator: '=', value: 'grafana' },
+        { key: 'instance', operator: '=', value: 'instance1' },
+      ];
+      setVariables(filtersWithKnownLabels);
+
+      const connector = createLabelsCrossReferenceConnector(mockScene);
+      const result = connector.getLokiQueryExpr();
+
+      expect(result).toBe('{service_name="grafana",service_instance_id="instance1"}');
+    });
+
+    it('should return empty string when no filters are present', () => {
+      setVariables([]);
+      const connector = createLabelsCrossReferenceConnector(mockScene);
+      const result = connector.getLokiQueryExpr();
+
+      expect(result).toBe('');
+    });
+
+    it('should handle missing filters variable', () => {
+      setVariables(null);
+      const connector = createLabelsCrossReferenceConnector(mockScene);
+      const result = connector.getLokiQueryExpr();
+
+      expect(result).toBe('');
+    });
+
+    it('should handle different filter operators', () => {
+      const filtersWithOperators: AdHocVariableFilter[] = [
+        { key: 'environment', operator: '!=', value: 'dev' },
+        { key: 'level', operator: '=~', value: 'error|warn' },
+      ];
+      setVariables(filtersWithOperators);
+
+      const connector = createLabelsCrossReferenceConnector(mockScene);
+      const result = connector.getLokiQueryExpr();
+
+      expect(result).toBe('{environment!="dev",level=~"error|warn"}');
+    });
+  });
+});

public/app/features/trails/Integrations/logs/labelsCrossReference.ts

@@ -0,0 +1,120 @@
+import { TimeRange, type AdHocVariableFilter } from '@grafana/data';
+import { getDataSourceSrv } from '@grafana/runtime';
+import { sceneGraph } from '@grafana/scenes';
+
+import { findHealthyLokiDataSources, RelatedLogsScene } from '../../RelatedLogs/RelatedLogsScene';
+import { VAR_FILTERS } from '../../shared';
+import { getTrailFor, isAdHocVariable } from '../../utils';
+
+import { createMetricsLogsConnector, type FoundLokiDataSource } from './base';
+
+const knownLabelNameDiscrepancies = {
+  job: 'service_name', // `service.name` is `job` in Mimir and `service_name` in Loki
+  instance: 'service_instance_id', // `service.instance.id` is `instance` in Mimir and `service_instance_id` in Loki
+} as const;
+
+function isLabelNameThatShouldBeReplaced(x: string): x is keyof typeof knownLabelNameDiscrepancies {
+  return x in knownLabelNameDiscrepancies;
+}
+
+function replaceKnownLabelNames(labelName: string): string {
+  if (isLabelNameThatShouldBeReplaced(labelName)) {
+    return knownLabelNameDiscrepancies[labelName];
+  }
+
+  return labelName;
+}
+
+/**
+ * Checks if a Loki data source has labels matching the current filters
+ */
+async function hasMatchingLabels(datasourceUid: string, filters: AdHocVariableFilter[], timeRange?: TimeRange) {
+  const ds = await getDataSourceSrv().get(datasourceUid);
+
+  // Get all available label keys for this data source
+  const labelKeys = await ds.getTagKeys?.({
+    timeRange,
+    filters: filters.map(({ key, operator, value }) => ({
+      key: replaceKnownLabelNames(key),
+      operator,
+      value,
+    })),
+  });
+
+  if (!Array.isArray(labelKeys)) {
+    return false;
+  }
+
+  const availableLabels = new Set(labelKeys.map((key) => key.text));
+
+  // Early return if none of our filter labels exist in this data source
+  const mappedFilterLabels = filters.map((f) => replaceKnownLabelNames(f.key));
+  const hasRequiredLabels = mappedFilterLabels.every((label) => availableLabels.has(label));
+  if (!hasRequiredLabels) {
+    return false;
+  }
+
+  // Check if each filter's value exists for its label
+  const results = await Promise.all(
+    filters.map(async (filter) => {
+      const lokiLabelName = replaceKnownLabelNames(filter.key);
+      const values = await ds.getTagValues?.({
+        key: lokiLabelName,
+        timeRange,
+        filters,
+      });
+
+      if (!Array.isArray(values)) {
+        return false;
+      }
+
+      return values.some((v) => v.text === filter.value);
+    })
+  );
+
+  // If any of the filters have no matching values, return false
+  return results.every(Boolean);
+}
+
+export const createLabelsCrossReferenceConnector = (scene: RelatedLogsScene) => {
+  return createMetricsLogsConnector({
+    name: 'labelsCrossReference',
+    async getDataSources(): Promise<FoundLokiDataSource[]> {
+      const trail = getTrailFor(scene);
+      const filtersVariable = sceneGraph.lookupVariable(VAR_FILTERS, trail);
+
+      if (!isAdHocVariable(filtersVariable) || !filtersVariable.state.filters.length) {
+        return [];
+      }
+
+      const filters = filtersVariable.state.filters.map(({ key, operator, value }) => ({ key, operator, value }));
+
+      // Get current time range if available
+      const timeRange = scene.state.$timeRange?.state.value;
+
+      const lokiDataSources = await findHealthyLokiDataSources();
+      const results = await Promise.all(
+        lokiDataSources.map(async ({ uid, name }) => {
+          const hasLabels = await hasMatchingLabels(uid, filters, timeRange);
+          return hasLabels ? { uid, name } : null;
+        })
+      );
+
+      return results.filter((ds): ds is FoundLokiDataSource => ds !== null);
+    },
+    getLokiQueryExpr(): string {
+      const trail = getTrailFor(scene);
+      const filtersVariable = sceneGraph.lookupVariable(VAR_FILTERS, trail);
+
+      if (!isAdHocVariable(filtersVariable) || !filtersVariable.state.filters.length) {
+        return '';
+      }
+
+      const labelValuePairs = filtersVariable.state.filters.map(
+        (filter) => `${replaceKnownLabelNames(filter.key)}${filter.operator}"${filter.value}"`
+      );
+
+      return `{${labelValuePairs.join(',')}}`; // e.g. `{environment="dev",region="us-west-1"}`
+    },
+  });
+};

public/app/features/trails/Integrations/logs/lokiRecordingRules.test.ts

@@ -105,7 +105,7 @@ jest.mock('@grafana/runtime', () => ({
   getBackendSrv: () => ({
     fetch: fetchSpy,
     delete: jest.fn(),
-    get: jest.fn(),
+    get: jest.fn().mockResolvedValue({ status: 'OK' }), // Mock successful health checks
     patch: jest.fn(),
     post: jest.fn(),
     put: jest.fn(),
@@ -121,7 +121,7 @@ describe('LokiRecordingRulesConnector', () => {
     getListSpy.mockClear();
     fetchSpy.mockClear();
     fetchSpy.mockImplementation(defaultFetchImpl);
-    consoleSpy = jest.spyOn(console, 'error').mockImplementation();
+    consoleSpy = jest.spyOn(console, 'warn').mockImplementation();
   });
 
   afterEach(() => {
@@ -138,7 +138,7 @@ describe('LokiRecordingRulesConnector', () => {
       expect(result).toContainEqual({ name: 'Loki Secondary', uid: 'loki2' });
 
       // Verify underlying calls
-      expect(getListSpy).toHaveBeenCalledWith({ logs: true });
+      expect(getListSpy).toHaveBeenCalledWith({ logs: true, type: 'loki', filter: expect.any(Function) });
       expect(fetchSpy).toHaveBeenCalledTimes(2);
     });
 

public/app/features/trails/Integrations/logs/lokiRecordingRules.ts

@@ -1,9 +1,11 @@
 import { lastValueFrom } from 'rxjs';
 
 import type { DataSourceInstanceSettings, DataSourceJsonData } from '@grafana/data';
-import { getBackendSrv, getDataSourceSrv, type BackendSrvRequest, type FetchResponse } from '@grafana/runtime';
+import { getBackendSrv, type BackendSrvRequest, type FetchResponse } from '@grafana/runtime';
 import { getLogQueryFromMetricsQuery } from 'app/plugins/datasource/loki/queryUtils';
 
+import { findHealthyLokiDataSources } from '../../RelatedLogs/RelatedLogsScene';
+
 import { createMetricsLogsConnector, type FoundLokiDataSource } from './base';
 
 export interface RecordingRuleGroup {
@@ -36,13 +38,18 @@ export interface ExtractedRecordingRules {
 async function fetchRecordingRuleGroups(datasourceSettings: DataSourceInstanceSettings<DataSourceJsonData>) {
   const recordingRuleUrl = `api/prometheus/${datasourceSettings.uid}/api/v1/rules`;
   const recordingRules: BackendSrvRequest = { url: recordingRuleUrl, showErrorAlert: false, showSuccessAlert: false };
-  const { data } = await lastValueFrom<
+  const res = await lastValueFrom<
     FetchResponse<{
       data: { groups: RecordingRuleGroup[] };
     }>
   >(getBackendSrv().fetch(recordingRules));
 
-  return data.data.groups;
+  if (!res.ok) {
+    console.warn(`Failed to fetch recording rules from Loki data source: ${datasourceSettings.name}`);
+    return [];
+  }
+
+  return res.data.data.groups;
 }
 
 /**
@@ -150,9 +157,7 @@ export function getLokiQueryForRelatedMetric(
  * @throws Will log an error to the console if fetching or extracting rules fails for any data source.
  */
 export async function fetchAndExtractLokiRecordingRules() {
-  const lokiDataSources = getDataSourceSrv()
-    .getList({ logs: true })
-    .filter((ds) => ds.type === 'loki');
+  const lokiDataSources = await findHealthyLokiDataSources();
   const extractedRecordingRules: ExtractedRecordingRules = {};
   await Promise.all(
     lokiDataSources.map(async (dataSource) => {
@@ -161,7 +166,7 @@ export async function fetchAndExtractLokiRecordingRules() {
         const extractedRules = extractRecordingRulesFromRuleGroups(ruleGroups, dataSource);
         extractedRecordingRules[dataSource.uid] = extractedRules;
       } catch (err) {
-        console.error(err);
+        console.warn(err);
       }
     })
   );
@@ -173,6 +178,7 @@ const createLokiRecordingRulesConnector = () => {
   let lokiRecordingRules: ExtractedRecordingRules = {};
 
   return createMetricsLogsConnector({
+    name: 'lokiRecordingRules',
     async getDataSources(selectedMetric: string): Promise<FoundLokiDataSource[]> {
       lokiRecordingRules = await fetchAndExtractLokiRecordingRules();
       const lokiDataSources = getDataSourcesWithRecordingRulesContainingMetric(selectedMetric, lokiRecordingRules);

public/app/features/trails/RelatedLogs/NoRelatedLogsFoundScene.tsx

@@ -1,9 +1,14 @@
+import { css } from '@emotion/css';
+
+import { GrafanaTheme2 } from '@grafana/data';
 import { SceneObjectBase, type SceneObjectState } from '@grafana/scenes';
-import { Stack, Text, TextLink } from '@grafana/ui';
+import { Stack, Text, TextLink, useStyles2 } from '@grafana/ui';
 import { Trans } from 'app/core/internationalization';
 
 export class NoRelatedLogsScene extends SceneObjectBase<SceneObjectState> {
   static readonly Component = () => {
+    const styles = useStyles2(getStyles);
+
     return (
       <Stack direction="column" gap={1}>
         <Text color="warning">
@@ -12,18 +17,28 @@ export class NoRelatedLogsScene extends SceneObjectBase<SceneObjectState> {
           </Trans>
         </Text>
         <Text>
-          <Trans i18nKey="explore-metrics.related-logs.relatedLogsUnavailableBeforeDocsLink">
-            Related logs are not available for this metric. Try selecting a metric created by a{' '}
-          </Trans>
-          <TextLink external href="https://grafana.com/docs/loki/latest/alert/#recording-rules">
-            <Trans i18nKey="explore-metrics.related-logs.docsLink">Loki Recording Rule</Trans>
-          </TextLink>
-          <Trans i18nKey="explore-metrics.related-logs.relatedLogsUnavailableAfterDocsLink">
-            , or check back later as we expand the various methods for establishing connections between metrics and
-            logs.
+          <Trans i18nKey="explore-metrics.related-logs.relatedLogsUnavailable">
+            No related logs found. To see related logs, you can either:
+            <ul className={styles.list}>
+              <li>adjust the label filter to find logs with the same labels as the currently-selected metric</li>
+              <li>
+                select a metric created by a{' '}
+                <TextLink external href="https://grafana.com/docs/loki/latest/alert/#recording-rules">
+                  <Trans i18nKey="explore-metrics.related-logs.LrrDocsLink">Loki Recording Rule</Trans>
+                </TextLink>
+              </li>
+            </ul>
           </Trans>
         </Text>
       </Stack>
     );
   };
 }
+
+function getStyles(theme: GrafanaTheme2) {
+  return {
+    list: css({
+      paddingLeft: theme.spacing(2),
+    }),
+  };
+}

public/app/features/trails/RelatedLogs/RelatedLogsScene.tsx

@@ -1,4 +1,5 @@
-import { config } from '@grafana/runtime';
+import { DataSourceInstanceSettings, DataSourceJsonData } from '@grafana/data';
+import { config, getBackendSrv, getDataSourceSrv } from '@grafana/runtime';
 import {
   CustomVariable,
   PanelBuilders,
@@ -19,9 +20,11 @@ import { Stack, LinkButton } from '@grafana/ui';
 import { Trans } from 'app/core/internationalization';
 
 import { MetricsLogsConnector } from '../Integrations/logs/base';
+import { createLabelsCrossReferenceConnector } from '../Integrations/logs/labelsCrossReference';
 import { lokiRecordingRulesConnector } from '../Integrations/logs/lokiRecordingRules';
 import { reportExploreMetrics } from '../interactions';
-import { VAR_LOGS_DATASOURCE, VAR_LOGS_DATASOURCE_EXPR, VAR_METRIC_EXPR } from '../shared';
+import { VAR_FILTERS, VAR_LOGS_DATASOURCE, VAR_LOGS_DATASOURCE_EXPR, VAR_METRIC, VAR_METRIC_EXPR } from '../shared';
+import { isConstantVariable, isCustomVariable } from '../utils';
 
 import { NoRelatedLogsScene } from './NoRelatedLogsFoundScene';
 
@@ -48,7 +51,7 @@ export class RelatedLogsScene extends SceneObjectBase<RelatedLogsSceneState> {
           }),
         ],
       }),
-      connectors: [lokiRecordingRulesConnector],
+      connectors: [],
       ...state,
     });
 
@@ -56,68 +59,99 @@ export class RelatedLogsScene extends SceneObjectBase<RelatedLogsSceneState> {
   }
 
   private onActivate() {
-    const selectedMetric = sceneGraph.interpolate(this, VAR_METRIC_EXPR);
-    Promise.all(this.state.connectors.map((connector) => connector.getDataSources(selectedMetric))).then((results) => {
-      const lokiDataSources = results.flat();
-      const logsPanelContainer = sceneGraph.findByKeyAndType(this, LOGS_PANEL_CONTAINER_KEY, SceneFlexItem);
+    this.setState({
+      connectors: [lokiRecordingRulesConnector, createLabelsCrossReferenceConnector(this)],
+    });
+    this.setLogsDataSourceVar();
+  }
 
-      if (!lokiDataSources?.length) {
-        logsPanelContainer.setState({
-          body: new NoRelatedLogsScene({}),
-        });
-      } else {
-        logsPanelContainer.setState({
-          body: PanelBuilders.logs()
-            .setTitle('Logs')
-            .setData(
-              new SceneQueryRunner({
-                datasource: { uid: VAR_LOGS_DATASOURCE_EXPR },
-                queries: [],
-                key: RELATED_LOGS_QUERY_KEY,
-              })
-            )
-            .build(),
-        });
-        this.setState({
-          $variables: new SceneVariableSet({
-            variables: [
-              new CustomVariable({
-                name: VAR_LOGS_DATASOURCE,
-                label: 'Logs data source',
-                query: lokiDataSources?.map((ds) => `${ds.name} : ${ds.uid}`).join(','),
-              }),
-            ],
-          }),
-          controls: [new VariableValueSelectors({ layout: 'vertical' })],
-        });
+  private setLogsDataSourceVar(): Promise<void> {
+    const selectedMetric = sceneGraph.interpolate(this, VAR_METRIC_EXPR);
+    return Promise.all(this.state.connectors.map((connector) => connector.getDataSources(selectedMetric))).then(
+      (results) => {
+        const lokiDataSources = results.flat().slice(0, 10); // limit to the first ten matching Loki data sources
+        const logsPanelContainer = sceneGraph.findByKeyAndType(this, LOGS_PANEL_CONTAINER_KEY, SceneFlexItem);
+
+        if (!lokiDataSources?.length) {
+          logsPanelContainer.setState({
+            body: new NoRelatedLogsScene({}),
+          });
+          this.setState({ $variables: undefined, controls: undefined });
+        } else {
+          logsPanelContainer.setState({
+            body: PanelBuilders.logs()
+              .setTitle('Logs')
+              .setData(
+                new SceneQueryRunner({
+                  datasource: { uid: VAR_LOGS_DATASOURCE_EXPR },
+                  queries: [],
+                  key: RELATED_LOGS_QUERY_KEY,
+                })
+              )
+              .build(),
+          });
+          this.setState({
+            $variables: new SceneVariableSet({
+              variables: [
+                new CustomVariable({
+                  name: VAR_LOGS_DATASOURCE,
+                  label: 'Logs data source',
+                  query: lokiDataSources?.map((ds) => `${ds.name} : ${ds.uid}`).join(','),
+                }),
+              ],
+            }),
+            controls: [new VariableValueSelectors({ layout: 'vertical' })],
+          });
+        }
       }
+    );
+  }
+
+  private updateLokiQuery() {
+    const selectedDatasourceVar = sceneGraph.lookupVariable(VAR_LOGS_DATASOURCE, this);
+    const selectedMetricVar = sceneGraph.lookupVariable(VAR_METRIC, this);
+
+    if (!isCustomVariable(selectedDatasourceVar) || !isConstantVariable(selectedMetricVar)) {
+      return;
+    }
+
+    const selectedMetric = selectedMetricVar.getValue();
+    const selectedDatasourceUid = selectedDatasourceVar.getValue();
+
+    if (typeof selectedMetric !== 'string' || typeof selectedDatasourceUid !== 'string') {
+      return;
+    }
+
+    const lokiQueries = this.state.connectors.reduce<Record<string, string>>((acc, connector, idx) => {
+      const lokiExpr = connector.getLokiQueryExpr(selectedMetric, selectedDatasourceUid);
+
+      if (lokiExpr) {
+        acc[connector.name ?? `connector-${idx}`] = lokiExpr;
+      }
+
+      return acc;
+    }, {});
+    const relatedLogsQuery = sceneGraph.findByKeyAndType(this, RELATED_LOGS_QUERY_KEY, SceneQueryRunner);
+    relatedLogsQuery.setState({
+      queries: Object.keys(lokiQueries).map((connectorName) => ({
+        refId: `RelatedLogs-${connectorName}`,
+        expr: lokiQueries[connectorName],
+        maxLines: 100,
+      })),
     });
   }
 
   protected _variableDependency = new VariableDependencyConfig(this, {
-    variableNames: [VAR_LOGS_DATASOURCE],
+    variableNames: [VAR_LOGS_DATASOURCE, VAR_FILTERS],
     onReferencedVariableValueChanged: (variable: SceneVariable) => {
       const { name } = variable.state;
 
       if (name === VAR_LOGS_DATASOURCE) {
-        const selectedMetric = sceneGraph.interpolate(this, VAR_METRIC_EXPR);
-        const selectedDatasourceUid = sceneGraph.interpolate(this, VAR_LOGS_DATASOURCE_EXPR);
-        const lokiQuery = this.state.connectors
-          .map((connector) => `(${connector.getLokiQueryExpr(selectedMetric, selectedDatasourceUid)})`)
-          .join(' and ');
+        this.updateLokiQuery();
+      }
 
-        if (lokiQuery) {
-          const relatedLogsQuery = sceneGraph.findByKeyAndType(this, RELATED_LOGS_QUERY_KEY, SceneQueryRunner);
-          relatedLogsQuery.setState({
-            queries: [
-              {
-                refId: 'A',
-                expr: lokiQuery,
-                maxLines: 100,
-              },
-            ],
-          });
-        }
+      if (name === VAR_FILTERS) {
+        this.setLogsDataSourceVar().then(() => this.updateLokiQuery());
       }
     },
   });
@@ -129,13 +163,10 @@ export class RelatedLogsScene extends SceneObjectBase<RelatedLogsSceneState> {
       <div>
         <Stack gap={1} direction={'column'} grow={1}>
           <Stack gap={1} direction={'row'} grow={1} justifyContent={'space-between'} alignItems={'start'}>
-            {controls && (
-              <Stack gap={1}>
-                {controls.map((control) => (
-                  <control.Component key={control.state.key} model={control} />
-                ))}
-              </Stack>
-            )}
+            <Stack gap={1}>
+              {controls?.map((control) => <control.Component key={control.state.key} model={control} />)}
+            </Stack>
+
             <LinkButton
               href={`${config.appSubUrl}/a/grafana-lokiexplore-app`} // We prefix with the appSubUrl for environments that don't host grafana at the root.
               target="_blank"
@@ -157,3 +188,35 @@ export class RelatedLogsScene extends SceneObjectBase<RelatedLogsSceneState> {
 export function buildRelatedLogsScene() {
   return new RelatedLogsScene({});
 }
+
+export async function findHealthyLokiDataSources() {
+  const lokiDataSources = getDataSourceSrv().getList({
+    logs: true,
+    type: 'loki',
+    filter: (ds) => ds.uid !== 'grafana',
+  });
+  const healthyLokiDataSources: Array<DataSourceInstanceSettings<DataSourceJsonData>> = [];
+  const unhealthyLokiDataSources: Array<DataSourceInstanceSettings<DataSourceJsonData>> = [];
+
+  await Promise.all(
+    lokiDataSources.map((ds) =>
+      getBackendSrv()
+        .get(`/api/datasources/${ds.id}/health`, undefined, undefined, {
+          showSuccessAlert: false,
+          showErrorAlert: false,
+        })
+        .then((health) =>
+          health?.status === 'OK' ? healthyLokiDataSources.push(ds) : unhealthyLokiDataSources.push(ds)
+        )
+        .catch(() => unhealthyLokiDataSources.push(ds))
+    )
+  );
+
+  if (unhealthyLokiDataSources.length) {
+    console.warn(
+      `Found ${unhealthyLokiDataSources.length} unhealthy Loki data sources: ${unhealthyLokiDataSources.map((ds) => ds.name).join(', ')}`
+    );
+  }
+
+  return healthyLokiDataSources;
+}

public/app/features/trails/utils.ts

@@ -14,6 +14,8 @@ import { getPrometheusTime } from '@grafana/prometheus/src/language_utils';
 import { config, FetchResponse, getBackendSrv, getDataSourceSrv } from '@grafana/runtime';
 import {
   AdHocFiltersVariable,
+  ConstantVariable,
+  CustomVariable,
   sceneGraph,
   SceneObject,
   SceneObjectState,
@@ -35,6 +37,18 @@ import { MetricDatasourceHelper } from './helpers/MetricDatasourceHelper';
 import { sortResources } from './otel/util';
 import { LOGS_METRIC, TRAILS_ROUTE, VAR_DATASOURCE_EXPR, VAR_OTEL_AND_METRIC_FILTERS } from './shared';
 
+export function isAdHocVariable(variable: SceneVariable | null): variable is AdHocFiltersVariable {
+  return variable !== null && variable.state.type === 'adhoc';
+}
+
+export function isCustomVariable(variable: SceneVariable | null): variable is CustomVariable {
+  return variable !== null && variable.state.type === 'custom';
+}
+
+export function isConstantVariable(variable: SceneVariable | null): variable is ConstantVariable {
+  return variable !== null && variable.state.type === 'constant';
+}
+
 export function getTrailFor(model: SceneObject): DataTrail {
   return sceneGraph.getAncestor(model, DataTrail);
 }

public/locales/en-US/grafana.json

@@ -1323,10 +1323,9 @@
       "sortBy": "Sort by"
     },
     "related-logs": {
-      "docsLink": "Loki Recording Rule",
+      "LrrDocsLink": "Loki Recording Rule",
       "openExploreLogs": "Open Explore Logs",
-      "relatedLogsUnavailableAfterDocsLink": ", or check back later as we expand the various methods for establishing connections between metrics and logs.",
-      "relatedLogsUnavailableBeforeDocsLink": "Related logs are not available for this metric. Try selecting a metric created by a ",
+      "relatedLogsUnavailable": "No related logs found. To see related logs, you can either:<1><0>adjust the label filter to find logs with the same labels as the currently-selected metric</0><1>select a metric created by a <2><0>Loki Recording Rule</0></2></1></1>",
       "warnExperimentalFeature": "Related logs is an experimental feature."
     },
     "viewBy": "View by"

public/locales/pseudo-LOCALE/grafana.json

@@ -1323,10 +1323,9 @@
       "sortBy": "Ŝőřŧ þy"
     },
     "related-logs": {
-      "docsLink": "Ŀőĸį Ŗęčőřđįʼnģ Ŗūľę",
+      "LrrDocsLink": "Ŀőĸį Ŗęčőřđįʼnģ Ŗūľę",
       "openExploreLogs": "Øpęʼn Ēχpľőřę Ŀőģş",
-      "relatedLogsUnavailableAfterDocsLink": ", őř čĥęčĸ þäčĸ ľäŧęř äş ŵę ęχpäʼnđ ŧĥę väřįőūş męŧĥőđş ƒőř ęşŧäþľįşĥįʼnģ čőʼnʼnęčŧįőʼnş þęŧŵęęʼn męŧřįčş äʼnđ ľőģş.",
-      "relatedLogsUnavailableBeforeDocsLink": "Ŗęľäŧęđ ľőģş äřę ʼnőŧ äväįľäþľę ƒőř ŧĥįş męŧřįč. Ŧřy şęľęčŧįʼnģ ä męŧřįč čřęäŧęđ þy ä ",
+      "relatedLogsUnavailable": "Ńő řęľäŧęđ ľőģş ƒőūʼnđ. Ŧő şęę řęľäŧęđ ľőģş, yőū čäʼn ęįŧĥęř:<1><0>äđĵūşŧ ŧĥę ľäþęľ ƒįľŧęř ŧő ƒįʼnđ ľőģş ŵįŧĥ ŧĥę şämę ľäþęľş äş ŧĥę čūřřęʼnŧľy-şęľęčŧęđ męŧřįč</0><1>şęľęčŧ ä męŧřįč čřęäŧęđ þy ä <2><0>Ŀőĸį Ŗęčőřđįʼnģ Ŗūľę</0></2></1></1>",
       "warnExperimentalFeature": "Ŗęľäŧęđ ľőģş įş äʼn ęχpęřįmęʼnŧäľ ƒęäŧūřę."
     },
     "viewBy": "Vįęŵ þy"