NodeGraph: Improve view traces for uninstrumented services (#98442)

* NodeGraph: Improve view traces for uninstrumented services

* Switch to onBuildUrl and more peer attributes

* Update unit tests

* Added test for new logic

* Open traces in same tab

* Update the tests

* bring back internal link

* Update public/app/plugins/datasource/tempo/datasource.ts

Co-authored-by: Joey <90795735+joey-grafana@users.noreply.github.com>

* Revert export of generateInternalHref

* Update tests after change from onBuildUrl to query function

---------

Co-authored-by: Domas Lapinskas <domasx2@gmail.com>
Co-authored-by: Joey <90795735+joey-grafana@users.noreply.github.com>

packages/grafana-data/src/types/dataLink.ts

@@ -1,3 +1,4 @@
+import { ScopedVars } from './ScopedVars';
 import { ExploreCorrelationHelperData, ExplorePanelsState } from './explore';
 import { InterpolateFunction } from './panel';
 import { DataQuery } from './query';
@@ -80,7 +81,7 @@ export interface DataLinkTransformationConfig {
 
 /** @internal */
 export interface InternalDataLink<T extends DataQuery = any> {
-  query: T;
+  query: T | ((options: { replaceVariables: InterpolateFunction; scopedVars: ScopedVars }) => T);
   datasourceUid: string;
   datasourceName: string; // used as a title if `DataLink.title` is empty
   panelsState?: ExplorePanelsState;

packages/grafana-data/src/utils/dataLinks.ts

@@ -38,7 +38,11 @@ export type LinkToExploreOptions = {
 export function mapInternalLinkToExplore(options: LinkToExploreOptions): LinkModel<Field> {
   const { onClickFn, replaceVariables, link, scopedVars, range, field, internalLink } = options;
 
-  const interpolatedQuery = interpolateObject(link.internal?.query, scopedVars, replaceVariables);
+  const query =
+    typeof link.internal?.query === 'function'
+      ? link.internal.query({ replaceVariables, scopedVars })
+      : internalLink.query;
+  const interpolatedQuery = interpolateObject(query, scopedVars, replaceVariables);
   const interpolatedPanelsState = interpolateObject(link.internal?.panelsState, scopedVars, replaceVariables);
   const interpolatedCorrelationData = interpolateObject(link.meta?.correlationData, scopedVars, replaceVariables);
   const title = link.title ? link.title : internalLink.datasourceName;

packages/grafana-data/src/utils/nodeGraph.ts

@@ -44,4 +44,7 @@ export enum NodeGraphDataFrameFieldNames {
 
   // Supplies a fixed Y position for the node to have in the finished graph.
   fixedY = 'fixedy',
+
+  // Whether the node is instrumented or not
+  isInstrumented = 'isinstrumented',
 }

public/app/plugins/datasource/tempo/datasource.test.ts

@@ -15,6 +15,8 @@ import {
   DataQueryRequest,
   getTimeZone,
   PluginMetaInfo,
+  DataLink,
+  NodeGraphDataFrameFieldNames,
 } from '@grafana/data';
 import {
   BackendDataSourceResponse,
@@ -544,6 +546,32 @@ describe('Tempo service graph view', () => {
     expect(response.data[1].fields[0]?.config?.links?.length).toBeGreaterThan(0);
     expect(response.data[1].fields[0]?.config?.links).toEqual(serviceGraphLinks);
 
+    const viewServicesLink = response.data[1].fields[0]?.config?.links.find(
+      (link: DataLink) => link.title === 'View traces'
+    );
+    expect(viewServicesLink).toBeDefined();
+    expect(viewServicesLink.internal.query({ replaceVariables: replaceVariablesInstrumented })).toEqual({
+      refId: 'A',
+      queryType: 'traceqlSearch',
+      filters: [
+        {
+          id: 'service-name',
+          operator: '=',
+          scope: 'resource',
+          tag: 'service.name',
+          value: 'my-service',
+          valueType: 'string',
+        },
+      ],
+    });
+    expect(viewServicesLink.internal.query({ replaceVariables: replaceVariablesUninstrumented })).toEqual({
+      refId: 'A',
+      queryType: 'traceql',
+      filters: [],
+      query:
+        '{span.db.name="my-service" || span.db.system="my-service" || span.peer.service="my-service" || span.messaging.system="my-service" || span.net.peer.name="my-service"}',
+    });
+
     expect(response.data[2].name).toBe('Edges');
     expect(response.data[2].fields[0].values.length).toBe(2);
   });
@@ -584,14 +612,26 @@ describe('Tempo service graph view', () => {
       'sum by (client, server) (rate(traces_service_graph_request_server_seconds_sum{ foo="bar" }[$__range]))'
     );
     expect(nthQuery(0).targets[1].expr).toBe(
-      'sum by (client, server) (rate(traces_service_graph_request_total{ foo="bar" }[$__range]))'
+      'group by (client, connection_type, server) (traces_service_graph_request_server_seconds_sum{ foo="bar" })'
     );
     expect(nthQuery(0).targets[2].expr).toBe(
-      'sum by (client, server) (rate(traces_service_graph_request_failed_total{ foo="bar" }[$__range]))'
+      'sum by (client, server) (rate(traces_service_graph_request_total{ foo="bar" }[$__range]))'
     );
     expect(nthQuery(0).targets[3].expr).toBe(
+      'group by (client, connection_type, server) (traces_service_graph_request_total{ foo="bar" })'
+    );
+    expect(nthQuery(0).targets[4].expr).toBe(
+      'sum by (client, server) (rate(traces_service_graph_request_failed_total{ foo="bar" }[$__range]))'
+    );
+    expect(nthQuery(0).targets[5].expr).toBe(
+      'group by (client, connection_type, server) (traces_service_graph_request_failed_total{ foo="bar" })'
+    );
+    expect(nthQuery(0).targets[6].expr).toBe(
       'sum by (client, server) (rate(traces_service_graph_request_server_seconds_bucket{ foo="bar" }[$__range]))'
     );
+    expect(nthQuery(0).targets[7].expr).toBe(
+      'group by (client, connection_type, server) (traces_service_graph_request_server_seconds_bucket{ foo="bar" })'
+    );
   });
 
   it('runs correct queries with multiple serviceMapQuery defined', async () => {
@@ -632,14 +672,26 @@ describe('Tempo service graph view', () => {
       'sum by (client, server) (rate(traces_service_graph_request_server_seconds_sum{ foo="bar" }[$__range])) OR sum by (client, server) (rate(traces_service_graph_request_server_seconds_sum{baz="bad"}[$__range]))'
     );
     expect(nthQuery(0).targets[1].expr).toBe(
-      'sum by (client, server) (rate(traces_service_graph_request_total{ foo="bar" }[$__range])) OR sum by (client, server) (rate(traces_service_graph_request_total{baz="bad"}[$__range]))'
+      'group by (client, connection_type, server) (traces_service_graph_request_server_seconds_sum{ foo="bar" }) OR group by (client, connection_type, server) (traces_service_graph_request_server_seconds_sum{baz="bad"})'
     );
     expect(nthQuery(0).targets[2].expr).toBe(
-      'sum by (client, server) (rate(traces_service_graph_request_failed_total{ foo="bar" }[$__range])) OR sum by (client, server) (rate(traces_service_graph_request_failed_total{baz="bad"}[$__range]))'
+      'sum by (client, server) (rate(traces_service_graph_request_total{ foo="bar" }[$__range])) OR sum by (client, server) (rate(traces_service_graph_request_total{baz="bad"}[$__range]))'
     );
     expect(nthQuery(0).targets[3].expr).toBe(
+      'group by (client, connection_type, server) (traces_service_graph_request_total{ foo="bar" }) OR group by (client, connection_type, server) (traces_service_graph_request_total{baz="bad"})'
+    );
+    expect(nthQuery(0).targets[4].expr).toBe(
+      'sum by (client, server) (rate(traces_service_graph_request_failed_total{ foo="bar" }[$__range])) OR sum by (client, server) (rate(traces_service_graph_request_failed_total{baz="bad"}[$__range]))'
+    );
+    expect(nthQuery(0).targets[5].expr).toBe(
+      'group by (client, connection_type, server) (traces_service_graph_request_failed_total{ foo="bar" }) OR group by (client, connection_type, server) (traces_service_graph_request_failed_total{baz="bad"})'
+    );
+    expect(nthQuery(0).targets[6].expr).toBe(
       'sum by (client, server) (rate(traces_service_graph_request_server_seconds_bucket{ foo="bar" }[$__range])) OR sum by (client, server) (rate(traces_service_graph_request_server_seconds_bucket{baz="bad"}[$__range]))'
     );
+    expect(nthQuery(0).targets[7].expr).toBe(
+      'group by (client, connection_type, server) (traces_service_graph_request_server_seconds_bucket{ foo="bar" }) OR group by (client, connection_type, server) (traces_service_graph_request_server_seconds_bucket{baz="bad"})'
+    );
   });
 
   it('should build expr correctly', () => {
@@ -803,27 +855,33 @@ describe('Tempo service graph view', () => {
           url: '',
           title: 'View traces',
           internal: {
-            query: {
-              refId: 'A',
-              queryType: 'traceqlSearch',
-              filters: [
-                {
-                  id: 'service-name',
-                  operator: '=',
-                  scope: 'resource',
-                  tag: 'service.name',
-                  value: '${__data.fields.target}',
-                  valueType: 'string',
-                },
-              ],
-            },
-            datasourceUid: 'EbPO1fYnz',
             datasourceName: '',
+            datasourceUid: 'EbPO1fYnz',
+            query: expect.any(Function),
           },
         },
       ],
     };
     expect(fieldConfig).toStrictEqual(resultObj);
+
+    const viewServicesLink: DataLink | undefined = fieldConfig.links.find(
+      (link: DataLink) => link.title === 'View traces'
+    );
+    expect(viewServicesLink).toBeDefined();
+    expect(viewServicesLink!.internal!.query({ replaceVariables: replaceVariablesInstrumented })).toEqual({
+      refId: 'A',
+      queryType: 'traceqlSearch',
+      filters: [
+        {
+          id: 'service-name',
+          operator: '=',
+          scope: 'resource',
+          tag: 'service.name',
+          value: 'my-service',
+          valueType: 'string',
+        },
+      ],
+    });
   });
 
   it('should get field config correctly when namespaces are present', () => {
@@ -894,35 +952,41 @@ describe('Tempo service graph view', () => {
           url: '',
           title: 'View traces',
           internal: {
-            query: {
-              queryType: 'traceqlSearch',
-              refId: 'A',
-              filters: [
-                {
-                  id: 'service-namespace',
-                  operator: '=',
-                  scope: 'resource',
-                  tag: 'service.namespace',
-                  value: '${__data.fields.targetNamespace}',
-                  valueType: 'string',
-                },
-                {
-                  id: 'service-name',
-                  operator: '=',
-                  scope: 'resource',
-                  tag: 'service.name',
-                  value: '${__data.fields.targetName}',
-                  valueType: 'string',
-                },
-              ],
-            },
-            datasourceUid: 'EbPO1fYnz',
             datasourceName: '',
+            datasourceUid: 'EbPO1fYnz',
+            query: expect.any(Function),
           },
         },
       ],
     };
     expect(fieldConfig).toStrictEqual(resultObj);
+
+    const viewServicesLink: DataLink | undefined = fieldConfig.links.find(
+      (link: DataLink) => link.title === 'View traces'
+    );
+    expect(viewServicesLink).toBeDefined();
+    expect(viewServicesLink!.internal!.query({ replaceVariables: replaceVariablesInstrumented })).toEqual({
+      refId: 'A',
+      queryType: 'traceqlSearch',
+      filters: [
+        {
+          id: 'service-namespace',
+          operator: '=',
+          scope: 'resource',
+          tag: 'service.namespace',
+          value: 'my-namespace',
+          valueType: 'string',
+        },
+        {
+          id: 'service-name',
+          operator: '=',
+          scope: 'resource',
+          tag: 'service.name',
+          value: 'my-service',
+          valueType: 'string',
+        },
+      ],
+    });
   });
 
   it('should get rate aligned values correctly', () => {
@@ -1435,26 +1499,31 @@ const serviceGraphLinks = [
     url: '',
     title: 'View traces',
     internal: {
-      query: {
+      query: expect.any(Function),
-        refId: 'A',
-        queryType: 'traceqlSearch',
-        filters: [
-          {
-            id: 'service-name',
-            operator: '=',
-            scope: 'resource',
-            tag: 'service.name',
-            value: '${__data.fields.id}',
-            valueType: 'string',
-          },
-        ],
-      } as TempoQuery,
       datasourceUid: 'gdev-tempo',
       datasourceName: 'Tempo',
     },
   },
 ];
 
+const replaceVariablesInstrumented = (variable: string): string => {
+  const variables: Record<string, string> = {
+    [`\${__data.fields.${NodeGraphDataFrameFieldNames.title}}`]: 'my-service',
+    [`\${__data.fields.${NodeGraphDataFrameFieldNames.subTitle}}`]: 'my-namespace',
+    [`\${__data.fields.${NodeGraphDataFrameFieldNames.isInstrumented}}`]: 'true',
+  };
+  return variables[variable];
+};
+
+const replaceVariablesUninstrumented = (variable: string): string => {
+  const variables: Record<string, string> = {
+    [`\${__data.fields.${NodeGraphDataFrameFieldNames.title}}`]: 'my-service',
+    [`\${__data.fields.${NodeGraphDataFrameFieldNames.subTitle}}`]: 'my-namespace',
+    [`\${__data.fields.${NodeGraphDataFrameFieldNames.isInstrumented}}`]: 'false',
+  };
+  return variables[variable];
+};
+
 interface PromQuery extends DataQuery {
   expr: string;
 }

public/app/plugins/datasource/tempo/datasource.ts

@@ -7,6 +7,7 @@ import {
   CoreApp,
   DataFrame,
   DataFrameDTO,
+  DataLink,
   DataQueryRequest,
   DataQueryResponse,
   DataQueryResponseData,
@@ -15,6 +16,7 @@ import {
   dateTime,
   FieldType,
   LoadingState,
+  NodeGraphDataFrameFieldNames,
   rangeUtil,
   ScopedVars,
   SelectableValue,
@@ -1122,13 +1124,7 @@ export function getFieldConfig(
         datasourceUid,
         false
       ),
-      makeTempoLink(
+      makeTempoLinkServiceMap('View traces', tempoDatasourceUid, !!namespaceFields?.targetNamespace),
-        'View traces',
-        namespaceFields !== undefined ? `\${${namespaceFields.targetNamespace}}` : '',
-        `\${${targetField}}`,
-        '',
-        tempoDatasourceUid
-      ),
     ],
   };
 }
@@ -1183,25 +1179,99 @@ export function makeTempoLink(
   };
 }
 
+function makeTempoLinkServiceMap(
+  title: string,
+  datasourceUid: string,
+  includeNamespace: boolean
+): DataLink<TempoQuery> {
+  return {
+    url: '',
+    title,
+    internal: {
+      datasourceUid,
+      datasourceName: getDataSourceSrv().getInstanceSettings(datasourceUid)?.name ?? '',
+      query: ({ replaceVariables, scopedVars }) => {
+        const serviceName = replaceVariables?.(`\${__data.fields.${NodeGraphDataFrameFieldNames.title}}`, scopedVars);
+        const serviceNamespace = replaceVariables?.(
+          `\${__data.fields.${NodeGraphDataFrameFieldNames.subTitle}}`,
+          scopedVars
+        );
+        const isInstrumented =
+          replaceVariables?.(`\${__data.fields.${NodeGraphDataFrameFieldNames.isInstrumented}}`, scopedVars) !==
+          'false';
+        const query: TempoQuery = { refId: 'A', queryType: 'traceqlSearch', filters: [] };
+
+        // Only do the peer query if service is actively set as not instrumented
+        if (isInstrumented === false) {
+          const filters = ['db.name', 'db.system', 'peer.service', 'messaging.system', 'net.peer.name']
+            .map((peerAttribute) => `span.${peerAttribute}="${serviceName}"`)
+            .join(' || ');
+          query.queryType = 'traceql';
+          query.query = `{${filters}}`;
+        } else {
+          if (includeNamespace && serviceNamespace) {
+            query.filters.push({
+              id: 'service-namespace',
+              scope: TraceqlSearchScope.Resource,
+              tag: 'service.namespace',
+              value: serviceNamespace,
+              operator: '=',
+              valueType: 'string',
+            });
+          }
+          if (serviceName) {
+            query.filters.push({
+              id: 'service-name',
+              scope: TraceqlSearchScope.Resource,
+              tag: 'service.name',
+              value: serviceName,
+              operator: '=',
+              valueType: 'string',
+            });
+          }
+        }
+
+        return query;
+      },
+    },
+  };
+}
+
 function makePromServiceMapRequest(options: DataQueryRequest<TempoQuery>): DataQueryRequest<PromQuery> {
   return {
     ...options,
-    targets: serviceMapMetrics.map((metric) => {
+    targets: serviceMapMetrics
-      const { serviceMapQuery, serviceMapIncludeNamespace: serviceMapIncludeNamespace } = options.targets[0];
+      .map<PromQuery[]>((metric) => {
-      const extraSumByFields = serviceMapIncludeNamespace ? ', client_service_namespace, server_service_namespace' : '';
+        const { serviceMapQuery, serviceMapIncludeNamespace: serviceMapIncludeNamespace } = options.targets[0];
-      const queries = Array.isArray(serviceMapQuery) ? serviceMapQuery : [serviceMapQuery];
+        const extraSumByFields = serviceMapIncludeNamespace
-      const subExprs = queries.map(
+          ? ', client_service_namespace, server_service_namespace'
-        (query) => `sum by (client, server${extraSumByFields}) (rate(${metric}${query || ''}[$__range]))`
+          : '';
-      );
+        const queries = Array.isArray(serviceMapQuery) ? serviceMapQuery : [serviceMapQuery];
-      return {
+        const sumSubExprs = queries.map(
-        format: 'table',
+          (query) => `sum by (client, server${extraSumByFields}) (rate(${metric}${query || ''}[$__range]))`
-        refId: metric,
+        );
-        // options.targets[0] is not correct here, but not sure what should happen if you have multiple queries for
+        const groupSubExprs = queries.map(
-        // service map at the same time anyway
+          (query) => `group by (client, connection_type, server${extraSumByFields}) (${metric}${query || ''})`
-        expr: subExprs.join(' OR '),
+        );
-        instant: true,
+
-      };
+        return [
-    }),
+          {
+            format: 'table',
+            refId: metric,
+            // options.targets[0] is not correct here, but not sure what should happen if you have multiple queries for
+            // service map at the same time anyway
+            expr: sumSubExprs.join(' OR '),
+            instant: true,
+          },
+          {
+            format: 'table',
+            refId: `${metric}_labels`,
+            expr: groupSubExprs.join(' OR '),
+            instant: true,
+          },
+        ];
+      })
+      .flat(),
   };
 }
 

public/app/plugins/datasource/tempo/graphTransform.test.ts

@@ -20,6 +20,7 @@ it('assigns correct field type even if values are numbers', async () => {
     { name: 'secondarystat', values: [10, 20], type: FieldType.number },
     { name: 'arc__success', values: [1, 1], type: FieldType.number },
     { name: 'arc__failed', values: [0, 0], type: FieldType.number },
+    { name: 'isinstrumented', values: [true, true], type: FieldType.boolean },
   ]);
 });
 
@@ -41,6 +42,7 @@ it('do not fail on response with empty list', async () => {
     { name: 'secondarystat', values: [], type: FieldType.number },
     { name: 'arc__success', values: [], type: FieldType.number },
     { name: 'arc__failed', values: [], type: FieldType.number },
+    { name: 'isinstrumented', values: [], type: FieldType.boolean },
   ]);
 });
 
@@ -66,6 +68,7 @@ describe('mapPromMetricsToServiceMap', () => {
       { name: 'secondarystat', values: [10, 20, NaN] },
       { name: 'arc__success', values: [0.8, 0.25, 1] },
       { name: 'arc__failed', values: [0.2, 0.75, 0] },
+      { name: 'isinstrumented', values: [true, true, true] },
     ]);
     expect(edges.fields).toMatchObject([
       { name: 'id', values: ['app_db', 'lb_app'] },
@@ -101,6 +104,7 @@ describe('mapPromMetricsToServiceMap', () => {
       { name: 'secondarystat', values: [10, 20, NaN] },
       { name: 'arc__success', values: [0.8, 0.25, 1] },
       { name: 'arc__failed', values: [0.2, 0.75, 0] },
+      { name: 'isinstrumented', values: [true, true, true] },
     ]);
     expect(edges.fields).toMatchObject([
       { name: 'id', values: ['ns1/app_ns3/db', 'ns2/lb_ns1/app'] },
@@ -138,6 +142,41 @@ describe('mapPromMetricsToServiceMap', () => {
       { name: 'secondarystat', values: [10, 20, NaN] },
       { name: 'arc__success', values: [0, 0, 1] },
       { name: 'arc__failed', values: [1, 1, 0] },
+      { name: 'isinstrumented', values: [true, true, true] },
+    ]);
+  });
+
+  it('handles setting isInstrumented based on the connection_type', () => {
+    const range = {
+      from: dateTime('2000-01-01T00:00:00'),
+      to: dateTime('2000-01-01T00:01:00'),
+    };
+    const { nodes } = mapPromMetricsToServiceMap(
+      [
+        {
+          data: [
+            totalsPromMetric(true),
+            secondsPromMetric(true),
+            secondsLabelsPromMetric(true),
+            failedPromMetric(true),
+          ],
+        },
+      ],
+      {
+        ...range,
+        raw: range,
+      }
+    );
+
+    expect(nodes.fields).toMatchObject([
+      { name: 'id', values: ['ns3/db', 'ns1/app', 'ns2/lb'] },
+      { name: 'title', values: ['db', 'app', 'lb'] },
+      { name: 'subtitle', values: ['ns3', 'ns1', 'ns2'] },
+      { name: 'mainstat', values: [1000, 2000, NaN] },
+      { name: 'secondarystat', values: [10, 20, NaN] },
+      { name: 'arc__success', values: [0.8, 0.25, 1] },
+      { name: 'arc__failed', values: [0.2, 0.75, 0] },
+      { name: 'isinstrumented', values: [true, false, true] },
     ]);
   });
 });
@@ -182,6 +221,27 @@ const secondsPromMetric = (namespace?: boolean) =>
     ],
   });
 
+const secondsLabelsPromMetric = (namespace?: boolean) =>
+  createDataFrame({
+    refId: 'traces_service_graph_request_server_seconds_sum_labels',
+    fields: [
+      { name: 'Time', values: [1628169788000, 1628169788000] },
+      { name: 'client', values: ['app', 'lb'] },
+      { name: 'instance', values: ['127.0.0.1:12345', '127.0.0.1:12345'] },
+      { name: 'job', values: ['local_scrape', 'local_scrape'] },
+      { name: 'server', values: ['db', 'app'] },
+      { name: 'tempo_config', values: ['default', 'default'] },
+      { name: 'Value #traces_service_graph_request_server_seconds_sum_label', values: [1, 1] },
+      { name: 'connection_type', values: ['messaging_system', 'virtual_node'] },
+      ...(namespace
+        ? [
+            { name: 'client_service_namespace', values: ['ns1', 'ns2'] },
+            { name: 'server_service_namespace', values: ['ns3', 'ns1'] },
+          ]
+        : []),
+    ],
+  });
+
 const failedPromMetric = (namespace?: boolean) =>
   createDataFrame({
     refId: 'traces_service_graph_request_failed_total',

public/app/plugins/datasource/tempo/graphTransform.ts

@@ -61,6 +61,10 @@ export function mapPromMetricsToServiceMap(
   collectMetricData(frames[secondsMetric], 'seconds', secondsMetric, nodesMap, edgesMap);
   collectMetricData(frames[failedMetric], 'failed', failedMetric, nodesMap, edgesMap);
 
+  collectIsInstrumented(frames[`${totalsMetric}_labels`], nodesMap);
+  collectIsInstrumented(frames[`${secondsMetric}_labels`], nodesMap);
+  collectIsInstrumented(frames[`${failedMetric}_labels`], nodesMap);
+
   return convertToDataFrames(nodesMap, edgesMap, range);
 }
 
@@ -97,6 +101,11 @@ function createServiceMapDataFrames() {
       config: { displayName: 'Failed', color: { fixedColor: 'red', mode: FieldColorModeId.Fixed } },
       values: [],
     },
+    {
+      name: Fields.isInstrumented,
+      type: FieldType.boolean,
+      values: [],
+    },
   ]);
   const edges = createDF('Edges', [
     { name: Fields.id, type: FieldType.string, values: [] },
@@ -145,6 +154,7 @@ type ServiceMapStatistics = {
 type NodeObject = ServiceMapStatistics & {
   name: string;
   namespace?: string;
+  isInstrumented?: boolean;
 };
 
 type EdgeObject = ServiceMapStatistics & {
@@ -240,6 +250,21 @@ function collectMetricData(
   }
 }
 
+function collectIsInstrumented(frame: DataFrameView | undefined, nodesMap: Record<string, NodeObject>) {
+  if (!frame) {
+    return;
+  }
+
+  for (let i = 0; i < frame.length; i++) {
+    const row = frame.get(i);
+    const serverId = row.server_service_namespace ? `${row.server_service_namespace}/${row.server}` : row.server;
+
+    if (nodesMap[serverId] && nodesMap[serverId].isInstrumented !== true) {
+      nodesMap[serverId].isInstrumented = row.connection_type === '' || row.connection_type === 'messaging_system';
+    }
+  }
+}
+
 function convertToDataFrames(
   nodesMap: Record<string, NodeObject>,
   edgesMap: Record<string, EdgeObject>,
@@ -258,6 +283,7 @@ function convertToDataFrames(
       [Fields.secondaryStat]: node.total ? Math.round(node.total * 100) / 100 : Number.NaN, // Request per second (to 2 decimals)
       [Fields.arc + 'success']: node.total ? (node.total - Math.min(node.failed || 0, node.total)) / node.total : 1,
       [Fields.arc + 'failed']: node.total ? Math.min(node.failed || 0, node.total) / node.total : 0,
+      [Fields.isInstrumented]: node.isInstrumented ?? true,
     });
   }
   for (const edgeId of Object.keys(edgesMap)) {

public/app/plugins/panel/nodeGraph/types.ts

@@ -17,6 +17,7 @@ export type NodeDatum = SimulationNodeDatum & {
   icon?: IconName;
   nodeRadius?: Field;
   highlighted: boolean;
+  isInstrumented?: boolean;
 };
 
 export type NodeDatumFromEdge = NodeDatum & { mainStatNumeric?: number; secondaryStatNumeric?: number };

public/app/plugins/panel/nodeGraph/utils.test.ts

@@ -19,7 +19,7 @@ describe('processNodes', () => {
 
   it('returns proper nodes and edges', async () => {
     const { nodes, edges, legend } = processNodes(
-      makeNodesDataFrame(3),
+      makeNodesDataFrame(3, [{ isinstrumented: false }]),
       makeEdgesDataFrame([
         { source: '0', target: '1' },
         { source: '0', target: '2' },
@@ -28,7 +28,7 @@ describe('processNodes', () => {
     );
 
     expect(nodes).toEqual([
-      makeNodeDatum(),
+      makeNodeDatum({ isInstrumented: false }),
       makeNodeDatum({ dataFrameRowIndex: 1, id: '1', incoming: 1, title: 'service:1' }),
       makeNodeDatum({ dataFrameRowIndex: 2, id: '2', incoming: 2, title: 'service:2' }),
     ]);
@@ -366,6 +366,7 @@ function makeNodeDatum(options: Partial<NodeDatum> = {}) {
       type: 'number',
       values: [40, 40, 40],
     },
+    isInstrumented: true,
     ...options,
   };
 }

public/app/plugins/panel/nodeGraph/utils.ts

@@ -58,6 +58,7 @@ export type NodeFields = {
   icon?: Field;
   nodeRadius?: Field;
   highlighted?: Field;
+  isInstrumented?: Field;
 };
 
 export function getNodeFields(nodes: DataFrame): NodeFields {
@@ -80,6 +81,7 @@ export function getNodeFields(nodes: DataFrame): NodeFields {
     highlighted: fieldsCache.getFieldByName(NodeGraphDataFrameFieldNames.highlighted.toLowerCase()),
     fixedX: fieldsCache.getFieldByName(NodeGraphDataFrameFieldNames.fixedX.toLowerCase()),
     fixedY: fieldsCache.getFieldByName(NodeGraphDataFrameFieldNames.fixedY.toLowerCase()),
+    isInstrumented: fieldsCache.getFieldByName(NodeGraphDataFrameFieldNames.isInstrumented.toLowerCase()),
   };
 }
 
@@ -364,6 +366,7 @@ function makeNodeDatum(id: string, nodeFields: NodeFields, index: number): NodeD
     highlighted: nodeFields.highlighted?.values[index] || false,
     x: nodeFields.fixedX?.values[index] ?? undefined,
     y: nodeFields.fixedY?.values[index] ?? undefined,
+    isInstrumented: nodeFields.isInstrumented?.values[index] ?? true,
   };
 }
 
@@ -384,16 +387,19 @@ export function statToString(config: FieldConfig, value: number | string): strin
  * Utilities mainly for testing
  */
 
-export function makeNodesDataFrame(count: number) {
+export function makeNodesDataFrame(
+  count: number,
+  partialNodes: Array<Partial<Record<NodeGraphDataFrameFieldNames, unknown>>> = []
+) {
   const frame = nodesFrame();
   for (let i = 0; i < count; i++) {
-    frame.add(makeNode(i));
+    frame.add(makeNode(i, partialNodes[i]));
   }
 
   return frame;
 }
 
-function makeNode(index: number) {
+function makeNode(index: number, partialNode: Partial<Record<NodeGraphDataFrameFieldNames, unknown>> = {}) {
   return {
     id: index.toString(),
     title: `service:${index}`,
@@ -405,6 +411,8 @@ function makeNode(index: number) {
     color: 0.5,
     icon: 'database',
     noderadius: 40,
+    isinstrumented: true,
+    ...partialNode,
   };
 }
 
@@ -453,6 +461,10 @@ function nodesFrame() {
       values: [],
       type: FieldType.number,
     },
+    [NodeGraphDataFrameFieldNames.isInstrumented]: {
+      values: [],
+      type: FieldType.boolean,
+    },
   };
 
   return new MutableDataFrame({