NodeGraph: Add msagl and the layered layout code (#88375)

jest.config.js

@@ -17,6 +17,7 @@ const esModules = [
   'monaco-promql',
   '@kusto/monaco-kusto',
   'monaco-editor',
+  '@msagl',
   'lodash-es',
   'vscode-languageserver-types',
 ].join('|');

package.json

@@ -271,6 +271,8 @@
     "@locker/near-membrane-dom": "0.13.6",
     "@locker/near-membrane-shared": "0.13.6",
     "@locker/near-membrane-shared-dom": "0.13.6",
+    "@msagl/core": "^1.1.19",
+    "@msagl/parser": "^1.1.19",
     "@opentelemetry/api": "1.8.0",
     "@opentelemetry/exporter-collector": "0.25.0",
     "@opentelemetry/semantic-conventions": "1.24.1",

public/app/plugins/panel/nodeGraph/createLayoutWorker.ts

@@ -1,3 +1,4 @@
 import { CorsWorker as Worker } from 'app/core/utils/CorsWorker';
 
 export const createWorker = () => new Worker(new URL('./layout.worker.js', import.meta.url));
+export const createMsaglWorker = () => new Worker(new URL('./layeredLayout.worker.js', import.meta.url));

public/app/plugins/panel/nodeGraph/layeredLayout.js

@@ -0,0 +1,227 @@
+import {
+  GeomGraph,
+  GeomEdge,
+  GeomNode,
+  Point,
+  CurveFactory,
+  SugiyamaLayoutSettings,
+  LayerDirectionEnum,
+  layoutGeomGraph,
+} from '@msagl/core';
+import { parseDot } from '@msagl/parser';
+
+/**
+ * Use d3 force layout to lay the nodes in a sensible way. This function modifies the nodes adding the x,y positions
+ * and also fills in node references in edges instead of node ids.
+ */
+export function layout(nodes, edges) {
+  const { mappedEdges, DOTToIdMap } = createMappings(nodes, edges);
+
+  const dot = graphToDOT(mappedEdges, DOTToIdMap);
+  const graph = parseDot(dot);
+  const geomGraph = new GeomGraph(graph);
+  for (const e of graph.deepEdges) {
+    new GeomEdge(e);
+  }
+
+  for (const n of graph.nodesBreadthFirst) {
+    const gn = new GeomNode(n);
+    gn.boundaryCurve = CurveFactory.mkCircle(50, new Point(0, 0));
+  }
+  geomGraph.layoutSettings = new SugiyamaLayoutSettings();
+  geomGraph.layoutSettings.layerDirection = LayerDirectionEnum.LR;
+  geomGraph.layoutSettings.LayerSeparation = 60;
+  geomGraph.layoutSettings.commonSettings.NodeSeparation = 40;
+  layoutGeomGraph(geomGraph);
+
+  const nodesMap = {};
+  for (const node of geomGraph.nodesBreadthFirst) {
+    nodesMap[DOTToIdMap[node.id]] = {
+      obj: node,
+    };
+  }
+
+  for (const node of nodes) {
+    nodesMap[node.id] = {
+      ...nodesMap[node.id],
+      datum: {
+        ...node,
+        x: nodesMap[node.id].obj.center.x,
+        y: nodesMap[node.id].obj.center.y,
+      },
+    };
+  }
+  const edgesMapped = edges.map((e) => {
+    return {
+      ...e,
+      source: nodesMap[e.source].datum,
+      target: nodesMap[e.target].datum,
+    };
+  });
+
+  // This section checks if there are separate disjointed subgraphs. If so it groups nodes for each and then aligns
+  // each subgraph, so it starts on a single vertical line. Otherwise, they are laid out randomly from left to right.
+  const subgraphs = [];
+  for (const e of edgesMapped) {
+    const sourceGraph = subgraphs.find((g) => g.nodes.has(e.source));
+    const targetGraph = subgraphs.find((g) => g.nodes.has(e.target));
+    if (sourceGraph && targetGraph) {
+      // if the node sets are not the same we merge them
+      if (sourceGraph !== targetGraph) {
+        targetGraph.nodes.forEach(sourceGraph.nodes.add, sourceGraph.nodes);
+        subgraphs.splice(subgraphs.indexOf(targetGraph), 1);
+        sourceGraph.top = Math.min(sourceGraph.top, targetGraph.top);
+        sourceGraph.bottom = Math.max(sourceGraph.bottom, targetGraph.bottom);
+        sourceGraph.left = Math.min(sourceGraph.left, targetGraph.left);
+        sourceGraph.right = Math.max(sourceGraph.right, targetGraph.right);
+      }
+      // if the sets are the same nothing to do.
+    } else if (sourceGraph) {
+      sourceGraph.nodes.add(e.target);
+      sourceGraph.top = Math.min(sourceGraph.top, e.target.y);
+      sourceGraph.bottom = Math.max(sourceGraph.bottom, e.target.y);
+      sourceGraph.left = Math.min(sourceGraph.left, e.target.x);
+      sourceGraph.right = Math.max(sourceGraph.right, e.target.x);
+    } else if (targetGraph) {
+      targetGraph.nodes.add(e.source);
+      targetGraph.top = Math.min(targetGraph.top, e.source.y);
+      targetGraph.bottom = Math.max(targetGraph.bottom, e.source.y);
+      targetGraph.left = Math.min(targetGraph.left, e.source.x);
+      targetGraph.right = Math.max(targetGraph.right, e.source.x);
+    } else {
+      // we don't have these nodes
+      subgraphs.push({
+        top: Math.min(e.source.y, e.target.y),
+        bottom: Math.max(e.source.y, e.target.y),
+        left: Math.min(e.source.x, e.target.x),
+        right: Math.max(e.source.x, e.target.x),
+        nodes: new Set([e.source, e.target]),
+      });
+    }
+  }
+
+  let top = 0;
+  let left = 0;
+  for (const g of subgraphs) {
+    if (top === 0) {
+      top = g.bottom + 200;
+      left = g.left;
+    } else {
+      const topDiff = top - g.top;
+      const leftDiff = left - g.left;
+      for (const n of g.nodes) {
+        n.x += leftDiff;
+        n.y += topDiff;
+      }
+      top += g.bottom - g.top + 200;
+    }
+  }
+
+  const finalNodes = Object.values(nodesMap).map((v) => v.datum);
+
+  centerNodes(finalNodes);
+  return [finalNodes, edgesMapped];
+}
+
+// We create mapping because the DOT language we use later to create the graph doesn't support arbitrary IDs. So we
+// map our IDs to just an index of the node so the IDs are safe for the DOT parser and also create and inverse mapping
+// for quick lookup.
+function createMappings(nodes, edges) {
+  // Edges where the source and target IDs are the indexes we use for layout
+  const mappedEdges = [];
+
+  // Key is an ID of the node and value is new ID which is just iteration index
+  const idToDOTMap = {};
+
+  // Key is an iteration index and value is actual ID of the node
+  const DOTToIdMap = {};
+
+  let index = 0;
+  for (const node of nodes) {
+    idToDOTMap[node.id] = index.toString(10);
+    DOTToIdMap[index.toString(10)] = node.id;
+    index++;
+  }
+
+  for (const edge of edges) {
+    mappedEdges.push({ source: idToDOTMap[edge.source], target: idToDOTMap[edge.target] });
+  }
+
+  return {
+    mappedEdges,
+    DOTToIdMap,
+    idToDOTMap,
+  };
+}
+
+function graphToDOT(edges, nodeIDsMap) {
+  let dot = `
+  digraph G {
+    rankdir="LR"; TBbalance="min"
+  `;
+  for (const edge of edges) {
+    dot += edge.source + '->' + edge.target + ' ' + '[ minlen=3 ]\n';
+  }
+  dot += nodesDOT(nodeIDsMap);
+  dot += '}';
+  return dot;
+}
+
+function nodesDOT(nodeIdsMap) {
+  let dot = '';
+  for (const node of Object.keys(nodeIdsMap)) {
+    dot += node + ' [fixedsize=true, width=1.2, height=1.7] \n';
+  }
+  return dot;
+}
+
+/**
+ * Makes sure that the center of the graph based on its bound is in 0, 0 coordinates.
+ * Modifies the nodes directly.
+ */
+function centerNodes(nodes) {
+  const bounds = graphBounds(nodes);
+  for (let node of nodes) {
+    node.x = node.x - bounds.center.x;
+    node.y = node.y - bounds.center.y;
+  }
+}
+
+/**
+ * Get bounds of the graph meaning the extent of the nodes in all directions.
+ */
+function graphBounds(nodes) {
+  if (nodes.length === 0) {
+    return { top: 0, right: 0, bottom: 0, left: 0, center: { x: 0, y: 0 } };
+  }
+
+  const bounds = nodes.reduce(
+    (acc, node) => {
+      if (node.x > acc.right) {
+        acc.right = node.x;
+      }
+      if (node.x < acc.left) {
+        acc.left = node.x;
+      }
+      if (node.y > acc.bottom) {
+        acc.bottom = node.y;
+      }
+      if (node.y < acc.top) {
+        acc.top = node.y;
+      }
+      return acc;
+    },
+    { top: Infinity, right: -Infinity, bottom: -Infinity, left: Infinity }
+  );
+
+  const y = bounds.top + (bounds.bottom - bounds.top) / 2;
+  const x = bounds.left + (bounds.right - bounds.left) / 2;
+
+  return {
+    ...bounds,
+    center: {
+      x,
+      y,
+    },
+  };
+}

public/app/plugins/panel/nodeGraph/layeredLayout.test.ts

@@ -0,0 +1,10 @@
+import { layout } from './layeredLayout';
+
+describe('layout', () => {
+  it('can render single node', () => {
+    const nodes = [{ id: 'A', incoming: 0 }];
+    const edges: unknown[] = [];
+    const graph = layout(nodes, edges);
+    expect(graph).toEqual([[{ id: 'A', incoming: 0, x: 0, y: 0 }], []]);
+  });
+});

public/app/plugins/panel/nodeGraph/layeredLayout.worker.js

@@ -0,0 +1,8 @@
+import { layout } from './layeredLayout';
+
+// Separate from main implementation so it does not trip out tests
+addEventListener('message', async (event) => {
+  const { nodes, edges, config } = event.data;
+  const [newNodes, newEdges] = layout(nodes, edges, config);
+  postMessage({ nodes: newNodes, edges: newEdges });
+});

public/app/plugins/panel/nodeGraph/layout.test.ts

@@ -27,6 +27,17 @@ jest.mock('./createLayoutWorker', () => {
       };
       return worker;
     },
+    createMsaglWorker: () => {
+      onmessage = jest.fn();
+      postMessage = jest.fn();
+      terminate = jest.fn();
+      worker = {
+        onmessage: onmessage,
+        postMessage: postMessage,
+        terminate: terminate,
+      };
+      return worker;
+    },
   };
 });
 

public/app/plugins/panel/nodeGraph/layout.ts

@@ -4,8 +4,9 @@ import { useUnmount } from 'react-use';
 import useMountedState from 'react-use/lib/useMountedState';
 
 import { Field } from '@grafana/data';
+import { config as grafanaConfig } from '@grafana/runtime';
 
-import { createWorker } from './createLayoutWorker';
+import { createWorker, createMsaglWorker } from './createLayoutWorker';
 import { EdgeDatum, EdgeDatumLayout, NodeDatum } from './types';
 import { useNodeLimit } from './useNodeLimit';
 import { graphBounds } from './utils';
@@ -102,10 +103,14 @@ export function useLayout(
       return;
     }
 
+    // Layered layout is better but also more expensive, so we switch to default force based layout for bigger graphs.
+    const layoutType =
+      grafanaConfig.featureToggles.nodeGraphDotLayout && rawNodes.length <= 500 ? 'layered' : 'default';
+
     setLoading(true);
     // This is async but as I wanted to still run the sync grid layout, and you cannot return promise from effect so
     // having callback seems ok here.
-    const cancel = layout(rawNodes, rawEdges, ({ nodes, edges }) => {
+    const cancel = layout(rawNodes, rawEdges, layoutType, ({ nodes, edges }) => {
       if (isMounted()) {
         setNodesGraph(nodes);
         setEdgesGraph(edges);
@@ -167,11 +172,10 @@ export function useLayout(
 function layout(
   nodes: NodeDatum[],
   edges: EdgeDatum[],
+  engine: 'default' | 'layered',
   done: (data: { nodes: NodeDatum[]; edges: EdgeDatumLayout[] }) => void
 ) {
-  // const worker = engine === 'default' ? createWorker() : createMsaglWorker();
-  // TODO: temp fix because of problem with msagl library https://github.com/grafana/grafana/issues/83318
-  const worker = createWorker();
+  const worker = engine === 'default' ? createWorker() : createMsaglWorker();
 
   worker.onmessage = (event: MessageEvent<{ nodes: NodeDatum[]; edges: EdgeDatumLayout[] }>) => {
     const nodesMap = fromPairs(nodes.map((node) => [node.id, node]));

scripts/webpack/webpack.common.js

@@ -109,6 +109,13 @@ module.exports = {
         test: /(unicons|mono|custom)[\\/].*\.svg$/,
         type: 'asset/source',
       },
+      {
+        // Required for msagl library (used in Nodegraph panel) to work
+        test: /\.m?js$/,
+        resolve: {
+          fullySpecified: false,
+        },
+      },
     ],
   },
   // https://webpack.js.org/plugins/split-chunks-plugin/#split-chunks-example-3

yarn.lock

@@ -4612,6 +4612,41 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@msagl/core@npm:^1.1.19":
+  version: 1.1.19
+  resolution: "@msagl/core@npm:1.1.19"
+  dependencies:
+    linked-list-typed: "npm:^1.52.0"
+    queue-typescript: "npm:^1.0.1"
+    reliable-random: "npm:^0.0.1"
+    stack-typescript: "npm:^1.0.4"
+    typescript-string-operations: "npm:^1.4.1"
+  checksum: 10/33536bb2ea00215010078faab2211af8be7bcab9e6067f035bed043ae21196bce98364ed554b97552b4cb674ae6c7c65e7f426a4f8cff7b4353c6d23968f1688
+  languageName: node
+  linkType: hard
+
+"@msagl/drawing@npm:^1.1.19":
+  version: 1.1.19
+  resolution: "@msagl/drawing@npm:1.1.19"
+  dependencies:
+    "@msagl/core": "npm:^1.1.19"
+  checksum: 10/b8963ab6f8dd7943a10d950abe11030996e12cd74503c6d704678dce54c1fd0f5ee1570cc893027e24150e475294d09a4ae4ac3be8f9e9f8d7382d7b7c5303a9
+  languageName: node
+  linkType: hard
+
+"@msagl/parser@npm:^1.1.19":
+  version: 1.1.19
+  resolution: "@msagl/parser@npm:1.1.19"
+  dependencies:
+    "@msagl/core": "npm:^1.1.19"
+    "@msagl/drawing": "npm:^1.1.19"
+    "@types/parse-color": "npm:^1.0.1"
+    dotparser: "npm:^1.1.1"
+    parse-color: "npm:^1.0.0"
+  checksum: 10/349dcd57a3365628699b45172359363b86a1b27f8300b5b7fde97ba65eb512191a7433b72908a544fbbdc1d91bcaa9ca498faef345078367529820a47af38609
+  languageName: node
+  linkType: hard
+
 "@mswjs/cookies@npm:^1.1.0":
   version: 1.1.0
   resolution: "@mswjs/cookies@npm:1.1.0"
@@ -8697,6 +8732,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/parse-color@npm:^1.0.1":
+  version: 1.0.3
+  resolution: "@types/parse-color@npm:1.0.3"
+  checksum: 10/d9c822f820f277502cc58a4c52f5758980e6972c9a4710df515ec93dc5dde05e8e3da6210ea6514035d49c1bf36a3f720c0e5ec9c214bdc0b8c3fdb9240e36ea
+  languageName: node
+  linkType: hard
+
 "@types/parse-json@npm:^4.0.0":
   version: 4.0.0
   resolution: "@types/parse-json@npm:4.0.0"
@@ -12001,6 +12043,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"color-convert@npm:~0.5.0":
+  version: 0.5.3
+  resolution: "color-convert@npm:0.5.3"
+  checksum: 10/00dc4256c07ed8760d7bbba234ff969c139eb964fe165853696852001002695c492e327d83ddb7a8cad8d27b49fa543d001328928c12474ee8ecb335bf5f2eb4
+  languageName: node
+  linkType: hard
+
 "color-name@npm:1.1.3":
   version: 1.1.3
   resolution: "color-name@npm:1.1.3"
@@ -13490,6 +13539,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"data-structure-typed@npm:^1.52.0":
+  version: 1.52.0
+  resolution: "data-structure-typed@npm:1.52.0"
+  checksum: 10/9adddcbd0aae44311e866ac093c353fe7f092c19216152329261ea85b5e5e33413ec0605d4766c2ed2939d0b5d1a41576f8c0d21c5c6fe407a7bbdd9b2b2f159
+  languageName: node
+  linkType: hard
+
 "data-urls@npm:^3.0.2":
   version: 3.0.2
   resolution: "data-urls@npm:3.0.2"
@@ -14170,6 +14226,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"dotparser@npm:^1.1.1":
+  version: 1.1.1
+  resolution: "dotparser@npm:1.1.1"
+  checksum: 10/a66a02f5add34b53356ee18b418589ccf055ef6ffec073e91230f5f34cb5ac9579ce20e843b9a3213f42a26bd89fce9232b9364cc08844dade575095dc40d88a
+  languageName: node
+  linkType: hard
+
 "duplexer@npm:^0.1.1, duplexer@npm:^0.1.2":
   version: 0.1.2
   resolution: "duplexer@npm:0.1.2"
@@ -16732,6 +16795,8 @@ __metadata:
     "@locker/near-membrane-shared": "npm:0.13.6"
     "@locker/near-membrane-shared-dom": "npm:0.13.6"
     "@manypkg/get-packages": "npm:^2.2.0"
+    "@msagl/core": "npm:^1.1.19"
+    "@msagl/parser": "npm:^1.1.19"
     "@opentelemetry/api": "npm:1.8.0"
     "@opentelemetry/exporter-collector": "npm:0.25.0"
     "@opentelemetry/semantic-conventions": "npm:1.24.1"
@@ -20092,6 +20157,22 @@ __metadata:
   languageName: node
   linkType: hard
 
+"linked-list-typed@npm:^1.52.0":
+  version: 1.52.0
+  resolution: "linked-list-typed@npm:1.52.0"
+  dependencies:
+    data-structure-typed: "npm:^1.52.0"
+  checksum: 10/8d378c1691f0fa6b3c045784510bc1bfea2cb4d9f51decccc321e39680e031352580423c84aa0b862bbea829db53a046a3f7752b82a83020c640d47c2b433994
+  languageName: node
+  linkType: hard
+
+"linked-list-typescript@npm:^1.0.11":
+  version: 1.0.15
+  resolution: "linked-list-typescript@npm:1.0.15"
+  checksum: 10/91c87ab00fe4bb9850f169adc62b811ae7e91a743bac1a9fc969a815f20ddd0fb285f84a4d31e98900512234f59297f7beef6b076dd382cf879a319c7a14c68b
+  languageName: node
+  linkType: hard
+
 "listr2@npm:^3.8.3":
   version: 3.14.0
   resolution: "listr2@npm:3.14.0"
@@ -23101,6 +23182,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"parse-color@npm:^1.0.0":
+  version: 1.0.0
+  resolution: "parse-color@npm:1.0.0"
+  dependencies:
+    color-convert: "npm:~0.5.0"
+  checksum: 10/9f988462af30929acbbce135286ce20dc2fb8efe48fd0693ed8bfd7da3a60e2465cffff6d35047fe94fd7863b43159bf41b377a9982939b7c65fdba1e30af24a
+  languageName: node
+  linkType: hard
+
 "parse-headers@npm:^2.0.2":
   version: 2.0.4
   resolution: "parse-headers@npm:2.0.4"
@@ -24301,6 +24391,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"queue-typescript@npm:^1.0.1":
+  version: 1.0.1
+  resolution: "queue-typescript@npm:1.0.1"
+  dependencies:
+    linked-list-typescript: "npm:^1.0.11"
+  checksum: 10/403e38579902af1ac533911876c31dd1fd62e94d352263b9ed9786db3677494b15236ad43182178e5103915605a8b07da5780d84b6cc075d7f54cfb056091489
+  languageName: node
+  linkType: hard
+
 "quick-lru@npm:^4.0.1":
   version: 4.0.1
   resolution: "quick-lru@npm:4.0.1"
@@ -25821,6 +25920,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"reliable-random@npm:^0.0.1":
+  version: 0.0.1
+  resolution: "reliable-random@npm:0.0.1"
+  checksum: 10/dbd1fe95dcea9004a0b64540a29c72832c605957a06d26eb762e6600835981ee4949ce7e15b303dcc5c44e9a976357d560f4b6f625fdfaf17f611c65a0150975
+  languageName: node
+  linkType: hard
+
 "remark-gfm@npm:^4.0.0":
   version: 4.0.0
   resolution: "remark-gfm@npm:4.0.0"
@@ -27431,6 +27537,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"stack-typescript@npm:^1.0.4":
+  version: 1.0.4
+  resolution: "stack-typescript@npm:1.0.4"
+  dependencies:
+    linked-list-typescript: "npm:^1.0.11"
+  checksum: 10/a3211a70b01ab4ece788a9c0c4612a132be4019427dc1d9daa7f289b5ab495d4a30cdc947888858eb9225b4c55428530ae60526c56b141f843047db1d878114e
+  languageName: node
+  linkType: hard
+
 "stack-utils@npm:^2.0.3":
   version: 2.0.5
   resolution: "stack-utils@npm:2.0.5"
@@ -28908,6 +29023,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"typescript-string-operations@npm:^1.4.1":
+  version: 1.5.1
+  resolution: "typescript-string-operations@npm:1.5.1"
+  checksum: 10/cec5ad55922ff6e5fc9f43bd7cdcbbb4049bf7b69b18d0ff508b7227519b4da2ce1873b913760b3bf12651833b67d314e0c1c8cee26a25cc0e138335e34bb250
+  languageName: node
+  linkType: hard
+
 "typescript@npm:5.2.2":
   version: 5.2.2
   resolution: "typescript@npm:5.2.2"