Auto-generate: Change dashboard save diff prompt, widen diff context (#76724)

Co-authored-by: nmarrs <nathanielmarrs@gmail.com>

public/app/features/dashboard/components/GenAI/GenAIButton.tsx

@@ -7,7 +7,7 @@ import { Button, Spinner, useStyles2, Tooltip, Toggletip, Text } from '@grafana/
 import { GenAIHistory } from './GenAIHistory';
 import { StreamStatus, useOpenAIStream } from './hooks';
 import { AutoGenerateItem, EventTrackingSrc, reportAutoGenerateInteraction } from './tracking';
-import { OPEN_AI_MODEL, Message, sanitizeReply } from './utils';
+import { OAI_MODEL, DEFAULT_OAI_MODEL, Message, sanitizeReply } from './utils';
 
 export interface GenAIButtonProps {
   // Button label text
@@ -24,6 +24,7 @@ export interface GenAIButtonProps {
   // Temperature for the LLM plugin. Default is 1.
   // Closer to 0 means more conservative, closer to 1 means more creative.
   temperature?: number;
+  model?: OAI_MODEL;
   // Event tracking source. Send as `src` to Rudderstack event
   eventTrackingSrc: EventTrackingSrc;
   // Whether the button should be disabled
@@ -35,6 +36,7 @@ export const GenAIButton = ({
   loadingText = 'Generating',
   toggleTipTitle = '',
   onClick: onClickProp,
+  model = DEFAULT_OAI_MODEL,
   messages,
   onGenerate,
   temperature = 1,
@@ -43,7 +45,7 @@ export const GenAIButton = ({
 }: GenAIButtonProps) => {
   const styles = useStyles2(getStyles);
 
-  const { setMessages, reply, value, error, streamStatus } = useOpenAIStream(OPEN_AI_MODEL, temperature);
+  const { setMessages, reply, value, error, streamStatus } = useOpenAIStream(model, temperature);
 
   const [history, setHistory] = useState<string[]>([]);
   const [showHistory, setShowHistory] = useState(true);

public/app/features/dashboard/components/GenAI/GenAIDashboardChangesButton.tsx

@@ -12,17 +12,25 @@ interface GenAIDashboardChangesButtonProps {
   disabled?: boolean;
 }
 
-const CHANGES_GENERATION_STANDARD_PROMPT = [
+const CHANGES_GENERATION_PREFIX_PROMPT = [
   'You are an expert in Grafana Dashboards',
-  'Your goal is to write a description of the changes for a dashboard',
+  'Your goal is to write a description of the changes for a dashboard to display to the user',
+  'You will be given human-readable diffs with most irrelevant lines filtered out',
+].join('.\n');
+
+const CHANGES_GENERATION_POSTFIX_PROMPT = [
+  `Respond only with the diff description, which is meant to be loaded directly into the application for the user.`,
+  `If there are no substantial user or migration changes, the correct description is "Minor changes only"`,
+  `If there are too many changes of either kind, and those changes have a message saying 'too long', the correct response for that section is "Too many changes to auto-summarize"`,
+  'In a diff, lines beginning with - are removed, and lines beginning with + are added.',
+  'Lines with neither + nor - are included for context. Be careful not to mark them as added or removed if they do not start with + or -.',
+  'If a line is changed, it will show a previous version removed and a new version added',
   'When referring to panel changes, use the panel title',
   'When using panel title, wrap it with double quotes',
-  'When the panel changes the position, just mention the panel has changed position and not the specific change',
+  'When the panel changes position, just mention that the panel has changed position',
   'When an entire panel is added or removed, use the panel title and only say it was added or removed and disregard the rest of the changes for that panel',
-  'Group changes when all panels are affected',
-  'Do not mention line number',
+  'Group together similar changes into one line when multiple panels are affected',
   'Refer to templating elements as variables',
-  'Ignore and never mention changes about plugin version',
   'Try to make the response as short as possible',
 ].join('.\n');
 
@@ -35,6 +43,7 @@ export const GenAIDashboardChangesButton = ({ dashboard, onGenerate, disabled }:
       onGenerate={onGenerate}
       loadingText={'Generating changes summary'}
       temperature={0}
+      model={'gpt-3.5-turbo-16k'}
       eventTrackingSrc={EventTrackingSrc.dashboardChanges}
       toggleTipTitle={'Improve your dashboard changes summary'}
       disabled={disabled}
@@ -43,33 +52,48 @@ export const GenAIDashboardChangesButton = ({ dashboard, onGenerate, disabled }:
 };
 
 function getMessages(dashboard: DashboardModel): Message[] {
-  const { userChanges, migrationChanges } = getDashboardChanges(dashboard);
+  let { userChanges, migrationChanges } = getDashboardChanges(dashboard);
+  if (userChanges.length > 8000) {
+    userChanges =
+      "User changes were too long, fill in the user changes section with 'User changes too long to auto-summarize'";
+  }
+
+  if (migrationChanges.split('\n').length < 10) {
+    migrationChanges = 'No significant migration changes';
+  } else if (migrationChanges.length > 8000) {
+    migrationChanges =
+      "Migration changes were too long, fill in the migration changes section with 'Migration changes too long to auto-summarize'";
+  }
 
   return [
     {
-      content: CHANGES_GENERATION_STANDARD_PROMPT,
+      content: CHANGES_GENERATION_PREFIX_PROMPT,
       role: Role.system,
     },
     {
-      content: `This is the list of panel names, when referring to a panel, please use the title: ${JSON.stringify(
-        dashboard.panels.map((panel) => panel.title)
-      )}`,
+      content: `Summarize the following user changes diff under "User changes":\n${userChanges}`,
       role: Role.system,
     },
     {
-      content: `Summarize the following user changes diff under a "User changes" heading with no special formatting as a bulleted list:\n${userChanges}`,
+      content:
+        `Be sure to only include substantial user changes, such as adding or removing entire panels, changing panel titles or descriptions, etc.\n` +
+        `Do not include "User Changes" section if there are no substantial user changes to report.`,
       role: Role.system,
     },
     {
-      content: `Be sure to only include substantial user changes, such as adding or removing entire panels, changing panel titles or descriptions, etc. Ignore other changes and do not include them in the summary. Do not include "User Changes" section if there are no substantial user changes to report.`,
+      content: `Summarize the following migration changes diff under "Migration changes":\n${migrationChanges}`,
       role: Role.system,
     },
     {
-      content: `Threshold step changes of value being removed and replaced with color "green" should always be ignored`,
+      content:
+        `Be sure to only include substantial migration changes, such as adding or removing entire panels, changing panel titles or descriptions, etc.\n` +
+        `Ignore any threshold step changes or templating list changes.\n` +
+        `Ignore other changes and do not include them in the summary. Do not include "Migration Changes" section if there are no substantial migration changes to report.\n` +
+        `If there are substantial migration changes, add "Some autogenerated changes are included to update the dashboard to the latest valid schema version" at the end.`,
       role: Role.system,
     },
     {
-      content: `In addition to summarizing the user diff changes, add the following sentence at the end of the response "Some autogenerated changes are included to update the dashboard to the latest valid schema version." Only add this sentence if the following migrations diff has substantial changes. Ignore any threshold step changes, templating list changes, and ignore the entire migration diff if it is less than 10 lines long. \n This is the migrations diff:\n${migrationChanges}`,
+      content: CHANGES_GENERATION_POSTFIX_PROMPT,
       role: Role.system,
     },
   ];

public/app/features/dashboard/components/GenAI/GenAIHistory.tsx

@@ -21,7 +21,7 @@ import { GenerationHistoryCarousel } from './GenerationHistoryCarousel';
 import { QuickFeedback } from './QuickFeedback';
 import { StreamStatus, useOpenAIStream } from './hooks';
 import { AutoGenerateItem, EventTrackingSrc, reportAutoGenerateInteraction } from './tracking';
-import { Message, OPEN_AI_MODEL, QuickFeedbackType, sanitizeReply } from './utils';
+import { Message, DEFAULT_OAI_MODEL, QuickFeedbackType, sanitizeReply } from './utils';
 
 export interface GenAIHistoryProps {
   history: string[];
@@ -46,7 +46,7 @@ export const GenAIHistory = ({
   const [showError, setShowError] = useState(false);
   const [customFeedback, setCustomPrompt] = useState('');
 
-  const { setMessages, reply, streamStatus, error } = useOpenAIStream(OPEN_AI_MODEL, temperature);
+  const { setMessages, reply, streamStatus, error } = useOpenAIStream(DEFAULT_OAI_MODEL, temperature);
 
   const isStreamGenerating = streamStatus === StreamStatus.GENERATING;
 

public/app/features/dashboard/components/GenAI/hooks.ts

@@ -6,7 +6,7 @@ import { llms } from '@grafana/experimental';
 import { logError } from '@grafana/runtime';
 import { useAppNotification } from 'app/core/copy/appNotification';
 
-import { isLLMPluginEnabled, OPEN_AI_MODEL } from './utils';
+import { isLLMPluginEnabled, DEFAULT_OAI_MODEL } from './utils';
 
 // Declared instead of imported from utils to make this hook modular
 // Ideally we will want to move the hook itself to a different scope later.
@@ -22,7 +22,7 @@ export const TIMEOUT = 10000;
 
 // TODO: Add tests
 export function useOpenAIStream(
-  model = OPEN_AI_MODEL,
+  model = DEFAULT_OAI_MODEL,
   temperature = 1
 ): {
   setMessages: React.Dispatch<React.SetStateAction<Message[]>>;

public/app/features/dashboard/components/GenAI/jsonDiffText.test.ts

@@ -1,7 +1,14 @@
 import { DASHBOARD_SCHEMA_VERSION } from '../../state/DashboardMigrator';
 import { createDashboardModelFixture, createPanelSaveModel } from '../../state/__fixtures__/dashboardFixtures';
 
-import { orderProperties, JSONArray, JSONValue, isObject, getDashboardStringDiff } from './jsonDiffText';
+import {
+  orderProperties,
+  JSONArray,
+  JSONValue,
+  isObject,
+  getDashboardStringDiff,
+  removeEmptyFields,
+} from './jsonDiffText';
 
 describe('orderProperties', () => {
   it('should sort simple objects', () => {
@@ -262,15 +269,13 @@ describe('getDashboardStringDiff', () => {
 
     expect(result).toEqual({
       migrationDiff:
-        'Index: Original Title\n' +
         '===================================================================\n' +
-        '--- Original Title\t\n' +
-        '+++ Original Title\t\n',
+        '--- Before migration changes\t\n' +
+        '+++ After migration changes\t\n',
       userDiff:
-        'Index: Original Title\n' +
         '===================================================================\n' +
-        '--- Original Title\t\n' +
-        '+++ Original Title\t\n',
+        '--- Before user changes\t\n' +
+        '+++ After user changes\t\n',
     });
   });
 
@@ -284,3 +289,151 @@ describe('getDashboardStringDiff', () => {
     expect(result.userDiff).toContain(`+  \"title\": \"New Title\",`);
   });
 });
+
+describe('removeEmptyFields', () => {
+  it('should remove "null" fields from the JSON object', () => {
+    const inputJSON = {
+      a: null,
+      b: 'Hello',
+      c: {
+        d: null,
+        e: 'World',
+      },
+    };
+
+    const result = removeEmptyFields(inputJSON);
+
+    expect(result).toEqual({
+      b: 'Hello',
+      c: {
+        e: 'World',
+      },
+    });
+  });
+
+  it('should remove empty arrays from the JSON object', () => {
+    const inputJSON = {
+      a: [1, 2, 3],
+      b: [],
+      c: [4, 5],
+    };
+
+    const result = removeEmptyFields(inputJSON);
+
+    expect(result).toEqual({
+      a: [1, 2, 3],
+      c: [4, 5],
+    });
+  });
+
+  it('should remove empty objects from the JSON object', () => {
+    const inputJSON = {
+      a: {},
+      b: 'Hello',
+      c: {
+        d: {},
+        e: 'World',
+      },
+    };
+
+    const result = removeEmptyFields(inputJSON);
+
+    expect(result).toEqual({
+      b: 'Hello',
+      c: {
+        e: 'World',
+      },
+    });
+  });
+
+  it('should handle a mix of "null", empty arrays, and empty objects', () => {
+    const inputJSON = {
+      a: null,
+      b: [],
+      c: {
+        d: null,
+        e: {},
+        f: [1, 2, 3],
+        g: 'Hello',
+      },
+    };
+
+    const result = removeEmptyFields(inputJSON);
+
+    expect(result).toEqual({
+      c: {
+        f: [1, 2, 3],
+        g: 'Hello',
+      },
+    });
+  });
+
+  it('should handle nested structures', () => {
+    const inputJSON = {
+      a: {
+        b: {
+          c: null,
+          d: {
+            e: [],
+            f: 'World',
+          },
+        },
+      },
+    };
+
+    const result = removeEmptyFields(inputJSON);
+
+    expect(result).toEqual({
+      a: {
+        b: {
+          d: {
+            f: 'World',
+          },
+        },
+      },
+    });
+  });
+
+  it('should handle complex JSON structure', () => {
+    const inputJSON = {
+      panels: [
+        {
+          fieldConfig: {
+            defaults: {
+              foo: 'bar',
+            },
+            overrides: [],
+          },
+          gridPos: {
+            h: 15,
+            w: 10,
+            x: 0,
+            y: 0,
+          },
+        },
+      ],
+      schemaVersion: 38,
+    };
+
+    const result = removeEmptyFields(inputJSON);
+
+    expect(result).toEqual({
+      panels: [
+        {
+          fieldConfig: {
+            defaults: {
+              foo: 'bar',
+            },
+          },
+          gridPos: {
+            h: 15,
+            w: 10,
+            x: 0,
+            y: 0,
+          },
+        },
+      ],
+      schemaVersion: 38,
+    });
+  });
+});

public/app/features/dashboard/components/GenAI/jsonDiffText.ts

@@ -135,6 +135,9 @@ export function fillBySimilarity(
           continue;
         }
         if (val1 === val2) {
+          if (key === 'id') {
+            score += 1000; // Can probably be caught earlier in the call tree.
+          }
           score++;
         }
       }
@@ -169,37 +172,99 @@ export function fillBySimilarity(
   }
 }
 
-export function jsonSanitize(obj: Dashboard | DashboardModel | null) {
+function shortenDiff(diffS: string) {
+  const diffLines = diffS.split('\n');
+  let headerEnd = diffS[0].startsWith('Index') ? 4 : 3;
+  let ret = diffLines.slice(0, headerEnd);
+
+  const titleOrBracket = /("title"|Title|\{|\}|\[|\])/i;
+  for (let i = headerEnd; i < diffLines.length; i++) {
+    let line = diffLines[i];
+    if (titleOrBracket.test(line)) {
+      ret.push(line);
+    } else if (line.startsWith('+') || line.startsWith('-')) {
+      ret.push(line);
+    }
+  }
+  return ret.join('\n') + '\n';
+}
+
+export function removeEmptyFields(input: JSONValue): JSONValue {
+  if (input === null || input === '') {
+    return null;
+  }
+
+  if (Array.isArray(input)) {
+    // Filter out empty values and recursively process the non-empty ones
+    const filteredArray = input.map((item) => removeEmptyFields(item)).filter((item) => item !== null);
+
+    return filteredArray.length > 0 ? filteredArray : null;
+  }
+
+  if (typeof input !== 'object') {
+    // If it's not an object, return as is
+    return input;
+  }
+
+  // For objects, recursively process each key-value pair
+  const result: JSONObject = {};
+  for (const key in input) {
+    const processedValue = removeEmptyFields(input[key]);
+
+    if (processedValue !== null) {
+      if (Array.isArray(processedValue) && processedValue.length === 0) {
+        continue;
+      }
+
+      if (typeof processedValue === 'object') {
+        const keys = Object.keys(processedValue);
+        if (keys.length === 0) {
+          continue;
+        }
+      }
+
+      result[key] = processedValue;
+    }
+  }
+
+  return Object.keys(result).length > 0 ? result : null;
+}
+
+function jsonSanitize(obj: Dashboard | DashboardModel | null) {
   return JSON.parse(JSON.stringify(obj, null, 2));
 }
 
-export function getDashboardStringDiff(dashboard: DashboardModel) {
-  const originalDashboard = jsonSanitize(dashboard.getOriginalDashboard());
+export function getDashboardStringDiff(dashboard: DashboardModel): { migrationDiff: string; userDiff: string } {
+  let originalDashboard = jsonSanitize(dashboard.getOriginalDashboard());
   let dashboardAfterMigration = jsonSanitize(new DashboardModel(originalDashboard).getSaveModelClone());
   let currentDashboard = jsonSanitize(dashboard.getSaveModelClone());
 
-  dashboardAfterMigration = orderProperties(originalDashboard, dashboardAfterMigration);
-  currentDashboard = orderProperties(dashboardAfterMigration, currentDashboard);
+  dashboardAfterMigration = removeEmptyFields(orderProperties(originalDashboard, dashboardAfterMigration));
+  currentDashboard = removeEmptyFields(orderProperties(dashboardAfterMigration, currentDashboard));
+  originalDashboard = removeEmptyFields(originalDashboard);
 
-  let migrationDiff = createTwoFilesPatch(
-    originalDashboard.title ?? 'Before migration changes',
-    dashboardAfterMigration.title ?? 'After migration changes',
+  let migrationDiff: string = createTwoFilesPatch(
+    'Before migration changes',
+    'After migration changes',
     JSON.stringify(originalDashboard, null, 2),
     JSON.stringify(dashboardAfterMigration, null, 2),
     '',
     '',
-    { context: 5 }
+    { context: 20 }
   );
 
-  let userDiff = createTwoFilesPatch(
-    dashboardAfterMigration.title ?? 'Before user changes',
-    currentDashboard.title ?? 'After user changes',
+  let userDiff: string = createTwoFilesPatch(
+    'Before user changes',
+    'After user changes',
     JSON.stringify(dashboardAfterMigration, null, 2),
     JSON.stringify(currentDashboard, null, 2),
     '',
     '',
-    { context: 5 }
+    { context: 20 }
   );
 
+  migrationDiff = shortenDiff(migrationDiff);
+  userDiff = shortenDiff(userDiff);
+
   return { migrationDiff, userDiff };
 }

public/app/features/dashboard/components/GenAI/utils.test.ts

@@ -41,49 +41,48 @@ describe('getDashboardChanges', () => {
     const result = getDashboardChanges(dashboard);
 
     // Assertions
-    expect(result.migrationChanges).toEqual(
-      '===================================================================\n' +
-        '--- Before migration changes\t\n' +
-        '+++ After migration changes\t\n' +
-        '@@ -1,9 +1,9 @@\n' +
-        ' {\n' +
-        '   "editable": true,\n' +
-        '   "graphTooltip": 0,\n' +
-        `-  "schemaVersion": ${deprecatedVersion},\n` +
-        `+  "schemaVersion": ${DASHBOARD_SCHEMA_VERSION},\n` +
-        '   "timezone": "",\n' +
-        '   "panels": [\n' +
-        '     {\n' +
-        '       "type": "timeseries",\n' +
-        '       "title": "Panel 1",\n'
+    expect(result.migrationChanges).toContain(
+      `-  "schemaVersion": ${deprecatedVersion},\n+  "schemaVersion": ${DASHBOARD_SCHEMA_VERSION},\n`
     );
-    expect(result.userChanges).toEqual(
-      '===================================================================\n' +
-        '--- Before user changes\t\n' +
-        '+++ After user changes\t\n' +
-        '@@ -3,16 +3,17 @@\n' +
-        '   "graphTooltip": 0,\n' +
-        `   "schemaVersion": ${DASHBOARD_SCHEMA_VERSION},\n` +
-        '   "timezone": "",\n' +
-        '   "panels": [\n' +
+
+    expect(result.migrationChanges).not.toContain(
+      '   "panels": [\n' +
         '     {\n' +
         '-      "type": "timeseries",\n' +
         '-      "title": "Panel 1",\n' +
-        '+      "id": 1,\n' +
-        '       "options": {\n' +
-        '         "legend": {\n' +
-        '           "displayMode": "hidden",\n' +
-        '           "showLegend": false\n' +
-        '         }\n' +
-        '-      }\n' +
+        '+      "id": 1,\n'
+    );
+
+    expect(result.migrationChanges).not.toContain(
+      '-      }\n' +
         '+      },\n' +
         '+      "title": "New title",\n' +
         '+      "type": "timeseries"\n' +
         '     }\n' +
         '   ]\n' +
-        ' }\n' +
-        '\\ No newline at end of file\n'
+        ' }\n'
     );
+
+    expect(result.userChanges).not.toContain('-  "schemaVersion": 37,\n' + '+  "schemaVersion": 38,\n');
+
+    expect(result.userChanges).toContain(
+      '   "panels": [\n' +
+        '     {\n' +
+        '-      "type": "timeseries",\n' +
+        '-      "title": "Panel 1",\n' +
+        '+      "id": 1,\n'
+    );
+
+    expect(result.userChanges).toContain(
+      '-      }\n' +
+        '+      },\n' +
+        '+      "title": "New title",\n' +
+        '+      "type": "timeseries"\n' +
+        '     }\n' +
+        '   ]\n' +
+        ' }\n'
+    );
+
     expect(result.migrationChanges).toBeDefined();
   });
 });

public/app/features/dashboard/components/GenAI/utils.ts

@@ -23,7 +23,9 @@ export enum QuickFeedbackType {
 /**
  * The OpenAI model to be used.
  */
-export const OPEN_AI_MODEL = 'gpt-4';
+export const DEFAULT_OAI_MODEL = 'gpt-4';
+
+export type OAI_MODEL = 'gpt-4' | 'gpt-4-32k' | 'gpt-3.5-turbo' | 'gpt-3.5-turbo-16k';
 
 /**
  * Sanitize the reply from OpenAI by removing the leading and trailing quotes.