DashGPT: Improve save dashboard description generation (#75699)

Co-authored-by: Aaron Sanders <aaron.sanders@grafana.com>
Co-authored-by: Ivan Ortega <ivanortegaalba@gmail.com>
This commit is contained in:
Nathan Marrs
2023-10-14 02:02:45 +02:00
committed by GitHub
parent 56c2d8ff40
commit 0238db42af
9 changed files with 578 additions and 41 deletions

View File

@@ -103,6 +103,7 @@
"@types/d3-force": "^3.0.0",
"@types/d3-scale-chromatic": "3.0.0",
"@types/debounce-promise": "3.1.6",
"@types/diff": "^5",
"@types/eslint": "8.44.0",
"@types/file-saver": "2.0.5",
"@types/glob": "^8.0.0",
@@ -314,6 +315,7 @@
"dangerously-set-html-content": "1.0.9",
"date-fns": "2.30.0",
"debounce-promise": "3.1.2",
"diff": "^5.1.0",
"emotion": "11.0.0",
"eventemitter3": "5.0.1",
"fast-deep-equal": "^3.1.3",

View File

@@ -26,6 +26,8 @@ export interface GenAIButtonProps {
temperature?: number;
// Event tracking source. Send as `src` to Rudderstack event
eventTrackingSrc: EventTrackingSrc;
// Whether the button should be disabled
disabled?: boolean;
}
export const GenAIButton = ({
@@ -37,6 +39,7 @@ export const GenAIButton = ({
onGenerate,
temperature = 1,
eventTrackingSrc,
disabled,
}: GenAIButtonProps) => {
const styles = useStyles2(getStyles);
@@ -47,7 +50,7 @@ export const GenAIButton = ({
const hasHistory = history.length > 0;
const isFirstHistoryEntry = streamStatus === StreamStatus.GENERATING && !hasHistory;
const isButtonDisabled = isFirstHistoryEntry || (value && !value.enabled && !error);
const isButtonDisabled = disabled || isFirstHistoryEntry || (value && !value.enabled && !error);
const reportInteraction = (item: AutoGenerateItem) => reportAutoGenerateInteraction(eventTrackingSrc, item);
const onClick = (e: React.MouseEvent<HTMLButtonElement>) => {

View File

@@ -9,6 +9,7 @@ import { getDashboardChanges, Message, Role } from './utils';
interface GenAIDashboardChangesButtonProps {
dashboard: DashboardModel;
onGenerate: (title: string) => void;
disabled?: boolean;
}
const CHANGES_GENERATION_STANDARD_PROMPT = [
@@ -16,16 +17,16 @@ const CHANGES_GENERATION_STANDARD_PROMPT = [
'Your goal is to write a description of the changes for a dashboard',
'When referring to panel changes, use the panel title',
'When using panel title, wrap it with double quotes',
'When the panel changes the position, just mention the panel title has changed position',
'When the panel changes the position, just mention the panel has changed position and not the specific change',
'When an entire panel is added or removed, use the panel title and only say it was added or removed and disregard the rest of the changes for that panel',
'Group changes when all panels are affected',
'Do not mention line number',
'Refer to templating elements as variables',
'Ignore and never mention changes about plugin version',
'Try to make it as short as possible.',
].join('. ');
'Try to make the response as short as possible',
].join('.\n');
export const GenAIDashboardChangesButton = ({ dashboard, onGenerate }: GenAIDashboardChangesButtonProps) => {
export const GenAIDashboardChangesButton = ({ dashboard, onGenerate, disabled }: GenAIDashboardChangesButtonProps) => {
const messages = useMemo(() => getMessages(dashboard), [dashboard]);
return (
@@ -36,6 +37,7 @@ export const GenAIDashboardChangesButton = ({ dashboard, onGenerate }: GenAIDash
temperature={0}
eventTrackingSrc={EventTrackingSrc.dashboardChanges}
toggleTipTitle={'Improve your dashboard changes summary'}
disabled={disabled}
/>
);
};
@@ -55,13 +57,19 @@ function getMessages(dashboard: DashboardModel): Message[] {
role: Role.system,
},
{
content: `Group the following diff under "User changes" as a bullet list: ${JSON.stringify(userChanges)}`,
content: `Summarize the following user changes diff under a "User changes" heading with no special formatting as a bulleted list:\n${userChanges}`,
role: Role.system,
},
{
content: `Group the following diff under "Migration changes" as a bullet list: ${JSON.stringify(
migrationChanges
)}`,
content: `Be sure to only include substantial user changes, such as adding or removing entire panels, changing panel titles or descriptions, etc. Ignore other changes and do not include them in the summary. Do not include "User Changes" section if there are no substantial user changes to report.`,
role: Role.system,
},
{
content: `Threshold step changes of value being removed and replaced with color "green" should always be ignored`,
role: Role.system,
},
{
content: `In addition to summarizing the user diff changes, add the following sentence at the end of the response "Some autogenerated changes are included to update the dashboard to the latest valid schema version." Only add this sentence if the following migrations diff has substantial changes. Ignore any threshold step changes, templating list changes, and ignore the entire migration diff if it is less than 10 lines long. \n This is the migrations diff:\n${migrationChanges}`,
role: Role.system,
},
];

View File

@@ -0,0 +1,285 @@
import { createDashboardModelFixture, createPanelSaveModel } from '../../state/__fixtures__/dashboardFixtures';
import { orderProperties, JSONArray, JSONValue, isObject, getDashboardStringDiff } from './jsonDiffText';
describe('orderProperties', () => {
it('should sort simple objects', () => {
// Simplest possible case
const before = {
firstProperty: 'foo',
secondProperty: 'bar',
};
const after = {
secondProperty: 'bar',
firstProperty: 'foo',
};
// Call the function to test
const result = orderProperties(before, after);
expect(result).toEqual({
firstProperty: 'foo',
secondProperty: 'bar',
});
});
it('should sort arrays', () => {
const result = orderProperties([0, 1], [1, 0]);
expect(result).toEqual([0, 1]);
});
it('should handle nested objects', () => {
const before = {
nested: {
firstProperty: 'foo',
secondProperty: 'bar',
},
};
const after = {
nested: {
secondProperty: 'bar',
firstProperty: 'foo',
},
};
const result = orderProperties(before, after);
expect(result).toEqual({
nested: {
firstProperty: 'foo',
secondProperty: 'bar',
},
});
});
it('should handle arrays of objects with different order', () => {
const before = [
{ id: 1, name: 'Alice' },
{ id: 2, name: 'Bob' },
];
const after = [
{ id: 2, name: 'Bob' },
{ id: 1, name: 'Alice' },
];
const result = orderProperties(before, after);
expect(result).toEqual([
{ id: 1, name: 'Alice' },
{ id: 2, name: 'Bob' },
]);
});
it('should handle null values', () => {
const before = {
a: null,
b: null,
};
const after = {
b: null,
a: null,
};
const result = orderProperties(before, after);
expect(result).toEqual({
a: null,
b: null,
});
});
it('should handle empty objects', () => {
const before = {};
const after = {};
const result = orderProperties(before, after);
expect(result).toEqual({});
});
it('should handle empty arrays', () => {
const before: JSONValue[] = [];
const after: JSONValue[] = [];
const result = orderProperties(before, after);
expect(result).toEqual([]);
});
it('should handle deeply nested objects', () => {
const before = {
a: {
b: {
c: 'foo',
},
},
d: 'bar',
};
const after = {
d: 'bar',
a: {
b: {
c: 'foo',
},
},
};
const result = orderProperties(before, after);
expect(result).toEqual({
a: {
b: {
c: 'foo',
},
},
d: 'bar',
});
});
it('should handle arrays of nested objects', () => {
const before = [
{ id: 1, nested: { name: 'Alice' } },
{ id: 2, nested: { name: 'Bob' } },
];
const after = [
{ id: 2, nested: { name: 'Bob' } },
{ id: 1, nested: { name: 'Alice' } },
];
const result = orderProperties(before, after);
expect(result).toEqual([
{ id: 1, nested: { name: 'Alice' } },
{ id: 2, nested: { name: 'Bob' } },
]);
});
it('should handle mixed arrays of objects and primitive values', () => {
const before = [{ id: 1 }, 42, [3, 2, 1]];
const after = [{ id: 1 }, [3, 2, 1], 42];
const result = orderProperties(before, after);
expect(result).toEqual([{ id: 1 }, 42, [3, 2, 1]]);
});
it('should handle arrays of objects with nested arrays', () => {
const before = [
{ id: 1, values: [3, 2, 1] },
{ id: 2, values: [6, 5, 4] },
];
const after = [
{ id: 2, values: [6, 5, 4] },
{ id: 1, values: [3, 2, 1] },
];
const result = orderProperties(before, after);
expect(result).toEqual([
{ id: 1, values: [3, 2, 1] },
{ id: 2, values: [6, 5, 4] },
]);
});
it('should handle arrays of arrays', () => {
const before = [
[1, 2, 3],
[4, 5, 6],
];
const after = [
[4, 5, 6],
[1, 2, 3],
];
const result = orderProperties(before, after);
expect(result).toEqual([
[1, 2, 3],
[4, 5, 6],
]);
});
it('should match reordered and modified arrays to nearest keys', () => {
const before = [
{ id: '1', name: 'Alice', country: 'England' },
{ id: '2', name: 'Bob', country: 'America' },
{ id: '3', name: 'Charlie', country: 'Foxtrot' },
];
const after: JSONArray = [{ name: 'Charlie', country: 'Foxtrot' }, { name: 'Alice' }];
const result = orderProperties(before, after);
expect(result).toEqual([{ name: 'Alice' }, { name: 'Charlie', country: 'Foxtrot' }]);
});
});
describe('isObject', () => {
it('should return true for non-array objects', () => {
expect(isObject({})).toBe(true);
expect(isObject({ foo: 'bar' })).toBe(true);
expect(isObject(null)).toBe(false);
expect(isObject([])).toBe(false);
expect(isObject('')).toBe(false);
expect(isObject(123)).toBe(false);
expect(isObject(true)).toBe(false);
});
});
describe('getDashboardStringDiff', () => {
const dashboard = {
title: 'Original Title',
schemaVersion: 38,
panels: [
createPanelSaveModel({
id: 1,
title: 'Original Panel Title',
gridPos: { x: 0, y: 0, w: 2, h: 2 },
}),
createPanelSaveModel({
id: 2,
title: 'Panel to be moved',
gridPos: { x: 2, y: 0, w: 2, h: 2 },
}),
],
};
it('should no return changes when nothing changes', () => {
const dashboardModel = createDashboardModelFixture(dashboard);
const result = getDashboardStringDiff(dashboardModel);
expect(result).toEqual({
migrationDiff:
'Index: Original Title\n' +
'===================================================================\n' +
'--- Original Title\t\n' +
'+++ Original Title\t\n',
userDiff:
'Index: Original Title\n' +
'===================================================================\n' +
'--- Original Title\t\n' +
'+++ Original Title\t\n',
});
});
it('should return a diff of the dashboard title as user change', () => {
const dashboardModel = createDashboardModelFixture(dashboard);
dashboardModel.title = 'New Title';
const result = getDashboardStringDiff(dashboardModel);
expect(result.userDiff).toContain(`- \"title\": \"Original Title\"`);
expect(result.userDiff).toContain(`+ \"title\": \"New Title\",`);
});
});

View File

@@ -0,0 +1,205 @@
import { createTwoFilesPatch } from 'diff';
import { Dashboard } from '@grafana/schema';
import { DashboardModel } from '../../state';
export type JSONValue = null | boolean | number | string | JSONArray | JSONObject;
export type JSONArray = JSONValue[];
export type JSONObject = {
[key: string]: JSONValue;
};
export function orderProperties(obj1: JSONValue, obj2: JSONValue) {
// If obj1 and obj2 are the same object, return obj2
if (obj1 === obj2) {
return obj2; // No need to order properties, they are already the same
}
if (Array.isArray(obj1) && Array.isArray(obj2)) {
// They are both arrays
return orderArrayProperties(obj1, obj2);
}
// Use a type guard to check if they are both non-array objects
else if (isObject(obj1) && isObject(obj2)) {
// Both non-array objects
return orderObjectProperties(obj1, obj2);
}
return obj2;
}
export function isObject(obj: JSONValue): obj is JSONObject {
return typeof obj === 'object' && !Array.isArray(obj) && obj !== null;
}
export function orderObjectProperties(obj1: JSONObject, obj2: JSONObject) {
const orderedProperties = Object.keys(obj1);
const orderedObj2: Record<string, JSONValue> = {};
for (const prop of orderedProperties) {
if (obj2.hasOwnProperty(prop)) {
if (Array.isArray(obj1[prop]) && Array.isArray(obj2[prop])) {
// Recursive call orderProperties for arrays
orderedObj2[prop] = orderProperties(obj1[prop], obj2[prop]);
} else if (typeof obj1[prop] === 'object' && typeof obj2[prop] === 'object') {
// Recursively call orderProperties for nested objects
orderedObj2[prop] = orderProperties(obj1[prop], obj2[prop]);
} else {
orderedObj2[prop] = obj2[prop];
}
}
}
return orderedObj2;
}
export function orderArrayProperties(obj1: JSONArray, obj2: JSONArray) {
const orderedObj2: JSONValue[] = new Array(obj1.length).fill(undefined);
const unseen1 = new Set<number>([...Array(obj1.length).keys()]);
const unseen2 = new Set<number>([...Array(obj2.length).keys()]);
// Loop to match up elements that match exactly
for (let i = 0; i < obj1.length; i++) {
if (unseen2.size === 0) {
break;
}
let item1 = obj1[i];
for (let j = 0; j < obj2.length; j++) {
if (!unseen2.has(j)) {
continue;
}
let item2 = obj2[j];
item2 = orderProperties(item1, item2);
if (JSON.stringify(item1) === JSON.stringify(item2)) {
unseen1.delete(i);
unseen2.delete(j);
orderedObj2[i] = item2;
}
}
}
fillBySimilarity(obj1, obj2, orderedObj2, unseen1, unseen2);
return orderedObj2.filter((value) => value !== undefined);
}
// Compare all pairings by similarity and match greedily from highest to lowest
// Similarity is simply measured by number of k:v pairs in fair
// O(n^2), which is more or less unavoidable
// Can be made a better match by using levenshtein distance and Hungarian matching
export function fillBySimilarity(
// TODO: Investigate not using any
// eslint-disable-next-line @typescript-eslint/no-explicit-any
obj1: any[],
// eslint-disable-next-line @typescript-eslint/no-explicit-any
obj2: any[],
// eslint-disable-next-line @typescript-eslint/no-explicit-any
orderedObj2: any[],
unseen1: Set<number>,
unseen2: Set<number>
): void {
let rankings: Record<number, number[][]> = {}; // Maps scores to arrays of value pairs
// Unpacking it because I'm not sure removing items while iterating is safe
unseen2.forEach((j: number) => {
// Index name matches calling function
let item2 = obj2[j];
// If not object, or if array, just push item2 to orderedObj2 and remove j from unseen2
if (typeof item2 !== 'object' || Array.isArray(item2)) {
orderedObj2.push(item2);
unseen2.delete(j);
return;
}
unseen1.forEach((i: number) => {
let item1 = obj1[i];
if (typeof item1 !== 'object' || Array.isArray(item1)) {
unseen1.delete(i);
return;
}
let score = 0;
for (const key in item1) {
let val1 = item1[key];
if (!item2.hasOwnProperty(key)) {
continue;
}
let val2 = item2[key];
if ((typeof val1 !== 'string' && typeof val1 !== 'number') || typeof val1 !== typeof val2) {
continue;
}
if (val1 === val2) {
score++;
}
}
if (score !== 0) {
if (rankings[score] === undefined) {
rankings[score] = [];
}
rankings[score].push([i, j]);
}
});
});
const keys: number[] = Object.keys(rankings).map(Number); // Get keys as an array of numbers
keys.sort((a, b) => b - a); // Sort in descending order
for (const key of keys) {
let pairs: number[][] = rankings[key];
for (const pair of pairs) {
const [i, j] = pair;
if (unseen1.has(i) && unseen2.has(j)) {
orderedObj2[i] = obj2[j];
unseen1.delete(i);
unseen2.delete(j);
}
}
}
// Get anything that had no matches whatsoever
for (const j of unseen2) {
orderedObj2.push(obj2[j]);
}
}
export function jsonSanitize(obj: Dashboard | DashboardModel | null) {
return JSON.parse(JSON.stringify(obj, null, 2));
}
export function getDashboardStringDiff(dashboard: DashboardModel) {
const originalDashboard = jsonSanitize(dashboard.getOriginalDashboard());
let dashboardAfterMigration = jsonSanitize(new DashboardModel(originalDashboard).getSaveModelClone());
let currentDashboard = jsonSanitize(dashboard.getSaveModelClone());
dashboardAfterMigration = orderProperties(originalDashboard, dashboardAfterMigration);
currentDashboard = orderProperties(dashboardAfterMigration, currentDashboard);
let migrationDiff = createTwoFilesPatch(
originalDashboard.title ?? 'Before migration changes',
dashboardAfterMigration.title ?? 'After migration changes',
JSON.stringify(originalDashboard, null, 2),
JSON.stringify(dashboardAfterMigration, null, 2),
'',
'',
{ context: 5 }
);
let userDiff = createTwoFilesPatch(
dashboardAfterMigration.title ?? 'Before user changes',
currentDashboard.title ?? 'After user changes',
JSON.stringify(dashboardAfterMigration, null, 2),
JSON.stringify(currentDashboard, null, 2),
'',
'',
{ context: 5 }
);
return { migrationDiff, userDiff };
}

View File

@@ -37,29 +37,50 @@ describe('getDashboardChanges', () => {
const result = getDashboardChanges(dashboard);
// Assertions
expect(result.userChanges).toEqual({
panels: [
{
op: 'replace',
originalValue: 'Panel 1',
value: 'New title',
startLineNumber: expect.any(Number),
path: ['panels', '0', 'title'],
},
],
});
expect(result.migrationChanges).toEqual(
'===================================================================\n' +
'--- Before migration changes\t\n' +
'+++ After migration changes\t\n' +
'@@ -1,9 +1,9 @@\n' +
' {\n' +
' "editable": true,\n' +
' "graphTooltip": 0,\n' +
'- "schemaVersion": 37,\n' +
'+ "schemaVersion": 38,\n' +
' "timezone": "",\n' +
' "panels": [\n' +
' {\n' +
' "type": "timeseries",\n' +
' "title": "Panel 1",\n'
);
expect(result.userChanges).toEqual(
'===================================================================\n' +
'--- Before user changes\t\n' +
'+++ After user changes\t\n' +
'@@ -3,16 +3,17 @@\n' +
' "graphTooltip": 0,\n' +
' "schemaVersion": 38,\n' +
' "timezone": "",\n' +
' "panels": [\n' +
' {\n' +
'- "type": "timeseries",\n' +
'- "title": "Panel 1",\n' +
'+ "id": 1,\n' +
' "options": {\n' +
' "legend": {\n' +
' "displayMode": "hidden",\n' +
' "showLegend": false\n' +
' }\n' +
'- }\n' +
'+ },\n' +
'+ "title": "New title",\n' +
'+ "type": "timeseries"\n' +
' }\n' +
' ]\n' +
' }\n' +
'\\ No newline at end of file\n'
);
expect(result.migrationChanges).toBeDefined();
expect(result.userChanges).not.toContain({
panels: [
{
op: 'replace',
originalValue: 'Panel 1',
value: 'New title',
startLineNumber: expect.any(Number),
path: ['panels', '0', 'title'],
},
],
});
});
});

View File

@@ -1,6 +1,6 @@
import { DashboardModel, PanelModel } from '../../state';
import { Diffs, jsonDiff } from '../VersionHistory/utils';
import { getDashboardStringDiff } from './jsonDiffText';
import { openai } from './llms';
export enum Role {
@@ -41,18 +41,14 @@ export const sanitizeReply = (reply: string) => {
* @returns user changes and migration changes
*/
export function getDashboardChanges(dashboard: DashboardModel): {
userChanges: Diffs;
migrationChanges: Diffs;
userChanges: string;
migrationChanges: string;
} {
// Re-parse the dashboard to remove functions and other non-serializable properties
const currentDashboard = dashboard.getSaveModelClone();
const originalDashboard = dashboard.getOriginalDashboard()!;
const dashboardAfterMigration = new DashboardModel(originalDashboard).getSaveModelClone();
const { migrationDiff, userDiff } = getDashboardStringDiff(dashboard);
return {
userChanges: jsonDiff(dashboardAfterMigration, currentDashboard),
migrationChanges: jsonDiff(originalDashboard, dashboardAfterMigration),
userChanges: userDiff,
migrationChanges: migrationDiff,
};
}

View File

@@ -100,6 +100,7 @@ export const SaveDashboardForm = ({
});
setMessage(text);
}}
disabled={!saveModel.hasChanges}
/>
)}
<TextArea

View File

@@ -8874,6 +8874,13 @@ __metadata:
languageName: node
linkType: hard
"@types/diff@npm:^5":
version: 5.0.5
resolution: "@types/diff@npm:5.0.5"
checksum: 5f093341d5488cd33079400d4b2ff463dfd4f589bf0a54fbb25f7c87a7bdc16bd1e6125f252c081fa9a3ad44fc78b156c0a5f38bd8128edceb40ba084e051885
languageName: node
linkType: hard
"@types/doctrine@npm:^0.0.3":
version: 0.0.3
resolution: "@types/doctrine@npm:0.0.3"
@@ -14666,6 +14673,13 @@ __metadata:
languageName: node
linkType: hard
"diff@npm:^5.1.0":
version: 5.1.0
resolution: "diff@npm:5.1.0"
checksum: c7bf0df7c9bfbe1cf8a678fd1b2137c4fb11be117a67bc18a0e03ae75105e8533dbfb1cda6b46beb3586ef5aed22143ef9d70713977d5fb1f9114e21455fba90
languageName: node
linkType: hard
"dir-glob@npm:^3.0.1":
version: 3.0.1
resolution: "dir-glob@npm:3.0.1"
@@ -17618,6 +17632,7 @@ __metadata:
"@types/d3-force": ^3.0.0
"@types/d3-scale-chromatic": 3.0.0
"@types/debounce-promise": 3.1.6
"@types/diff": ^5
"@types/eslint": 8.44.0
"@types/file-saver": 2.0.5
"@types/glob": ^8.0.0
@@ -17712,6 +17727,7 @@ __metadata:
dangerously-set-html-content: 1.0.9
date-fns: 2.30.0
debounce-promise: 3.1.2
diff: ^5.1.0
emotion: 11.0.0
esbuild: 0.18.12
esbuild-loader: 3.0.1