Analytics: Track usage of auto-generate functionality (#75267)

This commit is contained in:
Ivan Ortega Alba 2023-09-22 16:03:50 +02:00 committed by GitHub
parent 4155f0a92e
commit eba74f0408
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 121 additions and 65 deletions

View File

@ -16,9 +16,9 @@ jest.mock('./utils', () => ({
}));
describe('GenAIButton', () => {
const onReply = jest.fn();
const onGenerate = jest.fn();
function setup(props: GenAIButtonProps = { onReply, messages: [] }) {
function setup(props: GenAIButtonProps = { onGenerate, messages: [] }) {
return render(
<Router history={locationService.getHistory()}>
<GenAIButton text="Auto-generate" {...props} />
@ -99,23 +99,23 @@ describe('GenAIButton', () => {
replyHandler('Generated text', isDoneGeneratingMessage);
return new Promise(() => new Subscription());
});
const onReply = jest.fn();
setup({ onReply, messages: [] });
const onGenerate = jest.fn();
setup({ onGenerate, messages: [] });
const generateButton = await screen.findByRole('button');
// Click the button
await fireEvent.click(generateButton);
await waitFor(() => expect(generateButton).toBeEnabled());
await waitFor(() => expect(onReply).toHaveBeenCalledTimes(1));
await waitFor(() => expect(onGenerate).toHaveBeenCalledTimes(1));
// Wait for the loading state to be resolved
expect(onReply).toHaveBeenCalledTimes(1);
expect(onGenerate).toHaveBeenCalledTimes(1);
});
it('should call the LLM service with the messages configured and the right temperature', async () => {
const onReply = jest.fn();
const onGenerate = jest.fn();
const messages = [{ content: 'Generate X', role: 'system' as Role }];
setup({ onReply, messages, temperature: 3 });
setup({ onGenerate, messages, temperature: 3 });
const generateButton = await screen.findByRole('button');
await fireEvent.click(generateButton);
@ -123,5 +123,17 @@ describe('GenAIButton', () => {
await waitFor(() => expect(generateTextWithLLM).toHaveBeenCalledTimes(1));
await waitFor(() => expect(generateTextWithLLM).toHaveBeenCalledWith(messages, expect.any(Function), 3));
});
it('should call the onClick callback', async () => {
const onGenerate = jest.fn();
const onClick = jest.fn();
const messages = [{ content: 'Generate X', role: 'system' as Role }];
setup({ onGenerate, messages, temperature: 3, onClick });
const generateButton = await screen.findByRole('button');
await fireEvent.click(generateButton);
await waitFor(() => expect(onClick).toHaveBeenCalledTimes(1));
});
});
});

View File

@ -7,20 +7,27 @@ import { Button, Spinner, useStyles2, Link, Tooltip } from '@grafana/ui';
import { Message, generateTextWithLLM, isLLMPluginEnabled } from './utils';
export interface GenAIButtonProps {
// Button label text
text?: string;
// Button label text when loading
loadingText?: string;
// Button click handler
onClick?: (e: React.MouseEvent<HTMLButtonElement>) => void;
// Messages to send to the LLM plugin
messages: Message[];
onReply: (response: string, isDone: boolean) => void;
// Callback when the LLM plugin responds. It is sreaming, so it will be called multiple times.
onGenerate: (response: string, isDone: boolean) => void;
// Temperature for the LLM plugin. Default is 1.
// Closer to 0 means more conservative, closer to 1 means more creative.
temperature?: number;
}
export const GenAIButton = ({
text = 'Auto-generate',
loadingText = 'Generating',
onClick,
onClick: onClickProp,
messages,
onReply,
onGenerate,
temperature = 1,
}: GenAIButtonProps) => {
const styles = useStyles2(getStyles);
@ -29,11 +36,11 @@ export const GenAIButton = ({
const replyHandler = (response: string, isDone: boolean) => {
setLoading(!isDone);
onReply(response, isDone);
onGenerate(response, isDone);
};
const onGenerate = (e: React.MouseEvent<HTMLButtonElement>) => {
onClick?.(e);
const onClick = (e: React.MouseEvent<HTMLButtonElement>) => {
onClickProp?.(e);
setLoading(true);
generateTextWithLLM(messages, replyHandler, temperature);
};
@ -67,7 +74,7 @@ export const GenAIButton = ({
</span>
}
>
<Button icon={getIcon()} onClick={onGenerate} fill="text" size="sm" disabled={loading || !enabled}>
<Button icon={getIcon()} onClick={onClick} fill="text" size="sm" disabled={loading || !enabled}>
{!loading ? text : loadingText}
</Button>
</Tooltip>

View File

@ -3,6 +3,7 @@ import React from 'react';
import { DashboardModel } from '../../state';
import { GenAIButton } from './GenAIButton';
import { EventSource, reportGenerateAIButtonClicked } from './tracking';
import { Message, Role } from './utils';
interface GenAIDashDescriptionButtonProps {
@ -17,8 +18,11 @@ const DESCRIPTION_GENERATION_STANDARD_PROMPT =
export const GenAIDashDescriptionButton = ({ onGenerate, dashboard }: GenAIDashDescriptionButtonProps) => {
const messages = React.useMemo(() => getMessages(dashboard), [dashboard]);
const onClick = React.useCallback(() => reportGenerateAIButtonClicked(EventSource.dashboardDescription), []);
return <GenAIButton messages={messages} onReply={onGenerate} loadingText={'Generating description'} />;
return (
<GenAIButton messages={messages} onGenerate={onGenerate} onClick={onClick} loadingText={'Generating description'} />
);
};
function getMessages(dashboard: DashboardModel): Message[] {

View File

@ -3,6 +3,7 @@ import React from 'react';
import { DashboardModel } from '../../state';
import { GenAIButton } from './GenAIButton';
import { EventSource, reportGenerateAIButtonClicked } from './tracking';
import { Message, Role } from './utils';
interface GenAIDashTitleButtonProps {
@ -17,8 +18,9 @@ const DESCRIPTION_GENERATION_STANDARD_PROMPT =
export const GenAIDashTitleButton = ({ onGenerate, dashboard }: GenAIDashTitleButtonProps) => {
const messages = React.useMemo(() => getMessages(dashboard), [dashboard]);
const onClick = React.useCallback(() => reportGenerateAIButtonClicked(EventSource.dashboardTitle), []);
return <GenAIButton messages={messages} onReply={onGenerate} loadingText={'Generating title'} />;
return <GenAIButton messages={messages} onClick={onClick} onGenerate={onGenerate} loadingText={'Generating title'} />;
};
function getMessages(dashboard: DashboardModel): Message[] {

View File

@ -3,6 +3,7 @@ import React, { useMemo } from 'react';
import { DashboardModel } from '../../state';
import { GenAIButton } from './GenAIButton';
import { EventSource, reportGenerateAIButtonClicked } from './tracking';
import { getDashboardChanges, Message, Role } from './utils';
interface GenAIDashboardChangesButtonProps {
@ -26,9 +27,16 @@ const CHANGES_GENERATION_STANDARD_PROMPT = [
export const GenAIDashboardChangesButton = ({ dashboard, onGenerate }: GenAIDashboardChangesButtonProps) => {
const messages = useMemo(() => getMessages(dashboard), [dashboard]);
const onClick = React.useCallback(() => reportGenerateAIButtonClicked(EventSource.dashboardChanges), []);
return (
<GenAIButton messages={messages} onReply={onGenerate} loadingText={'Generating changes summary'} temperature={0} />
<GenAIButton
messages={messages}
onGenerate={onGenerate}
onClick={onClick}
loadingText={'Generating changes summary'}
temperature={0}
/>
);
};

View File

@ -4,6 +4,7 @@ import { getDashboardSrv } from '../../services/DashboardSrv';
import { PanelModel } from '../../state';
import { GenAIButton } from './GenAIButton';
import { EventSource, reportGenerateAIButtonClicked } from './tracking';
import { Message, Role } from './utils';
interface GenAIPanelDescriptionButtonProps {
@ -17,28 +18,33 @@ const DESCRIPTION_GENERATION_STANDARD_PROMPT =
'The description should be shorter than 140 characters.';
export const GenAIPanelDescriptionButton = ({ onGenerate, panel }: GenAIPanelDescriptionButtonProps) => {
function getMessages(): Message[] {
const dashboard = getDashboardSrv().getCurrent()!;
const messages = React.useMemo(() => getMessages(panel), [panel]);
const onClick = React.useCallback(() => reportGenerateAIButtonClicked(EventSource.panelDescription), []);
return [
{
content: DESCRIPTION_GENERATION_STANDARD_PROMPT,
role: Role.system,
},
{
content: `The panel is part of a dashboard with the title: ${dashboard.title}`,
role: Role.system,
},
{
content: `The panel is part of a dashboard with the description: ${dashboard.title}`,
role: Role.system,
},
{
content: `Use this JSON object which defines the panel: ${JSON.stringify(panel.getSaveModel())}`,
role: Role.user,
},
];
}
return <GenAIButton messages={getMessages()} onReply={onGenerate} loadingText={'Generating description'} />;
return (
<GenAIButton messages={messages} onClick={onClick} onGenerate={onGenerate} loadingText={'Generating description'} />
);
};
function getMessages(panel: PanelModel): Message[] {
const dashboard = getDashboardSrv().getCurrent()!;
return [
{
content: DESCRIPTION_GENERATION_STANDARD_PROMPT,
role: Role.system,
},
{
content: `The panel is part of a dashboard with the title: ${dashboard.title}`,
role: Role.system,
},
{
content: `The panel is part of a dashboard with the description: ${dashboard.title}`,
role: Role.system,
},
{
content: `Use this JSON object which defines the panel: ${JSON.stringify(panel.getSaveModel())}`,
role: Role.user,
},
];
}

View File

@ -4,6 +4,7 @@ import { getDashboardSrv } from '../../services/DashboardSrv';
import { PanelModel } from '../../state';
import { GenAIButton } from './GenAIButton';
import { EventSource, reportGenerateAIButtonClicked } from './tracking';
import { Message, Role } from './utils';
interface GenAIPanelTitleButtonProps {
@ -17,28 +18,31 @@ const TITLE_GENERATION_STANDARD_PROMPT =
'The title should be shorter than 50 characters.';
export const GenAIPanelTitleButton = ({ onGenerate, panel }: GenAIPanelTitleButtonProps) => {
function getMessages(): Message[] {
const dashboard = getDashboardSrv().getCurrent()!;
const messages = React.useMemo(() => getMessages(panel), [panel]);
const onClick = React.useCallback(() => reportGenerateAIButtonClicked(EventSource.panelTitle), []);
return [
{
content: TITLE_GENERATION_STANDARD_PROMPT,
role: Role.system,
},
{
content: `The panel is part of a dashboard with the title: ${dashboard.title}`,
role: Role.system,
},
{
content: `The panel is part of a dashboard with the description: ${dashboard.title}`,
role: Role.system,
},
{
content: `Use this JSON object which defines the panel: ${JSON.stringify(panel.getSaveModel())}`,
role: Role.user,
},
];
}
return <GenAIButton messages={getMessages()} onReply={onGenerate} loadingText={'Generating title'} />;
return <GenAIButton messages={messages} onClick={onClick} onGenerate={onGenerate} loadingText={'Generating title'} />;
};
function getMessages(panel: PanelModel): Message[] {
const dashboard = getDashboardSrv().getCurrent()!;
return [
{
content: TITLE_GENERATION_STANDARD_PROMPT,
role: Role.system,
},
{
content: `The panel is part of a dashboard with the title: ${dashboard.title}`,
role: Role.system,
},
{
content: `The panel is part of a dashboard with the description: ${dashboard.title}`,
role: Role.system,
},
{
content: `Use this JSON object which defines the panel: ${JSON.stringify(panel.getSaveModel())}`,
role: Role.user,
},
];
}

View File

@ -0,0 +1,13 @@
import { reportInteraction } from '@grafana/runtime';
export enum EventSource {
panelDescription = 'panel-description',
panelTitle = 'panel-title',
dashboardChanges = 'dashboard-changes',
dashboardTitle = 'dashboard-title',
dashboardDescription = 'dashboard-description',
}
export function reportGenerateAIButtonClicked(src: EventSource) {
reportInteraction('dashboards_autogenerate_clicked', { src });
}

View File

@ -42,7 +42,7 @@ export const OPEN_AI_MODEL = 'gpt-4';
*
* @param messages messages to send to LLM
* @param onReply callback to call when LLM replies. The reply will be streamed, so it will be called for every token received.
* @param temperature what temperature to use when calling the llm. default 1.
* @param temperature what temperature to use when calling the llm. default 1. Closer to 0 means more conservative, closer to 1 means more creative.
* @returns The subscription to the stream.
*/
export const generateTextWithLLM = async (