Alerting: Loki-based alert state history modal (#66595)

* adds alertstatehistory backend config to grafanaBootData

* add alertStateHistory api

* show different ASH modal when using loki implementation

* group log lines by instance (unique set of labels)
Co-Authored-By: Konrad Lalik <konrad.lalik@grafana.com>

* render log lines for each instance
Co-Authored-By: Konrad Lalik <konrad.lalik@grafana.com>

* Add visual improvements to the log record of state changes

* Add values to log records

* compute common labels and show unique labels

* Add state changes visualization

* fix common labels extraction

* Code cleanup

* Add timespan-based log record view

* WIP

* scroll to timestamp - poc

* Use SortedVector for timestamp field

* add conditional accessor for frames

* update some of the log formats and styles

* Timestamp-based visualization with scrolling

* minor improvements

* Split Loki's state history viewer into multiple files

* Add memoization to prevent graph rerender on filter updates

* make chart size shrink when fewer instances

* style updates

* show warning when instances are hidden

* Add basic label-based filtering

* Improve label-based filtering

* Add regex validation

* Improve no instances message when everything was filtered out

* Update warning message

* Move timeline viewer to a separate file, refactor handling timeline pointer changes

* Remove unused component, add comments

* Fix test snapshot, fix type error

* adds tests for common.ts

* Add tests for converting log records into data frames

* Add basic component test, fix type guards

* Use a constant for timeseries limit

* Improve a11y, update component test

* Memoize AlertStateTag, migrate from deprecated ArrayVector

* Update public/app/features/alerting/unified/components/rules/state-history/common.ts

* Move helper hook into a separate file. Add Search input component

* Change the limit of visible time series on the timeline

* Add LogRecordViewer perf improvements, refactor timeline cursor events tracking

* Use callback to pass timeline refs

* Add grouping tests for the log record viewer

---------

Co-authored-by: Gilles De Mey <gilles.de.mey@gmail.com>
This commit is contained in:
Konrad Lalik 2023-04-24 09:28:11 +02:00 committed by GitHub
parent fe5a07f336
commit 91704cf7de
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 1139 additions and 37 deletions

View File

@ -74,6 +74,8 @@ export interface GrafanaJavascriptAgentConfig {
export interface UnifiedAlertingConfig {
minInterval: string;
// will be undefined if alerStateHistory is not enabled
alertStateHistoryBackend?: string;
}
/** Supported OAuth services

View File

@ -133,7 +133,7 @@ export class GrafanaBootConfig implements GrafanaConfig {
geomapDefaultBaseLayerConfig?: MapLayerOptions;
geomapDisableCustomBaseLayer?: boolean;
unifiedAlertingEnabled = false;
unifiedAlerting = { minInterval: '' };
unifiedAlerting = { minInterval: '', alertStateHistoryBackend: undefined };
applicationInsightsConnectionString?: string;
applicationInsightsEndpointUrl?: string;
recordedQueries = {

View File

@ -61,7 +61,8 @@ type FrontendSettingsReportingDTO struct {
}
type FrontendSettingsUnifiedAlertingDTO struct {
MinInterval string `json:"minInterval"`
MinInterval string `json:"minInterval"`
AlertStateHistoryBackend string `json:"alertStateHistoryBackend,omitempty"`
}
// Enterprise-only

View File

@ -218,6 +218,10 @@ func (hs *HTTPServer) getFrontendSettings(c *contextmodel.ReqContext) (*dtos.Fro
SnapshotEnabled: hs.Cfg.SnapshotEnabled,
}
if hs.Cfg.UnifiedAlerting.StateHistory.Enabled {
frontendSettings.UnifiedAlerting.AlertStateHistoryBackend = hs.Cfg.UnifiedAlerting.StateHistory.Backend
}
if hs.Cfg.UnifiedAlerting.Enabled != nil {
frontendSettings.UnifiedAlertingEnabled = *hs.Cfg.UnifiedAlerting.Enabled
}

View File

@ -1,5 +1,5 @@
import { css } from '@emotion/css';
import { isEqual, orderBy, uniqWith } from 'lodash';
import { orderBy } from 'lodash';
import React, { useEffect, useState } from 'react';
import { useDebounce } from 'react-use';
@ -15,7 +15,7 @@ import { useCombinedRuleNamespaces } from './hooks/useCombinedRuleNamespaces';
import { usePagination } from './hooks/usePagination';
import { useURLSearchParams } from './hooks/useURLSearchParams';
import { fetchPromRulesAction, fetchRulerRulesAction } from './state/actions';
import { labelsMatchMatchers, matchersToString, parseMatcher, parseMatchers } from './utils/alertmanager';
import { combineMatcherStrings, labelsMatchMatchers, parseMatchers } from './utils/alertmanager';
import { GRAFANA_RULES_SOURCE_NAME } from './utils/datasource';
import { createViewLink } from './utils/misc';
@ -38,10 +38,7 @@ export const AlertsFolderView = ({ folder }: Props) => {
const dispatch = useDispatch();
const onTagClick = (tagName: string) => {
const matchers = parseMatchers(labelFilter);
const tagMatcherField = parseMatcher(tagName);
const uniqueMatchers = uniqWith([...matchers, tagMatcherField], isEqual);
const matchersString = matchersToString(uniqueMatchers);
const matchersString = combineMatcherStrings(labelFilter, tagName);
setLabelFilter(matchersString);
};

View File

@ -0,0 +1,16 @@
import { getUnixTime } from 'date-fns';
import { DataFrameJSON } from '@grafana/data';
import { alertingApi } from './alertingApi';
export const stateHistoryApi = alertingApi.injectEndpoints({
endpoints: (build) => ({
getRuleHistory: build.query<DataFrameJSON, { ruleUid: string; from: number; to?: number }>({
query: ({ ruleUid, from, to = getUnixTime(new Date()) }) => ({
url: '/api/v1/rules/history',
params: { ruleUID: ruleUid, from, to },
}),
}),
}),
});

View File

@ -9,12 +9,17 @@ export type State = 'good' | 'bad' | 'warning' | 'neutral' | 'info';
type Props = {
state: State;
size?: 'md' | 'sm';
muted?: boolean;
};
export const StateTag = ({ children, state, size = 'md' }: React.PropsWithChildren<Props>) => {
export const StateTag = ({ children, state, size = 'md', muted = false }: React.PropsWithChildren<Props>) => {
const styles = useStyles2(getStyles);
return <span className={cx(styles.common, styles[state], styles[size])}>{children || state}</span>;
return (
<span className={cx(styles.common, styles[state], styles[size], { [styles.muted]: muted })}>
{children || state}
</span>
);
};
const getStyles = (theme: GrafanaTheme2) => ({
@ -61,4 +66,7 @@ const getStyles = (theme: GrafanaTheme2) => ({
padding: ${theme.spacing(0.3, 0.5)};
min-width: 52px;
`,
muted: css`
opacity: 0.5;
`,
});

View File

@ -9,10 +9,12 @@ interface Props {
state: PromAlertingRuleState | GrafanaAlertState | GrafanaAlertStateWithReason | AlertState;
size?: 'md' | 'sm';
isPaused?: boolean;
muted?: boolean;
}
export const AlertStateTag = ({ state, isPaused = false, size = 'md' }: Props) => (
<StateTag state={alertStateToState(state)} size={size}>
export const AlertStateTag = React.memo(({ state, isPaused = false, size = 'md', muted = false }: Props) => (
<StateTag state={alertStateToState(state)} size={size} muted={muted}>
{alertStateToReadable(state)} {isPaused ? ' (Paused)' : ''}
</StateTag>
);
));
AlertStateTag.displayName = 'AlertStateTag';

View File

@ -37,8 +37,7 @@ interface Props {
export const RuleDetailsActionButtons = ({ rule, rulesSource, isViewMode }: Props) => {
const style = useStyles2(getStyles);
const { namespace, group, rulerRule } = rule;
const alertId = isGrafanaRulerRule(rule.rulerRule) ? rule.rulerRule.grafana_alert.id ?? '' : '';
const { StateHistoryModal, showStateHistoryModal } = useStateHistoryModal(alertId);
const { StateHistoryModal, showStateHistoryModal } = useStateHistoryModal();
const dispatch = useDispatch();
const location = useLocation();
const notifyApp = useAppNotification();
@ -159,10 +158,14 @@ export const RuleDetailsActionButtons = ({ rule, rulesSource, isViewMode }: Prop
);
}
if (alertId) {
if (isGrafanaRulerRule(rule.rulerRule)) {
buttons.push(
<Fragment key="history">
<Button size="sm" icon="history" onClick={() => showStateHistoryModal()}>
<Button
size="sm"
icon="history"
onClick={() => isGrafanaRulerRule(rule.rulerRule) && showStateHistoryModal(rule.rulerRule)}
>
Show state history
</Button>
{StateHistoryModal}

View File

@ -0,0 +1,35 @@
import { getByTestId, render } from '@testing-library/react';
import React from 'react';
import { byRole } from 'testing-library-selector';
import { LogRecordViewerByTimestamp } from './LogRecordViewer';
import { LogRecord } from './common';
const ui = {
log: byRole('list', { name: 'State history by timestamp' }),
};
describe('LogRecordViewerByTimestamp', () => {
it('should group the same timestamps into one group', () => {
const records: LogRecord[] = [
{ timestamp: 1681739580000, line: { current: 'Alerting', previous: 'Pending', labels: { foo: 'bar' } } },
{ timestamp: 1681739580000, line: { current: 'Alerting', previous: 'Pending', labels: { severity: 'warning' } } },
{ timestamp: 1681739600000, line: { current: 'Normal', previous: 'Alerting', labels: { foo: 'bar' } } },
{ timestamp: 1681739600000, line: { current: 'Normal', previous: 'Alerting', labels: { severity: 'warning' } } },
];
render(<LogRecordViewerByTimestamp records={records} commonLabels={[]} />);
const logElement = ui.log.get();
expect(logElement).toBeInTheDocument();
const entry1 = getByTestId(logElement, 1681739580000);
const entry2 = getByTestId(logElement, 1681739600000);
expect(entry1).toHaveTextContent('foo=bar');
expect(entry1).toHaveTextContent('severity=warning');
expect(entry2).toHaveTextContent('foo=bar');
expect(entry2).toHaveTextContent('severity=warning');
});
});

View File

@ -0,0 +1,174 @@
import { css } from '@emotion/css';
import { formatDistanceToNowStrict } from 'date-fns';
import { groupBy, uniqueId } from 'lodash';
import React, { useEffect } from 'react';
import { dateTimeFormat, GrafanaTheme2 } from '@grafana/data';
import { Stack } from '@grafana/experimental';
import { Icon, TagList, useStyles2 } from '@grafana/ui';
import { Label } from '../../Label';
import { AlertStateTag } from '../AlertStateTag';
import { LogRecord, omitLabels } from './common';
interface LogRecordViewerProps {
records: LogRecord[];
commonLabels: Array<[string, string]>;
onRecordsRendered?: (timestampRefs: Map<number, HTMLElement>) => void;
onLabelClick?: (label: string) => void;
}
export const LogRecordViewerByTimestamp = React.memo(
({ records, commonLabels, onLabelClick, onRecordsRendered }: LogRecordViewerProps) => {
const styles = useStyles2(getStyles);
// groupBy has been replaced by the reduce to avoid back and forth conversion of timestamp from number to string
const groupedLines = records.reduce((acc, current) => {
const tsGroup = acc.get(current.timestamp);
if (tsGroup) {
tsGroup.push(current);
} else {
acc.set(current.timestamp, [current]);
}
return acc;
}, new Map<number, LogRecord[]>());
const timestampRefs = new Map<number, HTMLElement>();
useEffect(() => {
onRecordsRendered && onRecordsRendered(timestampRefs);
});
return (
<ul className={styles.logsScrollable} aria-label="State history by timestamp">
{Array.from(groupedLines.entries()).map(([key, records]) => {
return (
<li
id={key.toString(10)}
key={key}
data-testid={key}
ref={(element) => element && timestampRefs.set(key, element)}
>
<Timestamp time={key} />
<div className={styles.logsContainer}>
{records.map(({ line }) => (
<React.Fragment key={uniqueId()}>
<AlertStateTag state={line.previous} size="sm" muted />
<Icon name="arrow-right" size="sm" />
<AlertStateTag state={line.current} />
<Stack direction="row">{line.values && <AlertInstanceValues record={line.values} />}</Stack>
<div>
{line.labels && (
<TagList
tags={omitLabels(Object.entries(line.labels), commonLabels).map(
([key, value]) => `${key}=${value}`
)}
onClick={onLabelClick}
/>
)}
</div>
</React.Fragment>
))}
</div>
</li>
);
})}
</ul>
);
}
);
LogRecordViewerByTimestamp.displayName = 'LogRecordViewerByTimestamp';
export function LogRecordViewerByInstance({ records, commonLabels }: LogRecordViewerProps) {
const styles = useStyles2(getStyles);
const groupedLines = groupBy(records, (record: LogRecord) => {
return JSON.stringify(record.line.labels);
});
return (
<>
{Object.entries(groupedLines).map(([key, records]) => {
return (
<Stack direction="column" key={key}>
<h4>
<TagList
tags={omitLabels(Object.entries(records[0].line.labels ?? {}), commonLabels).map(
([key, value]) => `${key}=${value}`
)}
/>
</h4>
<div className={styles.logsContainer}>
{records.map(({ line, timestamp }) => (
<div key={uniqueId()}>
<AlertStateTag state={line.previous} size="sm" muted />
<Icon name="arrow-right" size="sm" />
<AlertStateTag state={line.current} />
<Stack direction="row">{line.values && <AlertInstanceValues record={line.values} />}</Stack>
<div>{dateTimeFormat(timestamp)}</div>
</div>
))}
</div>
</Stack>
);
})}
</>
);
}
interface TimestampProps {
time: number; // epoch timestamp
}
const Timestamp = ({ time }: TimestampProps) => {
const dateTime = new Date(time);
const styles = useStyles2(getStyles);
return (
<div className={styles.timestampWrapper}>
<Stack direction="row" alignItems="center" gap={1}>
<Icon name="clock-nine" size="sm" />
<span className={styles.timestampText}>{dateTimeFormat(dateTime)}</span>
<small>({formatDistanceToNowStrict(dateTime)} ago)</small>
</Stack>
</div>
);
};
const AlertInstanceValues = React.memo(({ record }: { record: Record<string, number> }) => {
const values = Object.entries(record);
return (
<>
{values.map(([key, value]) => (
<Label key={key} label={key} value={value} />
))}
</>
);
});
AlertInstanceValues.displayName = 'AlertInstanceValues';
const getStyles = (theme: GrafanaTheme2) => ({
logsContainer: css`
display: grid;
grid-template-columns: max-content max-content max-content auto max-content;
gap: ${theme.spacing(2, 1)};
align-items: center;
`,
logsScrollable: css`
height: 500px;
overflow: scroll;
flex: 1;
`,
timestampWrapper: css`
color: ${theme.colors.text.secondary};
padding: ${theme.spacing(1)} 0;
`,
timestampText: css`
color: ${theme.colors.text.primary};
font-size: ${theme.typography.bodySmall.fontSize};
font-weight: ${theme.typography.fontWeightBold};
`,
});

View File

@ -0,0 +1,104 @@
import { noop } from 'lodash';
import React, { useEffect, useRef } from 'react';
import AutoSizer from 'react-virtualized-auto-sizer';
import { BehaviorSubject } from 'rxjs';
import { DataFrame, TimeRange } from '@grafana/data';
import { VisibilityMode } from '@grafana/schema';
import { LegendDisplayMode, UPlotConfigBuilder, useTheme2 } from '@grafana/ui';
import { TimelineChart } from 'app/core/components/TimelineChart/TimelineChart';
import { TimelineMode } from 'app/core/components/TimelineChart/utils';
interface LogTimelineViewerProps {
frames: DataFrame[];
timeRange: TimeRange;
onPointerMove?: (seriesIdx: number, pointerIdx: number) => void;
}
export const LogTimelineViewer = React.memo(({ frames, timeRange, onPointerMove = noop }: LogTimelineViewerProps) => {
const theme = useTheme2();
const { setupCursorTracking } = useCursorTimelinePosition(onPointerMove);
return (
<AutoSizer disableHeight>
{({ width }) => (
<TimelineChart
frames={frames}
timeRange={timeRange}
timeZone={'browser'}
mode={TimelineMode.Changes}
height={18 * frames.length + 50}
width={width}
showValue={VisibilityMode.Never}
theme={theme}
rowHeight={0.8}
legend={{
calcs: [],
displayMode: LegendDisplayMode.List,
placement: 'bottom',
showLegend: true,
}}
legendItems={[
{ label: 'Normal', color: theme.colors.success.main, yAxis: 1 },
{ label: 'Pending', color: theme.colors.warning.main, yAxis: 1 },
{ label: 'Alerting', color: theme.colors.error.main, yAxis: 1 },
{ label: 'NoData', color: theme.colors.info.main, yAxis: 1 },
]}
>
{(builder) => {
setupCursorTracking(builder);
return null;
}}
</TimelineChart>
)}
</AutoSizer>
);
});
function useCursorTimelinePosition(onPointerMove: (seriesIdx: number, pointIdx: number) => void) {
const pointerSubject = useRef(
new BehaviorSubject<{ seriesIdx: number; pointIdx: number }>({ seriesIdx: 0, pointIdx: 0 })
);
useEffect(() => {
const subscription = pointerSubject.current.subscribe(({ seriesIdx, pointIdx }) => {
onPointerMove && onPointerMove(seriesIdx, pointIdx);
});
return () => {
subscription.unsubscribe();
};
}, [onPointerMove]);
// Applies cursor tracking to the UPlot chart
const setupCursorTracking = (builder: UPlotConfigBuilder) => {
builder.setSync();
const interpolator = builder.getTooltipInterpolator();
// I found this in TooltipPlugin.tsx
if (interpolator) {
builder.addHook('setCursor', (u) => {
interpolator(
(seriesIdx) => {
if (seriesIdx) {
const currentPointer = pointerSubject.current.getValue();
pointerSubject.current.next({ ...currentPointer, seriesIdx });
}
},
(pointIdx) => {
if (pointIdx) {
const currentPointer = pointerSubject.current.getValue();
pointerSubject.current.next({ ...currentPointer, pointIdx });
}
},
() => {},
u
);
});
}
};
return { setupCursorTracking };
}
LogTimelineViewer.displayName = 'LogTimelineViewer';

View File

@ -0,0 +1,99 @@
import { render, waitFor } from '@testing-library/react';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import React from 'react';
import { byRole, byText } from 'testing-library-selector';
import 'whatwg-fetch';
import { DataFrameJSON } from '@grafana/data';
import { setBackendSrv } from '@grafana/runtime';
import { TestProvider } from '../../../../../../../test/helpers/TestProvider';
import { backendSrv } from '../../../../../../core/services/backend_srv';
import LokiStateHistory from './LokiStateHistory';
const server = setupServer();
beforeAll(() => {
setBackendSrv(backendSrv);
server.listen({ onUnhandledRequest: 'error' });
server.use(
rest.get('/api/v1/rules/history', (req, res, ctx) =>
res(
ctx.json<DataFrameJSON>({
data: {
values: [
[1681739580000, 1681739580000, 1681739580000],
[
{
previous: 'Normal',
current: 'Pending',
values: {
B: 0.010344684900897919,
C: 1,
},
labels: {
handler: '/api/prometheus/grafana/api/v1/rules',
},
},
{
previous: 'Normal',
current: 'Pending',
values: {
B: 0.010344684900897919,
C: 1,
},
dashboardUID: '',
panelID: 0,
labels: {
handler: '/api/live/ws',
},
},
{
previous: 'Normal',
current: 'Pending',
values: {
B: 0.010344684900897919,
C: 1,
},
labels: {
handler: '/api/folders/:uid/',
},
},
],
],
},
})
)
)
);
});
afterAll(() => {
server.close();
});
window.HTMLElement.prototype.scrollIntoView = jest.fn();
const ui = {
loadingIndicator: byText('Loading...'),
timestampViewer: byRole('list', { name: 'State history by timestamp' }),
record: byRole('listitem'),
};
describe('LokiStateHistory', () => {
it('should render history records', async () => {
render(<LokiStateHistory ruleUID="ABC123" />, { wrapper: TestProvider });
await waitFor(() => expect(ui.loadingIndicator.query()).not.toBeInTheDocument());
const timestampViewerElement = ui.timestampViewer.get();
expect(timestampViewerElement).toBeInTheDocument();
expect(timestampViewerElement).toHaveTextContent('/api/prometheus/grafana/api/v1/rules');
expect(timestampViewerElement).toHaveTextContent('/api/live/ws');
expect(timestampViewerElement).toHaveTextContent('/api/folders/:uid/');
});
});

View File

@ -0,0 +1,241 @@
import { css } from '@emotion/css';
import { isEmpty, sortBy, take, uniq } from 'lodash';
import React, { useCallback, useMemo, useRef, useState } from 'react';
import { useForm } from 'react-hook-form';
import { DataFrame, dateTime, GrafanaTheme2, TimeRange } from '@grafana/data';
import { Stack } from '@grafana/experimental';
import { Alert, Button, Field, Icon, Input, Label, TagList, Tooltip, useStyles2 } from '@grafana/ui';
import { stateHistoryApi } from '../../../api/stateHistoryApi';
import { combineMatcherStrings } from '../../../utils/alertmanager';
import { HoverCard } from '../../HoverCard';
import { LogRecordViewerByTimestamp } from './LogRecordViewer';
import { LogTimelineViewer } from './LogTimelineViewer';
import { useRuleHistoryRecords } from './useRuleHistoryRecords';
interface Props {
ruleUID: string;
}
const MAX_TIMELINE_SERIES = 12;
const LokiStateHistory = ({ ruleUID }: Props) => {
const styles = useStyles2(getStyles);
const [instancesFilter, setInstancesFilter] = useState('');
const logsRef = useRef<Map<number, HTMLElement>>(new Map<number, HTMLElement>());
const { getValues, setValue, register, handleSubmit } = useForm({ defaultValues: { query: '' } });
const { useGetRuleHistoryQuery } = stateHistoryApi;
const timeRange = useMemo(() => getDefaultTimeRange(), []);
const {
currentData: stateHistory,
isLoading,
isError,
error,
} = useGetRuleHistoryQuery({ ruleUid: ruleUID, from: timeRange.from.unix(), to: timeRange.to.unix() });
const { dataFrames, historyRecords, commonLabels, totalRecordsCount } = useRuleHistoryRecords(
stateHistory,
instancesFilter
);
const { frameSubset, frameSubsetTimestamps } = useFrameSubset(dataFrames);
const onLogRecordLabelClick = useCallback(
(label: string) => {
const matcherString = combineMatcherStrings(getValues('query'), label);
setInstancesFilter(matcherString);
setValue('query', matcherString);
},
[setInstancesFilter, setValue, getValues]
);
const onFilterCleared = useCallback(() => {
setInstancesFilter('');
setValue('query', '');
}, [setInstancesFilter, setValue]);
const onTimelinePointerMove = useCallback(
(seriesIdx: number, pointIdx: number) => {
const timestamp = frameSubsetTimestamps[pointIdx];
const refToScroll = logsRef.current.get(timestamp);
refToScroll?.scrollIntoView({ behavior: 'smooth', block: 'start' });
},
[frameSubsetTimestamps]
);
if (isLoading) {
return <div>Loading...</div>;
}
if (isError) {
return (
<Alert title="Error fetching the state history" severity="error">
{error instanceof Error ? error.message : 'Unable to fetch alert state history'}
</Alert>
);
}
const hasMoreInstances = frameSubset.length < dataFrames.length;
const emptyStateMessage =
totalRecordsCount > 0
? `No matches were found for the given filters among the ${totalRecordsCount} instances`
: 'No state transitions have occurred in the last 60 minutes';
return (
<div className={styles.fullSize}>
<form onSubmit={handleSubmit((data) => setInstancesFilter(data.query))}>
<SearchFieldInput
{...register('query')}
showClearFilterSuffix={!!instancesFilter}
onClearFilterClick={onFilterCleared}
/>
<input type="submit" hidden />
</form>
{!isEmpty(commonLabels) && (
<div className={styles.commonLabels}>
<Stack gap={1} alignItems="center">
<strong>Common labels</strong>
<Tooltip content="Common labels are the ones attached to all of the alert instances">
<Icon name="info-circle" />
</Tooltip>
</Stack>
<TagList tags={commonLabels.map((label) => label.join('='))} />
</div>
)}
{isEmpty(frameSubset) ? (
<>
<div className={styles.emptyState}>
{emptyStateMessage}
{totalRecordsCount > 0 && (
<Button variant="secondary" type="button" onClick={onFilterCleared}>
Clear filters
</Button>
)}
</div>
</>
) : (
<>
<div className={styles.graphWrapper}>
<LogTimelineViewer frames={frameSubset} timeRange={timeRange} onPointerMove={onTimelinePointerMove} />
</div>
{hasMoreInstances && (
<div className={styles.moreInstancesWarning}>
<Stack direction="row" alignItems="center" gap={1}>
<Icon name="exclamation-triangle" size="sm" />
<small>{`Only showing ${frameSubset.length} out of ${dataFrames.length} instances. Click on the labels to narrow down the results`}</small>
</Stack>
</div>
)}
<LogRecordViewerByTimestamp
records={historyRecords}
commonLabels={commonLabels}
onRecordsRendered={(recordRefs) => (logsRef.current = recordRefs)}
onLabelClick={onLogRecordLabelClick}
/>
</>
)}
</div>
);
};
function useFrameSubset(frames: DataFrame[]) {
return useMemo(() => {
const frameSubset = take(frames, MAX_TIMELINE_SERIES);
const frameSubsetTimestamps = sortBy(uniq(frameSubset.flatMap((frame) => frame.fields[0].values)));
return { frameSubset, frameSubsetTimestamps };
}, [frames]);
}
interface SearchFieldInputProps extends Omit<React.ComponentProps<typeof Input>, 'prefix' | 'suffix' | 'placeholder'> {
showClearFilterSuffix: boolean;
onClearFilterClick: () => void;
}
const SearchFieldInput = React.forwardRef<HTMLInputElement, SearchFieldInputProps>(
({ showClearFilterSuffix, onClearFilterClick, ...rest }: SearchFieldInputProps, ref) => {
return (
<Field
label={
<Label htmlFor="instancesSearchInput">
<Stack gap={0.5}>
<span>Filter instances</span>
<HoverCard
content={
<>
Use label matcher expression (like <code>{'{foo=bar}'}</code>) or click on an instance label to
filter instances
</>
}
>
<Icon name="info-circle" size="sm" />
</HoverCard>
</Stack>
</Label>
}
>
<Input
id="instancesSearchInput"
prefix={<Icon name="search" />}
suffix={
showClearFilterSuffix && (
<Button fill="text" icon="times" size="sm" onClick={onClearFilterClick}>
Clear
</Button>
)
}
placeholder="Filter instances"
ref={ref}
{...rest}
/>
</Field>
);
}
);
SearchFieldInput.displayName = 'SearchFieldInput';
function getDefaultTimeRange(): TimeRange {
const fromDateTime = dateTime().subtract(1, 'h');
const toDateTime = dateTime();
return {
from: fromDateTime,
to: toDateTime,
raw: { from: fromDateTime, to: toDateTime },
};
}
export const getStyles = (theme: GrafanaTheme2) => ({
fullSize: css`
min-width: 100%;
height: 100%;
display: flex;
flex-direction: column;
`,
graphWrapper: css`
padding: ${theme.spacing()} 0;
`,
emptyState: css`
color: ${theme.colors.text.secondary};
display: flex;
flex-direction: column;
gap: ${theme.spacing(2)};
align-items: center;
margin: auto auto;
`,
moreInstancesWarning: css`
color: ${theme.colors.warning.text};
padding: ${theme.spacing()};
`,
commonLabels: css`
display: grid;
grid-template-columns: max-content auto;
`,
});
export default LokiStateHistory;

View File

@ -8,11 +8,10 @@ import { Alert, Field, Icon, Input, Label, LoadingPlaceholder, Tooltip, useStyle
import { StateHistoryItem, StateHistoryItemData } from 'app/types/unified-alerting';
import { GrafanaAlertStateWithReason, PromAlertingRuleState } from 'app/types/unified-alerting-dto';
import { useManagedAlertStateHistory } from '../../hooks/useManagedAlertStateHistory';
import { AlertLabel } from '../AlertLabel';
import { DynamicTable, DynamicTableColumnProps, DynamicTableItemProps } from '../DynamicTable';
import { AlertStateTag } from './AlertStateTag';
import { useManagedAlertStateHistory } from '../../../hooks/useManagedAlertStateHistory';
import { AlertLabel } from '../../AlertLabel';
import { DynamicTable, DynamicTableColumnProps, DynamicTableItemProps } from '../../DynamicTable';
import { AlertStateTag } from '../AlertStateTag';
type StateHistoryRowItem = {
id: string;
@ -202,4 +201,4 @@ const getStyles = (theme: GrafanaTheme2) => ({
`,
});
export { StateHistory };
export default StateHistory;

View File

@ -0,0 +1,50 @@
import { extractCommonLabels, Label, omitLabels } from './common';
test('extractCommonLabels', () => {
const labels: Label[][] = [
[
['foo', 'bar'],
['baz', 'qux'],
],
[
['foo', 'bar'],
['baz', 'qux'],
['potato', 'tomato'],
],
];
expect(extractCommonLabels(labels)).toStrictEqual([
['foo', 'bar'],
['baz', 'qux'],
]);
});
test('extractCommonLabels with no common labels', () => {
const labels: Label[][] = [[['foo', 'bar']], [['potato', 'tomato']]];
expect(extractCommonLabels(labels)).toStrictEqual([]);
});
test('omitLabels', () => {
const labels: Label[] = [
['foo', 'bar'],
['baz', 'qux'],
['potato', 'tomato'],
];
const commonLabels: Label[] = [
['foo', 'bar'],
['baz', 'qux'],
];
expect(omitLabels(labels, commonLabels)).toStrictEqual([['potato', 'tomato']]);
});
test('omitLabels with no common labels', () => {
const labels: Label[] = [['potato', 'tomato']];
const commonLabels: Label[] = [
['foo', 'bar'],
['baz', 'qux'],
];
expect(omitLabels(labels, commonLabels)).toStrictEqual(labels);
});

View File

@ -0,0 +1,39 @@
import { isEqual, uniqBy } from 'lodash';
import { GrafanaAlertStateWithReason } from 'app/types/unified-alerting-dto';
export interface Line {
previous: GrafanaAlertStateWithReason;
current: GrafanaAlertStateWithReason;
values?: Record<string, number>;
labels?: Record<string, string>;
}
export interface LogRecord {
timestamp: number;
line: Line;
}
export type Label = [string, string];
// omit "common" labels from "labels"
export function omitLabels(labels: Label[], common: Label[]): Label[] {
return labels.filter((label) => {
return !common.find((commonLabel) => JSON.stringify(commonLabel) === JSON.stringify(label));
});
}
// find all common labels by looking at which ones occur in every record, then create a unique array of items for those
export function extractCommonLabels(labels: Label[][]): Label[] {
const flatLabels = labels.flatMap((label) => label);
const commonLabels = uniqBy(
flatLabels.filter((label) => {
const count = flatLabels.filter((l) => isEqual(label, l)).length;
return count === Object.keys(labels).length;
}),
(label) => JSON.stringify(label)
);
return commonLabels;
}

View File

@ -0,0 +1,104 @@
import { createTheme, FieldType } from '@grafana/data';
import { LogRecord } from './common';
import { logRecordsToDataFrame } from './useRuleHistoryRecords';
describe('logRecordsToDataFrame', () => {
const theme = createTheme();
it('should convert instance history records into a data frame', () => {
const instanceLabels = { foo: 'bar', severity: 'critical', cluster: 'dev-us' };
const records: LogRecord[] = [
{
timestamp: 1000000,
line: { previous: 'Normal', current: 'Alerting', labels: instanceLabels },
},
];
const frame = logRecordsToDataFrame(JSON.stringify(instanceLabels), records, [], theme);
expect(frame.fields).toHaveLength(2);
const timeField = frame.fields[0];
const stateChangeField = frame.fields[1];
expect(timeField.name).toBe('time');
expect(timeField.type).toBe(FieldType.time);
expect(stateChangeField.name).toBe('state');
expect(stateChangeField.type).toBe(FieldType.string);
// There should be an artificial element at the end meaning Date.now()
// It exist to draw the state change from when it happened to the current time
expect(timeField.values).toHaveLength(2);
expect(timeField.values[0]).toBe(1000000);
expect(stateChangeField.values).toHaveLength(2);
expect(stateChangeField.values).toEqual(['Alerting', 'Alerting']);
});
it('should configure value to color mappings', () => {
const instanceLabels = { foo: 'bar', severity: 'critical', cluster: 'dev-us' };
const records: LogRecord[] = [
{
timestamp: 1000000,
line: { previous: 'Normal', current: 'Alerting', labels: instanceLabels },
},
];
const frame = logRecordsToDataFrame(JSON.stringify(instanceLabels), records, [], theme);
const stateField = frame.fields[1];
expect(stateField.config.mappings).toHaveLength(1);
expect(stateField.config.mappings![0].options).toMatchObject({
Alerting: {
color: theme.colors.error.main,
},
Pending: {
color: theme.colors.warning.main,
},
Normal: {
color: theme.colors.success.main,
},
NoData: {
color: theme.colors.info.main,
},
});
});
it('should return correct data frame summary', () => {
const instanceLabels = { foo: 'bar', severity: 'critical', cluster: 'dev-us' };
const records: LogRecord[] = [
{
timestamp: 1000000,
line: { previous: 'Normal', current: 'Alerting', labels: instanceLabels },
},
];
const frame = logRecordsToDataFrame(JSON.stringify(instanceLabels), records, [], theme);
expect(frame.fields).toHaveLength(2);
expect(frame).toHaveLength(2);
});
it('should have only unique labels in display name', () => {
const instanceLabels = { foo: 'bar', severity: 'critical', cluster: 'dev-us' };
const records: LogRecord[] = [
{
timestamp: 1000000,
line: { previous: 'Normal', current: 'Alerting', labels: instanceLabels },
},
];
const frame = logRecordsToDataFrame(
JSON.stringify(instanceLabels),
records,
[
['foo', 'bar'],
['cluster', 'dev-us'],
],
theme
);
expect(frame.fields[1].config.displayName).toBe('severity=critical');
});
});

View File

@ -0,0 +1,155 @@
import { groupBy } from 'lodash';
import { useMemo } from 'react';
import {
DataFrame,
DataFrameJSON,
Field as DataFrameField,
FieldType,
getDisplayProcessor,
GrafanaTheme2,
} from '@grafana/data';
import { fieldIndexComparer } from '@grafana/data/src/field/fieldComparers';
import { MappingType, ThresholdsMode } from '@grafana/schema';
import { useTheme2 } from '@grafana/ui';
import { labelsMatchMatchers, parseMatchers } from '../../../utils/alertmanager';
import { extractCommonLabels, Line, LogRecord, omitLabels } from './common';
export function useRuleHistoryRecords(stateHistory?: DataFrameJSON, filter?: string) {
const theme = useTheme2();
return useMemo(() => {
// merge timestamp with "line"
const tsValues = stateHistory?.data?.values[0] ?? [];
const timestamps: number[] = isNumbers(tsValues) ? tsValues : [];
const lines = stateHistory?.data?.values[1] ?? [];
const logRecords = timestamps.reduce((acc: LogRecord[], timestamp: number, index: number) => {
const line = lines[index];
// values property can be undefined for some instance states (e.g. NoData)
if (isLine(line)) {
acc.push({ timestamp, line });
}
return acc;
}, []);
// group all records by alert instance (unique set of labels)
const logRecordsByInstance = groupBy(logRecords, (record: LogRecord) => {
return JSON.stringify(record.line.labels);
});
// CommonLabels should not be affected by the filter
// find common labels so we can extract those from the instances
const groupLabels = Object.keys(logRecordsByInstance);
const groupLabelsArray: Array<Array<[string, string]>> = groupLabels.map((label) => {
return Object.entries(JSON.parse(label));
});
const commonLabels = extractCommonLabels(groupLabelsArray);
const filterMatchers = filter ? parseMatchers(filter) : [];
const filteredGroupedLines = Object.entries(logRecordsByInstance).filter(([key]) => {
const labels = JSON.parse(key);
return labelsMatchMatchers(labels, filterMatchers);
});
const dataFrames: DataFrame[] = filteredGroupedLines.map<DataFrame>(([key, records]) => {
return logRecordsToDataFrame(key, records, commonLabels, theme);
});
return {
historyRecords: logRecords.filter(({ line }) => line.labels && labelsMatchMatchers(line.labels, filterMatchers)),
dataFrames,
commonLabels,
totalRecordsCount: logRecords.length,
};
}, [stateHistory, filter, theme]);
}
export function isNumbers(value: unknown[]): value is number[] {
return value.every((v) => typeof v === 'number');
}
export function isLine(value: unknown): value is Line {
return typeof value === 'object' && value !== null && 'current' in value && 'previous' in value;
}
// Each alert instance is represented by a data frame
// Each frame consists of two fields: timestamp and state change
export function logRecordsToDataFrame(
instanceLabels: string,
records: LogRecord[],
commonLabels: Array<[string, string]>,
theme: GrafanaTheme2
): DataFrame {
const parsedInstanceLabels = Object.entries<string>(JSON.parse(instanceLabels));
// There is an artificial element at the end meaning Date.now()
// It exist to draw the state change from when it happened to the current time
const timeField: DataFrameField = {
name: 'time',
type: FieldType.time,
values: [...records.map((record) => record.timestamp), Date.now()],
config: { displayName: 'Time', custom: { fillOpacity: 100 } },
};
const timeIndex = timeField.values.map((_, index) => index);
timeIndex.sort(fieldIndexComparer(timeField));
const stateValues = [...records.map((record) => record.line.current), records.at(-1)?.line.current];
const frame: DataFrame = {
fields: [
{
...timeField,
values: timeField.values.map((_, i) => timeField.values[timeIndex[i]]),
},
{
name: 'state',
type: FieldType.string,
values: stateValues.map((_, i) => stateValues[timeIndex[i]]),
config: {
displayName: omitLabels(parsedInstanceLabels, commonLabels)
.map(([key, label]) => `${key}=${label}`)
.join(', '),
color: { mode: 'thresholds' },
custom: { fillOpacity: 100 },
mappings: [
{
type: MappingType.ValueToText,
options: {
Alerting: {
color: theme.colors.error.main,
},
Pending: {
color: theme.colors.warning.main,
},
Normal: {
color: theme.colors.success.main,
},
NoData: {
color: theme.colors.info.main,
},
},
},
],
thresholds: {
mode: ThresholdsMode.Absolute,
steps: [],
},
},
},
],
length: timeField.values.length,
name: instanceLabels,
};
frame.fields.forEach((field) => {
field.display = getDisplayProcessor({ field, theme });
});
return frame;
}

View File

@ -1,32 +1,83 @@
import React, { useMemo, useState } from 'react';
import { css } from '@emotion/css';
import React, { lazy, Suspense, useCallback, useMemo, useState } from 'react';
import { Modal } from '@grafana/ui';
import { GrafanaTheme2 } from '@grafana/data';
import { config } from '@grafana/runtime';
import { Modal, useStyles2 } from '@grafana/ui';
import { RulerGrafanaRuleDTO } from 'app/types/unified-alerting-dto';
import { StateHistory } from '../components/rules/StateHistory';
const AnnotationsStateHistory = lazy(() => import('../components/rules/state-history/StateHistory'));
const LokiStateHistory = lazy(() => import('../components/rules/state-history/LokiStateHistory'));
function useStateHistoryModal(alertId: string) {
enum StateHistoryImplementation {
Loki = 'loki',
Annotations = 'annotations',
}
function useStateHistoryModal() {
const [showModal, setShowModal] = useState<boolean>(false);
const [rule, setRule] = useState<RulerGrafanaRuleDTO | undefined>();
const StateHistoryModal = useMemo(
() => (
const styles = useStyles2(getStyles);
const implementation =
config.unifiedAlerting.alertStateHistoryBackend === StateHistoryImplementation.Loki
? StateHistoryImplementation.Loki
: StateHistoryImplementation.Annotations;
const dismissModal = useCallback(() => {
setRule(undefined);
setShowModal(false);
}, []);
const openModal = useCallback((rule: RulerGrafanaRuleDTO) => {
setRule(rule);
setShowModal(true);
}, []);
const StateHistoryModal = useMemo(() => {
if (!rule) {
return null;
}
return (
<Modal
isOpen={showModal}
onDismiss={() => setShowModal(false)}
onDismiss={dismissModal}
closeOnBackdropClick={true}
closeOnEscape={true}
title="State history"
className={styles.modal}
contentClassName={styles.modalContent}
>
<StateHistory alertId={alertId} />
<Suspense fallback={'Loading...'}>
{implementation === StateHistoryImplementation.Loki && <LokiStateHistory ruleUID={rule.grafana_alert.uid} />}
{implementation === StateHistoryImplementation.Annotations && (
<AnnotationsStateHistory alertId={rule.grafana_alert.id ?? ''} />
)}
</Suspense>
</Modal>
),
[alertId, showModal]
);
);
}, [rule, showModal, dismissModal, implementation, styles]);
return {
StateHistoryModal,
showStateHistoryModal: () => setShowModal(true),
hideStateHistoryModal: () => setShowModal(false),
showStateHistoryModal: openModal,
hideStateHistoryModal: dismissModal,
};
}
const getStyles = (theme: GrafanaTheme2) => ({
modal: css`
width: 80%;
height: 80%;
min-width: 800px;
`,
modalContent: css`
height: 100%;
width: 100%;
padding: ${theme.spacing(2)};
`,
});
export { useStateHistoryModal };

View File

@ -1,3 +1,5 @@
import { isEqual, uniqWith } from 'lodash';
import { SelectableValue } from '@grafana/data';
import {
AlertManagerCortexConfig,
@ -173,7 +175,7 @@ export function parseMatchers(matcherQueryString: string): Matcher[] {
const isRegex = operator === MatcherOperator.regex || operator === MatcherOperator.notRegex;
matchers.push({
name: key,
value: value.trim(),
value: isRegex ? getValidRegexString(value.trim()) : value.trim(),
isEqual,
isRegex,
});
@ -183,6 +185,16 @@ export function parseMatchers(matcherQueryString: string): Matcher[] {
return matchers;
}
function getValidRegexString(regex: string): string {
// Regexes provided by users might be invalid, so we need to catch the error
try {
new RegExp(regex);
return regex;
} catch (error) {
return '';
}
}
export function labelsMatchMatchers(labels: Labels, matchers: Matcher[]): boolean {
return matchers.every(({ name, value, isRegex, isEqual }) => {
return Object.entries(labels).some(([labelKey, labelValue]) => {
@ -206,6 +218,12 @@ export function labelsMatchMatchers(labels: Labels, matchers: Matcher[]): boolea
});
}
export function combineMatcherStrings(...matcherStrings: string[]): string {
const matchers = matcherStrings.map(parseMatchers).flat();
const uniqueMatchers = uniqWith(matchers, isEqual);
return matchersToString(uniqueMatchers);
}
export function getAllAlertmanagerDataSources() {
return getAllDataSources().filter((ds) => ds.type === DataSourceType.Alertmanager);
}