Transformation: added support for excluding/including rows based on their values. (#26884)

* Adding FilterByValue transformer skeleton

* Connecting options with Editor

* Improving UI and making deep copy of options on change.

* Improving Transformation Editor UI

* Implementing Regex filtering

* Adding valueFilters.ts and creating filter registry

* Connecting the test function

* Correcting TypeScript errors

* Using FilterInstance instead of simple Filter test function

* Adding field.type as filter options

* Improving UI. Adding custom placeholder depending on filter.

* Implementing a few more filter types

* Implementing more filters

* Return original data if no filter were processed

* Improving UI

* Correcting TS errors

* Making sure inequality transform are invalid until the filterExpression is not empty

* Cleanup in the UI file

* Improving UI (highlight invalid fields)

* Only show filterType that are supported for the selected field

* Adding tests + correction of a filter

* Adding transformer test

* Adding doc

* Cleanup

* Typing props for FilterSelectorRow component

Co-authored-by: Marcus Andersson <systemvetaren@gmail.com>

* Moving rendering in the JSX

Co-authored-by: Marcus Andersson <systemvetaren@gmail.com>

* Memoizing filterTypeOptions computation

Co-authored-by: Marcus Andersson <systemvetaren@gmail.com>

* Improve code compactness

Co-authored-by: Marcus Andersson <systemvetaren@gmail.com>

* Cleanup + solving TS errors

* Updating some labels

* Wrapping stuff around useMemo and useCallback

* Using cloneDeep from lodash

* Don't highlight field name input if null

* Removing time type fields in selectable options

* We want loose equality in this scenario.

* Adding `onChange` to useCallback dependencies

Co-authored-by: Marcus Andersson <systemvetaren@gmail.com>

* Include or exclude matching any or all conditions

* Correcting field name matching

* Cleanup

* Don't highlight the filterExpression input when its empty

* Adding Range filter

* Updating doc

* Correcting TS error

* Only showing the Match All/Match Any option if more than one condition

* Two inputs for the Range filter instead of one

* Improving invalid highlight for Range filter type

* Cleanup

* Improving labels in UI

* Using ButtonSelect to improve UI

* editor UI updates.

* Updating tests

* Adding component for Regex

* Improve TS typing

* Adding components for the other filter types.

* Cleanup

* Correct error

* Updating valueFilter.test.ts

* Updating filterByValue.test.ts

* Reverting and removing Range filter

* Update docs/sources/panels/transformations.md

* starting to implement poc.

* added a small poc.

* wip

* added tests.

* added structure for dynamic value matcher editors.

* added more support.

* added some more value matchers.

* removed unused value filters.

* added some more matchers.

* adding more matchers.

* added a range matcher.

* fixing some tests.

* fixing tests.

* remove unused dep.

* making the matching a bit more performant.

* UX improvements and alignment fixes

* fixed delete button.

* fixed some spacing in the UI.

* added docs for matchers.

* adding docs and exposing value matcher types.

* will store dateTime as string.

* updated docs according to feedback.

* moved filter by value in transformation list.

* Improved description.

* added regex value filter.

* added support for regex.

* fixing failing tests.

Co-authored-by: Marcus Andersson <systemvetaren@gmail.com>
Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com>
Co-authored-by: Torkel Ödegaard <torkel@grafana.com>

public/app/core/components/TransformersUI/FilterByValueTransformer/FilterByValueFilterEditor.tsx

@@ -0,0 +1,171 @@
+import React, { useCallback } from 'react';
+import { Button, Select } from '@grafana/ui';
+import { Field, SelectableValue, valueMatchers } from '@grafana/data';
+import { FilterByValueFilter } from '@grafana/data/src/transformations/transformers/filterByValue';
+import { valueMatchersUI } from './ValueMatchers/valueMatchersUI';
+
+interface Props {
+  onDelete: () => void;
+  onChange: (filter: FilterByValueFilter) => void;
+  filter: FilterByValueFilter;
+  fieldsInfo: DataFrameFieldsInfo;
+}
+
+export interface DataFrameFieldsInfo {
+  fieldsAsOptions: Array<SelectableValue<string>>;
+  fieldByDisplayName: Record<string, Field>;
+}
+
+export const FilterByValueFilterEditor: React.FC<Props> = props => {
+  const { onDelete, onChange, filter, fieldsInfo } = props;
+  const { fieldsAsOptions, fieldByDisplayName } = fieldsInfo;
+  const fieldName = getFieldName(filter, fieldsAsOptions) ?? '';
+  const field = fieldByDisplayName[fieldName];
+
+  if (!field) {
+    return null;
+  }
+
+  const matcherOptions = getMatcherOptions(field);
+  const matcherId = getSelectedMatcherId(filter, matcherOptions);
+  const editor = valueMatchersUI.getIfExists(matcherId);
+
+  if (!editor || !editor.component) {
+    return null;
+  }
+
+  const onChangeField = useCallback(
+    (selectable?: SelectableValue<string>) => {
+      if (!selectable?.value) {
+        return;
+      }
+      onChange({
+        ...filter,
+        fieldName: selectable.value,
+      });
+    },
+    [onChange, filter]
+  );
+
+  const onChangeMatcher = useCallback(
+    (selectable?: SelectableValue<string>) => {
+      if (!selectable?.value) {
+        return;
+      }
+
+      const id = selectable.value;
+      const options = valueMatchers.get(id).getDefaultOptions(field);
+
+      onChange({
+        ...filter,
+        config: { id, options },
+      });
+    },
+    [onChange, filter, field]
+  );
+
+  const onChangeMatcherOptions = useCallback(
+    options => {
+      onChange({
+        ...filter,
+        config: {
+          ...filter.config,
+          options,
+        },
+      });
+    },
+    [onChange, filter]
+  );
+
+  return (
+    <div className="gf-form-inline">
+      <div className="gf-form gf-form-spacing">
+        <div className="gf-form-label width-7">Field</div>
+        <Select
+          className="min-width-15 max-width-24"
+          placeholder="Field Name"
+          options={fieldsAsOptions}
+          value={filter.fieldName}
+          onChange={onChangeField}
+          menuPlacement="bottom"
+        />
+      </div>
+      <div className="gf-form gf-form-spacing">
+        <div className="gf-form-label">Match</div>
+        <Select
+          className="width-12"
+          placeholder="Select test"
+          options={matcherOptions}
+          value={matcherId}
+          onChange={onChangeMatcher}
+          menuPlacement="bottom"
+        />
+      </div>
+      <div className="gf-form gf-form--grow gf-form-spacing">
+        <div className="gf-form-label">Value</div>
+        <editor.component field={field} options={filter.config.options ?? {}} onChange={onChangeMatcherOptions} />
+      </div>
+      <div className="gf-form">
+        <Button icon="times" onClick={onDelete} variant="secondary" />
+      </div>
+    </div>
+  );
+};
+
+const getMatcherOptions = (field: Field): Array<SelectableValue<string>> => {
+  const options = [];
+
+  for (const matcher of valueMatchers.list()) {
+    if (!matcher.isApplicable(field)) {
+      continue;
+    }
+
+    const editor = valueMatchersUI.getIfExists(matcher.id);
+
+    if (!editor) {
+      continue;
+    }
+
+    options.push({
+      value: matcher.id,
+      label: matcher.name,
+      description: matcher.description,
+    });
+  }
+
+  return options;
+};
+
+const getSelectedMatcherId = (
+  filter: FilterByValueFilter,
+  matcherOptions: Array<SelectableValue<string>>
+): string | undefined => {
+  const matcher = matcherOptions.find(m => m.value === filter.config.id);
+
+  if (matcher && matcher.value) {
+    return matcher.value;
+  }
+
+  if (matcherOptions[0]?.value) {
+    return matcherOptions[0]?.value;
+  }
+
+  return;
+};
+
+const getFieldName = (
+  filter: FilterByValueFilter,
+  fieldOptions: Array<SelectableValue<string>>
+): string | undefined => {
+  const fieldName = fieldOptions.find(m => m.value === filter.fieldName);
+
+  if (fieldName && fieldName.value) {
+    return fieldName.value;
+  }
+
+  if (fieldOptions[0]?.value) {
+    return fieldOptions[0]?.value;
+  }
+
+  return;
+};

public/app/core/components/TransformersUI/FilterByValueTransformer/FilterByValueTransformerEditor.tsx

@@ -0,0 +1,180 @@
+import React, { useMemo, useCallback } from 'react';
+import { css } from 'emotion';
+import {
+  DataTransformerID,
+  standardTransformers,
+  TransformerRegistyItem,
+  TransformerUIProps,
+  getFieldDisplayName,
+  DataFrame,
+  SelectableValue,
+  FieldType,
+  ValueMatcherID,
+  valueMatchers,
+} from '@grafana/data';
+import { Button, RadioButtonGroup, stylesFactory } from '@grafana/ui';
+import cloneDeep from 'lodash/cloneDeep';
+import {
+  FilterByValueFilter,
+  FilterByValueMatch,
+  FilterByValueTransformerOptions,
+  FilterByValueType,
+} from '@grafana/data/src/transformations/transformers/filterByValue';
+
+import { DataFrameFieldsInfo, FilterByValueFilterEditor } from './FilterByValueFilterEditor';
+
+const filterTypes: Array<SelectableValue<FilterByValueType>> = [
+  { label: 'Include', value: FilterByValueType.include },
+  { label: 'Exclude', value: FilterByValueType.exclude },
+];
+
+const filterMatch: Array<SelectableValue<FilterByValueMatch>> = [
+  { label: 'Match all', value: FilterByValueMatch.all },
+  { label: 'Match any', value: FilterByValueMatch.any },
+];
+
+export const FilterByValueTransformerEditor: React.FC<TransformerUIProps<FilterByValueTransformerOptions>> = props => {
+  const { input, options, onChange } = props;
+  const styles = getEditorStyles();
+  const fieldsInfo = useFieldsInfo(input);
+
+  const onAddFilter = useCallback(() => {
+    const frame = input[0];
+    const field = frame.fields.find(f => f.type !== FieldType.time);
+
+    if (!field) {
+      return;
+    }
+
+    const filters = cloneDeep(options.filters);
+    const matcher = valueMatchers.get(ValueMatcherID.greater);
+
+    filters.push({
+      fieldName: getFieldDisplayName(field, frame, input),
+      config: {
+        id: matcher.id,
+        options: matcher.getDefaultOptions(field),
+      },
+    });
+    onChange({ ...options, filters });
+  }, [onChange, options, valueMatchers, input]);
+
+  const onDeleteFilter = useCallback(
+    (index: number) => {
+      let filters = cloneDeep(options.filters);
+      filters.splice(index, 1);
+      onChange({ ...options, filters });
+    },
+    [options, onChange]
+  );
+
+  const onChangeFilter = useCallback(
+    (filter: FilterByValueFilter, index: number) => {
+      let filters = cloneDeep(options.filters);
+      filters[index] = filter;
+      onChange({ ...options, filters });
+    },
+    [options, onChange]
+  );
+
+  const onChangeType = useCallback(
+    (type?: FilterByValueType) => {
+      onChange({
+        ...options,
+        type: type ?? FilterByValueType.include,
+      });
+    },
+    [options, onChange]
+  );
+
+  const onChangeMatch = useCallback(
+    (match?: FilterByValueMatch) => {
+      onChange({
+        ...options,
+        match: match ?? FilterByValueMatch.all,
+      });
+    },
+    [options, onChange]
+  );
+
+  return (
+    <div>
+      <div className="gf-form gf-form-inline">
+        <div className="gf-form-label width-8">Filter type</div>
+        <div className="width-15">
+          <RadioButtonGroup options={filterTypes} value={options.type} onChange={onChangeType} fullWidth />
+        </div>
+      </div>
+      <div className="gf-form gf-form-inline">
+        <div className="gf-form-label width-8">Conditions</div>
+        <div className="width-15">
+          <RadioButtonGroup options={filterMatch} value={options.match} onChange={onChangeMatch} fullWidth />
+        </div>
+      </div>
+      <div className={styles.conditions}>
+        {options.filters.map((filter, idx) => (
+          <FilterByValueFilterEditor
+            key={idx}
+            filter={filter}
+            fieldsInfo={fieldsInfo}
+            onChange={filter => onChangeFilter(filter, idx)}
+            onDelete={() => onDeleteFilter(idx)}
+          />
+        ))}
+        <div className="gf-form">
+          <Button icon="plus" size="sm" onClick={onAddFilter} variant="secondary">
+            Add condition
+          </Button>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export const filterByValueTransformRegistryItem: TransformerRegistyItem<FilterByValueTransformerOptions> = {
+  id: DataTransformerID.filterByValue,
+  editor: FilterByValueTransformerEditor,
+  transformation: standardTransformers.filterByValueTransformer,
+  name: standardTransformers.filterByValueTransformer.name,
+  description:
+    'Removes rows of the query results using user definied filters. This is useful if you can not filter your data in the data source.',
+};
+
+const getEditorStyles = stylesFactory(() => ({
+  conditions: css`
+    padding-left: 16px;
+  `,
+}));
+
+const useFieldsInfo = (data: DataFrame[]): DataFrameFieldsInfo => {
+  return useMemo(() => {
+    const meta = {
+      fieldsAsOptions: [],
+      fieldByDisplayName: {},
+    };
+
+    if (!Array.isArray(data)) {
+      return meta;
+    }
+
+    return data.reduce((meta: DataFrameFieldsInfo, frame) => {
+      return frame.fields.reduce((meta, field) => {
+        const fieldName = getFieldDisplayName(field, frame, data);
+
+        if (meta.fieldByDisplayName[fieldName]) {
+          return meta;
+        }
+
+        meta.fieldsAsOptions.push({
+          label: fieldName,
+          value: fieldName,
+          type: field.type,
+        });
+
+        meta.fieldByDisplayName[fieldName] = field;
+
+        return meta;
+      }, meta);
+    }, meta);
+  }, [data]);
+};

public/app/core/components/TransformersUI/FilterByValueTransformer/ValueMatchers/BasicMatcherEditor.tsx

@@ -0,0 +1,104 @@
+import React, { useCallback, useState } from 'react';
+import { Input } from '@grafana/ui';
+import { ValueMatcherID, BasicValueMatcherOptions } from '@grafana/data';
+import { ValueMatcherEditorConfig, ValueMatcherUIProps, ValueMatcherUIRegistryItem } from './types';
+import { convertToType } from './utils';
+
+export function basicMatcherEditor<T = any>(
+  config: ValueMatcherEditorConfig
+): React.FC<ValueMatcherUIProps<BasicValueMatcherOptions<T>>> {
+  return ({ options, onChange, field }) => {
+    const { validator, converter = convertToType } = config;
+    const { value } = options;
+    const [isInvalid, setInvalid] = useState(!validator(value));
+
+    const onChangeValue = useCallback(
+      (event: React.FormEvent<HTMLInputElement>) => {
+        setInvalid(!validator(event.currentTarget.value));
+      },
+      [setInvalid, validator]
+    );
+
+    const onChangeOptions = useCallback(
+      (event: React.FocusEvent<HTMLInputElement>) => {
+        if (isInvalid) {
+          return;
+        }
+
+        const { value } = event.currentTarget;
+
+        onChange({
+          ...options,
+          value: converter(value, field),
+        });
+      },
+      [options, onChange, isInvalid, field, converter]
+    );
+
+    return (
+      <Input
+        className="flex-grow-1"
+        invalid={isInvalid}
+        defaultValue={String(options.value)}
+        placeholder="Value"
+        onChange={onChangeValue}
+        onBlur={onChangeOptions}
+      />
+    );
+  };
+}
+
+export const getBasicValueMatchersUI = (): Array<ValueMatcherUIRegistryItem<BasicValueMatcherOptions>> => {
+  return [
+    {
+      name: 'Is greater',
+      id: ValueMatcherID.greater,
+      component: basicMatcherEditor<number>({
+        validator: value => !isNaN(value),
+      }),
+    },
+    {
+      name: 'Is greater or equal',
+      id: ValueMatcherID.greaterOrEqual,
+      component: basicMatcherEditor<number>({
+        validator: value => !isNaN(value),
+      }),
+    },
+    {
+      name: 'Is lower',
+      id: ValueMatcherID.lower,
+      component: basicMatcherEditor<number>({
+        validator: value => !isNaN(value),
+      }),
+    },
+    {
+      name: 'Is lower or equal',
+      id: ValueMatcherID.lowerOrEqual,
+      component: basicMatcherEditor<number>({
+        validator: value => !isNaN(value),
+      }),
+    },
+    {
+      name: 'Is equal',
+      id: ValueMatcherID.equal,
+      component: basicMatcherEditor<any>({
+        validator: () => true,
+      }),
+    },
+    {
+      name: 'Is not equal',
+      id: ValueMatcherID.notEqual,
+      component: basicMatcherEditor<any>({
+        validator: () => true,
+      }),
+    },
+    {
+      name: 'Regex',
+      id: ValueMatcherID.regex,
+      component: basicMatcherEditor<string>({
+        validator: () => true,
+        converter: (value: any) => String(value),
+      }),
+    },
+  ];
+};

public/app/core/components/TransformersUI/FilterByValueTransformer/ValueMatchers/NoopMatcherEditor.tsx

@@ -0,0 +1,22 @@
+import { ValueMatcherID } from '@grafana/data';
+import React from 'react';
+import { ValueMatcherUIProps, ValueMatcherUIRegistryItem } from './types';
+
+export const NoopMatcherEditor: React.FC<ValueMatcherUIProps<any>> = () => {
+  return null;
+};
+
+export const getNoopValueMatchersUI = (): Array<ValueMatcherUIRegistryItem<any>> => {
+  return [
+    {
+      name: 'Is null',
+      id: ValueMatcherID.isNull,
+      component: NoopMatcherEditor,
+    },
+    {
+      name: 'Is not null',
+      id: ValueMatcherID.isNotNull,
+      component: NoopMatcherEditor,
+    },
+  ];
+};

public/app/core/components/TransformersUI/FilterByValueTransformer/ValueMatchers/RangeMatcherEditor.tsx

@@ -0,0 +1,81 @@
+import React, { useCallback, useState } from 'react';
+import { Input } from '@grafana/ui';
+import { ValueMatcherID, RangeValueMatcherOptions } from '@grafana/data';
+import { ValueMatcherEditorConfig, ValueMatcherUIProps, ValueMatcherUIRegistryItem } from './types';
+import { convertToType } from './utils';
+
+type PropNames = 'from' | 'to';
+
+export function rangeMatcherEditor<T = any>(
+  config: ValueMatcherEditorConfig
+): React.FC<ValueMatcherUIProps<RangeValueMatcherOptions<T>>> {
+  return ({ options, onChange, field }) => {
+    const { validator } = config;
+    const [isInvalid, setInvalid] = useState({
+      from: !validator(options.from),
+      to: !validator(options.to),
+    });
+
+    const onChangeValue = useCallback(
+      (event: React.FormEvent<HTMLInputElement>, prop: PropNames) => {
+        setInvalid({
+          ...isInvalid,
+          [prop]: !validator(event.currentTarget.value),
+        });
+      },
+      [setInvalid, validator, isInvalid]
+    );
+
+    const onChangeOptions = useCallback(
+      (event: React.FocusEvent<HTMLInputElement>, prop: PropNames) => {
+        if (isInvalid[prop]) {
+          return;
+        }
+
+        const { value } = event.currentTarget;
+
+        onChange({
+          ...options,
+          [prop]: convertToType(value, field),
+        });
+      },
+      [options, onChange, isInvalid, field]
+    );
+
+    return (
+      <>
+        <Input
+          className="flex-grow-1 gf-form-spacing"
+          invalid={isInvalid['from']}
+          defaultValue={String(options.from)}
+          placeholder="From"
+          onChange={event => onChangeValue(event, 'from')}
+          onBlur={event => onChangeOptions(event, 'from')}
+        />
+        <div className="gf-form-label">and</div>
+        <Input
+          className="flex-grow-1"
+          invalid={isInvalid['to']}
+          defaultValue={String(options.to)}
+          placeholder="To"
+          onChange={event => onChangeValue(event, 'to')}
+          onBlur={event => onChangeOptions(event, 'to')}
+        />
+      </>
+    );
+  };
+}
+
+export const getRangeValueMatchersUI = (): Array<ValueMatcherUIRegistryItem<RangeValueMatcherOptions>> => {
+  return [
+    {
+      name: 'Is between',
+      id: ValueMatcherID.between,
+      component: rangeMatcherEditor<number>({
+        validator: value => {
+          return !isNaN(value);
+        },
+      }),
+    },
+  ];
+};

public/app/core/components/TransformersUI/FilterByValueTransformer/ValueMatchers/types.ts

@@ -0,0 +1,14 @@
+import { Field, RegistryItem } from '@grafana/data';
+export interface ValueMatcherUIRegistryItem<TOptions> extends RegistryItem {
+  component: React.ComponentType<ValueMatcherUIProps<TOptions>>;
+}
+
+export interface ValueMatcherUIProps<TOptions> {
+  options: TOptions;
+  onChange: (options: TOptions) => void;
+  field: Field;
+}
+export interface ValueMatcherEditorConfig {
+  validator: (value: any) => boolean;
+  converter?: (value: any, field: Field) => any;
+}

public/app/core/components/TransformersUI/FilterByValueTransformer/ValueMatchers/utils.ts

@@ -0,0 +1,34 @@
+import { Field, FieldType } from '@grafana/data';
+import { isString, isUndefined } from 'lodash';
+
+export function convertToType(value: any, field: Field): any {
+  switch (field.type) {
+    case FieldType.boolean:
+      if (isUndefined(value)) {
+        return false;
+      }
+      return convertToBool(value);
+
+    case FieldType.number:
+      if (isNaN(value)) {
+        return 0;
+      }
+      return parseFloat(value);
+
+    case FieldType.string:
+      if (!value) {
+        return '';
+      }
+      return String(value);
+
+    default:
+      return value;
+  }
+}
+
+const convertToBool = (value: any): boolean => {
+  if (isString(value)) {
+    return !(value[0] === 'F' || value[0] === 'f' || value[0] === '0');
+  }
+  return !!value;
+};

public/app/core/components/TransformersUI/FilterByValueTransformer/ValueMatchers/valueMatchersUI.ts

@@ -0,0 +1,9 @@
+import { Registry } from '@grafana/data';
+import { getBasicValueMatchersUI } from './BasicMatcherEditor';
+import { getNoopValueMatchersUI } from './NoopMatcherEditor';
+import { getRangeValueMatchersUI } from './RangeMatcherEditor';
+import { ValueMatcherUIRegistryItem } from './types';
+
+export const valueMatchersUI = new Registry<ValueMatcherUIRegistryItem<any>>(() => {
+  return [...getBasicValueMatchersUI(), ...getNoopValueMatchersUI(), ...getRangeValueMatchersUI()];
+});