Transformation: Added variance and standard deviation (#48844)

* Transformation: Added variance and standard deviation for sample and population. Modified mean calculation approach

* Transformation: Removed existing mean calculation

* Transformation: Added testcases for variance and Standard deviation

* Update docs/sources/panels/calculation-types.md

Co-authored-by: brendamuir <100768211+brendamuir@users.noreply.github.com>

Co-authored-by: brendamuir <100768211+brendamuir@users.noreply.github.com>

docs/sources/panels/calculation-types.md

@@ -6,23 +6,27 @@ weight = 1100
 
 # Reference: Calculation types
 
-Refer to the following list of calculations you can perform in Grafana. You can find these calculations in the **Transform** tab and in the bar gauge, gauge, and stat visualizations.
+You can perform the following calculations in Grafana. Navigate to the **Transform** tab and in the bar gauge, gauge, and stat visualizations.
 
-| Calculation        | Description                                               |
+| Calculation                     | Description                                                       |
-| :----------------- | :-------------------------------------------------------- |
+| :------------------------------ | :---------------------------------------------------------------- |
-| All nulls          | True when all values are null                             |
+| All nulls                       | True when all values are null                                     |
-| All zeros          | True when all values are 0                                |
+| All zeros                       | True when all values are 0                                        |
-| Change count       | Number of times the field's value changes                 |
+| Change count                    | Number of times the field's value changes                         |
-| Count              | Number of values in a field                               |
+| Count                           | Number of values in a field                                       |
-| Delta              | Cumulative change in value, only counts increments        |
+| Delta                           | Cumulative change in value, only counts increments                |
-| Difference         | Difference between first and last value of a field        |
+| Difference                      | Difference between first and last value of a field                |
-| Difference percent | Percentage change between first and last value of a field |
+| Difference percent              | Percentage change between first and last value of a field         |
-| Distinct count     | Number of unique values in a field                        |
+| Distinct count                  | Number of unique values in a field                                |
-| First (not null)   | First, not null value in a field                          |
+| First (not null)                | First, not null value in a field                                  |
-| Max                | Maximum value of a field                                  |
+| Max                             | Maximum value of a field                                          |
-| Mean               | Mean value of all values in a field                       |
+| Mean                            | Mean value of all values in a field                               |
-| Min                | Minimum value of a field                                  |
+| Variance (Population)           | Variance (based on population) of all values in a field           |
-| Min (above zero)   | Minimum, positive value of a field                        |
+| Standard deviation (Population) | Standard deviation (based on population) of all values in a field |
-| Range              | Difference between maximum and minimum values of a field  |
+| Variance (Sample)               | Variance (based on sample) of all values in a field               |
-| Step               | Minimal interval between values of a field                |
+| Standard deviation (Sample)     | Standard deviation (based on sample) of all values in a field     |
-| Total              | Sum of all values in a field                              |
+| Min                             | Minimum value of a field                                          |
+| Min (above zero)                | Minimum, positive value of a field                                |
+| Range                           | Difference between maximum and minimum values of a field          |
+| Step                            | Minimal interval between values of a field                        |
+| Total                           | Sum of all values in a field                                      |

packages/grafana-data/src/transformations/fieldReducer.test.ts

@@ -56,13 +56,26 @@ describe('Stats Calculators', () => {
   it('should calculate basic stats', () => {
     const stats = reduceField({
       field: basicTable.fields[0],
-      reducers: ['first', 'last', 'mean', 'count'],
+      reducers: [
+        'first',
+        'last',
+        'mean',
+        'count',
+        'Variance (Population)',
+        'Variance (Sample)',
+        'Standard deviation (Population)',
+        'Standard deviation (Sample)',
+      ],
     });
 
     expect(stats.first).toEqual(10);
     expect(stats.last).toEqual(20);
     expect(stats.mean).toEqual(15);
     expect(stats.count).toEqual(2);
+    expect(stats.variancePopulation).toEqual(25);
+    expect(stats.varianceSample).toEqual(50);
+    expect(stats.stddevPopulation).toEqual(5);
+    expect(stats.stddevSample).toBeCloseTo(7.0710678, 5);
   });
 
   it('should support a single stat also', () => {

packages/grafana-data/src/transformations/fieldReducer.ts

@@ -10,6 +10,10 @@ export enum ReducerID {
   min = 'min',
   logmin = 'logmin',
   mean = 'mean',
+  variancePopulation = 'variancePopulation',
+  stddevPopulation = 'stddevPopulation',
+  varianceSample = 'varianceSample',
+  stddevSample = 'stddevSample',
   last = 'last',
   first = 'first',
   count = 'count',
@@ -152,6 +156,30 @@ export const fieldReducers = new Registry<FieldReducerInfo>(() => [
   { id: ReducerID.min, name: 'Min', description: 'Minimum Value', standard: true },
   { id: ReducerID.max, name: 'Max', description: 'Maximum Value', standard: true },
   { id: ReducerID.mean, name: 'Mean', description: 'Average Value', standard: true, aliasIds: ['avg'] },
+  {
+    id: ReducerID.variancePopulation,
+    name: 'Variance (Population)',
+    description: 'Variance (based on population) of all values in a field',
+    standard: true,
+  },
+  {
+    id: ReducerID.stddevPopulation,
+    name: 'Standard deviation (Population)',
+    description: 'Standard deviation (based on population) of all values in a field',
+    standard: true,
+  },
+  {
+    id: ReducerID.varianceSample,
+    name: 'Variance (Sample)',
+    description: 'Variance (based on sample) of all values in a field',
+    standard: true,
+  },
+  {
+    id: ReducerID.stddevSample,
+    name: 'Standard deviation (Sample)',
+    description: 'Standard deviation (based on sample) of all values in a field',
+    standard: true,
+  },
   {
     id: ReducerID.sum,
     name: 'Total',
@@ -256,6 +284,10 @@ export function doStandardCalcs(field: Field, ignoreNulls: boolean, nullAsZero:
     min: Number.MAX_VALUE,
     logmin: Number.MAX_VALUE,
     mean: null,
+    variancePopulation: null,
+    stddevPopulation: null,
+    varianceSample: null,
+    stddevSample: null,
     last: null,
     first: null,
     lastNotNull: null,
@@ -274,6 +306,8 @@ export function doStandardCalcs(field: Field, ignoreNulls: boolean, nullAsZero:
     previousDeltaUp: true,
   } as FieldCalcs;
 
+  let squareSum = 0;
+
   const data = field.values;
   calcs.count = data.length;
 
@@ -343,6 +377,10 @@ export function doStandardCalcs(field: Field, ignoreNulls: boolean, nullAsZero:
         if (currentValue < calcs.logmin && currentValue > 0) {
           calcs.logmin = currentValue;
         }
+
+        let _oldMean = calcs.mean;
+        calcs.mean += (currentValue - _oldMean) / calcs.nonNullCount;
+        squareSum += (currentValue - _oldMean) * (currentValue - calcs.mean);
       }
 
       if (currentValue !== 0) {
@@ -366,7 +404,19 @@ export function doStandardCalcs(field: Field, ignoreNulls: boolean, nullAsZero:
   }
 
   if (calcs.nonNullCount > 0) {
-    calcs.mean = calcs.sum! / calcs.nonNullCount;
+    calcs.variancePopulation = squareSum / calcs.nonNullCount;
+  }
+
+  if (calcs.nonNullCount > 0) {
+    calcs.stddevPopulation = Math.sqrt(calcs.variancePopulation);
+  }
+
+  if (calcs.nonNullCount > 0) {
+    calcs.varianceSample = squareSum / (calcs.nonNullCount - 1);
+  }
+
+  if (calcs.nonNullCount > 0) {
+    calcs.stddevSample = Math.sqrt(calcs.varianceSample);
   }
 
   if (calcs.allIsNull) {