From 0b02c84e33cedc70312906c838b91b1b19776593 Mon Sep 17 00:00:00 2001
From: Mathieu <70369997+MathieuRA@users.noreply.github.com>
Date: Mon, 15 Aug 2022 09:59:22 +0200
Subject: [PATCH] feat(lite): xapiStat with fetchStats composable (#6361)
---
@xen-orchestra/lite/.eslintrc.cjs | 6 +-
@xen-orchestra/lite/.prettierrc.cjs | 1 +
@xen-orchestra/lite/package.json | 8 +
.../src/composables/fetch-stats.composable.md | 23 +
.../src/composables/fetch-stats.composable.ts | 27 ++
@xen-orchestra/lite/src/libs/utils.ts | 11 +
@xen-orchestra/lite/src/libs/xapi-stats.ts | 450 ++++++++++++++++++
@xen-orchestra/lite/src/libs/xen-api.ts | 26 +
@xen-orchestra/lite/src/stores/host.store.ts | 28 +-
@xen-orchestra/lite/src/stores/vm.store.ts | 23 +
.../lite/src/stores/xen-api.store.ts | 11 +
.../src/types/decorator-synchronized.d.ts | 3 +
.../types/limit-concurrency-decorator.d.ts | 3 +
@xen-orchestra/lite/tsconfig.json | 1 +
yarn.lock | 63 ++-
15 files changed, 660 insertions(+), 24 deletions(-)
create mode 100644 @xen-orchestra/lite/src/composables/fetch-stats.composable.md
create mode 100644 @xen-orchestra/lite/src/composables/fetch-stats.composable.ts
create mode 100644 @xen-orchestra/lite/src/libs/xapi-stats.ts
create mode 100644 @xen-orchestra/lite/src/types/decorator-synchronized.d.ts
create mode 100644 @xen-orchestra/lite/src/types/limit-concurrency-decorator.d.ts
diff --git a/@xen-orchestra/lite/.eslintrc.cjs b/@xen-orchestra/lite/.eslintrc.cjs
index bda9cb6c5..7e6b870d9 100644
--- a/@xen-orchestra/lite/.eslintrc.cjs
+++ b/@xen-orchestra/lite/.eslintrc.cjs
@@ -12,15 +12,13 @@ module.exports = {
"@vue/eslint-config-typescript/recommended",
"@vue/eslint-config-prettier",
],
- plugins: [
- "@limegrass/import-alias"
- ],
+ plugins: ["@limegrass/import-alias"],
rules: {
"@typescript-eslint/no-non-null-assertion": "off",
"@typescript-eslint/no-explicit-any": "off",
"@limegrass/import-alias/import-alias": [
"error",
- { aliasConfigPath: require("path").join(__dirname, "tsconfig.json") }
+ { aliasConfigPath: require("path").join(__dirname, "tsconfig.json") },
],
},
};
diff --git a/@xen-orchestra/lite/.prettierrc.cjs b/@xen-orchestra/lite/.prettierrc.cjs
index bfa205e91..41d9f66db 100644
--- a/@xen-orchestra/lite/.prettierrc.cjs
+++ b/@xen-orchestra/lite/.prettierrc.cjs
@@ -11,4 +11,5 @@ module.exports = {
],
importOrderSeparation: false,
importOrderSortSpecifiers: true,
+ importOrderParserPlugins: ["typescript", "decorators-legacy"],
};
diff --git a/@xen-orchestra/lite/package.json b/@xen-orchestra/lite/package.json
index 10fdbf574..e22297d97 100644
--- a/@xen-orchestra/lite/package.json
+++ b/@xen-orchestra/lite/package.json
@@ -17,9 +17,17 @@
"@fortawesome/pro-thin-svg-icons": "^6.1.2",
"@fortawesome/vue-fontawesome": "^3.0.1",
"@novnc/novnc": "^1.3.0",
+ "@types/d3-time-format": "^4.0.0",
+ "@types/lodash-es": "^4.17.6",
"@vueuse/core": "^8.7.5",
"complex-matcher": "^0.7.0",
+ "d3-time-format": "^4.1.0",
+ "decorator-synchronized": "^0.6.0",
"json-rpc-2.0": "^1.3.0",
+ "json5": "^2.2.1",
+ "limit-concurrency-decorator": "^0.5.0",
+ "lodash-es": "^4.17.21",
+ "make-error": "^1.3.6",
"pinia": "^2.0.14",
"vue": "^3.2.37",
"vue-router": "^4.0.16"
diff --git a/@xen-orchestra/lite/src/composables/fetch-stats.composable.md b/@xen-orchestra/lite/src/composables/fetch-stats.composable.md
new file mode 100644
index 000000000..36d198fec
--- /dev/null
+++ b/@xen-orchestra/lite/src/composables/fetch-stats.composable.md
@@ -0,0 +1,23 @@
+# useFetchStats composable
+
+```vue
+
+```
diff --git a/@xen-orchestra/lite/src/composables/fetch-stats.composable.ts b/@xen-orchestra/lite/src/composables/fetch-stats.composable.ts
new file mode 100644
index 000000000..81c99b765
--- /dev/null
+++ b/@xen-orchestra/lite/src/composables/fetch-stats.composable.ts
@@ -0,0 +1,27 @@
+import { type Ref, ref } from "vue";
+import { promiseTimeout, useIntervalFn } from "@vueuse/core";
+import type { GRANULARITY, XapiStatsResponse } from "@/libs/xapi-stats";
+import { useHostStore } from "@/stores/host.store";
+import { useVmStore } from "@/stores/vm.store";
+
+const STORES_BY_OBJECT_TYPE = {
+ host: useHostStore,
+ vm: useVmStore,
+};
+
+export default function useFetchStats(
+ type: "host" | "vm",
+ id: string,
+ granularity: GRANULARITY
+) {
+ const stats = ref();
+ const fetch = STORES_BY_OBJECT_TYPE[type]().getStats;
+
+ const fetchStats = async () => {
+ stats.value = await fetch(id, granularity);
+ await promiseTimeout(stats.value.interval * 1000);
+ };
+ useIntervalFn(fetchStats);
+
+ return { stats } as { stats: Ref> | undefined };
+}
diff --git a/@xen-orchestra/lite/src/libs/utils.ts b/@xen-orchestra/lite/src/libs/utils.ts
index 95ff44258..cd9c88133 100644
--- a/@xen-orchestra/lite/src/libs/utils.ts
+++ b/@xen-orchestra/lite/src/libs/utils.ts
@@ -1,3 +1,4 @@
+import { utcParse } from "d3-time-format";
import type { Filter } from "@/types/filter";
import { faSquareCheck } from "@fortawesome/pro-regular-svg-icons";
import { faFont, faHashtag, faList } from "@fortawesome/pro-solid-svg-icons";
@@ -41,3 +42,13 @@ export function getFilterIcon(filter: Filter | undefined) {
return iconsByType[filter.type];
}
+
+export function parseDateTime(dateTime: string) {
+ const date = utcParse("%Y%m%dT%H:%M:%SZ")(dateTime);
+ if (date === null) {
+ throw new RangeError(
+ `unable to parse XAPI datetime ${JSON.stringify(dateTime)}`
+ );
+ }
+ return date.getTime();
+}
diff --git a/@xen-orchestra/lite/src/libs/xapi-stats.ts b/@xen-orchestra/lite/src/libs/xapi-stats.ts
new file mode 100644
index 000000000..1e3da37a3
--- /dev/null
+++ b/@xen-orchestra/lite/src/libs/xapi-stats.ts
@@ -0,0 +1,450 @@
+import { synchronized } from "decorator-synchronized";
+import JSON5 from "json5";
+import { limitConcurrency } from "limit-concurrency-decorator";
+import { defaults, findKey, forEach, identity, map } from "lodash-es";
+import { BaseError } from "make-error";
+import type XenApi from "@/libs/xen-api";
+import type { XenApiHost } from "@/libs/xen-api";
+
+class FaultyGranularity extends BaseError {}
+
+// -------------------------------------------------------------------
+
+// according to https://xapi-project.github.io/xen-api/metrics.html
+// The values are stored at intervals of:
+// - 5 seconds for the past 10 minutes
+// - one minute for the past 2 hours
+// - one hour for the past week
+// - one day for the past year
+enum RRD_STEP {
+ Seconds = 5,
+ Minutes = 60,
+ Hours = 3600,
+ Days = 86400,
+}
+
+export enum GRANULARITY {
+ Seconds = "seconds",
+ Minutes = "minutes",
+ Hours = "hours",
+ Days = "days",
+}
+
+const RRD_STEP_FROM_STRING: { [key in GRANULARITY]: RRD_STEP } = {
+ [GRANULARITY.Seconds]: RRD_STEP.Seconds,
+ [GRANULARITY.Minutes]: RRD_STEP.Minutes,
+ [GRANULARITY.Hours]: RRD_STEP.Hours,
+ [GRANULARITY.Days]: RRD_STEP.Days,
+};
+
+// points = intervalInSeconds / step
+const RRD_POINTS_PER_STEP: { [key in RRD_STEP]: number } = {
+ [RRD_STEP.Seconds]: 120,
+ [RRD_STEP.Minutes]: 120,
+ [RRD_STEP.Hours]: 168,
+ [RRD_STEP.Days]: 366,
+};
+
+// -------------------------------------------------------------------
+// Utils
+// -------------------------------------------------------------------
+
+function convertNanToNull(value: number) {
+ return isNaN(value) ? null : value;
+}
+
+// -------------------------------------------------------------------
+// Stats
+// -------------------------------------------------------------------
+
+const computeValues = (
+ dataRow: any,
+ legendIndex: number,
+ transformValue = identity
+) =>
+ map(dataRow, ({ values }) =>
+ transformValue(convertNanToNull(values[legendIndex]))
+ );
+
+const createGetProperty = (
+ obj: object,
+ property: string,
+ defaultValue: unknown
+) => defaults(obj, { [property]: defaultValue })[property] as any;
+
+const testMetric = (
+ test:
+ | string
+ | { exec: (type: string) => boolean }
+ | { (type: string): boolean },
+ type: string
+): boolean =>
+ typeof test === "string"
+ ? test === type
+ : typeof test === "function"
+ ? test(type)
+ : test.exec(type);
+
+const findMetric = (metrics: any, metricType: string) => {
+ let testResult;
+ let metric;
+
+ forEach(metrics, (current) => {
+ if (current.test === undefined) {
+ const newValues = findMetric(current, metricType);
+
+ metric = newValues.metric;
+ if (metric !== undefined) {
+ testResult = newValues.testResult;
+ return false;
+ }
+ } else if ((testResult = testMetric(current.test, metricType))) {
+ metric = current;
+ return false;
+ }
+ });
+
+ return { metric, testResult };
+};
+
+// -------------------------------------------------------------------
+
+// The metrics:
+// test: can be a function, regexp or string, default to: currentKey
+// getPath: default to: () => currentKey
+// transformValue: default to: identity
+const STATS: { [key: string]: object } = {
+ host: {
+ load: {
+ test: "loadavg",
+ },
+ memoryFree: {
+ test: "memory_free_kib",
+ transformValue: (value: number) => value * 1024,
+ },
+ memory: {
+ test: "memory_total_kib",
+ transformValue: (value: number) => value * 1024,
+ },
+ cpus: {
+ test: /^cpu(\d+)$/,
+ getPath: (matches: any) => ["cpus", matches[1]],
+ transformValue: (value: number) => value * 1e2,
+ },
+ pifs: {
+ rx: {
+ test: /^pif_eth(\d+)_rx$/,
+ getPath: (matches: unknown[]) => ["pifs", "rx", matches[1]],
+ },
+ tx: {
+ test: /^pif_eth(\d+)_tx$/,
+ getPath: (matches: unknown[]) => ["pifs", "tx", matches[1]],
+ },
+ },
+ iops: {
+ r: {
+ test: /^iops_read_(\w+)$/,
+ getPath: (matches: unknown[]) => ["iops", "r", matches[1]],
+ },
+ w: {
+ test: /^iops_write_(\w+)$/,
+ getPath: (matches: unknown[]) => ["iops", "w", matches[1]],
+ },
+ },
+ ioThroughput: {
+ r: {
+ test: /^io_throughput_read_(\w+)$/,
+ getPath: (matches: unknown[]) => ["ioThroughput", "r", matches[1]],
+ transformValue: (value: number) => value * 2 ** 20,
+ },
+ w: {
+ test: /^io_throughput_write_(\w+)$/,
+ getPath: (matches: unknown[]) => ["ioThroughput", "w", matches[1]],
+ transformValue: (value: number) => value * 2 ** 20,
+ },
+ },
+ latency: {
+ r: {
+ test: /^read_latency_(\w+)$/,
+ getPath: (matches: unknown[]) => ["latency", "r", matches[1]],
+ transformValue: (value: number) => value / 1e3,
+ },
+ w: {
+ test: /^write_latency_(\w+)$/,
+ getPath: (matches: unknown[]) => ["latency", "w", matches[1]],
+ transformValue: (value: number) => value / 1e3,
+ },
+ },
+ iowait: {
+ test: /^iowait_(\w+)$/,
+ getPath: (matches: unknown[]) => ["iowait", matches[1]],
+ },
+ },
+ vm: {
+ memoryFree: {
+ test: "memory_internal_free",
+ transformValue: (value: number) => value * 1024,
+ },
+ memory: {
+ test: (metricType: string) => metricType.endsWith("memory"),
+ },
+ cpus: {
+ test: /^cpu(\d+)$/,
+ getPath: (matches: unknown[]) => ["cpus", matches[1]],
+ transformValue: (value: number) => value * 1e2,
+ },
+ vifs: {
+ rx: {
+ test: /^vif_(\d+)_rx$/,
+ getPath: (matches: unknown[]) => ["vifs", "rx", matches[1]],
+ },
+ tx: {
+ test: /^vif_(\d+)_tx$/,
+ getPath: (matches: unknown[]) => ["vifs", "tx", matches[1]],
+ },
+ },
+ xvds: {
+ r: {
+ test: /^vbd_xvd(.)_read$/,
+ getPath: (matches: unknown[]) => ["xvds", "r", matches[1]],
+ },
+ w: {
+ test: /^vbd_xvd(.)_write$/,
+ getPath: (matches: unknown[]) => ["xvds", "w", matches[1]],
+ },
+ },
+ iops: {
+ r: {
+ test: /^vbd_xvd(.)_iops_read$/,
+ getPath: (matches: unknown[]) => ["iops", "r", matches[1]],
+ },
+ w: {
+ test: /^vbd_xvd(.)_iops_write$/,
+ getPath: (matches: unknown[]) => ["iops", "w", matches[1]],
+ },
+ },
+ },
+};
+
+// -------------------------------------------------------------------
+
+// RRD
+// json: {
+// meta: {
+// start: Number,
+// step: Number,
+// end: Number,
+// rows: Number,
+// columns: Number,
+// legend: String[rows]
+// },
+// data: Item[columns] // Item = { t: Number, values: Number[rows] }
+// }
+
+// Local cache
+// _statsByObject : {
+// [uuid]: {
+// [step]: {
+// endTimestamp: Number, // the timestamp of the last statistic point
+// interval: Number, // step
+// stats: T
+// }
+// }
+// }
+
+export type VmStats = {
+ cpus: Record;
+ iops: {
+ r: Record;
+ w: Record;
+ };
+ memory: number[];
+ memoryFree: number[];
+ vifs: {
+ rx: Record;
+ tx: Record;
+ };
+ xvds: {
+ w: Record;
+ r: Record;
+ };
+};
+
+export type HostStats = {
+ cpus: Record;
+ ioThroughput: {
+ r: Record;
+ w: Record;
+ };
+ iops: {
+ r: Record;
+ w: Record;
+ };
+ iowait: Record;
+ latency: {
+ r: Record;
+ w: Record;
+ };
+ load: number[];
+ memory: number[];
+ memoryFree: number[];
+ pifs: {
+ rx: Record;
+ tx: Record;
+ };
+};
+
+export type XapiStatsResponse = {
+ endTimestamp: number;
+ interval: number;
+ stats: T;
+};
+
+export default class XapiStats {
+ #xapi;
+ #statsByObject: {
+ [uuid: string]: {
+ [step: string]: XapiStatsResponse;
+ };
+ } = {};
+ constructor(xapi: XenApi) {
+ this.#xapi = xapi;
+ }
+
+ // Execute one http request on a XenServer for get stats
+ // Return stats (Json format) or throws got exception
+ @limitConcurrency(3)
+ async _getJson(host: XenApiHost, timestamp: any, step: any) {
+ const resp = await this.#xapi.getResource("/rrd_updates", {
+ host,
+ query: {
+ cf: "AVERAGE",
+ host: "true",
+ interval: step,
+ json: "true",
+ start: timestamp,
+ },
+ });
+ return JSON5.parse(await resp.text());
+ }
+
+ // To avoid multiple requests, we keep a cache for the stats and
+ // only return it if we not exceed a step
+ #getCachedStats(uuid: any, step: any, currentTimeStamp: any) {
+ const statsByObject = this.#statsByObject;
+
+ const stats = statsByObject[uuid]?.[step];
+ if (stats === undefined) {
+ return;
+ }
+
+ if (stats.endTimestamp + step < currentTimeStamp) {
+ delete statsByObject[uuid][step];
+ return;
+ }
+
+ return stats;
+ }
+
+ @synchronized.withKey(({ host }: { host: XenApiHost }) => host.uuid)
+ async _getAndUpdateStats({
+ host,
+ uuid,
+ granularity,
+ }: {
+ host: XenApiHost;
+ uuid: any;
+ granularity: GRANULARITY;
+ }) {
+ const step =
+ granularity === undefined
+ ? RRD_STEP.Seconds
+ : RRD_STEP_FROM_STRING[granularity];
+ if (step === undefined) {
+ throw new FaultyGranularity(
+ `Unknown granularity: '${granularity}'. Use 'seconds', 'minutes', 'hours', or 'days'.`
+ );
+ }
+ const currentTimeStamp = await this.#xapi.getHostServertime(host);
+
+ const stats = this.#getCachedStats(uuid, step, currentTimeStamp);
+ if (stats !== undefined) {
+ return stats;
+ }
+
+ const maxDuration = step * RRD_POINTS_PER_STEP[step];
+
+ // To avoid crossing over the boundary, we ask for one less step
+ const optimumTimestamp = currentTimeStamp - maxDuration + step;
+ const json = await this._getJson(host, optimumTimestamp, step);
+
+ const actualStep = json.meta.step as number;
+
+ if (json.data.length > 0) {
+ // fetched data is organized from the newest to the oldest
+ // but this implementation requires it in the other direction
+ json.data.reverse();
+ json.meta.legend.forEach((legend: any, index: number) => {
+ const [, type, uuid, metricType] = /^AVERAGE:([^:]+):(.+):(.+)$/.exec(
+ legend
+ ) as any;
+
+ const metrics = STATS[type] as any;
+ if (metrics === undefined) {
+ return;
+ }
+
+ const { metric, testResult } = findMetric(metrics, metricType) as any;
+ if (metric === undefined) {
+ return;
+ }
+
+ const xoObjectStats = createGetProperty(this.#statsByObject, uuid, {});
+ let stepStats = xoObjectStats[actualStep];
+ if (
+ stepStats === undefined ||
+ stepStats.endTimestamp !== json.meta.end
+ ) {
+ stepStats = xoObjectStats[actualStep] = {
+ endTimestamp: json.meta.end,
+ interval: actualStep,
+ };
+ }
+
+ const path =
+ metric.getPath !== undefined
+ ? metric.getPath(testResult)
+ : [findKey(metrics, metric)];
+
+ const lastKey = path.length - 1;
+ let metricStats = createGetProperty(stepStats, "stats", {});
+ path.forEach((property: any, key: number) => {
+ if (key === lastKey) {
+ metricStats[property] = computeValues(
+ json.data,
+ index,
+ metric.transformValue
+ );
+ return;
+ }
+
+ metricStats = createGetProperty(metricStats, property, {});
+ });
+ });
+ }
+
+ if (actualStep !== step) {
+ throw new FaultyGranularity(
+ `Unable to get the true granularity: ${actualStep}`
+ );
+ }
+
+ return (
+ this.#statsByObject[uuid]?.[step] ?? {
+ endTimestamp: currentTimeStamp,
+ interval: step,
+ stats: {},
+ }
+ );
+ }
+}
diff --git a/@xen-orchestra/lite/src/libs/xen-api.ts b/@xen-orchestra/lite/src/libs/xen-api.ts
index 29613977a..9cd2cb30d 100644
--- a/@xen-orchestra/lite/src/libs/xen-api.ts
+++ b/@xen-orchestra/lite/src/libs/xen-api.ts
@@ -1,4 +1,5 @@
import { JSONRPCClient } from "json-rpc-2.0";
+import { parseDateTime } from "@/libs/utils";
export type RawObjectType =
| "Bond"
@@ -72,6 +73,7 @@ export interface XenApiPool extends XenApiRecord {
}
export interface XenApiHost extends XenApiRecord {
+ address: string;
name_label: string;
metrics: string;
resident_VMs: string[];
@@ -190,6 +192,30 @@ export default class XenApi {
return this.#client.request(method, args);
}
+ async getHostServertime(host: XenApiHost) {
+ const serverLocaltime = (await this.#call("host.get_servertime", [
+ this.sessionId,
+ host.$ref,
+ ])) as string;
+ return Math.floor(parseDateTime(serverLocaltime) / 1e3);
+ }
+
+ async getResource(
+ pathname: string,
+ { host, query }: { host: XenApiHost; query: any }
+ ) {
+ const url = new URL("http://localhost");
+ url.protocol = window.location.protocol;
+ url.hostname = host.address;
+ url.pathname = pathname;
+ url.search = new URLSearchParams({
+ ...query,
+ session_id: this.#sessionId,
+ }).toString();
+
+ return fetch(url);
+ }
+
async loadRecords(
type: RawObjectType
): Promise