feat(xo-server/xapi-stats): new implementation (#2648)

packages/xo-server-usage-report/src/index.js

@@ -159,7 +159,7 @@ function computeMean (values) {
   return sum / n
 }
 
-const computeDoubleMean = val => computeMean(val.map(computeMean))
+const computeDoubleMean = val => computeMean(map(val, computeMean))
 
 function computeMeans (objects, options) {
   return zipObject(
@@ -212,6 +212,10 @@ function getDiff (oldElements, newElements) {
   }
 }
 
+function getMemoryUsedMetric ({ memory, memoryFree = memory }) {
+  return map(memory, (value, key) => value - memoryFree[key])
+}
+
 // ===================================================================
 
 async function getVmsStats ({ runningVms, xo }) {
@@ -223,7 +227,7 @@ async function getVmsStats ({ runningVms, xo }) {
           uuid: vm.uuid,
           name: vm.name_label,
           cpu: computeDoubleMean(vmStats.stats.cpus),
-          ram: computeMean(vmStats.stats.memoryUsed) / gibPower,
+          ram: computeMean(getMemoryUsedMetric(vmStats.stats)) / gibPower,
           diskRead: computeDoubleMean(values(vmStats.stats.xvds.r)) / mibPower,
           diskWrite: computeDoubleMean(values(vmStats.stats.xvds.w)) / mibPower,
           netReception: computeDoubleMean(vmStats.stats.vifs.rx) / kibPower,
@@ -245,7 +249,7 @@ async function getHostsStats ({ runningHosts, xo }) {
           uuid: host.uuid,
           name: host.name_label,
           cpu: computeDoubleMean(hostStats.stats.cpus),
-          ram: computeMean(hostStats.stats.memoryUsed) / gibPower,
+          ram: computeMean(getMemoryUsedMetric(hostStats.stats)) / gibPower,
           load: computeMean(hostStats.stats.load),
           netReception: computeDoubleMean(hostStats.stats.pifs.rx) / kibPower,
           netTransmission:

packages/xo-server/src/api/host.js

@@ -242,7 +242,7 @@ emergencyShutdownHost.resolve = {
 // -------------------------------------------------------------------
 
 export function stats ({ host, granularity }) {
-  return this.getXapiHostStats(host, granularity)
+  return this.getXapiHostStats(host._xapiId, granularity)
 }
 
 stats.description = 'returns statistic of the host'

packages/xo-server/src/api/vm.js

@@ -1351,7 +1351,7 @@ detachPci.resolve = {
 // -------------------------------------------------------------------
 
 export function stats ({ vm, granularity }) {
-  return this.getXapiVmStats(vm, granularity)
+  return this.getXapiVmStats(vm._xapiId, granularity)
 }
 
 stats.description = 'returns statistics about the VM'

packages/xo-server/src/xapi-stats.js

@@ -1,10 +1,20 @@
-import endsWith from 'lodash/endsWith'
 import JSON5 from 'json5'
 import limitConcurrency from 'limit-concurrency-decorator'
 import { BaseError } from 'make-error'
+import { endsWith, findKey, forEach, get, identity, map } from 'lodash'
 
 import { parseDateTime } from './xapi'
 
+export class FaultyGranularity extends BaseError {}
+
+// -------------------------------------------------------------------
+
+// according to https://xapi-project.github.io/xen-api/metrics.html
+// The values are stored at intervals of:
+//  - 5 seconds for the past 10 minutes
+//  - one minute for the past 2 hours
+//  - one hour for the past week
+//  - one day for the past year
 const RRD_STEP_SECONDS = 5
 const RRD_STEP_MINUTES = 60
 const RRD_STEP_HOURS = 3600
@@ -17,6 +27,7 @@ const RRD_STEP_FROM_STRING = {
   days: RRD_STEP_DAYS,
 }
 
+// points = intervalInSeconds / step
 const RRD_POINTS_PER_STEP = {
   [RRD_STEP_SECONDS]: 120,
   [RRD_STEP_MINUTES]: 120,
@@ -24,16 +35,6 @@ const RRD_POINTS_PER_STEP = {
   [RRD_STEP_DAYS]: 366,
 }
 
-export class XapiStatsError extends BaseError {}
-
-export class UnknownLegendFormat extends XapiStatsError {
-  constructor (line) {
-    super('Unknown legend line: ' + line)
-  }
-}
-
-export class FaultyGranularity extends XapiStatsError {}
-
 // -------------------------------------------------------------------
 // Utils
 // -------------------------------------------------------------------
@@ -47,353 +48,143 @@ function convertNanToNull (value) {
   return isNaN(value) ? null : value
 }
 
-async function getServerTimestamp (xapi, host) {
+async function getServerTimestamp (xapi, hostRef) {
-  const serverLocalTime = await xapi.call('host.get_servertime', host.$ref)
+  const serverLocalTime = await xapi.call('host.get_servertime', hostRef)
-  return Math.floor(parseDateTime(serverLocalTime).getTime() / 1000)
+  return Math.floor(parseDateTime(serverLocalTime).getTime() / 1e3)
 }
 
 // -------------------------------------------------------------------
 // Stats
 // -------------------------------------------------------------------
 
-function getNewHostStats () {
+const computeValues = (dataRow, legendIndex, transformValue = identity) =>
-  return {
+  map(dataRow, ({ values }) =>
-    cpus: [],
+    transformValue(convertNanToNull(values[legendIndex]))
-    pifs: {
+  )
-      rx: [],
+
-      tx: [],
+// It browse the object in depth and initialise it's properties
-    },
+// The targerPath can be a string or an array containing the depth
-    load: [],
+// targetPath: [a, b, c] => a.b.c
-    memory: [],
+const getValuesFromDepth = (obj, targetPath) => {
-    memoryFree: [],
+  if (typeof targetPath === 'string') {
-    memoryUsed: [],
+    return (obj[targetPath] = [])
   }
-}
 
-function getNewVmStats () {
+  forEach(targetPath, (path, key) => {
-  return {
+    if (obj[path] === undefined) {
-    cpus: [],
+      obj = obj[path] = targetPath.length - 1 === key ? [] : {}
-    vifs: {
+      return
-      rx: [],
-      tx: [],
-    },
-    xvds: {
-      r: {},
-      w: {},
-    },
-    memory: [],
-    memoryFree: [],
-    memoryUsed: [],
-  }
-}
-
-// -------------------------------------------------------------------
-// Stats legends
-// -------------------------------------------------------------------
-
-function getNewHostLegends () {
-  return {
-    cpus: [],
-    pifs: {
-      rx: [],
-      tx: [],
-    },
-    load: null,
-    memoryFree: null,
-    memory: null,
-  }
-}
-
-function getNewVmLegends () {
-  return {
-    cpus: [],
-    vifs: {
-      rx: [],
-      tx: [],
-    },
-    xvds: {
-      r: [],
-      w: [],
-    },
-    memoryFree: null,
-    memory: null,
-  }
-}
-
-// Compute one legend line for one host
-function parseOneHostLegend (hostLegend, type, index) {
-  let resReg
-
-  if ((resReg = /^cpu([0-9]+)$/.exec(type)) !== null) {
-    hostLegend.cpus[resReg[1]] = index
-  } else if ((resReg = /^pif_eth([0-9]+)_(rx|tx)$/.exec(type)) !== null) {
-    if (resReg[2] === 'rx') {
-      hostLegend.pifs.rx[resReg[1]] = index
-    } else {
-      hostLegend.pifs.tx[resReg[1]] = index
     }
-  } else if (type === 'loadavg') {
+    obj = obj[path]
-    hostLegend.load = index
+  })
-  } else if (type === 'memory_free_kib') {
+  return obj
-    hostLegend.memoryFree = index
-  } else if (type === 'memory_total_kib') {
-    hostLegend.memory = index
-  }
 }
 
-// Compute one legend line for one vm
+const testMetric = (test, type) =>
-function parseOneVmLegend (vmLegend, type, index) {
+  typeof test === 'string'
-  let resReg
+    ? test === type
+    : typeof test === 'function' ? test(type) : test.exec(type)
 
-  if ((resReg = /^cpu([0-9]+)$/.exec(type)) !== null) {
+const findMetric = (metrics, metricType) => {
-    vmLegend.cpus[resReg[1]] = index
+  let testResult
-  } else if ((resReg = /^vif_([0-9]+)_(rx|tx)$/.exec(type)) !== null) {
+  let metric
-    if (resReg[2] === 'rx') {
-      vmLegend.vifs.rx[resReg[1]] = index
-    } else {
-      vmLegend.vifs.tx[resReg[1]] = index
-    }
-  } else if ((resReg = /^vbd_xvd(.)_(read|write)$/.exec(type))) {
-    if (resReg[2] === 'read') {
-      vmLegend.xvds.r[resReg[1]] = index
-    } else {
-      vmLegend.xvds.w[resReg[1]] = index
-    }
-  } else if (type === 'memory_internal_free') {
-    vmLegend.memoryFree = index
-  } else if (endsWith(type, 'memory')) {
-    vmLegend.memory = index
-  }
-}
 
-// Compute Stats Legends for host and vms from RRD update
+  forEach(metrics, (current, key) => {
-function parseLegends (json) {
+    if (current.test === undefined) {
-  const hostLegends = getNewHostLegends()
+      const newValues = findMetric(current, metricType)
-  const vmsLegends = {}
 
-  json.meta.legend.forEach((value, index) => {
+      metric = newValues.metric
-    const parsedLine = /^AVERAGE:(host|vm):(.+):(.+)$/.exec(value)
+      if (metric !== undefined) {
-
+        testResult = newValues.testResult
-    if (parsedLine === null) {
+        return false
-      throw new UnknownLegendFormat(value)
-    }
-
-    const [, name, uuid, type] = parsedLine
-
-    if (name !== 'vm') {
-      parseOneHostLegend(hostLegends, type, index)
-    } else {
-      if (vmsLegends[uuid] === undefined) {
-        vmsLegends[uuid] = getNewVmLegends()
       }
-
+    } else if ((testResult = testMetric(current.test, metricType))) {
-      parseOneVmLegend(vmsLegends[uuid], type, index)
+      metric = current
+      return false
     }
   })
 
-  return [hostLegends, vmsLegends]
+  return { metric, testResult }
 }
 
+// -------------------------------------------------------------------
+
+// The metrics:
+//  test: can be a function, regexp or string, default to: currentKey
+//  getPath: default to: () => currentKey
+//  transformValue: default to: identity
+const STATS = {
+  host: {
+    load: {
+      test: 'loadavg',
+    },
+    memoryFree: {
+      test: 'memory_free_kib',
+      transformValue: value => value * 1024,
+    },
+    memory: {
+      test: 'memory_total_kib',
+      transformValue: value => value * 1024,
+    },
+    cpus: {
+      test: /^cpu(\d+)$/,
+      getPath: matches => ['cpus', matches[1]],
+      transformValue: value => value * 1e2,
+    },
+    pifs: {
+      rx: {
+        test: /^pif_eth(\d+)_rx$/,
+        getPath: matches => ['pifs', 'rx', matches[1]],
+      },
+      tx: {
+        test: /^pif_eth(\d+)_tx$/,
+        getPath: matches => ['pifs', 'tx', matches[1]],
+      },
+    },
+  },
+  vm: {
+    memoryFree: {
+      test: 'memory_internal_free',
+      transformValue: value => value * 1024,
+    },
+    memory: {
+      test: metricType => endsWith(metricType, 'memory'),
+    },
+    cpus: {
+      test: /^cpu(\d+)$/,
+      getPath: matches => ['cpus', matches[1]],
+      transformValue: value => value * 1e2,
+    },
+    vifs: {
+      rx: {
+        test: /^vif_(\d+)_rx$/,
+        getPath: matches => ['vifs', 'rx', matches[1]],
+      },
+      tx: {
+        test: /^vif_(\d+)_tx$/,
+        getPath: matches => ['vifs', 'tx', matches[1]],
+      },
+    },
+    xvds: {
+      r: {
+        test: /^vbd_xvd(.)_read$/,
+        getPath: matches => ['xvds', 'r', matches[1]],
+      },
+      w: {
+        test: /^vbd_xvd(.)_write$/,
+        getPath: matches => ['xvds', 'w', matches[1]],
+      },
+    },
+  },
+}
+
+// -------------------------------------------------------------------
+
 export default class XapiStats {
   constructor () {
-    this._vms = {}
+    this._statsByObject = {}
-    this._hosts = {}
   }
 
-  // -------------------------------------------------------------------
-  // Remove stats (Helper)
-  // -------------------------------------------------------------------
-
-  _removeOlderStats (source, dest, pointsPerStep) {
-    for (const key in source) {
-      if (key === 'cpus') {
-        for (const cpuIndex in source.cpus) {
-          dest.cpus[cpuIndex].splice(
-            0,
-            dest.cpus[cpuIndex].length - pointsPerStep
-          )
-        }
-
-        // If the number of cpus has been decreased, remove !
-        let offset
-
-        if ((offset = dest.cpus.length - source.cpus.length) > 0) {
-          dest.cpus.splice(-offset)
-        }
-      } else if (endsWith(key, 'ifs')) {
-        // For each pif or vif
-        for (const ifType in source[key]) {
-          for (const pifIndex in source[key][ifType]) {
-            dest[key][ifType][pifIndex].splice(
-              0,
-              dest[key][ifType][pifIndex].length - pointsPerStep
-            )
-          }
-
-          // If the number of pifs has been decreased, remove !
-          let offset
-
-          if (
-            (offset = dest[key][ifType].length - source[key][ifType].length) > 0
-          ) {
-            dest[key][ifType].splice(-offset)
-          }
-        }
-      } else if (key === 'xvds') {
-        for (const xvdType in source.xvds) {
-          for (const xvdLetter in source.xvds[xvdType]) {
-            dest.xvds[xvdType][xvdLetter].splice(
-              0,
-              dest.xvds[xvdType][xvdLetter].length - pointsPerStep
-            )
-          }
-
-          // If the number of xvds has been decreased, remove !
-          // FIXME
-        }
-      } else if (key === 'load') {
-        dest.load.splice(0, dest[key].length - pointsPerStep)
-      } else if (key === 'memory') {
-        // Load, memory, memoryFree, memoryUsed
-        const length = dest.memory.length - pointsPerStep
-        dest.memory.splice(0, length)
-        dest.memoryFree.splice(0, length)
-        dest.memoryUsed.splice(0, length)
-      }
-    }
-  }
-
-  // -------------------------------------------------------------------
-  // HOST: Computation and stats update
-  // -------------------------------------------------------------------
-
-  // Compute one stats row for one host
-  _parseRowHostStats (hostLegends, hostStats, values) {
-    // Cpus
-    hostLegends.cpus.forEach((cpuIndex, index) => {
-      if (hostStats.cpus[index] === undefined) {
-        hostStats.cpus[index] = []
-      }
-
-      hostStats.cpus[index].push(values[cpuIndex] * 100)
-    })
-
-    // Pifs
-    for (const pifType in hostLegends.pifs) {
-      hostLegends.pifs[pifType].forEach((pifIndex, index) => {
-        if (hostStats.pifs[pifType][index] === undefined) {
-          hostStats.pifs[pifType][index] = []
-        }
-
-        hostStats.pifs[pifType][index].push(convertNanToNull(values[pifIndex]))
-      })
-    }
-
-    // Load
-    hostStats.load.push(convertNanToNull(values[hostLegends.load]))
-
-    // Memory.
-    // WARNING! memory/memoryFree are in kB.
-    const memory = values[hostLegends.memory] * 1024
-    const memoryFree = values[hostLegends.memoryFree] * 1024
-
-    hostStats.memory.push(memory)
-
-    if (hostLegends.memoryFree !== undefined) {
-      hostStats.memoryFree.push(memoryFree)
-      hostStats.memoryUsed.push(memory - memoryFree)
-    }
-  }
-
-  // Compute stats for host from RRD update
-  _parseHostStats (json, hostname, hostLegends, step) {
-    const host = this._hosts[hostname][step]
-
-    if (host.stats === undefined) {
-      host.stats = getNewHostStats()
-    }
-
-    for (const row of json.data) {
-      this._parseRowHostStats(hostLegends, host.stats, row.values)
-    }
-  }
-
-  // -------------------------------------------------------------------
-  // VM: Computation and stats update
-  // -------------------------------------------------------------------
-
-  // Compute stats for vms from RRD update
-  _parseRowVmStats (vmLegends, vmStats, values) {
-    // Cpus
-    vmLegends.cpus.forEach((cpuIndex, index) => {
-      if (vmStats.cpus[index] === undefined) {
-        vmStats.cpus[index] = []
-      }
-
-      vmStats.cpus[index].push(values[cpuIndex] * 100)
-    })
-
-    // Vifs
-    for (const vifType in vmLegends.vifs) {
-      vmLegends.vifs[vifType].forEach((vifIndex, index) => {
-        if (vmStats.vifs[vifType][index] === undefined) {
-          vmStats.vifs[vifType][index] = []
-        }
-
-        vmStats.vifs[vifType][index].push(convertNanToNull(values[vifIndex]))
-      })
-    }
-
-    // Xvds
-    for (const xvdType in vmLegends.xvds) {
-      for (const index in vmLegends.xvds[xvdType]) {
-        if (vmStats.xvds[xvdType][index] === undefined) {
-          vmStats.xvds[xvdType][index] = []
-        }
-
-        vmStats.xvds[xvdType][index].push(
-          convertNanToNull(values[vmLegends.xvds[xvdType][index]])
-        )
-      }
-    }
-
-    // Memory
-    // WARNING! memoryFree is in Kb not in b, memory is in b
-    const memory = values[vmLegends.memory]
-    const memoryFree = values[vmLegends.memoryFree] * 1024
-
-    vmStats.memory.push(memory)
-
-    if (vmLegends.memoryFree !== undefined) {
-      vmStats.memoryFree.push(memoryFree)
-      vmStats.memoryUsed.push(memory - memoryFree)
-    }
-  }
-
-  // Compute stats for vms
-  _parseVmsStats (json, hostname, vmsLegends, step) {
-    if (this._vms[hostname][step] === undefined) {
-      this._vms[hostname][step] = {}
-    }
-
-    const vms = this._vms[hostname][step]
-
-    for (const uuid in vmsLegends) {
-      if (vms[uuid] === undefined) {
-        vms[uuid] = getNewVmStats()
-      }
-    }
-
-    for (const row of json.data) {
-      for (const uuid in vmsLegends) {
-        this._parseRowVmStats(vmsLegends[uuid], vms[uuid], row.values)
-      }
-    }
-  }
-
-  // -------------------------------------------------------------------
-  // -------------------------------------------------------------------
-
   // Execute one http request on a XenServer for get stats
   // Return stats (Json format) or throws got exception
   @limitConcurrency(3)
@@ -411,40 +202,46 @@ export default class XapiStats {
       .then(response => response.readAll().then(JSON5.parse))
   }
 
-  async _getLastTimestamp (xapi, host, step) {
+  async _getNextTimestamp (xapi, host, step) {
-    if (this._hosts[host.address][step] === undefined) {
+    const currentTimeStamp = await getServerTimestamp(xapi, host.$ref)
-      const serverTimeStamp = await getServerTimestamp(xapi, host)
+    const maxDuration = step * RRD_POINTS_PER_STEP[step]
-      return serverTimeStamp - step * RRD_POINTS_PER_STEP[step] + step
+    const lastTimestamp = get(this._statsByObject, [
-    }
+      host.uuid,
+      step,
+      'endTimestamp',
+    ])
 
-    return this._hosts[host.address][step].endTimestamp
+    if (
+      lastTimestamp === undefined ||
+      currentTimeStamp - lastTimestamp + step > maxDuration
+    ) {
+      return currentTimeStamp - maxDuration + step
+    }
+    return lastTimestamp
   }
 
-  _getPoints (hostname, step, vmId) {
+  _getStats (hostUuid, step, vmUuid) {
-    const hostStats = this._hosts[hostname][step]
+    const hostStats = this._statsByObject[hostUuid][step]
 
-    // Return host points
+    // Return host stats
-    if (vmId === undefined) {
+    if (vmUuid === undefined) {
       return {
         interval: step,
         ...hostStats,
       }
     }
 
-    const vmsStats = this._vms[hostname][step]
+    // Return vm stats
-
-    // Return vm points
     return {
       interval: step,
       endTimestamp: hostStats.endTimestamp,
-      stats: (vmsStats && vmsStats[vmId]) || getNewVmStats(),
+      ...this._statsByObject[vmUuid][step],
     }
   }
 
-  async _getAndUpdatePoints (xapi, host, vmId, granularity) {
+  async _getAndUpdateStats (xapi, { host, vmUuid, granularity }) {
-    // Get granularity to use
     const step =
-      granularity === undefined || granularity === 0
+      granularity === undefined
         ? RRD_STEP_SECONDS
         : RRD_STEP_FROM_STRING[granularity]
 
@@ -455,59 +252,21 @@ export default class XapiStats {
     }
 
     // Limit the number of http requests
-    const hostname = host.address
+    const hostUuid = host.uuid
-
-    if (this._hosts[hostname] === undefined) {
-      this._hosts[hostname] = {}
-      this._vms[hostname] = {}
-    }
 
     if (
-      this._hosts[hostname][step] !== undefined &&
+      get(this._statsByObject, [hostUuid, step, 'localTimestamp']) + step >
-      this._hosts[hostname][step].localTimestamp + step > getCurrentTimestamp()
+      getCurrentTimestamp()
     ) {
-      return this._getPoints(hostname, step, vmId)
+      return this._getStats(hostUuid, step, vmUuid)
     }
 
-    // Check if we are in the good interval, use this._hosts[hostname][step].localTimestamp
+    const timestamp = await this._getNextTimestamp(xapi, host, step)
-    // for avoid bad requests
+    const json = await this._getJson(xapi, host, timestamp)
-    // TODO
-
-    // Get json
-    const timestamp = await this._getLastTimestamp(xapi, host, step)
-    let json = await this._getJson(xapi, host, timestamp)
-
-    // Check if the granularity is linked to 'step'
-    // If it's not the case, we retry other url with the json timestamp
     if (json.meta.step !== step) {
-      console.log(
+      throw new FaultyGranularity(
-        `RRD call: Expected step: ${step}, received step: ${
+        `Unable to get the true granularity: ${json.meta.step}`
-          json.meta.step
-        }. Retry with other timestamp`
       )
-      const serverTimestamp = await getServerTimestamp(xapi, host)
-
-      // Approximately: half points are asked
-      // FIXME: Not the best solution
-      json = await this._getJson(
-        xapi,
-        host,
-        serverTimestamp - step * (RRD_POINTS_PER_STEP[step] / 2) + step
-      )
-
-      if (json.meta.step !== step) {
-        throw new FaultyGranularity(
-          `Unable to get the true granularity: ${json.meta.step}`
-        )
-      }
-    }
-
-    // Make new backup slot if necessary
-    if (this._hosts[hostname][step] === undefined) {
-      this._hosts[hostname][step] = {
-        endTimestamp: 0,
-        localTimestamp: 0,
-      }
     }
 
     // It exists data
@@ -516,70 +275,90 @@ export default class XapiStats {
       // timestamp of the oldest data value
       // So, we use the timestamp of the oldest data value !
       const startTimestamp = json.data[json.meta.rows - 1].t
+      const endTimestamp = get(this._statsByObject, [
+        hostUuid,
+        step,
+        'endTimestamp',
+      ])
 
-      // Remove useless data and reorder
+      const statsOffset = endTimestamp - startTimestamp + step
-      // Note: Older values are at end of json.data.row
+      if (endTimestamp !== undefined && statsOffset > 0) {
-      const parseOffset =
+        const parseOffset = statsOffset / step
-        (this._hosts[hostname][step].endTimestamp - startTimestamp + step) /
+        // Remove useless data
-        step
+        // Note: Older values are at end of json.data.row
-
+        json.data.splice(json.data.length - parseOffset)
-      json.data.splice(json.data.length - parseOffset)
+      }
-      json.data.reverse()
 
       // It exists useful data
       if (json.data.length > 0) {
-        const [hostLegends, vmsLegends] = parseLegends(json)
+        // reorder data
-
+        json.data.reverse()
-        // Compute and update host/vms stats
+        forEach(json.meta.legend, (legend, index) => {
-        this._parseVmsStats(json, hostname, vmsLegends, step)
+          const [, type, uuid, metricType] = /^AVERAGE:([^:]+):(.+):(.+)$/.exec(
-        this._parseHostStats(json, hostname, hostLegends, step)
+            legend
-
-        // Remove older stats
-        this._removeOlderStats(
-          hostLegends,
-          this._hosts[hostname][step].stats,
-          RRD_POINTS_PER_STEP[step]
-        )
-
-        for (const uuid in vmsLegends) {
-          this._removeOlderStats(
-            vmsLegends[uuid],
-            this._vms[hostname][step][uuid],
-            RRD_POINTS_PER_STEP[step]
           )
-        }
+
+          const metrics = STATS[type]
+          if (metrics === undefined) {
+            return
+          }
+
+          const { metric, testResult } = findMetric(metrics, metricType)
+
+          if (metric === undefined) {
+            return
+          }
+
+          const path =
+            metric.getPath !== undefined
+              ? metric.getPath(testResult)
+              : [findKey(metrics, metric)]
+
+          const metricValues = getValuesFromDepth(this._statsByObject, [
+            uuid,
+            step,
+            'stats',
+            ...path,
+          ])
+
+          metricValues.push(
+            ...computeValues(json.data, index, metric.transformValue)
+          )
+
+          // remove older Values
+          metricValues.splice(
+            0,
+            metricValues.length - RRD_POINTS_PER_STEP[step]
+          )
+        })
       }
     }
 
     // Update timestamp
-    this._hosts[hostname][step].endTimestamp = json.meta.end
+    const hostStats = this._statsByObject[hostUuid][step]
-    this._hosts[hostname][step].localTimestamp = getCurrentTimestamp()
+    hostStats.endTimestamp = json.meta.end
-
+    hostStats.localTimestamp = getCurrentTimestamp()
-    return this._getPoints(hostname, step, vmId)
+    return this._getStats(hostUuid, step, vmUuid)
   }
 
-  // -------------------------------------------------------------------
+  getHostStats (xapi, hostId, granularity) {
-  // -------------------------------------------------------------------
+    return this._getAndUpdateStats(xapi, {
-
+      host: xapi.getObject(hostId),
-  // Warning: This functions returns one reference on internal data
+      granularity,
-  // So, data can be changed by a parallel call on this functions
+    })
-  // It is forbidden to modify the returned data
-
-  // Return host stats
-  async getHostPoints (xapi, hostId, granularity) {
-    const host = xapi.getObject(hostId)
-    return this._getAndUpdatePoints(xapi, host, undefined, granularity)
   }
 
-  // Return vms stats
+  getVmStats (xapi, vmId, granularity) {
-  async getVmPoints (xapi, vmId, granularity) {
     const vm = xapi.getObject(vmId)
     const host = vm.$resident_on
-
     if (!host) {
       throw new Error(`VM ${vmId} is halted or host could not be found.`)
     }
 
-    return this._getAndUpdatePoints(xapi, host, vm.uuid, granularity)
+    return this._getAndUpdateStats(xapi, {
+      host,
+      vmUuid: vm.uuid,
+      granularity,
+    })
   }
 }

packages/xo-server/src/xo-mixins/xen-servers.js

@@ -392,14 +392,12 @@ export default class {
     return servers
   }
 
-  getXapiVmStats (vm, granularity) {
+  getXapiVmStats (vmId, granularity) {
-    const xapi = this.getXapi(vm)
+    return this._stats.getVmStats(this.getXapi(vmId), vmId, granularity)
-    return this._stats.getVmPoints(xapi, vm._xapiId, granularity)
   }
 
-  getXapiHostStats (host, granularity) {
+  getXapiHostStats (hostId, granularity) {
-    const xapi = this.getXapi(host)
+    return this._stats.getHostStats(this.getXapi(hostId), hostId, granularity)
-    return this._stats.getHostPoints(xapi, host._xapiId, granularity)
   }
 
   async mergeXenPools (sourceId, targetId, force = false) {

packages/xo-web/src/common/utils.js

@@ -524,3 +524,8 @@ export const createFakeProgress = (() => {
 export const ShortDate = ({ timestamp }) => (
   <FormattedDate value={timestamp} month='short' day='numeric' year='numeric' />
 )
+
+// ===================================================================
+
+export const getMemoryUsedMetric = ({ memory, memoryFree = memory }) =>
+  map(memory, (value, key) => value - memoryFree[key])

packages/xo-web/src/common/xo-line-chart/index.js

@@ -8,7 +8,7 @@ import { find, flatten, floor, map, max, size, sum, values } from 'lodash'
 
 import propTypes from '../prop-types-decorator'
 import { computeArraysSum } from '../xo-stats'
-import { formatSize } from '../utils'
+import { formatSize, getMemoryUsedMetric } from '../utils'
 
 import styles from './index.css'
 
@@ -225,7 +225,8 @@ export const MemoryLineChart = injectIntl(
     data: propTypes.object.isRequired,
     options: propTypes.object,
   })(({ data, options = {}, intl }) => {
-    const { memory, memoryUsed } = data.stats
+    const { memory } = data.stats
+    const memoryUsed = getMemoryUsedMetric(data.stats)
 
     if (!memory || !memoryUsed) {
       return templateError
@@ -265,7 +266,8 @@ export const PoolMemoryLineChart = injectIntl(
     options: propTypes.object,
   })(({ addSumSeries, data, options = {}, intl }) => {
     const firstHostData = data[0]
-    const { memory, memoryUsed } = firstHostData.stats
+    const { memory } = firstHostData.stats
+    const memoryUsed = getMemoryUsedMetric(firstHostData.stats)
 
     if (!memory || !memoryUsed) {
       return templateError
@@ -273,13 +275,15 @@ export const PoolMemoryLineChart = injectIntl(
 
     const series = map(data, ({ host, stats }) => ({
       name: host,
-      data: stats.memoryUsed,
+      data: getMemoryUsedMetric(stats),
     }))
 
     if (addSumSeries) {
       series.push({
         name: intl.formatMessage(messages.poolAllHosts),
-        data: computeArraysSum(map(data, 'stats.memoryUsed')),
+        data: computeArraysSum(
+          map(data, ({ stats }) => getMemoryUsedMetric(stats))
+        ),
         className: styles.dashedLine,
       })
     }
@@ -298,7 +302,7 @@ export const PoolMemoryLineChart = injectIntl(
         options={{
           ...makeOptions({
             intl,
-            nValues: firstHostData.stats.memoryUsed.length,
+            nValues: memoryUsed.length,
             endTimestamp: firstHostData.endTimestamp,
             interval: firstHostData.interval,
             valueTransform: formatSize,

packages/xo-web/src/common/xo-sparklines.js

@@ -3,6 +3,7 @@ import { Sparklines, SparklinesLine } from 'react-sparklines'
 
 import propTypes from './prop-types-decorator'
 import { computeArraysAvg, computeObjectsAvg } from './xo-stats'
+import { getMemoryUsedMetric } from './utils'
 
 const STYLE = {}
 
@@ -50,8 +51,8 @@ export const CpuSparkLines = propTypes({
 export const MemorySparkLines = propTypes({
   data: propTypes.object.isRequired,
 })(({ data, width = WIDTH, height = HEIGHT, strokeWidth = STROKE_WIDTH }) => {
-  const { memory, memoryUsed } = data.stats
+  const { memory } = data.stats
-
+  const memoryUsed = getMemoryUsedMetric(data.stats)
   if (!memory || !memoryUsed) {
     return templateError
   }

packages/xo-web/src/xo-app/dashboard/stats/index.js

@@ -16,7 +16,7 @@ import { Container, Row, Col } from 'grid'
 import { error } from 'notification'
 import { SelectHostVm } from 'select-objects'
 import { createGetObjectsOfType } from 'selectors'
-import { connectStore, formatSize, mapPlus } from 'utils'
+import { connectStore, formatSize, getMemoryUsedMetric, mapPlus } from 'utils'
 import { fetchHostStats, fetchVmStats } from 'xo'
 
 // ===================================================================
@@ -246,6 +246,7 @@ class SelectMetric extends Component {
                 (result.endTimestamp - 3600 * (stats.memory.length - 1)) * 1000,
             }
 
+            stats.memoryUsed = getMemoryUsedMetric(stats)
             forEach(stats, (stats, type) => {
               const fnc = STATS_TYPE_TO_COMPUTE_FNC[type]