Merge branch 'master' into nr-delete-fs-createOutputStream

Because it's not a full block but an identifier.

.eslintrc.js

@@ -48,9 +48,5 @@ module.exports = {
     'lines-between-class-members': 'off',
 
     'no-console': ['error', { allow: ['warn', 'error'] }],
-    'no-var': 'error',
-    'node/no-extraneous-import': 'error',
-    'node/no-extraneous-require': 'error',
-    'prefer-const': 'error',
   },
 }

.prettierrc.js

@@ -1,6 +1,11 @@
 module.exports = {
+  arrowParens: 'avoid',
   jsxSingleQuote: true,
   semi: false,
   singleQuote: true,
-  trailingComma: 'es5',
+
+  // 2020-11-24: Requested by nraynaud and approved by the rest of the team
+  //
+  // https://team.vates.fr/vates/pl/a1i8af1b9id7pgzm3jcg4toacy
+  printWidth: 120,
 }

.travis.yml

@@ -1,7 +1,6 @@
 language: node_js
 node_js:
-  #- stable # disable for now due to an issue of indirect dep upath with Node 9
+  - 12
-  - 8
 
 # Use containers.
 # http://docs.travis-ci.com/user/workers/container-based-infrastructure/

@vates/coalesce-calls/README.md

@@ -0,0 +1,46 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/coalesce-calls
+
+[![Package Version](https://badgen.net/npm/v/@vates/coalesce-calls)](https://npmjs.org/package/@vates/coalesce-calls) ![License](https://badgen.net/npm/license/@vates/coalesce-calls) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/coalesce-calls)](https://bundlephobia.com/result?p=@vates/coalesce-calls) [![Node compatibility](https://badgen.net/npm/node/@vates/coalesce-calls)](https://npmjs.org/package/@vates/coalesce-calls)
+
+> Wraps an async function so that concurrent calls will be coalesced
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/coalesce-calls):
+
+```
+> npm install --save @vates/coalesce-calls
+```
+
+## Usage
+
+```js
+import { coalesceCalls } from '@vates/coalesce-calls'
+
+const connect = coalesceCalls(async function () {
+  // async operation
+})
+
+connect()
+
+// the previous promise result will be returned if the operation is not
+// complete yet
+connect()
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/coalesce-calls/USAGE.md

@@ -0,0 +1,13 @@
+```js
+import { coalesceCalls } from '@vates/coalesce-calls'
+
+const connect = coalesceCalls(async function () {
+  // async operation
+})
+
+connect()
+
+// the previous promise result will be returned if the operation is not
+// complete yet
+connect()
+```

@vates/coalesce-calls/index.js

@@ -0,0 +1,14 @@
+exports.coalesceCalls = function (fn) {
+  let promise
+  const clean = () => {
+    promise = undefined
+  }
+  return function () {
+    if (promise !== undefined) {
+      return promise
+    }
+    promise = fn.apply(this, arguments)
+    promise.then(clean, clean)
+    return promise
+  }
+}

@vates/coalesce-calls/index.spec.js

@@ -0,0 +1,33 @@
+/* eslint-env jest */
+
+const { coalesceCalls } = require('./')
+
+const pDefer = () => {
+  const r = {}
+  r.promise = new Promise((resolve, reject) => {
+    r.reject = reject
+    r.resolve = resolve
+  })
+  return r
+}
+
+describe('coalesceCalls', () => {
+  it('decorates an async function', async () => {
+    const fn = coalesceCalls(promise => promise)
+
+    const defer1 = pDefer()
+    const promise1 = fn(defer1.promise)
+    const defer2 = pDefer()
+    const promise2 = fn(defer2.promise)
+
+    defer1.resolve('foo')
+    expect(await promise1).toBe('foo')
+    expect(await promise2).toBe('foo')
+
+    const defer3 = pDefer()
+    const promise3 = fn(defer3.promise)
+
+    defer3.resolve('bar')
+    expect(await promise3).toBe('bar')
+  })
+})

@vates/coalesce-calls/package.json

@@ -0,0 +1,38 @@
+{
+  "private": false,
+  "name": "@vates/coalesce-calls",
+  "description": "Wraps an async function so that concurrent calls will be coalesced",
+  "keywords": [
+    "async",
+    "calls",
+    "coalesce",
+    "decorate",
+    "decorator",
+    "merge",
+    "promise",
+    "wrap",
+    "wrapper"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/coalesce-calls",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/coalesce-calls",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "files": [
+    "index.js"
+  ],
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.1.0",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/decorate-with/README.md

@@ -0,0 +1,45 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/decorate-with
+
+[![Package Version](https://badgen.net/npm/v/@vates/decorate-with)](https://npmjs.org/package/@vates/decorate-with) ![License](https://badgen.net/npm/license/@vates/decorate-with) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/decorate-with)](https://bundlephobia.com/result?p=@vates/decorate-with) [![Node compatibility](https://badgen.net/npm/node/@vates/decorate-with)](https://npmjs.org/package/@vates/decorate-with)
+
+> Creates a decorator from a function wrapper
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/decorate-with):
+
+```
+> npm install --save @vates/decorate-with
+```
+
+## Usage
+
+For instance, allows using Lodash's functions as decorators:
+
+```js
+import { decorateWith } from '@vates/decorate-with'
+
+class Foo {
+  @decorateWith(lodash.debounce, 150)
+  bar() {
+    // body
+  }
+}
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/decorate-with/USAGE.md

@@ -0,0 +1,12 @@
+For instance, allows using Lodash's functions as decorators:
+
+```js
+import { decorateWith } from '@vates/decorate-with'
+
+class Foo {
+  @decorateWith(lodash.debounce, 150)
+  bar() {
+    // body
+  }
+}
+```

@vates/decorate-with/index.js

@@ -0,0 +1,4 @@
+exports.decorateWith = (fn, ...args) => (target, name, descriptor) => ({
+  ...descriptor,
+  value: fn(descriptor.value, ...args),
+})

@vates/decorate-with/package.json

@@ -0,0 +1,30 @@
+{
+  "private": false,
+  "name": "@vates/decorate-with",
+  "description": "Creates a decorator from a function wrapper",
+  "keywords": [
+    "apply",
+    "decorator",
+    "factory",
+    "wrapper"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/decorate-with",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/decorate-with",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.0.1",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/multi-key-map/README.md

@@ -0,0 +1,53 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/multi-key-map
+
+[![Package Version](https://badgen.net/npm/v/@vates/multi-key-map)](https://npmjs.org/package/@vates/multi-key-map) ![License](https://badgen.net/npm/license/@vates/multi-key-map) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/multi-key-map)](https://bundlephobia.com/result?p=@vates/multi-key-map) [![Node compatibility](https://badgen.net/npm/node/@vates/multi-key-map)](https://npmjs.org/package/@vates/multi-key-map)
+
+> Create map with values affected to multiple keys
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/multi-key-map):
+
+```
+> npm install --save @vates/multi-key-map
+```
+
+## Usage
+
+```js
+import { MultiKeyMap } from '@vates/multi-key-map'
+
+const map = new MultiKeyMap()
+
+const OBJ = {}
+map.set([], 0)
+map.set(['foo'], 1)
+map.set(['foo', 'bar'], 2)
+map.set(['bar', 'foo'], 3)
+map.set([OBJ], 4)
+map.set([{}], 5)
+
+map.get([]) // 0
+map.get(['foo']) // 1
+map.get(['foo', 'bar']) // 2
+map.get(['bar', 'foo']) // 3
+map.get([OBJ]) // 4
+map.get([{}]) // undefined
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/multi-key-map/USAGE.md

@@ -0,0 +1,20 @@
+```js
+import { MultiKeyMap } from '@vates/multi-key-map'
+
+const map = new MultiKeyMap()
+
+const OBJ = {}
+map.set([], 0)
+map.set(['foo'], 1)
+map.set(['foo', 'bar'], 2)
+map.set(['bar', 'foo'], 3)
+map.set([OBJ], 4)
+map.set([{}], 5)
+
+map.get([]) // 0
+map.get(['foo']) // 1
+map.get(['foo', 'bar']) // 2
+map.get(['bar', 'foo']) // 3
+map.get([OBJ]) // 4
+map.get([{}]) // undefined
+```

@vates/multi-key-map/index.js

@@ -67,7 +67,7 @@ function set(node, i, keys, value) {
   return node
 }
 
-export default class MultiKeyMap {
+exports.MultiKeyMap = class MultiKeyMap {
   constructor() {
     // each node is either a value or a Node if it contains children
     this._root = undefined

@vates/multi-key-map/index.spec.js

@@ -1,6 +1,6 @@
 /* eslint-env jest */
 
-import MultiKeyMap from './_MultiKeyMap'
+const { MultiKeyMap } = require('./')
 
 describe('MultiKeyMap', () => {
   it('works', () => {

@vates/multi-key-map/package.json

@@ -0,0 +1,28 @@
+{
+  "private": false,
+  "name": "@vates/multi-key-map",
+  "description": "Create map with values affected to multiple keys",
+  "keywords": [
+    "cache",
+    "map"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/multi-key-map",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/multi-key-map",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.1.0",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/parse-duration/README.md

@@ -0,0 +1,47 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/parse-duration
+
+[![Package Version](https://badgen.net/npm/v/@vates/parse-duration)](https://npmjs.org/package/@vates/parse-duration) ![License](https://badgen.net/npm/license/@vates/parse-duration) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/parse-duration)](https://bundlephobia.com/result?p=@vates/parse-duration) [![Node compatibility](https://badgen.net/npm/node/@vates/parse-duration)](https://npmjs.org/package/@vates/parse-duration)
+
+> Small wrapper around ms to parse a duration
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/parse-duration):
+
+```
+> npm install --save @vates/parse-duration
+```
+
+## Usage
+
+`ms` without magic: always parse a duration and throws if invalid.
+
+```js
+import { parseDuration } from '@vates/parse-duration'
+
+parseDuration('2 days')
+// 172800000
+
+parseDuration(172800000)
+// 172800000
+
+parseDuration(undefined)
+// throws TypeError('not a valid duration: undefined')
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

@vates/parse-duration/USAGE.md

@@ -0,0 +1,14 @@
+`ms` without magic: always parse a duration and throws if invalid.
+
+```js
+import { parseDuration } from '@vates/parse-duration'
+
+parseDuration('2 days')
+// 172800000
+
+parseDuration(172800000)
+// 172800000
+
+parseDuration(undefined)
+// throws TypeError('not a valid duration: undefined')
+```

@vates/parse-duration/index.js

@@ -1,6 +1,6 @@
-import ms from 'ms'
+const ms = require('ms')
 
-export default value => {
+exports.parseDuration = value => {
   if (typeof value === 'number') {
     return value
   }

@vates/parse-duration/package.json

@@ -0,0 +1,32 @@
+{
+  "private": false,
+  "name": "@vates/parse-duration",
+  "description": "Small wrapper around ms to parse a duration",
+  "keywords": [
+    "duration",
+    "ms",
+    "parse"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/parse-duration",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/parse-duration",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "AGPL-3.0-or-later",
+  "version": "0.1.0",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "dependencies": {
+    "ms": "^2.1.2"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/read-chunk/README.md

@@ -0,0 +1,46 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/read-chunk
+
+[![Package Version](https://badgen.net/npm/v/@vates/read-chunk)](https://npmjs.org/package/@vates/read-chunk) ![License](https://badgen.net/npm/license/@vates/read-chunk) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/read-chunk)](https://bundlephobia.com/result?p=@vates/read-chunk) [![Node compatibility](https://badgen.net/npm/node/@vates/read-chunk)](https://npmjs.org/package/@vates/read-chunk)
+
+> Read a chunk of a Node stream
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/read-chunk):
+
+```
+> npm install --save @vates/read-chunk
+```
+
+## Usage
+
+- returns the next available chunk of data
+- like `stream.read()`, a number of bytes can be specified
+- returns `null` if the stream has ended
+
+```js
+import { readChunk } from '@vates/read-chunk'
+;(async () => {
+  let chunk
+  while ((chunk = await readChunk(stream, 1024)) !== null) {
+    // do something with chunk
+  }
+})()
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/read-chunk/USAGE.md

@@ -0,0 +1,13 @@
+- returns the next available chunk of data
+- like `stream.read()`, a number of bytes can be specified
+- returns `null` if the stream has ended
+
+```js
+import { readChunk } from '@vates/read-chunk'
+;(async () => {
+  let chunk
+  while ((chunk = await readChunk(stream, 1024)) !== null) {
+    // do something with chunk
+  }
+})()
+```

@vates/read-chunk/index.js

@@ -0,0 +1,27 @@
+exports.readChunk = (stream, size) =>
+  new Promise((resolve, reject) => {
+    function onEnd() {
+      resolve(null)
+      removeListeners()
+    }
+    function onError(error) {
+      reject(error)
+      removeListeners()
+    }
+    function onReadable() {
+      const data = stream.read(size)
+      if (data !== null) {
+        resolve(data)
+        removeListeners()
+      }
+    }
+    function removeListeners() {
+      stream.removeListener('end', onEnd)
+      stream.removeListener('error', onError)
+      stream.removeListener('readable', onReadable)
+    }
+    stream.on('end', onEnd)
+    stream.on('error', onError)
+    stream.on('readable', onReadable)
+    onReadable()
+  })

@vates/read-chunk/package.json

@@ -0,0 +1,33 @@
+{
+  "private": false,
+  "name": "@vates/read-chunk",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/read-chunk",
+  "description": "Read a chunk of a Node stream",
+  "license": "ISC",
+  "keywords": [
+    "async",
+    "chunk",
+    "data",
+    "node",
+    "promise",
+    "read",
+    "stream"
+  ],
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/read-chunk",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "version": "0.1.1",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  }
+}

@xen-orchestra/async-map/.babelrc.js

@@ -1,3 +1 @@
-module.exports = require('../../@xen-orchestra/babel-config')(
+module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))
-  require('./package.json')
-)

@xen-orchestra/async-map/README.md

@@ -1,6 +1,10 @@
-# @xen-orchestra/async-map [![Build Status](https://travis-ci.org/vatesfr/xen-orchestra.png?branch=master)](https://travis-ci.org/vatesfr/xen-orchestra)
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
 
-> ${pkg.description}
+# @xen-orchestra/async-map
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/async-map)](https://npmjs.org/package/@xen-orchestra/async-map) ![License](https://badgen.net/npm/license/@xen-orchestra/async-map) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/async-map)](https://bundlephobia.com/result?p=@xen-orchestra/async-map) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/async-map)](https://npmjs.org/package/@xen-orchestra/async-map)
+
+> Similar to Promise.all + lodash.map but wait for all promises to be settled
 
 ## Install
 
@@ -12,30 +16,15 @@ Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/async
 
 ## Usage
 
-**TODO**
+```js
+import asyncMap from '@xen-orchestra/async-map'
 
-## Development
+const array = await asyncMap(collection, iteratee)
-
-```
-# Install dependencies
-> yarn
-
-# Run the tests
-> yarn test
-
-# Continuously compile
-> yarn dev
-
-# Continuously run the tests
-> yarn dev-test
-
-# Build for production (automatically called by npm install)
-> yarn build
 ```
 
 ## Contributions
 
-Contributions are *very* welcomed, either on the documentation or on
+Contributions are _very_ welcomed, either on the documentation or on
 the code.
 
 You may:
@@ -46,4 +35,4 @@ You may:
 
 ## License
 
-ISC © [Vates SAS](https://vates.fr)
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@xen-orchestra/async-map/USAGE.md

@@ -0,0 +1,5 @@
+```js
+import asyncMap from '@xen-orchestra/async-map'
+
+const array = await asyncMap(collection, iteratee)
+```

@xen-orchestra/async-map/package.json

@@ -1,8 +1,9 @@
 {
+  "private": false,
   "name": "@xen-orchestra/async-map",
   "version": "0.0.0",
   "license": "ISC",
-  "description": "",
+  "description": "Similar to Promise.all + lodash.map but wait for all promises to be settled",
   "keywords": [],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/async-map",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
@@ -12,8 +13,8 @@
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
   "author": {
-    "name": "Julien Fontanet",
+    "name": "Vates SAS",
-    "email": "julien.fontanet@isonoe.net"
+    "url": "https://vates.fr"
   },
   "preferGlobal": false,
   "main": "dist/",
@@ -36,7 +37,7 @@
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
     "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^6.0.3",
+    "cross-env": "^7.0.2",
     "rimraf": "^3.0.0"
   },
   "scripts": {
@@ -45,7 +46,6 @@
     "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
     "prebuild": "yarn run clean",
     "predev": "yarn run prebuild",
-    "prepare": "yarn run build",
     "prepublishOnly": "yarn run build",
     "postversion": "npm publish"
   }

@xen-orchestra/audit-core/.babelrc.js

@@ -0,0 +1 @@
+module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))

@xen-orchestra/audit-core/.npmignore

@@ -0,0 +1,24 @@
+/benchmark/
+/benchmarks/
+*.bench.js
+*.bench.js.map
+
+/examples/
+example.js
+example.js.map
+*.example.js
+*.example.js.map
+
+/fixture/
+/fixtures/
+*.fixture.js
+*.fixture.js.map
+*.fixtures.js
+*.fixtures.js.map
+
+/test/
+/tests/
+*.spec.js
+*.spec.js.map
+
+__snapshots__/

@xen-orchestra/audit-core/README.md

@@ -0,0 +1,28 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @xen-orchestra/audit-core
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/audit-core)](https://npmjs.org/package/@xen-orchestra/audit-core) ![License](https://badgen.net/npm/license/@xen-orchestra/audit-core) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/audit-core)](https://bundlephobia.com/result?p=@xen-orchestra/audit-core) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/audit-core)](https://npmjs.org/package/@xen-orchestra/audit-core)
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/audit-core):
+
+```
+> npm install --save @xen-orchestra/audit-core
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

@xen-orchestra/audit-core/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@xen-orchestra/audit-core",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/audit-core",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@xen-orchestra/audit-core",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "version": "0.2.0",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "main": "dist/",
+  "scripts": {
+    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
+    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
+    "postversion": "npm publish --access public",
+    "prebuild": "rimraf dist/",
+    "predev": "yarn run prebuild",
+    "prepublishOnly": "yarn run build"
+  },
+  "devDependencies": {
+    "@babel/cli": "^7.7.4",
+    "@babel/core": "^7.7.4",
+    "@babel/plugin-proposal-decorators": "^7.8.0",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.0",
+    "@babel/preset-env": "^7.7.4",
+    "cross": "^1.0.0",
+    "rimraf": "^3.0.0"
+  },
+  "dependencies": {
+    "@xen-orchestra/log": "^0.2.0",
+    "core-js": "^3.6.4",
+    "golike-defer": "^0.4.1",
+    "lodash": "^4.17.15",
+    "object-hash": "^2.0.1"
+  },
+  "private": false,
+  "license": "AGPL-3.0-or-later",
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  }
+}

@xen-orchestra/audit-core/src/index.js

@@ -0,0 +1,193 @@
+// see https://github.com/babel/babel/issues/8450
+import 'core-js/features/symbol/async-iterator'
+
+import assert from 'assert'
+import createLogger from '@xen-orchestra/log'
+import defer from 'golike-defer'
+import hash from 'object-hash'
+
+const log = createLogger('xo:audit-core')
+
+export class Storage {
+  constructor() {
+    this._lock = Promise.resolve()
+  }
+
+  async acquireLock() {
+    const lock = this._lock
+    let releaseLock
+    this._lock = new Promise(resolve => {
+      releaseLock = resolve
+    })
+    await lock
+    return releaseLock
+  }
+}
+
+// Format: $<algorithm>$<salt>$<encrypted>
+//
+// http://man7.org/linux/man-pages/man3/crypt.3.html#NOTES
+const ID_TO_ALGORITHM = {
+  5: 'sha256',
+}
+
+export class AlteredRecordError extends Error {
+  constructor(id, nValid, record) {
+    super('altered record')
+
+    this.id = id
+    this.nValid = nValid
+    this.record = record
+  }
+}
+
+export class MissingRecordError extends Error {
+  constructor(id, nValid) {
+    super('missing record')
+
+    this.id = id
+    this.nValid = nValid
+  }
+}
+
+export const NULL_ID = 'nullId'
+
+const HASH_ALGORITHM_ID = '5'
+const createHash = (data, algorithmId = HASH_ALGORITHM_ID) =>
+  `$${algorithmId}$$${hash(data, {
+    algorithm: ID_TO_ALGORITHM[algorithmId],
+    excludeKeys: key => key === 'id',
+  })}`
+
+export class AuditCore {
+  constructor(storage) {
+    assert.notStrictEqual(storage, undefined)
+    this._storage = storage
+  }
+
+  @defer
+  async add($defer, subject, event, data) {
+    const time = Date.now()
+    $defer(await this._storage.acquireLock())
+    return this._addUnsafe({
+      data,
+      event,
+      subject,
+      time,
+    })
+  }
+
+  async _addUnsafe({ data, event, subject, time }) {
+    const storage = this._storage
+
+    // delete "undefined" properties and normalize data with JSON.stringify
+    const record = JSON.parse(
+      JSON.stringify({
+        data,
+        event,
+        previousId: (await storage.getLastId()) ?? NULL_ID,
+        subject,
+        time,
+      })
+    )
+    record.id = createHash(record)
+    await storage.put(record)
+    await storage.setLastId(record.id)
+    return record
+  }
+
+  async checkIntegrity(oldest, newest) {
+    const storage = this._storage
+
+    // handle separated chains case
+    if (newest !== (await storage.getLastId())) {
+      let isNewestAccessible = false
+      for await (const { id } of this.getFrom()) {
+        if (id === newest) {
+          isNewestAccessible = true
+          break
+        }
+      }
+      if (!isNewestAccessible) {
+        throw new MissingRecordError(newest, 0)
+      }
+    }
+
+    let nValid = 0
+    while (newest !== oldest) {
+      const record = await storage.get(newest)
+      if (record === undefined) {
+        throw new MissingRecordError(newest, nValid)
+      }
+      if (newest !== createHash(record, newest.slice(1, newest.indexOf('$', 1)))) {
+        throw new AlteredRecordError(newest, nValid, record)
+      }
+      newest = record.previousId
+      nValid++
+    }
+    return nValid
+  }
+
+  async *getFrom(newest) {
+    const storage = this._storage
+
+    let id = newest ?? (await storage.getLastId())
+    if (id === undefined) {
+      return
+    }
+
+    let record
+    while ((record = await storage.get(id)) !== undefined) {
+      yield record
+      id = record.previousId
+    }
+  }
+
+  async deleteFrom(newest) {
+    assert.notStrictEqual(newest, undefined)
+    for await (const { id } of this.getFrom(newest)) {
+      await this._storage.del(id)
+    }
+  }
+
+  @defer
+  async deleteRangeAndRewrite($defer, newest, oldest) {
+    assert.notStrictEqual(newest, undefined)
+    assert.notStrictEqual(oldest, undefined)
+
+    const storage = this._storage
+    $defer(await storage.acquireLock())
+
+    assert.notStrictEqual(await storage.get(newest), undefined)
+    const oldestRecord = await storage.get(oldest)
+    assert.notStrictEqual(oldestRecord, undefined)
+
+    const lastId = await storage.getLastId()
+    const recentRecords = []
+    for await (const record of this.getFrom(lastId)) {
+      if (record.id === newest) {
+        break
+      }
+
+      recentRecords.push(record)
+    }
+
+    for await (const record of this.getFrom(newest)) {
+      await storage.del(record.id)
+      if (record.id === oldest) {
+        break
+      }
+    }
+
+    await storage.setLastId(oldestRecord.previousId)
+
+    for (const record of recentRecords) {
+      try {
+        await this._addUnsafe(record)
+        await storage.del(record.id)
+      } catch (error) {
+        log.error(error)
+      }
+    }
+  }
+}

@xen-orchestra/audit-core/src/index.spec.js

@@ -0,0 +1,115 @@
+/* eslint-env jest */
+
+import { AlteredRecordError, AuditCore, MissingRecordError, NULL_ID, Storage } from '.'
+
+const asyncIteratorToArray = async asyncIterator => {
+  const array = []
+  for await (const entry of asyncIterator) {
+    array.push(entry)
+  }
+  return array
+}
+
+class DB extends Storage {
+  constructor() {
+    super()
+
+    this._db = new Map()
+    this._lastId = undefined
+  }
+
+  async put(record) {
+    this._db.set(record.id, record)
+  }
+
+  async setLastId(id) {
+    this._lastId = id
+  }
+
+  async getLastId() {
+    return this._lastId
+  }
+
+  async del(id) {
+    this._db.delete(id)
+  }
+
+  async get(id) {
+    return this._db.get(id)
+  }
+
+  _clear() {
+    return this._db.clear()
+  }
+}
+
+const DATA = [
+  [
+    {
+      name: 'subject0',
+    },
+    'event0',
+    {},
+  ],
+  [
+    {
+      name: 'subject1',
+    },
+    'event1',
+    {},
+  ],
+  [
+    {
+      name: 'subject2',
+    },
+    'event2',
+    {},
+  ],
+]
+
+const db = new DB()
+const auditCore = new AuditCore(db)
+const storeAuditRecords = async () => {
+  await Promise.all(DATA.map(data => auditCore.add(...data)))
+  const records = await asyncIteratorToArray(auditCore.getFrom())
+  expect(records.length).toBe(DATA.length)
+  return records
+}
+
+describe('auditCore', () => {
+  afterEach(() => db._clear())
+
+  it('detects that a record is missing', async () => {
+    const [newestRecord, deletedRecord] = await storeAuditRecords()
+
+    const nValidRecords = await auditCore.checkIntegrity(NULL_ID, newestRecord.id)
+    expect(nValidRecords).toBe(DATA.length)
+
+    await db.del(deletedRecord.id)
+    await expect(auditCore.checkIntegrity(NULL_ID, newestRecord.id)).rejects.toEqual(
+      new MissingRecordError(deletedRecord.id, 1)
+    )
+  })
+
+  it('detects that a record has been altered', async () => {
+    const [newestRecord, alteredRecord] = await storeAuditRecords()
+
+    alteredRecord.event = ''
+    await db.put(alteredRecord)
+
+    await expect(auditCore.checkIntegrity(NULL_ID, newestRecord.id)).rejects.toEqual(
+      new AlteredRecordError(alteredRecord.id, 1, alteredRecord)
+    )
+  })
+
+  it('confirms interval integrity after deletion of records outside of the interval', async () => {
+    const [thirdRecord, secondRecord, firstRecord] = await storeAuditRecords()
+
+    await auditCore.deleteFrom(secondRecord.id)
+
+    expect(await db.get(firstRecord.id)).toBe(undefined)
+    expect(await db.get(secondRecord.id)).toBe(undefined)
+
+    await auditCore.checkIntegrity(secondRecord.id, thirdRecord.id)
+  })
+})

@xen-orchestra/audit-core/src/specification.ts

@@ -0,0 +1,26 @@
+class Storage {
+  acquire: () => Promise<() => undefined>
+  del: (id: string) => Promise<void>
+  get: (id: string) => Promise<Record | void>
+  getLastId: () => Promise<string | void>
+  put: (record: Record) => Promise<void>
+  setLastId: (id: string) => Promise<void>
+}
+
+interface Record {
+  data: object
+  event: string
+  id: string
+  previousId: string
+  subject: object
+  time: number
+}
+
+export class AuditCore {
+  constructor(storage: Storage) { }
+  public add(subject: any, event: string, data: any): Promise<Record> { }
+  public checkIntegrity(oldest: string, newest: string): Promise<number> { }
+  public getFrom(newest?: string): AsyncIterator { }
+  public deleteFrom(newest: string): Promise<void> { }
+  public deleteRangeAndRewrite(newest: string, oldest: string): Promise<void> { }
+}

@xen-orchestra/babel-config/README.md

@@ -0,0 +1,18 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @xen-orchestra/babel-config
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

@xen-orchestra/babel-config/index.js

@@ -32,27 +32,19 @@ const configs = {
         }
         return { browsers: pkg.browserslist, node }
       })(),
-      useBuiltIns: '@babel/polyfill' in (pkg.dependencies || {}) && 'usage',
     }
   },
 }
 
 const getConfig = (key, ...args) => {
   const config = configs[key]
-  return config === undefined
+  return config === undefined ? {} : typeof config === 'function' ? config(...args) : config
-    ? {}
-    : typeof config === 'function'
-    ? config(...args)
-    : config
 }
 
 // some plugins must be used in a specific order
-const pluginsOrder = [
+const pluginsOrder = ['@babel/plugin-proposal-decorators', '@babel/plugin-proposal-class-properties']
-  '@babel/plugin-proposal-decorators',
-  '@babel/plugin-proposal-class-properties',
-]
 
-module.exports = function(pkg, plugins, presets) {
+module.exports = function (pkg, plugins, presets) {
   plugins === undefined && (plugins = {})
   presets === undefined && (presets = {})
 

@xen-orchestra/babel-config/package.json

@@ -11,5 +11,10 @@
   },
   "engines": {
     "node": ">=6"
+  },
+  "license": "AGPL-3.0-or-later",
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
   }
 }

@xen-orchestra/backups-cli/README.md

@@ -0,0 +1,28 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @xen-orchestra/backups-cli
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/backups-cli)](https://npmjs.org/package/@xen-orchestra/backups-cli) ![License](https://badgen.net/npm/license/@xen-orchestra/backups-cli) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/backups-cli)](https://bundlephobia.com/result?p=@xen-orchestra/backups-cli) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/backups-cli)](https://npmjs.org/package/@xen-orchestra/backups-cli)
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/backups-cli):
+
+```
+> npm install --global @xen-orchestra/backups-cli
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

@xen-orchestra/backups-cli/_asyncMap.js

@@ -0,0 +1,5 @@
+const curryRight = require('lodash/curryRight')
+
+module.exports = curryRight((iterable, fn) =>
+  Promise.all(Array.isArray(iterable) ? iterable.map(fn) : Array.from(iterable, fn))
+)

@xen-orchestra/backups-cli/_composeCommands.js

@@ -3,7 +3,7 @@ const getopts = require('getopts')
 const { version } = require('./package.json')
 
 module.exports = commands =>
-  async function(args, prefix) {
+  async function (args, prefix) {
     const opts = getopts(args, {
       alias: {
         help: 'h',

@xen-orchestra/backups-cli/_fs.js

@@ -0,0 +1,69 @@
+const { dirname } = require('path')
+
+const fs = require('promise-toolbox/promisifyAll')(require('fs'))
+module.exports = fs
+
+fs.getSize = path =>
+  fs.stat(path).then(
+    _ => _.size,
+    error => {
+      if (error.code === 'ENOENT') {
+        return 0
+      }
+      throw error
+    }
+  )
+
+fs.mktree = async function mkdirp(path) {
+  try {
+    await fs.mkdir(path)
+  } catch (error) {
+    const { code } = error
+    if (code === 'EEXIST') {
+      await fs.readdir(path)
+      return
+    }
+    if (code === 'ENOENT') {
+      await mkdirp(dirname(path))
+      return mkdirp(path)
+    }
+    throw error
+  }
+}
+
+// - easier:
+//   - single param for direct use in `Array#map`
+//   - files are prefixed with directory path
+// - safer: returns empty array if path is missing or not a directory
+fs.readdir2 = path =>
+  fs.readdir(path).then(
+    entries => {
+      entries.forEach((entry, i) => {
+        entries[i] = `${path}/${entry}`
+      })
+
+      return entries
+    },
+    error => {
+      const { code } = error
+      if (code === 'ENOENT') {
+        // do nothing
+      } else if (code === 'ENOTDIR') {
+        console.warn('WARN: readdir(%s)', path, error)
+      } else {
+        throw error
+      }
+      return []
+    }
+  )
+
+fs.symlink2 = async (target, path) => {
+  try {
+    await fs.symlink(target, path)
+  } catch (error) {
+    if (error.code === 'EEXIST' && (await fs.readlink(path)) === target) {
+      return
+    }
+    throw error
+  }
+}

@xen-orchestra/backups-cli/commands/clean-vms.js

@@ -1,122 +1,36 @@
 #!/usr/bin/env node
 
 // assigned when options are parsed by the main function
-let force
+let merge, remove
 
 // -----------------------------------------------------------------------------
 
 const assert = require('assert')
+const flatten = require('lodash/flatten')
 const getopts = require('getopts')
+const limitConcurrency = require('limit-concurrency-decorator').default
 const lockfile = require('proper-lockfile')
-const { default: Vhd } = require('vhd-lib')
+const pipe = require('promise-toolbox/pipe')
-const { curryRight, flatten } = require('lodash')
+const { default: Vhd, mergeVhd } = require('vhd-lib')
 const { dirname, resolve } = require('path')
 const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants')
-const { pipe, promisifyAll } = require('promise-toolbox')
+const { isValidXva } = require('@xen-orchestra/backups/isValidXva')
+
+const asyncMap = require('../_asyncMap')
+const fs = require('../_fs')
 
-const fs = promisifyAll(require('fs'))
 const handler = require('@xen-orchestra/fs').getHandler({ url: 'file://' })
 
 // -----------------------------------------------------------------------------
 
-const asyncMap = curryRight((iterable, fn) =>
-  Promise.all(
-    Array.isArray(iterable) ? iterable.map(fn) : Array.from(iterable, fn)
-  )
-)
-
-const filter = (...args) => thisArg => thisArg.filter(...args)
-
-const isGzipFile = async fd => {
-  // https://tools.ietf.org/html/rfc1952.html#page-5
-  const magicNumber = Buffer.allocUnsafe(2)
-  assert.strictEqual(
-    await fs.read(fd, magicNumber, 0, magicNumber.length, 0),
-    magicNumber.length
-  )
-  return magicNumber[0] === 31 && magicNumber[1] === 139
-}
-
-// TODO: better check?
-//
-// our heuristic is not good enough, there has been some false positives
-// (detected as invalid by us but valid by `tar` and imported with success),
-// either THOUGH THEY MAY HAVE BEEN COMPRESSED FILES:
-// - these files were normal but the check is incorrect
-// - these files were invalid but without data loss
-// - these files were invalid but with silent data loss
-//
-// maybe reading the end of the file looking for a file named
-// /^Ref:\d+/\d+\.checksum$/ and then validating the tar structure from it
-//
-// https://github.com/npm/node-tar/issues/234#issuecomment-538190295
-const isValidTar = async (size, fd) => {
-  if (size <= 1024 || size % 512 !== 0) {
-    return false
-  }
-
-  const buf = Buffer.allocUnsafe(1024)
-  assert.strictEqual(
-    await fs.read(fd, buf, 0, buf.length, size - buf.length),
-    buf.length
-  )
-  return buf.every(_ => _ === 0)
-}
-
-// TODO: find an heuristic for compressed files
-const isValidXva = async path => {
-  try {
-    const fd = await fs.open(path, 'r')
-    try {
-      const { size } = await fs.fstat(fd)
-      if (size < 20) {
-        // neither a valid gzip not tar
-        return false
-      }
-
-      return (await isGzipFile(fd))
-        ? true // gzip files cannot be validated at this time
-        : await isValidTar(size, fd)
-    } finally {
-      fs.close(fd).catch(noop)
-    }
-  } catch (error) {
-    // never throw, log and report as valid to avoid side effects
-    console.error('isValidXva', path, error)
-    return true
-  }
-}
-
-const noop = Function.prototype
-
-const readDir = path =>
-  fs.readdir(path).then(
-    entries => {
-      entries.forEach((entry, i) => {
-        entries[i] = `${path}/${entry}`
-      })
-
-      return entries
-    },
-    error => {
-      // a missing dir is by definition empty
-      if (error != null && error.code === 'ENOENT') {
-        return []
-      }
-      throw error
-    }
-  )
-
-// -----------------------------------------------------------------------------
-
 // chain is an array of VHDs from child to parent
 //
 // the whole chain will be merged into parent, parent will be renamed to child
 // and all the others will deleted
-async function mergeVhdChain(chain) {
+const mergeVhdChain = limitConcurrency(1)(async function mergeVhdChain(chain) {
   assert(chain.length >= 2)
 
-  const child = chain[0]
+  let child = chain[0]
   const parent = chain[chain.length - 1]
   const children = chain.slice(0, -1).reverse()
 
@@ -127,42 +41,63 @@ async function mergeVhdChain(chain) {
     .forEach(parent => {
       console.warn('  ', parent)
     })
-  force && console.warn('  merging…')
+  merge && console.warn('  merging…')
   console.warn('')
-  if (force) {
+  if (merge) {
     // `mergeVhd` does not work with a stream, either
     // - make it accept a stream
     // - or create synthetic VHD which is not a stream
-    return console.warn('TODO: implement merge')
+    if (children.length !== 1) {
-    // await mergeVhd(
+      console.warn('TODO: implement merging multiple children')
-    //   handler,
+      children.length = 1
-    //   parent,
+      child = children[0]
-    //   handler,
+    }
-    //   children.length === 1
+
-    //     ? child
+    let done, total
-    //     : await createSyntheticStream(handler, children)
+    const handle = setInterval(() => {
-    // )
+      if (done !== undefined) {
+        console.log('merging %s: %s/%s', child, done, total)
+      }
+    }, 10e3)
+
+    await mergeVhd(
+      handler,
+      parent,
+      handler,
+      child,
+      // children.length === 1
+      //   ? child
+      //   : await createSyntheticStream(handler, children),
+      {
+        onProgress({ done: d, total: t }) {
+          done = d
+          total = t
+        },
+      }
+    )
+
+    clearInterval(handle)
   }
 
   await Promise.all([
-    force && fs.rename(parent, child),
+    remove && fs.rename(parent, child),
     asyncMap(children.slice(0, -1), child => {
       console.warn('Unused VHD', child)
-      force && console.warn('  deleting…')
+      remove && console.warn('  deleting…')
       console.warn('')
-      return force && handler.unlink(child)
+      return remove && handler.unlink(child)
     }),
   ])
-}
+})
 
 const listVhds = pipe([
   vmDir => vmDir + '/vdis',
-  readDir,
+  fs.readdir2,
-  asyncMap(readDir),
+  asyncMap(fs.readdir2),
   flatten,
-  asyncMap(readDir),
+  asyncMap(fs.readdir2),
   flatten,
-  filter(_ => _.endsWith('.vhd')),
+  _ => _.filter(_ => _.endsWith('.vhd')),
 ])
 
 async function handleVm(vmDir) {
@@ -180,9 +115,7 @@ async function handleVm(vmDir) {
         const parent = resolve(dirname(path), vhd.header.parentUnicodeName)
         vhdParents[path] = parent
         if (parent in vhdChildren) {
-          const error = new Error(
+          const error = new Error('this script does not support multiple VHD children')
-            'this script does not support multiple VHD children'
-          )
           error.parent = parent
           error.child1 = vhdChildren[parent]
           error.child2 = path
@@ -194,9 +127,9 @@ async function handleVm(vmDir) {
       console.warn('Error while checking VHD', path)
       console.warn('  ', error)
       if (error != null && error.code === 'ERR_ASSERTION') {
-        force && console.warn('  deleting…')
+        remove && console.warn('  deleting…')
         console.warn('')
-        force && (await handler.unlink(path))
+        remove && (await handler.unlink(path))
       }
     }
   })
@@ -222,9 +155,9 @@ async function handleVm(vmDir) {
 
         console.warn('Error while checking VHD', vhd)
         console.warn('  missing parent', parent)
-        force && console.warn('  deleting…')
+        remove && console.warn('  deleting…')
         console.warn('')
-        force && deletions.push(handler.unlink(vhd))
+        remove && deletions.push(handler.unlink(vhd))
       }
     }
 
@@ -239,10 +172,13 @@ async function handleVm(vmDir) {
     await Promise.all(deletions)
   }
 
-  const [jsons, xvas] = await readDir(vmDir).then(entries => [
+  const [jsons, xvas, xvaSums] = await fs
-    entries.filter(_ => _.endsWith('.json')),
+    .readdir2(vmDir)
-    new Set(entries.filter(_ => _.endsWith('.xva'))),
+    .then(entries => [
-  ])
+      entries.filter(_ => _.endsWith('.json')),
+      new Set(entries.filter(_ => _.endsWith('.xva'))),
+      entries.filter(_ => _.endsWith('.xva.cheksum')),
+    ])
 
   await asyncMap(xvas, async path => {
     // check is not good enough to delete the file, the best we can do is report
@@ -269,9 +205,9 @@ async function handleVm(vmDir) {
       } else {
         console.warn('Error while checking backup', json)
         console.warn('  missing file', linkedXva)
-        force && console.warn('  deleting…')
+        remove && console.warn('  deleting…')
         console.warn('')
-        force && (await handler.unlink(json))
+        remove && (await handler.unlink(json))
       }
     } else if (mode === 'delta') {
       const linkedVhds = (() => {
@@ -286,17 +222,13 @@ async function handleVm(vmDir) {
       } else {
         console.warn('Error while checking backup', json)
         const missingVhds = linkedVhds.filter(_ => !vhds.has(_))
-        console.warn(
+        console.warn('  %i/%i missing VHDs', missingVhds.length, linkedVhds.length)
-          '  %i/%i missing VHDs',
-          missingVhds.length,
-          linkedVhds.length
-        )
         missingVhds.forEach(vhd => {
           console.warn('  ', vhd)
         })
-        force && console.warn('  deleting…')
+        remove && console.warn('  deleting…')
         console.warn('')
-        force && (await handler.unlink(json))
+        remove && (await handler.unlink(json))
       }
     }
   })
@@ -334,9 +266,9 @@ async function handleVm(vmDir) {
       }
 
       console.warn('Unused VHD', vhd)
-      force && console.warn('  deleting…')
+      remove && console.warn('  deleting…')
       console.warn('')
-      force && unusedVhdsDeletion.push(handler.unlink(vhd))
+      remove && unusedVhdsDeletion.push(handler.unlink(vhd))
     }
 
     toCheck.forEach(vhd => {
@@ -355,9 +287,18 @@ async function handleVm(vmDir) {
     unusedVhdsDeletion,
     asyncMap(unusedXvas, path => {
       console.warn('Unused XVA', path)
-      force && console.warn('  deleting…')
+      remove && console.warn('  deleting…')
       console.warn('')
-      return force && handler.unlink(path)
+      return remove && handler.unlink(path)
+    }),
+    asyncMap(xvaSums, path => {
+      // no need to handle checksums for XVAs deleted by the script, they will be handled by `unlink()`
+      if (!xvas.has(path.slice(0, -'.checksum'.length))) {
+        console.warn('Unused XVA checksum', path)
+        remove && console.warn('  deleting…')
+        console.warn('')
+        return remove && handler.unlink(path)
+      }
     }),
   ])
 }
@@ -367,15 +308,17 @@ async function handleVm(vmDir) {
 module.exports = async function main(args) {
   const opts = getopts(args, {
     alias: {
-      force: 'f',
+      remove: 'r',
+      merge: 'm',
     },
-    boolean: ['force'],
+    boolean: ['merge', 'remove'],
     default: {
-      force: false,
+      merge: false,
+      remove: false,
     },
   })
 
-  ;({ force } = opts)
+  ;({ remove, merge } = opts)
   await asyncMap(opts._, async vmDir => {
     vmDir = resolve(vmDir)
 

@xen-orchestra/backups-cli/commands/create-symlink-index.js

@@ -0,0 +1,28 @@
+const filenamify = require('filenamify')
+const get = require('lodash/get')
+const { dirname, join, relative } = require('path')
+
+const asyncMap = require('../_asyncMap')
+const { mktree, readdir2, readFile, symlink2 } = require('../_fs')
+
+module.exports = async function createSymlinkIndex([backupDir, fieldPath]) {
+  const indexDir = join(backupDir, 'indexes', filenamify(fieldPath))
+  await mktree(indexDir)
+
+  await asyncMap(await readdir2(backupDir), async vmDir =>
+    asyncMap(
+      (await readdir2(vmDir)).filter(_ => _.endsWith('.json')),
+      async json => {
+        const metadata = JSON.parse(await readFile(json))
+        const value = get(metadata, fieldPath)
+        if (value !== undefined) {
+          const target = relative(indexDir, dirname(json))
+          const path = join(indexDir, filenamify(String(value)))
+          await symlink2(target, path).catch(error => {
+            console.warn('symlink(%s, %s)', target, path, error)
+          })
+        }
+      }
+    )
+  )
+}

@xen-orchestra/backups-cli/commands/info.js

@@ -0,0 +1,54 @@
+const groupBy = require('lodash/groupBy')
+const { createHash } = require('crypto')
+const { dirname, resolve } = require('path')
+
+const asyncMap = require('../_asyncMap')
+const { readdir2, readFile, getSize } = require('../_fs')
+
+const sha512 = str => createHash('sha512').update(str).digest('hex')
+const sum = values => values.reduce((a, b) => a + b)
+
+module.exports = async function info(vmDirs) {
+  const jsonFiles = (
+    await asyncMap(vmDirs, async vmDir => (await readdir2(vmDir)).filter(_ => _.endsWith('.json')))
+  ).flat()
+
+  const hashes = { __proto__: null }
+
+  const info = (
+    await asyncMap(jsonFiles, async jsonFile => {
+      try {
+        const jsonDir = dirname(jsonFile)
+        const json = await readFile(jsonFile)
+
+        const hash = sha512(json)
+        if (hash in hashes) {
+          console.log(jsonFile, 'duplicate of', hashes[hash])
+          return
+        }
+        hashes[hash] = jsonFile
+
+        const metadata = JSON.parse(json)
+
+        return {
+          jsonDir,
+          jsonFile,
+          metadata,
+          size:
+            json.length +
+            (await (metadata.mode === 'delta'
+              ? asyncMap(Object.values(metadata.vhds), _ => getSize(resolve(jsonDir, _))).then(sum)
+              : getSize(resolve(jsonDir, metadata.xva)))),
+        }
+      } catch (error) {
+        console.error(jsonFile, error)
+      }
+    })
+  ).filter(_ => _ !== undefined)
+  const byJobs = groupBy(info, 'metadata.jobId')
+  Object.keys(byJobs)
+    .sort()
+    .forEach(jobId => {
+      console.log(jobId, sum(byJobs[jobId].map(_ => _.size)))
+    })
+}

@xen-orchestra/backups-cli/index.js

@@ -5,7 +5,19 @@ require('./_composeCommands')({
     get main() {
       return require('./commands/clean-vms')
     },
-    usage: '[--force] xo-vm-backups/*',
+    usage: '[--force] [--merge] xo-vm-backups/*',
+  },
+  'create-symlink-index': {
+    get main() {
+      return require('./commands/create-symlink-index')
+    },
+    usage: 'xo-vm-backups <field path>',
+  },
+  info: {
+    get main() {
+      return require('./commands/info')
+    },
+    usage: 'xo-vm-backups/*',
   },
 })(process.argv.slice(2), 'xo-backups').catch(error => {
   console.error('main', error)

@xen-orchestra/backups-cli/package.json

@@ -1,19 +1,28 @@
 {
+  "private": false,
   "bin": {
     "xo-backups": "index.js"
   },
+  "preferGlobal": true,
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
-    "@xen-orchestra/fs": "^0.10.2",
+    "@xen-orchestra/backups": "^0.1.1",
+    "@xen-orchestra/fs": "^0.12.1",
+    "filenamify": "^4.1.0",
     "getopts": "^2.2.5",
+    "limit-concurrency-decorator": "^0.4.0",
     "lodash": "^4.17.15",
-    "promise-toolbox": "^0.14.0",
+    "promise-toolbox": "^0.16.0",
     "proper-lockfile": "^4.1.1",
-    "vhd-lib": "^0.7.2"
+    "vhd-lib": "^1.0.0"
   },
   "engines": {
     "node": ">=7.10.1"
   },
+  "files": [
+    "commands",
+    "*.js"
+  ],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/backups-cli",
   "name": "@xen-orchestra/backups-cli",
   "repository": {
@@ -24,5 +33,10 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "0.0.0"
+  "version": "0.3.0",
+  "license": "AGPL-3.0-or-later",
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  }
 }

@xen-orchestra/backups/README.md

@@ -0,0 +1,28 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @xen-orchestra/backups
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/backups)](https://npmjs.org/package/@xen-orchestra/backups) ![License](https://badgen.net/npm/license/@xen-orchestra/backups) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/backups)](https://bundlephobia.com/result?p=@xen-orchestra/backups) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/backups)](https://npmjs.org/package/@xen-orchestra/backups)
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/backups):
+
+```
+> npm install --save @xen-orchestra/backups
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

@xen-orchestra/backups/extractIdsFromSimplePattern.js

@@ -0,0 +1,30 @@
+function extractIdsFromSimplePattern(pattern) {
+  if (pattern === undefined) {
+    return []
+  }
+
+  if (pattern !== null && typeof pattern === 'object') {
+    let keys = Object.keys(pattern)
+
+    if (keys.length === 1 && keys[0] === 'id') {
+      pattern = pattern.id
+      if (typeof pattern === 'string') {
+        return [pattern]
+      }
+      if (pattern !== null && typeof pattern === 'object') {
+        keys = Object.keys(pattern)
+        if (
+          keys.length === 1 &&
+          keys[0] === '__or' &&
+          Array.isArray((pattern = pattern.__or)) &&
+          pattern.every(_ => typeof _ === 'string')
+        ) {
+          return pattern
+        }
+      }
+    }
+  }
+
+  throw new Error('invalid pattern')
+}
+exports.extractIdsFromSimplePattern = extractIdsFromSimplePattern

@xen-orchestra/backups/filenameDate.js

@@ -0,0 +1,6 @@
+const { utcFormat, utcParse } = require('d3-time-format')
+
+// Format a date in ISO 8601 in a safe way to be used in filenames
+// (even on Windows).
+exports.formatFilenameDate = utcFormat('%Y%m%dT%H%M%SZ')
+exports.parseFilenameDate = utcParse('%Y%m%dT%H%M%SZ')

@xen-orchestra/backups/getOldEntries.js

@@ -0,0 +1,3 @@
+// returns all entries but the last retention-th
+exports.getOldEntries = (retention, entries) =>
+  entries === undefined ? [] : retention > 0 ? entries.slice(0, -retention) : entries

@xen-orchestra/backups/isValidXva.js

@@ -0,0 +1,59 @@
+const assert = require('assert')
+const fs = require('fs-extra')
+
+const isGzipFile = async fd => {
+  // https://tools.ietf.org/html/rfc1952.html#page-5
+  const magicNumber = Buffer.allocUnsafe(2)
+  assert.strictEqual((await fs.read(fd, magicNumber, 0, magicNumber.length, 0)).bytesRead, magicNumber.length)
+  return magicNumber[0] === 31 && magicNumber[1] === 139
+}
+
+// TODO: better check?
+//
+// our heuristic is not good enough, there has been some false positives
+// (detected as invalid by us but valid by `tar` and imported with success),
+// either THOUGH THEY MAY HAVE BEEN COMPRESSED FILES:
+// - these files were normal but the check is incorrect
+// - these files were invalid but without data loss
+// - these files were invalid but with silent data loss
+//
+// maybe reading the end of the file looking for a file named
+// /^Ref:\d+/\d+\.checksum$/ and then validating the tar structure from it
+//
+// https://github.com/npm/node-tar/issues/234#issuecomment-538190295
+const isValidTar = async (size, fd) => {
+  if (size <= 1024 || size % 512 !== 0) {
+    return false
+  }
+
+  const buf = Buffer.allocUnsafe(1024)
+  assert.strictEqual((await fs.read(fd, buf, 0, buf.length, size - buf.length)).bytesRead, buf.length)
+  return buf.every(_ => _ === 0)
+}
+
+// TODO: find an heuristic for compressed files
+const isValidXva = async path => {
+  try {
+    const fd = await fs.open(path, 'r')
+    try {
+      const { size } = await fs.fstat(fd)
+      if (size < 20) {
+        // neither a valid gzip not tar
+        return false
+      }
+
+      return (await isGzipFile(fd))
+        ? true // gzip files cannot be validated at this time
+        : await isValidTar(size, fd)
+    } finally {
+      fs.close(fd).catch(noop)
+    }
+  } catch (error) {
+    // never throw, log and report as valid to avoid side effects
+    console.error('isValidXva', path, error)
+    return true
+  }
+}
+exports.isValidXva = isValidXva
+
+const noop = Function.prototype

@xen-orchestra/backups/package.json

@@ -0,0 +1,27 @@
+{
+  "private": false,
+  "name": "@xen-orchestra/backups",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/backups",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@xen-orchestra/backups",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "version": "0.1.1",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  },
+  "dependencies": {
+    "d3-time-format": "^3.0.0",
+    "fs-extra": "^9.0.0"
+  },
+  "license": "AGPL-3.0-or-later",
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  }
+}

@xen-orchestra/backups/watchStreamSize.js

@@ -0,0 +1,11 @@
+exports.watchStreamSize = stream => {
+  const container = { size: 0 }
+  const isPaused = stream.isPaused()
+  stream.on('data', data => {
+    container.size += data.length
+  })
+  if (isPaused) {
+    stream.pause()
+  }
+  return container
+}

@xen-orchestra/cr-seed-cli/README.md

@@ -0,0 +1,28 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @xen-orchestra/cr-seed-cli
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/cr-seed-cli)](https://npmjs.org/package/@xen-orchestra/cr-seed-cli) ![License](https://badgen.net/npm/license/@xen-orchestra/cr-seed-cli) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/cr-seed-cli)](https://bundlephobia.com/result?p=@xen-orchestra/cr-seed-cli) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/cr-seed-cli)](https://npmjs.org/package/@xen-orchestra/cr-seed-cli)
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/cr-seed-cli):
+
+```
+> npm install --global @xen-orchestra/cr-seed-cli
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

@xen-orchestra/cr-seed-cli/index.js

@@ -5,7 +5,7 @@ const { NULL_REF, Xapi } = require('xen-api')
 
 const pkg = require('./package.json')
 
-Xapi.prototype.getVmDisks = async function(vm) {
+Xapi.prototype.getVmDisks = async function (vm) {
   const disks = { __proto__: null }
   await Promise.all([
     ...vm.VBDs.map(async vbdRef => {
@@ -32,14 +32,7 @@ ${cliName} v${pkg.version}
     )
   }
 
-  const [
+  const [srcXapiUrl, srcSnapshotUuid, tgtXapiUrl, tgtVmUuid, jobId, scheduleId] = args
-    srcXapiUrl,
-    srcSnapshotUuid,
-    tgtXapiUrl,
-    tgtVmUuid,
-    jobId,
-    scheduleId,
-  ] = args
 
   const srcXapi = new Xapi({
     allowUnauthorized: true,
@@ -70,16 +63,10 @@ ${cliName} v${pkg.version}
     'xo:backup:vm': srcVm.uuid,
   }
 
-  const [srcDisks, tgtDisks] = await Promise.all([
+  const [srcDisks, tgtDisks] = await Promise.all([srcXapi.getVmDisks(srcSnapshot), tgtXapi.getVmDisks(tgtVm)])
-    srcXapi.getVmDisks(srcSnapshot),
-    tgtXapi.getVmDisks(tgtVm),
-  ])
   const userDevices = Object.keys(tgtDisks)
 
-  const tgtSr = await tgtXapi.getRecord(
+  const tgtSr = await tgtXapi.getRecord('SR', tgtDisks[Object.keys(tgtDisks)[0]].SR)
-    'SR',
-    tgtDisks[Object.keys(tgtDisks)[0]].SR
-  )
 
   await Promise.all([
     srcSnapshot.update_other_config(metadata),
@@ -90,10 +77,7 @@ ${cliName} v${pkg.version}
       'xo:backup:sr': tgtSr.uuid,
       'xo:copy_of': srcSnapshotUuid,
     }),
-    tgtVm.update_blocked_operations(
+    tgtVm.update_blocked_operations('start', 'Start operation for this vm is blocked, clone it if you want to use it.'),
-      'start',
-      'Start operation for this vm is blocked, clone it if you want to use it.'
-    ),
     Promise.all(
       userDevices.map(userDevice => {
         const srcDisk = srcDisks[userDevice]

@xen-orchestra/cr-seed-cli/package.json

@@ -1,4 +1,5 @@
 {
+  "private": false,
   "name": "@xen-orchestra/cr-seed-cli",
   "version": "0.2.0",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/cr-seed-cli",
@@ -14,11 +15,17 @@
   "bin": {
     "xo-cr-seed": "./index.js"
   },
+  "preferGlobal": true,
   "dependencies": {
     "golike-defer": "^0.4.1",
-    "xen-api": "^0.27.3"
+    "xen-api": "^0.29.0"
   },
   "scripts": {
     "postversion": "npm publish"
+  },
+  "license": "AGPL-3.0-or-later",
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
   }
 }

@xen-orchestra/cron/.babelrc.js

@@ -1,3 +1 @@
-module.exports = require('../../@xen-orchestra/babel-config')(
+module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))
-  require('./package.json')
-)

@xen-orchestra/cron/README.md

@@ -1,4 +1,8 @@
-# @xen-orchestra/cron [![Build Status](https://travis-ci.org/vatesfr/xen-orchestra.png?branch=master)](https://travis-ci.org/vatesfr/xen-orchestra)
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @xen-orchestra/cron
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/cron)](https://npmjs.org/package/@xen-orchestra/cron) ![License](https://badgen.net/npm/license/@xen-orchestra/cron) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/cron)](https://bundlephobia.com/result?p=@xen-orchestra/cron) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/cron)](https://npmjs.org/package/@xen-orchestra/cron)
 
 > Focused, well maintained, cron parser/scheduler
 
@@ -10,13 +14,14 @@ Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/cron)
 > npm install --save @xen-orchestra/cron
 ```
 
+## Usage
+
 ### Pattern syntax
 
 ```
 <minute> <hour> <day of month> <month> <day of week>
 ```
 
-
 Each entry can be:
 
 - a single value
@@ -29,12 +34,12 @@ A wildcard (`*`) can be used as a shortcut for the whole range
 Step values can be used in conjunctions with ranges. For instance,
 `1-7/2` is the same as `1,3,5,7`.
 
-| Field            | Allowed values |
+| Field            | Allowed values                                                     |
-|------------------|----------------|
+| ---------------- | ------------------------------------------------------------------ |
-| minute           | 0-59           |
+| minute           | 0-59                                                               |
-| hour             | 0-23           |
+| hour             | 0-23                                                               |
-| day of the month | 1-31 or 3-letter names (`jan`, `feb`, …) |
+| day of the month | 1-31 or 3-letter names (`jan`, `feb`, …)                           |
-| month            | 0-11           |
+| month            | 0-11                                                               |
 | day of week      | 0-7 (0 and 7 both mean Sunday) or 3-letter names (`mon`, `tue`, …) |
 
 > Note: the month range is 0-11 to be compatible with
@@ -110,28 +115,9 @@ job.start()
 job.stop()
 ```
 
-## Development
-
-```
-# Install dependencies
-> yarn
-
-# Run the tests
-> yarn test
-
-# Continuously compile
-> yarn dev
-
-# Continuously run the tests
-> yarn dev-test
-
-# Build for production (automatically called by npm install)
-> yarn build
-```
-
 ## Contributions
 
-Contributions are *very* welcomed, either on the documentation or on
+Contributions are _very_ welcomed, either on the documentation or on
 the code.
 
 You may:
@@ -142,4 +128,4 @@ You may:
 
 ## License
 
-ISC © [Vates SAS](https://vates.fr)
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@xen-orchestra/cron/USAGE.md

@@ -0,0 +1,98 @@
+### Pattern syntax
+
+```
+<minute> <hour> <day of month> <month> <day of week>
+```
+
+Each entry can be:
+
+- a single value
+- a range (`0-23` or `*/2`)
+- a list of values/ranges (`1,8-12`)
+
+A wildcard (`*`) can be used as a shortcut for the whole range
+(`first-last`).
+
+Step values can be used in conjunctions with ranges. For instance,
+`1-7/2` is the same as `1,3,5,7`.
+
+| Field            | Allowed values                                                     |
+| ---------------- | ------------------------------------------------------------------ |
+| minute           | 0-59                                                               |
+| hour             | 0-23                                                               |
+| day of the month | 1-31 or 3-letter names (`jan`, `feb`, …)                           |
+| month            | 0-11                                                               |
+| day of week      | 0-7 (0 and 7 both mean Sunday) or 3-letter names (`mon`, `tue`, …) |
+
+> Note: the month range is 0-11 to be compatible with
+> [cron](https://github.com/kelektiv/node-cron), it does not appear to
+> be very standard though.
+
+### API
+
+`createSchedule(pattern: string, zone: string = 'utc'): Schedule`
+
+> Create a new schedule.
+
+- `pattern`: the pattern to use, see [the syntax](#pattern-syntax)
+- `zone`: the timezone to use, use `'local'` for the local timezone
+
+```js
+import { createSchedule } from '@xen-orchestra/cron'
+
+const schedule = createSchedule('0 0 * * sun', 'America/New_York')
+```
+
+`Schedule#createJob(fn: Function): Job`
+
+> Create a new job from this schedule.
+
+- `fn`: function to execute, if it returns a promise, it will be
+  awaited before scheduling the next run.
+
+```js
+const job = schedule.createJob(() => {
+  console.log(new Date())
+})
+```
+
+`Schedule#next(n: number): Array<Date>`
+
+> Returns the next dates matching this schedule.
+
+- `n`: number of dates to return
+
+```js
+schedule.next(2)
+// [ 2018-02-11T05:00:00.000Z, 2018-02-18T05:00:00.000Z ]
+```
+
+`Schedule#startJob(fn: Function): () => void`
+
+> Start a new job from this schedule and return a function to stop it.
+
+- `fn`: function to execute, if it returns a promise, it will be
+  awaited before scheduling the next run.
+
+```js
+const stopJob = schedule.startJob(() => {
+  console.log(new Date())
+})
+stopJob()
+```
+
+`Job#start(): void`
+
+> Start this job.
+
+```js
+job.start()
+```
+
+`Job#stop(): void`
+
+> Stop this job.
+
+```js
+job.stop()
+```

@xen-orchestra/cron/package.json

@@ -1,4 +1,5 @@
 {
+  "private": false,
   "name": "@xen-orchestra/cron",
   "version": "1.0.6",
   "license": "ISC",
@@ -22,8 +23,8 @@
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
   "author": {
-    "name": "Julien Fontanet",
+    "name": "Vates SAS",
-    "email": "julien.fontanet@isonoe.net"
+    "url": "https://vates.fr"
   },
   "preferGlobal": false,
   "main": "dist/",
@@ -46,7 +47,7 @@
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
-    "cross-env": "^6.0.3",
+    "cross-env": "^7.0.2",
     "rimraf": "^3.0.0"
   },
   "scripts": {

@xen-orchestra/cron/src/index.js

@@ -42,10 +42,7 @@ class Job {
         const now = schedule._createDate()
         scheduledDate = +next(schedule._schedule, now)
         const delay = scheduledDate - now
-        this._timeout =
+        this._timeout = delay < MAX_DELAY ? setTimeout(wrapper, delay) : setTimeout(scheduleNext, MAX_DELAY)
-          delay < MAX_DELAY
-            ? setTimeout(wrapper, delay)
-            : setTimeout(scheduleNext, MAX_DELAY)
       }
     }
 
@@ -73,12 +70,7 @@ class Job {
 class Schedule {
   constructor(pattern, zone = 'utc') {
     this._schedule = parse(pattern)
-    this._createDate =
+    this._createDate = zone.toLowerCase() === 'utc' ? moment.utc : zone === 'local' ? moment : () => moment.tz(zone)
-      zone.toLowerCase() === 'utc'
-        ? moment.utc
-        : zone === 'local'
-        ? moment
-        : () => moment.tz(zone)
   }
 
   createJob(fn) {

@xen-orchestra/cron/src/next.spec.js

@@ -37,9 +37,7 @@ describe('next()', () => {
   })
 
   it('fails when no solutions has been found', () => {
-    expect(() => N('0 0 30 feb *')).toThrow(
+    expect(() => N('0 0 30 feb *')).toThrow('no solutions found for this schedule')
-      'no solutions found for this schedule'
-    )
   })
 
   it('select the first sunday of the month', () => {

@xen-orchestra/cron/src/parse.js

@@ -66,9 +66,7 @@ const createParser = ({ fields: [...fields], presets: { ...presets } }) => {
       aliasesRegExp.lastIndex = i
       const matches = aliasesRegExp.exec(pattern)
       if (matches === null) {
-        throw new SyntaxError(
+        throw new SyntaxError(`${field.name}: missing alias or integer at character ${i}`)
-          `${field.name}: missing alias or integer at character ${i}`
-        )
       }
       const [alias] = matches
       i += alias.length
@@ -77,9 +75,7 @@ const createParser = ({ fields: [...fields], presets: { ...presets } }) => {
 
     const { range } = field
     if (value < range[0] || value > range[1]) {
-      throw new SyntaxError(
+      throw new SyntaxError(`${field.name}: ${value} is not between ${range[0]} and ${range[1]}`)
-        `${field.name}: ${value} is not between ${range[0]} and ${range[1]}`
-      )
     }
     return value
   }
@@ -117,9 +113,7 @@ const createParser = ({ fields: [...fields], presets: { ...presets } }) => {
     {
       const schedule = presets[p]
       if (schedule !== undefined) {
-        return typeof schedule === 'string'
+        return typeof schedule === 'string' ? (presets[p] = parse(schedule)) : schedule
-          ? (presets[p] = parse(schedule))
-          : schedule
       }
     }
 
@@ -142,9 +136,7 @@ const createParser = ({ fields: [...fields], presets: { ...presets } }) => {
 
       consumeWhitespaces()
       if (i !== n) {
-        throw new SyntaxError(
+        throw new SyntaxError(`unexpected character at offset ${i}, expected end`)
-          `unexpected character at offset ${i}, expected end`
-        )
       }
 
       return schedule

@xen-orchestra/cron/src/parse.spec.js

@@ -33,9 +33,7 @@ describe('parse()', () => {
   })
 
   it('reports invalid aliases', () => {
-    expect(() => parse('* * * jan-foo *')).toThrow(
+    expect(() => parse('* * * jan-foo *')).toThrow('month: missing alias or integer at character 10')
-      'month: missing alias or integer at character 10'
-    )
   })
 
   it('dayOfWeek: 0 and 7 bind to sunday', () => {

@xen-orchestra/defined/.babelrc.js

@@ -1,3 +1 @@
-module.exports = require('../../@xen-orchestra/babel-config')(
+module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))
-  require('./package.json')
-)

@xen-orchestra/defined/README.md

@@ -1,6 +1,8 @@
-# @xen-orchestra/defined [![Build Status](https://travis-ci.org/${pkg.shortGitHubPath}.png?branch=master)](https://travis-ci.org/${pkg.shortGitHubPath})
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
 
-> ${pkg.description}
+# @xen-orchestra/defined
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/defined)](https://npmjs.org/package/@xen-orchestra/defined) ![License](https://badgen.net/npm/license/@xen-orchestra/defined) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/defined)](https://bundlephobia.com/result?p=@xen-orchestra/defined) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/defined)](https://npmjs.org/package/@xen-orchestra/defined)
 
 ## Install
 
@@ -10,32 +12,9 @@ Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/defin
 > npm install --save @xen-orchestra/defined
 ```
 
-## Usage
-
-**TODO**
-
-## Development
-
-```
-# Install dependencies
-> yarn
-
-# Run the tests
-> yarn test
-
-# Continuously compile
-> yarn dev
-
-# Continuously run the tests
-> yarn dev-test
-
-# Build for production (automatically called by npm install)
-> yarn build
-```
-
 ## Contributions
 
-Contributions are *very* welcomed, either on the documentation or on
+Contributions are _very_ welcomed, either on the documentation or on
 the code.
 
 You may:
@@ -46,4 +25,4 @@ You may:
 
 ## License
 
-ISC © [Vates SAS](https://vates.fr)
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@xen-orchestra/defined/package.json

@@ -1,4 +1,5 @@
 {
+  "private": false,
   "name": "@xen-orchestra/defined",
   "version": "0.0.0",
   "license": "ISC",
@@ -12,8 +13,8 @@
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
   "author": {
-    "name": "Julien Fontanet",
+    "name": "Vates SAS",
-    "email": "julien.fontanet@vates.fr"
+    "url": "https://vates.fr"
   },
   "preferGlobal": false,
   "main": "dist/",
@@ -34,7 +35,7 @@
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
     "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^6.0.3",
+    "cross-env": "^7.0.2",
     "rimraf": "^3.0.0"
   },
   "scripts": {

@xen-orchestra/defined/src/index.js

@@ -60,5 +60,4 @@ export const get = (accessor: (input: ?any) => any, arg: ?any) => {
 //   _ => new ProxyAgent(_)
 // )
 // ```
-export const ifDef = (value: ?any, thenFn: (value: any) => any) =>
+export const ifDef = (value: ?any, thenFn: (value: any) => any) => (value !== undefined ? thenFn(value) : value)
-  value !== undefined ? thenFn(value) : value

@xen-orchestra/emit-async/.babelrc.js

@@ -1,3 +1 @@
-module.exports = require('../../@xen-orchestra/babel-config')(
+module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))
-  require('./package.json')
-)

@xen-orchestra/emit-async/README.md

@@ -1,6 +1,8 @@
-# @xen-orchestra/emit-async [![Build Status](https://travis-ci.org/${pkg.shortGitHubPath}.png?branch=master)](https://travis-ci.org/${pkg.shortGitHubPath})
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
 
-> ${pkg.description}
+# @xen-orchestra/emit-async
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/emit-async)](https://npmjs.org/package/@xen-orchestra/emit-async) ![License](https://badgen.net/npm/license/@xen-orchestra/emit-async) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/emit-async)](https://bundlephobia.com/result?p=@xen-orchestra/emit-async) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/emit-async)](https://npmjs.org/package/@xen-orchestra/emit-async)
 
 ## Install
 
@@ -17,6 +19,11 @@ import EE from 'events'
 import emitAsync from '@xen-orchestra/emit-async'
 
 const ee = new EE()
+
+// exposing emitAsync on our event emitter
+//
+// it's not required though and we could have used directly via
+// emitAsync.call(ee, event, args...)
 ee.emitAsync = emitAsync
 
 ee.on('start', async function () {
@@ -24,40 +31,24 @@ ee.on('start', async function () {
 })
 
 // similar to EventEmmiter#emit() but returns a promise which resolves when all
-// listeners have resolved
+// listeners have settled
 await ee.emitAsync('start')
 
 // by default, it will rejects as soon as one listener reject, you can customise
 // error handling though:
-await ee.emitAsync({
+await ee.emitAsync(
-  onError (error) {
+  {
-    console.warn(error)
+    onError(error) {
-  }
+      console.warn(error)
-}, 'start')
+    },
-```
+  },
-
+  'start'
-## Development
+)
-
-```
-# Install dependencies
-> yarn
-
-# Run the tests
-> yarn test
-
-# Continuously compile
-> yarn dev
-
-# Continuously run the tests
-> yarn dev-test
-
-# Build for production (automatically called by npm install)
-> yarn build
 ```
 
 ## Contributions
 
-Contributions are *very* welcomed, either on the documentation or on
+Contributions are _very_ welcomed, either on the documentation or on
 the code.
 
 You may:
@@ -68,4 +59,4 @@ You may:
 
 ## License
 
-ISC © [Vates SAS](https://vates.fr)
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@xen-orchestra/emit-async/USAGE.md

@@ -0,0 +1,31 @@
+```js
+import EE from 'events'
+import emitAsync from '@xen-orchestra/emit-async'
+
+const ee = new EE()
+
+// exposing emitAsync on our event emitter
+//
+// it's not required though and we could have used directly via
+// emitAsync.call(ee, event, args...)
+ee.emitAsync = emitAsync
+
+ee.on('start', async function () {
+  // whatever
+})
+
+// similar to EventEmmiter#emit() but returns a promise which resolves when all
+// listeners have settled
+await ee.emitAsync('start')
+
+// by default, it will rejects as soon as one listener reject, you can customise
+// error handling though:
+await ee.emitAsync(
+  {
+    onError(error) {
+      console.warn(error)
+    },
+  },
+  'start'
+)
+```

@xen-orchestra/emit-async/package.json

@@ -1,4 +1,5 @@
 {
+  "private": false,
   "name": "@xen-orchestra/emit-async",
   "version": "0.0.0",
   "license": "ISC",
@@ -12,8 +13,8 @@
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
   "author": {
-    "name": "Julien Fontanet",
+    "name": "Vates SAS",
-    "email": "julien.fontanet@vates.fr"
+    "url": "https://vates.fr"
   },
   "preferGlobal": false,
   "main": "dist/",
@@ -33,7 +34,7 @@
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^6.0.3",
+    "cross-env": "^7.0.2",
     "rimraf": "^3.0.0"
   },
   "scripts": {

@xen-orchestra/fs/.babelrc.js

@@ -1,3 +1 @@
-module.exports = require('../../@xen-orchestra/babel-config')(
+module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))
-  require('./package.json')
-)

@xen-orchestra/fs/README.md

@@ -0,0 +1,30 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @xen-orchestra/fs
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/fs)](https://npmjs.org/package/@xen-orchestra/fs) ![License](https://badgen.net/npm/license/@xen-orchestra/fs) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/fs)](https://bundlephobia.com/result?p=@xen-orchestra/fs) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/fs)](https://npmjs.org/package/@xen-orchestra/fs)
+
+> The File System for Xen Orchestra backups.
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/fs):
+
+```
+> npm install --global @xen-orchestra/fs
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

@xen-orchestra/fs/package.json

@@ -1,7 +1,8 @@
 {
+  "private": false,
   "name": "@xen-orchestra/fs",
-  "version": "0.10.2",
+  "version": "0.12.1",
-  "license": "AGPL-3.0",
+  "license": "AGPL-3.0-or-later",
   "description": "The File System for Xen Orchestra backups.",
   "keywords": [],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
@@ -21,20 +22,22 @@
     "node": ">=8.10"
   },
   "dependencies": {
-    "@marsaud/smb2": "^0.14.0",
+    "@marsaud/smb2": "^0.17.2",
     "@sindresorhus/df": "^3.1.1",
+    "@sullux/aws-sdk": "^1.0.5",
     "@xen-orchestra/async-map": "^0.0.0",
+    "aws-sdk": "^2.686.0",
     "decorator-synchronized": "^0.5.0",
-    "execa": "^3.2.0",
+    "execa": "^5.0.0",
-    "fs-extra": "^8.0.1",
+    "fs-extra": "^9.0.0",
-    "get-stream": "^5.1.0",
+    "get-stream": "^6.0.0",
     "limit-concurrency-decorator": "^0.4.0",
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.14.0",
+    "promise-toolbox": "^0.16.0",
     "readable-stream": "^3.0.6",
-    "through2": "^3.0.0",
+    "through2": "^4.0.2",
-    "tmp": "^0.1.0",
+    "tmp": "^0.2.1",
-    "xo-remote-parser": "^0.5.0"
+    "xo-remote-parser": "^0.6.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",
@@ -46,7 +49,7 @@
     "@babel/preset-flow": "^7.0.0",
     "async-iterator-to-stream": "^1.1.0",
     "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^6.0.3",
+    "cross-env": "^7.0.2",
     "dotenv": "^8.0.0",
     "index-modules": "^0.3.0",
     "rimraf": "^3.0.0"
@@ -57,7 +60,11 @@
     "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
     "prebuild": "yarn run clean",
     "predev": "yarn run clean",
-    "prepare": "yarn run build",
+    "prepublishOnly": "yarn run build",
     "postversion": "npm publish"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
   }
 }

@xen-orchestra/fs/src/_mount.js

@@ -6,36 +6,19 @@ import { tmpdir } from 'os'
 
 import LocalHandler from './local'
 
-const sudoExeca = (command, args, opts) =>
+const sudoExeca = (command, args, opts) => execa('sudo', [command, ...args], opts)
-  execa('sudo', [command, ...args], opts)
 
 export default class MountHandler extends LocalHandler {
-  constructor(
+  constructor(remote, { mountsDir = join(tmpdir(), 'xo-fs-mounts'), useSudo = false, ...opts } = {}, params) {
-    remote,
-    {
-      mountsDir = join(tmpdir(), 'xo-fs-mounts'),
-      useSudo = false,
-      ...opts
-    } = {},
-    params
-  ) {
     super(remote, opts)
 
     this._execa = useSudo ? sudoExeca : execa
     this._keeper = undefined
     this._params = {
       ...params,
-      options: [params.options, remote.options]
+      options: [params.options, remote.options ?? params.defaultOptions].filter(_ => _ !== undefined).join(','),
-        .filter(_ => _ !== undefined)
-        .join(','),
     }
-    this._realPath = join(
+    this._realPath = join(mountsDir, remote.id || Math.random().toString(36).slice(2))
-      mountsDir,
-      remote.id ||
-        Math.random()
-          .toString(36)
-          .slice(2)
-    )
   }
 
   async _forget() {
@@ -75,16 +58,15 @@ export default class MountHandler extends LocalHandler {
 
     try {
       const { type, device, options, env } = this._params
-      await this._execa(
+
-        'mount',
+      // Linux mount is more flexible in which order the mount arguments appear.
-        ['-t', type, device, realPath, '-o', options],
+      // But FreeBSD requires this order of the arguments.
-        {
+      await this._execa('mount', ['-o', options, '-t', type, device, realPath], {
-          env: {
+        env: {
-            LANG: 'C',
+          LANG: 'C',
-            ...env,
+          ...env,
-          },
+        },
-        }
+      })
-      )
     } catch (error) {
       try {
         // the failure may mean it's already mounted, use `findmnt` to check
@@ -99,9 +81,7 @@ export default class MountHandler extends LocalHandler {
 
     // keep an open file on the mount to prevent it from being unmounted if used
     // by another handler/process
-    const keeperPath = `${realPath}/.keeper_${Math.random()
+    const keeperPath = `${realPath}/.keeper_${Math.random().toString(36).slice(2)}`
-      .toString(36)
-      .slice(2)}`
     this._keeper = await fs.open(keeperPath, 'w')
     ignoreErrors.call(fs.unlink(keeperPath))
   }

@xen-orchestra/fs/src/abstract.js

@@ -4,8 +4,9 @@
 import getStream from 'get-stream'
 
 import asyncMap from '@xen-orchestra/async-map'
+import CancelToken from 'promise-toolbox/CancelToken'
 import limit from 'limit-concurrency-decorator'
-import path from 'path'
+import path, { basename } from 'path'
 import synchronized from 'decorator-synchronized'
 import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
 import { parse } from 'xo-remote-parser'
@@ -86,9 +87,7 @@ export default class RemoteHandlerAbstract {
     }
     ;({ timeout: this._timeout = DEFAULT_TIMEOUT } = options)
 
-    const sharedLimit = limit(
+    const sharedLimit = limit(options.maxParallelOperations ?? DEFAULT_MAX_PARALLEL_OPERATIONS)
-      options.maxParallelOperations ?? DEFAULT_MAX_PARALLEL_OPERATIONS
-    )
     this.closeFile = sharedLimit(this.closeFile)
     this.getInfo = sharedLimit(this.getInfo)
     this.getSize = sharedLimit(this.getSize)
@@ -118,46 +117,7 @@ export default class RemoteHandlerAbstract {
   }
 
   async closeFile(fd: FileDescriptor): Promise<void> {
-    await timeout.call(this._closeFile(fd.fd), this._timeout)
+    await this.__closeFile(fd)
-  }
-
-  async createOutputStream(
-    file: File,
-    { checksum = false, ...options }: Object = {}
-  ): Promise<LaxWritable> {
-    if (typeof file === 'string') {
-      file = normalizePath(file)
-    }
-    const path = typeof file === 'string' ? file : file.path
-    const streamP = timeout.call(
-      this._createOutputStream(file, {
-        flags: 'wx',
-        ...options,
-      }),
-      this._timeout
-    )
-
-    if (!checksum) {
-      return streamP
-    }
-
-    const checksumStream = createChecksumStream()
-    const forwardError = error => {
-      checksumStream.emit('error', error)
-    }
-
-    const stream = await streamP
-    stream.on('error', forwardError)
-    checksumStream.pipe(stream)
-
-    // $FlowFixMe
-    checksumStream.checksumWritten = checksumStream.checksum
-      .then(value =>
-        this._outputFile(checksumFile(path), value, { flags: 'wx' })
-      )
-      .catch(forwardError)
-
-    return checksumStream
   }
 
   createReadStream(
@@ -168,30 +128,24 @@ export default class RemoteHandlerAbstract {
       file = normalizePath(file)
     }
     const path = typeof file === 'string' ? file : file.path
-    const streamP = timeout
+    const streamP = timeout.call(this._createReadStream(file, options), this._timeout).then(stream => {
-      .call(this._createReadStream(file, options), this._timeout)
+      // detect early errors
-      .then(stream => {
+      let promise = fromEvent(stream, 'readable')
-        // detect early errors
-        let promise = fromEvent(stream, 'readable')
 
-        // try to add the length prop if missing and not a range stream
+      // try to add the length prop if missing and not a range stream
-        if (
+      if (stream.length === undefined && options.end === undefined && options.start === undefined) {
-          stream.length === undefined &&
+        promise = Promise.all([
-          options.end === undefined &&
+          promise,
-          options.start === undefined
+          ignoreErrors.call(
-        ) {
+            this._getSize(file).then(size => {
-          promise = Promise.all([
+              stream.length = size
-            promise,
+            })
-            ignoreErrors.call(
+          ),
-              this._getSize(file).then(size => {
+        ])
-                stream.length = size
+      }
-              })
-            ),
-          ])
-        }
 
-        return promise.then(() => stream)
+      return promise.then(() => stream)
-      })
+    })
 
     if (!checksum) {
       return streamP
@@ -204,10 +158,7 @@ export default class RemoteHandlerAbstract {
       checksum =>
         streamP.then(stream => {
           const { length } = stream
-          stream = (validChecksumOfReadStream(
+          stream = (validChecksumOfReadStream(stream, String(checksum).trim()): LaxReadable)
-            stream,
-            String(checksum).trim()
-          ): LaxReadable)
           stream.length = length
 
           return stream
@@ -221,19 +172,18 @@ export default class RemoteHandlerAbstract {
     )
   }
 
-  createWriteStream(
+  // write a stream to a file using a temporary file
-    file: File,
+  async outputStream(
-    options: { end?: number, flags?: string, start?: number } = {}
+    path: string,
-  ): Promise<LaxWritable> {
+    input: Readable | Promise<Readable>,
-    return timeout.call(
+    { checksum = true, dirMode, cancelToken = CancelToken.none }: { checksum?: boolean, dirMode?: number } = {}
-      this._createWriteStream(
+  ): Promise<void> {
-        typeof file === 'string' ? normalizePath(file) : file,
+    path = normalizePath(path)
-        {
+    return this._outputStream(await input, normalizePath(path), {
-          flags: 'wx',
+      checksum,
-          ...options,
+      dirMode,
-        }
+      cancelToken,
-      )
+    })
-    )
   }
 
   // Free the resources possibly dedicated to put the remote at work, when it
@@ -252,18 +202,12 @@ export default class RemoteHandlerAbstract {
   }
 
   async getSize(file: File): Promise<number> {
-    return timeout.call(
+    return timeout.call(this._getSize(typeof file === 'string' ? normalizePath(file) : file), this._timeout)
-      this._getSize(typeof file === 'string' ? normalizePath(file) : file),
-      this._timeout
-    )
   }
 
   async list(
     dir: string,
-    {
+    { filter, prependDir = false }: { filter?: (name: string) => boolean, prependDir?: boolean } = {}
-      filter,
-      prependDir = false,
-    }: { filter?: (name: string) => boolean, prependDir?: boolean } = {}
   ): Promise<string[]> {
     const virtualDir = normalizePath(dir)
     dir = normalizePath(dir)
@@ -282,95 +226,47 @@ export default class RemoteHandlerAbstract {
     return entries
   }
 
-  async mkdir(dir: string): Promise<void> {
+  async mkdir(dir: string, { mode }: { mode?: number } = {}): Promise<void> {
-    dir = normalizePath(dir)
+    await this.__mkdir(normalizePath(dir), { mode })
-    try {
-      await this._mkdir(dir)
-    } catch (error) {
-      if (error == null || error.code !== 'EEXIST') {
-        throw error
-      }
-
-      // this operation will throw if it's not already a directory
-      await this._list(dir)
-    }
   }
 
-  async mktree(dir: string): Promise<void> {
+  async mktree(dir: string, { mode }: { mode?: number } = {}): Promise<void> {
-    await this._mktree(normalizePath(dir))
+    await this._mktree(normalizePath(dir), { mode })
   }
 
-  async openFile(path: string, flags: string): Promise<FileDescriptor> {
+  openFile(path: string, flags: string): Promise<FileDescriptor> {
-    path = normalizePath(path)
+    return this.__openFile(path, flags)
-
-    return {
-      fd: await timeout.call(this._openFile(path, flags), this._timeout),
-      path,
-    }
   }
 
   async outputFile(
     file: string,
     data: Data,
-    { flags = 'wx' }: { flags?: string } = {}
+    { dirMode, flags = 'wx' }: { dirMode?: number, flags?: string } = {}
   ): Promise<void> {
-    await this._outputFile(normalizePath(file), data, { flags })
+    await this._outputFile(normalizePath(file), data, { dirMode, flags })
   }
 
-  async read(
+  async read(file: File, buffer: Buffer, position?: number): Promise<{| bytesRead: number, buffer: Buffer |}> {
-    file: File,
+    return this._read(typeof file === 'string' ? normalizePath(file) : file, buffer, position)
-    buffer: Buffer,
-    position?: number
-  ): Promise<{| bytesRead: number, buffer: Buffer |}> {
-    return this._read(
-      typeof file === 'string' ? normalizePath(file) : file,
-      buffer,
-      position
-    )
   }
 
-  async readFile(
+  async readFile(file: string, { flags = 'r' }: { flags?: string } = {}): Promise<Buffer> {
-    file: string,
-    { flags = 'r' }: { flags?: string } = {}
-  ): Promise<Buffer> {
     return this._readFile(normalizePath(file), { flags })
   }
 
-  async refreshChecksum(path: string): Promise<void> {
+  async rename(oldPath: string, newPath: string, { checksum = false }: Object = {}) {
-    path = normalizePath(path)
-
-    const stream = (await this._createReadStream(path, { flags: 'r' })).pipe(
-      createChecksumStream()
-    )
-    stream.resume() // start reading the whole file
-    await this._outputFile(checksumFile(path), await stream.checksum, {
-      flags: 'wx',
-    })
-  }
-
-  async rename(
-    oldPath: string,
-    newPath: string,
-    { checksum = false }: Object = {}
-  ) {
     oldPath = normalizePath(oldPath)
     newPath = normalizePath(newPath)
 
     let p = timeout.call(this._rename(oldPath, newPath), this._timeout)
     if (checksum) {
-      p = Promise.all([
+      p = Promise.all([p, this._rename(checksumFile(oldPath), checksumFile(newPath))])
-        p,
-        this._rename(checksumFile(oldPath), checksumFile(newPath)),
-      ])
     }
     return p
   }
 
   async rmdir(dir: string): Promise<void> {
-    await timeout.call(
+    await timeout.call(this._rmdir(normalizePath(dir)).catch(ignoreEnoent), this._timeout)
-      this._rmdir(normalizePath(dir)).catch(ignoreEnoent),
-      this._timeout
-    )
   }
 
   async rmtree(dir: string): Promise<void> {
@@ -435,33 +331,49 @@ export default class RemoteHandlerAbstract {
     await this._unlink(file).catch(ignoreEnoent)
   }
 
-  async write(
+  async write(file: File, buffer: Buffer, position: number): Promise<{| bytesWritten: number, buffer: Buffer |}> {
-    file: File,
+    await this._write(typeof file === 'string' ? normalizePath(file) : file, buffer, position)
-    buffer: Buffer,
-    position: number
-  ): Promise<{| bytesWritten: number, buffer: Buffer |}> {
-    await this._write(
-      typeof file === 'string' ? normalizePath(file) : file,
-      buffer,
-      position
-    )
   }
 
-  async writeFile(
+  async writeFile(file: string, data: Data, { flags = 'wx' }: { flags?: string } = {}): Promise<void> {
-    file: string,
-    data: Data,
-    { flags = 'wx' }: { flags?: string } = {}
-  ): Promise<void> {
     await this._writeFile(normalizePath(file), data, { flags })
   }
 
+  // Methods that can be called by private methods to avoid parallel limit on public methods
+
+  async __closeFile(fd: FileDescriptor): Promise<void> {
+    await timeout.call(this._closeFile(fd.fd), this._timeout)
+  }
+
+  async __mkdir(dir: string, { mode }: { mode?: number } = {}): Promise<void> {
+    try {
+      await this._mkdir(dir, { mode })
+    } catch (error) {
+      if (error == null || error.code !== 'EEXIST') {
+        throw error
+      }
+
+      // this operation will throw if it's not already a directory
+      await this._list(dir)
+    }
+  }
+
+  async __openFile(path: string, flags: string): Promise<FileDescriptor> {
+    path = normalizePath(path)
+
+    return {
+      fd: await timeout.call(this._openFile(path, flags), this._timeout),
+      path,
+    }
+  }
+
   // Methods that can be implemented by inheriting classes
 
   async _closeFile(fd: mixed): Promise<void> {
     throw new Error('Not implemented')
   }
 
-  async _createOutputStream(file: File, options: Object): Promise<LaxWritable> {
+  async _createOutputStream(file: File, { dirMode, ...options }: Object = {}): Promise<LaxWritable> {
     try {
       return await this._createWriteStream(file, options)
     } catch (error) {
@@ -470,7 +382,7 @@ export default class RemoteHandlerAbstract {
       }
     }
 
-    await this._mktree(dirname(file))
+    await this._mktree(dirname(file), { mode: dirMode })
     return this._createOutputStream(file, options)
   }
 
@@ -501,45 +413,94 @@ export default class RemoteHandlerAbstract {
     throw new Error('Not implemented')
   }
 
-  async _mktree(dir: string): Promise<void> {
+  async _mktree(dir: string, { mode }: { mode?: number } = {}): Promise<void> {
     try {
-      return await this.mkdir(dir)
+      return await this.__mkdir(dir, { mode })
     } catch (error) {
       if (error.code !== 'ENOENT') {
         throw error
       }
     }
 
-    await this._mktree(dirname(dir))
+    await this._mktree(dirname(dir), { mode })
-    return this._mktree(dir)
+    return this._mktree(dir, { mode })
   }
 
   async _openFile(path: string, flags: string): Promise<mixed> {
     throw new Error('Not implemented')
   }
 
-  async _outputFile(
+  async _outputFile(file: string, data: Data, { dirMode, flags }: { dirMode?: number, flags?: string }): Promise<void> {
-    file: string,
-    data: Data,
-    options: { flags?: string }
-  ): Promise<void> {
     try {
-      return await this._writeFile(file, data, options)
+      return await this._writeFile(file, data, { flags })
     } catch (error) {
       if (error.code !== 'ENOENT') {
         throw error
       }
     }
 
-    await this._mktree(dirname(file))
+    await this._mktree(dirname(file), { mode: dirMode })
-    return this._outputFile(file, data, options)
+    return this._outputFile(file, data, { flags })
   }
 
-  _read(
+  async _createOutputStreamChecksum(file: File, { checksum = false, ...options }: Object = {}): Promise<LaxWritable> {
-    file: File,
+    if (typeof file === 'string') {
-    buffer: Buffer,
+      file = normalizePath(file)
-    position?: number
+    }
-  ): Promise<{| bytesRead: number, buffer: Buffer |}> {
+    const path = typeof file === 'string' ? file : file.path
+    const streamP = timeout.call(
+      this._createOutputStream(file, {
+        flags: 'wx',
+        ...options,
+      }),
+      this._timeout
+    )
+
+    if (!checksum) {
+      return streamP
+    }
+
+    const checksumStream = createChecksumStream()
+    const forwardError = error => {
+      checksumStream.emit('error', error)
+    }
+
+    const stream = await streamP
+    stream.on('error', forwardError)
+    checksumStream.pipe(stream)
+
+    // $FlowFixMe
+    checksumStream.checksumWritten = checksumStream.checksum
+      .then(value => this._outputFile(checksumFile(path), value, { flags: 'wx' }))
+      .catch(forwardError)
+
+    return checksumStream
+  }
+
+  async _outputStream(
+    input: Readable,
+    path: string,
+    { checksum, dirMode, cancelToken = CancelToken.none }: { checksum?: boolean, dirMode?: number }
+  ) {
+    const tmpPath = `${dirname(path)}/.${basename(path)}`
+    const output = await this._createOutputStreamChecksum(tmpPath, { checksum })
+    try {
+      cancelToken.promise.then(reason => {
+        input.destroy(reason)
+      })
+      input.pipe(output)
+      await fromEvent(output, 'finish')
+      await output.checksumWritten
+      // $FlowFixMe
+      await input.task
+      await this.rename(tmpPath, path, { checksum })
+    } catch (error) {
+      await this.unlink(tmpPath, { checksum })
+      throw error
+    }
+  }
+
+  _read(file: File, buffer: Buffer, position?: number): Promise<{| bytesRead: number, buffer: Buffer |}> {
     throw new Error('Not implemented')
   }
 
@@ -586,30 +547,22 @@ export default class RemoteHandlerAbstract {
   async _write(file: File, buffer: Buffer, position: number): Promise<void> {
     const isPath = typeof file === 'string'
     if (isPath) {
-      file = await this.openFile(file, 'r+')
+      file = await this.__openFile(file, 'r+')
     }
     try {
       return await this._writeFd(file, buffer, position)
     } finally {
       if (isPath) {
-        await this.closeFile(file)
+        await this.__closeFile(file)
       }
     }
   }
 
-  async _writeFd(
+  async _writeFd(fd: FileDescriptor, buffer: Buffer, position: number): Promise<void> {
-    fd: FileDescriptor,
-    buffer: Buffer,
-    position: number
-  ): Promise<void> {
     throw new Error('Not implemented')
   }
 
-  async _writeFile(
+  async _writeFile(file: string, data: Data, options: { flags?: string }): Promise<void> {
-    file: string,
-    data: Data,
-    options: { flags?: string }
-  ): Promise<void> {
     throw new Error('Not implemented')
   }
 }
@@ -629,13 +582,12 @@ function createPrefixWrapperMethods() {
     if (
       hasOwnProperty.call(pPw, name) ||
       name[0] === '_' ||
-      typeof (value = (descriptor = getOwnPropertyDescriptor(pRha, name))
+      typeof (value = (descriptor = getOwnPropertyDescriptor(pRha, name)).value) !== 'function'
-        .value) !== 'function'
     ) {
       return
     }
 
-    descriptor.value = function() {
+    descriptor.value = function () {
       let path
       if (arguments.length !== 0 && typeof (path = arguments[0]) === 'string') {
         arguments[0] = this._resolve(path)

@xen-orchestra/fs/src/abstract.spec.js

@@ -30,30 +30,6 @@ describe('closeFile()', () => {
   })
 })
 
-describe('createOutputStream()', () => {
-  it(`throws in case of timeout`, async () => {
-    const testHandler = new TestHandler({
-      createOutputStream: () => new Promise(() => {}),
-    })
-
-    const promise = testHandler.createOutputStream('File')
-    jest.advanceTimersByTime(TIMEOUT)
-    await expect(promise).rejects.toThrowError(TimeoutError)
-  })
-})
-
-describe('createReadStream()', () => {
-  it(`throws in case of timeout`, async () => {
-    const testHandler = new TestHandler({
-      createReadStream: () => new Promise(() => {}),
-    })
-
-    const promise = testHandler.createReadStream('file')
-    jest.advanceTimersByTime(TIMEOUT)
-    await expect(promise).rejects.toThrowError(TimeoutError)
-  })
-})
-
 describe('getInfo()', () => {
   it('throws in case of timeout', async () => {
     const testHandler = new TestHandler({

@xen-orchestra/fs/src/checksum.js

@@ -27,9 +27,7 @@ const ID_TO_ALGORITHM = invert(ALGORITHM_TO_ID)
 //    const checksumStream = source.pipe(createChecksumStream())
 //    checksumStream.resume() // make the data flow without an output
 //    console.log(await checksumStream.checksum)
-export const createChecksumStream = (
+export const createChecksumStream = (algorithm: string = 'md5'): Transform & { checksum: Promise<string> } => {
-  algorithm: string = 'md5'
-): Transform & { checksum: Promise<string> } => {
   const algorithmId = ALGORITHM_TO_ID[algorithm]
 
   if (!algorithmId) {
@@ -60,10 +58,7 @@ export const validChecksumOfReadStream = (
   stream: Readable,
   expectedChecksum: string
 ): Readable & { checksumVerified: Promise<void> } => {
-  const algorithmId = expectedChecksum.slice(
+  const algorithmId = expectedChecksum.slice(1, expectedChecksum.indexOf('$', 1))
-    1,
-    expectedChecksum.indexOf('$', 1)
-  )
 
   if (!algorithmId) {
     throw new Error(`unknown algorithm: ${algorithmId}`)
@@ -82,11 +77,7 @@ export const validChecksumOfReadStream = (
         const checksum = `$${algorithmId}$$${hash.digest('hex')}`
 
         callback(
-          checksum !== expectedChecksum
+          checksum !== expectedChecksum ? new Error(`Bad checksum (${checksum}), expected: ${expectedChecksum}`) : null
-            ? new Error(
-                `Bad checksum (${checksum}), expected: ${expectedChecksum}`
-              )
-            : null
         )
       }
     )

@xen-orchestra/fs/src/fs.spec.js

@@ -2,13 +2,10 @@
 
 import 'dotenv/config'
 import asyncIteratorToStream from 'async-iterator-to-stream'
-import getStream from 'get-stream'
 import { forOwn, random } from 'lodash'
-import { fromCallback } from 'promise-toolbox'
-import { pipeline } from 'readable-stream'
 import { tmpdir } from 'os'
 
-import { getHandler } from '.'
+import { getHandler } from './'
 
 // https://gist.github.com/julien-f/3228c3f34fdac01ade09
 const unsecureRandomBytes = n => {
@@ -28,7 +25,7 @@ const unsecureRandomBytes = n => {
 
 const TEST_DATA_LEN = 1024
 const TEST_DATA = unsecureRandomBytes(TEST_DATA_LEN)
-const createTestDataStream = asyncIteratorToStream(function*() {
+const createTestDataStream = asyncIteratorToStream(function* () {
   yield TEST_DATA
 })
 
@@ -83,39 +80,13 @@ handlers.forEach(url => {
       })
     })
 
-    describe('#createOutputStream()', () => {
+    describe('#outputStream()', () => {
       it('creates parent dir if missing', async () => {
-        const stream = await handler.createOutputStream('dir/file')
+        await handler.outputStream('dir/file', createTestDataStream())
-        await fromCallback(pipeline, createTestDataStream(), stream)
         await expect(await handler.readFile('dir/file')).toEqual(TEST_DATA)
       })
     })
 
-    describe('#createReadStream()', () => {
-      beforeEach(() => handler.outputFile('file', TEST_DATA))
-
-      testWithFileDescriptor('file', 'r', async ({ file, flags }) => {
-        await expect(
-          await getStream.buffer(
-            await handler.createReadStream(file, { flags })
-          )
-        ).toEqual(TEST_DATA)
-      })
-    })
-
-    describe('#createWriteStream()', () => {
-      testWithFileDescriptor('file', 'wx', async ({ file, flags }) => {
-        const stream = await handler.createWriteStream(file, { flags })
-        await fromCallback(pipeline, createTestDataStream(), stream)
-        await expect(await handler.readFile('file')).toEqual(TEST_DATA)
-      })
-
-      it('fails if parent dir is missing', async () => {
-        const error = await rejectionOf(handler.createWriteStream('dir/file'))
-        expect(error.code).toBe('ENOENT')
-      })
-    })
-
     describe('#getInfo()', () => {
       let info
       beforeAll(async () => {
@@ -152,16 +123,12 @@ handlers.forEach(url => {
 
       it('can prepend the directory to entries', async () => {
         await handler.outputFile('dir/file', '')
-        expect(await handler.list('dir', { prependDir: true })).toEqual([
+        expect(await handler.list('dir', { prependDir: true })).toEqual(['/dir/file'])
-          '/dir/file',
-        ])
       })
 
       it('can prepend the directory to entries', async () => {
         await handler.outputFile('dir/file', '')
-        expect(await handler.list('dir', { prependDir: true })).toEqual([
+        expect(await handler.list('dir', { prependDir: true })).toEqual(['/dir/file'])
-          '/dir/file',
-        ])
       })
     })
 
@@ -219,6 +186,12 @@ handlers.forEach(url => {
         const error = await rejectionOf(handler.outputFile('file', ''))
         expect(error.code).toBe('EEXIST')
       })
+
+      it("shouldn't timeout in case of the respect of the parallel execution restriction", async () => {
+        const handler = getHandler({ url }, { maxParallelOperations: 1 })
+        await handler.sync()
+        await handler.outputFile(`xo-fs-tests-${Date.now()}/test`, '')
+      }, 40)
     })
 
     describe('#read()', () => {
@@ -328,10 +301,7 @@ handlers.forEach(url => {
             return { offset, expected }
           })(),
           'increase file size': (() => {
-            const offset = random(
+            const offset = random(TEST_DATA_LEN - PATCH_DATA_LEN + 1, TEST_DATA_LEN)
-              TEST_DATA_LEN - PATCH_DATA_LEN + 1,
-              TEST_DATA_LEN
-            )
 
             const expected = Buffer.alloc(offset + PATCH_DATA_LEN)
             TEST_DATA.copy(expected)

@xen-orchestra/fs/src/index.js

@@ -4,6 +4,7 @@ import execa from 'execa'
 import type RemoteHandler from './abstract'
 import RemoteHandlerLocal from './local'
 import RemoteHandlerNfs from './nfs'
+import RemoteHandlerS3 from './s3'
 import RemoteHandlerSmb from './smb'
 import RemoteHandlerSmbMount from './smb-mount'
 
@@ -13,6 +14,7 @@ export type Remote = { url: string }
 const HANDLERS = {
   file: RemoteHandlerLocal,
   nfs: RemoteHandlerNfs,
+  s3: RemoteHandlerS3,
 }
 
 try {

@xen-orchestra/fs/src/local.js

@@ -63,9 +63,7 @@ export default class LocalHandler extends RemoteHandlerAbstract {
   }
 
   async _getSize(file) {
-    const stats = await fs.stat(
+    const stats = await fs.stat(this._getFilePath(typeof file === 'string' ? file : file.path))
-      this._getFilePath(typeof file === 'string' ? file : file.path)
-    )
     return stats.size
   }
 
@@ -73,8 +71,8 @@ export default class LocalHandler extends RemoteHandlerAbstract {
     return fs.readdir(this._getFilePath(dir))
   }
 
-  _mkdir(dir) {
+  _mkdir(dir, { mode }) {
-    return fs.mkdir(this._getFilePath(dir))
+    return fs.mkdir(this._getFilePath(dir), { mode })
   }
 
   async _openFile(path, flags) {
@@ -85,13 +83,7 @@ export default class LocalHandler extends RemoteHandlerAbstract {
     const needsClose = typeof file === 'string'
     file = needsClose ? await fs.open(this._getFilePath(file), 'r') : file.fd
     try {
-      return await fs.read(
+      return await fs.read(file, buffer, 0, buffer.length, position === undefined ? null : position)
-        file,
-        buffer,
-        0,
-        buffer.length,
-        position === undefined ? null : position
-      )
     } finally {
       if (needsClose) {
         await fs.close(file)

@xen-orchestra/fs/src/nfs.js

@@ -2,15 +2,13 @@ import { parse } from 'xo-remote-parser'
 
 import MountHandler from './_mount'
 
-const DEFAULT_NFS_OPTIONS = 'vers=3'
-
 export default class NfsHandler extends MountHandler {
   constructor(remote, opts) {
     const { host, port, path } = parse(remote.url)
     super(remote, opts, {
       type: 'nfs',
       device: `${host}${port !== undefined ? ':' + port : ''}:${path}`,
-      options: DEFAULT_NFS_OPTIONS,
+      defaultOptions: 'vers=3',
     })
   }
 

@xen-orchestra/fs/src/s3.js

@@ -0,0 +1,303 @@
+import aws from '@sullux/aws-sdk'
+import assert from 'assert'
+import http from 'http'
+import { parse } from 'xo-remote-parser'
+
+import RemoteHandlerAbstract from './abstract'
+import { createChecksumStream } from './checksum'
+import CancelToken from 'promise-toolbox/CancelToken'
+
+// endpoints https://docs.aws.amazon.com/general/latest/gr/s3.html
+
+// limits: https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html
+const MIN_PART_SIZE = 1024 * 1024 * 5 // 5MB
+const MAX_PART_SIZE = 1024 * 1024 * 1024 * 5 // 5GB
+const MAX_PARTS_COUNT = 10000
+const MAX_OBJECT_SIZE = 1024 * 1024 * 1024 * 1024 * 5 // 5TB
+const IDEAL_FRAGMENT_SIZE = Math.ceil(MAX_OBJECT_SIZE / MAX_PARTS_COUNT) // the smallest fragment size that still allows a 5TB upload in 10000 fragments, about 524MB
+
+const USE_SSL = true
+export default class S3Handler extends RemoteHandlerAbstract {
+  constructor(remote, _opts) {
+    super(remote)
+    const { host, path, username, password } = parse(remote.url)
+    // https://www.zenko.io/blog/first-things-first-getting-started-scality-s3-server/
+    const params = {
+      accessKeyId: username,
+      apiVersion: '2006-03-01',
+      endpoint: host,
+      s3ForcePathStyle: true,
+      secretAccessKey: password,
+      signatureVersion: 'v4',
+      httpOptions: {
+        timeout: 600000,
+      },
+    }
+    if (!USE_SSL) {
+      params.httpOptions.agent = new http.Agent()
+      params.sslEnabled = false
+    }
+    this._s3 = aws(params).s3
+
+    const splitPath = path.split('/').filter(s => s.length)
+    this._bucket = splitPath.shift()
+    this._dir = splitPath.join('/')
+  }
+
+  get type() {
+    return 's3'
+  }
+
+  _createParams(file) {
+    return { Bucket: this._bucket, Key: this._dir + file }
+  }
+
+  async _outputStream(input, path, { checksum, cancelToken = CancelToken.none }) {
+    cancelToken.promise.then(reason => {
+      input.destroy(reason)
+    })
+    let inputStream = input
+    if (checksum) {
+      const checksumStream = createChecksumStream()
+      const forwardError = error => {
+        checksumStream.emit('error', error)
+      }
+      input.pipe(checksumStream)
+      input.on('error', forwardError)
+      inputStream = checksumStream
+    }
+    await this._s3.upload(
+      {
+        ...this._createParams(path),
+        Body: inputStream,
+      },
+      { partSize: IDEAL_FRAGMENT_SIZE, queueSize: 1 }
+    )
+    if (checksum) {
+      const checksum = await inputStream.checksum
+      const params = {
+        ...this._createParams(path + '.checksum'),
+        Body: checksum,
+      }
+      await this._s3.upload(params)
+    }
+    await input.task
+  }
+
+  async _writeFile(file, data, options) {
+    return this._s3.putObject({ ...this._createParams(file), Body: data })
+  }
+
+  async _createReadStream(file, options) {
+    // https://github.com/Sullux/aws-sdk/issues/11
+    return this._s3.getObject.raw(this._createParams(file)).createReadStream()
+  }
+
+  async _unlink(file) {
+    return this._s3.deleteObject(this._createParams(file))
+  }
+
+  async _list(dir) {
+    function splitPath(path) {
+      return path.split('/').filter(d => d.length)
+    }
+
+    const prefix = [this._dir, dir].join('/')
+    const splitPrefix = splitPath(prefix)
+    const result = await this._s3.listObjectsV2({
+      Bucket: this._bucket,
+      Prefix: splitPrefix.join('/'),
+    })
+    const uniq = new Set()
+    for (const entry of result.Contents) {
+      const line = splitPath(entry.Key)
+      if (line.length > splitPrefix.length) {
+        uniq.add(line[splitPrefix.length])
+      }
+    }
+    return [...uniq]
+  }
+
+  async _rename(oldPath, newPath) {
+    const size = await this._getSize(oldPath)
+    const multipartParams = await this._s3.createMultipartUpload({ ...this._createParams(newPath) })
+    const param2 = { ...multipartParams, CopySource: `/${this._bucket}/${this._dir}${oldPath}` }
+    try {
+      const parts = []
+      let start = 0
+      while (start < size) {
+        const range = `bytes=${start}-${Math.min(start + MAX_PART_SIZE, size) - 1}`
+        const partParams = { ...param2, PartNumber: parts.length + 1, CopySourceRange: range }
+        const upload = await this._s3.uploadPartCopy(partParams)
+        parts.push({ ETag: upload.CopyPartResult.ETag, PartNumber: partParams.PartNumber })
+        start += MAX_PART_SIZE
+      }
+      await this._s3.completeMultipartUpload({ ...multipartParams, MultipartUpload: { Parts: parts } })
+    } catch (e) {
+      await this._s3.abortMultipartUpload(multipartParams)
+      throw e
+    }
+    await this._s3.deleteObject(this._createParams(oldPath))
+  }
+
+  async _getSize(file) {
+    if (typeof file !== 'string') {
+      file = file.fd
+    }
+    const result = await this._s3.headObject(this._createParams(file))
+    return +result.ContentLength
+  }
+
+  async _read(file, buffer, position = 0) {
+    if (typeof file !== 'string') {
+      file = file.fd
+    }
+    const params = this._createParams(file)
+    params.Range = `bytes=${position}-${position + buffer.length - 1}`
+    const result = await this._s3.getObject(params)
+    result.Body.copy(buffer)
+    return { bytesRead: result.Body.length, buffer }
+  }
+
+  async _write(file, buffer, position) {
+    if (typeof file !== 'string') {
+      file = file.fd
+    }
+    const uploadParams = this._createParams(file)
+    let fileSize
+    try {
+      fileSize = +(await this._s3.headObject(uploadParams)).ContentLength
+    } catch (e) {
+      if (e.code === 'NotFound') {
+        fileSize = 0
+      } else {
+        throw e
+      }
+    }
+    if (fileSize < MIN_PART_SIZE) {
+      const resultBuffer = Buffer.alloc(Math.max(fileSize, position + buffer.length))
+      const fileContent = fileSize !== 0 ? (await this._s3.getObject(uploadParams)).Body : Buffer.alloc(0)
+      fileContent.copy(resultBuffer)
+      buffer.copy(resultBuffer, position)
+      await this._s3.putObject({ ...uploadParams, Body: resultBuffer })
+      return { buffer, bytesWritten: buffer.length }
+    } else {
+      // using this trick: https://stackoverflow.com/a/38089437/72637
+      // multipart fragments have a minimum size of 5Mo and a max of 5Go unless they are last
+      // splitting the file in 3 parts: [prefix, edit, suffix]
+      // if `prefix` is bigger than 5Mo, it will be sourced from uploadPartCopy()
+      // otherwise otherwise it will be downloaded, concatenated to `edit`
+      // `edit` will always be an upload part
+      // `suffix` will always be sourced from uploadPartCopy()
+      // Then everything will be sliced in 5Gb parts before getting uploaded
+      const multipartParams = await this._s3.createMultipartUpload(uploadParams)
+      const copyMultipartParams = {
+        ...multipartParams,
+        CopySource: `/${this._bucket}/${this._dir + file}`,
+      }
+      try {
+        const parts = []
+        const prefixSize = position
+        let suffixOffset = prefixSize + buffer.length
+        let suffixSize = Math.max(0, fileSize - suffixOffset)
+        let hasSuffix = suffixSize > 0
+        let editBuffer = buffer
+        let editBufferOffset = position
+        let partNumber = 1
+        let prefixPosition = 0
+        // use floor() so that last fragment is handled in the if bellow
+        let fragmentsCount = Math.floor(prefixSize / MAX_PART_SIZE)
+        const prefixFragmentSize = MAX_PART_SIZE
+        let prefixLastFragmentSize = prefixSize - prefixFragmentSize * fragmentsCount
+        if (prefixLastFragmentSize >= MIN_PART_SIZE) {
+          // the last fragment of the prefix is smaller than MAX_PART_SIZE, but bigger than the minimum
+          // so we can copy it too
+          fragmentsCount++
+          prefixLastFragmentSize = 0
+        }
+        for (let i = 0; i < fragmentsCount; i++) {
+          const fragmentEnd = Math.min(prefixPosition + prefixFragmentSize, prefixSize)
+          assert.strictEqual(fragmentEnd - prefixPosition <= MAX_PART_SIZE, true)
+          const range = `bytes=${prefixPosition}-${fragmentEnd - 1}`
+          const copyPrefixParams = { ...copyMultipartParams, PartNumber: partNumber++, CopySourceRange: range }
+          const part = await this._s3.uploadPartCopy(copyPrefixParams)
+          parts.push({ ETag: part.CopyPartResult.ETag, PartNumber: copyPrefixParams.PartNumber })
+          prefixPosition += prefixFragmentSize
+        }
+        if (prefixLastFragmentSize) {
+          // grab everything from the prefix that was too small to be copied, download and merge to the edit buffer.
+          const downloadParams = { ...uploadParams, Range: `bytes=${prefixPosition}-${prefixSize - 1}` }
+          const prefixBuffer = prefixSize > 0 ? (await this._s3.getObject(downloadParams)).Body : Buffer.alloc(0)
+          editBuffer = Buffer.concat([prefixBuffer, buffer])
+          editBufferOffset -= prefixLastFragmentSize
+        }
+        if (hasSuffix && editBuffer.length < MIN_PART_SIZE) {
+          // the edit fragment is too short and is not the last fragment
+          // let's steal from the suffix fragment to reach the minimum size
+          // the suffix might be too short and itself entirely absorbed in the edit fragment, making it the last one.
+          const complementSize = Math.min(MIN_PART_SIZE - editBuffer.length, suffixSize)
+          const complementOffset = editBufferOffset + editBuffer.length
+          suffixOffset += complementSize
+          suffixSize -= complementSize
+          hasSuffix = suffixSize > 0
+          const prefixRange = `bytes=${complementOffset}-${complementOffset + complementSize - 1}`
+          const downloadParams = { ...uploadParams, Range: prefixRange }
+          const complementBuffer = (await this._s3.getObject(downloadParams)).Body
+          editBuffer = Buffer.concat([editBuffer, complementBuffer])
+        }
+        const editParams = { ...multipartParams, Body: editBuffer, PartNumber: partNumber++ }
+        const editPart = await this._s3.uploadPart(editParams)
+        parts.push({ ETag: editPart.ETag, PartNumber: editParams.PartNumber })
+        if (hasSuffix) {
+          // use ceil because the last fragment can be arbitrarily small.
+          const suffixFragments = Math.ceil(suffixSize / MAX_PART_SIZE)
+          let suffixFragmentOffset = suffixOffset
+          for (let i = 0; i < suffixFragments; i++) {
+            const fragmentEnd = suffixFragmentOffset + MAX_PART_SIZE
+            assert.strictEqual(Math.min(fileSize, fragmentEnd) - suffixFragmentOffset <= MAX_PART_SIZE, true)
+            const suffixRange = `bytes=${suffixFragmentOffset}-${Math.min(fileSize, fragmentEnd) - 1}`
+            const copySuffixParams = { ...copyMultipartParams, PartNumber: partNumber++, CopySourceRange: suffixRange }
+            const suffixPart = (await this._s3.uploadPartCopy(copySuffixParams)).CopyPartResult
+            parts.push({ ETag: suffixPart.ETag, PartNumber: copySuffixParams.PartNumber })
+            suffixFragmentOffset = fragmentEnd
+          }
+        }
+        await this._s3.completeMultipartUpload({
+          ...multipartParams,
+          MultipartUpload: { Parts: parts },
+        })
+      } catch (e) {
+        await this._s3.abortMultipartUpload(multipartParams)
+        throw e
+      }
+    }
+  }
+
+  async _openFile(path, flags) {
+    return path
+  }
+
+  async _closeFile(fd) {}
+
+  // https://stackoverflow.com/a/48955582/72637
+  async _rmtree(dir) {
+    const listParams = {
+      Bucket: this._bucket,
+      Prefix: this._dir + dir,
+    }
+    let listedObjects = {}
+    do {
+      listedObjects = await this._s3.listObjectsV2({
+        ...listParams,
+        ContinuationToken: listedObjects.NextContinuationToken,
+      })
+      if (listedObjects.Contents.length === 0) {
+        return
+      }
+      await this._s3.deleteObjects({
+        Bucket: this._bucket,
+        Delete: { Objects: listedObjects.Contents.map(({ Key }) => ({ Key })) },
+      })
+    } while (listedObjects.IsTruncated)
+  }
+}

@xen-orchestra/fs/src/smb-mount.js

@@ -5,9 +5,7 @@ import normalizePath from './_normalizePath'
 
 export default class SmbMountHandler extends MountHandler {
   constructor(remote, opts) {
-    const { domain = 'WORKGROUP', host, password, path, username } = parse(
+    const { domain = 'WORKGROUP', host, password, path, username } = parse(remote.url)
-      remote.url
-    )
     super(remote, opts, {
       type: 'cifs',
       device: '//' + host + normalizePath(path),

@xen-orchestra/fs/src/smb.js

@@ -17,8 +17,7 @@ const normalizeError = (error, shouldBeDirectory) => {
     ? wrapError(error, 'EISDIR')
     : code === 'STATUS_NOT_A_DIRECTORY'
     ? wrapError(error, 'ENOTDIR')
-    : code === 'STATUS_OBJECT_NAME_NOT_FOUND' ||
+    : code === 'STATUS_OBJECT_NAME_NOT_FOUND' || code === 'STATUS_OBJECT_PATH_NOT_FOUND'
-      code === 'STATUS_OBJECT_PATH_NOT_FOUND'
     ? wrapError(error, 'ENOENT')
     : code === 'STATUS_OBJECT_NAME_COLLISION'
     ? wrapError(error, 'EEXIST')
@@ -44,12 +43,7 @@ export default class SmbHandler extends RemoteHandlerAbstract {
   }
 
   _getFilePath(file) {
-    return (
+    return this._prefix + (typeof file === 'string' ? file : file.path).slice(1).replace(/\//g, '\\')
-      this._prefix +
-      (typeof file === 'string' ? file : file.path)
-        .slice(1)
-        .replace(/\//g, '\\')
-    )
   }
 
   _dirname(file) {
@@ -96,15 +90,13 @@ export default class SmbHandler extends RemoteHandlerAbstract {
     return this._client.readdir(this._getFilePath(dir)).catch(normalizeDirError)
   }
 
-  _mkdir(dir) {
+  _mkdir(dir, { mode }) {
-    return this._client.mkdir(this._getFilePath(dir)).catch(normalizeDirError)
+    return this._client.mkdir(this._getFilePath(dir), mode).catch(normalizeDirError)
   }
 
   // TODO: add flags
   _openFile(path, flags) {
-    return this._client
+    return this._client.open(this._getFilePath(path), flags).catch(normalizeError)
-      .open(this._getFilePath(path), flags)
-      .catch(normalizeError)
   }
 
   async _read(file, buffer, position) {
@@ -123,9 +115,7 @@ export default class SmbHandler extends RemoteHandlerAbstract {
   }
 
   _readFile(file, options) {
-    return this._client
+    return this._client.readFile(this._getFilePath(file), options).catch(normalizeError)
-      .readFile(this._getFilePath(file), options)
-      .catch(normalizeError)
   }
 
   _rename(oldPath, newPath) {
@@ -156,9 +146,7 @@ export default class SmbHandler extends RemoteHandlerAbstract {
   }
 
   _truncate(file, len) {
-    return this._client
+    return this._client.truncate(this._getFilePath(file), len).catch(normalizeError)
-      .truncate(this._getFilePath(file), len)
-      .catch(normalizeError)
   }
 
   _unlink(file) {
@@ -170,8 +158,6 @@ export default class SmbHandler extends RemoteHandlerAbstract {
   }
 
   _writeFile(file, data, options) {
-    return this._client
+    return this._client.writeFile(this._getFilePath(file), data, options).catch(normalizeError)
-      .writeFile(this._getFilePath(file), data, options)
-      .catch(normalizeError)
   }
 }

@xen-orchestra/log/.babelrc.js

@@ -1,3 +1 @@
-module.exports = require('../../@xen-orchestra/babel-config')(
+module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))
-  require('./package.json')
-)

@xen-orchestra/log/README.md

@@ -1,6 +1,8 @@
-# @xen-orchestra/log [![Build Status](https://travis-ci.org/vatesfr/xen-orchestra.png?branch=master)](https://travis-ci.org/vatesfr/xen-orchestra)
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
 
-> ${pkg.description}
+# @xen-orchestra/log
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/log)](https://npmjs.org/package/@xen-orchestra/log) ![License](https://badgen.net/npm/license/@xen-orchestra/log) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/log)](https://bundlephobia.com/result?p=@xen-orchestra/log) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/log)](https://npmjs.org/package/@xen-orchestra/log)
 
 ## Install
 
@@ -51,13 +53,10 @@ const transport = transportEmail({
   service: 'gmail',
   auth: {
     user: 'jane.smith@gmail.com',
-    pass: 'H&NbECcpXF|pyXe#%ZEb'
+    pass: 'H&NbECcpXF|pyXe#%ZEb',
   },
   from: 'jane.smith@gmail.com',
-  to: [
+  to: ['jane.smith@gmail.com', 'sam.doe@yahoo.com'],
-    'jane.smith@gmail.com',
-    'sam.doe@yahoo.com'
-  ]
 })
 
 configure([
@@ -67,14 +66,14 @@ configure([
     // matched against the namespace of the logs
     filter: process.env.DEBUG,
 
-    transport: transportConsole()
+    transport: transportConsole(),
   },
   {
     // only levels >= warn
     level: 'warn',
 
-    transport
+    transport,
-  }
+  },
 ])
 
 // send all global errors (uncaught exceptions, warnings, unhandled rejections)
@@ -105,18 +104,17 @@ Configuration:
 ```js
 import transportEmail from '@xen-orchestra/log/transports/email'
 
-configure(transportEmail({
+configure(
-  service: 'gmail',
+  transportEmail({
-  auth: {
+    service: 'gmail',
-    user: 'jane.smith@gmail.com',
+    auth: {
-    pass: 'H&NbECcpXF|pyXe#%ZEb'
+      user: 'jane.smith@gmail.com',
-  },
+      pass: 'H&NbECcpXF|pyXe#%ZEb',
-  from: 'jane.smith@gmail.com',
+    },
-  to: [
+    from: 'jane.smith@gmail.com',
-    'jane.smith@gmail.com',
+    to: ['jane.smith@gmail.com', 'sam.doe@yahoo.com'],
-    'sam.doe@yahoo.com'
+  })
-  ]
+)
-}))
 ```
 
 #### Syslog
@@ -139,36 +137,17 @@ configure(transportSyslog())
 configure(transportSyslog('tcp://syslog.company.lan'))
 ```
 
-## Development
-
-```
-# Install dependencies
-> yarn
-
-# Run the tests
-> yarn test
-
-# Continuously compile
-> yarn dev
-
-# Continuously run the tests
-> yarn dev-test
-
-# Build for production (automatically called by npm install)
-> yarn build
-```
-
 ## Contributions
 
-Contributions are *very* welcomed, either on the documentation or on
+Contributions are _very_ welcomed, either on the documentation or on
 the code.
 
 You may:
 
-- report any [issue](https://github.com/vatesfr/xo-web/issues/)
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
   you've encountered;
 - fork and create a pull request.
 
 ## License
 
-ISC © [Vates SAS](https://vates.fr)
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@xen-orchestra/log/USAGE.md

@@ -0,0 +1,122 @@
+Everywhere something should be logged:
+
+```js
+import { createLogger } from '@xen-orchestra/log'
+
+const log = createLogger('my-module')
+
+log.debug('only useful for debugging')
+log.info('this information is relevant to the user')
+log.warn('something went wrong but did not prevent current action')
+log.error('something went wrong')
+log.fatal('service/app is going down')
+
+// you can add contextual info
+log.debug('new API request', {
+  method: 'foo',
+  params: [ 'bar', 'baz' ]
+  user: 'qux'
+})
+
+// by convention, errors go into the `error` field
+log.error('could not join server', {
+  error,
+  server: 'example.org',
+})
+```
+
+Then, at application level, configure the logs are handled:
+
+```js
+import { createLogger } from '@xen-orchestra/log'
+import { configure, catchGlobalErrors } from '@xen-orchestra/log/configure'
+import transportConsole from '@xen-orchestra/log/transports/console'
+import transportEmail from '@xen-orchestra/log/transports/email'
+
+const transport = transportEmail({
+  service: 'gmail',
+  auth: {
+    user: 'jane.smith@gmail.com',
+    pass: 'H&NbECcpXF|pyXe#%ZEb',
+  },
+  from: 'jane.smith@gmail.com',
+  to: ['jane.smith@gmail.com', 'sam.doe@yahoo.com'],
+})
+
+configure([
+  {
+    // if filter is a string, then it is pattern
+    // (https://github.com/visionmedia/debug#wildcards) which is
+    // matched against the namespace of the logs
+    filter: process.env.DEBUG,
+
+    transport: transportConsole(),
+  },
+  {
+    // only levels >= warn
+    level: 'warn',
+
+    transport,
+  },
+])
+
+// send all global errors (uncaught exceptions, warnings, unhandled rejections)
+// to this logger
+catchGlobalErrors(createLogger('app'))
+```
+
+### Transports
+
+#### Console
+
+```js
+import transportConsole from '@xen-orchestra/log/transports/console'
+
+configure(transportConsole())
+```
+
+#### Email
+
+Optional dependency:
+
+```
+> yarn add nodemailer pretty-format
+```
+
+Configuration:
+
+```js
+import transportEmail from '@xen-orchestra/log/transports/email'
+
+configure(
+  transportEmail({
+    service: 'gmail',
+    auth: {
+      user: 'jane.smith@gmail.com',
+      pass: 'H&NbECcpXF|pyXe#%ZEb',
+    },
+    from: 'jane.smith@gmail.com',
+    to: ['jane.smith@gmail.com', 'sam.doe@yahoo.com'],
+  })
+)
+```
+
+#### Syslog
+
+Optional dependency:
+
+```
+> yarn add split-host syslog-client
+```
+
+Configuration:
+
+```js
+import transportSyslog from '@xen-orchestra/log/transports/syslog'
+
+// By default, log to udp://localhost:514
+configure(transportSyslog())
+
+// But TCP, a different host, or a different port can be used
+configure(transportSyslog('tcp://syslog.company.lan'))
+```

@xen-orchestra/log/package.json

@@ -1,4 +1,5 @@
 {
+  "private": false,
   "name": "@xen-orchestra/log",
   "version": "0.2.0",
   "license": "ISC",
@@ -12,8 +13,8 @@
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
   "author": {
-    "name": "Julien Fontanet",
+    "name": "Vates SAS",
-    "email": "julien.fontanet@vates.fr"
+    "url": "https://vates.fr"
   },
   "preferGlobal": false,
   "main": "dist/",
@@ -31,14 +32,14 @@
   },
   "dependencies": {
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.14.0"
+    "promise-toolbox": "^0.16.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^6.0.3",
+    "cross-env": "^7.0.2",
     "index-modules": "^0.3.0",
     "rimraf": "^3.0.0"
   },
@@ -48,7 +49,7 @@
     "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
     "prebuild": "yarn run clean",
     "predev": "yarn run prebuild",
-    "prepare": "yarn run build",
+    "prepublishOnly": "yarn run build",
     "postversion": "npm publish"
   }
 }

@xen-orchestra/log/src/configure.js

@@ -12,7 +12,7 @@ const createTransport = config => {
   if (Array.isArray(config)) {
     const transports = config.map(createTransport)
     const { length } = transports
-    return function() {
+    return function () {
       for (let i = 0; i < length; ++i) {
         transports[i].apply(this, arguments)
       }
@@ -30,14 +30,14 @@ const createTransport = config => {
     }
 
     const orig = transport
-    transport = function(log) {
+    transport = function (log) {
       if ((level !== undefined && log.level >= level) || filter(log)) {
         return orig.apply(this, arguments)
       }
     }
   } else if (level !== undefined) {
     const orig = transport
-    transport = function(log) {
+    transport = function (log) {
       if (log.level >= level) {
         return orig.apply(this, arguments)
       }
@@ -47,10 +47,7 @@ const createTransport = config => {
   return transport
 }
 
-const symbol =
+const symbol = typeof Symbol !== 'undefined' ? Symbol.for('@xen-orchestra/log') : '@@@xen-orchestra/log'
-  typeof Symbol !== 'undefined'
-    ? Symbol.for('@xen-orchestra/log')
-    : '@@@xen-orchestra/log'
 
 const { env } = process
 global[symbol] = createTransport({