feat(xo): export/import config (#427)

See vatesfr/xo-web#786

package.json

@@ -98,6 +98,7 @@
     "tar-stream": "^1.5.2",
     "through2": "^2.0.0",
     "trace": "^2.0.1",
+    "uuid": "^2.0.3",
     "ws": "^1.1.1",
     "xen-api": "^0.9.4",
     "xml2js": "~0.4.6",

src/api/job.js

@@ -18,7 +18,9 @@ get.params = {
 }
 
 export async function create ({job}) {
-  return (await this.createJob(this.session.get('user_id'), job)).id
+  job.userId = this.session.get('user_id')
+
+  return (await this.createJob(job)).id
 }
 
 create.permission = 'admin'

src/api/user.js

@@ -4,7 +4,7 @@ import { getUserPublicProperties, mapToArray } from '../utils'
 // ===================================================================
 
 export async function create ({email, password, permission}) {
-  return (await this.createUser(email, {password, permission})).id
+  return (await this.createUser({email, password, permission})).id
 }
 
 create.description = 'creates a new user'

src/api/xo.js

@@ -1,5 +1,49 @@
+import { streamToBuffer } from '../utils'
+
+// ===================================================================
+
+export function clean () {
+  return this.clean()
+}
+
+clean.permission = 'admin'
+
+// -------------------------------------------------------------------
+
+export async function exportConfig () {
+  return {
+    $getFrom: await this.registerHttpRequest((req, res) => {
+      res.writeHead(200, 'OK', {
+        'content-disposition': 'attachment'
+      })
+
+      return this.exportConfig()
+    },
+    undefined,
+    { suffix: '/config.json' })
+  }
+}
+
+exportConfig.permission = 'admin'
+
+// -------------------------------------------------------------------
+
 export function getAllObjects () {
   return this.getObjects()
 }
 
 getAllObjects.permission = ''
+
+// -------------------------------------------------------------------
+
+export async function importConfig () {
+  return {
+    $sendTo: await this.registerHttpRequest(async (req, res) => {
+      await this.importConfig(JSON.parse(await streamToBuffer(req)))
+
+      res.end('config successfully imported')
+    })
+  }
+}
+
+importConfig.permission = 'admin'

src/collection/redis.js

@@ -3,6 +3,7 @@ import difference from 'lodash/difference'
 import filter from 'lodash/filter'
 import getKey from 'lodash/keys'
 import {createClient as createRedisClient} from 'redis'
+import {v4 as generateUuid} from 'uuid'
 
 import {
   forEach,
@@ -68,12 +69,12 @@ export default class Redis extends Collection {
     // TODO: remove “replace” which is a temporary measure, implement
     // “set()” instead.
 
-    const {indexes, prefix, redis, idPrefix = ''} = this
+    const {indexes, prefix, redis} = this
 
     return Promise.all(mapToArray(models, async model => {
       // Generate a new identifier if necessary.
       if (model.id === undefined) {
-        model.id = idPrefix + String(await redis.incr(prefix + '_id'))
+        model.id = generateUuid()
       }
 
       const success = await redis.sadd(prefix + '_ids', model.id)

src/models/group.js

@@ -14,10 +14,6 @@ export class Groups extends Collection {
     return Group
   }
 
-  get idPrefix () {
-    return 'group:'
-  }
-
   create (name) {
     return this.add(new Group({
       name,

src/models/job.js

@@ -11,12 +11,7 @@ export class Jobs extends Collection {
     return Job
   }
 
-  get idPrefix () {
-    return 'job:'
-  }
-
-  async create (userId, job) {
-    job.userId = userId
+  async create (job) {
     // Serializes.
     job.paramsVector = JSON.stringify(job.paramsVector)
     return /* await */ this.add(new Job(job))

src/models/plugin-metadata.js

@@ -13,10 +13,6 @@ export class PluginsMetadata extends Collection {
     return PluginMetadata
   }
 
-  get idPrefix () {
-    return 'plugin-metadata:'
-  }
-
   async save ({ id, autoload, configuration }) {
     return /* await */ this.update({
       id,

src/models/remote.js

@@ -13,10 +13,6 @@ export class Remotes extends Collection {
     return Remote
   }
 
-  get idPrefix () {
-    return 'remote-'
-  }
-
   create (name, url) {
     return this.add(new Remote({
       name,

src/models/schedule.js

@@ -11,10 +11,6 @@ export class Schedules extends Collection {
     return Schedule
   }
 
-  get idPrefix () {
-    return 'schedule:'
-  }
-
   create (userId, job, cron, enabled, name = undefined, timezone = undefined) {
     return this.add(new Schedule({
       userId,

src/models/user.js

@@ -31,15 +31,14 @@ export class Users extends Collection {
     return User
   }
 
-  async create (email, properties = {}) {
+  async create (properties) {
+    const { email } = properties
+
     // Avoid duplicates.
     if (await this.exists({email})) {
       throw new Error(`the user ${email} already exists`)
     }
 
-    // Adds the email to the user's properties.
-    properties.email = email
-
     // Create the user object.
     const user = new User(properties)
 

src/xo-mixins/acls.js

@@ -24,6 +24,15 @@ export default class {
       prefix: 'xo:acl',
       indexes: ['subject', 'object']
     })
+
+    xo.on('start', () => {
+      xo.addConfigManager('acls',
+        () => this.getAllAcls(),
+        acls => Promise.all(mapToArray(acls, acl =>
+          this.addAcl(acl.subjectId, acl.objectId, acl.action)
+        ))
+      )
+    })
   }
 
   async _getAclsForUser (userId) {

src/xo-mixins/authentication.js

@@ -4,6 +4,7 @@ import {
 } from '../api-errors'
 import {
   createRawObject,
+  forEach,
   generateToken,
   pCatch,
   noop
@@ -30,7 +31,7 @@ export default class {
     this._providers = new Set()
 
     // Creates persistent collections.
-    this._tokens = new Tokens({
+    const tokensDb = this._tokens = new Tokens({
       connection: xo._redis,
       prefix: 'xo:token',
       indexes: ['user_id']
@@ -65,6 +66,25 @@ export default class {
         return
       }
     })
+
+    xo.on('clean', async () => {
+      const tokens = await tokensDb.get()
+      const toRemove = []
+      const now = Date.now()
+      forEach(tokens, ({ expiration, id }) => {
+        if (!expiration || expiration < now) {
+          toRemove.push(id)
+        }
+      })
+      await tokensDb.remove(toRemove)
+    })
+
+    xo.on('start', () => {
+      xo.addConfigManager('authTokens',
+        () => tokensDb.get(),
+        tokens => tokensDb.update(tokens)
+      )
+    })
   }
 
   registerAuthenticationProvider (provider) {

src/xo-mixins/config-management.js

@@ -0,0 +1,33 @@
+import { map, noop } from '../utils'
+
+import { all as pAll } from 'promise-toolbox'
+
+export default class ConfigManagement {
+  constructor () {
+    this._managers = { __proto__: null }
+  }
+
+  addConfigManager (id, exporter, importer) {
+    const managers = this._managers
+    if (id in managers) {
+      throw new Error(`${id} is already taken`)
+    }
+
+    this._managers[id] = { exporter, importer }
+  }
+
+  exportConfig () {
+    return map(this._managers, ({ exporter }, key) => exporter())::pAll()
+  }
+
+  importConfig (config) {
+    const managers = this._managers
+
+    return map(config, (entry, key) => {
+      const manager = managers[key]
+      if (manager) {
+        return manager.importer(entry)
+      }
+    })::pAll().then(noop)
+  }
+}

src/xo-mixins/ip-pools.js

@@ -54,6 +54,11 @@ export default class IpPools {
 
     xo.on('start', async () => {
       this._store = await xo.getStore('ipPools')
+
+      xo.addConfigManager('ipPools',
+        () => this.getAllIpPools(),
+        ipPools => Promise.all(mapToArray(ipPools, ipPool => this._save(ipPool)))
+      )
     })
   }
 

src/xo-mixins/jobs.js

@@ -1,6 +1,8 @@
 import assign from 'lodash/assign'
+
 import JobExecutor from '../job-executor'
 import { Jobs } from '../models/job'
+import { mapToArray } from '../utils'
 import {
   GenericError,
   NoSuchObject
@@ -19,11 +21,20 @@ class NoSuchJob extends NoSuchObject {
 export default class {
   constructor (xo) {
     this._executor = new JobExecutor(xo)
-    this._jobs = new Jobs({
+    const jobsDb = this._jobs = new Jobs({
       connection: xo._redis,
       prefix: 'xo:job',
       indexes: ['user_id', 'key']
     })
+
+    xo.on('start', () => {
+      xo.addConfigManager('jobs',
+        () => jobsDb.get(),
+        jobs => Promise.all(mapToArray(jobs, job =>
+          jobsDb.save(job)
+        ))
+      )
+    })
   }
 
   async getAllJobs () {
@@ -39,9 +50,9 @@ export default class {
     return job.properties
   }
 
-  async createJob (userId, job) {
+  async createJob (job) {
     // TODO: use plain objects
-    const job_ = await this._jobs.create(userId, job)
+    const job_ = await this._jobs.create(job)
     return job_.properties
   }
 

src/xo-mixins/plugins.js

@@ -29,6 +29,15 @@ export default class {
       connection: xo._redis,
       prefix: 'xo:plugin-metadata'
     })
+
+    xo.on('start', () => {
+      xo.addConfigManager('plugins',
+        () => this._pluginsMetadata.get(),
+        plugins => Promise.all(mapToArray(plugins, plugin =>
+          this._pluginsMetadata.save(plugin)
+        ))
+      )
+    })
   }
 
   _getRawPlugin (id) {

src/xo-mixins/remotes.js

@@ -2,7 +2,8 @@ import RemoteHandlerLocal from '../remote-handlers/local'
 import RemoteHandlerNfs from '../remote-handlers/nfs'
 import RemoteHandlerSmb from '../remote-handlers/smb'
 import {
-  forEach
+  forEach,
+  mapToArray
 } from '../utils'
 import {
   NoSuchObject
@@ -30,6 +31,13 @@ export default class {
     })
 
     xo.on('start', async () => {
+      xo.addConfigManager('remotes',
+        () => this._remotes.get(),
+        remotes => Promise.all(mapToArray(remotes, remote =>
+          this._remotes.save(remote)
+        ))
+      )
+
       await this.initRemotes()
       await this.syncAllRemotes()
     })

src/xo-mixins/resource-sets.js

@@ -85,6 +85,13 @@ export default class {
 
     this._store = null
     xo.on('start', async () => {
+      xo.addConfigManager('resourceSets',
+        () => this.getAllResourceSets(),
+        resourceSets => Promise.all(mapToArray(resourceSets, resourceSet =>
+          this._save(resourceSet)
+        ))
+      )
+
       this._store = await xo.getStore('resourceSets')
     })
   }

src/xo-mixins/scheduling.js

@@ -4,6 +4,7 @@ import { Schedules } from '../models/schedule'
 
 import {
   forEach,
+  mapToArray,
   scheduleFn
 } from '../utils'
 
@@ -42,14 +43,23 @@ export class ScheduleAlreadyEnabled extends SchedulerError {
 export default class {
   constructor (xo) {
     this.xo = xo
-    this._redisSchedules = new Schedules({
+    const schedules = this._redisSchedules = new Schedules({
       connection: xo._redis,
       prefix: 'xo:schedule',
       indexes: ['user_id', 'job']
     })
     this._scheduleTable = undefined
 
-    xo.on('start', () => this._loadSchedules())
+    xo.on('start', () => {
+      xo.addConfigManager('schedules',
+        () => schedules.get(),
+        schedules_ => Promise.all(mapToArray(schedules_, schedule =>
+          schedules.save(schedule)
+        ))
+      )
+
+      return this._loadSchedules()
+    })
     xo.on('stop', () => this._disableAll())
   }
 

src/xo-mixins/subjects.js

@@ -52,22 +52,39 @@ export default class {
 
     const redis = xo._redis
 
-    this._groups = new Groups({
+    const groupsDb = this._groups = new Groups({
       connection: redis,
       prefix: 'xo:group'
     })
-    const users = this._users = new Users({
+    const usersDb = this._users = new Users({
       connection: redis,
       prefix: 'xo:user',
       indexes: ['email']
     })
 
     xo.on('start', async () => {
-      if (!await users.exists()) {
+      xo.addConfigManager('groups',
+        () => groupsDb.get(),
+        groups => Promise.all(mapToArray(groups, group => groupsDb.save(group)))
+      )
+      xo.addConfigManager('users',
+        () => usersDb.get(),
+        users => Promise.all(mapToArray(users, async user => {
+          const conflictUsers = await usersDb.get({ email: user.email })
+          if (!isEmpty(conflictUsers)) {
+            await Promise.all(mapToArray(conflictUsers, user =>
+              this.deleteUser(user.id)
+            ))
+          }
+          return usersDb.save(user)
+        }))
+      )
+
+      if (!await usersDb.exists()) {
         const email = 'admin@admin.net'
         const password = 'admin'
 
-        await this.createUser(email, {password, permission: 'admin'})
+        await this.createUser({email, password, permission: 'admin'})
         console.log('[INFO] Default user created:', email, ' with password', password)
       }
     })
@@ -75,13 +92,17 @@ export default class {
 
   // -----------------------------------------------------------------
 
-  async createUser (email, { password, ...properties }) {
+  async createUser ({ name, password, ...properties }) {
+    if (name) {
+      properties.email = name
+    }
+
     if (password) {
       properties.pw_hash = await hash(password)
     }
 
     // TODO: use plain objects
-    const user = await this._users.create(email, properties)
+    const user = await this._users.create(properties)
 
     return user.properties
   }
@@ -210,7 +231,8 @@ export default class {
       throw new Error(`registering ${name} user is forbidden`)
     }
 
-    return /* await */ this.createUser(name, {
+    return /* await */ this.createUser({
+      name,
       _provider: provider
     })
   }

src/xo-mixins/xen-servers.js

@@ -32,7 +32,7 @@ class NoSuchXenServer extends NoSuchObject {
 export default class {
   constructor (xo) {
     this._objectConflicts = createRawObject() // TODO: clean when a server is disconnected.
-    this._servers = new Servers({
+    const serversDb = this._servers = new Servers({
       connection: xo._redis,
       prefix: 'xo:server',
       indexes: ['host']
@@ -43,8 +43,13 @@ export default class {
     this._xo = xo
 
     xo.on('start', async () => {
+      xo.addConfigManager('xenServers',
+        () => serversDb.get(),
+        servers => serversDb.update(servers)
+      )
+
       // Connects to existing servers.
-      const servers = await this._servers.get()
+      const servers = await serversDb.get()
       for (let server of servers) {
         if (server.enabled) {
           this.connectXenServer(server.id).catch(error => {

src/xo.js

@@ -48,6 +48,24 @@ export default class Xo extends EventEmitter {
 
   // -----------------------------------------------------------------
 
+  async clean () {
+    const handleCleanError = error => {
+      console.error(
+        '[WARN] clean error:',
+        error && error.stack || error
+      )
+    }
+    await Promise.all(mapToArray(
+      this.listeners('clean'),
+
+      listener => new Promise(resolve => {
+        resolve(listener.call(this))
+      }).catch(handleCleanError)
+    ))
+  }
+
+  // -----------------------------------------------------------------
+
   async start () {
     this.start = noop