feat(xo): export/import config (#427)

See vatesfr/xo-web#786
This commit is contained in:
Julien Fontanet 2016-10-27 18:48:19 +02:00 committed by GitHub
parent 0afd506a41
commit 9d1d6ea4c5
23 changed files with 229 additions and 46 deletions

View File

@ -98,6 +98,7 @@
"tar-stream": "^1.5.2", "tar-stream": "^1.5.2",
"through2": "^2.0.0", "through2": "^2.0.0",
"trace": "^2.0.1", "trace": "^2.0.1",
"uuid": "^2.0.3",
"ws": "^1.1.1", "ws": "^1.1.1",
"xen-api": "^0.9.4", "xen-api": "^0.9.4",
"xml2js": "~0.4.6", "xml2js": "~0.4.6",

View File

@ -18,7 +18,9 @@ get.params = {
} }
export async function create ({job}) { export async function create ({job}) {
return (await this.createJob(this.session.get('user_id'), job)).id job.userId = this.session.get('user_id')
return (await this.createJob(job)).id
} }
create.permission = 'admin' create.permission = 'admin'

View File

@ -4,7 +4,7 @@ import { getUserPublicProperties, mapToArray } from '../utils'
// =================================================================== // ===================================================================
export async function create ({email, password, permission}) { export async function create ({email, password, permission}) {
return (await this.createUser(email, {password, permission})).id return (await this.createUser({email, password, permission})).id
} }
create.description = 'creates a new user' create.description = 'creates a new user'

View File

@ -1,5 +1,49 @@
import { streamToBuffer } from '../utils'
// ===================================================================
export function clean () {
return this.clean()
}
clean.permission = 'admin'
// -------------------------------------------------------------------
export async function exportConfig () {
return {
$getFrom: await this.registerHttpRequest((req, res) => {
res.writeHead(200, 'OK', {
'content-disposition': 'attachment'
})
return this.exportConfig()
},
undefined,
{ suffix: '/config.json' })
}
}
exportConfig.permission = 'admin'
// -------------------------------------------------------------------
export function getAllObjects () { export function getAllObjects () {
return this.getObjects() return this.getObjects()
} }
getAllObjects.permission = '' getAllObjects.permission = ''
// -------------------------------------------------------------------
export async function importConfig () {
return {
$sendTo: await this.registerHttpRequest(async (req, res) => {
await this.importConfig(JSON.parse(await streamToBuffer(req)))
res.end('config successfully imported')
})
}
}
importConfig.permission = 'admin'

View File

@ -3,6 +3,7 @@ import difference from 'lodash/difference'
import filter from 'lodash/filter' import filter from 'lodash/filter'
import getKey from 'lodash/keys' import getKey from 'lodash/keys'
import {createClient as createRedisClient} from 'redis' import {createClient as createRedisClient} from 'redis'
import {v4 as generateUuid} from 'uuid'
import { import {
forEach, forEach,
@ -68,12 +69,12 @@ export default class Redis extends Collection {
// TODO: remove “replace” which is a temporary measure, implement // TODO: remove “replace” which is a temporary measure, implement
// “set()” instead. // “set()” instead.
const {indexes, prefix, redis, idPrefix = ''} = this const {indexes, prefix, redis} = this
return Promise.all(mapToArray(models, async model => { return Promise.all(mapToArray(models, async model => {
// Generate a new identifier if necessary. // Generate a new identifier if necessary.
if (model.id === undefined) { if (model.id === undefined) {
model.id = idPrefix + String(await redis.incr(prefix + '_id')) model.id = generateUuid()
} }
const success = await redis.sadd(prefix + '_ids', model.id) const success = await redis.sadd(prefix + '_ids', model.id)

View File

@ -14,10 +14,6 @@ export class Groups extends Collection {
return Group return Group
} }
get idPrefix () {
return 'group:'
}
create (name) { create (name) {
return this.add(new Group({ return this.add(new Group({
name, name,

View File

@ -11,12 +11,7 @@ export class Jobs extends Collection {
return Job return Job
} }
get idPrefix () { async create (job) {
return 'job:'
}
async create (userId, job) {
job.userId = userId
// Serializes. // Serializes.
job.paramsVector = JSON.stringify(job.paramsVector) job.paramsVector = JSON.stringify(job.paramsVector)
return /* await */ this.add(new Job(job)) return /* await */ this.add(new Job(job))

View File

@ -13,10 +13,6 @@ export class PluginsMetadata extends Collection {
return PluginMetadata return PluginMetadata
} }
get idPrefix () {
return 'plugin-metadata:'
}
async save ({ id, autoload, configuration }) { async save ({ id, autoload, configuration }) {
return /* await */ this.update({ return /* await */ this.update({
id, id,

View File

@ -13,10 +13,6 @@ export class Remotes extends Collection {
return Remote return Remote
} }
get idPrefix () {
return 'remote-'
}
create (name, url) { create (name, url) {
return this.add(new Remote({ return this.add(new Remote({
name, name,

View File

@ -11,10 +11,6 @@ export class Schedules extends Collection {
return Schedule return Schedule
} }
get idPrefix () {
return 'schedule:'
}
create (userId, job, cron, enabled, name = undefined, timezone = undefined) { create (userId, job, cron, enabled, name = undefined, timezone = undefined) {
return this.add(new Schedule({ return this.add(new Schedule({
userId, userId,

View File

@ -31,15 +31,14 @@ export class Users extends Collection {
return User return User
} }
async create (email, properties = {}) { async create (properties) {
const { email } = properties
// Avoid duplicates. // Avoid duplicates.
if (await this.exists({email})) { if (await this.exists({email})) {
throw new Error(`the user ${email} already exists`) throw new Error(`the user ${email} already exists`)
} }
// Adds the email to the user's properties.
properties.email = email
// Create the user object. // Create the user object.
const user = new User(properties) const user = new User(properties)

View File

@ -24,6 +24,15 @@ export default class {
prefix: 'xo:acl', prefix: 'xo:acl',
indexes: ['subject', 'object'] indexes: ['subject', 'object']
}) })
xo.on('start', () => {
xo.addConfigManager('acls',
() => this.getAllAcls(),
acls => Promise.all(mapToArray(acls, acl =>
this.addAcl(acl.subjectId, acl.objectId, acl.action)
))
)
})
} }
async _getAclsForUser (userId) { async _getAclsForUser (userId) {

View File

@ -4,6 +4,7 @@ import {
} from '../api-errors' } from '../api-errors'
import { import {
createRawObject, createRawObject,
forEach,
generateToken, generateToken,
pCatch, pCatch,
noop noop
@ -30,7 +31,7 @@ export default class {
this._providers = new Set() this._providers = new Set()
// Creates persistent collections. // Creates persistent collections.
this._tokens = new Tokens({ const tokensDb = this._tokens = new Tokens({
connection: xo._redis, connection: xo._redis,
prefix: 'xo:token', prefix: 'xo:token',
indexes: ['user_id'] indexes: ['user_id']
@ -65,6 +66,25 @@ export default class {
return return
} }
}) })
xo.on('clean', async () => {
const tokens = await tokensDb.get()
const toRemove = []
const now = Date.now()
forEach(tokens, ({ expiration, id }) => {
if (!expiration || expiration < now) {
toRemove.push(id)
}
})
await tokensDb.remove(toRemove)
})
xo.on('start', () => {
xo.addConfigManager('authTokens',
() => tokensDb.get(),
tokens => tokensDb.update(tokens)
)
})
} }
registerAuthenticationProvider (provider) { registerAuthenticationProvider (provider) {

View File

@ -0,0 +1,33 @@
import { map, noop } from '../utils'
import { all as pAll } from 'promise-toolbox'
export default class ConfigManagement {
constructor () {
this._managers = { __proto__: null }
}
addConfigManager (id, exporter, importer) {
const managers = this._managers
if (id in managers) {
throw new Error(`${id} is already taken`)
}
this._managers[id] = { exporter, importer }
}
exportConfig () {
return map(this._managers, ({ exporter }, key) => exporter())::pAll()
}
importConfig (config) {
const managers = this._managers
return map(config, (entry, key) => {
const manager = managers[key]
if (manager) {
return manager.importer(entry)
}
})::pAll().then(noop)
}
}

View File

@ -54,6 +54,11 @@ export default class IpPools {
xo.on('start', async () => { xo.on('start', async () => {
this._store = await xo.getStore('ipPools') this._store = await xo.getStore('ipPools')
xo.addConfigManager('ipPools',
() => this.getAllIpPools(),
ipPools => Promise.all(mapToArray(ipPools, ipPool => this._save(ipPool)))
)
}) })
} }

View File

@ -1,6 +1,8 @@
import assign from 'lodash/assign' import assign from 'lodash/assign'
import JobExecutor from '../job-executor' import JobExecutor from '../job-executor'
import { Jobs } from '../models/job' import { Jobs } from '../models/job'
import { mapToArray } from '../utils'
import { import {
GenericError, GenericError,
NoSuchObject NoSuchObject
@ -19,11 +21,20 @@ class NoSuchJob extends NoSuchObject {
export default class { export default class {
constructor (xo) { constructor (xo) {
this._executor = new JobExecutor(xo) this._executor = new JobExecutor(xo)
this._jobs = new Jobs({ const jobsDb = this._jobs = new Jobs({
connection: xo._redis, connection: xo._redis,
prefix: 'xo:job', prefix: 'xo:job',
indexes: ['user_id', 'key'] indexes: ['user_id', 'key']
}) })
xo.on('start', () => {
xo.addConfigManager('jobs',
() => jobsDb.get(),
jobs => Promise.all(mapToArray(jobs, job =>
jobsDb.save(job)
))
)
})
} }
async getAllJobs () { async getAllJobs () {
@ -39,9 +50,9 @@ export default class {
return job.properties return job.properties
} }
async createJob (userId, job) { async createJob (job) {
// TODO: use plain objects // TODO: use plain objects
const job_ = await this._jobs.create(userId, job) const job_ = await this._jobs.create(job)
return job_.properties return job_.properties
} }

View File

@ -29,6 +29,15 @@ export default class {
connection: xo._redis, connection: xo._redis,
prefix: 'xo:plugin-metadata' prefix: 'xo:plugin-metadata'
}) })
xo.on('start', () => {
xo.addConfigManager('plugins',
() => this._pluginsMetadata.get(),
plugins => Promise.all(mapToArray(plugins, plugin =>
this._pluginsMetadata.save(plugin)
))
)
})
} }
_getRawPlugin (id) { _getRawPlugin (id) {

View File

@ -2,7 +2,8 @@ import RemoteHandlerLocal from '../remote-handlers/local'
import RemoteHandlerNfs from '../remote-handlers/nfs' import RemoteHandlerNfs from '../remote-handlers/nfs'
import RemoteHandlerSmb from '../remote-handlers/smb' import RemoteHandlerSmb from '../remote-handlers/smb'
import { import {
forEach forEach,
mapToArray
} from '../utils' } from '../utils'
import { import {
NoSuchObject NoSuchObject
@ -30,6 +31,13 @@ export default class {
}) })
xo.on('start', async () => { xo.on('start', async () => {
xo.addConfigManager('remotes',
() => this._remotes.get(),
remotes => Promise.all(mapToArray(remotes, remote =>
this._remotes.save(remote)
))
)
await this.initRemotes() await this.initRemotes()
await this.syncAllRemotes() await this.syncAllRemotes()
}) })

View File

@ -85,6 +85,13 @@ export default class {
this._store = null this._store = null
xo.on('start', async () => { xo.on('start', async () => {
xo.addConfigManager('resourceSets',
() => this.getAllResourceSets(),
resourceSets => Promise.all(mapToArray(resourceSets, resourceSet =>
this._save(resourceSet)
))
)
this._store = await xo.getStore('resourceSets') this._store = await xo.getStore('resourceSets')
}) })
} }

View File

@ -4,6 +4,7 @@ import { Schedules } from '../models/schedule'
import { import {
forEach, forEach,
mapToArray,
scheduleFn scheduleFn
} from '../utils' } from '../utils'
@ -42,14 +43,23 @@ export class ScheduleAlreadyEnabled extends SchedulerError {
export default class { export default class {
constructor (xo) { constructor (xo) {
this.xo = xo this.xo = xo
this._redisSchedules = new Schedules({ const schedules = this._redisSchedules = new Schedules({
connection: xo._redis, connection: xo._redis,
prefix: 'xo:schedule', prefix: 'xo:schedule',
indexes: ['user_id', 'job'] indexes: ['user_id', 'job']
}) })
this._scheduleTable = undefined this._scheduleTable = undefined
xo.on('start', () => this._loadSchedules()) xo.on('start', () => {
xo.addConfigManager('schedules',
() => schedules.get(),
schedules_ => Promise.all(mapToArray(schedules_, schedule =>
schedules.save(schedule)
))
)
return this._loadSchedules()
})
xo.on('stop', () => this._disableAll()) xo.on('stop', () => this._disableAll())
} }

View File

@ -52,22 +52,39 @@ export default class {
const redis = xo._redis const redis = xo._redis
this._groups = new Groups({ const groupsDb = this._groups = new Groups({
connection: redis, connection: redis,
prefix: 'xo:group' prefix: 'xo:group'
}) })
const users = this._users = new Users({ const usersDb = this._users = new Users({
connection: redis, connection: redis,
prefix: 'xo:user', prefix: 'xo:user',
indexes: ['email'] indexes: ['email']
}) })
xo.on('start', async () => { xo.on('start', async () => {
if (!await users.exists()) { xo.addConfigManager('groups',
() => groupsDb.get(),
groups => Promise.all(mapToArray(groups, group => groupsDb.save(group)))
)
xo.addConfigManager('users',
() => usersDb.get(),
users => Promise.all(mapToArray(users, async user => {
const conflictUsers = await usersDb.get({ email: user.email })
if (!isEmpty(conflictUsers)) {
await Promise.all(mapToArray(conflictUsers, user =>
this.deleteUser(user.id)
))
}
return usersDb.save(user)
}))
)
if (!await usersDb.exists()) {
const email = 'admin@admin.net' const email = 'admin@admin.net'
const password = 'admin' const password = 'admin'
await this.createUser(email, {password, permission: 'admin'}) await this.createUser({email, password, permission: 'admin'})
console.log('[INFO] Default user created:', email, ' with password', password) console.log('[INFO] Default user created:', email, ' with password', password)
} }
}) })
@ -75,13 +92,17 @@ export default class {
// ----------------------------------------------------------------- // -----------------------------------------------------------------
async createUser (email, { password, ...properties }) { async createUser ({ name, password, ...properties }) {
if (name) {
properties.email = name
}
if (password) { if (password) {
properties.pw_hash = await hash(password) properties.pw_hash = await hash(password)
} }
// TODO: use plain objects // TODO: use plain objects
const user = await this._users.create(email, properties) const user = await this._users.create(properties)
return user.properties return user.properties
} }
@ -210,7 +231,8 @@ export default class {
throw new Error(`registering ${name} user is forbidden`) throw new Error(`registering ${name} user is forbidden`)
} }
return /* await */ this.createUser(name, { return /* await */ this.createUser({
name,
_provider: provider _provider: provider
}) })
} }

View File

@ -32,7 +32,7 @@ class NoSuchXenServer extends NoSuchObject {
export default class { export default class {
constructor (xo) { constructor (xo) {
this._objectConflicts = createRawObject() // TODO: clean when a server is disconnected. this._objectConflicts = createRawObject() // TODO: clean when a server is disconnected.
this._servers = new Servers({ const serversDb = this._servers = new Servers({
connection: xo._redis, connection: xo._redis,
prefix: 'xo:server', prefix: 'xo:server',
indexes: ['host'] indexes: ['host']
@ -43,8 +43,13 @@ export default class {
this._xo = xo this._xo = xo
xo.on('start', async () => { xo.on('start', async () => {
xo.addConfigManager('xenServers',
() => serversDb.get(),
servers => serversDb.update(servers)
)
// Connects to existing servers. // Connects to existing servers.
const servers = await this._servers.get() const servers = await serversDb.get()
for (let server of servers) { for (let server of servers) {
if (server.enabled) { if (server.enabled) {
this.connectXenServer(server.id).catch(error => { this.connectXenServer(server.id).catch(error => {

View File

@ -48,6 +48,24 @@ export default class Xo extends EventEmitter {
// ----------------------------------------------------------------- // -----------------------------------------------------------------
async clean () {
const handleCleanError = error => {
console.error(
'[WARN] clean error:',
error && error.stack || error
)
}
await Promise.all(mapToArray(
this.listeners('clean'),
listener => new Promise(resolve => {
resolve(listener.call(this))
}).catch(handleCleanError)
))
}
// -----------------------------------------------------------------
async start () { async start () {
this.start = noop this.start = noop