From b7d746ee0965647133c0db8ecb8c41215e520d67 Mon Sep 17 00:00:00 2001 From: Julien Fontanet Date: Mon, 29 May 2017 15:07:24 +0200 Subject: [PATCH] feat: improve HTTP handling (#546) `Xo#httpRequest()` is used to make a request respecting XO settings like the proxy. `Xapi#getResource()` and `Xapi#putResource()` are used to receive and send resources to/from a XenServer. --- package.json | 6 ++-- src/decorators.js | 2 +- src/http-proxy.js | 13 ------- src/index.js | 10 ++---- src/utils.js | 32 +++++++---------- src/xapi-stats.js | 22 ++++++------ src/xapi/index.js | 69 +++++++++++------------------------- src/xapi/mixins/patching.js | 21 ++++------- src/xapi/utils.js | 32 ----------------- src/xo-mixins/http.js | 23 ++++++++++++ src/xo-mixins/xen-servers.js | 2 ++ yarn.lock | 38 ++++++++++---------- 12 files changed, 100 insertions(+), 170 deletions(-) delete mode 100644 src/http-proxy.js create mode 100644 src/xo-mixins/http.js diff --git a/package.json b/package.json index 2fcf7981b..76dd2844b 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "helmet": "^3.6.0", "highland": "^2.10.5", "http-proxy": "^1.16.2", - "http-request-plus": "^0.1.3", + "http-request-plus": "^0.1.5", "http-server-plus": "^0.8.0", "human-format": "^0.8.0", "is-my-json-valid": "^2.16.0", @@ -95,7 +95,7 @@ "passport": "^0.3.2", "passport-local": "^1.0.0", "pretty-format": "^20.0.1", - "promise-toolbox": "^0.8.2", + "promise-toolbox": "^0.9.2", "proxy-agent": "^2.0.0", "pug": "^2.0.0-rc.1", "pw": "^0.0.4", @@ -110,7 +110,7 @@ "tmp": "^0.0.31", "uuid": "^3.0.1", "ws": "^3.0.0", - "xen-api": "^0.12.0", + "xen-api": "^0.12.1", "xml2js": "~0.4.17", "xo-acl-resolver": "^0.2.3", "xo-collection": "^0.4.1", diff --git a/src/decorators.js b/src/decorators.js index 4473c3611..c96b5bc62 100644 --- a/src/decorators.js +++ b/src/decorators.js @@ -116,7 +116,7 @@ export const mixin = MixIns => Class => { for (const MixIn of MixIns) { const { prototype } = MixIn - const mixinInstance = new MixIn(instance) + const mixinInstance = new MixIn(instance, ...args) const descriptors = { __proto__: null } for (const prop of _ownKeys(prototype)) { if (_isIgnoredProperty(prop)) { diff --git a/src/http-proxy.js b/src/http-proxy.js deleted file mode 100644 index 78298e65c..000000000 --- a/src/http-proxy.js +++ /dev/null @@ -1,13 +0,0 @@ -import ProxyAgent from 'proxy-agent' - -let agent -export { agent as default } - -export function setup (uri) { - agent = uri != null - ? new ProxyAgent(uri) - : undefined -} - -const { env } = process -setup(env.http_proxy || env.HTTP_PROXY) diff --git a/src/index.js b/src/index.js index d4dbfdbc2..cff8ea0e5 100644 --- a/src/index.js +++ b/src/index.js @@ -25,9 +25,6 @@ import { import WebServer from 'http-server-plus' import Xo from './xo' -import { - setup as setupHttpProxy -} from './http-proxy' import { createRawObject, forEach, @@ -234,7 +231,8 @@ async function registerPlugin (pluginPath, pluginName) { getDataDir: () => { const dir = `${this._config.datadir}/${pluginName}` return ensureDir(dir).then(() => dir) - }}) + } + }) : factory await this.registerPlugin( @@ -558,10 +556,6 @@ export default async function main (args) { warn('Failed to change user/group:', error) } - if (config.httpProxy) { - setupHttpProxy(config.httpProxy) - } - // Creates main object. const xo = new Xo(config) diff --git a/src/utils.js b/src/utils.js index 6ad63161c..aa40b6933 100644 --- a/src/utils.js +++ b/src/utils.js @@ -23,7 +23,6 @@ import 'moment-timezone' import through2 from 'through2' import { CronJob } from 'cron' -import { Readable } from 'stream' import { utcFormat, utcParse } from 'd3-time-format' import { all as pAll, @@ -63,24 +62,6 @@ export const asyncMap = (collection, iteratee) => { // ------------------------------------------------------------------- -export function bufferToStream (buf) { - const stream = new Readable() - - let i = 0 - const { length } = buf - stream._read = function (size) { - if (i === length) { - return this.push(null) - } - - const newI = Math.min(i + size, length) - this.push(buf.slice(i, newI)) - i = newI - } - - return stream -} - export streamToBuffer from './stream-to-new-buffer' // ------------------------------------------------------------------- @@ -226,6 +207,19 @@ export function extractProperty (obj, prop) { // ------------------------------------------------------------------- +// Returns the first defined (non-undefined) value. +export const firstDefined = function () { + const n = arguments.length + for (let i = 0; i < n; ++i) { + const arg = arguments[i] + if (arg !== undefined) { + return arg + } + } +} + +// ------------------------------------------------------------------- + export const getUserPublicProperties = user => pick( user.properties || user, 'id', 'email', 'groups', 'permission', 'preferences', 'provider' diff --git a/src/xapi-stats.js b/src/xapi-stats.js index b793413b1..4f9077613 100644 --- a/src/xapi-stats.js +++ b/src/xapi-stats.js @@ -1,5 +1,4 @@ import endsWith from 'lodash/endsWith' -import httpRequest from 'http-request-plus' import JSON5 from 'json5' import { BaseError } from 'make-error' @@ -38,10 +37,6 @@ export class FaultyGranularity extends XapiStatsError {} // Utils // ------------------------------------------------------------------- -function makeUrl (hostname, sessionId, timestamp) { - return `https://${hostname}/rrd_updates?session_id=${sessionId}&start=${timestamp}&cf=AVERAGE&host=true&json=true` -} - // Return current local timestamp in seconds function getCurrentTimestamp () { return Date.now() / 1000 @@ -387,9 +382,16 @@ export default class XapiStats { // Execute one http request on a XenServer for get stats // Return stats (Json format) or throws got exception - async _getJson (url) { - const body = await httpRequest(url, { rejectUnauthorized: false }).readAll() - return JSON5.parse(body) + _getJson (xapi, host, timestamp) { + return xapi.getResource('/rrd_updates', { + host, + query: { + cf: 'AVERAGE', + host: 'true', + json: 'true', + start: timestamp + } + }).readAll().then(JSON5.parse) } async _getLastTimestamp (xapi, host, step) { @@ -450,7 +452,7 @@ export default class XapiStats { // Get json const timestamp = await this._getLastTimestamp(xapi, host, step) - let json = await this._getJson(makeUrl(hostname, xapi.sessionId, timestamp)) + let json = await this._getJson(xapi, host, timestamp) // Check if the granularity is linked to 'step' // If it's not the case, we retry other url with the json timestamp @@ -460,7 +462,7 @@ export default class XapiStats { // Approximately: half points are asked // FIXME: Not the best solution - json = await this._getJson(makeUrl(hostname, xapi.sessionId, serverTimestamp - step * (RRD_POINTS_PER_STEP[step] / 2) + step)) + json = await this._getJson(xapi, host, serverTimestamp - step * (RRD_POINTS_PER_STEP[step] / 2) + step) if (json.meta.step !== step) { throw new FaultyGranularity(`Unable to get the true granularity: ${json.meta.step}`) diff --git a/src/xapi/index.js b/src/xapi/index.js index d815b1812..ca67518b0 100644 --- a/src/xapi/index.js +++ b/src/xapi/index.js @@ -1,7 +1,6 @@ /* eslint-disable camelcase */ import deferrable from 'golike-defer' import fatfs from 'fatfs' -import httpRequest from 'http-request-plus' import synchronized from 'decorator-synchronized' import tarStream from 'tar-stream' import vmdkToVhd from 'xo-vmdk-to-vhd' @@ -31,7 +30,6 @@ import createSizeStream from '../size-stream' import fatfsBuffer, { init as fatfsBufferInit } from '../fatfs-buffer' import { mixin } from '../decorators' import { - bufferToStream, camelToSnakeCase, createRawObject, ensureArray, @@ -61,8 +59,7 @@ import { isVmRunning, NULL_REF, optional, - prepareXapiParam, - put + prepareXapiParam } from './utils' // =================================================================== @@ -760,8 +757,6 @@ export default class Xapi extends XapiBase { if (isVmRunning(vm) && !onlyMetadata) { host = vm.$resident_on snapshotRef = (await this._snapshotVm(vm)).$ref - } else { - host = this.pool.$master } const taskRef = await this._createTask('VM Export', vm.name_label) @@ -771,17 +766,13 @@ export default class Xapi extends XapiBase { }) } - return httpRequest({ - hostname: host.address, - path: onlyMetadata ? '/export_metadata/' : '/export/', - protocol: 'https', + return this.getResource(onlyMetadata ? '/export_metadata/' : '/export/', { + host, query: { ref: snapshotRef || vm.$ref, - session_id: this.sessionId, task_id: taskRef, use_compression: compress ? 'true' : 'false' - }, - rejectUnauthorized: false + } }) } @@ -1228,7 +1219,6 @@ export default class Xapi extends XapiBase { force: onlyMetadata ? 'true' : undefined, - session_id: this.sessionId, task_id: taskRef } @@ -1236,12 +1226,8 @@ export default class Xapi extends XapiBase { if (sr) { host = sr.$PBDs[0].$host query.sr_id = sr.$ref - } else { - host = this.pool.$master } - const path = onlyMetadata ? '/import_metadata/' : '/import/' - if (onVmCreation) { this._waitObject( obj => obj && obj.current_operations && taskRef in obj.current_operations @@ -1250,13 +1236,11 @@ export default class Xapi extends XapiBase { const [ vmRef ] = await Promise.all([ this._watchTask(taskRef).then(extractOpaqueRef), - put(stream, { - hostname: host.address, - path, - protocol: 'https', - query, - rejectUnauthorized: false - }) + this.putResource( + stream, + onlyMetadata ? '/import_metadata/' : '/import/', + { host, query } + ) ]) // Importing a metadata archive of running VMs is currently @@ -1867,7 +1851,6 @@ export default class Xapi extends XapiBase { const query = { format, - session_id: this.sessionId, task_id: taskRef, vdi: vdi.$ref } @@ -1881,12 +1864,9 @@ export default class Xapi extends XapiBase { }`) const task = this._watchTask(taskRef) - return httpRequest($cancelToken, { - hostname: host.address, - path: '/export_raw_vdi/', - protocol: 'https', - query, - rejectUnauthorized: false + return this.getResource($cancelToken, '/export_raw_vdi/', { + host, + query }).then(response => { response.task = task @@ -1911,13 +1891,6 @@ export default class Xapi extends XapiBase { async _importVdiContent (vdi, stream, format = VDI_FORMAT_VHD) { const taskRef = await this._createTask('VDI Content Import', vdi.name_label) - const query = { - session_id: this.sessionId, - task_id: taskRef, - format, - vdi: vdi.$ref - } - const pbd = find(vdi.$SR.$PBDs, 'currently_attached') if (!pbd) { throw new Error('no valid PBDs found') @@ -1927,12 +1900,13 @@ export default class Xapi extends XapiBase { await Promise.all([ stream.checksumVerified, task, - put(stream, { - hostname: pbd.$host.address, - path: '/import_raw_vdi/', - protocol: 'https', - query, - rejectUnauthorized: false + this.putResource(stream, '/import_raw_vdi/', { + host: pbd.host, + query: { + format, + task_id: taskRef, + vdi: vdi.$ref + } }) ]) } @@ -2171,10 +2145,7 @@ export default class Xapi extends XapiBase { await fs.writeFile('openstack/latest/user_data', config) // Transform the buffer into a stream - const stream = bufferToStream(buffer) - await this.importVdiContent(vdi.$id, stream, { - format: VDI_FORMAT_RAW - }) + await this._importVdiContent(vdi, buffer, VDI_FORMAT_RAW) await this._createVbd(vm, vdi) } diff --git a/src/xapi/mixins/patching.js b/src/xapi/mixins/patching.js index f0d81f18c..bcc4cd22a 100644 --- a/src/xapi/mixins/patching.js +++ b/src/xapi/mixins/patching.js @@ -1,12 +1,10 @@ import deferrable from 'golike-defer' import filter from 'lodash/filter' -import httpRequest from 'http-request-plus' import includes from 'lodash/includes' import some from 'lodash/some' import sortBy from 'lodash/sortBy' import unzip from 'julien-f-unzip' -import httpProxy from '../../http-proxy' import { debounce } from '../../decorators' import { createRawObject, @@ -19,7 +17,6 @@ import { import { debug, - put, useUpdateSystem } from '../utils' @@ -27,9 +24,8 @@ export default { // FIXME: should be static @debounce(24 * 60 * 60 * 1000) async _getXenUpdates () { - const { readAll, statusCode } = await httpRequest( - 'http://updates.xensource.com/XenServer/updates.xml', - { agent: httpProxy } + const { readAll, statusCode } = await this.xo.httpRequest( + 'http://updates.xensource.com/XenServer/updates.xml' ) if (statusCode !== 200) { @@ -211,15 +207,10 @@ export default { const task = this._watchTask(taskRef) const [ patchRef ] = await Promise.all([ task, - put(stream, { - hostname: this.pool.$master.address, - path: '/pool_patch_upload', - protocol: 'https', + this.putResource(stream, '/pool_patch_upload', { query: { - session_id: this.sessionId, task_id: taskRef - }, - rejectUnauthorized: false + } }) ]) @@ -238,7 +229,7 @@ export default { throw new Error('no such patch ' + uuid) } - let stream = await httpRequest(patchInfo.url, { agent: httpProxy }) + let stream = await this.xo.httpRequest(patchInfo.url) stream = await new Promise((resolve, reject) => { const PATCH_RE = /\.xsupdate$/ stream.pipe(unzip.Parse()).on('entry', entry => { @@ -280,7 +271,7 @@ export default { throw new Error('no such patch ' + uuid) } - let stream = await httpRequest(patchInfo.url, { agent: httpProxy }) + let stream = await this.xo.httpRequest(patchInfo.url) stream = await new Promise((resolve, reject) => { stream.pipe(unzip.Parse()).on('entry', entry => { entry.length = entry.size diff --git a/src/xapi/utils.js b/src/xapi/utils.js index 3395575c2..e1c82d0f2 100644 --- a/src/xapi/utils.js +++ b/src/xapi/utils.js @@ -1,7 +1,6 @@ // import isFinite from 'lodash/isFinite' import camelCase from 'lodash/camelCase' import createDebug from 'debug' -import httpRequest from 'http-request-plus' import isEqual from 'lodash/isEqual' import isPlainObject from 'lodash/isPlainObject' import pickBy from 'lodash/pickBy' @@ -348,37 +347,6 @@ export const NULL_REF = 'OpaqueRef:NULL' // =================================================================== -// HTTP put, use an ugly hack if the length is not known because XAPI -// does not support chunk encoding. -export const put = (stream, { - headers: { ...headers } = {}, - ...opts -}) => { - const makeRequest = () => httpRequest.put(opts, { - body: stream, - headers - }) - - // Xen API does not support chunk encoding. - if (stream.length == null) { - // add a fake huge content length (1 PiB) - headers['content-length'] = '1125899906842624' - - const promise = makeRequest() - - // when the data has been emitted, close the connection - stream.on('end', () => { - setTimeout(() => { - promise.cancel() - }, 1e3) - }) - - return promise.readAll() - } - - return makeRequest().readAll() -} - export const useUpdateSystem = host => { // Match Xen Center's condition: https://github.com/xenserver/xenadmin/blob/f3a64fc54bbff239ca6f285406d9034f57537d64/XenModel/Utils/Helpers.cs#L420 return versionSatisfies(host.software_version.platform_version, '^2.1.1') diff --git a/src/xo-mixins/http.js b/src/xo-mixins/http.js new file mode 100644 index 000000000..e7f20189c --- /dev/null +++ b/src/xo-mixins/http.js @@ -0,0 +1,23 @@ +import hrp from 'http-request-plus' +import ProxyAgent from 'proxy-agent' + +import { + firstDefined +} from '../utils' + +export default class Http { + constructor (_, { + httpProxy = firstDefined( + process.env.http_proxy, + process.env.HTTP_PROXY + ) + }) { + this._proxy = httpProxy && new ProxyAgent(httpProxy) + } + + httpRequest (...args) { + return hrp({ + agent: this._proxy + }, ...args) + } +} diff --git a/src/xo-mixins/xen-servers.js b/src/xo-mixins/xen-servers.js index f68f89810..6c48adeca 100644 --- a/src/xo-mixins/xen-servers.js +++ b/src/xo-mixins/xen-servers.js @@ -266,6 +266,8 @@ export default class { } return { + httpRequest: this._xo.httpRequest.bind(this), + install () { objects.on('add', onAddOrUpdate) objects.on('update', onAddOrUpdate) diff --git a/yarn.lock b/yarn.lock index d5b783551..a26604082 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3233,21 +3233,13 @@ http-proxy@^1.16.2: eventemitter3 "1.x.x" requires-port "1.x.x" -http-request-plus@^0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/http-request-plus/-/http-request-plus-0.1.3.tgz#44b90f6d3b391cf5f21c1dbfefc115c86975cdd7" +http-request-plus@^0.1.5: + version "0.1.5" + resolved "https://registry.yarnpkg.com/http-request-plus/-/http-request-plus-0.1.5.tgz#9aead8b230586397928ecbb9a6bed96a3bfe2210" dependencies: is-redirect "^1.0.0" lodash "^4.17.4" - promise-toolbox "^0.8.2" - -http-request-plus@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/http-request-plus/-/http-request-plus-0.1.4.tgz#c99cd36366e96c13f92da5954b3a2fd2ce2c531e" - dependencies: - is-redirect "^1.0.0" - lodash "^4.17.4" - promise-toolbox "^0.8.2" + promise-toolbox "^0.9.1" http-server-plus@^0.8.0: version "0.8.0" @@ -4435,11 +4427,11 @@ ltgt@^2.1.2, ltgt@~2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/ltgt/-/ltgt-2.1.3.tgz#10851a06d9964b971178441c23c9e52698eece34" -make-error@^1, make-error@^1.2.2: +make-error@^1: version "1.2.3" resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.2.3.tgz#6c4402df732e0977ac6faf754a5074b3d2b1d19d" -make-error@^1.0.2, make-error@^1.2.0, make-error@^1.2.1, make-error@^1.3.0: +make-error@^1.0.2, make-error@^1.2.0, make-error@^1.2.1, make-error@^1.2.2, make-error@^1.2.3, make-error@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.0.tgz#52ad3a339ccf10ce62b4040b708fe707244b8b96" @@ -5171,12 +5163,18 @@ promise-polyfill@^6.0.1: version "6.0.2" resolved "https://registry.yarnpkg.com/promise-polyfill/-/promise-polyfill-6.0.2.tgz#d9c86d3dc4dc2df9016e88946defd69b49b41162" -promise-toolbox@^0.8.0, promise-toolbox@^0.8.2: +promise-toolbox@^0.8.0: version "0.8.2" resolved "https://registry.yarnpkg.com/promise-toolbox/-/promise-toolbox-0.8.2.tgz#91722a364e6a2d6d13319491da3068b7de0b348f" dependencies: make-error "^1.2.2" +promise-toolbox@^0.9.1, promise-toolbox@^0.9.2: + version "0.9.2" + resolved "https://registry.yarnpkg.com/promise-toolbox/-/promise-toolbox-0.9.2.tgz#3577b6815f1b5c5a26cdd7517bf928a0c478eb4a" + dependencies: + make-error "^1.2.3" + promise@^7.0.1: version "7.1.1" resolved "https://registry.yarnpkg.com/promise/-/promise-7.1.1.tgz#489654c692616b8aa55b0724fa809bb7db49c5bf" @@ -6795,23 +6793,23 @@ xdg-basedir@^2.0.0: dependencies: os-homedir "^1.0.0" -xen-api@^0.12.0: - version "0.12.0" - resolved "https://registry.yarnpkg.com/xen-api/-/xen-api-0.12.0.tgz#1eefa8da0fe25c5a5e104d548579f329c4de22cc" +xen-api@^0.12.1: + version "0.12.1" + resolved "https://registry.yarnpkg.com/xen-api/-/xen-api-0.12.1.tgz#b9da5489ce9a0d7fe61f2c3d6e509c3e95094f36" dependencies: babel-polyfill "^6.23.0" blocked "^1.2.1" debug "^2.6.8" event-to-promise "^0.8.0" exec-promise "^0.7.0" - http-request-plus "^0.1.4" + http-request-plus "^0.1.5" json-rpc-protocol "^0.11.2" kindof "^2.0.0" lodash "^4.17.4" make-error "^1.3.0" minimist "^1.2.0" ms "^2.0.0" - promise-toolbox "^0.8.2" + promise-toolbox "^0.9.1" pw "0.0.4" xmlrpc "^1.3.2" xo-collection "^0.4.1"