NuKVM/xen-orchestra
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: improve HTTP handling (#546)

Browse Source 
`Xo#httpRequest()` is used to make a request respecting XO settings like the proxy.

`Xapi#getResource()` and `Xapi#putResource()` are used to receive and send resources to/from a XenServer.
This commit is contained in:
Julien Fontanet 2017-05-29 15:07:24 +02:00 committed by GitHub
parent 83f3fcd913
commit b7d746ee09
12 changed files with 100 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
package.json
View File

@ -67,7 +67,7 @@
"helmet": "^3.6.0",
"highland": "^2.10.5",
"http-proxy": "^1.16.2",
"http-request-plus": "^0.1.3",
"http-request-plus": "^0.1.5",
"http-server-plus": "^0.8.0",
"human-format": "^0.8.0",
"is-my-json-valid": "^2.16.0",
@ -95,7 +95,7 @@
"passport": "^0.3.2",
"passport-local": "^1.0.0",
"pretty-format": "^20.0.1",
"promise-toolbox": "^0.8.2",
"promise-toolbox": "^0.9.2",
"proxy-agent": "^2.0.0",
"pug": "^2.0.0-rc.1",
"pw": "^0.0.4",
@ -110,7 +110,7 @@
"tmp": "^0.0.31",
"uuid": "^3.0.1",
"ws": "^3.0.0",
"xen-api": "^0.12.0",
"xen-api": "^0.12.1",
"xml2js": "~0.4.17",
"xo-acl-resolver": "^0.2.3",
"xo-collection": "^0.4.1",

2
src/decorators.js
View File

@ -116,7 +116,7 @@ export const mixin = MixIns => Class => {
for (const MixIn of MixIns) {
const { prototype } = MixIn
const mixinInstance = new MixIn(instance)
const mixinInstance = new MixIn(instance, ...args)
const descriptors = { __proto__: null }
for (const prop of _ownKeys(prototype)) {
if (_isIgnoredProperty(prop)) {

13
src/http-proxy.js
View File

@ -1,13 +0,0 @@
import ProxyAgent from 'proxy-agent'
let agent
export { agent as default }
export function setup (uri) {
agent = uri != null
? new ProxyAgent(uri)
: undefined
}
const { env } = process
setup(env.http_proxy || env.HTTP_PROXY)

10
src/index.js
View File

@ -25,9 +25,6 @@ import {
import WebServer from 'http-server-plus'
import Xo from './xo'
import {
setup as setupHttpProxy
} from './http-proxy'
import {
createRawObject,
forEach,
@ -234,7 +231,8 @@ async function registerPlugin (pluginPath, pluginName) {
getDataDir: () => {
const dir = `${this._config.datadir}/${pluginName}`
return ensureDir(dir).then(() => dir)
}})
}
})
: factory
await this.registerPlugin(
@ -558,10 +556,6 @@ export default async function main (args) {
warn('Failed to change user/group:', error)
}
if (config.httpProxy) {
setupHttpProxy(config.httpProxy)
}
// Creates main object.
const xo = new Xo(config)

32
src/utils.js
View File

@ -23,7 +23,6 @@ import 'moment-timezone'
import through2 from 'through2'
import { CronJob } from 'cron'
import { Readable } from 'stream'
import { utcFormat, utcParse } from 'd3-time-format'
import {
all as pAll,
@ -63,24 +62,6 @@ export const asyncMap = (collection, iteratee) => {
// -------------------------------------------------------------------
export function bufferToStream (buf) {
const stream = new Readable()
let i = 0
const { length } = buf
stream._read = function (size) {
if (i === length) {
return this.push(null)
}
const newI = Math.min(i + size, length)
this.push(buf.slice(i, newI))
i = newI
}
return stream
}
export streamToBuffer from './stream-to-new-buffer'
// -------------------------------------------------------------------
@ -226,6 +207,19 @@ export function extractProperty (obj, prop) {
// -------------------------------------------------------------------
// Returns the first defined (non-undefined) value.
export const firstDefined = function () {
const n = arguments.length
for (let i = 0; i < n; ++i) {
const arg = arguments[i]
if (arg !== undefined) {
return arg
}
}
}
// -------------------------------------------------------------------
export const getUserPublicProperties = user => pick(
user.properties || user,
'id', 'email', 'groups', 'permission', 'preferences', 'provider'

22
src/xapi-stats.js
View File

@ -1,5 +1,4 @@
import endsWith from 'lodash/endsWith'
import httpRequest from 'http-request-plus'
import JSON5 from 'json5'
import { BaseError } from 'make-error'
@ -38,10 +37,6 @@ export class FaultyGranularity extends XapiStatsError {}
// Utils
// -------------------------------------------------------------------
function makeUrl (hostname, sessionId, timestamp) {
return `https://${hostname}/rrd_updates?session_id=${sessionId}&start=${timestamp}&cf=AVERAGE&host=true&json=true`
}
// Return current local timestamp in seconds
function getCurrentTimestamp () {
return Date.now() / 1000
@ -387,9 +382,16 @@ export default class XapiStats {
// Execute one http request on a XenServer for get stats
// Return stats (Json format) or throws got exception
async _getJson (url) {
const body = await httpRequest(url, { rejectUnauthorized: false }).readAll()
return JSON5.parse(body)
_getJson (xapi, host, timestamp) {
return xapi.getResource('/rrd_updates', {
host,
query: {
cf: 'AVERAGE',
host: 'true',
json: 'true',
start: timestamp
}
}).readAll().then(JSON5.parse)
}
async _getLastTimestamp (xapi, host, step) {
@ -450,7 +452,7 @@ export default class XapiStats {
// Get json
const timestamp = await this._getLastTimestamp(xapi, host, step)
let json = await this._getJson(makeUrl(hostname, xapi.sessionId, timestamp))
let json = await this._getJson(xapi, host, timestamp)
// Check if the granularity is linked to 'step'
// If it's not the case, we retry other url with the json timestamp
@ -460,7 +462,7 @@ export default class XapiStats {
// Approximately: half points are asked
// FIXME: Not the best solution
json = await this._getJson(makeUrl(hostname, xapi.sessionId, serverTimestamp - step * (RRD_POINTS_PER_STEP[step] / 2) + step))
json = await this._getJson(xapi, host, serverTimestamp - step * (RRD_POINTS_PER_STEP[step] / 2) + step)
if (json.meta.step !== step) {
throw new FaultyGranularity(`Unable to get the true granularity: ${json.meta.step}`)

69
src/xapi/index.js
View File

@ -1,7 +1,6 @@
/* eslint-disable camelcase */
import deferrable from 'golike-defer'
import fatfs from 'fatfs'
import httpRequest from 'http-request-plus'
import synchronized from 'decorator-synchronized'
import tarStream from 'tar-stream'
import vmdkToVhd from 'xo-vmdk-to-vhd'
@ -31,7 +30,6 @@ import createSizeStream from '../size-stream'
import fatfsBuffer, { init as fatfsBufferInit } from '../fatfs-buffer'
import { mixin } from '../decorators'
import {
bufferToStream,
camelToSnakeCase,
createRawObject,
ensureArray,
@ -61,8 +59,7 @@ import {
isVmRunning,
NULL_REF,
optional,
prepareXapiParam,
put
prepareXapiParam
} from './utils'
// ===================================================================
@ -760,8 +757,6 @@ export default class Xapi extends XapiBase {
if (isVmRunning(vm) && !onlyMetadata) {
host = vm.$resident_on
snapshotRef = (await this._snapshotVm(vm)).$ref
} else {
host = this.pool.$master
}
const taskRef = await this._createTask('VM Export', vm.name_label)
@ -771,17 +766,13 @@ export default class Xapi extends XapiBase {
})
}
return httpRequest({
hostname: host.address,
path: onlyMetadata ? '/export_metadata/' : '/export/',
protocol: 'https',
return this.getResource(onlyMetadata ? '/export_metadata/' : '/export/', {
host,
query: {
ref: snapshotRef || vm.$ref,
session_id: this.sessionId,
task_id: taskRef,
use_compression: compress ? 'true' : 'false'
},
rejectUnauthorized: false
}
})
}
@ -1228,7 +1219,6 @@ export default class Xapi extends XapiBase {
force: onlyMetadata
? 'true'
: undefined,
session_id: this.sessionId,
task_id: taskRef
}
@ -1236,12 +1226,8 @@ export default class Xapi extends XapiBase {
if (sr) {
host = sr.$PBDs[0].$host
query.sr_id = sr.$ref
} else {
host = this.pool.$master
}
const path = onlyMetadata ? '/import_metadata/' : '/import/'
if (onVmCreation) {
this._waitObject(
obj => obj && obj.current_operations && taskRef in obj.current_operations
@ -1250,13 +1236,11 @@ export default class Xapi extends XapiBase {
const [ vmRef ] = await Promise.all([
this._watchTask(taskRef).then(extractOpaqueRef),
put(stream, {
hostname: host.address,
path,
protocol: 'https',
query,
rejectUnauthorized: false
})
this.putResource(
stream,
onlyMetadata ? '/import_metadata/' : '/import/',
{ host, query }
)
])
// Importing a metadata archive of running VMs is currently
@ -1867,7 +1851,6 @@ export default class Xapi extends XapiBase {
const query = {
format,
session_id: this.sessionId,
task_id: taskRef,
vdi: vdi.$ref
}
@ -1881,12 +1864,9 @@ export default class Xapi extends XapiBase {
}`)
const task = this._watchTask(taskRef)
return httpRequest($cancelToken, {
hostname: host.address,
path: '/export_raw_vdi/',
protocol: 'https',
query,
rejectUnauthorized: false
return this.getResource($cancelToken, '/export_raw_vdi/', {
host,
query
}).then(response => {
response.task = task
@ -1911,13 +1891,6 @@ export default class Xapi extends XapiBase {
async _importVdiContent (vdi, stream, format = VDI_FORMAT_VHD) {
const taskRef = await this._createTask('VDI Content Import', vdi.name_label)
const query = {
session_id: this.sessionId,
task_id: taskRef,
format,
vdi: vdi.$ref
}
const pbd = find(vdi.$SR.$PBDs, 'currently_attached')
if (!pbd) {
throw new Error('no valid PBDs found')
@ -1927,12 +1900,13 @@ export default class Xapi extends XapiBase {
await Promise.all([
stream.checksumVerified,
task,
put(stream, {
hostname: pbd.$host.address,
path: '/import_raw_vdi/',
protocol: 'https',
query,
rejectUnauthorized: false
this.putResource(stream, '/import_raw_vdi/', {
host: pbd.host,
query: {
format,
task_id: taskRef,
vdi: vdi.$ref
}
})
])
}
@ -2171,10 +2145,7 @@ export default class Xapi extends XapiBase {
await fs.writeFile('openstack/latest/user_data', config)
// Transform the buffer into a stream
const stream = bufferToStream(buffer)
await this.importVdiContent(vdi.$id, stream, {
format: VDI_FORMAT_RAW
})
await this._importVdiContent(vdi, buffer, VDI_FORMAT_RAW)
await this._createVbd(vm, vdi)
}

21
src/xapi/mixins/patching.js
View File

@ -1,12 +1,10 @@
import deferrable from 'golike-defer'
import filter from 'lodash/filter'
import httpRequest from 'http-request-plus'
import includes from 'lodash/includes'
import some from 'lodash/some'
import sortBy from 'lodash/sortBy'
import unzip from 'julien-f-unzip'
import httpProxy from '../../http-proxy'
import { debounce } from '../../decorators'
import {
createRawObject,
@ -19,7 +17,6 @@ import {
import {
debug,
put,
useUpdateSystem
} from '../utils'
@ -27,9 +24,8 @@ export default {
// FIXME: should be static
@debounce(24 * 60 * 60 * 1000)
async _getXenUpdates () {
const { readAll, statusCode } = await httpRequest(
'http://updates.xensource.com/XenServer/updates.xml',
{ agent: httpProxy }
const { readAll, statusCode } = await this.xo.httpRequest(
'http://updates.xensource.com/XenServer/updates.xml'
)
if (statusCode !== 200) {
@ -211,15 +207,10 @@ export default {
const task = this._watchTask(taskRef)
const [ patchRef ] = await Promise.all([
task,
put(stream, {
hostname: this.pool.$master.address,
path: '/pool_patch_upload',
protocol: 'https',
this.putResource(stream, '/pool_patch_upload', {
query: {
session_id: this.sessionId,
task_id: taskRef
},
rejectUnauthorized: false
}
})
])
@ -238,7 +229,7 @@ export default {
throw new Error('no such patch ' + uuid)
}
let stream = await httpRequest(patchInfo.url, { agent: httpProxy })
let stream = await this.xo.httpRequest(patchInfo.url)
stream = await new Promise((resolve, reject) => {
const PATCH_RE = /\.xsupdate$/
stream.pipe(unzip.Parse()).on('entry', entry => {
@ -280,7 +271,7 @@ export default {
throw new Error('no such patch ' + uuid)
}
let stream = await httpRequest(patchInfo.url, { agent: httpProxy })
let stream = await this.xo.httpRequest(patchInfo.url)
stream = await new Promise((resolve, reject) => {
stream.pipe(unzip.Parse()).on('entry', entry => {
entry.length = entry.size

32
src/xapi/utils.js
View File

@ -1,7 +1,6 @@
// import isFinite from 'lodash/isFinite'
import camelCase from 'lodash/camelCase'
import createDebug from 'debug'
import httpRequest from 'http-request-plus'
import isEqual from 'lodash/isEqual'
import isPlainObject from 'lodash/isPlainObject'
import pickBy from 'lodash/pickBy'
@ -348,37 +347,6 @@ export const NULL_REF = 'OpaqueRef:NULL'
// ===================================================================
// HTTP put, use an ugly hack if the length is not known because XAPI
// does not support chunk encoding.
export const put = (stream, {
headers: { ...headers } = {},
...opts
}) => {
const makeRequest = () => httpRequest.put(opts, {
body: stream,
headers
})
// Xen API does not support chunk encoding.
if (stream.length == null) {
// add a fake huge content length (1 PiB)
headers['content-length'] = '1125899906842624'
const promise = makeRequest()
// when the data has been emitted, close the connection
stream.on('end', () => {
setTimeout(() => {
promise.cancel()
}, 1e3)
})
return promise.readAll()
}
return makeRequest().readAll()
}
export const useUpdateSystem = host => {
// Match Xen Center's condition: https://github.com/xenserver/xenadmin/blob/f3a64fc54bbff239ca6f285406d9034f57537d64/XenModel/Utils/Helpers.cs#L420
return versionSatisfies(host.software_version.platform_version, '^2.1.1')

23
src/xo-mixins/http.js Normal file
View File

@ -0,0 +1,23 @@
import hrp from 'http-request-plus'
import ProxyAgent from 'proxy-agent'
import {
firstDefined
} from '../utils'
export default class Http {
constructor (_, {
httpProxy = firstDefined(
process.env.http_proxy,
process.env.HTTP_PROXY
)
}) {
this._proxy = httpProxy && new ProxyAgent(httpProxy)
}
httpRequest (...args) {
return hrp({
agent: this._proxy
}, ...args)
}
}

2
src/xo-mixins/xen-servers.js
View File

@ -266,6 +266,8 @@ export default class {
}
return {
httpRequest: this._xo.httpRequest.bind(this),
install () {
objects.on('add', onAddOrUpdate)
objects.on('update', onAddOrUpdate)

38
yarn.lock
View File

@ -3233,21 +3233,13 @@ http-proxy@^1.16.2:
eventemitter3 "1.x.x"
requires-port "1.x.x"
http-request-plus@^0.1.3:
version "0.1.3"
resolved "https://registry.yarnpkg.com/http-request-plus/-/http-request-plus-0.1.3.tgz#44b90f6d3b391cf5f21c1dbfefc115c86975cdd7"
http-request-plus@^0.1.5:
version "0.1.5"
resolved "https://registry.yarnpkg.com/http-request-plus/-/http-request-plus-0.1.5.tgz#9aead8b230586397928ecbb9a6bed96a3bfe2210"
dependencies:
is-redirect "^1.0.0"
lodash "^4.17.4"
promise-toolbox "^0.8.2"
http-request-plus@^0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/http-request-plus/-/http-request-plus-0.1.4.tgz#c99cd36366e96c13f92da5954b3a2fd2ce2c531e"
dependencies:
is-redirect "^1.0.0"
lodash "^4.17.4"
promise-toolbox "^0.8.2"
promise-toolbox "^0.9.1"
http-server-plus@^0.8.0:
version "0.8.0"
@ -4435,11 +4427,11 @@ ltgt@^2.1.2, ltgt@~2.1.1:
version "2.1.3"
resolved "https://registry.yarnpkg.com/ltgt/-/ltgt-2.1.3.tgz#10851a06d9964b971178441c23c9e52698eece34"
make-error@^1, make-error@^1.2.2:
make-error@^1:
version "1.2.3"
resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.2.3.tgz#6c4402df732e0977ac6faf754a5074b3d2b1d19d"
make-error@^1.0.2, make-error@^1.2.0, make-error@^1.2.1, make-error@^1.3.0:
make-error@^1.0.2, make-error@^1.2.0, make-error@^1.2.1, make-error@^1.2.2, make-error@^1.2.3, make-error@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.0.tgz#52ad3a339ccf10ce62b4040b708fe707244b8b96"
@ -5171,12 +5163,18 @@ promise-polyfill@^6.0.1:
version "6.0.2"
resolved "https://registry.yarnpkg.com/promise-polyfill/-/promise-polyfill-6.0.2.tgz#d9c86d3dc4dc2df9016e88946defd69b49b41162"
promise-toolbox@^0.8.0, promise-toolbox@^0.8.2:
promise-toolbox@^0.8.0:
version "0.8.2"
resolved "https://registry.yarnpkg.com/promise-toolbox/-/promise-toolbox-0.8.2.tgz#91722a364e6a2d6d13319491da3068b7de0b348f"
dependencies:
make-error "^1.2.2"
promise-toolbox@^0.9.1, promise-toolbox@^0.9.2:
version "0.9.2"
resolved "https://registry.yarnpkg.com/promise-toolbox/-/promise-toolbox-0.9.2.tgz#3577b6815f1b5c5a26cdd7517bf928a0c478eb4a"
dependencies:
make-error "^1.2.3"
promise@^7.0.1:
version "7.1.1"
resolved "https://registry.yarnpkg.com/promise/-/promise-7.1.1.tgz#489654c692616b8aa55b0724fa809bb7db49c5bf"
@ -6795,23 +6793,23 @@ xdg-basedir@^2.0.0:
dependencies:
os-homedir "^1.0.0"
xen-api@^0.12.0:
version "0.12.0"
resolved "https://registry.yarnpkg.com/xen-api/-/xen-api-0.12.0.tgz#1eefa8da0fe25c5a5e104d548579f329c4de22cc"
xen-api@^0.12.1:
version "0.12.1"
resolved "https://registry.yarnpkg.com/xen-api/-/xen-api-0.12.1.tgz#b9da5489ce9a0d7fe61f2c3d6e509c3e95094f36"
dependencies:
babel-polyfill "^6.23.0"
blocked "^1.2.1"
debug "^2.6.8"
event-to-promise "^0.8.0"
exec-promise "^0.7.0"
http-request-plus "^0.1.4"
http-request-plus "^0.1.5"
json-rpc-protocol "^0.11.2"
kindof "^2.0.0"
lodash "^4.17.4"
make-error "^1.3.0"
minimist "^1.2.0"
ms "^2.0.0"
promise-toolbox "^0.8.2"
promise-toolbox "^0.9.1"
pw "0.0.4"
xmlrpc "^1.3.2"
xo-collection "^0.4.1"