feat(mixins/HttpProxy): HTTP/HTTP CONNECT proxy (#6201)

@xen-orchestra/mixins/HttpProxy.js

@@ -0,0 +1,137 @@
+'use strict'
+
+const { debug, warn } = require('@xen-orchestra/log').createLogger('xo:mixins:HttpProxy')
+const { EventListenersManager } = require('@vates/event-listeners-manager')
+const { pipeline } = require('stream')
+const { ServerResponse, request } = require('http')
+const assert = require('assert')
+const fromCallback = require('promise-toolbox/fromCallback')
+const fromEvent = require('promise-toolbox/fromEvent')
+const net = require('net')
+
+const { parseBasicAuth } = require('./_parseBasicAuth.js')
+
+const IGNORED_HEADERS = new Set([
+  // https://datatracker.ietf.org/doc/html/rfc2616#section-13.5.1
+  'connection',
+  'keep-alive',
+  'proxy-authenticate',
+  'proxy-authorization',
+  'te',
+  'trailers',
+  'transfer-encoding',
+  'upgrade',
+
+  // don't forward original host
+  'host',
+])
+
+module.exports = class HttpProxy {
+  #app
+
+  constructor(app, { httpServer }) {
+    this.#app = app
+
+    const events = new EventListenersManager(httpServer)
+    app.config.watch('http.proxy.enabled', (enabled = false) => {
+      events.removeAll()
+      if (enabled) {
+        events.add('connect', this.#handleConnect.bind(this)).add('request', this.#handleRequest.bind(this))
+      }
+    })
+  }
+
+  async #handleAuthentication(req, res, next) {
+    const auth = parseBasicAuth(req.headers['proxy-authorization'])
+
+    let authenticated = false
+
+    if (auth !== undefined) {
+      const app = this.#app
+
+      if (app.authenticateUser !== undefined) {
+        // xo-server
+        try {
+          const { user } = await app.authenticateUser(auth)
+          authenticated = user.permission === 'admin'
+        } catch (error) {}
+      } else {
+        // xo-proxy
+        authenticated = (await app.authentication.findProfile(auth)) !== undefined
+      }
+    }
+
+    if (authenticated) {
+      return next()
+    }
+
+    // https://datatracker.ietf.org/doc/html/rfc7235#section-3.2
+    res.statusCode = '407'
+    res.setHeader('proxy-authenticate', 'Basic realm="proxy"')
+    return res.end('Proxy Authentication Required')
+  }
+
+  // https://nodejs.org/api/http.html#event-connect
+  async #handleConnect(req, clientSocket, head) {
+    const { url } = req
+
+    debug('CONNECT proxy', { url })
+
+    // https://github.com/TooTallNate/proxy/blob/d677ef31fd4ca9f7e868b34c18b9cb22b0ff69da/proxy.js#L391-L398
+    const res = new ServerResponse(req)
+    res.assignSocket(clientSocket)
+
+    try {
+      await this.#handleAuthentication(req, res, async () => {
+        const { port, hostname } = new URL('http://' + req.url)
+        const serverSocket = net.connect(port || 80, hostname)
+
+        await fromEvent(serverSocket, 'connect')
+
+        clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n')
+        serverSocket.write(head)
+        fromCallback(pipeline, clientSocket, serverSocket).catch(warn)
+        fromCallback(pipeline, serverSocket, clientSocket).catch(warn)
+      })
+    } catch (error) {
+      warn(error)
+      clientSocket.end()
+    }
+  }
+
+  async #handleRequest(req, res) {
+    const { url } = req
+
+    if (url.startsWith('/')) {
+      // not a proxy request
+      return
+    }
+
+    debug('HTTP proxy', { url })
+
+    try {
+      assert(url.startsWith('http:'), 'HTTPS should use connect')
+
+      await this.#handleAuthentication(req, res, async () => {
+        const { headers } = req
+        const pHeaders = {}
+        for (const key of Object.keys(headers)) {
+          if (!IGNORED_HEADERS.has(key)) {
+            pHeaders[key] = headers[key]
+          }
+        }
+
+        const pReq = request(url, { headers: pHeaders, method: req.method })
+        fromCallback(pipeline, req, pReq).catch(warn)
+
+        const pRes = await fromEvent(pReq, 'response')
+        res.writeHead(pRes.statusCode, pRes.statusMessage, pRes.headers)
+        await fromCallback(pipeline, pRes, res)
+      })
+    } catch (error) {
+      res.statusCode = 500
+      res.end('Internal Server Error')
+      warn(error)
+    }
+  }
+}

@xen-orchestra/mixins/_parseBasicAuth.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const RE = /^\s*basic\s+(.+?)\s*$/i
+
+exports.parseBasicAuth = function parseBasicAuth(header) {
+  if (header === undefined) {
+    return
+  }
+
+  const matches = RE.exec(header)
+  if (matches === null) {
+    return
+  }
+
+  let credentials = Buffer.from(matches[1], 'base64').toString()
+
+  const i = credentials.indexOf(':')
+  if (i === -1) {
+    credentials = { token: credentials }
+  } else {
+    // https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.1
+    credentials = {
+      username: credentials.slice(0, i),
+      password: credentials.slice(i + 1),
+    }
+  }
+
+  return credentials
+}

@xen-orchestra/mixins/docs/HttpProxy.md

@@ -0,0 +1,74 @@
+> This module provides an HTTP and HTTPS proxy for `xo-proxy` and `xo-server`.
+
+- [Set up](#set-up)
+- [Usage](#usage)
+  - [`xo-proxy`](#xo-proxy)
+  - [`xo-server`](#xo-server)
+- [Use cases](#use-cases)
+  - [Access hosts in a private network](#access-hosts-in-a-private-network)
+  - [Allow upgrading xo-proxy via xo-server](#allow-upgrading-xo-proxy-via-xo-server)
+
+## Set up
+
+The proxy is disabled by default, to enable it, add the following lines to your config:
+
+```toml
+[http.proxy]
+enabled = true
+```
+
+## Usage
+
+For safety reasons, the proxy requires authentication to be used.
+
+### `xo-proxy`
+
+Use the authentication token:
+
+```
+$ cat ~/.config/xo-proxy/config.z-auto.json
+{"authenticationToken":"J0BgKritQgPxoyZrBJ5ViafQfLk06YoyFwC3fmfO5wU"}
+```
+
+Proxy URL to use:
+
+```
+https://J0BgKritQgPxoyZrBJ5ViafQfLk06YoyFwC3fmfO5wU@xo-proxy.company.lan
+```
+
+### `xo-server`
+
+> Only available for admin users.
+
+You can use your credentials:
+
+```
+https://user:password@xo.company.lan
+```
+
+Or create a dedicated token with `xo-cli`:
+
+```
+$ xo-cli --createToken xoa.company.lan admin@admin.net
+Password: ********
+Successfully logged with admin@admin.net
+Authentication token created
+
+DiYBFavJwf9GODZqQJs23eAx9eh3KlsRhBi8RcoX0KM
+```
+
+And use it in the URL:
+
+```
+https://DiYBFavJwf9GODZqQJs23eAx9eh3KlsRhBi8RcoX0KM@xo.company.lan
+```
+
+## Use cases
+
+### Access hosts in a private network
+
+To access hosts in a private network, deploy an XO Proxy in this network, expose its port 443 and use it as an HTTP proxy to connect to your servers in XO.
+
+### Allow upgrading xo-proxy via xo-server
+
+If your xo-proxy does not have direct Internet access, you can use xo-server as an HTTP proxy to make upgrades possible.

@xen-orchestra/mixins/package.json

@@ -19,11 +19,13 @@
     "node": ">=12"
   },
   "dependencies": {
+    "@vates/event-listeners-manager": "^0.0.0",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/emit-async": "^0.1.0",
     "@xen-orchestra/log": "^0.3.0",
     "app-conf": "^2.1.0",
-    "lodash": "^4.17.21"
+    "lodash": "^4.17.21",
+    "promise-toolbox": "^0.21.0"
   },
   "scripts": {
     "postversion": "npm publish --access public"

@xen-orchestra/proxy/app/index.mjs

@@ -1,5 +1,6 @@
 import Config from '@xen-orchestra/mixins/Config.js'
 import Hooks from '@xen-orchestra/mixins/Hooks.js'
+import HttpProxy from '@xen-orchestra/mixins/HttpProxy.js'
 import mixin from '@xen-orchestra/mixin'
 import { createDebounceResource } from '@vates/disposable/debounceResource.js'
 
@@ -13,7 +14,9 @@ import ReverseProxy from './mixins/reverseProxy.mjs'
 
 export default class App {
   constructor(opts) {
-    mixin(this, { Api, Appliance, Authentication, Backups, Config, Hooks, Logs, Remotes, ReverseProxy }, [opts])
+    mixin(this, { Api, Appliance, Authentication, Backups, Config, Hooks, HttpProxy, Logs, Remotes, ReverseProxy }, [
+      opts,
+    ])
 
     const debounceResource = createDebounceResource()
     this.config.watchDuration('resourceCacheDelay', delay => {

@xen-orchestra/proxy/app/mixins/api.mjs

@@ -52,7 +52,7 @@ export default class Api {
       ctx.req.setTimeout(0)
 
       const profile = await app.authentication.findProfile({
-        authenticationToken: ctx.cookies.get('authenticationToken'),
+        token: ctx.cookies.get('authenticationToken'),
       })
       if (profile === undefined) {
         ctx.status = 401

@xen-orchestra/proxy/app/mixins/authentication.mjs

@@ -52,7 +52,7 @@ export default class Authentication {
   }
 
   async findProfile(credentials) {
-    if (credentials?.authenticationToken === this.#token) {
+    if (credentials?.token === this.#token) {
       return new Profile()
     }
   }

CHANGELOG.unreleased.md

@@ -12,6 +12,7 @@
 - [VM migrate] Allow to choose a private network for VIFs network (PR [#6200](https://github.com/vatesfr/xen-orchestra/pull/6200))
 - [Proxy] Disable "Deploy proxy" button for source users (PR [#6199](https://github.com/vatesfr/xen-orchestra/pull/6199))
 - [Import] Feat import `iso` disks (PR [#6180](https://github.com/vatesfr/xen-orchestra/pull/6180))
+- New HTTP/HTTPS proxy implemented in xo-proxy and xo-server, [see the documentation](https://github.com/vatesfr/xen-orchestra/blob/master/@xen-orchestra/mixins/docs/HttpProxy.md) (PR [#6201](https://github.com/vatesfr/xen-orchestra/pull/6201))
 
 ### Bug fixes
 
@@ -41,10 +42,11 @@
 - @vates/cached-dns.lookup major
 - @vates/event-listeners-manager major
 - xen-api minor
+- @xen-orchestra/mixins minor
 - xo-vmdk-to-vhd minor
 - @xen-orchestra/fs patch
 - @xen-orchestra/backups patch
-- @xen-orchestra/proxy patch
+- @xen-orchestra/proxy minor
 - xo-server minor
 - xo-web minor
 - vhd-cli patch

package.json

@@ -61,6 +61,7 @@
       "/@vates/predicates/",
       "/@xen-orchestra/audit-core/",
       "/dist/",
+      "/xen-api/",
       "/xo-server/",
       "/xo-server-test/",
       "/xo-web/"

packages/xen-api/_parseUrl.spec.js

@@ -0,0 +1,52 @@
+'use strict'
+
+const t = require('tap')
+
+const parseUrl = require('./dist/_parseUrl.js').default
+
+const data = {
+  'xcp.company.lan': {
+    hostname: 'xcp.company.lan',
+    pathname: '/',
+    protocol: 'https:',
+  },
+  '[::1]': {
+    hostname: '::1',
+    pathname: '/',
+    protocol: 'https:',
+  },
+  'http://username:password@xcp.company.lan': {
+    auth: 'username:password',
+    hostname: 'xcp.company.lan',
+    password: 'password',
+    pathname: '/',
+    protocol: 'http:',
+    username: 'username',
+  },
+  'https://username@xcp.company.lan': {
+    auth: 'username',
+    hostname: 'xcp.company.lan',
+    pathname: '/',
+    protocol: 'https:',
+    username: 'username',
+  },
+}
+
+t.test('invalid url', function (t) {
+  t.throws(() => parseUrl(''))
+  t.end()
+})
+
+for (const url of Object.keys(data)) {
+  t.test(url, function (t) {
+    const parsed = parseUrl(url)
+    for (const key of Object.keys(parsed)) {
+      if (parsed[key] === undefined) {
+        delete parsed[key]
+      }
+    }
+
+    t.same(parsed, data[url])
+    t.end()
+  })
+}

packages/xen-api/package.json

@@ -56,7 +56,8 @@
     "@babel/plugin-proposal-decorators": "^7.0.0",
     "@babel/preset-env": "^7.8.0",
     "cross-env": "^7.0.2",
-    "rimraf": "^3.0.0"
+    "rimraf": "^3.0.0",
+    "tap": "^16.1.0"
   },
   "scripts": {
     "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
@@ -65,6 +66,7 @@
     "prebuild": "rimraf dist/",
     "predev": "yarn run prebuild",
     "prepublishOnly": "yarn run build",
-    "postversion": "npm publish"
+    "postversion": "npm publish",
+    "test": "tap"
   }
 }

packages/xen-api/src/_parseUrl.js

@@ -1,4 +1,4 @@
-const URL_RE = /^(?:(https?:)\/*)?(?:([^:]+):([^@]+)@)?(?:\[([^\]]+)\]|([^:/]+))(?::([0-9]+))?(\/[^?#]*)?$/
+const URL_RE = /^(?:(https?:)\/*)?(?:(([^:]*)(?::([^@]*))?)@)?(?:\[([^\]]+)\]|([^:/]+))(?::([0-9]+))?(\/[^?#]*)?$/
 
 export default url => {
   const matches = URL_RE.exec(url)
@@ -6,12 +6,21 @@ export default url => {
     throw new Error('invalid URL: ' + url)
   }
 
-  const [, protocol = 'https:', username, password, ipv6, hostname = ipv6, port, pathname = '/'] = matches
-  const parsedUrl = { protocol, hostname, port, pathname }
-  if (username !== undefined) {
+  const [, protocol = 'https:', auth, username = '', password = '', ipv6, hostname = ipv6, port, pathname = '/'] =
+    matches
+  const parsedUrl = {
+    protocol,
+    hostname,
+    port,
+    pathname,
+
+    // compat with url.parse
+    auth,
+  }
+  if (username !== '') {
     parsedUrl.username = decodeURIComponent(username)
   }
-  if (password !== undefined) {
+  if (password !== '') {
     parsedUrl.password = decodeURIComponent(password)
   }
   return parsedUrl

packages/xen-api/src/index.js

@@ -122,8 +122,12 @@ export class Xapi extends EventEmitter {
     }
 
     this._allowUnauthorized = opts.allowUnauthorized
-    const { httpProxy } = opts
+    let { httpProxy } = opts
     if (httpProxy !== undefined) {
+      if (httpProxy.startsWith('https:')) {
+        httpProxy = parseUrl(httpProxy)
+        httpProxy.rejectUnauthorized = !opts.allowUnauthorized
+      }
       this._httpAgent = new ProxyAgent(httpProxy)
     }
     this._setUrl(url)

packages/xo-server/src/index.mjs

@@ -758,7 +758,12 @@ export default async function main(args) {
   }
 
   // Attaches express to the web server.
-  webServer.on('request', express)
+  webServer.on('request', (req, res) => {
+    // don't handle proxy requests
+    if (req.url.startsWith('/')) {
+      return express(req, res)
+    }
+  })
   webServer.on('upgrade', (req, socket, head) => {
     express.emit('upgrade', req, socket, head)
   })
@@ -772,6 +777,7 @@ export default async function main(args) {
     appVersion: APP_VERSION,
     config,
     express,
+    httpServer: webServer,
     safeMode,
   })
 

packages/xo-server/src/xo.mjs

@@ -1,6 +1,7 @@
 import Config from '@xen-orchestra/mixins/Config.js'
 import forEach from 'lodash/forEach.js'
 import Hooks from '@xen-orchestra/mixins/Hooks.js'
+import HttpProxy from '@xen-orchestra/mixins/HttpProxy.js'
 import includes from 'lodash/includes.js'
 import isEmpty from 'lodash/isEmpty.js'
 import iteratee from 'lodash/iteratee.js'
@@ -28,7 +29,7 @@ export default class Xo extends EventEmitter {
   constructor(opts) {
     super()
 
-    mixin(this, { Config, Hooks }, [opts])
+    mixin(this, { Config, Hooks, HttpProxy }, [opts])
 
     // a lot of mixins adds listener for start/stop/… events
     this.hooks.setMaxListeners(0)

yarn.lock

@@ -17442,7 +17442,7 @@ tap-yaml@^1.0.0:
   dependencies:
     yaml "^1.5.0"
 
-tap@^16.0.1:
+tap@^16.0.1, tap@^16.1.0:
   version "16.1.0"
   resolved "https://registry.yarnpkg.com/tap/-/tap-16.1.0.tgz#85e989313afb318e6447dfa74c8aeb01b1770278"
   integrity sha512-EFERYEEDCLjvsT+B+z/qAVuxh5JPEmtn0aGh1ZT/2BN5nVLm6VbcL9fR/Y2FtsxvHuEC3Q2xLc1n1h7mnWVP9w==