feat(mixins/SslCertificate): Let's Encrypt support (#6320)

@xen-orchestra/mixins/SslCertificate.mjs

@@ -0,0 +1,219 @@
+import { createLogger } from '@xen-orchestra/log'
+import { createSecureContext } from 'tls'
+import { dirname } from 'node:path'
+import { X509Certificate } from 'node:crypto'
+import acme from 'acme-client'
+import fs from 'node:fs/promises'
+import get from 'lodash/get.js'
+
+const { debug, info, warn } = createLogger('xo:mixins:sslCertificate')
+
+acme.setLogger(message => {
+  debug(message)
+})
+
+// - create any missing parent directories
+// - replace existing files
+// - secure permissions (read-only for the owner)
+async function outputFile(path, content) {
+  await fs.mkdir(dirname(path), { recursive: true })
+  try {
+    await fs.unlink(path)
+  } catch (error) {
+    if (error.code !== 'ENOENT') {
+      throw error
+    }
+  }
+  await fs.writeFile(path, content, { flag: 'wx', mode: 0o400 })
+}
+
+// from https://github.com/publishlab/node-acme-client/blob/master/examples/auto.js
+class SslCertificate {
+  #cert
+  #challengeCreateFn
+  #challengeRemoveFn
+  #delayBeforeRenewal = 30 * 24 * 60 * 60 * 1000 // 30 days
+  #secureContext
+  #updateSslCertificatePromise
+
+  constructor({ challengeCreateFn, challengeRemoveFn }, cert, key) {
+    this.#challengeCreateFn = challengeCreateFn
+    this.#challengeRemoveFn = challengeRemoveFn
+
+    this.#set(cert, key)
+  }
+
+  get #isValid() {
+    const cert = this.#cert
+    return cert !== undefined && Date.parse(cert.validTo) > Date.now() && cert.issuer !== cert.subject
+  }
+
+  get #shouldBeRenewed() {
+    return !(this.#isValid && Date.parse(this.#cert.validTo) > Date.now() + this.#delayBeforeRenewal)
+  }
+
+  #set(cert, key) {
+    this.#cert = new X509Certificate(cert)
+    this.#secureContext = createSecureContext({ cert, key })
+  }
+
+  async getSecureContext(httpsDomainName, config) {
+    // something changed in configuration or there is a network misconfiguration
+    // don't generate new let's encrypt challenges or invalid certificates
+    if (config?.acmeDomain !== httpsDomainName) {
+      warn(`certificates is configured for a domain, but receive http request from another`, {
+        acmeDomain: config?.acmeDomain,
+        httpsDomainName,
+      })
+      // fallback to self signed certificate to not lock user out
+      return undefined
+    }
+
+    if (!this.#shouldBeRenewed) {
+      return this.#secureContext
+    }
+
+    if (this.#updateSslCertificatePromise === undefined) {
+      // not currently updating certificate
+      //
+      // ensure we only refresh certificate once at a time
+      //
+      // promise is cleaned by #updateSslCertificate itself
+      this.#updateSslCertificatePromise = this.#updateSslCertificate(config)
+    }
+
+    // old certificate is still here, return it while updating
+    if (this.#isValid) {
+      return this.#secureContext
+    }
+
+    return this.#updateSslCertificatePromise
+  }
+
+  async #save(certPath, cert, keyPath, key) {
+    try {
+      await Promise.all([outputFile(keyPath, key), outputFile(certPath, cert)])
+      info('new certificate generated', { cert: certPath, key: keyPath })
+    } catch (error) {
+      warn(`couldn't write let's encrypt certificates to disk `, { error })
+    }
+  }
+
+  async #updateSslCertificate(config) {
+    const { cert: certPath, key: keyPath, acmeEmail, acmeDomain } = config
+    try {
+      let { acmeCa = 'letsencrypt/production' } = config
+      if (!(acmeCa.startsWith('http:') || acmeCa.startsWith('https:'))) {
+        acmeCa = get(acme.directory, acmeCa.split('/'))
+      }
+
+      /* Init client */
+      const client = new acme.Client({
+        directoryUrl: acmeCa,
+        accountKey: await acme.forge.createPrivateKey(),
+      })
+
+      /* Create CSR */
+      let [key, csr] = await acme.forge.createCsr({
+        commonName: acmeDomain,
+      })
+      csr = csr.toString()
+      key = key.toString()
+      debug('Successfully generated key and csr')
+
+      /* Certificate */
+      const cert = await client.auto({
+        challengeCreateFn: this.#challengeCreateFn,
+        challengePriority: ['http-01'],
+        challengeRemoveFn: this.#challengeRemoveFn,
+        csr,
+        email: acmeEmail,
+        skipChallengeVerification: true,
+        termsOfServiceAgreed: true,
+      })
+      debug('Successfully generated certificate')
+
+      this.#set(cert, key)
+
+      // don't wait for this
+      this.#save(certPath, cert, keyPath, key)
+
+      return this.#secureContext
+    } catch (error) {
+      warn(`couldn't renew ssl certificate`, { acmeDomain, error })
+    } finally {
+      this.#updateSslCertificatePromise = undefined
+    }
+  }
+}
+
+export default class SslCertificates {
+  #app
+  #challenges = new Map()
+  #challengeHandlers = {
+    challengeCreateFn: (authz, challenge, keyAuthorization) => {
+      this.#challenges.set(challenge.token, keyAuthorization)
+    },
+    challengeRemoveFn: (authz, challenge, keyAuthorization) => {
+      this.#challenges.delete(challenge.token)
+    },
+  }
+  #handlers = new Map()
+
+  constructor(app, { httpServer }) {
+    // don't setup the proxy if httpServer is not present
+    //
+    // that can happen when the app is instanciated in another context like xo-server-recover-account
+    if (httpServer === undefined) {
+      return
+    }
+    const prefix = '/.well-known/acme-challenge/'
+    httpServer.on('request', (req, res) => {
+      const { url } = req
+      if (url.startsWith(prefix)) {
+        const token = url.slice(prefix.length)
+        this.#acmeChallendMiddleware(req, res, token)
+      }
+    })
+
+    this.#app = app
+
+    httpServer.getSecureContext = this.getSecureContext.bind(this)
+  }
+
+  async getSecureContext(httpsDomainName, configKey, initialCert, initialKey) {
+    const config = this.#app.config.get(['http', 'listen', configKey])
+    const handlers = this.#handlers
+
+    // not a let's encrypt protected end point, sommething changed in the configuration
+    if (config.acmeDomain === undefined) {
+      warn(`config don't have acmeDomain, mandatory for let's encrypt`, { config })
+      handlers.delete(configKey)
+      return
+    }
+
+    let handler = handlers.get(configKey)
+    if (handler === undefined) {
+      // register the handler for this domain
+      handler = new SslCertificate(this.#challengeHandlers, initialCert, initialKey)
+      handlers.set(configKey, handler)
+    }
+    return handler.getSecureContext(httpsDomainName, config)
+  }
+
+  // middleware that will serve the http challenge to let's encrypt servers
+  #acmeChallendMiddleware(req, res, token) {
+    debug('fetching challenge for token ', token)
+    const challenge = this.#challenges.get(token)
+    debug('challenge content is ', challenge)
+    if (challenge === undefined) {
+      res.statusCode = 404
+      res.end()
+      return
+    }
+
+    res.write(challenge)
+    res.end()
+    debug('successfully answered challenge ')
+  }
+}

@xen-orchestra/mixins/docs/SslCertificate.md

@@ -0,0 +1,40 @@
+> This module provides [Let's Encrypt](https://letsencrypt.org/) integration to `xo-proxy` and `xo-server`.
+
+First of all, make sure your server is listening on HTTP on port 80 and on HTTPS 443.
+
+In `xo-server`, to avoid HTTP access, enable the redirection to HTTPs:
+
+```toml
+[http]
+redirectToHttps = true
+```
+
+Your server must be reachable with the configured domain to the certificate provider (e.g. Let's Encrypt), it usually means publicly reachable.
+
+Finally, add the following entries to your HTTPS configuration.
+
+```toml
+# Must be set to true for this feature
+autoCert = true
+
+# ACME (e.g. Let's Encrypt, ZeroSSL) CA directory
+#
+# Specifies the URL to the ACME CA's directory. It is strongly recommended to
+# set this to `letsencrypt/staging` for testing or development.
+#
+# A identifier `provider/directory` can be passed instead of a URL, see the
+# list of supported directories here: https://www.npmjs.com/package/acme-client#directory-urls
+#
+# Default is 'letsencrypt/production'
+acmeCa = 'letsencrypt/staging'
+
+# Domain for which the certificate should be created.
+#
+# This entry is required.
+acmeDomain = 'my.domain.net'
+
+# Optional email address which will be used for the certificate creation.
+#
+# It will be notified of any issues.
+acmeEmail = 'admin@my.domain.net'
+```

@xen-orchestra/mixins/package.json

@@ -16,13 +16,14 @@
   "license": "AGPL-3.0-or-later",
   "version": "0.6.0",
   "engines": {
-    "node": ">=12"
+    "node": ">=15.6"
   },
   "dependencies": {
     "@vates/event-listeners-manager": "^1.0.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/emit-async": "^1.0.0",
     "@xen-orchestra/log": "^0.3.0",
+    "acme-client": "^4.2.5",
     "app-conf": "^2.1.0",
     "lodash": "^4.17.21",
     "promise-toolbox": "^0.21.0"

@xen-orchestra/proxy/app/index.mjs

@@ -1,6 +1,7 @@
 import Config from '@xen-orchestra/mixins/Config.mjs'
 import Hooks from '@xen-orchestra/mixins/Hooks.mjs'
 import HttpProxy from '@xen-orchestra/mixins/HttpProxy.mjs'
+import SslCertificate from '@xen-orchestra/mixins/SslCertificate.mjs'
 import mixin from '@xen-orchestra/mixin'
 import { createDebounceResource } from '@vates/disposable/debounceResource.js'
 
@@ -14,9 +15,23 @@ import ReverseProxy from './mixins/reverseProxy.mjs'
 
 export default class App {
   constructor(opts) {
-    mixin(this, { Api, Appliance, Authentication, Backups, Config, Hooks, HttpProxy, Logs, Remotes, ReverseProxy }, [
+    mixin(
-      opts,
+      this,
-    ])
+      {
+        Api,
+        Appliance,
+        Authentication,
+        Backups,
+        Config,
+        Hooks,
+        HttpProxy,
+        Logs,
+        Remotes,
+        ReverseProxy,
+        SslCertificate,
+      },
+      [opts]
+    )
 
     const debounceResource = createDebounceResource()
     this.config.watchDuration('resourceCacheDelay', delay => {

@xen-orchestra/proxy/index.mjs

@@ -56,11 +56,32 @@ ${APP_NAME} v${APP_VERSION}
     createSecureServer: opts => createSecureServer({ ...opts, allowHTTP1: true }),
   })
 
-  forOwn(config.http.listen, async ({ autoCert, cert, key, ...opts }) => {
+  forOwn(config.http.listen, async ({ autoCert, cert, key, ...opts }, configKey) => {
+    const useAcme = autoCert && opts.acmeDomain !== undefined
+
+    // don't pass these entries to httpServer.listen(opts)
+    for (const key of Object.keys(opts).filter(_ => _.startsWith('acme'))) {
+      delete opts[key]
+    }
+
     try {
-      const niceAddress = await pRetry(
+      let niceAddress
-        async () => {
       if (cert !== undefined && key !== undefined) {
+        if (useAcme) {
+          opts.SNICallback = async (serverName, callback) => {
+            try {
+              // injected by mixins/SslCertificate
+              const secureContext = await httpServer.getSecureContext(serverName, configKey, opts.cert, opts.key)
+              callback(null, secureContext)
+            } catch (error) {
+              warn(error)
+              callback(error, null)
+            }
+          }
+        }
+
+        niceAddress = await pRetry(
+          async () => {
             try {
               opts.cert = fse.readFileSync(cert)
               opts.key = fse.readFileSync(key)
@@ -76,7 +97,6 @@ ${APP_NAME} v${APP_VERSION}
               opts.cert = pems.cert
               opts.key = pems.key
             }
-          }
 
             return httpServer.listen(opts)
           },
@@ -90,6 +110,9 @@ ${APP_NAME} v${APP_VERSION}
             },
           }
         )
+      } else {
+        niceAddress = await httpServer.listen(opts)
+      }
 
       info(`Web server listening on ${niceAddress}`)
     } catch (error) {

CHANGELOG.unreleased.md

@@ -7,6 +7,8 @@
 
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
+- HTTPS server can acquire SSL certificate from Let's Encrypt (PR [#6320](https://github.com/vatesfr/xen-orchestra/pull/6320))
+
 ### Bug fixes
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”
@@ -27,4 +29,8 @@
 
 <!--packages-start-->
 
+- @xen-orchestra/mixins minor
+- @xen-orchestra/proxy minor
+- xo-server minor
+
 <!--packages-end-->

packages/xo-server/src/index.mjs

@@ -400,11 +400,34 @@ async function makeWebServerListen(
     cert = certificate,
 
     key,
+
+    configKey,
+
     ...opts
   }
 ) {
   try {
+    const useAcme = autoCert && opts.acmeDomain !== undefined
+
+    // don't pass these entries to httpServer.listen(opts)
+    for (const key of Object.keys(opts).filter(_ => _.startsWith('acme'))) {
+      delete opts[key]
+    }
+
     if (cert && key) {
+      if (useAcme) {
+        opts.SNICallback = async (serverName, callback) => {
+          try {
+            // injected by mixins/SslCertificate
+            const secureContext = await webServer.getSecureContext(serverName, configKey, opts.cert, opts.key)
+            callback(null, secureContext)
+          } catch (error) {
+            log.warn(error)
+            callback(error, null)
+          }
+        }
+      }
+
       try {
         ;[opts.cert, opts.key] = await Promise.all([fse.readFile(cert), fse.readFile(key)])
         if (opts.key.includes('ENCRYPTED')) {
@@ -419,7 +442,6 @@ async function makeWebServerListen(
         if (!(autoCert && error.code === 'ENOENT')) {
           throw error
         }
-
         const pems = await genSelfSignedCert()
         await Promise.all([
           fse.outputFile(cert, pems.cert, { flag: 'wx', mode: 0o400 }),
@@ -452,8 +474,9 @@ async function makeWebServerListen(
 
 async function createWebServer({ listen, listenOptions }) {
   const webServer = stoppable(new WebServer())
-
+  await Promise.all(
-  await Promise.all(map(listen, opts => makeWebServerListen(webServer, { ...listenOptions, ...opts })))
+    map(listen, (opts, configKey) => makeWebServerListen(webServer, { ...listenOptions, ...opts, configKey }))
+  )
 
   return webServer
 }
@@ -760,6 +783,10 @@ export default async function main(args) {
 
   // Attaches express to the web server.
   webServer.on('request', (req, res) => {
+    // don't redirect let's encrypt challenge to https
+    if (req.url.startsWith('/.well')) {
+      return
+    }
     // don't handle proxy requests
     if (req.url.startsWith('/')) {
       return express(req, res)

packages/xo-server/src/xo.mjs

@@ -8,6 +8,7 @@ import iteratee from 'lodash/iteratee.js'
 import mixin from '@xen-orchestra/mixin'
 import mixinLegacy from '@xen-orchestra/mixin/legacy.js'
 import stubTrue from 'lodash/stubTrue.js'
+import SslCertificate from '@xen-orchestra/mixins/SslCertificate.mjs'
 import { Collection as XoCollection } from 'xo-collection'
 import { createClient as createRedisClient } from 'redis'
 import { createDebounceResource } from '@vates/disposable/debounceResource.js'
@@ -29,8 +30,7 @@ export default class Xo extends EventEmitter {
   constructor(opts) {
     super()
 
-    mixin(this, { Config, Hooks, HttpProxy }, [opts])
+    mixin(this, { Config, Hooks, HttpProxy, SslCertificate }, [opts])
-
     // a lot of mixins adds listener for start/stop/… events
     this.hooks.setMaxListeners(0)
 

yarn.lock

@@ -3294,6 +3294,17 @@ accepts@^1.3.5, accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.8:
     mime-types "~2.1.34"
     negotiator "0.6.3"
 
+acme-client@^4.2.5:
+  version "4.2.5"
+  resolved "https://registry.yarnpkg.com/acme-client/-/acme-client-4.2.5.tgz#d18e29aadb38fbc8c6d4ce289f26392b51b5a698"
+  integrity sha512-dtnck4sdZ2owFLTC73Ewjx0kmvsRjTRgaOc8UztCNODT+lr1DXj0tiuUXjeY4LAzZryXCtCib/E+KD8NYeP1aw==
+  dependencies:
+    axios "0.26.1"
+    backo2 "^1.0.0"
+    bluebird "^3.5.0"
+    debug "^4.1.1"
+    node-forge "^1.3.0"
+
 acorn-globals@^3.0.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/acorn-globals/-/acorn-globals-3.1.0.tgz#fd8270f71fbb4996b004fa880ee5d46573a731bf"
@@ -4024,6 +4035,13 @@ aws4@^1.8.0:
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59"
   integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==
 
+axios@0.26.1:
+  version "0.26.1"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.26.1.tgz#1ede41c51fcf51bbbd6fd43669caaa4f0495aaa9"
+  integrity sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==
+  dependencies:
+    follow-redirects "^1.14.8"
+
 babel-jest@^28.1.2:
   version "28.1.2"
   resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-28.1.2.tgz#2b37fb81439f14d34d8b2cc4a4bd7efabf9acbfe"
@@ -4203,6 +4221,11 @@ bach@^1.0.0:
     async-settle "^1.0.0"
     now-and-later "^2.0.0"
 
+backo2@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/backo2/-/backo2-1.0.2.tgz#31ab1ac8b129363463e35b3ebb69f4dfcfba7947"
+  integrity sha512-zj6Z6M7Eq+PBZ7PQxl5NT665MvJdAkzp0f60nAJ+sLaSCBPMwVak5ZegFbgVCzFcCJTKFoMizvM5Ld7+JrRJHA==
+
 backoff@~2.3.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/backoff/-/backoff-2.3.0.tgz#ee7c7e38093f92e472859db635e7652454fc21ea"
@@ -4335,7 +4358,7 @@ blocked@^1.2.1:
   resolved "https://registry.yarnpkg.com/blocked/-/blocked-1.3.0.tgz#f91bbbcced7aa26835942ef0722d51c625ae1a01"
   integrity sha512-tAb98b4F01wLnKIjCpp17hheKIKnd7j+SgxwgNHQNjQ+EcvOCRZ1HPVNZt3/XnpMjFymVdIZlBQysi+s7OltLw==
 
-bluebird@^3.1.1, bluebird@^3.5.1, bluebird@^3.5.5:
+bluebird@^3.1.1, bluebird@^3.5.0, bluebird@^3.5.1, bluebird@^3.5.5:
   version "3.7.2"
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
   integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
@@ -8518,7 +8541,7 @@ flush-write-stream@^1.0.0, flush-write-stream@^1.0.2:
     inherits "^2.0.3"
     readable-stream "^2.3.6"
 
-follow-redirects@^1.0.0:
+follow-redirects@^1.0.0, follow-redirects@^1.14.8:
   version "1.15.1"
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.1.tgz#0ca6a452306c9b276e4d3127483e29575e207ad5"
   integrity sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==
@@ -12930,6 +12953,11 @@ node-forge@^0.10.0:
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3"
   integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==
 
+node-forge@^1.3.0:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
+  integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==
+
 node-gyp-build@^4.3.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.5.0.tgz#7a64eefa0b21112f89f58379da128ac177f20e40"