feat(@vates/cached-dns.lookup): small DNS cache (#6196)

See https://xcp-ng.org/forum/topic/5775/dns-queries-during-backup-job

@vates/cached-dns.lookup/.USAGE.md

@@ -0,0 +1,30 @@
+Node does not cache queries to `dns.lookup`, which can lead application doing a lot of connections to have perf issues and to saturate Node threads pool.
+
+This library attempts to mitigate these problems by providing a version of this function with a version short cache, applied on both errors and results.
+
+> Limitation: `verbatim: false` option is not supported.
+
+It has exactly the same API as the native method and can be used directly:
+
+```js
+import { createCachedLookup } from '@vates/cached-dns.lookup'
+
+const lookup = createCachedLookup()
+
+lookup('example.net', { all: true, family: 0 }, (error, result) => {
+  if (error != null) {
+    return console.warn(error)
+  }
+  console.log(result)
+})
+```
+
+Or it can be used to replace the native implementation and speed up the whole app:
+
+```js
+// assign our cached implementation to dns.lookup
+const restore = createCachedLookup().patchGlobal()
+
+// to restore the previous implementation
+restore()
+```

@vates/cached-dns.lookup/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/cached-dns.lookup/README.md

@@ -0,0 +1,63 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/cached-dns.lookup
+
+[![Package Version](https://badgen.net/npm/v/@vates/cached-dns.lookup)](https://npmjs.org/package/@vates/cached-dns.lookup) ![License](https://badgen.net/npm/license/@vates/cached-dns.lookup) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/cached-dns.lookup)](https://bundlephobia.com/result?p=@vates/cached-dns.lookup) [![Node compatibility](https://badgen.net/npm/node/@vates/cached-dns.lookup)](https://npmjs.org/package/@vates/cached-dns.lookup)
+
+> Cached implementation of dns.lookup
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/cached-dns.lookup):
+
+```
+> npm install --save @vates/cached-dns.lookup
+```
+
+## Usage
+
+Node does not cache queries to `dns.lookup`, which can leads application doing a lot of connections to have perf issues and to saturate Node threads pool.
+
+This library attemps to mitigate these problems by providing a version of this function with a version short cache, applied on both errors and results.
+
+> Limitation: `verbatim: false` option is not supported.
+
+I has exactly the same API as the native method and can be used directly:
+
+```js
+import { createCachedLookup } from '@vates/cached-dns.lookup'
+
+const lookup = createCachedLookup()
+
+lookup('example.net', { all: true, family: 0 }, (error, result) => {
+  if (error != null) {
+    return console.warn(error)
+  }
+  console.log(result)
+})
+```
+
+Or it can be used to replace the native implementation and speed up the whole app:
+
+```js
+// assign our cached implementation to dns.lookup
+const restore = createCachedLookup().patchGlobal()
+
+// to restore the previous implementation
+restore()
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/cached-dns.lookup/index.js

@@ -0,0 +1,72 @@
+'use strict'
+
+const assert = require('assert/strict')
+const dns = require('dns')
+const LRU = require('lru-cache')
+
+function reportResults(all, results, callback) {
+  if (all) {
+    callback(null, results)
+  } else {
+    const first = results[0]
+    callback(null, first.address, first.family)
+  }
+}
+
+exports.createCachedLookup = function createCachedLookup({ lookup = dns.lookup } = {}) {
+  const cache = new LRU({
+    max: 500,
+
+    // 1 minute: long enough to be effective, short enough so there is no need to bother with DNS TTLs
+    ttl: 60e3,
+  })
+
+  function cachedLookup(hostname, options, callback) {
+    let all = false
+    let family = 0
+    if (typeof options === 'function') {
+      callback = options
+    } else if (typeof options === 'number') {
+      family = options
+    } else if (options != null) {
+      assert.notEqual(options.verbatim, false, 'not supported by this implementation')
+      ;({ all = all, family = family } = options)
+    }
+
+    // cache by family option because there will be an error if there is no
+    // entries for the requestion family so we cannot easily cache all families
+    // and filter on reporting back
+    const key = hostname + '/' + family
+
+    const result = cache.get(key)
+    if (result !== undefined) {
+      setImmediate(reportResults, all, result, callback)
+    } else {
+      lookup(hostname, { all: true, family, verbatim: true }, function onLookup(error, results) {
+        // errors are not cached because this will delay recovery after DNS/network issues
+        //
+        // there are no reliable way to detect if the error is real or simply
+        // that there are no results for the requested hostname
+        //
+        // there should be much fewer errors than success, therefore it should
+        // not be a big deal to not cache them
+        if (error != null) {
+          return callback(error)
+        }
+
+        cache.set(key, results)
+        reportResults(all, results, callback)
+      })
+    }
+  }
+  cachedLookup.patchGlobal = function patchGlobal() {
+    const previous = dns.lookup
+    dns.lookup = cachedLookup
+    return function restoreGlobal() {
+      assert.equal(dns.lookup, cachedLookup)
+      dns.lookup = previous
+    }
+  }
+
+  return cachedLookup
+}

@vates/cached-dns.lookup/package.json

@@ -0,0 +1,32 @@
+{
+  "engines": {
+    "node": ">=8"
+  },
+  "dependencies": {
+    "lru-cache": "^7.0.4"
+  },
+  "private": false,
+  "name": "@vates/cached-dns.lookup",
+  "description": "Cached implementation of dns.lookup",
+  "keywords": [
+    "cache",
+    "dns",
+    "lookup"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/cached-dns.lookup",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/cached-dns.lookup",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.0.0",
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@xen-orchestra/backups/_backupWorker.js

@@ -4,6 +4,8 @@ require('@xen-orchestra/log/configure.js').catchGlobalErrors(
   require('@xen-orchestra/log').createLogger('xo:backups:worker')
 )
 
+require('@vates/cached-dns.lookup').createCachedLookup().patchGlobal()
+
 const Disposable = require('promise-toolbox/Disposable')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { compose } = require('@vates/compose')

@xen-orchestra/backups/package.json

@@ -16,6 +16,7 @@
     "postversion": "npm publish --access public"
   },
   "dependencies": {
+    "@vates/cached-dns.lookup": "^0.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.1",

@xen-orchestra/proxy/index.mjs

@@ -6,6 +6,7 @@ import getopts from 'getopts'
 import pRetry from 'promise-toolbox/retry'
 import { catchGlobalErrors } from '@xen-orchestra/log/configure.js'
 import { create as createServer } from 'http-server-plus'
+import { createCachedLookup } from '@vates/cached-dns.lookup'
 import { createLogger } from '@xen-orchestra/log'
 import { createSecureServer } from 'http2'
 import { genSelfSignedCert } from '@xen-orchestra/self-signed'
@@ -15,6 +16,8 @@ import { load as loadConfig } from 'app-conf'
 
 catchGlobalErrors(createLogger('xo:proxy'))
 
+createCachedLookup().patchGlobal()
+
 const { fatal, info, warn } = createLogger('xo:proxy:bootstrap')
 
 const APP_DIR = new URL('.', import.meta.url).pathname

@xen-orchestra/proxy/package.json

@@ -27,6 +27,7 @@
   "dependencies": {
     "@iarna/toml": "^2.2.0",
     "@koa/router": "^10.0.0",
+    "@vates/cached-dns.lookup": "^0.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.1",

CHANGELOG.unreleased.md

@@ -33,6 +33,7 @@
 >
 > In case of conflict, the highest (lowest in previous list) `$version` wins.
 
+- @vates/cached-dns.lookup major
 - @vates/event-listeners-manager major
 - xo-vmdk-to-vhd minor
 - xo-server minor

packages/xo-server/package.json

@@ -29,6 +29,7 @@
   "dependencies": {
     "@iarna/toml": "^2.2.1",
     "@vates/async-each": "^0.1.0",
+    "@vates/cached-dns.lookup": "^0.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.1",

packages/xo-server/src/cli.mjs

@@ -4,6 +4,7 @@ import * as SourceMapSupport from 'source-map-support'
 import Bluebird from 'bluebird'
 import execPromise from 'exec-promise'
 import { catchGlobalErrors } from '@xen-orchestra/log/configure.js'
+import { createCachedLookup } from '@vates/cached-dns.lookup'
 import { createLogger } from '@xen-orchestra/log'
 
 import main from './index.mjs'
@@ -33,4 +34,6 @@ global.Promise = Bluebird
 
 catchGlobalErrors(createLogger('xo:xo-server'))
 
+createCachedLookup().patchGlobal()
+
 execPromise(main)

yarn.lock

@@ -12241,6 +12241,11 @@ lru-cache@^6.0.0:
   dependencies:
     yallist "^4.0.0"
 
+lru-cache@^7.0.4:
+  version "7.8.1"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.8.1.tgz#68ee3f4807a57d2ba185b7fd90827d5c21ce82bb"
+  integrity sha512-E1v547OCgJvbvevfjgK9sNKIVXO96NnsTsFPBlg4ZxjhsJSODoH9lk8Bm0OxvHNm6Vm5Yqkl/1fErDxhYL8Skg==
+
 lru-queue@^0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/lru-queue/-/lru-queue-0.1.0.tgz#2738bd9f0d3cf4f84490c5736c48699ac632cda3"
@@ -17481,7 +17486,7 @@ tapable@^1.0.0, tapable@^1.1.3:
   resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2"
   integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==
 
-tar-stream@^2.0.1:
+tar-stream@^2.0.1, tar-stream@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"
   integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==