feat(backup/s3): add http and region parameters to S3 (#5658)

@xen-orchestra/fs/src/index.js

@@ -1,4 +1,5 @@
 import execa from 'execa'
+import { parse } from 'xo-remote-parser'
 
 import RemoteHandlerLocal from './local'
 import RemoteHandlerNfs from './nfs'
@@ -20,10 +21,7 @@ try {
 }
 
 export const getHandler = (remote, ...rest) => {
-  // FIXME: should be done in xo-remote-parser.
+  const Handler = HANDLERS[parse(remote.url).type]
-  const type = remote.url.split('://')[0]
-
-  const Handler = HANDLERS[type]
   if (!Handler) {
     throw new Error('Unhandled remote type')
   }

@xen-orchestra/fs/src/s3.js

@@ -1,5 +1,6 @@
 import aws from '@sullux/aws-sdk'
 import assert from 'assert'
+import http from 'http'
 import { parse } from 'xo-remote-parser'
 
 import RemoteHandlerAbstract from './abstract'
@@ -16,9 +17,8 @@ const IDEAL_FRAGMENT_SIZE = Math.ceil(MAX_OBJECT_SIZE / MAX_PARTS_COUNT) // the
 export default class S3Handler extends RemoteHandlerAbstract {
   constructor(remote, _opts) {
     super(remote)
-    const { host, path, username, password } = parse(remote.url)
+    const { host, path, username, password, protocol, region } = parse(remote.url)
-    // https://www.zenko.io/blog/first-things-first-getting-started-scality-s3-server/
+    const params = {
-    this._s3 = aws({
       accessKeyId: username,
       apiVersion: '2006-03-01',
       endpoint: host,
@@ -28,7 +28,16 @@ export default class S3Handler extends RemoteHandlerAbstract {
       httpOptions: {
         timeout: 600000,
       },
-    }).s3
+    }
+    if (protocol === 'http') {
+      params.httpOptions.agent = new http.Agent()
+      params.sslEnabled = false
+    }
+    if (region !== undefined) {
+      params.region = region
+    }
+
+    this._s3 = aws(params).s3
 
     const splitPath = path.split('/').filter(s => s.length)
     this._bucket = splitPath.shift()

CHANGELOG.unreleased.md

@@ -14,6 +14,7 @@
 - [Backup] Lock VM directory during backup to avoid race conditions (PR [#5746](https://github.com/vatesfr/xen-orchestra/pull/5746))
 - [XOA] Notify user when proxies need to be upgraded (PR [#5717](https://github.com/vatesfr/xen-orchestra/pull/5717))
 - [Host/network] Identify the management network [#5731](https://github.com/vatesfr/xen-orchestra/issues/5731) (PR [#5743](https://github.com/vatesfr/xen-orchestra/pull/5743))
+- [Backup/S3] Support for HTTP protocol and choice of region (PR [#5658](https://github.com/vatesfr/xen-orchestra/pull/5658))
 
 ### Bug fixes
 
@@ -44,6 +45,7 @@
 > In case of conflict, the highest (lowest in previous list) `$version` wins.
 
 - xo-server-perf-alert patch
+- xo-remote-parser minor
 - @xen-orchestra/fs minor
 - @xen-orchestra/xapi patch
 - @xen-orchestra/backups minor

packages/xo-remote-parser/src/index.js

@@ -37,9 +37,11 @@ export const parse = string => {
     object.domain = domain
     object.username = username
     object.password = password
-  } else if (type === 's3') {
+  } else if (type === 's3' || type === 's3+http') {
     const parsed = new Url(string)
+    object.protocol = parsed.protocol === 's3:' ? 'https' : 'http'
     object.type = 's3'
+    object.region = parsed.hash.length === 0 ? undefined : parsed.hash.slice(1) // remove '#'
     object.host = parsed.host
     object.path = parsed.pathname
     object.username = parsed.username
@@ -48,7 +50,7 @@ export const parse = string => {
   return object
 }
 
-export const format = ({ type, host, path, port, username, password, domain }) => {
+export const format = ({ type, host, path, port, username, password, domain, protocol = type, region }) => {
   type === 'local' && (type = 'file')
   let string = `${type}://`
   if (type === 'nfs') {
@@ -58,6 +60,7 @@ export const format = ({ type, host, path, port, username, password, domain }) =
     string += `${username}:${password}@${domain}\\\\${host}`
   }
   if (type === 's3') {
+    string = protocol === 'https' ? 's3://' : 's3+http://'
     string += `${username}:${encodeURIComponent(password)}@${host}`
   }
   path = sanitizePath(path)
@@ -68,5 +71,8 @@ export const format = ({ type, host, path, port, username, password, domain }) =
     path = `/${path}`
   }
   string += path
+  if (type === 's3' && region !== undefined) {
+    string += `#${region}`
+  }
   return string
 }

packages/xo-remote-parser/src/index.spec.js

@@ -48,10 +48,12 @@ const data = deepFreeze({
     string: 's3://AKIAS:XSuBupZ0mJlu%2B@s3-us-west-2.amazonaws.com/test-bucket/dir',
     object: {
       type: 's3',
+      protocol: 'https',
       host: 's3-us-west-2.amazonaws.com',
       path: '/test-bucket/dir',
       username: 'AKIAS',
       password: 'XSuBupZ0mJlu+',
+      region: undefined,
     },
   },
 })
@@ -85,6 +87,18 @@ const parseData = deepFreeze({
       password: 'pas:sw@ord',
     },
   },
+  'S3 with http and region': {
+    string: 's3+http://Administrator:password@192.168.100.225/bucket/dir#reg1',
+    object: {
+      type: 's3',
+      host: '192.168.100.225',
+      protocol: 'http',
+      path: '/bucket/dir',
+      region: 'reg1',
+      username: 'Administrator',
+      password: 'password',
+    },
+  },
 })
 
 const formatData = deepFreeze({

packages/xo-web/src/common/intl/messages.js

@@ -580,7 +580,9 @@ const messages = {
   remoteSmbPlaceHolderAddressShare: '<address>\\\\<share>',
   remoteSmbPlaceHolderOptions: 'Custom mount options',
   remoteS3PlaceHolderBucket: 'AWS S3 bucket name',
-  remoteS3PlaceHolderDirectory: 'directory',
+  remoteS3PlaceHolderDirectory: 'Directory',
+  remoteS3Region: 'Region, leave blank for default',
+  remoteS3TooltipProtocol: 'Check if you want HTTP instead of HTTPS',
   remotePlaceHolderPassword: 'Password(fill to edit)',
 
   // ------ New Storage -----

packages/xo-web/src/xo-app/settings/remotes/remote.js

@@ -3,6 +3,7 @@ import ActionButton from 'action-button'
 import decorate from 'apply-decorators'
 import Icon from 'icon'
 import React from 'react'
+import Tooltip from 'tooltip'
 import { addSubscriptions, resolveId } from 'utils'
 import { alert, confirm } from 'modal'
 import { createRemote, editRemote, subscribeRemotes } from 'xo'
@@ -11,7 +12,7 @@ import { format } from 'xo-remote-parser'
 import { generateId, linkState } from 'reaclette-utils'
 import { injectState, provideState } from 'reaclette'
 import { map, some, trimStart } from 'lodash'
-import { Password, Number } from 'form'
+import { Password, Number, Toggle } from 'form'
 import { SelectProxy } from 'select-objects'
 
 const remoteTypes = {
@@ -39,6 +40,8 @@ export default decorate([
       username: undefined,
       directory: undefined,
       bucket: undefined,
+      protocol: undefined,
+      region: undefined,
     }),
     effects: {
       linkState,
@@ -60,6 +63,8 @@ export default decorate([
           proxyId = remote.proxy,
           type = remote.type,
           username = remote.username,
+          protocol = remote.protocol || 'https',
+          region = remote.region,
         } = state
         let { path = remote.path } = state
         if (type === 's3') {
@@ -76,6 +81,8 @@ export default decorate([
             port: port || undefined,
             type,
             username,
+            protocol,
+            region,
           }),
           options: options !== '' ? options : null,
           proxy: proxyId,
@@ -133,6 +140,9 @@ export default decorate([
       setSecretKey(_, { target: { value } }) {
         this.state.password = value
       },
+      setInsecure(_, value) {
+        this.state.protocol = value ? 'http' : 'https'
+      },
     },
     computed: {
       formId: generateId,
@@ -149,6 +159,8 @@ export default decorate([
       name = remote.name || '',
       options = remote.options || '',
       password = remote.password || '',
+      protocol = remote.protocol || 'https',
+      region = remote.region || '',
       parsedPath,
       path = parsedPath || '',
       parsedBucket = parsedPath != null && parsedPath.split('/')[0],
@@ -326,7 +338,12 @@ export default decorate([
           )}
           {type === 's3' && (
             <fieldset className='form-group form-group'>
-              <div className='input-group '>
+              <div className='input-group form-group'>
+                <span className='input-group-addon'>
+                  <Tooltip content={formatMessage(messages.remoteS3TooltipProtocol)}>
+                    <Toggle iconSize={1} onChange={effects.setInsecure} value={protocol === 'http'} />
+                  </Tooltip>
+                </span>
                 <input
                   className='form-control'
                   name='host'
@@ -338,7 +355,18 @@ export default decorate([
                   value={host}
                 />
               </div>
-              <div className='input-group '>
+              <div className='input-group form-group'>
+                <input
+                  className='form-control'
+                  name='region'
+                  onChange={effects.linkState}
+                  pattern='[a-z0-9-]+'
+                  placeholder={formatMessage(messages.remoteS3Region)}
+                  type='text'
+                  value={region}
+                />
+              </div>
+              <div className='input-group form-group'>
                 <input
                   className='form-control'
                   name='bucket'
@@ -363,7 +391,7 @@ export default decorate([
                   value={directory}
                 />
               </div>
-              <div className='input-group'>
+              <div className='input-group form-group'>
                 <input
                   className='form-control'
                   name='username'
@@ -374,14 +402,13 @@ export default decorate([
                   value={username}
                 />
               </div>
-              <div className='input-group'>
+              <div className='input-group form-group'>
                 <input
                   className='form-control'
                   name='password'
                   onChange={effects.setSecretKey}
                   placeholder='Paste secret here to change it'
                   autoComplete='off'
-                  required
                   type='text'
                 />
               </div>