feat(xo-server,xo-web/patching): support new XS Updates system (#7044)

See Zammad#13416

Support for new XenServer Updates system with authentication:
- User downloads Client ID JSON file from XenServer account
- User uploads it to XO in their user preferences
- XO uses `username` and `apikey` from that file to authenticate and download updates

CHANGELOG.unreleased.md

@@ -9,6 +9,7 @@
 
 - [Netbox] Don't delete VMs that have been created manually in XO-synced cluster [Forum#7639](https://xcp-ng.org/forum/topic/7639) (PR [#7008](https://github.com/vatesfr/xen-orchestra/pull/7008))
 - [Kubernetes] *Search domains* field is now optional [#7028](https://github.com/vatesfr/xen-orchestra/pull/7028)
+- [Patches] Support new XenServer Updates system. See [our documentation](https://xen-orchestra.com/docs/updater.html#xenserver-updates). (PR [#7044](https://github.com/vatesfr/xen-orchestra/pull/7044))
 
 ### Bug fixes
 
@@ -41,10 +42,10 @@
 - @xen-orchestra/backups patch
 - vhd-lib minor
 - xo-vmdk-to-vhd patch
-- xo-server patch
+- xo-server minor
 - xo-server-auth-github patch
 - xo-server-auth-google patch
 - xo-server-netbox minor
-- xo-web patch
+- xo-web minor
 
 <!--packages-end-->

docs/updater.md

@@ -125,3 +125,21 @@ If you can't fetch updates, perform a few checks from your XOA:
 - if not, check your `/etc/resolv.conf` file and modify it if necessary (give a correct DNS server)
 - use `ifconfig` to check your network configuration
 - check your firewall(s) and allow XOA to reach xen-orchestra.com (port 443)
+
+## XenServer Updates
+
+Starting September 2023, XenServer Updates require authentication:
+
+1. Make sure your XenServer hosts have [the proper licenses](https://docs.xenserver.com/en-us/citrix-hypervisor/overview-licensing.html)
+
+2. Go to any XenServer Update URL like [this one](https://support.citrix.com/article/CTX277443/hotfix-xs81e006-for-citrix-hypervisor-81) and log in to check that your account has permissions to download updates. You should see a "Download" button.
+
+3. Go to this URL: [https://support.citrix.com/xencenterclientiddownload](https://support.citrix.com/xencenterclientiddownload) and click "Download Client ID"
+
+![Download XenServer Client ID](./assets/xs-client-id-download.png)
+
+4. In Xen Orchestra, go to your User Settings page (bottom left-hand corner) and upload the file `xencenter_client_id.json` you just downloaded from the "XenServer Client ID" section
+
+![Upload XenServer Client ID](./assets/xs-client-id-upload.png)
+
+5. Go to a pool's "Patches" page. You can now install XenServer Updates. If you get a `LICENCE_RESTRICTION` error, it means that [you're missing XenServer licenses on your hosts](https://docs.xenserver.com/en-us/citrix-hypervisor/overview-licensing.html).

packages/xo-server/src/api/pool.mjs

@@ -129,7 +129,7 @@ listMissingPatches.resolve = {
 // -------------------------------------------------------------------
 
 export async function installPatches({ pool, patches, hosts }) {
-  const opts = { patches }
+  const opts = { patches, xsCredentials: this.apiContext.user.preferences.xsCredentials }
   let xapi
   if (pool !== undefined) {
     pool = this.getXapiObject(pool, 'pool')

packages/xo-server/src/xapi/mixins/patching.mjs

@@ -65,7 +65,7 @@ export default {
     return this
   })
   async _getXenUpdates() {
-    const response = await this.xo.httpRequest('https://updates.xensource.com/XenServer/updates.xml')
+    const response = await this.xo.httpRequest('https://updates.ops.xenserver.com/xenserver/updates.xml')
 
     const data = parseXml(await response.buffer()).patchdata
 
@@ -329,7 +329,7 @@ export default {
   },
 
   // Legacy XS patches: upload a patch on a pool before installing it
-  async _legacyUploadPatch(uuid) {
+  async _legacyUploadPatch(uuid, xsCredentials) {
     // check if the patch has already been uploaded
     try {
       return this.getObjectByUuid(uuid)
@@ -342,7 +342,8 @@ export default {
       throw new Error('no such patch ' + uuid)
     }
 
-    let stream = await this.xo.httpRequest(patchInfo.url)
+    const { username, apikey } = xsCredentials
+    let stream = await this.xo.httpRequest(patchInfo.url, { auth: `${username}:${apikey}` })
     stream = await new Promise((resolve, reject) => {
       const PATCH_RE = /\.xsupdate$/
       stream
@@ -367,7 +368,7 @@ export default {
   // ----------
 
   // upload patch on a VDI on a shared SR
-  async _uploadPatch($defer, uuid) {
+  async _uploadPatch($defer, uuid, xsCredentials) {
     log.debug(`downloading patch ${uuid}`)
 
     const patchInfo = (await this._getXenUpdates()).patches[uuid]
@@ -375,7 +376,8 @@ export default {
       throw new Error('no such patch ' + uuid)
     }
 
-    let stream = await this.xo.httpRequest(patchInfo.url)
+    const { username, apikey } = xsCredentials
+    let stream = await this.xo.httpRequest(patchInfo.url, { auth: `${username}:${apikey}` })
     stream = await new Promise((resolve, reject) => {
       stream
         .pipe(unzip.Parse())
@@ -402,13 +404,13 @@ export default {
     return vdi
   },
 
-  _poolWideInstall: deferrable(async function ($defer, patches) {
+  _poolWideInstall: deferrable(async function ($defer, patches, xsCredentials) {
     // Legacy XS patches
     if (!useUpdateSystem(this.pool.$master)) {
       // for each patch: pool_patch.pool_apply
       for (const p of patches) {
         const [patch] = await Promise.all([
-          this._legacyUploadPatch(p.uuid),
+          this._legacyUploadPatch(p.uuid, xsCredentials),
           this._ejectToolsIsos(this.pool.$master.$ref),
         ])
 
@@ -418,15 +420,27 @@ export default {
     }
     // ----------
 
+    // New XS patching system: https://support.citrix.com/article/CTX473972/upcoming-changes-in-xencenter
+    if (xsCredentials?.username === undefined || xsCredentials?.apikey === undefined) {
+      throw new Error('XenServer credentials not found. See https://xen-orchestra.com/docs/updater.html#xenserver-updates')
+    }
+
     // for each patch: pool_update.introduce → pool_update.pool_apply
     for (const p of patches) {
-      const [vdi] = await Promise.all([this._uploadPatch($defer, p.uuid), this._ejectToolsIsos()])
+      const [vdi] = await Promise.all([this._uploadPatch($defer, p.uuid, xsCredentials), this._ejectToolsIsos()])
       if (vdi === undefined) {
         throw new Error('patch could not be uploaded')
       }
 
+      const updateRef = await this.call('pool_update.introduce', vdi.$ref)
+
+      // Checks for license restrictions (and other conditions?)
+      await Promise.all(filter(this.objects.all, { $type: 'host' }).map(host =>
+        this.call('pool_update.precheck', updateRef, host.$ref)
+      ))
+
       log.debug(`installing patch ${p.uuid}`)
-      await this.call('pool_update.pool_apply', await this.call('pool_update.introduce', vdi.$ref))
+      await this.call('pool_update.pool_apply', updateRef)
     }
   }),
 
@@ -449,7 +463,7 @@ export default {
   //
   // XS pool-wide optimization only works when no hosts are specified
   // it may install more patches that specified if some of them require other patches
-  async installPatches({ patches, hosts } = {}) {
+  async installPatches({ patches, hosts, xsCredentials } = {}) {
     // XCP
     if (_isXcp(this.pool.$master)) {
       return this._xcpUpdate(hosts)
@@ -467,7 +481,7 @@ export default {
         installablePatches.map(patch => patch.uuid)
       )
 
-      return this._poolWideInstall(installablePatches)
+      return this._poolWideInstall(installablePatches, xsCredentials)
     }
 
     // for each host

packages/xo-web/src/common/intl/messages.js

@@ -37,6 +37,7 @@ const messages = {
   errorUnknownItem: 'Unknown {type}',
   generateNewMacAddress: 'Generate new MAC addresses',
   memoryFree: '{memoryFree} RAM free',
+  configured: 'Configured',
   notConfigured: 'Not configured',
   utcDate: 'UTC date',
   utcTime: 'UTC time',
@@ -132,6 +133,7 @@ const messages = {
   deleteCustomField: 'Delete custom field',
   onlyAvailableXoaUsers: 'Only available to XOA users',
   xcpNg: 'XCP-ng',
+  noFileSelected: 'No file selected',
 
   // ----- Modals -----
   alertOk: 'OK',
@@ -248,6 +250,14 @@ const messages = {
 
   // ----- User Profile -----
   editUserProfile: 'Edit my settings {username}',
+  xsClientId: 'XenServer Client ID',
+  uploadClientId: 'Upload Client ID file',
+  forgetClientId: 'Forget Client ID',
+  forgetXsCredentialsConfirm: 'Are you sure you want to forget your XenServer Client ID?',
+  forgetXsCredentialsError: 'Could not forget Client ID',
+  forgetXsCredentialsSuccess: 'Client ID forgotten',
+  setXsCredentialsError: 'Could not upload Client ID',
+  setXsCredentialsSuccess: 'Client ID uploaded',
 
   // ----- Home view ------
   allVms: 'All VMs',
@@ -1101,6 +1111,9 @@ const messages = {
   vmsHaveCds: '{nVms, number} VM{nVms, plural, one {} other {s}} {nVms, plural, one {has} other {have}} CDs',
   ejectCds: 'Eject CDs',
   highAvailabilityNotDisabledTooltip: 'High Availability must be disabled',
+  xsCredentialsMissing:
+    'In order to install XenServer updates, you first need to configure your XenServer Client ID. See {link}.',
+  xsCredentialsMissingShort: 'Missing XenServer Client ID',
 
   // ----- Pool storage tabs -----
   defaultSr: 'Default SR',

packages/xo-web/src/common/select-files.js

@@ -5,6 +5,10 @@ import PropTypes from 'prop-types'
 import React from 'react'
 import omit from 'lodash/omit.js'
 
+const STYLE = {
+  marginBottom: 0
+}
+
 export default class SelectFiles extends Component {
   static propTypes = {
     multi: PropTypes.bool,
@@ -21,7 +25,7 @@ export default class SelectFiles extends Component {
 
   render() {
     return (
-      <label className='btn btn-secondary btn-file hidden'>
+      <label className='btn btn-secondary btn-file hidden' style={STYLE}>
         <Icon icon='file' /> {this.props.label || _('browseFiles')}
         <input
           {...omit(this.props, ['hidden', 'label', 'onChange', 'multi'])}

packages/xo-web/src/common/xo/index.js

@@ -3131,6 +3131,11 @@ export const editAuthToken = ({ description, id }) =>
     id,
   })::tap(subscribeUserAuthTokens.forceRefresh)
 
+export const editXsCredentials = xsCredentials =>
+  _setUserPreferences({
+    xsCredentials,
+  })
+
 // User filters --------------------------------------------------
 
 import AddUserFilterModalBody from './add-user-filter-modal' // eslint-disable-line import/first

packages/xo-web/src/xo-app/pool/tab-patches.js

@@ -9,7 +9,13 @@ import { Col, Container, Row } from 'grid'
 import { createGetObjectsOfType } from 'selectors'
 import { FormattedRelative, FormattedTime } from 'react-intl'
 import { getXoaPlan, ENTERPRISE } from 'xoa-plans'
-import { installAllPatchesOnPool, installPatches, rollingPoolUpdate, subscribeHostMissingPatches } from 'xo'
+import {
+  installAllPatchesOnPool,
+  installPatches,
+  rollingPoolUpdate,
+  subscribeCurrentUser,
+  subscribeHostMissingPatches,
+} from 'xo'
 import isEmpty from 'lodash/isEmpty.js'
 
 const ROLLING_POOL_UPDATES_AVAILABLE = getXoaPlan().value >= ENTERPRISE.value
@@ -48,7 +54,7 @@ const MISSING_PATCH_COLUMNS = [
 
 const ACTIONS = [
   {
-    disabled: (_, { pool }) => pool.HA_enabled,
+    disabled: (_, { pool, needsCredentials }) => pool.HA_enabled || needsCredentials,
     handler: (patches, { pool }) => installPatches(patches, pool),
     icon: 'host-patch-update',
     label: _('install'),
@@ -156,6 +162,7 @@ const INSTALLED_PATCH_COLUMNS = [
 
 @addSubscriptions(({ master }) => ({
   missingPatches: cb => subscribeHostMissingPatches(master, cb),
+  userPreferences: cb => subscribeCurrentUser(user => cb(user.preferences)),
 }))
 @connectStore({
   hostPatches: createGetObjectsOfType('patch').pick((_, { master }) => master.patches),
@@ -164,11 +171,14 @@ export default class TabPatches extends Component {
   render() {
     const {
       hostPatches,
+      master: { productBrand },
       missingPatches = [],
       pool,
-      master: { productBrand },
+      userPreferences,
     } = this.props
 
+    const needsCredentials = productBrand !== 'XCP-ng' && userPreferences.xsCredentials === undefined
+
     return (
       <Upgrade place='poolPatches' required={2}>
         <Container>
@@ -187,11 +197,17 @@ export default class TabPatches extends Component {
               <TabButton
                 btnStyle='primary'
                 data-pool={pool}
-                disabled={isEmpty(missingPatches) || pool.HA_enabled}
+                disabled={isEmpty(missingPatches) || pool.HA_enabled || needsCredentials}
                 handler={installAllPatchesOnPool}
                 icon='host-patch-update'
                 labelId='installPoolPatches'
-                tooltip={pool.HA_enabled ? _('highAvailabilityNotDisabledTooltip') : undefined}
+                tooltip={
+                  pool.HA_enabled
+                    ? _('highAvailabilityNotDisabledTooltip')
+                    : needsCredentials
+                    ? _('xsCredentialsMissingShort')
+                    : undefined
+                }
               />
             </Col>
           </Row>
@@ -212,11 +228,27 @@ export default class TabPatches extends Component {
               <Row>
                 <Col>
                   <h3>{_('hostMissingPatches')}</h3>
+                  {needsCredentials && (
+                    <div className='alert alert-danger'>
+                      {_('xsCredentialsMissing', {
+                        link: (
+                          <a
+                            href='https://xen-orchestra.com/docs/updater.html#xenserver-updates'
+                            target='_blank'
+                            rel='noreferrer'
+                          >
+                            https://xen-orchestra.com/docs/updater.html
+                          </a>
+                        ),
+                      })}
+                    </div>
+                  )}
                   <SortedTable
                     actions={ACTIONS}
                     collection={missingPatches}
                     columns={MISSING_PATCH_COLUMNS}
                     data-pool={pool}
+                    data-needsCredentials={needsCredentials}
                     stateUrlParam='s_missing'
                   />
                 </Col>

packages/xo-web/src/xo-app/user/index.js

@@ -7,11 +7,13 @@ import homeFilters from 'home-filters'
 import Icon from 'icon'
 import PropTypes from 'prop-types'
 import React from 'react'
+import SelectFiles from 'select-files'
 import SortedTable from 'sorted-table'
 import Tooltip from 'tooltip'
 import { Text } from 'editable'
-import { alert } from 'modal'
+import { alert, confirm } from 'modal'
 import { Container, Row, Col } from 'grid'
+import { error, success } from 'notification'
 import { getLang } from 'selectors'
 import { isEmpty, map } from 'lodash'
 import { injectIntl } from 'react-intl'
@@ -27,6 +29,7 @@ import {
   deleteSshKey,
   deleteSshKeys,
   editAuthToken,
+  editXsCredentials,
   editCustomFilter,
   removeCustomFilter,
   setDefaultHomeFilter,
@@ -76,6 +79,81 @@ const getUserPreferences = user => user.preferences || {}
 
 // ===================================================================
 
+@addSubscriptions({
+  user: subscribeCurrentUser,
+})
+class XsClientId extends Component {
+  async editXsCredentials(file) {
+    if (file === undefined) {
+      error(_('noFileSelected'))
+      return
+    }
+
+    try {
+      await new Promise((resolve, reject) => {
+        const fr = new window.FileReader()
+        fr.onload = event => {
+          try {
+            const { username, apikey } = JSON.parse(event.target.result)
+            if (username === undefined || apikey === undefined) {
+              reject(new Error('Could not find username and apikey in file'))
+            }
+
+            editXsCredentials({ username, apikey }).then(resolve, reject)
+          } catch (err) {
+            reject(err)
+          }
+        }
+        fr.readAsText(file)
+      })
+      success(_('setXsCredentialsSuccess'))
+    } catch (err) {
+      error(_('setXsCredentialsError'), err.message)
+    }
+  }
+
+  async deleteXsCredentials() {
+    await confirm({
+      icon: 'delete',
+      title: _('forgetClientId'),
+      body: _('forgetXsCredentialsConfirm'),
+    })
+    try {
+      await editXsCredentials(null)
+      success(_('forgetXsCredentialsSuccess'))
+    } catch (err) {
+      error('forgetXsCredentialsError', err.message)
+    }
+  }
+
+  render() {
+    const isConfigured = this.props.user?.preferences?.xsCredentials !== undefined
+    return (
+      <Container>
+        <Row>
+          <Col smallSize={2}>
+            <strong>{_('xsClientId')}</strong>{' '}
+            <a href='https://xen-orchestra.com/docs/updater.html#xenserver-updates' target='_blank' rel='noreferrer'>
+              <Icon icon='info' />
+            </a>
+          </Col>
+          <Col smallSize={10}>
+            <span className='mr-1'>{isConfigured ? _('configured') : _('notConfigured')}</span>
+            <SelectFiles onChange={this.editXsCredentials} label={_('uploadClientId')} />{' '}
+            {isConfigured && (
+              <ActionButton btnStyle='danger' handler={this.deleteXsCredentials} icon='delete'>
+                {_('forgetClientId')}
+              </ActionButton>
+            )}
+          </Col>
+        </Row>
+      </Container>
+    )
+  }
+}
+
+// ===================================================================
+
 class DefaultFilterPicker extends Component {
   static propTypes = {
     customFilters: PropTypes.object,
@@ -494,6 +572,8 @@ export default class User extends Component {
           <Otp user={user} key='otp' />,
           <hr key='hr' />,
         ]}
+        <XsClientId user={user} />
+        <hr />
         <SshKeys />
         <hr />
         <UserAuthTokens />