Implement user import/export in server

server/core/controllers/api/config.ts

@@ -355,6 +355,16 @@ function customConfig (): CustomConfig {
       videoChannelSynchronization: {
         enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED,
         maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER
+      },
+      users: {
+        enabled: CONFIG.IMPORT.USERS.ENABLED
+      }
+    },
+    export: {
+      users: {
+        enabled: CONFIG.EXPORT.USERS.ENABLED,
+        exportExpiration: CONFIG.EXPORT.USERS.EXPORT_EXPIRATION,
+        maxUserVideoQuota: CONFIG.EXPORT.USERS.MAX_USER_VIDEO_QUOTA
       }
     },
     trending: {

server/core/controllers/api/index.ts

@@ -50,7 +50,6 @@ apiRouter.use('/custom-pages', customPageRouter)
 apiRouter.use('/blocklist', blocklistRouter)
 apiRouter.use('/runners', runnersRouter)
 
-// apiRouter.use(apiRateLimiter)
 apiRouter.use('/ping', pong)
 apiRouter.use('/*', badRequest)
 

server/core/controllers/api/runners/jobs-files.ts

@@ -9,7 +9,7 @@ import {
   runnerJobGetVideoStudioTaskFileValidator,
   runnerJobGetVideoTranscodingFileValidator
 } from '@server/middlewares/validators/runners/job-files.js'
-import { RunnerJobState, VideoStorage } from '@peertube/peertube-models'
+import { RunnerJobState, FileStorage } from '@peertube/peertube-models'
 
 const lTags = loggerTagsFactory('api', 'runner')
 
@@ -57,7 +57,7 @@ async function getMaxQualityVideoFile (req: express.Request, res: express.Respon
 
   const file = video.getMaxQualityFile()
 
-  if (file.storage === VideoStorage.OBJECT_STORAGE) {
+  if (file.storage === FileStorage.OBJECT_STORAGE) {
     if (file.isHLS()) {
       return proxifyHLS({
         req,

server/core/controllers/api/search/search-videos.ts

@@ -151,7 +151,7 @@ async function searchVideoURI (url: string, res: express.Response) {
       logger.info('Cannot search remote video %s.', url, { err })
     }
   } else {
-    video = await searchLocalUrl(sanitizeLocalUrl(url), url => VideoModel.loadByUrlAndPopulateAccount(url))
+    video = await searchLocalUrl(sanitizeLocalUrl(url), url => VideoModel.loadByUrlAndPopulateAccountAndFiles(url))
   }
 
   return res.json({

server/core/controllers/api/server/debug.ts

@@ -7,6 +7,7 @@ import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-ch
 import { VideoViewsBufferScheduler } from '@server/lib/schedulers/video-views-buffer-scheduler.js'
 import { VideoViewsManager } from '@server/lib/views/video-views-manager.js'
 import { authenticate, ensureUserHasRight } from '../../../middlewares/index.js'
+import { RemoveExpiredUserExportsScheduler } from '@server/lib/schedulers/remove-expired-user-exports-scheduler.js'
 
 const debugRouter = express.Router()
 
@@ -42,6 +43,7 @@ async function runCommand (req: express.Request, res: express.Response) {
 
   const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
     'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
+    'remove-expired-user-exports': () => RemoveExpiredUserExportsScheduler.Instance.execute(),
     'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
     'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
     'process-update-videos-scheduler': () => UpdateVideosScheduler.Instance.execute(),

server/core/controllers/api/server/server-blocklist.ts

@@ -1,9 +1,7 @@
 import 'multer'
 import express from 'express'
 import { HttpStatusCode, UserRight } from '@peertube/peertube-models'
-import { logger } from '@server/helpers/logger.js'
 import { getServerActor } from '@server/models/application/application.js'
-import { UserNotificationModel } from '@server/models/user/user-notification.js'
 import { getFormattedObjects } from '../../../helpers/utils.js'
 import {
   addAccountInBlocklist,
@@ -105,15 +103,9 @@ async function blockAccount (req: express.Request, res: express.Response) {
   const serverActor = await getServerActor()
   const accountToBlock = res.locals.account
 
-  await addAccountInBlocklist(serverActor.Account.id, accountToBlock.id)
+  await addAccountInBlocklist({ byAccountId: serverActor.Account.id, targetAccountId: accountToBlock.id, removeNotificationOfUserId: null })
 
-  UserNotificationModel.removeNotificationsOf({
-    id: accountToBlock.id,
-    type: 'account',
-    forUserId: null // For all users
-  }).catch(err => logger.error('Cannot remove notifications after an account mute.', { err }))
-
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function unblockAccount (req: express.Request, res: express.Response) {
@@ -121,7 +113,7 @@ async function unblockAccount (req: express.Request, res: express.Response) {
 
   await removeAccountFromBlocklist(accountBlock)
 
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function listBlockedServers (req: express.Request, res: express.Response) {
@@ -142,15 +134,13 @@ async function blockServer (req: express.Request, res: express.Response) {
   const serverActor = await getServerActor()
   const serverToBlock = res.locals.server
 
-  await addServerInBlocklist(serverActor.Account.id, serverToBlock.id)
+  await addServerInBlocklist({
+    byAccountId: serverActor.Account.id,
+    targetServerId: serverToBlock.id,
+    removeNotificationOfUserId: null
+  })
 
-  UserNotificationModel.removeNotificationsOf({
-    id: serverToBlock.id,
-    type: 'server',
-    forUserId: null // For all users
-  }).catch(err => logger.error('Cannot remove notifications after a server mute.', { err }))
-
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function unblockServer (req: express.Request, res: express.Response) {
@@ -158,5 +148,5 @@ async function unblockServer (req: express.Request, res: express.Response) {
 
   await removeServerFromBlocklist(serverBlock)
 
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }

server/core/controllers/api/users/index.ts

@@ -47,6 +47,8 @@ import { mySubscriptionsRouter } from './my-subscriptions.js'
 import { myVideoPlaylistsRouter } from './my-video-playlists.js'
 import { registrationsRouter } from './registrations.js'
 import { twoFactorRouter } from './two-factor.js'
+import { userExportsRouter } from './user-exports.js'
+import { userImportRouter } from './user-imports.js'
 
 const auditLogger = auditLoggerFactory('users')
 
@@ -55,6 +57,8 @@ const usersRouter = express.Router()
 usersRouter.use(apiRateLimiter)
 
 usersRouter.use('/', emailVerificationRouter)
+usersRouter.use('/', userExportsRouter)
+usersRouter.use('/', userImportRouter)
 usersRouter.use('/', registrationsRouter)
 usersRouter.use('/', twoFactorRouter)
 usersRouter.use('/', tokensRouter)

server/core/controllers/api/users/me.ts

@@ -262,11 +262,12 @@ async function updateMyAvatar (req: express.Request, res: express.Response) {
 
   const userAccount = await AccountModel.load(user.Account.id)
 
-  const avatars = await updateLocalActorImageFiles(
-    userAccount,
-    avatarPhysicalFile,
-    ActorImageType.AVATAR
-  )
+  const avatars = await updateLocalActorImageFiles({
+    accountOrChannel: userAccount,
+    imagePhysicalFile: avatarPhysicalFile,
+    type: ActorImageType.AVATAR,
+    sendActorUpdate: true
+  })
 
   return res.json({
     avatars: avatars.map(avatar => avatar.toFormattedJSON())

server/core/controllers/api/users/my-blocklist.ts

@@ -1,8 +1,6 @@
 import 'multer'
 import express from 'express'
 import { HttpStatusCode } from '@peertube/peertube-models'
-import { logger } from '@server/helpers/logger.js'
-import { UserNotificationModel } from '@server/models/user/user-notification.js'
 import { getFormattedObjects } from '../../../helpers/utils.js'
 import {
   addAccountInBlocklist,
@@ -97,15 +95,9 @@ async function blockAccount (req: express.Request, res: express.Response) {
   const user = res.locals.oauth.token.User
   const accountToBlock = res.locals.account
 
-  await addAccountInBlocklist(user.Account.id, accountToBlock.id)
+  await addAccountInBlocklist({ byAccountId: user.Account.id, targetAccountId: accountToBlock.id, removeNotificationOfUserId: user.id })
 
-  UserNotificationModel.removeNotificationsOf({
-    id: accountToBlock.id,
-    type: 'account',
-    forUserId: user.id
-  }).catch(err => logger.error('Cannot remove notifications after an account mute.', { err }))
-
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function unblockAccount (req: express.Request, res: express.Response) {
@@ -134,15 +126,13 @@ async function blockServer (req: express.Request, res: express.Response) {
   const user = res.locals.oauth.token.User
   const serverToBlock = res.locals.server
 
-  await addServerInBlocklist(user.Account.id, serverToBlock.id)
+  await addServerInBlocklist({
+    byAccountId: user.Account.id,
+    targetServerId: serverToBlock.id,
+    removeNotificationOfUserId: user.id
+  })
 
-  UserNotificationModel.removeNotificationsOf({
-    id: serverToBlock.id,
-    type: 'server',
-    forUserId: user.id
-  }).catch(err => logger.error('Cannot remove notifications after a server mute.', { err }))
-
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function unblockServer (req: express.Request, res: express.Response) {
@@ -150,5 +140,5 @@ async function unblockServer (req: express.Request, res: express.Response) {
 
   await removeServerFromBlocklist(serverBlock)
 
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }

server/core/controllers/api/users/my-notifications.ts

@@ -16,7 +16,7 @@ import {
   listUserNotificationsValidator,
   markAsReadUserNotificationsValidator,
   updateNotificationSettingsValidator
-} from '../../../middlewares/validators/user-notifications.js'
+} from '../../../middlewares/validators/users/user-notifications.js'
 import { UserNotificationSettingModel } from '../../../models/user/user-notification-setting.js'
 import { meRouter } from './me.js'
 
@@ -59,12 +59,6 @@ async function updateNotificationSettings (req: express.Request, res: express.Re
   const user = res.locals.oauth.token.User
   const body = req.body as UserNotificationSetting
 
-  const query = {
-    where: {
-      userId: user.id
-    }
-  }
-
   const values: UserNotificationSetting = {
     newVideoFromSubscription: body.newVideoFromSubscription,
     newCommentOnMyVideo: body.newCommentOnMyVideo,
@@ -85,9 +79,9 @@ async function updateNotificationSettings (req: express.Request, res: express.Re
     myVideoStudioEditionFinished: body.myVideoStudioEditionFinished
   }
 
-  await UserNotificationSettingModel.update(values, query)
+  await UserNotificationSettingModel.updateUserSettings(values, user.id)
 
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function listUserNotifications (req: express.Request, res: express.Response) {
@@ -103,7 +97,7 @@ async function markAsReadUserNotifications (req: express.Request, res: express.R
 
   await UserNotificationModel.markAsRead(user.id, req.body.ids)
 
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
 async function markAsReadAllUserNotifications (req: express.Request, res: express.Response) {
@@ -111,5 +105,5 @@ async function markAsReadAllUserNotifications (req: express.Request, res: expres
 
   await UserNotificationModel.markAllAsRead(user.id)
 
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }

server/core/controllers/api/users/user-exports.ts

@@ -0,0 +1,100 @@
+import express from 'express'
+import { FileStorage, HttpStatusCode, UserExportRequest, UserExportRequestResult, UserExportState } from '@peertube/peertube-models'
+import {
+  apiRateLimiter,
+  asyncMiddleware,
+  authenticate,
+  userExportDeleteValidator,
+  userExportRequestValidator,
+  userExportsListValidator
+} from '../../../middlewares/index.js'
+import { UserExportModel } from '@server/models/user/user-export.js'
+import { getFormattedObjects } from '@server/helpers/utils.js'
+import { sequelizeTypescript } from '@server/initializers/database.js'
+import { JobQueue } from '@server/lib/job-queue/job-queue.js'
+import { CONFIG } from '@server/initializers/config.js'
+
+const userExportsRouter = express.Router()
+
+userExportsRouter.use(apiRateLimiter)
+
+userExportsRouter.post('/:userId/exports/request',
+  authenticate,
+  asyncMiddleware(userExportRequestValidator),
+  asyncMiddleware(requestExport)
+)
+
+userExportsRouter.get('/:userId/exports',
+  authenticate,
+  asyncMiddleware(userExportsListValidator),
+  asyncMiddleware(listUserExports)
+)
+
+userExportsRouter.delete('/:userId/exports/:id',
+  authenticate,
+  asyncMiddleware(userExportDeleteValidator),
+  asyncMiddleware(deleteUserExport)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  userExportsRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function requestExport (req: express.Request, res: express.Response) {
+  const body = req.body as UserExportRequest
+
+  const exportModel = new UserExportModel({
+    state: UserExportState.PENDING,
+    withVideoFiles: body.withVideoFiles,
+
+    storage: CONFIG.OBJECT_STORAGE.ENABLED
+      ? FileStorage.OBJECT_STORAGE
+      : FileStorage.FILE_SYSTEM,
+
+    userId: res.locals.user.id,
+    createdAt: new Date()
+  })
+  exportModel.generateAndSetFilename()
+
+  await sequelizeTypescript.transaction(async transaction => {
+    await exportModel.save({ transaction })
+  })
+
+  await JobQueue.Instance.createJob({ type: 'create-user-export', payload: { userExportId: exportModel.id } })
+
+  return res.json({
+    export: {
+      id: exportModel.id
+    }
+  } as UserExportRequestResult)
+}
+
+async function listUserExports (req: express.Request, res: express.Response) {
+  const resultList = await UserExportModel.listForApi({
+    start: req.query.start,
+    count: req.query.count,
+    user: res.locals.user
+  })
+
+  return res.json(getFormattedObjects(resultList.data, resultList.total))
+}
+
+async function deleteUserExport (req: express.Request, res: express.Response) {
+  const userExport = res.locals.userExport
+
+  await sequelizeTypescript.transaction(async transaction => {
+    await userExport.reload({ transaction })
+
+    if (!userExport.canBeSafelyRemoved()) {
+      return res.sendStatus(HttpStatusCode.CONFLICT_409)
+    }
+
+    await userExport.destroy({ transaction })
+  })
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}

server/core/controllers/api/users/user-imports.ts

@@ -0,0 +1,90 @@
+import express from 'express'
+import {
+  apiRateLimiter,
+  asyncMiddleware,
+  authenticate
+} from '../../../middlewares/index.js'
+import { uploadx } from '@server/lib/uploadx.js'
+import {
+  getLatestImportStatusValidator,
+  userImportRequestResumableInitValidator,
+  userImportRequestResumableValidator
+} from '@server/middlewares/validators/users/user-import.js'
+import { HttpStatusCode, UserImportState, UserImportUploadResult } from '@peertube/peertube-models'
+import { logger } from '@server/helpers/logger.js'
+import { UserImportModel } from '@server/models/user/user-import.js'
+import { getFSUserImportFilePath } from '@server/lib/paths.js'
+import { move } from 'fs-extra/esm'
+import { JobQueue } from '@server/lib/job-queue/job-queue.js'
+import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
+
+const userImportRouter = express.Router()
+
+userImportRouter.use(apiRateLimiter)
+
+userImportRouter.post('/:userId/imports/import-resumable',
+  authenticate,
+  asyncMiddleware(userImportRequestResumableInitValidator),
+  (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
+)
+
+userImportRouter.delete('/:userId/imports/import-resumable',
+  authenticate,
+  (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
+)
+
+userImportRouter.put('/:userId/imports/import-resumable',
+  authenticate,
+  uploadx.upload, // uploadx doesn't next() before the file upload completes
+  asyncMiddleware(userImportRequestResumableValidator),
+  asyncMiddleware(addUserImportResumable)
+)
+
+userImportRouter.get('/:userId/imports/latest',
+  authenticate,
+  asyncMiddleware(getLatestImportStatusValidator),
+  asyncMiddleware(getLatestImport)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  userImportRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function addUserImportResumable (req: express.Request, res: express.Response) {
+  const file = res.locals.importUserFileResumable
+  const user = res.locals.user
+
+  // Move import
+  const userImport = new UserImportModel({
+    state: UserImportState.PENDING,
+    userId: user.id,
+    createdAt: new Date()
+  })
+  userImport.generateAndSetFilename()
+
+  await move(file.path, getFSUserImportFilePath(userImport))
+
+  await saveInTransactionWithRetries(userImport)
+
+  // Create job
+  await JobQueue.Instance.createJob({ type: 'import-user-archive', payload: { userImportId: userImport.id } })
+
+  logger.info('User import request job created for user ' + user.username)
+
+  return res.json({
+    userImport: {
+      id: userImport.id
+    }
+  } as UserImportUploadResult)
+}
+
+async function getLatestImport (req: express.Request, res: express.Response) {
+  const userImport = await UserImportModel.loadLatestByUserId(res.locals.user.id)
+  if (!userImport) return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
+
+  return res.json(userImport.toFormattedJSON())
+}

server/core/controllers/api/video-channel.ts

@@ -213,7 +213,12 @@ async function updateVideoChannelBanner (req: express.Request, res: express.Resp
   const videoChannel = res.locals.videoChannel
   const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON())
 
-  const banners = await updateLocalActorImageFiles(videoChannel, bannerPhysicalFile, ActorImageType.BANNER)
+  const banners = await updateLocalActorImageFiles({
+    accountOrChannel: videoChannel,
+    imagePhysicalFile: bannerPhysicalFile,
+    type: ActorImageType.BANNER,
+    sendActorUpdate: true
+  })
 
   auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys)
 
@@ -227,7 +232,13 @@ async function updateVideoChannelAvatar (req: express.Request, res: express.Resp
   const videoChannel = res.locals.videoChannel
   const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON())
 
-  const avatars = await updateLocalActorImageFiles(videoChannel, avatarPhysicalFile, ActorImageType.AVATAR)
+  const avatars = await updateLocalActorImageFiles({
+    accountOrChannel: videoChannel,
+    imagePhysicalFile: avatarPhysicalFile,
+    type: ActorImageType.AVATAR,
+    sendActorUpdate: true
+  })
+
   auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys)
 
   return res.json({

server/core/controllers/api/video-playlist.ts

@@ -192,7 +192,6 @@ async function addVideoPlaylist (req: express.Request, res: express.Response) {
     const videoPlaylistCreated = await videoPlaylist.save({ transaction: t }) as MVideoPlaylistFull
 
     if (thumbnailModel) {
-      thumbnailModel.automaticallyGenerated = false
       await videoPlaylistCreated.setAndSaveThumbnail(thumbnailModel, t)
     }
 

server/core/controllers/api/videos/rate.ts

@@ -1,12 +1,8 @@
 import express from 'express'
 import { HttpStatusCode, UserVideoRateUpdate } from '@peertube/peertube-models'
 import { logger } from '../../../helpers/logger.js'
-import { VIDEO_RATE_TYPES } from '../../../initializers/constants.js'
-import { sequelizeTypescript } from '../../../initializers/database.js'
-import { getLocalRateUrl, sendVideoRateChange } from '../../../lib/activitypub/video-rates.js'
 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoUpdateRateValidator } from '../../../middlewares/index.js'
-import { AccountModel } from '../../../models/account/account.js'
-import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js'
+import { userRateVideo } from '@server/lib/rate.js'
 
 const rateVideoRouter = express.Router()
 
@@ -25,63 +21,16 @@ export {
 // ---------------------------------------------------------------------------
 
 async function rateVideo (req: express.Request, res: express.Response) {
-  const body: UserVideoRateUpdate = req.body
-  const rateType = body.rating
-  const videoInstance = res.locals.videoAll
-  const userAccount = res.locals.oauth.token.User.Account
+  const user = res.locals.oauth.token.User
+  const video = res.locals.videoAll
 
-  await sequelizeTypescript.transaction(async t => {
-    const sequelizeOptions = { transaction: t }
-
-    const accountInstance = await AccountModel.load(userAccount.id, t)
-    const previousRate = await AccountVideoRateModel.load(accountInstance.id, videoInstance.id, t)
-
-    // Same rate, nothing do to
-    if (rateType === 'none' && !previousRate || previousRate?.type === rateType) return
-
-    let likesToIncrement = 0
-    let dislikesToIncrement = 0
-
-    if (rateType === VIDEO_RATE_TYPES.LIKE) likesToIncrement++
-    else if (rateType === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement++
-
-    // There was a previous rate, update it
-    if (previousRate) {
-      // We will remove the previous rate, so we will need to update the video count attribute
-      if (previousRate.type === 'like') likesToIncrement--
-      else if (previousRate.type === 'dislike') dislikesToIncrement--
-
-      if (rateType === 'none') { // Destroy previous rate
-        await previousRate.destroy(sequelizeOptions)
-      } else { // Update previous rate
-        previousRate.type = rateType
-        previousRate.url = getLocalRateUrl(rateType, userAccount.Actor, videoInstance)
-        await previousRate.save(sequelizeOptions)
-      }
-    } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate
-      const query = {
-        accountId: accountInstance.id,
-        videoId: videoInstance.id,
-        type: rateType,
-        url: getLocalRateUrl(rateType, userAccount.Actor, videoInstance)
-      }
-
-      await AccountVideoRateModel.create(query, sequelizeOptions)
-    }
-
-    const incrementQuery = {
-      likes: likesToIncrement,
-      dislikes: dislikesToIncrement
-    }
-
-    await videoInstance.increment(incrementQuery, sequelizeOptions)
-
-    await sendVideoRateChange(accountInstance, videoInstance, likesToIncrement, dislikesToIncrement, t)
-
-    logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name)
+  await userRateVideo({
+    account: user.Account,
+    rateType: (req.body as UserVideoRateUpdate).rating,
+    video
   })
 
-  return res.type('json')
-            .status(HttpStatusCode.NO_CONTENT_204)
-            .end()
+  logger.info('Account video rate for video %s of account %s updated.', video.name, user.username)
+
+  return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }

server/core/controllers/api/videos/source.ts

@@ -5,7 +5,7 @@ import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-q
 import { Hooks } from '@server/lib/plugins/hooks.js'
 import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js'
 import { uploadx } from '@server/lib/uploadx.js'
-import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video.js'
+import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
 import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
 import { buildNewFile } from '@server/lib/video-file.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'

server/core/controllers/api/videos/update.ts

@@ -6,7 +6,7 @@ import { exists } from '@server/helpers/custom-validators/misc.js'
 import { changeVideoChannelShare } from '@server/lib/activitypub/share.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { setVideoPrivacy } from '@server/lib/video-privacy.js'
-import { addVideoJobsAfterUpdate, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js'
+import { buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js'
 import { openapiOperationDoc } from '@server/middlewares/doc.js'
 import { VideoPasswordModel } from '@server/models/video/video-password.js'
 import { FilteredModelAttributes } from '@server/types/index.js'
@@ -23,6 +23,7 @@ import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosU
 import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update.js'
 import { VideoModel } from '../../../models/video/video.js'
 import { replaceChaptersFromDescriptionIfNeeded } from '@server/lib/video-chapters.js'
+import { addVideoJobsAfterUpdate } from '@server/lib/video-jobs.js'
 
 const lTags = loggerTagsFactory('api', 'video')
 const auditLogger = auditLoggerFactory('videos')

server/core/controllers/api/videos/upload.ts

@@ -3,14 +3,10 @@ import { move } from 'fs-extra/esm'
 import { basename } from 'path'
 import { getResumableUploadPath } from '@server/helpers/upload.js'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js'
-import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js'
 import { Redis } from '@server/lib/redis.js'
 import { uploadx } from '@server/lib/uploadx.js'
 import {
-  buildLocalVideoFromReq,
-  buildMoveJob,
-  buildStoryboardJobIfNeeded,
-  buildVideoThumbnailsFromReq,
+  buildLocalVideoFromReq, buildVideoThumbnailsFromReq,
   setVideoTags
 } from '@server/lib/video.js'
 import { buildNewFile } from '@server/lib/video-file.js'
@@ -21,7 +17,7 @@ import { VideoPasswordModel } from '@server/models/video/video-password.js'
 import { VideoSourceModel } from '@server/models/video/video-source.js'
 import { MVideoFile, MVideoFullLight, MVideoThumbnail } from '@server/types/models/index.js'
 import { uuidToShort } from '@peertube/peertube-node-utils'
-import { HttpStatusCode, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@peertube/peertube-models'
+import { HttpStatusCode, ThumbnailType, VideoCreate, VideoPrivacy } from '@peertube/peertube-models'
 import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger.js'
 import { createReqFiles } from '../../../helpers/express-utils.js'
 import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
@@ -43,6 +39,7 @@ import { VideoModel } from '../../../models/video/video.js'
 import { ffprobePromise, getChaptersFromContainer } from '@peertube/peertube-ffmpeg'
 import { replaceChapters, replaceChaptersFromDescriptionIfNeeded } from '@server/lib/video-chapters.js'
 import { FfprobeData } from 'fluent-ffmpeg'
+import { addVideoJobsAfterCreation } from '@server/lib/video-jobs.js'
 
 const lTags = loggerTagsFactory('api', 'video')
 const auditLogger = auditLoggerFactory('videos')
@@ -230,7 +227,7 @@ async function addVideo (options: {
   // Channel has a new content, set as updated
   await videoCreated.VideoChannel.setAsUpdated()
 
-  addVideoJobsAfterUpload(videoCreated, videoFile)
+  addVideoJobsAfterCreation({ video: videoCreated, videoFile })
     .catch(err => logger.error('Cannot build new video jobs of %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
 
   Hooks.runAction('action:api.video.uploaded', { video: videoCreated, req, res })
@@ -244,55 +241,6 @@ async function addVideo (options: {
   }
 }
 
-async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile) {
-  const jobs: (CreateJobArgument & CreateJobOptions)[] = [
-    {
-      type: 'manage-video-torrent' as 'manage-video-torrent',
-      payload: {
-        videoId: video.id,
-        videoFileId: videoFile.id,
-        action: 'create'
-      }
-    },
-
-    buildStoryboardJobIfNeeded({ video, federate: false }),
-
-    {
-      type: 'notify',
-      payload: {
-        action: 'new-video',
-        videoUUID: video.uuid
-      }
-    },
-
-    {
-      type: 'federate-video' as 'federate-video',
-      payload: {
-        videoUUID: video.uuid,
-        isNewVideoForFederation: true
-      }
-    }
-  ]
-
-  if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
-    jobs.push(await buildMoveJob({ video, previousVideoState: undefined, type: 'move-to-object-storage' }))
-  }
-
-  if (video.state === VideoState.TO_TRANSCODE) {
-    jobs.push({
-      type: 'transcoding-job-builder' as 'transcoding-job-builder',
-      payload: {
-        videoUUID: video.uuid,
-        optimizeJob: {
-          isNewVideo: true
-        }
-      }
-    })
-  }
-
-  return JobQueue.Instance.createSequentialJobFlow(...jobs)
-}
-
 async function deleteUploadResumableCache (req: express.Request, res: express.Response, next: express.NextFunction) {
   await Redis.Instance.deleteUploadSession(req.query.upload_id)
 

server/core/controllers/download.ts

@@ -2,14 +2,30 @@ import cors from 'cors'
 import express from 'express'
 import { logger } from '@server/helpers/logger.js'
 import { VideoTorrentsSimpleFileCache } from '@server/lib/files-cache/index.js'
-import { generateHLSFilePresignedUrl, generateWebVideoPresignedUrl } from '@server/lib/object-storage/index.js'
+import {
+  generateHLSFilePresignedUrl,
+  generateUserExportPresignedUrl,
+  generateWebVideoPresignedUrl
+} from '@server/lib/object-storage/index.js'
 import { Hooks } from '@server/lib/plugins/hooks.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
-import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
+import {
+  MStreamingPlaylist,
+  MStreamingPlaylistVideo,
+  MUserExport,
+  MVideo,
+  MVideoFile,
+  MVideoFullLight
+} from '@server/types/models/index.js'
 import { forceNumber } from '@peertube/peertube-core-utils'
-import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models'
+import { HttpStatusCode, FileStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models'
 import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants.js'
-import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares/index.js'
+import {
+  asyncMiddleware, optionalAuthenticate,
+  userExportDownloadValidator,
+  videosDownloadValidator
+} from '../middlewares/index.js'
+import { getFSUserExportFilePath } from '@server/lib/paths.js'
 
 const downloadRouter = express.Router()
 
@@ -34,6 +50,12 @@ downloadRouter.use(
   asyncMiddleware(downloadHLSVideoFile)
 )
 
+downloadRouter.use(
+  STATIC_DOWNLOAD_PATHS.USER_EXPORT + ':filename',
+  asyncMiddleware(userExportDownloadValidator), // Include JWT token authentication
+  asyncMiddleware(downloadUserExport)
+)
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -99,8 +121,8 @@ async function downloadVideoFile (req: express.Request, res: express.Response) {
   const videoName = video.name.replace(/[/\\]/g, '_')
   const downloadFilename = `${videoName}-${videoFile.resolution}p${videoFile.extname}`
 
-  if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
-    return redirectToObjectStorage({ req, res, video, file: videoFile, downloadFilename })
+  if (videoFile.storage === FileStorage.OBJECT_STORAGE) {
+    return redirectVideoDownloadToObjectStorage({ res, video, file: videoFile, downloadFilename })
   }
 
   await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => {
@@ -140,8 +162,8 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
   const videoName = video.name.replace(/\//g, '_')
   const downloadFilename = `${videoName}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}`
 
-  if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
-    return redirectToObjectStorage({ req, res, video, streamingPlaylist, file: videoFile, downloadFilename })
+  if (videoFile.storage === FileStorage.OBJECT_STORAGE) {
+    return redirectVideoDownloadToObjectStorage({ res, video, streamingPlaylist, file: videoFile, downloadFilename })
   }
 
   await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => {
@@ -149,6 +171,21 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
   })
 }
 
+function downloadUserExport (req: express.Request, res: express.Response) {
+  const userExport = res.locals.userExport
+
+  const downloadFilename = userExport.filename
+
+  if (userExport.storage === FileStorage.OBJECT_STORAGE) {
+    return redirectUserExportToObjectStorage({ res, userExport, downloadFilename })
+  }
+
+  res.download(getFSUserExportFilePath(userExport), downloadFilename)
+  return Promise.resolve()
+}
+
+// ---------------------------------------------------------------------------
+
 function getVideoFile (req: express.Request, files: MVideoFile[]) {
   const resolution = forceNumber(req.params.resolution)
   return files.find(f => f.resolution === resolution)
@@ -194,8 +231,7 @@ function checkAllowResult (res: express.Response, allowParameters: any, result?:
   return true
 }
 
-async function redirectToObjectStorage (options: {
-  req: express.Request
+async function redirectVideoDownloadToObjectStorage (options: {
   res: express.Response
   video: MVideo
   file: MVideoFile
@@ -212,3 +248,17 @@ async function redirectToObjectStorage (options: {
 
   return res.redirect(url)
 }
+
+async function redirectUserExportToObjectStorage (options: {
+  res: express.Response
+  downloadFilename: string
+  userExport: MUserExport
+}) {
+  const { res, downloadFilename, userExport } = options
+
+  const url = await generateUserExportPresignedUrl({ userExport, downloadFilename })
+
+  logger.debug('Generating pre-signed URL %s for user export %s', url, userExport.filename)
+
+  return res.redirect(url)
+}