Add user history in import/export

packages/models/src/import-export/peertube-export-format/index.ts

@@ -8,5 +8,6 @@ export * from './followers-export.model.js'
 export * from './following-export.model.js'
 export * from './likes-export.model.js'
 export * from './user-settings-export.model.js'
+export * from './user-video-history-export.js'
 export * from './video-export.model.js'
 export * from './video-playlists-export.model.js'

packages/models/src/import-export/peertube-export-format/user-video-history-export.ts

@@ -0,0 +1,10 @@
+export interface UserVideoHistoryExportJSON {
+  watchedVideos: {
+    videoUrl: string
+    lastTimecode: number
+    createdAt: string
+    updatedAt: string
+  }[]
+
+  archiveFiles?: never
+}

packages/models/src/import-export/user-import-result.model.ts

@@ -16,5 +16,7 @@ export interface UserImportResultSummary {
 
     account: Summary
     userSettings: Summary
+
+    userVideoHistory: Summary
   }
 }

packages/tests/src/api/users/user-export.ts

@@ -24,6 +24,7 @@ import {
   UserExportState,
   UserNotificationSettingValue,
   UserSettingsExportJSON,
+  UserVideoHistoryExportJSON,
   VideoChapterObject,
   VideoCommentObject,
   VideoCreateResult,
@@ -468,6 +469,22 @@ function runTest (withObjectStorage: boolean) {
       }
     }
 
+    {
+      const json = await parseZIPJSONFile<UserVideoHistoryExportJSON>(zip, 'peertube/video-history.json')
+
+      expect(json.watchedVideos).to.have.lengthOf(2)
+
+      expect(json.watchedVideos[0].createdAt).to.exist
+      expect(json.watchedVideos[0].updatedAt).to.exist
+      expect(json.watchedVideos[0].lastTimecode).to.equal(4)
+      expect(json.watchedVideos[0].videoUrl).to.equal(server.url + '/videos/watch/' + noahVideo.uuid)
+
+      expect(json.watchedVideos[1].createdAt).to.exist
+      expect(json.watchedVideos[1].updatedAt).to.exist
+      expect(json.watchedVideos[1].lastTimecode).to.equal(2)
+      expect(json.watchedVideos[1].videoUrl).to.equal(remoteServer.url + '/videos/watch/' + externalVideo.uuid)
+    }
+
     {
       const json = await parseZIPJSONFile<VideoExportJSON>(zip, 'peertube/videos.json')
 

packages/tests/src/api/users/user-import.ts

@@ -330,6 +330,18 @@ function runTest (withObjectStorage: boolean) {
     }
   })
 
+  it('Should have correctly imported user video history', async function () {
+    const { data } = await remoteServer.history.list({ token: remoteNoahToken })
+
+    expect(data).to.have.lengthOf(2)
+
+    expect(data[0].userHistory.currentTime).to.equal(2)
+    expect(data[0].url).to.equal(remoteServer.url + '/videos/watch/' + externalVideo.uuid)
+
+    expect(data[1].userHistory.currentTime).to.equal(4)
+    expect(data[1].url).to.equal(server.url + '/videos/watch/' + noahVideo.uuid)
+  })
+
   it('Should have correctly imported user videos', async function () {
     const { data } = await remoteServer.videos.listMyVideos({ token: remoteNoahToken })
     expect(data).to.have.lengthOf(5)

packages/tests/src/shared/import-export.ts

@@ -311,6 +311,10 @@ export async function prepareImportExportTests (options: {
     token: noahToken
   })
 
+  // Views
+  await server.views.view({ id: noahVideo.uuid, token: noahToken, currentTime: 4 })
+  await server.views.view({ id: externalVideo.uuid, token: noahToken, currentTime: 2 })
+
   return {
     rootId,
 

server/core/initializers/constants.ts

@@ -817,6 +817,10 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
 
 // ---------------------------------------------------------------------------
 
+const USER_EXPORT_MAX_ITEMS = 1000
+
+// ---------------------------------------------------------------------------
+
 // Express static paths (router)
 const STATIC_PATHS = {
   // TODO: deprecated in v6, to remove
@@ -1255,6 +1259,7 @@ export {
   STATIC_MAX_AGE,
   VIEWER_SYNC_REDIS,
   STATIC_PATHS,
+  USER_EXPORT_MAX_ITEMS,
   VIDEO_IMPORT_TIMEOUT,
   VIDEO_PLAYLIST_TYPES,
   MAX_LOGS_OUTPUT_CHARACTERS,

server/core/lib/emails/user-import-completed/html.pug

@@ -44,3 +44,6 @@ block content
     li
       strong Videos:
       +displaySummary(resultStats.videos)
+    li
+      strong Video history:
+      +displaySummary(resultStats.userVideoHistory)

server/core/lib/user-import-export/exporters/dislikes-exporter.ts

@@ -9,7 +9,7 @@ import { activityPubContextify } from '@server/helpers/activity-pub-utils.js'
 export class DislikesExporter extends AbstractUserExporter <DislikesExportJSON> {
 
   async export () {
-    const dislikes = await AccountVideoRateModel.listRatesOfAccountId(this.user.Account.id, 'dislike')
+    const dislikes = await AccountVideoRateModel.listRatesOfAccountIdForExport(this.user.Account.id, 'dislike')
 
     return {
       json: {

server/core/lib/user-import-export/exporters/index.ts

@@ -8,5 +8,6 @@ export * from './following-exporter.js'
 export * from './likes-exporter.js'
 export * from './abstract-user-exporter.js'
 export * from './user-settings-exporter.js'
+export * from './user-video-history-exporter.js'
 export * from './video-playlists-exporter.js'
 export * from './videos-exporter.js'

server/core/lib/user-import-export/exporters/likes-exporter.ts

@@ -9,7 +9,7 @@ import { getContextFilter } from '@server/lib/activitypub/context.js'
 export class LikesExporter extends AbstractUserExporter <LikesExportJSON> {
 
   async export () {
-    const likes = await AccountVideoRateModel.listRatesOfAccountId(this.user.Account.id, 'like')
+    const likes = await AccountVideoRateModel.listRatesOfAccountIdForExport(this.user.Account.id, 'like')
 
     return {
       json: {

server/core/lib/user-import-export/exporters/user-video-history-exporter.ts

@@ -0,0 +1,23 @@
+import { UserVideoHistoryExportJSON } from '@peertube/peertube-models'
+import { AbstractUserExporter } from './abstract-user-exporter.js'
+import { UserVideoHistoryModel } from '@server/models/user/user-video-history.js'
+
+export class UserVideoHistoryExporter extends AbstractUserExporter <UserVideoHistoryExportJSON> {
+
+  async export () {
+    const videos = await UserVideoHistoryModel.listForExport(this.user)
+
+    return {
+      json: {
+        watchedVideos: videos.map(v => ({
+          videoUrl: v.videoUrl,
+          lastTimecode: v.currentTime,
+          createdAt: v.createdAt.toISOString(),
+          updatedAt: v.updatedAt.toISOString()
+        }))
+      } as UserVideoHistoryExportJSON,
+
+      staticFiles: []
+    }
+  }
+}

server/core/lib/user-import-export/exporters/videos-exporter.ts

@@ -28,6 +28,7 @@ import { MVideoSource } from '@server/types/models/video/video-source.js'
 import { VideoSourceModel } from '@server/models/video/video-source.js'
 import { VideoChapterModel } from '@server/models/video/video-chapter.js'
 import { buildChaptersAPHasPart } from '@server/lib/activitypub/video-chapters.js'
+import { USER_EXPORT_MAX_ITEMS } from '@server/initializers/constants.js'
 
 export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
 
@@ -45,7 +46,7 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
     const channels = await VideoChannelModel.listAllByAccount(this.user.Account.id)
 
     for (const channel of channels) {
-      const videoIds = await VideoModel.getAllIdsFromChannel(channel)
+      const videoIds = await VideoModel.getAllIdsFromChannel(channel, USER_EXPORT_MAX_ITEMS)
 
       await Bluebird.map(videoIds, async id => {
         try {

server/core/lib/user-import-export/importers/index.ts

@@ -5,5 +5,6 @@ export * from './dislikes-importer.js'
 export * from './following-importer.js'
 export * from './likes-importer.js'
 export * from './user-settings-importer.js'
+export * from './user-video-history-importer.js'
 export * from './video-playlists-importer.js'
 export * from './videos-importer.js'

server/core/lib/user-import-export/importers/user-video-history-importer.ts

@@ -0,0 +1,41 @@
+import { UserVideoHistoryExportJSON } from '@peertube/peertube-models'
+import { AbstractRatesImporter } from './abstract-rates-importer.js'
+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'
+import { pick } from '@peertube/peertube-core-utils'
+import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js'
+import { UserVideoHistoryModel } from '@server/models/user/user-video-history.js'
+
+type SanitizedObject = Pick<UserVideoHistoryExportJSON['watchedVideos'][0], 'videoUrl' | 'lastTimecode'>
+
+// eslint-disable-next-line max-len
+export class UserVideoHistoryImporter extends AbstractRatesImporter <UserVideoHistoryExportJSON, UserVideoHistoryExportJSON['watchedVideos'][0]> {
+
+  protected getImportObjects (json: UserVideoHistoryExportJSON) {
+    return json.watchedVideos
+  }
+
+  protected sanitize (data: UserVideoHistoryExportJSON['watchedVideos'][0]) {
+    if (!isUrlValid(data.videoUrl)) return undefined
+
+    return pick(data, [ 'videoUrl', 'lastTimecode' ])
+  }
+
+  protected async importObject (data: SanitizedObject) {
+    if (!this.user.videosHistoryEnabled) return { duplicate: false }
+
+    const videoUrl = data.videoUrl
+    const videoImmutable = await loadOrCreateVideoIfAllowedForUser(videoUrl)
+
+    if (!videoImmutable) {
+      throw new Error(`Cannot get or create video ${videoUrl} to import user history`)
+    }
+
+    await UserVideoHistoryModel.upsert({
+      videoId: videoImmutable.id,
+      userId: this.user.id,
+      currentTime: data.lastTimecode
+    })
+
+    return { duplicate: false }
+  }
+}

server/core/lib/user-import-export/user-exporter.ts

@@ -11,7 +11,8 @@ import {
   LikesExporter, AbstractUserExporter,
   UserSettingsExporter,
   VideoPlaylistsExporter,
-  VideosExporter
+  VideosExporter,
+  UserVideoHistoryExporter
 } from './exporters/index.js'
 import { MUserDefault, MUserExport } from '@server/types/models/index.js'
 import archiver, { Archiver } from 'archiver'
@@ -236,6 +237,10 @@ export class UserExporter {
 
           relativeStaticDirPath: '../files/video-playlists'
         })
+      },
+      {
+        jsonFilename: 'video-history.json',
+        exporter: new UserVideoHistoryExporter(options)
       }
     ] as { jsonFilename: string, exporter: AbstractUserExporter<any> }[]
   }

server/core/lib/user-import-export/user-importer.ts

@@ -17,6 +17,7 @@ import { FollowingImporter } from './importers/following-importer.js'
 import { LikesImporter } from './importers/likes-importer.js'
 import { DislikesImporter } from './importers/dislikes-importer.js'
 import { VideoPlaylistsImporter } from './importers/video-playlists-importer.js'
+import { UserVideoHistoryImporter } from './importers/user-video-history-importer.js'
 
 const lTags = loggerTagsFactory('user-import')
 
@@ -34,7 +35,8 @@ export class UserImporter {
         videoPlaylists: this.buildSummary(),
         videos: this.buildSummary(),
         account: this.buildSummary(),
-        userSettings: this.buildSummary()
+        userSettings: this.buildSummary(),
+        userVideoHistory: this.buildSummary()
       }
     }
 
@@ -127,6 +129,10 @@ export class UserImporter {
       {
         name: 'videoPlaylists' as 'videoPlaylists',
         importer: new VideoPlaylistsImporter(this.buildImporterOptions(user, 'video-playlists.json'))
+      },
+      {
+        name: 'userVideoHistory' as 'userVideoHistory',
+        importer: new UserVideoHistoryImporter(this.buildImporterOptions(user, 'video-history.json'))
       }
     ]
   }

server/core/models/account/account-video-rate.ts

@@ -9,7 +9,7 @@ import {
 import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize'
 import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Table, UpdatedAt } from 'sequelize-typescript'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc.js'
-import { CONSTRAINTS_FIELDS, VIDEO_RATE_TYPES } from '../../initializers/constants.js'
+import { CONSTRAINTS_FIELDS, USER_EXPORT_MAX_ITEMS, VIDEO_RATE_TYPES } from '../../initializers/constants.js'
 import { ActorModel } from '../actor/actor.js'
 import { SequelizeModel, getSort, throwIfNotValid } from '../shared/index.js'
 import { SummaryOptions, VideoChannelModel, ScopeNames as VideoChannelScopeNames } from '../video/video-channel.js'
@@ -252,8 +252,8 @@ export class AccountVideoRateModel extends SequelizeModel<AccountVideoRateModel>
     ]).then(([ total, data ]) => ({ total, data }))
   }
 
-  static listRatesOfAccountId (accountId: number, rateType: VideoRateType): Promise<MAccountVideoRateVideoUrl[]> {
-    const query = {
+  static listRatesOfAccountIdForExport (accountId: number, rateType: VideoRateType): Promise<MAccountVideoRateVideoUrl[]> {
+    return AccountVideoRateModel.findAll({
       where: {
         accountId,
         type: rateType
@@ -264,10 +264,9 @@ export class AccountVideoRateModel extends SequelizeModel<AccountVideoRateModel>
           model: VideoModel,
           required: true
         }
-      ]
-    }
-
-    return AccountVideoRateModel.findAll(query)
+      ],
+      limit: USER_EXPORT_MAX_ITEMS
+    })
   }
 
   // ---------------------------------------------------------------------------

server/core/models/actor/actor-follow.ts

@@ -30,7 +30,14 @@ import {
   UpdatedAt
 } from 'sequelize-typescript'
 import { logger } from '../../helpers/logger.js'
-import { ACTOR_FOLLOW_SCORE, CONSTRAINTS_FIELDS, FOLLOW_STATES, SERVER_ACTOR_NAME, SORTABLE_COLUMNS } from '../../initializers/constants.js'
+import {
+  ACTOR_FOLLOW_SCORE,
+  CONSTRAINTS_FIELDS,
+  FOLLOW_STATES,
+  SERVER_ACTOR_NAME,
+  SORTABLE_COLUMNS,
+  USER_EXPORT_MAX_ITEMS
+} from '../../initializers/constants.js'
 import { AccountModel } from '../account/account.js'
 import { ServerModel } from '../server/server.js'
 import { SequelizeModel, buildSQLAttributes, createSafeIn, getSort, searchAttribute, throwIfNotValid } from '../shared/index.js'
@@ -510,8 +517,8 @@ export class ActorFollowModel extends SequelizeModel<ActorFollowModel> {
     }).then(({ data, total }) => ({ total, data: data.map(d => d.selectionUrl) }))
   }
 
-  static listAcceptedFollowersForExport (targetActorId: number) {
-    const query = {
+  static async listAcceptedFollowersForExport (targetActorId: number) {
+    const data = await ActorFollowModel.findAll({
       where: {
         state: 'accepted',
         targetActorId
@@ -530,17 +537,15 @@ export class ActorFollowModel extends SequelizeModel<ActorFollowModel> {
             }
           ]
         }
-      ]
-    }
+      ],
+      limit: USER_EXPORT_MAX_ITEMS
+    })
 
-    return ActorFollowModel.findAll(query)
-      .then(data => {
-        return data.map(f => ({
-          createdAt: f.createdAt,
-          followerHandle: f.ActorFollower.getFullIdentifier(),
-          followerUrl: f.ActorFollower.url
-        }))
-      })
+    return data.map(f => ({
+      createdAt: f.createdAt,
+      followerHandle: f.ActorFollower.getFullIdentifier(),
+      followerUrl: f.ActorFollower.url
+    }))
   }
 
   // ---------------------------------------------------------------------------
@@ -550,8 +555,8 @@ export class ActorFollowModel extends SequelizeModel<ActorFollowModel> {
       .then(({ data, total }) => ({ total, data: data.map(d => d.selectionUrl) }))
   }
 
-  static listAcceptedFollowingForExport (actorId: number) {
-    const query = {
+  static async listAcceptedFollowingForExport (actorId: number) {
+    const data = await ActorFollowModel.findAll({
       where: {
         state: 'accepted',
         actorId
@@ -570,17 +575,15 @@ export class ActorFollowModel extends SequelizeModel<ActorFollowModel> {
             }
           ]
         }
-      ]
-    }
+      ],
+      limit: USER_EXPORT_MAX_ITEMS
+    })
 
-    return ActorFollowModel.findAll(query)
-      .then(data => {
-        return data.map(f => ({
-          createdAt: f.createdAt,
-          followingHandle: f.ActorFollowing.getFullIdentifier(),
-          followingUrl: f.ActorFollowing.url
-        }))
-      })
+    return data.map(f => ({
+      createdAt: f.createdAt,
+      followingHandle: f.ActorFollowing.getFullIdentifier(),
+      followingUrl: f.ActorFollowing.url
+    }))
   }
 
   // ---------------------------------------------------------------------------

server/core/models/user/user-video-history.ts

@@ -5,6 +5,8 @@ import { MUserAccountId, MUserId } from '@server/types/models/index.js'
 import { VideoModel } from '../video/video.js'
 import { UserModel } from './user.js'
 import { SequelizeModel } from '../shared/sequelize-type.js'
+import { USER_EXPORT_MAX_ITEMS } from '@server/initializers/constants.js'
+import { getSort } from '../shared/sort.js'
 
 @Table({
   tableName: 'userVideoHistory',
@@ -71,6 +73,26 @@ export class UserVideoHistoryModel extends SequelizeModel<UserVideoHistoryModel>
     })
   }
 
+  static async listForExport (user: MUserId) {
+    const rows = await UserVideoHistoryModel.findAll({
+      attributes: [ 'createdAt', 'updatedAt', 'currentTime' ],
+      where: {
+        userId: user.id
+      },
+      limit: USER_EXPORT_MAX_ITEMS,
+      include: [
+        {
+          attributes: [ 'url' ],
+          model: VideoModel.unscoped(),
+          required: true
+        }
+      ],
+      order: getSort('updatedAt')
+    })
+
+    return rows.map(r => ({ createdAt: r.createdAt, updatedAt: r.updatedAt, currentTime: r.currentTime, videoUrl: r.Video.url }))
+  }
+
   static removeUserHistoryElement (user: MUserId, videoId: number) {
     const query: DestroyOptions = {
       where: {

server/core/models/video/video-comment.ts

@@ -17,7 +17,7 @@ import { extractMentions } from '@server/helpers/mentions.js'
 import { getServerActor } from '@server/models/application/application.js'
 import { MAccount, MAccountId, MUserAccountId } from '@server/types/models/index.js'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc.js'
-import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { CONSTRAINTS_FIELDS, USER_EXPORT_MAX_ITEMS } from '../../initializers/constants.js'
 import {
   MComment,
   MCommentAdminFormattable,
@@ -456,7 +456,7 @@ export class VideoCommentModel extends SequelizeModel<VideoCommentModel> {
   }
 
   static listForExport (ofAccountId: number): Promise<MCommentExport[]> {
-    const query = {
+    return VideoCommentModel.findAll({
       attributes: [ 'url', 'text', 'createdAt' ],
       where: {
         accountId: ofAccountId,
@@ -474,10 +474,9 @@ export class VideoCommentModel extends SequelizeModel<VideoCommentModel> {
           model: VideoCommentModel,
           as: 'InReplyToVideoComment'
         }
-      ]
-    }
-
-    return VideoCommentModel.findAll(query)
+      ],
+      limit: USER_EXPORT_MAX_ITEMS
+    })
   }
 
   static async getStats () {

server/core/models/video/video-playlist-element.ts

@@ -31,7 +31,7 @@ import {
   MVideoPlaylistElementVideoUrl
 } from '@server/types/models/video/video-playlist-element.js'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc.js'
-import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { CONSTRAINTS_FIELDS, USER_EXPORT_MAX_ITEMS } from '../../initializers/constants.js'
 import { AccountModel } from '../account/account.js'
 import { SequelizeModel, getSort, throwIfNotValid } from '../shared/index.js'
 import { VideoPlaylistModel } from './video-playlist.js'
@@ -258,7 +258,7 @@ export class VideoPlaylistElementModel extends SequelizeModel<VideoPlaylistEleme
   }
 
   static listElementsForExport (videoPlaylistId: number): Promise<MVideoPlaylistElementVideoUrl[]> {
-    const query = {
+    return VideoPlaylistElementModel.findAll({
       where: {
         videoPlaylistId
       },
@@ -269,10 +269,9 @@ export class VideoPlaylistElementModel extends SequelizeModel<VideoPlaylistEleme
           required: true
         }
       ],
-      order: getSort('position')
-    }
-
-    return VideoPlaylistElementModel.findAll(query)
+      order: getSort('position'),
+      limit: USER_EXPORT_MAX_ITEMS
+    })
   }
 
   // ---------------------------------------------------------------------------

server/core/models/video/video-playlist.ts

@@ -39,6 +39,7 @@ import {
   CONSTRAINTS_FIELDS,
   LAZY_STATIC_PATHS,
   THUMBNAILS_SIZE,
+  USER_EXPORT_MAX_ITEMS,
   VIDEO_PLAYLIST_PRIVACIES,
   VIDEO_PLAYLIST_TYPES,
   WEBSERVER
@@ -496,15 +497,14 @@ export class VideoPlaylistModel extends SequelizeModel<VideoPlaylistModel> {
   }
 
   static listPlaylistForExport (accountId: number): Promise<MVideoPlaylistFull[]> {
-    const query = {
-      where: {
-        ownerAccountId: accountId
-      }
-    }
-
     return VideoPlaylistModel
       .scope([ ScopeNames.WITH_ACCOUNT_AND_CHANNEL, ScopeNames.WITH_VIDEOS_LENGTH, ScopeNames.WITH_THUMBNAIL ])
-      .findAll(query)
+      .findAll({
+        where: {
+          ownerAccountId: accountId
+        },
+        limit: USER_EXPORT_MAX_ITEMS
+      })
   }
 
   // ---------------------------------------------------------------------------

server/core/models/video/video.ts

@@ -1593,16 +1593,16 @@ export class VideoModel extends SequelizeModel<VideoModel> {
     return VideoModel.update({ support: ofChannel.support }, options)
   }
 
-  static getAllIdsFromChannel (videoChannel: MChannelId): Promise<number[]> {
-    const query = {
+  static async getAllIdsFromChannel (videoChannel: MChannelId, limit?: number): Promise<number[]> {
+    const videos = await VideoModel.findAll({
       attributes: [ 'id' ],
       where: {
         channelId: videoChannel.id
-      }
-    }
+      },
+      limit
+    })
 
-    return VideoModel.findAll(query)
-                     .then(videos => videos.map(v => v.id))
+    return videos.map(v => v.id)
   }
 
   // threshold corresponds to how many video the field should have to be returned