Support object storage in prune script

Also prune original files and user exports

server/core/helpers/logger.ts

@@ -1,11 +1,11 @@
+import { context } from '@opentelemetry/api'
+import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils.js'
+import { omit } from '@peertube/peertube-core-utils'
 import { stat } from 'fs/promises'
 import { join } from 'path'
 import { format as sqlFormat } from 'sql-formatter'
 import { createLogger, format, transports } from 'winston'
 import { FileTransportOptions } from 'winston/lib/winston/transports'
-import { context } from '@opentelemetry/api'
-import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils.js'
-import { omit } from '@peertube/peertube-core-utils'
 import { CONFIG } from '../initializers/config.js'
 import { LOG_FILENAME } from '../initializers/constants.js'
 
@@ -60,7 +60,7 @@ if (CONFIG.LOG.ROTATION.ENABLED) {
 
 function buildLogger (labelSuffix?: string) {
   return createLogger({
-    level: CONFIG.LOG.LEVEL,
+    level: process.env.LOGGER_LEVEL ?? CONFIG.LOG.LEVEL,
     defaultMeta: {
       get traceId () { return getSpanContext(context.active())?.traceId },
       get spanId () { return getSpanContext(context.active())?.spanId },
@@ -154,18 +154,10 @@ async function mtimeSortFilesDesc (files: string[], basePath: string) {
 // ---------------------------------------------------------------------------
 
 export {
-  type LoggerTagsFn,
-  type LoggerTags,
 
-  buildLogger,
-  timestampFormatter,
-  labelFormatter,
-  consoleLoggerFormat,
-  jsonLoggerFormat,
-  mtimeSortFilesDesc,
-  logger,
-  loggerTagsFactory,
-  bunyanLogger
+  buildLogger, bunyanLogger, consoleLoggerFormat,
+  jsonLoggerFormat, labelFormatter, logger,
+  loggerTagsFactory, mtimeSortFilesDesc, timestampFormatter, type LoggerTags, type LoggerTagsFn
 }
 
 // ---------------------------------------------------------------------------

server/core/initializers/constants.ts

@@ -845,6 +845,7 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
 // ---------------------------------------------------------------------------
 
 const USER_EXPORT_MAX_ITEMS = 1000
+const USER_EXPORT_FILE_PREFIX = 'user-export-'
 
 // ---------------------------------------------------------------------------
 
@@ -1268,6 +1269,7 @@ export {
   CONSTRAINTS_FIELDS,
   EMBED_SIZE,
   REDUNDANCY,
+  USER_EXPORT_FILE_PREFIX,
   JOB_CONCURRENCY,
   JOB_ATTEMPTS,
   AP_CLEANER,

server/core/lib/object-storage/object-storage-helpers.ts

@@ -7,9 +7,9 @@ import { createReadStream, createWriteStream } from 'fs'
 import { ensureDir } from 'fs-extra/esm'
 import { dirname } from 'path'
 import { Readable } from 'stream'
-import { getInternalUrl } from '../urls.js'
-import { getClient } from './client.js'
-import { lTags } from './logger.js'
+import { getInternalUrl } from './urls.js'
+import { getClient } from './shared/client.js'
+import { lTags } from './shared/logger.js'
 
 import type { _Object, ObjectCannedACL, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3'
 
@@ -18,7 +18,7 @@ type BucketInfo = {
   PREFIX?: string
 }
 
-async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
+async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo, continuationToken?: string) {
   const s3Client = await getClient()
 
   const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
@@ -26,14 +26,21 @@ async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
   const commandPrefix = bucketInfo.PREFIX + prefix
   const listCommand = new ListObjectsV2Command({
     Bucket: bucketInfo.BUCKET_NAME,
-    Prefix: commandPrefix
+    Prefix: commandPrefix,
+    ContinuationToken: continuationToken
   })
 
   const listedObjects = await s3Client.send(listCommand)
 
   if (isArray(listedObjects.Contents) !== true) return []
 
-  return listedObjects.Contents.map(c => c.Key)
+  let keys = listedObjects.Contents.map(c => c.Key)
+
+  if (listedObjects.IsTruncated) {
+    keys = keys.concat(await listKeysOfPrefix(prefix, bucketInfo, listedObjects.NextContinuationToken))
+  }
+
+  return keys
 }
 
 // ---------------------------------------------------------------------------
@@ -145,7 +152,7 @@ function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
   return removeObjectByFullKey(key, bucketInfo)
 }
 
-async function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) {
+async function removeObjectByFullKey (fullKey: string, bucketInfo: Pick<BucketInfo, 'BUCKET_NAME'>) {
   logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags())
 
   const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')

server/core/lib/object-storage/shared/index.ts

@@ -1,3 +1,3 @@
 export * from './client.js'
 export * from './logger.js'
-export * from './object-storage-helpers.js'
+export * from '../object-storage-helpers.js'

server/core/models/user/user-export.ts

@@ -1,23 +1,25 @@
-import { FindOptions, Op } from 'sequelize'
-import { AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
-import { MUserAccountId, MUserExport } from '@server/types/models/index.js'
-import { UserModel } from './user.js'
-import { getSort } from '../shared/sort.js'
-import { UserExportState, type UserExport, type UserExportStateType, type FileStorageType, FileStorage } from '@peertube/peertube-models'
+import { FileStorage, UserExportState, type FileStorageType, type UserExport, type UserExportStateType } from '@peertube/peertube-models'
 import { logger } from '@server/helpers/logger.js'
-import { remove } from 'fs-extra/esm'
-import { getFSUserExportFilePath } from '@server/lib/paths.js'
+import { CONFIG } from '@server/initializers/config.js'
 import {
   JWT_TOKEN_USER_EXPORT_FILE_LIFETIME,
   STATIC_DOWNLOAD_PATHS,
+  USER_EXPORT_FILE_PREFIX,
   USER_EXPORT_STATES,
   WEBSERVER
 } from '@server/initializers/constants.js'
-import { join } from 'path'
-import jwt from 'jsonwebtoken'
-import { CONFIG } from '@server/initializers/config.js'
 import { removeUserExportObjectStorage } from '@server/lib/object-storage/user-export.js'
+import { getFSUserExportFilePath } from '@server/lib/paths.js'
+import { MUserAccountId, MUserExport } from '@server/types/models/index.js'
+import { remove } from 'fs-extra/esm'
+import jwt from 'jsonwebtoken'
+import { join } from 'path'
+import { FindOptions, Op } from 'sequelize'
+import { AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
+import { doesExist } from '../shared/query.js'
 import { SequelizeModel } from '../shared/sequelize-type.js'
+import { getSort } from '../shared/sort.js'
+import { UserModel } from './user.js'
 
 @Table({
   tableName: 'userExport',
@@ -147,11 +149,20 @@ export class UserExportModel extends SequelizeModel<UserExportModel> {
 
   // ---------------------------------------------------------------------------
 
+  static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
+    const query = 'SELECT 1 FROM "userExport" ' +
+      `WHERE "filename" = $filename AND "storage" = $storage LIMIT 1`
+
+    return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
+  }
+
+  // ---------------------------------------------------------------------------
+
   generateAndSetFilename () {
     if (!this.userId) throw new Error('Cannot generate filename without userId')
     if (!this.createdAt) throw new Error('Cannot generate filename without createdAt')
 
-    this.filename = `user-export-${this.userId}-${this.createdAt.toISOString()}.zip`
+    this.filename = `${USER_EXPORT_FILE_PREFIX}${this.userId}-${this.createdAt.toISOString()}.zip`
   }
 
   canBeSafelyRemoved () {

server/core/models/video/video-file.ts

@@ -289,11 +289,11 @@ export class VideoFileModel extends SequelizeModel<VideoFileModel> {
     return doesExist({ sequelize: this.sequelize, query, bind: { filename } })
   }
 
-  static async doesOwnedWebVideoFileExist (filename: string) {
+  static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
     const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
-                  `WHERE "filename" = $filename AND "storage" = ${FileStorage.FILE_SYSTEM} LIMIT 1`
+      `WHERE "filename" = $filename AND "storage" = $storage LIMIT 1`
 
-    return doesExist({ sequelize: this.sequelize, query, bind: { filename } })
+    return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
   }
 
   static loadByFilename (filename: string) {

server/core/models/video/video-source.ts

@@ -1,12 +1,12 @@
-import type { FileStorageType, VideoSource } from '@peertube/peertube-models'
+import { type FileStorageType, type VideoSource } from '@peertube/peertube-models'
 import { STATIC_DOWNLOAD_PATHS, WEBSERVER } from '@server/initializers/constants.js'
+import { MVideoSource } from '@server/types/models/video/video-source.js'
 import { join } from 'path'
 import { Transaction } from 'sequelize'
 import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
-import { SequelizeModel, getSort } from '../shared/index.js'
+import { SequelizeModel, doesExist, getSort } from '../shared/index.js'
 import { getResolutionLabel } from './formatter/video-api-format.js'
 import { VideoModel } from './video.js'
-import { MVideoSource } from '@server/types/models/video/video-source.js'
 
 @Table({
   tableName: 'videoSource',
@@ -103,6 +103,18 @@ export class VideoSourceModel extends SequelizeModel<VideoSourceModel> {
     })
   }
 
+  // ---------------------------------------------------------------------------
+
+  static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
+    const query = 'SELECT 1 FROM "videoSource" ' +
+      'INNER JOIN "video" ON "video"."id" = "videoSource"."videoId" AND "video"."remote" IS FALSE ' +
+      `WHERE "keptOriginalFilename" = $filename AND "storage" = $storage LIMIT 1`
+
+    return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
+  }
+
+  // ---------------------------------------------------------------------------
+
   getFileDownloadUrl () {
     if (!this.keptOriginalFilename) return null
 

server/core/models/video/video-streaming-playlist.ts

@@ -229,13 +229,13 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
     return Object.assign(playlist, { Video: video })
   }
 
-  static doesOwnedHLSPlaylistExist (videoUUID: string) {
+  static doesOwnedVideoUUIDExist (videoUUID: string, storage: FileStorageType) {
     const query = `SELECT 1 FROM "videoStreamingPlaylist" ` +
       `INNER JOIN "video" ON "video"."id" = "videoStreamingPlaylist"."videoId" ` +
       `AND "video"."remote" IS FALSE AND "video"."uuid" = $videoUUID ` +
-      `AND "storage" = ${FileStorage.FILE_SYSTEM} LIMIT 1`
+      `AND "storage" = $storage LIMIT 1`
 
-    return doesExist({ sequelize: this.sequelize, query, bind: { videoUUID } })
+    return doesExist({ sequelize: this.sequelize, query, bind: { videoUUID, storage } })
   }
 
   assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) {

server/scripts/prune-storage.ts

@@ -1,13 +1,16 @@
+import { uniqify } from '@peertube/peertube-core-utils'
+import { FileStorage, ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models'
+import { DIRECTORIES, USER_EXPORT_FILE_PREFIX } from '@server/initializers/constants.js'
+import { listKeysOfPrefix, removeObjectByFullKey } from '@server/lib/object-storage/object-storage-helpers.js'
+import { UserExportModel } from '@server/models/user/user-export.js'
+import { VideoFileModel } from '@server/models/video/video-file.js'
+import { VideoSourceModel } from '@server/models/video/video-source.js'
+import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
 import Bluebird from 'bluebird'
 import { remove } from 'fs-extra/esm'
 import { readdir, stat } from 'fs/promises'
-import { basename, join } from 'path'
+import { basename, dirname, join } from 'path'
 import prompt from 'prompt'
-import { uniqify } from '@peertube/peertube-core-utils'
-import { ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models'
-import { DIRECTORIES } from '@server/initializers/constants.js'
-import { VideoFileModel } from '@server/models/video/video-file.js'
-import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
 import { getUUIDFromFilename } from '../core/helpers/utils.js'
 import { CONFIG } from '../core/initializers/config.js'
 import { initDatabaseModels } from '../core/initializers/database.js'
@@ -24,141 +27,276 @@ run()
   })
 
 async function run () {
-  const dirs = Object.values(CONFIG.STORAGE)
-
-  if (uniqify(dirs).length !== dirs.length) {
-    console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
-    process.exit(0)
-  }
-
   await initDatabaseModels(true)
 
-  let toDelete: string[] = []
+  await new FSPruner().prune()
 
-  console.log('Detecting files to remove, it could take a while...')
+  console.log('\n')
 
-  toDelete = toDelete.concat(
-    await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PUBLIC, doesWebVideoFileExist()),
-    await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PRIVATE, doesWebVideoFileExist()),
+  await new ObjectStoragePruner().prune()
+}
 
-    await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, doesHLSPlaylistExist()),
-    await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, doesHLSPlaylistExist()),
+// ---------------------------------------------------------------------------
+// Object storage
+// ---------------------------------------------------------------------------
 
-    await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
+class ObjectStoragePruner {
+  private readonly keysToDelete: { bucket: string, key: string }[] = []
 
-    await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
+  async prune () {
+    if (!CONFIG.OBJECT_STORAGE.ENABLED) return
 
-    await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)),
-    await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)),
+    console.log('Pruning object storage.')
 
-    await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES_DIR, doesActorImageExist)
-  )
+    await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.WEB_VIDEOS, this.doesWebVideoFileExistFactory())
+    await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, this.doesStreamingPlaylistFileExistFactory())
+    await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES, this.doesOriginalFileExistFactory())
+    await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.USER_EXPORTS, this.doesUserExportFileExistFactory())
 
-  const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
-  toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
+    if (this.keysToDelete.length === 0) {
+      console.log('No unknown object storage files to delete.')
+      return
+    }
 
-  if (toDelete.length === 0) {
-    console.log('No files to delete.')
-    return
+    const formattedKeysToDelete = this.keysToDelete.map(({ bucket, key }) => ` In bucket ${bucket}: ${key}`).join('\n')
+    console.log(`${this.keysToDelete.length} unknown files from object storage can be deleted:\n${formattedKeysToDelete}\n`)
+
+    const res = await askConfirmation()
+    if (res !== true) {
+      console.log('Exiting without deleting object storage files.')
+      return
+    }
+
+    console.log('Deleting object storage files...\n')
+
+    for (const { bucket, key } of this.keysToDelete) {
+      await removeObjectByFullKey(key, { BUCKET_NAME: bucket })
+    }
+
+    console.log(`${this.keysToDelete.length} object storage files deleted.`)
   }
 
-  console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
+  private async findFilesToDelete (
+    config: { BUCKET_NAME: string, PREFIX?: string },
+    existFun: (file: string) => Promise<boolean> | boolean
+  ) {
+    try {
+      const keys = await listKeysOfPrefix('', config)
 
-  const res = await askConfirmation()
-  if (res === true) {
-    console.log('Processing delete...\n')
+      await Bluebird.map(keys, async key => {
+        if (await existFun(key) !== true) {
+          this.keysToDelete.push({ bucket: config.BUCKET_NAME, key })
+        }
+      }, { concurrency: 20 })
+    } catch (err) {
+      const prefixMessage = config.PREFIX
+        ? ` and prefix ${config.PREFIX}`
+        : ''
 
-    for (const path of toDelete) {
+      console.error('Cannot find files to delete in bucket ' + config.BUCKET_NAME + prefixMessage)
+    }
+  }
+
+  private doesWebVideoFileExistFactory () {
+    return (key: string) => {
+      const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
+
+      return VideoFileModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
+    }
+  }
+
+  private doesStreamingPlaylistFileExistFactory () {
+    return (key: string) => {
+      const uuid = basename(dirname(this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)))
+
+      return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(uuid, FileStorage.OBJECT_STORAGE)
+    }
+  }
+
+  private doesOriginalFileExistFactory () {
+    return (key: string) => {
+      const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES)
+
+      return VideoSourceModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
+    }
+  }
+
+  private doesUserExportFileExistFactory () {
+    return (key: string) => {
+      const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.USER_EXPORTS)
+
+      return UserExportModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
+    }
+  }
+
+  private sanitizeKey (key: string, config: { PREFIX: string }) {
+    return key.replace(new RegExp(`^${config.PREFIX}`), '')
+  }
+}
+
+// ---------------------------------------------------------------------------
+// FS
+// ---------------------------------------------------------------------------
+
+class FSPruner {
+  private pathsToDelete: string[] = []
+
+  async prune () {
+    const dirs = Object.values(CONFIG.STORAGE)
+
+    if (uniqify(dirs).length !== dirs.length) {
+      console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
+      process.exit(0)
+    }
+
+    console.log('Pruning filesystem storage.')
+
+    console.log('Detecting files to remove, it can take a while...')
+
+    await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PUBLIC, this.doesWebVideoFileExistFactory())
+    await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PRIVATE, this.doesWebVideoFileExistFactory())
+
+    await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, this.doesHLSPlaylistExistFactory())
+    await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, this.doesHLSPlaylistExistFactory())
+
+    await this.findFilesToDelete(DIRECTORIES.ORIGINAL_VIDEOS, this.doesOriginalVideoExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.TORRENTS_DIR, this.doesTorrentFileExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.REDUNDANCY_DIR, this.doesRedundancyExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.PREVIEWS_DIR, this.doesThumbnailExistFactory(true, ThumbnailType.PREVIEW))
+    await this.findFilesToDelete(CONFIG.STORAGE.THUMBNAILS_DIR, this.doesThumbnailExistFactory(false, ThumbnailType.MINIATURE))
+
+    await this.findFilesToDelete(CONFIG.STORAGE.ACTOR_IMAGES_DIR, this.doesActorImageExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.TMP_PERSISTENT_DIR, this.doesUserExportExistFactory())
+
+    const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
+    this.pathsToDelete = [ ...this.pathsToDelete, ...tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)) ]
+
+    if (this.pathsToDelete.length === 0) {
+      console.log('No unknown filesystem files to delete.')
+      return
+    }
+
+    const formattedKeysToDelete = this.pathsToDelete.map(p => ` ${p}`).join('\n')
+    console.log(`${this.pathsToDelete.length} unknown files from filesystem can be deleted:\n${formattedKeysToDelete}\n`)
+
+    const res = await askConfirmation()
+    if (res !== true) {
+      console.log('Exiting without deleting filesystem files.')
+      return
+    }
+
+    console.log('Deleting filesystem files...\n')
+
+    for (const path of this.pathsToDelete) {
       await remove(path)
     }
 
-    console.log('Done!')
-  } else {
-    console.log('Exiting without deleting files.')
+    console.log(`${this.pathsToDelete.length} filesystem files deleted.`)
   }
-}
 
-type ExistFun = (file: string) => Promise<boolean> | boolean
-async function pruneDirectory (directory: string, existFun: ExistFun) {
-  const files = await readdir(directory)
+  private async findFilesToDelete (directory: string, existFun: (file: string) => Promise<boolean> | boolean) {
+    const files = await readdir(directory)
 
-  const toDelete: string[] = []
-  await Bluebird.map(files, async file => {
-    const filePath = join(directory, file)
+    await Bluebird.map(files, async file => {
+      const filePath = join(directory, file)
 
-    if (await existFun(filePath) !== true) {
-      toDelete.push(filePath)
+      if (await existFun(filePath) !== true) {
+        this.pathsToDelete.push(filePath)
+      }
+    }, { concurrency: 20 })
+  }
+
+  private doesWebVideoFileExistFactory () {
+    return (filePath: string) => {
+      // Don't delete private directory
+      if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true
+
+      return VideoFileModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
     }
-  }, { concurrency: 20 })
-
-  return toDelete
-}
-
-function doesWebVideoFileExist () {
-  return (filePath: string) => {
-    // Don't delete private directory
-    if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true
-
-    return VideoFileModel.doesOwnedWebVideoFileExist(basename(filePath))
   }
-}
 
-function doesHLSPlaylistExist () {
-  return (hlsPath: string) => {
-    // Don't delete private directory
-    if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true
+  private doesHLSPlaylistExistFactory () {
+    return (hlsPath: string) => {
+      // Don't delete private directory
+      if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true
 
-    return VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath))
-  }
-}
-
-function doesTorrentFileExist () {
-  return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
-}
-
-function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType_Type) {
-  return async (filePath: string) => {
-    const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
-    if (!thumbnail) return false
-
-    if (keepOnlyOwned) {
-      const video = await VideoModel.load(thumbnail.videoId)
-      if (video.isOwned() === false) return false
+      return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(basename(hlsPath), FileStorage.FILE_SYSTEM)
     }
-
-    return true
-  }
-}
-
-async function doesActorImageExist (filePath: string) {
-  const image = await ActorImageModel.loadByName(basename(filePath))
-
-  return !!image
-}
-
-async function doesRedundancyExist (filePath: string) {
-  const isPlaylist = (await stat(filePath)).isDirectory()
-
-  if (isPlaylist) {
-    // Don't delete HLS redundancy directory
-    if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true
-
-    const uuid = getUUIDFromFilename(filePath)
-    const video = await VideoModel.loadWithFiles(uuid)
-    if (!video) return false
-
-    const p = video.getHLSPlaylist()
-    if (!p) return false
-
-    const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
-    return !!redundancy
   }
 
-  const file = await VideoFileModel.loadByFilename(basename(filePath))
-  if (!file) return false
+  private doesOriginalVideoExistFactory () {
+    return (filePath: string) => {
+      return VideoSourceModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
+    }
+  }
 
-  const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
-  return !!redundancy
+  private doesTorrentFileExistFactory () {
+    return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
+  }
+
+  private doesThumbnailExistFactory (keepOnlyOwned: boolean, type: ThumbnailType_Type) {
+    return async (filePath: string) => {
+      const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
+      if (!thumbnail) return false
+
+      if (keepOnlyOwned) {
+        const video = await VideoModel.load(thumbnail.videoId)
+        if (video.isOwned() === false) return false
+      }
+
+      return true
+    }
+  }
+
+  private doesActorImageExistFactory () {
+    return async (filePath: string) => {
+      const image = await ActorImageModel.loadByName(basename(filePath))
+
+      return !!image
+    }
+  }
+
+  private doesRedundancyExistFactory () {
+    return async (filePath: string) => {
+      const isPlaylist = (await stat(filePath)).isDirectory()
+
+      if (isPlaylist) {
+        // Don't delete HLS redundancy directory
+        if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true
+
+        const uuid = getUUIDFromFilename(filePath)
+        const video = await VideoModel.loadWithFiles(uuid)
+        if (!video) return false
+
+        const p = video.getHLSPlaylist()
+        if (!p) return false
+
+        const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
+        return !!redundancy
+      }
+
+      const file = await VideoFileModel.loadByFilename(basename(filePath))
+      if (!file) return false
+
+      const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
+      return !!redundancy
+    }
+  }
+
+  private doesUserExportExistFactory () {
+    return (filePath: string) => {
+      const filename = basename(filePath)
+
+      // Only detect non-existing user export
+      if (!filename.startsWith(USER_EXPORT_FILE_PREFIX)) return true
+
+      return UserExportModel.doesOwnedFileExist(filename, FileStorage.FILE_SYSTEM)
+    }
+  }
 }
 
 async function askConfirmation () {
@@ -169,10 +307,12 @@ async function askConfirmation () {
       properties: {
         confirm: {
           type: 'string',
-          description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
+          description: 'These unknown files can be deleted, but please check your backups first (bugs happen).' +
             ' Notice PeerTube must have been stopped when your ran this script.' +
-            ' Can we delete these files?',
+            ' Can we delete these files? (y/n)',
           default: 'n',
+          validator: /y[es]*|n[o]?/,
+          warning: 'Must respond yes or no',
           required: true
         }
       }