Generate a name for thumbnails

Allows aggressive caching

scripts/prune-storage.ts

@@ -13,6 +13,7 @@ import { getUUIDFromFilename } from '../server/helpers/utils'
 import { ThumbnailModel } from '../server/models/video/thumbnail'
 import { AvatarModel } from '../server/models/avatar/avatar'
 import { uniq, values } from 'lodash'
+import { ThumbnailType } from '@shared/models'
 
 run()
   .then(() => process.exit(0))
@@ -39,8 +40,8 @@ async function run () {
 
     await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
 
-    await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true)),
-    await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false)),
+    await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)),
+    await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)),
 
     await pruneDirectory(CONFIG.STORAGE.AVATARS_DIR, doesAvatarExist)
   )
@@ -92,9 +93,9 @@ function doesVideoExist (keepOnlyOwned: boolean) {
   }
 }
 
-function doesThumbnailExist (keepOnlyOwned: boolean) {
+function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
   return async (file: string) => {
-    const thumbnail = await ThumbnailModel.loadByName(file)
+    const thumbnail = await ThumbnailModel.loadWithVideoByName(file, type)
     if (!thumbnail) return false
 
     if (keepOnlyOwned) {

server/controllers/lazy-static.ts

@@ -18,7 +18,7 @@ lazyStaticRouter.use(
 )
 
 lazyStaticRouter.use(
-  LAZY_STATIC_PATHS.PREVIEWS + ':uuid.jpg',
+  LAZY_STATIC_PATHS.PREVIEWS + ':filename',
   asyncMiddleware(getPreview)
 )
 
@@ -71,7 +71,7 @@ async function getAvatar (req: express.Request, res: express.Response) {
 }
 
 async function getPreview (req: express.Request, res: express.Response) {
-  const result = await VideosPreviewCache.Instance.getFilePath(req.params.uuid)
+  const result = await VideosPreviewCache.Instance.getFilePath(req.params.filename)
   if (!result) return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
 
   return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER })

server/controllers/static.ts

@@ -1,5 +1,15 @@
 import * as cors from 'cors'
 import * as express from 'express'
+import { join } from 'path'
+import { getRegisteredPlugins, getRegisteredThemes } from '@server/controllers/api/config'
+import { serveIndexHTML } from '@server/lib/client-html'
+import { getTorrentFilePath, getVideoFilePath } from '@server/lib/video-paths'
+import { MVideoFile, MVideoFullLight } from '@server/types/models'
+import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes'
+import { VideoStreamingPlaylistType } from '@shared/models/videos/video-streaming-playlist.type'
+import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo'
+import { root } from '../helpers/core-utils'
+import { CONFIG, isEmailEnabled } from '../initializers/config'
 import {
   CONSTRAINTS_FIELDS,
   DEFAULT_THEME_NAME,
@@ -11,24 +21,13 @@ import {
   STATIC_PATHS,
   WEBSERVER
 } from '../initializers/constants'
-import { cacheRoute } from '../middlewares/cache'
-import { asyncMiddleware, videosDownloadValidator } from '../middlewares'
-import { VideoModel } from '../models/video/video'
-import { UserModel } from '../models/account/user'
-import { VideoCommentModel } from '../models/video/video-comment'
-import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo'
-import { join } from 'path'
-import { root } from '../helpers/core-utils'
-import { getEnabledResolutions } from '../lib/video-transcoding'
-import { CONFIG, isEmailEnabled } from '../initializers/config'
-import { getPreview, getVideoCaption } from './lazy-static'
-import { VideoStreamingPlaylistType } from '@shared/models/videos/video-streaming-playlist.type'
-import { MVideoFile, MVideoFullLight } from '@server/types/models'
-import { getTorrentFilePath, getVideoFilePath } from '@server/lib/video-paths'
 import { getThemeOrDefault } from '../lib/plugins/theme-utils'
-import { getRegisteredPlugins, getRegisteredThemes } from '@server/controllers/api/config'
-import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes'
-import { serveIndexHTML } from '@server/lib/client-html'
+import { getEnabledResolutions } from '../lib/video-transcoding'
+import { asyncMiddleware, videosDownloadValidator } from '../middlewares'
+import { cacheRoute } from '../middlewares/cache'
+import { UserModel } from '../models/account/user'
+import { VideoModel } from '../models/video/video'
+import { VideoCommentModel } from '../models/video/video-comment'
 
 const staticRouter = express.Router()
 

server/initializers/constants.ts

@@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 570
+const LAST_MIGRATION_VERSION = 575
 
 // ---------------------------------------------------------------------------
 

server/initializers/migrations/0575-duplicate-thumbnail.ts

@@ -0,0 +1,24 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+  db: any
+}): Promise<void> {
+  {
+    const query = 'DELETE FROM "thumbnail" s1 ' +
+      'USING (SELECT MIN(id) as id, "filename", "type" FROM "thumbnail" GROUP BY "filename", "type" HAVING COUNT(*) > 1) s2 ' +
+      'WHERE s1."filename" = s2."filename" AND s1."type" = s2."type" AND s1.id <> s2.id'
+    await utils.sequelize.query(query)
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}

server/lib/activitypub/videos.ts

@@ -5,6 +5,7 @@ import { join } from 'path'
 import * as request from 'request'
 import * as sequelize from 'sequelize'
 import { VideoLiveModel } from '@server/models/video/video-live'
+import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 import {
   ActivityHashTagObject,
   ActivityMagnetUrlObject,
@@ -15,7 +16,7 @@ import {
   ActivityUrlObject,
   ActivityVideoUrlObject
 } from '../../../shared/index'
-import { VideoObject } from '../../../shared/models/activitypub/objects'
+import { ActivityIconObject, VideoObject } from '../../../shared/models/activitypub/objects'
 import { VideoPrivacy } from '../../../shared/models/videos'
 import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
@@ -76,7 +77,6 @@ import { sendCreateVideo, sendUpdateVideo } from './send'
 import { addVideoShares, shareVideoByServerAndChannel } from './share'
 import { addVideoComments } from './video-comments'
 import { createRates } from './video-rates'
-import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 
 async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) {
   const video = videoArg as MVideoAP
@@ -360,7 +360,7 @@ async function updateVideoFromAP (options: {
       if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
 
       if (videoUpdated.getPreview()) {
-        const previewUrl = videoUpdated.getPreview().getFileUrl(videoUpdated)
+        const previewUrl = getPreviewUrl(getPreviewFromIcons(videoObject), video)
         const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
         await videoUpdated.addAndSaveThumbnail(previewModel, t)
       }
@@ -597,9 +597,7 @@ async function createVideo (videoObject: VideoObject, channel: MChannelAccountLi
     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
 
     const previewIcon = getPreviewFromIcons(videoObject)
-    const previewUrl = previewIcon
-      ? previewIcon.url
-      : buildRemoteVideoBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
+    const previewUrl = getPreviewUrl(previewIcon, videoCreated)
     const previewModel = createPlaceholderThumbnail(previewUrl, videoCreated, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
 
     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
@@ -822,7 +820,11 @@ function getThumbnailFromIcons (videoObject: VideoObject) {
 function getPreviewFromIcons (videoObject: VideoObject) {
   const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth)
 
-  // FIXME: don't put a fallback here for compatibility with PeerTube <2.2
-
   return maxBy(validIcons, 'width')
 }
+
+function getPreviewUrl (previewIcon: ActivityIconObject, video: MVideoAccountLight) {
+  return previewIcon
+    ? previewIcon.url
+    : buildRemoteVideoBaseUrl(video, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
+}

server/lib/files-cache/videos-preview-cache.ts

@@ -3,6 +3,9 @@ import { FILES_CACHE } from '../../initializers/constants'
 import { VideoModel } from '../../models/video/video'
 import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
 import { doRequestAndSaveToFile } from '@server/helpers/requests'
+import { ThumbnailModel } from '@server/models/video/thumbnail'
+import { ThumbnailType } from '@shared/models'
+import { logger } from '@server/helpers/logger'
 
 class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
 
@@ -16,13 +19,13 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
     return this.instance || (this.instance = new this())
   }
 
-  async getFilePathImpl (videoUUID: string) {
-    const video = await VideoModel.loadByUUID(videoUUID)
-    if (!video) return undefined
+  async getFilePathImpl (filename: string) {
+    const thumbnail = await ThumbnailModel.loadWithVideoByName(filename, ThumbnailType.PREVIEW)
+    if (!thumbnail) return undefined
 
-    if (video.isOwned()) return { isOwned: true, path: video.getPreview().getPath() }
+    if (thumbnail.Video.isOwned()) return { isOwned: true, path: thumbnail.getPath() }
 
-    return this.loadRemoteFile(videoUUID)
+    return this.loadRemoteFile(thumbnail.Video.uuid)
   }
 
   protected async loadRemoteFile (key: string) {
@@ -37,6 +40,8 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
     const remoteUrl = preview.getFileUrl(video)
     await doRequestAndSaveToFile({ uri: remoteUrl }, destPath)
 
+    logger.debug('Fetched remote preview %s to %s.', remoteUrl, destPath)
+
     return { isOwned: false, path: destPath }
   }
 }

server/lib/thumbnail.ts

@@ -27,18 +27,28 @@ function createPlaylistMiniatureFromExisting (
   return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, automaticallyGenerated, existingThumbnail })
 }
 
-function createPlaylistMiniatureFromUrl (fileUrl: string, playlist: MVideoPlaylistThumbnail, size?: ImageSize) {
+function createPlaylistMiniatureFromUrl (downloadUrl: string, playlist: MVideoPlaylistThumbnail, size?: ImageSize) {
   const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size)
   const type = ThumbnailType.MINIATURE
 
-  const thumbnailCreator = () => downloadImage(fileUrl, basePath, filename, { width, height })
+  // Only save the file URL if it is a remote playlist
+  const fileUrl = playlist.isOwned()
+    ? null
+    : downloadUrl
+
+  const thumbnailCreator = () => downloadImage(downloadUrl, basePath, filename, { width, height })
   return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl })
 }
 
-function createVideoMiniatureFromUrl (fileUrl: string, video: MVideoThumbnail, type: ThumbnailType, size?: ImageSize) {
+function createVideoMiniatureFromUrl (downloadUrl: string, video: MVideoThumbnail, type: ThumbnailType, size?: ImageSize) {
   const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
-  const thumbnailCreator = () => downloadImage(fileUrl, basePath, filename, { width, height })
 
+  // Only save the file URL if it is a remote video
+  const fileUrl = video.isOwned()
+    ? null
+    : downloadUrl
+
+  const thumbnailCreator = () => downloadImage(downloadUrl, basePath, filename, { width, height })
   return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl })
 }
 

server/models/video/thumbnail.ts

@@ -1,3 +1,4 @@
+import { remove } from 'fs-extra'
 import { join } from 'path'
 import {
   AfterDestroy,
@@ -12,15 +13,14 @@ import {
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
-import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
+import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
+import { MThumbnailVideo, MVideoAccountLight } from '@server/types/models'
+import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
 import { logger } from '../../helpers/logger'
-import { remove } from 'fs-extra'
 import { CONFIG } from '../../initializers/config'
+import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
 import { VideoModel } from './video'
 import { VideoPlaylistModel } from './video-playlist'
-import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
-import { MVideoAccountLight } from '@server/types/models'
-import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
 
 @Table({
   tableName: 'thumbnail',
@@ -31,6 +31,10 @@ import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
     {
       fields: [ 'videoPlaylistId' ],
       unique: true
+    },
+    {
+      fields: [ 'filename', 'type' ],
+      unique: true
     }
   ]
 })
@@ -114,20 +118,23 @@ export class ThumbnailModel extends Model {
             .catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, err))
   }
 
-  static loadByName (filename: string) {
+  static loadWithVideoByName (filename: string, thumbnailType: ThumbnailType): Promise<MThumbnailVideo> {
     const query = {
       where: {
-        filename
-      }
+        filename,
+        type: thumbnailType
+      },
+      include: [
+        {
+          model: VideoModel.unscoped(),
+          required: true
+        }
+      ]
     }
 
     return ThumbnailModel.findOne(query)
   }
 
-  static generateDefaultPreviewName (videoUUID: string) {
-    return videoUUID + '.jpg'
-  }
-
   getFileUrl (video: MVideoAccountLight) {
     const staticPath = ThumbnailModel.types[this.type].staticPath + this.filename
 

server/models/video/video.ts

@@ -130,6 +130,7 @@ import { VideoShareModel } from './video-share'
 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
 import { VideoTagModel } from './video-tag'
 import { VideoViewModel } from './video-view'
+import { v4 as uuidv4 } from 'uuid'
 
 export enum ScopeNames {
   AVAILABLE_FOR_LIST_IDS = 'AVAILABLE_FOR_LIST_IDS',
@@ -1827,7 +1828,7 @@ export class VideoModel extends Model {
   }
 
   generateThumbnailName () {
-    return this.uuid + '.jpg'
+    return uuidv4() + '.jpg'
   }
 
   getMiniature () {
@@ -1837,7 +1838,7 @@ export class VideoModel extends Model {
   }
 
   generatePreviewName () {
-    return this.uuid + '.jpg'
+    return uuidv4() + '.jpg'
   }
 
   hasPreview () {

server/types/models/video/thumbnail.ts

@@ -1,3 +1,15 @@
+import { PickWith } from '@shared/core-utils'
 import { ThumbnailModel } from '../../../models/video/thumbnail'
+import { MVideo } from './video'
+
+type Use<K extends keyof ThumbnailModel, M> = PickWith<ThumbnailModel, K, M>
+
+// ############################################################################
 
 export type MThumbnail = Omit<ThumbnailModel, 'Video' | 'VideoPlaylist'>
+
+// ############################################################################
+
+export type MThumbnailVideo =
+  MThumbnail &
+  Use<'Video', MVideo>