Improve real world script

client/src/app/videos/shared/video.model.ts

@@ -5,6 +5,7 @@ export class Video implements VideoServerModel {
   author: string
   by: string
   createdAt: Date
+  updatedAt: Date
   categoryLabel: string
   category: number
   licenceLabel: string

server/controllers/api/remote/videos.ts

@@ -17,7 +17,7 @@ import {
 } from '../../../middlewares'
 import { logger, retryTransactionWrapper } from '../../../helpers'
 import { quickAndDirtyUpdatesVideoToFriends } from '../../../lib'
-import { PodInstance } from '../../../models'
+import { PodInstance, VideoFileInstance } from '../../../models'
 import {
   RemoteVideoRequest,
   RemoteVideoCreateData,
@@ -81,7 +81,7 @@ function remoteVideos (req: express.Request, res: express.Response, next: expres
     // Get the function we need to call in order to process the request
     const fun = functionsHash[request.type]
     if (fun === undefined) {
-      logger.error('Unkown remote request type %s.', request.type)
+      logger.error('Unknown remote request type %s.', request.type)
       return
     }
 
@@ -176,7 +176,7 @@ function processVideosEvents (eventData: RemoteVideoEventData, fromPod: PodInsta
         return quickAndDirtyUpdatesVideoToFriends(qadusParams, t)
       })
   })
-  .then(() => logger.info('Remote video event processed for video %s.', eventData.uuid))
+  .then(() => logger.info('Remote video event processed for video with uuid %s.', eventData.uuid))
   .catch(err => {
     logger.debug('Cannot process a video event.', err)
     throw err
@@ -193,14 +193,14 @@ function quickAndDirtyUpdateVideoRetryWrapper (videoData: RemoteQaduVideoData, f
 }
 
 function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) {
-  let videoName
+  let videoUUID = ''
 
   return db.sequelize.transaction(t => {
     return fetchVideoByHostAndUUID(fromPod.host, videoData.uuid)
       .then(videoInstance => {
         const options = { transaction: t }
 
-        videoName = videoInstance.name
+        videoUUID = videoInstance.uuid
 
         if (videoData.views) {
           videoInstance.set('views', videoData.views)
@@ -217,7 +217,7 @@ function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodI
         return videoInstance.save(options)
       })
   })
-  .then(() => logger.info('Remote video %s quick and dirty updated', videoName))
+  .then(() => logger.info('Remote video with uuid %s quick and dirty updated', videoUUID))
   .catch(err => logger.debug('Cannot quick and dirty update the remote video.', err))
 }
 
@@ -315,7 +315,7 @@ function addRemoteVideo (videoToCreateData: RemoteVideoCreateData, fromPod: PodI
         return videoCreated.setTags(tagInstances, options)
       })
   })
-  .then(() => logger.info('Remote video %s inserted.', videoToCreateData.name))
+  .then(() => logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid))
   .catch(err => {
     logger.debug('Cannot insert the remote video.', err)
     throw err
@@ -361,7 +361,17 @@ function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, from
         return videoInstance.save(options).then(() => ({ videoInstance, tagInstances }))
       })
       .then(({ tagInstances, videoInstance }) => {
-        const tasks = []
+        const tasks: Promise<void>[] = []
+
+        // Remove old video files
+        videoInstance.VideoFiles.forEach(videoFile => {
+          tasks.push(videoFile.destroy())
+        })
+
+        return Promise.all(tasks).then(() => ({ tagInstances, videoInstance }))
+      })
+      .then(({ tagInstances, videoInstance }) => {
+        const tasks: Promise<VideoFileInstance>[] = []
         const options = {
           transaction: t
         }
@@ -386,7 +396,7 @@ function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, from
         return videoInstance.setTags(tagInstances, options)
       })
   })
-  .then(() => logger.info('Remote video %s updated', videoAttributesToUpdate.name))
+  .then(() => logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid))
   .catch(err => {
     // This is just a debug because we will retry the insert
     logger.debug('Cannot update the remote video.', err)
@@ -398,7 +408,7 @@ function removeRemoteVideo (videoToRemoveData: RemoteVideoRemoveData, fromPod: P
   // We need the instance because we have to remove some other stuffs (thumbnail etc)
   return fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid)
     .then(video => {
-      logger.debug('Removing remote video %s.', video.uuid)
+      logger.debug('Removing remote video with uuid %s.', video.uuid)
       return video.destroy()
     })
     .catch(err => {

server/controllers/api/videos/index.ts

@@ -157,6 +157,7 @@ function addVideoRetryWrapper (req: express.Request, res: express.Response, next
 
 function addVideo (req: express.Request, res: express.Response, videoPhysicalFile: Express.Multer.File) {
   const videoInfo: VideoCreate = req.body
+  let videoUUID = ''
 
   return db.sequelize.transaction(t => {
     const user = res.locals.oauth.token.User
@@ -241,6 +242,7 @@ function addVideo (req: express.Request, res: express.Response, videoPhysicalFil
           .then(videoCreated => {
             // Do not forget to add Author information to the created video
             videoCreated.Author = author
+            videoUUID = videoCreated.uuid
 
             return { tagInstances, video: videoCreated, videoFile }
           })
@@ -274,7 +276,7 @@ function addVideo (req: express.Request, res: express.Response, videoPhysicalFil
           })
       })
   })
-  .then(() => logger.info('Video with name %s created.', videoInfo.name))
+  .then(() => logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoUUID))
   .catch((err: Error) => {
     logger.debug('Cannot insert the video.', err)
     throw err
@@ -342,7 +344,7 @@ function updateVideo (req: express.Request, res: express.Response) {
       })
   })
   .then(() => {
-    logger.info('Video with name %s updated.', videoInstance.name)
+    logger.info('Video with name %s and uuid %s updated.', videoInstance.name, videoInstance.uuid)
   })
   .catch(err => {
     logger.debug('Cannot update the video.', err)
@@ -398,7 +400,10 @@ function removeVideo (req: express.Request, res: express.Response, next: express
   const videoInstance = res.locals.video
 
   videoInstance.destroy()
-    .then(() => res.type('json').status(204).end())
+    .then(() => {
+      logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid)
+      res.type('json').status(204).end()
+    })
     .catch(err => {
       logger.error('Errors when removed the video.', err)
       return next(err)

server/middlewares/validators/videos.ts

@@ -109,8 +109,6 @@ function videosRemoveValidator (req: express.Request, res: express.Response, nex
 
   checkErrors(req, res, () => {
     checkVideoExists(req.params.id, res, () => {
-      // We need to make additional checks
-
       // Check if the user who did the request is able to delete the video
       checkUserCanDeleteVideo(res.locals.oauth.token.User.id, res, () => {
         next()
@@ -205,17 +203,15 @@ function checkUserCanDeleteVideo (userId: number, res: express.Response, callbac
   // Retrieve the user who did the request
   db.User.loadById(userId)
     .then(user => {
+      if (res.locals.video.isOwned() === false) {
+        return res.status(403).send('Cannot remove video of another pod, blacklist it')
+      }
+
       // Check if the user can delete the video
       // The user can delete it if s/he is an admin
       // Or if s/he is the video's author
-      if (user.isAdmin() === false) {
+      if (user.isAdmin() === false && res.locals.video.Author.userId !== res.locals.oauth.token.User.id) {
-        if (res.locals.video.isOwned() === false) {
+        return res.status(403).send('Cannot remove video of another user')
-          return res.status(403).send('Cannot remove video of another pod')
-        }
-
-        if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) {
-          return res.status(403).send('Cannot remove video of another user')
-        }
       }
 
       // If we reach this comment, we can delete the video

server/tests/real-world/real-world.ts

@@ -1,11 +1,10 @@
 import * as program from 'commander'
-import { isEqual, differenceWith } from 'lodash'
 
 // /!\ Before imports /!\
 process.env.NODE_ENV = 'test'
 
 import { REQUESTS_INTERVAL } from '../../initializers/constants'
-import { Video, VideoRateType } from '../../../shared'
+import { Video, VideoRateType, VideoFile } from '../../../shared'
 import {
   ServerInfo as DefaultServerInfo,
   flushAndRunMultipleServers,
@@ -121,12 +120,14 @@ async function start () {
     checking = true
 
     const waitingInterval = setInterval(async () => {
-      const awaitingRequests = await isThereAwaitingRequests(servers)
+      const pendingRequests = await isTherePendingRequests(servers)
-      if (awaitingRequests === true) {
+      if (pendingRequests === true) {
-        console.log('A server has awaiting requests, waiting...')
+        console.log('A server has pending requests, waiting...')
         return
       }
 
+      // Even if there are no pending request, wait some potential processes
+      await wait(2000)
       await checkIntegrity(servers)
 
       initializeRequestsPerServer(servers)
@@ -212,7 +213,7 @@ async function update (servers: ServerInfo[], numServer: number) {
 
 async function remove (servers: ServerInfo[], numServer: number) {
   const res = await getVideosList(servers[numServer].url)
-  const videos = res.body.data
+  const videos = res.body.data.filter(video => video.isLocal === true)
   if (videos.length === 0) return undefined
 
   const toRemove = videos[getRandomInt(0, videos.length)].id
@@ -259,19 +260,7 @@ async function checkIntegrity (servers: ServerInfo[]) {
 
   // Fetch all videos and remove some fields that can differ between pods
   for (const server of servers) {
-    const p = getAllVideosListBy(server.url).then(res => {
+    const p = getAllVideosListBy(server.url).then(res => videos.push(res.body.data))
-      const serverVideos = res.body.data
-      for (const serverVideo of serverVideos) {
-        delete serverVideo.id
-        delete serverVideo.isLocal
-        delete serverVideo.thumbnailPath
-        delete serverVideo.updatedAt
-        delete serverVideo.views
-      }
-
-      videos.push(serverVideos)
-    })
-
     tasks.push(p)
   }
 
@@ -279,12 +268,12 @@ async function checkIntegrity (servers: ServerInfo[]) {
 
   let i = 0
   for (const video of videos) {
-    if (!isEqual(video, videos[0])) {
+    const differences = areDifferences(video, videos[0])
+    if (differences !== undefined) {
       console.error('Integrity not ok with server %d!', i + 1)
 
       if (displayDiffOnFail) {
-        console.log(differenceWith(videos[0], video, isEqual))
+        console.log(differences)
-        console.log(differenceWith(video, videos[0], isEqual))
       }
 
       process.exit(-1)
@@ -296,15 +285,74 @@ async function checkIntegrity (servers: ServerInfo[]) {
   console.log('Integrity ok.')
 }
 
+function areDifferences (videos1: Video[], videos2: Video[]) {
+  // Remove some keys we don't want to compare
+  videos1.concat(videos2).forEach(video => {
+    delete video.id
+    delete video.isLocal
+    delete video.thumbnailPath
+    delete video.updatedAt
+    delete video.views
+  })
+
+  if (videos1.length !== videos2.length) {
+    return `Videos length are different (${videos1.length}/${videos2.length}).`
+  }
+
+  for (const video1 of videos1) {
+    const video2 = videos2.find(video => video.uuid === video1.uuid)
+
+    if (!video2) return 'Video ' + video1.uuid + ' is missing.'
+
+    for (const videoKey of Object.keys(video1)) {
+      const attribute1 = video1[videoKey]
+      const attribute2 = video2[videoKey]
+
+      if (videoKey === 'tags') {
+        if (attribute1.length !== attribute2.length) {
+          return 'Tags are different.'
+        }
+
+        attribute1.forEach(tag1 => {
+          if (attribute2.indexOf(tag1) === -1) {
+            return 'Tag ' + tag1 + ' is missing.'
+          }
+        })
+      } else if (videoKey === 'files') {
+        if (attribute1.length !== attribute2.length) {
+          return 'Video files are different.'
+        }
+
+        attribute1.forEach((videoFile1: VideoFile) => {
+          const videoFile2: VideoFile = attribute2.find(videoFile => videoFile.magnetUri === videoFile1.magnetUri)
+          if (!videoFile2) {
+            return `Video ${video1.uuid} has missing video file ${videoFile1.magnetUri}.`
+          }
+
+          if (videoFile1.size !== videoFile2.size || videoFile1.resolutionLabel !== videoFile2.resolutionLabel) {
+            return `Video ${video1.uuid} has different video file ${videoFile1.magnetUri}.`
+          }
+        })
+      } else {
+        if (attribute1 !== attribute2) {
+          return `Video ${video1.uuid} has different value for attribute ${videoKey}.`
+        }
+      }
+    }
+  }
+
+  return undefined
+}
+
 function goodbye () {
   return process.exit(-1)
 }
 
-async function isThereAwaitingRequests (servers: ServerInfo[]) {
+async function isTherePendingRequests (servers: ServerInfo[]) {
   const tasks: Promise<any>[] = []
-  let awaitingRequests = false
+  let pendingRequests = false
 
-  // Check if each server has awaiting request
+  // Check if each server has pending request
   for (const server of servers) {
     const p = getRequestsStats(server).then(res => {
       const stats = res.body
@@ -314,7 +362,7 @@ async function isThereAwaitingRequests (servers: ServerInfo[]) {
         stats.requestVideoEventScheduler.totalRequests !== 0 ||
         stats.requestVideoQaduScheduler.totalRequests !== 0
       ) {
-        awaitingRequests = true
+        pendingRequests = true
       }
     })
 
@@ -323,5 +371,5 @@ async function isThereAwaitingRequests (servers: ServerInfo[]) {
 
   await Promise.all(tasks)
 
-  return awaitingRequests
+  return pendingRequests
 }

shared/models/videos/video.model.ts

@@ -10,6 +10,7 @@ export interface Video {
   uuid: string
   author: string
   createdAt: Date
+  updatedAt: Date
   categoryLabel: string
   category: number
   licenceLabel: string