Add migrate-to-object-storage script (#4481)

* add migrate-to-object-storage-script

closes #4467

* add migrate-to-unique-playlist-filenames script

* fix(migrate-to-unique-playlist-filenames): update master/segments256

run updateMasterHLSPlaylist and updateSha256VODSegments after
file rename.

* Improve move to object storage scripts

* PR remarks

Co-authored-by: Chocobozzz <me@florianbigard.com>

scripts/create-import-video-file-job.ts

@@ -47,7 +47,7 @@ async function run () {
     filePath: resolve(options.import)
   }
 
-  JobQueue.Instance.init()
+  JobQueue.Instance.init(true)
   await JobQueue.Instance.createJobWithPromise({ type: 'video-file-import', payload: dataInput })
   console.log('Import job for video %s created.', video.uuid)
 }

scripts/create-move-video-storage-job.ts

@@ -0,0 +1,86 @@
+import { registerTSPaths } from '../server/helpers/register-ts-paths'
+registerTSPaths()
+
+import { program } from 'commander'
+import { VideoModel } from '@server/models/video/video'
+import { initDatabaseModels } from '@server/initializers/database'
+import { VideoStorage } from '@shared/models'
+import { moveToExternalStorageState } from '@server/lib/video-state'
+import { JobQueue } from '@server/lib/job-queue'
+import { CONFIG } from '@server/initializers/config'
+
+program
+  .description('Move videos to another storage.')
+  .option('-o, --to-object-storage', 'Move videos in object storage')
+  .option('-v, --video [videoUUID]', 'Move a specific video')
+  .option('-a, --all-videos', 'Migrate all videos')
+  .parse(process.argv)
+
+const options = program.opts()
+
+if (!options['toObjectStorage']) {
+  console.error('You need to choose where to send video files.')
+  process.exit(-1)
+}
+
+if (!options['video'] && !options['allVideos']) {
+  console.error('You need to choose which videos to move.')
+  process.exit(-1)
+}
+
+if (options['toObjectStorage'] && !CONFIG.OBJECT_STORAGE.ENABLED) {
+  console.error('Object storage is not enabled on this instance.')
+  process.exit(-1)
+}
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => console.error(err))
+
+async function run () {
+  await initDatabaseModels(true)
+
+  JobQueue.Instance.init(true)
+
+  let ids: number[] = []
+
+  if (options['video']) {
+    const video = await VideoModel.load(options['video'])
+
+    if (!video) {
+      console.error('Unknown video ' + options['video'])
+      process.exit(-1)
+    }
+
+    if (video.remote === true) {
+      console.error('Cannot process a remote video')
+      process.exit(-1)
+    }
+
+    ids.push(video.id)
+  } else {
+    ids = await VideoModel.listLocalIds()
+  }
+
+  for (const id of ids) {
+    const videoFull = await VideoModel.loadAndPopulateAccountAndServerAndTags(id)
+
+    const files = videoFull.VideoFiles || []
+    const hls = videoFull.getHLSPlaylist()
+
+    if (files.some(f => f.storage === VideoStorage.FILE_SYSTEM) || hls?.storage === VideoStorage.FILE_SYSTEM) {
+      console.log('Processing video %s.', videoFull.name)
+
+      const success = await moveToExternalStorageState(videoFull, false, undefined)
+
+      if (!success) {
+        console.error(
+          'Cannot create move job for %s: job creation may have failed or there may be pending transcoding jobs for this video',
+          videoFull.name
+        )
+      }
+    }
+
+    console.log(`Created move-to-object-storage job for ${videoFull.name}.`)
+  }
+}

scripts/create-transcoding-job.ts

@@ -91,7 +91,7 @@ async function run () {
     }
   }
 
-  JobQueue.Instance.init()
+  JobQueue.Instance.init(true)
 
   video.state = VideoState.TO_TRANSCODE
   await video.save()

scripts/migrations/peertube-4.0.ts

@@ -0,0 +1,107 @@
+import { registerTSPaths } from '../../server/helpers/register-ts-paths'
+registerTSPaths()
+
+import { join } from 'path'
+import { JobQueue } from '@server/lib/job-queue'
+import { initDatabaseModels } from '../../server/initializers/database'
+import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
+import { VideoPathManager } from '@server/lib/video-path-manager'
+import { VideoModel } from '@server/models/video/video'
+import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
+import { move, readFile, writeFile } from 'fs-extra'
+import Bluebird from 'bluebird'
+import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  console.log('Migrate old HLS paths to new format.')
+
+  await initDatabaseModels(true)
+
+  JobQueue.Instance.init(true)
+
+  const ids = await VideoModel.listLocalIds()
+
+  await Bluebird.map(ids, async id => {
+    try {
+      await processVideo(id)
+    } catch (err) {
+      console.error('Cannot process video %s.', { err })
+    }
+  }, { concurrency: 5 })
+
+  console.log('Migration finished!')
+}
+
+async function processVideo (videoId: number) {
+  const video = await VideoModel.loadWithFiles(videoId)
+
+  const hls = video.getHLSPlaylist()
+  if (!hls || hls.playlistFilename !== 'master.m3u8' || hls.VideoFiles.length === 0) {
+    return
+  }
+
+  console.log(`Renaming HLS playlist files of video ${video.name}.`)
+
+  const playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
+  const hlsDirPath = VideoPathManager.Instance.getFSHLSOutputPath(video)
+
+  const masterPlaylistPath = join(hlsDirPath, playlist.playlistFilename)
+  let masterPlaylistContent = await readFile(masterPlaylistPath, 'utf8')
+
+  for (const videoFile of hls.VideoFiles) {
+    const srcName = `${videoFile.resolution}.m3u8`
+    const dstName = getHlsResolutionPlaylistFilename(videoFile.filename)
+
+    const src = join(hlsDirPath, srcName)
+    const dst = join(hlsDirPath, dstName)
+
+    try {
+      await move(src, dst)
+
+      masterPlaylistContent = masterPlaylistContent.replace(new RegExp('^' + srcName + '$', 'm'), dstName)
+    } catch (err) {
+      console.error('Cannot move video file %s to %s.', src, dst, err)
+    }
+  }
+
+  await writeFile(masterPlaylistPath, masterPlaylistContent)
+
+  if (playlist.segmentsSha256Filename === 'segments-sha256.json') {
+    try {
+      const newName = generateHlsSha256SegmentsFilename(video.isLive)
+
+      const dst = join(hlsDirPath, newName)
+      await move(join(hlsDirPath, playlist.segmentsSha256Filename), dst)
+      playlist.segmentsSha256Filename = newName
+    } catch (err) {
+      console.error(`Cannot rename ${video.name} segments-sha256.json file to a new name`, err)
+    }
+  }
+
+  if (playlist.playlistFilename === 'master.m3u8') {
+    try {
+      const newName = generateHLSMasterPlaylistFilename(video.isLive)
+
+      const dst = join(hlsDirPath, newName)
+      await move(join(hlsDirPath, playlist.playlistFilename), dst)
+      playlist.playlistFilename = newName
+    } catch (err) {
+      console.error(`Cannot rename ${video.name} master.m3u8 file to a new name`, err)
+    }
+  }
+
+  // Everything worked, we can save the playlist now
+  await playlist.save()
+
+  const allVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.id)
+  await federateVideoIfNeeded(allVideo, false)
+
+  console.log(`Successfully moved HLS files of ${video.name}.`)
+}

scripts/regenerate-thumbnails.ts

@@ -7,7 +7,6 @@ import { pathExists, remove } from 'fs-extra'
 import { generateImageFilename, processImage } from '@server/helpers/image-utils'
 import { THUMBNAILS_SIZE } from '@server/initializers/constants'
 import { VideoModel } from '@server/models/video/video'
-import { MVideo } from '@server/types/models'
 import { initDatabaseModels } from '@server/initializers/database'
 
 program
@@ -21,16 +20,16 @@ run()
 async function run () {
   await initDatabaseModels(true)
 
-  const videos = await VideoModel.listLocal()
+  const ids = await VideoModel.listLocalIds()
 
-  await map(videos, v => {
-    return processVideo(v)
-      .catch(err => console.error('Cannot process video %s.', v.url, err))
+  await map(ids, id => {
+    return processVideo(id)
+      .catch(err => console.error('Cannot process video %d.', id, err))
   }, { concurrency: 20 })
 }
 
-async function processVideo (videoArg: MVideo) {
-  const video = await VideoModel.loadWithFiles(videoArg.id)
+async function processVideo (id: number) {
+  const video = await VideoModel.loadWithFiles(id)
 
   console.log('Processing video %s.', video.name)
 

scripts/update-host.ts

@@ -115,9 +115,9 @@ async function run () {
 
   console.log('Updating video and torrent files.')
 
-  const localVideos = await VideoModel.listLocal()
-  for (const localVideo of localVideos) {
-    const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(localVideo.id)
+  const ids = await VideoModel.listLocalIds()
+  for (const id of ids) {
+    const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(id)
 
     console.log('Updating video ' + video.uuid)