Add hls support on server

2025-02-25 18:55:32 -06:00 · 2019-01-29 08:37:25 +01:00
parent 4348a27d25
commit 0920929696
81 changed files with 2000 additions and 407 deletions
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -1,11 +1,28 @@
-import { CacheFileObject } from '../../../shared/index'
+import { ActivityPlaylistUrlObject, ActivityVideoUrlObject, CacheFileObject } from '../../../shared/index'
 import { VideoModel } from '../../models/video/video'
 import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
 import { Transaction } from 'sequelize'
+import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'

 function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
-  const url = cacheFileObject.url

+  if (cacheFileObject.url.mediaType === 'application/x-mpegURL') {
+    const url = cacheFileObject.url
+
+    const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS)
+    if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
+
+    return {
+      expiresOn: new Date(cacheFileObject.expires),
+      url: cacheFileObject.id,
+      fileUrl: url.href,
+      strategy: null,
+      videoStreamingPlaylistId: playlist.id,
+      actorId: byActor.id
+    }
+  }
+
+  const url = cacheFileObject.url
  const videoFile = video.VideoFiles.find(f => {
    return f.resolution === url.height && f.fps === url.fps
  })
@@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
  return {
    expiresOn: new Date(cacheFileObject.expires),
    url: cacheFileObject.id,
-    fileUrl: cacheFileObject.url.href,
+    fileUrl: url.href,
    strategy: null,
    videoFileId: videoFile.id,
    actorId: byActor.id
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -1,6 +1,6 @@
 import { Transaction } from 'sequelize'
 import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub'
-import { VideoPrivacy } from '../../../../shared/models/videos'
+import { Video, VideoPrivacy } from '../../../../shared/models/videos'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { VideoModel } from '../../../models/video/video'
 import { VideoAbuseModel } from '../../../models/video/video-abuse'
@@ -39,17 +39,14 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
  return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
 }

-async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) {
+async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) {
  logger.info('Creating job to send file cache of %s.', fileRedundancy.url)

-  const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
-  const redundancyObject = fileRedundancy.toActivityPubObject()
-
  return sendVideoRelatedCreateActivity({
    byActor,
    video,
    url: fileRedundancy.url,
-    object: redundancyObject
+    object: fileRedundancy.toActivityPubObject()
  })
 }

--- a/server/lib/activitypub/send/send-undo.ts
+++ b/server/lib/activitypub/send/send-undo.ts
@@ -73,7 +73,8 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans
 async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
  logger.info('Creating job to undo cache file %s.', redundancyModel.url)

-  const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
+  const videoId = redundancyModel.getVideo().id
+  const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
  const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())

  return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t })
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -61,7 +61,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
 async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
  logger.info('Creating job to update cache file %s.', redundancyModel.url)

-  const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
+  const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id)

  const activityBuilder = (audience: ActivityAudience) => {
    const redundancyObject = redundancyModel.toActivityPubObject()
--- a/server/lib/activitypub/url.ts
+++ b/server/lib/activitypub/url.ts
@@ -5,6 +5,8 @@ import { VideoModel } from '../../models/video/video'
 import { VideoAbuseModel } from '../../models/video/video-abuse'
 import { VideoCommentModel } from '../../models/video/video-comment'
 import { VideoFileModel } from '../../models/video/video-file'
+import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
+import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'

 function getVideoActivityPubUrl (video: VideoModel) {
  return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid
@@ -16,6 +18,10 @@ function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) {
  return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}`
 }

+function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) {
+  return `${CONFIG.WEBSERVER.URL}/redundancy/video-playlists/${playlist.getStringType()}/${video.uuid}`
+}
+
 function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) {
  return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id
 }
@@ -92,6 +98,7 @@ function getUndoActivityPubUrl (originalUrl: string) {

 export {
  getVideoActivityPubUrl,
+  getVideoCacheStreamingPlaylistActivityPubUrl,
  getVideoChannelActivityPubUrl,
  getAccountActivityPubUrl,
  getVideoAbuseActivityPubUrl,
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -2,7 +2,14 @@ import * as Bluebird from 'bluebird'
 import * as sequelize from 'sequelize'
 import * as magnetUtil from 'magnet-uri'
 import * as request from 'request'
-import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index'
+import {
+  ActivityIconObject,
+  ActivityPlaylistSegmentHashesObject,
+  ActivityPlaylistUrlObject,
+  ActivityUrlObject,
+  ActivityVideoUrlObject,
+  VideoState
+} from '../../../shared/index'
 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
 import { VideoPrivacy } from '../../../shared/models/videos'
 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
@@ -30,6 +37,9 @@ import { AccountModel } from '../../models/account/account'
 import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
 import { Notifier } from '../notifier'
+import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
+import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
+import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'

 async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
  // If the video is not private and published, we federate it
@@ -263,6 +273,25 @@ async function updateVideoFromAP (options: {
        options.video.VideoFiles = await Promise.all(upsertTasks)
      }

+      {
+        const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject)
+        const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
+
+        // Remove video files that do not exist anymore
+        const destroyTasks = options.video.VideoStreamingPlaylists
+                                    .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f)))
+                                    .map(f => f.destroy(sequelizeOptions))
+        await Promise.all(destroyTasks)
+
+        // Update or add other one
+        const upsertTasks = streamingPlaylistAttributes.map(a => {
+          return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t })
+                               .then(([ streamingPlaylist ]) => streamingPlaylist)
+        })
+
+        options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks)
+      }
+
      {
        // Update Tags
        const tags = options.videoObject.tag.map(tag => tag.name)
@@ -367,13 +396,25 @@ export {

 // ---------------------------------------------------------------------------

-function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
+function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
  const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)

  const urlMediaType = url.mediaType || url.mimeType
  return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
 }

+function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
+  const urlMediaType = url.mediaType || url.mimeType
+
+  return urlMediaType === 'application/x-mpegURL'
+}
+
+function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
+  const urlMediaType = tag.mediaType || tag.mimeType
+
+  return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json'
+}
+
 async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
  logger.debug('Adding remote video %s.', videoObject.id)

@@ -394,8 +435,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
    const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
    await Promise.all(videoFilePromises)

+    const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject)
+    const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
+    await Promise.all(playlistPromises)
+
    // Process tags
-    const tags = videoObject.tag.map(t => t.name)
+    const tags = videoObject.tag
+                            .filter(t => t.type === 'Hashtag')
+                            .map(t => t.name)
    const tagInstances = await TagModel.findOrCreateTags(tags, t)
    await videoCreated.$set('Tags', tagInstances, sequelizeOptions)

@@ -473,13 +520,13 @@ async function videoActivityObjectToDBAttributes (
 }

 function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
-  const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
+  const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]

  if (fileUrls.length === 0) {
    throw new Error('Cannot find video files for ' + video.url)
  }

-  const attributes: VideoFileModel[] = []
+  const attributes: FilteredModelAttributes<VideoFileModel>[] = []
  for (const fileUrl of fileUrls) {
    // Fetch associated magnet uri
    const magnet = videoObject.url.find(u => {
@@ -502,7 +549,45 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid
      size: fileUrl.size,
      videoId: video.id,
      fps: fileUrl.fps || -1
-    } as VideoFileModel
+    }
+
+    attributes.push(attribute)
+  }
+
+  return attributes
+}
+
+function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
+  const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
+  if (playlistUrls.length === 0) return []
+
+  const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
+  for (const playlistUrlObject of playlistUrls) {
+    const p2pMediaLoaderInfohashes = playlistUrlObject.tag
+                                                      .filter(t => t.type === 'Infohash')
+                                                      .map(t => t.name)
+    if (p2pMediaLoaderInfohashes.length === 0) {
+      logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject })
+      continue
+    }
+
+    const segmentsSha256UrlObject = playlistUrlObject.tag
+                                                     .find(t => {
+                                                       return isAPPlaylistSegmentHashesUrlObject(t)
+                                                     }) as ActivityPlaylistSegmentHashesObject
+    if (!segmentsSha256UrlObject) {
+      logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
+      continue
+    }
+
+    const attribute = {
+      type: VideoStreamingPlaylistType.HLS,
+      playlistUrl: playlistUrlObject.href,
+      segmentsSha256Url: segmentsSha256UrlObject.href,
+      p2pMediaLoaderInfohashes,
+      videoId: video.id
+    }
+
    attributes.push(attribute)
  }

--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -0,0 +1,110 @@
+import { VideoModel } from '../models/video/video'
+import { basename, dirname, join } from 'path'
+import { HLS_PLAYLIST_DIRECTORY, CONFIG } from '../initializers'
+import { outputJSON, pathExists, readdir, readFile, remove, writeFile, move } from 'fs-extra'
+import { getVideoFileSize } from '../helpers/ffmpeg-utils'
+import { sha256 } from '../helpers/core-utils'
+import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
+import HLSDownloader from 'hlsdownloader'
+import { logger } from '../helpers/logger'
+import { parse } from 'url'
+
+async function updateMasterHLSPlaylist (video: VideoModel) {
+  const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
+  const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
+  const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
+
+  for (const file of video.VideoFiles) {
+    // If we did not generated a playlist for this resolution, skip
+    const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
+    if (await pathExists(filePlaylistPath) === false) continue
+
+    const videoFilePath = video.getVideoFilePath(file)
+
+    const size = await getVideoFileSize(videoFilePath)
+
+    const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
+    const resolution = `RESOLUTION=${size.width}x${size.height}`
+
+    let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
+    if (file.fps) line += ',FRAME-RATE=' + file.fps
+
+    masterPlaylists.push(line)
+    masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
+  }
+
+  await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
+}
+
+async function updateSha256Segments (video: VideoModel) {
+  const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
+  const files = await readdir(directory)
+  const json: { [filename: string]: string} = {}
+
+  for (const file of files) {
+    if (file.endsWith('.ts') === false) continue
+
+    const buffer = await readFile(join(directory, file))
+    const filename = basename(file)
+
+    json[filename] = sha256(buffer)
+  }
+
+  const outputPath = join(directory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
+  await outputJSON(outputPath, json)
+}
+
+function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
+  let timer
+
+  logger.info('Importing HLS playlist %s', playlistUrl)
+
+  const params = {
+    playlistURL: playlistUrl,
+    destination: CONFIG.STORAGE.TMP_DIR
+  }
+  const downloader = new HLSDownloader(params)
+
+  const hlsDestinationDir = join(CONFIG.STORAGE.TMP_DIR, dirname(parse(playlistUrl).pathname))
+
+  return new Promise<string>(async (res, rej) => {
+    downloader.startDownload(err => {
+      clearTimeout(timer)
+
+      if (err) {
+        deleteTmpDirectory(hlsDestinationDir)
+
+        return rej(err)
+      }
+
+      move(hlsDestinationDir, destinationDir, { overwrite: true })
+        .then(() => res())
+        .catch(err => {
+          deleteTmpDirectory(hlsDestinationDir)
+
+          return rej(err)
+        })
+    })
+
+    timer = setTimeout(() => {
+      deleteTmpDirectory(hlsDestinationDir)
+
+      return rej(new Error('HLS download timeout.'))
+    }, timeout)
+
+    function deleteTmpDirectory (directory: string) {
+      remove(directory)
+        .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
+    }
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  updateMasterHLSPlaylist,
+  updateSha256Segments,
+  downloadPlaylistSegments
+}
+
+// ---------------------------------------------------------------------------
--- a/server/lib/job-queue/handlers/video-file.ts
+++ b/server/lib/job-queue/handlers/video-file.ts
@@ -5,17 +5,18 @@ import { VideoModel } from '../../../models/video/video'
 import { JobQueue } from '../job-queue'
 import { federateVideoIfNeeded } from '../../activitypub'
 import { retryTransactionWrapper } from '../../../helpers/database-utils'
-import { sequelizeTypescript } from '../../../initializers'
+import { sequelizeTypescript, CONFIG } from '../../../initializers'
 import * as Bluebird from 'bluebird'
 import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
-import { importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding'
+import { generateHlsPlaylist, importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding'
 import { Notifier } from '../../notifier'

 export type VideoFilePayload = {
  videoUUID: string
-  isNewVideo?: boolean
  resolution?: VideoResolution
+  isNewVideo?: boolean
  isPortraitMode?: boolean
+  generateHlsPlaylist?: boolean
 }

 export type VideoFileImportPayload = {
@@ -51,21 +52,38 @@ async function processVideoFile (job: Bull.Job) {
    return undefined
  }

-  // Transcoding in other resolution
-  if (payload.resolution) {
+  if (payload.generateHlsPlaylist) {
+    await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false)
+
+    await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video)
+  } else if (payload.resolution) { // Transcoding in other resolution
    await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false)

-    await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video)
+    await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video, payload)
  } else {
    await optimizeVideofile(video)

-    await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo)
+    await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload)
  }

  return video
 }

-async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) {
+async function onHlsPlaylistGenerationSuccess (video: VideoModel) {
+  if (video === undefined) return undefined
+
+  await sequelizeTypescript.transaction(async t => {
+    // Maybe the video changed in database, refresh it
+    let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
+    // Video does not exist anymore
+    if (!videoDatabase) return undefined
+
+    // If the video was not published, we consider it is a new one for other instances
+    await federateVideoIfNeeded(videoDatabase, false, t)
+  })
+}
+
+async function onVideoFileTranscoderOrImportSuccess (video: VideoModel, payload?: VideoFilePayload) {
  if (video === undefined) return undefined

  const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
@@ -96,9 +114,11 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) {
    Notifier.Instance.notifyOnNewVideo(videoDatabase)
    Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase)
  }
+
+  await createHlsJobIfEnabled(payload)
 }

-async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) {
+async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoFilePayload) {
  if (videoArg === undefined) return undefined

  // Outside the transaction (IO on disk)
@@ -145,7 +165,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo
      logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy })
    }

-    await federateVideoIfNeeded(videoDatabase, isNewVideo, t)
+    await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t)

    return { videoDatabase, videoPublished }
  })
@@ -155,6 +175,8 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo
    if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase)
    if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase)
  }
+
+  await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution }))
 }

 // ---------------------------------------------------------------------------
@@ -163,3 +185,20 @@ export {
  processVideoFile,
  processVideoFileImport
 }
+
+// ---------------------------------------------------------------------------
+
+function createHlsJobIfEnabled (payload?: VideoFilePayload) {
+  // Generate HLS playlist?
+  if (payload && CONFIG.TRANSCODING.HLS.ENABLED) {
+    const hlsTranscodingPayload = {
+      videoUUID: payload.videoUUID,
+      resolution: payload.resolution,
+      isPortraitMode: payload.isPortraitMode,
+
+      generateHlsPlaylist: true
+    }
+
+    return JobQueue.Instance.createJob({ type: 'video-file', payload: hlsTranscodingPayload })
+  }
+}
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,5 +1,5 @@
 import { AbstractScheduler } from './abstract-scheduler'
-import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers'
+import { CONFIG, HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers'
 import { logger } from '../../helpers/logger'
 import { VideosRedundancy } from '../../../shared/models/redundancy'
 import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
@@ -9,9 +9,19 @@ import { join } from 'path'
 import { move } from 'fs-extra'
 import { getServerActor } from '../../helpers/utils'
 import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
-import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
+import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
 import { removeVideoRedundancy } from '../redundancy'
 import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
+import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
+import { VideoModel } from '../../models/video/video'
+import { downloadPlaylistSegments } from '../hls'
+
+type CandidateToDuplicate = {
+  redundancy: VideosRedundancy,
+  video: VideoModel,
+  files: VideoFileModel[],
+  streamingPlaylists: VideoStreamingPlaylistModel[]
+}

 export class VideosRedundancyScheduler extends AbstractScheduler {

@@ -24,28 +34,32 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
  }

  protected async internalExecute () {
-    for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
-      logger.info('Running redundancy scheduler for strategy %s.', obj.strategy)
+    for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
+      logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy)

      try {
-        const videoToDuplicate = await this.findVideoToDuplicate(obj)
+        const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig)
        if (!videoToDuplicate) continue

-        const videoFiles = videoToDuplicate.VideoFiles
-        videoFiles.forEach(f => f.Video = videoToDuplicate)
+        const candidateToDuplicate = {
+          video: videoToDuplicate,
+          redundancy: redundancyConfig,
+          files: videoToDuplicate.VideoFiles,
+          streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
+        }

-        await this.purgeCacheIfNeeded(obj, videoFiles)
+        await this.purgeCacheIfNeeded(candidateToDuplicate)

-        if (await this.isTooHeavy(obj, videoFiles)) {
+        if (await this.isTooHeavy(candidateToDuplicate)) {
          logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
          continue
        }

-        logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy)
+        logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy)

-        await this.createVideoRedundancy(obj, videoFiles)
+        await this.createVideoRedundancies(candidateToDuplicate)
      } catch (err) {
-        logger.error('Cannot run videos redundancy %s.', obj.strategy, { err })
+        logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err })
      }
    }

@@ -63,25 +77,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler {

    for (const redundancyModel of expired) {
      try {
-        await this.extendsOrDeleteRedundancy(redundancyModel)
+        const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
+        const candidate = {
+          redundancy: redundancyConfig,
+          video: null,
+          files: [],
+          streamingPlaylists: []
+        }
+
+        // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it
+        if (!redundancyConfig || await this.isTooHeavy(candidate)) {
+          logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy)
+          await removeVideoRedundancy(redundancyModel)
+        } else {
+          await this.extendsRedundancy(redundancyModel)
+        }
      } catch (err) {
-        logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel))
+        logger.error(
+          'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel),
+          { err }
+        )
      }
    }
  }

-  private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) {
-    // Refresh the video, maybe it was deleted
-    const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url)
-
-    if (!video) {
-      logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url)
-
-      await redundancyModel.destroy()
-      return
-    }
-
+  private async extendsRedundancy (redundancyModel: VideoRedundancyModel) {
    const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
+    // Redundancy strategy disabled, remove our redundancy instead of extending expiration
+    if (!redundancy) await removeVideoRedundancy(redundancyModel)
+
    await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
  }

@@ -112,49 +136,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
    }
  }

-  private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
-    const serverActor = await getServerActor()
+  private async createVideoRedundancies (data: CandidateToDuplicate) {
+    const video = await this.loadAndRefreshVideo(data.video.url)

-    for (const file of filesToDuplicate) {
-      const video = await this.loadAndRefreshVideo(file.Video.url)
+    if (!video) {
+      logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url)

+      return
+    }
+
+    for (const file of data.files) {
      const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
      if (existingRedundancy) {
-        await this.extendsOrDeleteRedundancy(existingRedundancy)
+        await this.extendsRedundancy(existingRedundancy)

        continue
      }

-      if (!video) {
-        logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url)
-
-        continue
-      }
-
-      logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
-
-      const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
-      const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
-
-      const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
-
-      const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file))
-      await move(tmpPath, destPath)
-
-      const createdModel = await VideoRedundancyModel.create({
-        expiresOn: this.buildNewExpiration(redundancy.minLifetime),
-        url: getVideoCacheFileActivityPubUrl(file),
-        fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
-        strategy: redundancy.strategy,
-        videoFileId: file.id,
-        actorId: serverActor.id
-      })
-      createdModel.VideoFile = file
-
-      await sendCreateCacheFile(serverActor, createdModel)
-
-      logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url)
+      await this.createVideoFileRedundancy(data.redundancy, video, file)
    }
+
+    for (const streamingPlaylist of data.streamingPlaylists) {
+      const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id)
+      if (existingRedundancy) {
+        await this.extendsRedundancy(existingRedundancy)
+
+        continue
+      }
+
+      await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist)
+    }
+  }
+
+  private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) {
+    file.Video = video
+
+    const serverActor = await getServerActor()
+
+    logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
+
+    const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
+    const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
+
+    const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
+
+    const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file))
+    await move(tmpPath, destPath)
+
+    const createdModel = await VideoRedundancyModel.create({
+      expiresOn: this.buildNewExpiration(redundancy.minLifetime),
+      url: getVideoCacheFileActivityPubUrl(file),
+      fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
+      strategy: redundancy.strategy,
+      videoFileId: file.id,
+      actorId: serverActor.id
+    })
+
+    createdModel.VideoFile = file
+
+    await sendCreateCacheFile(serverActor, video, createdModel)
+
+    logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url)
+  }
+
+  private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) {
+    playlist.Video = video
+
+    const serverActor = await getServerActor()
+
+    logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy)
+
+    const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
+    await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
+
+    const createdModel = await VideoRedundancyModel.create({
+      expiresOn: this.buildNewExpiration(redundancy.minLifetime),
+      url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
+      fileUrl: playlist.getVideoRedundancyUrl(CONFIG.WEBSERVER.URL),
+      strategy: redundancy.strategy,
+      videoStreamingPlaylistId: playlist.id,
+      actorId: serverActor.id
+    })
+
+    createdModel.VideoStreamingPlaylist = playlist
+
+    await sendCreateCacheFile(serverActor, video, createdModel)
+
+    logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url)
  }

  private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) {
@@ -168,8 +236,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
    await sendUpdateCacheFile(serverActor, redundancy)
  }

-  private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
-    while (this.isTooHeavy(redundancy, filesToDuplicate)) {
+  private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) {
+    while (this.isTooHeavy(candidateToDuplicate)) {
+      const redundancy = candidateToDuplicate.redundancy
      const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime)
      if (!toDelete) return

@@ -177,11 +246,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
    }
  }

-  private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
-    const maxSize = redundancy.size
+  private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) {
+    const maxSize = candidateToDuplicate.redundancy.size

-    const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy)
-    const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate)
+    const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy)
+    const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists)

    return totalWillDuplicate > maxSize
  }
@@ -191,13 +260,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
  }

  private buildEntryLogId (object: VideoRedundancyModel) {
-    return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
+    if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
+
+    return `${object.VideoStreamingPlaylist.playlistUrl}`
  }

-  private getTotalFileSizes (files: VideoFileModel[]) {
+  private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) {
    const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size

-    return files.reduce(fileReducer, 0)
+    return files.reduce(fileReducer, 0) * playlists.length
  }

  private async loadAndRefreshVideo (videoUrl: string) {
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -1,11 +1,14 @@
-import { CONFIG } from '../initializers'
+import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers'
 import { extname, join } from 'path'
 import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils'
-import { copy, remove, move, stat } from 'fs-extra'
+import { copy, ensureDir, move, remove, stat } from 'fs-extra'
 import { logger } from '../helpers/logger'
 import { VideoResolution } from '../../shared/models/videos'
 import { VideoFileModel } from '../models/video/video-file'
 import { VideoModel } from '../models/video/video'
+import { updateMasterHLSPlaylist, updateSha256Segments } from './hls'
+import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
+import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type'

 async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) {
  const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
@@ -17,7 +20,8 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi

  const transcodeOptions = {
    inputPath: videoInputPath,
-    outputPath: videoTranscodedPath
+    outputPath: videoTranscodedPath,
+    resolution: inputVideoFile.resolution
  }

  // Could be very long!
@@ -47,7 +51,7 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi
  }
 }

-async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) {
+async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) {
  const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
  const extname = '.mp4'

@@ -60,13 +64,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
    size: 0,
    videoId: video.id
  })
-  const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile))
+  const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile))

  const transcodeOptions = {
    inputPath: videoInputPath,
    outputPath: videoOutputPath,
    resolution,
-    isPortraitMode
+    isPortraitMode: isPortrait
  }

  await transcode(transcodeOptions)
@@ -84,6 +88,38 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
  video.VideoFiles.push(newVideoFile)
 }

+async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) {
+  const baseHlsDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
+  await ensureDir(join(HLS_PLAYLIST_DIRECTORY, video.uuid))
+
+  const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile()))
+  const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
+
+  const transcodeOptions = {
+    inputPath: videoInputPath,
+    outputPath,
+    resolution,
+    isPortraitMode,
+    generateHlsPlaylist: true
+  }
+
+  await transcode(transcodeOptions)
+
+  await updateMasterHLSPlaylist(video)
+  await updateSha256Segments(video)
+
+  const playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
+
+  await VideoStreamingPlaylistModel.upsert({
+    videoId: video.id,
+    playlistUrl,
+    segmentsSha256Url: CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid),
+    p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles),
+
+    type: VideoStreamingPlaylistType.HLS
+  })
+}
+
 async function importVideoFile (video: VideoModel, inputFilePath: string) {
  const { videoFileResolution } = await getVideoFileResolution(inputFilePath)
  const { size } = await stat(inputFilePath)
@@ -125,6 +161,7 @@ async function importVideoFile (video: VideoModel, inputFilePath: string) {
 }

 export {
+  generateHlsPlaylist,
  optimizeVideofile,
  transcodeOriginalVideofile,
  importVideoFile