Merge branch 'release/5.0.0' into develop

server/controllers/object-storage-proxy.ts

@@ -1,7 +1,10 @@
 import cors from 'cors'
 import express from 'express'
+import { PassThrough, pipeline } from 'stream'
 import { logger } from '@server/helpers/logger'
+import { StreamReplacer } from '@server/helpers/stream-replacer'
 import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants'
+import { injectQueryToPlaylistUrls } from '@server/lib/hls'
 import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage'
 import {
   asyncMiddleware,
@@ -11,6 +14,7 @@ import {
   optionalAuthenticate
 } from '@server/middlewares'
 import { HttpStatusCode } from '@shared/models'
+import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist'
 
 const objectStorageProxyRouter = express.Router()
 
@@ -67,7 +71,20 @@ async function proxifyHLS (req: express.Request, res: express.Response) {
       rangeHeader: req.header('range')
     })
 
-    return stream.pipe(res)
+    const streamReplacer = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req)
+      ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req)))
+      : new PassThrough()
+
+    return pipeline(
+      stream,
+      streamReplacer,
+      res,
+      err => {
+        if (!err) return
+
+        handleObjectStorageFailure(res, err)
+      }
+    )
   } catch (err) {
     return handleObjectStorageFailure(res, err)
   }
@@ -75,6 +92,7 @@ async function proxifyHLS (req: express.Request, res: express.Response) {
 
 function handleObjectStorageFailure (res: express.Response, err: Error) {
   if (err.name === 'NoSuchKey') {
+    logger.debug('Could not find key in object storage to proxify private HLS video file.', { err })
     return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
   }
 

server/controllers/shared/m3u8-playlist.ts

@@ -0,0 +1,14 @@
+import express from 'express'
+
+function doReinjectVideoFileToken (req: express.Request) {
+  return req.query.videoFileToken && req.query.reinjectVideoFileToken
+}
+
+function buildReinjectVideoFileTokenQuery (req: express.Request) {
+  return 'videoFileToken=' + req.query.videoFileToken
+}
+
+export {
+  doReinjectVideoFileToken,
+  buildReinjectVideoFileTokenQuery
+}

server/controllers/static.ts

@@ -1,5 +1,8 @@
 import cors from 'cors'
 import express from 'express'
+import { readFile } from 'fs-extra'
+import { join } from 'path'
+import { injectQueryToPlaylistUrls } from '@server/lib/hls'
 import {
   asyncMiddleware,
   ensureCanAccessPrivateVideoHLSFiles,
@@ -7,8 +10,10 @@ import {
   handleStaticError,
   optionalAuthenticate
 } from '@server/middlewares'
+import { HttpStatusCode } from '@shared/models'
 import { CONFIG } from '../initializers/config'
 import { DIRECTORIES, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants'
+import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist'
 
 const staticRouter = express.Router()
 
@@ -49,6 +54,12 @@ const privateHLSStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AU
   ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles) ]
   : []
 
+staticRouter.use(
+  STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:playlistName.m3u8',
+  ...privateHLSStaticMiddlewares,
+  asyncMiddleware(servePrivateM3U8)
+)
+
 staticRouter.use(
   STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS,
   ...privateHLSStaticMiddlewares,
@@ -74,3 +85,31 @@ staticRouter.use(
 export {
   staticRouter
 }
+
+// ---------------------------------------------------------------------------
+
+async function servePrivateM3U8 (req: express.Request, res: express.Response) {
+  const path = join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, req.params.videoUUID, req.params.playlistName + '.m3u8')
+
+  let playlistContent: string
+
+  try {
+    playlistContent = await readFile(path, 'utf-8')
+  } catch (err) {
+    if (err.message.includes('ENOENT')) {
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'File not found'
+      })
+    }
+
+    throw err
+  }
+
+  // Inject token in playlist so players that cannot alter the HTTP request can still watch the video
+  const transformedContent = doReinjectVideoFileToken(req)
+    ? injectQueryToPlaylistUrls(playlistContent, buildReinjectVideoFileTokenQuery(req))
+    : playlistContent
+
+  return res.set('content-type', 'application/vnd.apple.mpegurl').send(transformedContent).end()
+}

server/helpers/stream-replacer.ts

@@ -0,0 +1,58 @@
+import { Transform, TransformCallback } from 'stream'
+
+// Thanks: https://stackoverflow.com/a/45126242
+class StreamReplacer extends Transform {
+  private pendingChunk: Buffer
+
+  constructor (private readonly replacer: (line: string) => string) {
+    super()
+  }
+
+  _transform (chunk: Buffer, _encoding: BufferEncoding, done: TransformCallback) {
+    try {
+      this.pendingChunk = this.pendingChunk?.length
+        ? Buffer.concat([ this.pendingChunk, chunk ])
+        : chunk
+
+      let index: number
+
+      // As long as we keep finding newlines, keep making slices of the buffer and push them to the
+      // readable side of the transform stream
+      while ((index = this.pendingChunk.indexOf('\n')) !== -1) {
+        // The `end` parameter is non-inclusive, so increase it to include the newline we found
+        const line = this.pendingChunk.slice(0, ++index)
+
+        // `start` is inclusive, but we are already one char ahead of the newline -> all good
+        this.pendingChunk = this.pendingChunk.slice(index)
+
+        // We have a single line here! Prepend the string we want
+        this.push(this.doReplace(line))
+      }
+
+      return done()
+    } catch (err) {
+      return done(err)
+    }
+  }
+
+  _flush (done: TransformCallback) {
+    // If we have any remaining data in the cache, send it out
+    if (!this.pendingChunk?.length) return done()
+
+    try {
+      return done(null, this.doReplace(this.pendingChunk))
+    } catch (err) {
+      return done(err)
+    }
+  }
+
+  private doReplace (buffer: Buffer) {
+    const line = this.replacer(buffer.toString('utf8'))
+
+    return Buffer.from(line, 'utf8')
+  }
+}
+
+export {
+  StreamReplacer
+}

server/lib/hls.ts

@@ -234,13 +234,20 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
 
 // ---------------------------------------------------------------------------
 
+function injectQueryToPlaylistUrls (content: string, queryString: string) {
+  return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString)
+}
+
+// ---------------------------------------------------------------------------
+
 export {
   updateMasterHLSPlaylist,
   updateSha256VODSegments,
   buildSha256Segment,
   downloadPlaylistSegments,
   updateStreamingPlaylistsInfohashesIfNeeded,
-  updatePlaylistAfterFileChange
+  updatePlaylistAfterFileChange,
+  injectQueryToPlaylistUrls
 }
 
 // ---------------------------------------------------------------------------

server/middlewares/validators/static.ts

@@ -2,7 +2,7 @@ import express from 'express'
 import { query } from 'express-validator'
 import LRUCache from 'lru-cache'
 import { basename, dirname } from 'path'
-import { exists, isUUIDValid } from '@server/helpers/custom-validators/misc'
+import { exists, isUUIDValid, toBooleanOrNull } from '@server/helpers/custom-validators/misc'
 import { logger } from '@server/helpers/logger'
 import { LRU_CACHE } from '@server/initializers/constants'
 import { VideoModel } from '@server/models/video/video'
@@ -60,7 +60,14 @@ const ensureCanAccessVideoPrivateWebTorrentFiles = [
 ]
 
 const ensureCanAccessPrivateVideoHLSFiles = [
-  query('videoFileToken').optional().custom(exists),
+  query('videoFileToken')
+    .optional()
+    .custom(exists),
+
+  query('reinjectVideoFileToken')
+    .optional()
+    .customSanitizer(toBooleanOrNull)
+    .isBoolean().withMessage('Should be a valid reinjectVideoFileToken boolean'),
 
   async (req: express.Request, res: express.Response, next: express.NextFunction) => {
     if (areValidationErrors(req, res)) return

server/middlewares/validators/users.ts

@@ -44,7 +44,7 @@ const usersListValidator = [
   query('blocked')
     .optional()
     .customSanitizer(toBooleanOrNull)
-    .isBoolean().withMessage('Should be a valid blocked boolena'),
+    .isBoolean().withMessage('Should be a valid blocked boolean'),
 
   (req: express.Request, res: express.Response, next: express.NextFunction) => {
     if (areValidationErrors(req, res)) return

server/tests/api/object-storage/video-static-file-privacy.ts

@@ -2,7 +2,7 @@
 
 import { expect } from 'chai'
 import { basename } from 'path'
-import { expectStartWith } from '@server/tests/shared'
+import { checkVideoFileTokenReinjection, expectStartWith } from '@server/tests/shared'
 import { areScalewayObjectStorageTestsDisabled, getAllFiles, getHLS } from '@shared/core-utils'
 import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models'
 import {
@@ -191,6 +191,20 @@ describe('Object storage for video static file privacy', function () {
       }
     })
 
+    it('Should reinject video file token', async function () {
+      this.timeout(120000)
+
+      const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID })
+
+      await checkVideoFileTokenReinjection({
+        server,
+        videoUUID: privateVideoUUID,
+        videoFileToken,
+        resolutions: [ 240, 720 ],
+        isLive: false
+      })
+    })
+
     it('Should update public video to private', async function () {
       this.timeout(60000)
 
@@ -315,6 +329,26 @@ describe('Object storage for video static file privacy', function () {
       await checkLiveFiles(permanentLive, permanentLiveId)
     })
 
+    it('Should reinject video file token in permanent live', async function () {
+      this.timeout(240000)
+
+      const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: permanentLive.rtmpUrl, streamKey: permanentLive.streamKey })
+      await server.live.waitUntilPublished({ videoId: permanentLiveId })
+
+      const video = await server.videos.getWithToken({ id: permanentLiveId })
+      const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid })
+
+      await checkVideoFileTokenReinjection({
+        server,
+        videoUUID: permanentLiveId,
+        videoFileToken,
+        resolutions: [ 720 ],
+        isLive: true
+      })
+
+      await stopFfmpeg(ffmpegCommand)
+    })
+
     it('Should have created a replay of the normal live with a private static path', async function () {
       this.timeout(240000)
 

server/tests/api/videos/video-static-file-privacy.ts

@@ -2,7 +2,7 @@
 
 import { expect } from 'chai'
 import { decode } from 'magnet-uri'
-import { expectStartWith } from '@server/tests/shared'
+import { checkVideoFileTokenReinjection, expectStartWith } from '@server/tests/shared'
 import { getAllFiles, wait } from '@shared/core-utils'
 import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models'
 import {
@@ -248,6 +248,35 @@ describe('Test video static file privacy', function () {
       await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken })
     })
 
+    it('Should reinject video file token', async function () {
+      this.timeout(120000)
+
+      const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
+
+      const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
+      await waitJobs([ server ])
+
+      const video = await server.videos.getWithToken({ id: uuid })
+      const hls = video.streamingPlaylists[0]
+
+      {
+        const query = { videoFileToken }
+        const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 })
+
+        expect(text).to.not.include(videoFileToken)
+      }
+
+      {
+        await checkVideoFileTokenReinjection({
+          server,
+          videoUUID: uuid,
+          videoFileToken,
+          resolutions: [ 240, 720 ],
+          isLive: false
+        })
+      }
+    })
+
     it('Should be able to access a private video of another user with an admin OAuth token or file token', async function () {
       this.timeout(120000)
 
@@ -360,6 +389,36 @@ describe('Test video static file privacy', function () {
       await checkLiveFiles(permanentLive, permanentLiveId)
     })
 
+    it('Should reinject video file token on permanent live', async function () {
+      this.timeout(240000)
+
+      const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: permanentLive.rtmpUrl, streamKey: permanentLive.streamKey })
+      await server.live.waitUntilPublished({ videoId: permanentLiveId })
+
+      const video = await server.videos.getWithToken({ id: permanentLiveId })
+      const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid })
+      const hls = video.streamingPlaylists[0]
+
+      {
+        const query = { videoFileToken }
+        const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 })
+
+        expect(text).to.not.include(videoFileToken)
+      }
+
+      {
+        await checkVideoFileTokenReinjection({
+          server,
+          videoUUID: permanentLiveId,
+          videoFileToken,
+          resolutions: [ 720 ],
+          isLive: true
+        })
+      }
+
+      await stopFfmpeg(ffmpegCommand)
+    })
+
     it('Should have created a replay of the normal live with a private static path', async function () {
       this.timeout(240000)
 

server/tests/shared/checks.ts

@@ -23,6 +23,12 @@ function expectNotStartWith (str: string, start: string) {
   expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.false
 }
 
+function expectEndWith (str: string, end: string) {
+  expect(str.endsWith(end), `${str} does not end with ${end}`).to.be.true
+}
+
+// ---------------------------------------------------------------------------
+
 async function expectLogDoesNotContain (server: PeerTubeServer, str: string) {
   const content = await server.servers.getLogContent()
 
@@ -103,6 +109,7 @@ export {
   testFileExistsOrNot,
   expectStartWith,
   expectNotStartWith,
+  expectEndWith,
   checkBadStartPagination,
   checkBadCountPagination,
   checkBadSortPagination,

server/tests/shared/index.ts

@@ -6,7 +6,7 @@ export * from './directories'
 export * from './generate'
 export * from './live'
 export * from './notifications'
-export * from './playlists'
+export * from './video-playlists'
 export * from './plugins'
 export * from './requests'
 export * from './streaming-playlists'

server/tests/shared/streaming-playlists.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
 import { expect } from 'chai'
-import { basename } from 'path'
+import { basename, dirname, join } from 'path'
 import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
 import { sha256 } from '@shared/extra-utils'
 import { HttpStatusCode, VideoStreamingPlaylist, VideoStreamingPlaylistType } from '@shared/models'
@@ -188,9 +188,55 @@ async function completeCheckHlsPlaylist (options: {
   }
 }
 
+async function checkVideoFileTokenReinjection (options: {
+  server: PeerTubeServer
+  videoUUID: string
+  videoFileToken: string
+  resolutions: number[]
+  isLive: boolean
+}) {
+  const { server, resolutions, videoFileToken, videoUUID, isLive } = options
+
+  const video = await server.videos.getWithToken({ id: videoUUID })
+  const hls = video.streamingPlaylists[0]
+
+  const query = { videoFileToken, reinjectVideoFileToken: 'true' }
+  const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 })
+
+  for (let i = 0; i < resolutions.length; i++) {
+    const resolution = resolutions[i]
+
+    const suffix = isLive
+      ? i
+      : `-${resolution}`
+
+    expect(text).to.contain(`${suffix}.m3u8?videoFileToken=${videoFileToken}`)
+  }
+
+  const resolutionPlaylists = extractResolutionPlaylistUrls(hls.playlistUrl, text)
+  expect(resolutionPlaylists).to.have.lengthOf(resolutions.length)
+
+  for (const url of resolutionPlaylists) {
+    const { text } = await makeRawRequest({ url, query, expectedStatus: HttpStatusCode.OK_200 })
+
+    const extension = isLive
+      ? '.ts'
+      : '.mp4'
+
+    expect(text).to.contain(`${extension}?videoFileToken=${videoFileToken}`)
+  }
+}
+
+function extractResolutionPlaylistUrls (masterPath: string, masterContent: string) {
+  return masterContent.match(/^([^.]+\.m3u8.*)/mg)
+    .map(filename => join(dirname(masterPath), filename))
+}
+
 export {
   checkSegmentHash,
   checkLiveSegmentHash,
   checkResolutionsInMasterPlaylist,
-  completeCheckHlsPlaylist
+  completeCheckHlsPlaylist,
+  extractResolutionPlaylistUrls,
+  checkVideoFileTokenReinjection
 }