Merge branch 'develop' into pr/1285

shared/models/activitypub/activity.ts

@@ -5,12 +5,14 @@ import { DislikeObject } from './objects/dislike-object'
 import { VideoAbuseObject } from './objects/video-abuse-object'
 import { VideoCommentObject } from './objects/video-comment-object'
 import { ViewObject } from './objects/view-object'
+import { APObject } from './objects/object.model'
 
 export type Activity = ActivityCreate | ActivityUpdate |
   ActivityDelete | ActivityFollow | ActivityAccept | ActivityAnnounce |
-  ActivityUndo | ActivityLike | ActivityReject
+  ActivityUndo | ActivityLike | ActivityReject | ActivityView | ActivityDislike | ActivityFlag
 
-export type ActivityType = 'Create' | 'Update' | 'Delete' | 'Follow' | 'Accept' | 'Announce' | 'Undo' | 'Like' | 'Reject'
+export type ActivityType = 'Create' | 'Update' | 'Delete' | 'Follow' | 'Accept' | 'Announce' | 'Undo' | 'Like' | 'Reject' |
+  'View' | 'Dislike' | 'Flag'
 
 export interface ActivityAudience {
   to: string[]
@@ -59,15 +61,34 @@ export interface ActivityReject extends BaseActivity {
 
 export interface ActivityAnnounce extends BaseActivity {
   type: 'Announce'
-  object: string | { id: string }
+  object: APObject
 }
 
 export interface ActivityUndo extends BaseActivity {
   type: 'Undo',
-  object: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce
+  object: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce
 }
 
 export interface ActivityLike extends BaseActivity {
   type: 'Like',
-  object: string
+  object: APObject
+}
+
+export interface ActivityView extends BaseActivity {
+  type: 'View',
+  actor: string
+  object: APObject
+}
+
+export interface ActivityDislike extends BaseActivity {
+  id: string
+  type: 'Dislike'
+  actor: string
+  object: APObject
+}
+
+export interface ActivityFlag extends BaseActivity {
+  type: 'Flag',
+  content: string,
+  object: APObject
 }

shared/models/activitypub/activitypub-ordered-collection.ts

@@ -2,6 +2,9 @@ export interface ActivityPubOrderedCollection<T> {
   '@context': string[]
   type: 'OrderedCollection' | 'OrderedCollectionPage'
   totalItems: number
-  partOf?: string
   orderedItems: T[]
+
+  partOf?: string
+  next?: string
+  first?: string
 }

shared/models/activitypub/objects/cache-file-object.ts

@@ -1,9 +1,9 @@
-import { ActivityVideoUrlObject } from './common-objects'
+import { ActivityVideoUrlObject, ActivityPlaylistUrlObject } from './common-objects'
 
 export interface CacheFileObject {
   id: string
   type: 'CacheFile',
   object: string
   expires: string
-  url: ActivityVideoUrlObject
+  url: ActivityVideoUrlObject | ActivityPlaylistUrlObject
 }

shared/models/activitypub/objects/common-objects.ts

@@ -28,25 +28,47 @@ export type ActivityVideoUrlObject = {
   fps: number
 }
 
-export type ActivityUrlObject =
-  ActivityVideoUrlObject
-  |
-  {
-    type: 'Link'
-    // TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
-    mimeType?: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
-    mediaType: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
-    href: string
-    height: number
-  }
-  |
-  {
-    type: 'Link'
-    // TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
-    mimeType?: 'text/html'
-    mediaType: 'text/html'
-    href: string
-  }
+export type ActivityPlaylistSegmentHashesObject = {
+  type: 'Link'
+  name: 'sha256'
+  // TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
+  mimeType?: 'application/json'
+  mediaType: 'application/json'
+  href: string
+}
+
+export type ActivityPlaylistInfohashesObject = {
+  type: 'Infohash'
+  name: string
+}
+
+export type ActivityPlaylistUrlObject = {
+  type: 'Link'
+  // TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
+  mimeType?: 'application/x-mpegURL'
+  mediaType: 'application/x-mpegURL'
+  href: string
+  tag?: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[]
+}
+
+export type ActivityBitTorrentUrlObject = {
+  type: 'Link'
+  // TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
+  mimeType?: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
+  mediaType: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
+  href: string
+  height: number
+}
+
+export type ActivityHtmlUrlObject = {
+  type: 'Link'
+  // TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
+  mimeType?: 'text/html'
+  mediaType: 'text/html'
+  href: string
+}
+
+export type ActivityUrlObject = ActivityVideoUrlObject | ActivityPlaylistUrlObject | ActivityBitTorrentUrlObject | ActivityHtmlUrlObject
 
 export interface ActivityPubAttributedTo {
   type: 'Group' | 'Person'

shared/models/activitypub/objects/object.model.ts

@@ -0,0 +1 @@
+export type APObject = string | { id: string }

shared/models/activitypub/objects/video-torrent-object.ts

@@ -20,7 +20,8 @@ export interface VideoTorrentObject {
   subtitleLanguage: ActivityIdentifierObject[]
   views: number
   sensitive: boolean
-  commentsEnabled: boolean
+  commentsEnabled: boolean,
+  downloadEnabled: boolean,
   waitTranscoding: boolean
   state: VideoState
   published: string

shared/models/actors/actor.model.ts

@@ -10,5 +10,5 @@ export interface Actor {
   followersCount: number
   createdAt: Date | string
   updatedAt: Date | string
-  avatar: Avatar
+  avatar?: Avatar
 }

shared/models/server/custom-config.model.ts

@@ -61,6 +61,9 @@ export interface CustomConfig {
       '720p': boolean
       '1080p': boolean
     }
+    hls: {
+      enabled: boolean
+    }
   }
 
   import: {

shared/models/server/server-config.model.ts

@@ -25,11 +25,15 @@ export interface ServerConfig {
 
   signup: {
     allowed: boolean,
-    allowedForCurrentIP: boolean,
+    allowedForCurrentIP: boolean
     requiresEmailVerification: boolean
   }
 
   transcoding: {
+    hls: {
+      enabled: boolean
+    }
+
     enabledResolutions: number[]
   }
 
@@ -48,7 +52,7 @@ export interface ServerConfig {
     file: {
       size: {
         max: number
-      },
+      }
       extensions: string[]
     }
   }
@@ -78,4 +82,10 @@ export interface ServerConfig {
     videoQuota: number
     videoQuotaDaily: number
   }
+
+  trending: {
+    videos: {
+      intervalDays: number
+    }
+  }
 }

shared/models/server/server-stats.model.ts

@@ -5,6 +5,7 @@ export interface ServerStats {
   totalLocalVideos: number
   totalLocalVideoViews: number
   totalLocalVideoComments: number
+  totalLocalVideoFilesSize: number
 
   totalVideos: number
   totalVideoComments: number

shared/models/users/user-notification.model.ts

@@ -22,16 +22,23 @@ export interface VideoInfo {
   name: string
 }
 
+export interface ActorInfo {
+  id: number
+  displayName: string
+  name: string
+  host: string
+  avatar?: {
+    path: string
+  }
+}
+
 export interface UserNotification {
   id: number
   type: UserNotificationType
   read: boolean
 
   video?: VideoInfo & {
-    channel: {
-      id: number
-      displayName: string
-    }
+    channel: ActorInfo
   }
 
   videoImport?: {
@@ -45,10 +52,7 @@ export interface UserNotification {
   comment?: {
     id: number
     threadId: number
-    account: {
-      id: number
-      displayName: string
-    }
+    account: ActorInfo
     video: VideoInfo
   }
 
@@ -62,18 +66,11 @@ export interface UserNotification {
     video: VideoInfo
   }
 
-  account?: {
-    id: number
-    displayName: string
-    name: string
-  }
+  account?: ActorInfo
 
   actorFollow?: {
     id: number
-    follower: {
-      name: string
-      displayName: string
-    }
+    follower: ActorInfo
     following: {
       type: 'account' | 'channel'
       name: string

shared/models/users/user-update.model.ts

@@ -1,6 +1,7 @@
 import { UserRole } from './user-role'
 
 export interface UserUpdate {
+  password?: string
   email?: string
   emailVerified?: boolean
   videoQuota?: number

shared/models/videos/blacklist/video-blacklist-create.model.ts

@@ -1,3 +1,4 @@
 export interface VideoBlacklistCreate {
   reason?: string
+  unfederate?: boolean
 }

shared/models/videos/blacklist/video-blacklist.model.ts

@@ -2,6 +2,7 @@ export interface VideoBlacklist {
   id: number
   createdAt: Date
   updatedAt: Date
+  unfederated: boolean
   reason?: string
 
   video: {

shared/models/videos/video-create.model.ts

@@ -13,6 +13,7 @@ export interface VideoCreate {
   name: string
   tags?: string[]
   commentsEnabled?: boolean
+  downloadEnabled?: boolean
   privacy: VideoPrivacy
   scheduleUpdate?: VideoScheduleUpdate
   originallyPublishedAt: Date | string

shared/models/videos/video-streaming-playlist.model.ts

@@ -0,0 +1,12 @@
+import { VideoStreamingPlaylistType } from './video-streaming-playlist.type'
+
+export class VideoStreamingPlaylist {
+  id: number
+  type: VideoStreamingPlaylistType
+  playlistUrl: string
+  segmentsSha256Url: string
+
+  redundancies: {
+    baseUrl: string
+  }[]
+}

shared/models/videos/video-streaming-playlist.type.ts

@@ -0,0 +1,3 @@
+export enum VideoStreamingPlaylistType {
+  HLS = 1
+}

shared/models/videos/video-update.model.ts

@@ -11,6 +11,7 @@ export interface VideoUpdate {
   privacy?: VideoPrivacy
   tags?: string[]
   commentsEnabled?: boolean
+  downloadEnabled?: boolean
   nsfw?: boolean
   waitTranscoding?: boolean
   channelId?: number

shared/models/videos/video.model.ts

@@ -5,6 +5,7 @@ import { VideoChannel } from './channel/video-channel.model'
 import { VideoPrivacy } from './video-privacy.enum'
 import { VideoScheduleUpdate } from './video-schedule-update.model'
 import { VideoConstant } from './video-constant.model'
+import { VideoStreamingPlaylist } from './video-streaming-playlist.model'
 
 export interface VideoFile {
   magnetUri: string
@@ -24,7 +25,7 @@ export interface VideoChannelAttribute {
   displayName: string
   url: string
   host: string
-  avatar: Avatar
+  avatar?: Avatar
 }
 
 export interface AccountAttribute {
@@ -34,7 +35,7 @@ export interface AccountAttribute {
   displayName: string
   url: string
   host: string
-  avatar: Avatar
+  avatar?: Avatar
 }
 
 export interface Video {
@@ -83,8 +84,13 @@ export interface VideoDetails extends Video {
   files: VideoFile[]
   account: Account
   commentsEnabled: boolean
+  downloadEnabled: boolean
 
   // Not optional in details (unlike in Video)
   waitTranscoding: boolean
   state: VideoConstant<VideoState>
+
+  trackerUrls: string[]
+
+  streamingPlaylists: VideoStreamingPlaylist[]
 }

shared/utils/index.ts

@@ -17,6 +17,8 @@ export * from './users/users'
 export * from './videos/video-abuses'
 export * from './videos/video-blacklist'
 export * from './videos/video-channels'
+export * from './videos/video-comments'
+export * from './videos/video-playlists'
 export * from './videos/videos'
 export * from './videos/video-change-ownership'
 export * from './feeds/feeds'

shared/utils/requests/requests.ts

@@ -1,24 +1,32 @@
 import * as request from 'supertest'
 import { buildAbsoluteFixturePath, root } from '../miscs/miscs'
 import { isAbsolute, join } from 'path'
+import { parse } from 'url'
+
+function makeRawRequest (url: string, statusCodeExpected?: number, range?: string) {
+  const { host, protocol, pathname } = parse(url)
+
+  return makeGetRequest({ url: `${protocol}//${host}`, path: pathname, statusCodeExpected, range })
+}
 
 function makeGetRequest (options: {
   url: string,
-  path: string,
+  path?: string,
   query?: any,
   token?: string,
   statusCodeExpected?: number,
-  contentType?: string
+  contentType?: string,
+  range?: string
 }) {
   if (!options.statusCodeExpected) options.statusCodeExpected = 400
   if (options.contentType === undefined) options.contentType = 'application/json'
 
-  const req = request(options.url)
-    .get(options.path)
+  const req = request(options.url).get(options.path)
 
   if (options.contentType) req.set('Accept', options.contentType)
   if (options.token) req.set('Authorization', 'Bearer ' + options.token)
   if (options.query) req.query(options.query)
+  if (options.range) req.set('Range', options.range)
 
   return req.expect(options.statusCodeExpected)
 }
@@ -164,5 +172,6 @@ export {
   makePostBodyRequest,
   makePutBodyRequest,
   makeDeleteRequest,
+  makeRawRequest,
   updateAvatarRequest
 }

shared/utils/server/config.ts

@@ -97,6 +97,9 @@ function updateCustomSubConfig (url: string, token: string, newConfig: any) {
         '480p': true,
         '720p': false,
         '1080p': false
+      },
+      hls: {
+        enabled: false
       }
     },
     import: {

shared/utils/server/servers.ts

@@ -145,8 +145,16 @@ function runServer (serverNumber: number, configOverride?: Object, args = []) {
       if (dontContinue === true) return
 
       server.app.stdout.removeListener('data', onStdout)
+
+      process.on('exit', () => {
+        try {
+          process.kill(server.app.pid)
+        } catch { /* empty */ }
+      })
+
       res(server)
     })
+
   })
 }
 
@@ -158,9 +166,13 @@ async function reRunServer (server: ServerInfo, configOverride?: any) {
 }
 
 async function checkTmpIsEmpty (server: ServerInfo) {
+  return checkDirectoryIsEmpty(server, 'tmp')
+}
+
+async function checkDirectoryIsEmpty (server: ServerInfo, directory: string) {
   const testDirectory = 'test' + server.serverNumber
 
-  const directoryPath = join(root(), testDirectory, 'tmp')
+  const directoryPath = join(root(), testDirectory, directory)
 
   const directoryExists = existsSync(directoryPath)
   expect(directoryExists).to.be.true
@@ -191,6 +203,7 @@ async function waitUntilLog (server: ServerInfo, str: string, count = 1) {
 // ---------------------------------------------------------------------------
 
 export {
+  checkDirectoryIsEmpty,
   checkTmpIsEmpty,
   ServerInfo,
   flushAndRunMultipleServers,

shared/utils/users/user-notifications.ts

@@ -146,6 +146,7 @@ function checkVideo (video: any, videoName?: string, videoUUID?: string) {
 function checkActor (actor: any) {
   expect(actor.displayName).to.be.a('string')
   expect(actor.displayName).to.not.be.empty
+  expect(actor.host).to.not.be.undefined
 }
 
 function checkComment (comment: any, commentId: number, threadId: number) {
@@ -273,8 +274,8 @@ async function checkNewActorFollow (
       checkActor(notification.actorFollow.follower)
       expect(notification.actorFollow.follower.displayName).to.equal(followerDisplayName)
       expect(notification.actorFollow.follower.name).to.equal(followerName)
+      expect(notification.actorFollow.follower.host).to.not.be.undefined
 
-      checkActor(notification.actorFollow.following)
       expect(notification.actorFollow.following.displayName).to.equal(followingDisplayName)
       expect(notification.actorFollow.following.type).to.equal(followType)
     } else {

shared/utils/users/users.ts

@@ -213,11 +213,13 @@ function updateUser (options: {
   emailVerified?: boolean,
   videoQuota?: number,
   videoQuotaDaily?: number,
+  password?: string,
   role?: UserRole
 }) {
   const path = '/api/v1/users/' + options.userId
 
   const toSend = {}
+  if (options.password !== undefined && options.password !== null) toSend['password'] = options.password
   if (options.email !== undefined && options.email !== null) toSend['email'] = options.email
   if (options.emailVerified !== undefined && options.emailVerified !== null) toSend['emailVerified'] = options.emailVerified
   if (options.videoQuota !== undefined && options.videoQuota !== null) toSend['videoQuota'] = options.videoQuota

shared/utils/videos/video-blacklist.ts

@@ -1,11 +1,18 @@
 import * as request from 'supertest'
 
-function addVideoToBlacklist (url: string, token: string, videoId: number | string, reason?: string, specialStatus = 204) {
+function addVideoToBlacklist (
+  url: string,
+  token: string,
+  videoId: number | string,
+  reason?: string,
+  unfederate?: boolean,
+  specialStatus = 204
+) {
   const path = '/api/v1/videos/' + videoId + '/blacklist'
 
   return request(url)
           .post(path)
-          .send({ reason })
+          .send({ reason, unfederate })
           .set('Accept', 'application/json')
           .set('Authorization', 'Bearer ' + token)
           .expect(specialStatus)

shared/utils/videos/video-playlists.ts

@@ -0,0 +1,51 @@
+import { makeRawRequest } from '../requests/requests'
+import { sha256 } from '../../../server/helpers/core-utils'
+import { VideoStreamingPlaylist } from '../../models/videos/video-streaming-playlist.model'
+import { expect } from 'chai'
+
+function getPlaylist (url: string, statusCodeExpected = 200) {
+  return makeRawRequest(url, statusCodeExpected)
+}
+
+function getSegment (url: string, statusCodeExpected = 200, range?: string) {
+  return makeRawRequest(url, statusCodeExpected, range)
+}
+
+function getSegmentSha256 (url: string, statusCodeExpected = 200) {
+  return makeRawRequest(url, statusCodeExpected)
+}
+
+async function checkSegmentHash (
+  baseUrlPlaylist: string,
+  baseUrlSegment: string,
+  videoUUID: string,
+  resolution: number,
+  hlsPlaylist: VideoStreamingPlaylist
+) {
+  const res = await getPlaylist(`${baseUrlPlaylist}/${videoUUID}/${resolution}.m3u8`)
+  const playlist = res.text
+
+  const videoName = `${videoUUID}-${resolution}-fragmented.mp4`
+
+  const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist)
+
+  const length = parseInt(matches[1], 10)
+  const offset = parseInt(matches[2], 10)
+  const range = `${offset}-${offset + length - 1}`
+
+  const res2 = await getSegment(`${baseUrlSegment}/${videoUUID}/${videoName}`, 206, `bytes=${range}`)
+
+  const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url)
+
+  const sha256Server = resSha.body[ videoName ][range]
+  expect(sha256(res2.body)).to.equal(sha256Server)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getPlaylist,
+  getSegment,
+  getSegmentSha256,
+  checkSegmentHash
+}

shared/utils/videos/videos.ts

@@ -28,6 +28,7 @@ type VideoAttributes = {
   language?: string
   nsfw?: boolean
   commentsEnabled?: boolean
+  downloadEnabled?: boolean
   waitTranscoding?: boolean
   description?: string
   tags?: string[]
@@ -271,7 +272,16 @@ function removeVideo (url: string, token: string, id: number | string, expectedS
 async function checkVideoFilesWereRemoved (
   videoUUID: string,
   serverNumber: number,
-  directories = [ 'redundancy', 'videos', 'thumbnails', 'torrents', 'previews', 'captions' ]
+  directories = [
+    'redundancy',
+    'videos',
+    'thumbnails',
+    'torrents',
+    'previews',
+    'captions',
+    join('playlists', 'hls'),
+    join('redundancy', 'hls')
+  ]
 ) {
   const testDirectory = 'test' + serverNumber
 
@@ -279,7 +289,7 @@ async function checkVideoFilesWereRemoved (
     const directoryPath = join(root(), testDirectory, directory)
 
     const directoryExists = existsSync(directoryPath)
-    expect(directoryExists).to.be.true
+    if (!directoryExists) continue
 
     const files = await readdir(directoryPath)
     for (const file of files) {
@@ -311,6 +321,7 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg
     tags: [ 'tag' ],
     privacy: VideoPrivacy.PUBLIC,
     commentsEnabled: true,
+    downloadEnabled: true,
     fixture: 'video_short.webm'
   }, videoAttributesArg)
 
@@ -321,6 +332,7 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg
               .field('name', attributes.name)
               .field('nsfw', JSON.stringify(attributes.nsfw))
               .field('commentsEnabled', JSON.stringify(attributes.commentsEnabled))
+              .field('downloadEnabled', JSON.stringify(attributes.downloadEnabled))
               .field('waitTranscoding', JSON.stringify(attributes.waitTranscoding))
               .field('privacy', attributes.privacy.toString())
               .field('channelId', attributes.channelId)
@@ -371,6 +383,7 @@ function updateVideo (url: string, accessToken: string, id: number | string, att
   if (attributes.language) body['language'] = attributes.language
   if (attributes.nsfw !== undefined) body['nsfw'] = JSON.stringify(attributes.nsfw)
   if (attributes.commentsEnabled !== undefined) body['commentsEnabled'] = JSON.stringify(attributes.commentsEnabled)
+  if (attributes.downloadEnabled !== undefined) body['downloadEnabled'] = JSON.stringify(attributes.downloadEnabled)
   if (attributes.description) body['description'] = attributes.description
   if (attributes.tags) body['tags'] = attributes.tags
   if (attributes.privacy) body['privacy'] = attributes.privacy
@@ -436,6 +449,7 @@ async function completeVideoCheck (
     language: string
     nsfw: boolean
     commentsEnabled: boolean
+    downloadEnabled: boolean
     description: string
     publishedAt?: string
     support: string
@@ -510,6 +524,7 @@ async function completeVideoCheck (
   expect(dateIsValid(videoDetails.channel.createdAt.toString())).to.be.true
   expect(dateIsValid(videoDetails.channel.updatedAt.toString())).to.be.true
   expect(videoDetails.commentsEnabled).to.equal(attributes.commentsEnabled)
+  expect(videoDetails.downloadEnabled).to.equal(attributes.downloadEnabled)
 
   for (const attributeFile of attributes.files) {
     const file = videoDetails.files.find(f => f.resolution.id === attributeFile.resolution)