chore(refactor): remove shared folder dependencies to the server

Many files from the `shared` folder were importing files from the `server` folder.
When attempting to use Typescript project references to describe dependencies,
it highlighted a circular dependency beetween `shared` <-> `server`.

The Typescript project forbid such usages.
Using project references greatly improve performance by rebuilding only
the updated project and not all source files.
> see https://www.typescriptlang.org/docs/handbook/project-references.html

.eslintrc.json

@@ -102,6 +102,9 @@
   "parserOptions": {
     "project": [
       "./tsconfig.json",
+      "./shared/tsconfig.json",
+      "./scripts/tsconfig.json",
+      "./server/tsconfig.json",
       "./server/tools/tsconfig.json"
     ]
   }

.gitignore

@@ -50,3 +50,6 @@ yarn-error.log
 /server/tools/import-mediacore.ts
 /docker-volume/
 /init.mp4
+
+# TypeScript
+*.tsbuildinfo

scripts/build/server.sh

@@ -4,7 +4,10 @@ set -eu
 
 rm -rf ./dist
 
-npm run tsc
-cp "./tsconfig.json" "./dist"
+npm run tsc -- -b --verbose
+cp "./tsconfig.base.json" "./tsconfig.json" "./dist/"
+cp "./scripts/tsconfig.json" "./dist/scripts/"
+cp "./server/tsconfig.json" "./dist/server/"
+cp "./shared/tsconfig.json" "./dist/shared/"
 cp -r "./server/static" "./server/assets" "./dist/server"
 cp -r "./server/lib/emails" "./dist/server/lib"

scripts/client-build-stats.ts

@@ -3,7 +3,7 @@ registerTSPaths()
 
 import { readdir, stat } from 'fs-extra'
 import { join } from 'path'
-import { root } from '@server/helpers/core-utils'
+import { root } from '@shared/core-utils'
 
 async function run () {
   const result = {

scripts/setup/cli.sh

@@ -11,6 +11,6 @@ rm -rf ./dist/server/tools/
     yarn install --pure-lockfile
 )
 
-npm run tsc -- --build ./server/tools/tsconfig.json
+npm run tsc -- --build --verbose ./server/tools/tsconfig.json
 cp -r "./server/tools/node_modules" "./dist/server/tools"
 cp "./tsconfig.json" "./dist"

scripts/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "../dist/scripts",
+  },
+  "references": [
+    { "path": "../shared" },
+    { "path": "../server" }
+  ]
+}

server/controllers/api/users/token.ts

@@ -1,7 +1,7 @@
 import express from 'express'
 import RateLimit from 'express-rate-limit'
 import { logger } from '@server/helpers/logger'
-import { buildUUID } from '@server/helpers/uuid'
+import { buildUUID } from '@shared/core-utils/uuid'
 import { CONFIG } from '@server/initializers/config'
 import { getAuthNameFromRefreshGrant, getBypassFromExternalAuth, getBypassFromPasswordGrant } from '@server/lib/auth/external-auth'
 import { handleOAuthToken } from '@server/lib/auth/oauth'

server/controllers/api/video-playlist.ts

@@ -1,6 +1,6 @@
 import express from 'express'
 import { join } from 'path'
-import { uuidToShort } from '@server/helpers/uuid'
+import { uuidToShort } from '@shared/core-utils/uuid'
 import { scheduleRefreshIfNeeded } from '@server/lib/activitypub/playlists'
 import { Hooks } from '@server/lib/plugins/hooks'
 import { getServerActor } from '@server/models/application/application'

server/controllers/api/videos/live.ts

@@ -1,6 +1,6 @@
 import express from 'express'
 import { createReqFiles } from '@server/helpers/express-utils'
-import { buildUUID, uuidToShort } from '@server/helpers/uuid'
+import { buildUUID, uuidToShort } from '@shared/core-utils/uuid'
 import { CONFIG } from '@server/initializers/config'
 import { ASSETS_PATH, MIMETYPES } from '@server/initializers/constants'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'

server/controllers/api/videos/upload.ts

@@ -1,9 +1,9 @@
 import express from 'express'
 import { move } from 'fs-extra'
 import { basename } from 'path'
-import { getLowercaseExtension } from '@server/helpers/core-utils'
+import { getLowercaseExtension } from '@shared/core-utils'
 import { getResumableUploadPath } from '@server/helpers/upload'
-import { uuidToShort } from '@server/helpers/uuid'
+import { uuidToShort } from '@shared/core-utils/uuid'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
 import { generateWebTorrentVideoFilename } from '@server/lib/paths'

server/controllers/client.ts

@@ -7,7 +7,7 @@ import { CONFIG } from '@server/initializers/config'
 import { Hooks } from '@server/lib/plugins/hooks'
 import { buildFileLocale, getCompleteLocale, is18nLocale, LOCALE_FILES } from '@shared/core-utils/i18n'
 import { HttpStatusCode } from '@shared/models'
-import { root } from '../helpers/core-utils'
+import { root } from '@shared/core-utils'
 import { STATIC_MAX_AGE } from '../initializers/constants'
 import { ClientHtml, sendHTML, serveIndexHTML } from '../lib/client-html'
 import { asyncMiddleware, embedCSP } from '../middlewares'

server/controllers/static.ts

@@ -5,7 +5,7 @@ import { serveIndexHTML } from '@server/lib/client-html'
 import { ServerConfigManager } from '@server/lib/server-config-manager'
 import { HttpStatusCode } from '@shared/models'
 import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo/nodeinfo.model'
-import { root } from '../helpers/core-utils'
+import { root } from '@shared/core-utils'
 import { CONFIG, isEmailEnabled } from '../initializers/config'
 import {
   CONSTRAINTS_FIELDS,

server/helpers/core-utils.ts

@@ -6,9 +6,8 @@
 */
 
 import { exec, ExecOptions } from 'child_process'
-import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto'
+import { randomBytes } from 'crypto'
 import { truncate } from 'lodash'
-import { basename, extname, isAbsolute, join, resolve } from 'path'
 import { createPrivateKey as createPrivateKey_1, getPublicKey as getPublicKey_1 } from 'pem'
 import { pipeline } from 'stream'
 import { URL } from 'url'
@@ -159,34 +158,6 @@ function getAppNumber () {
 
 // ---------------------------------------------------------------------------
 
-let rootPath: string
-
-function root () {
-  if (rootPath) return rootPath
-
-  rootPath = __dirname
-
-  if (basename(rootPath) === 'helpers') rootPath = resolve(rootPath, '..')
-  if (basename(rootPath) === 'server') rootPath = resolve(rootPath, '..')
-  if (basename(rootPath) === 'dist') rootPath = resolve(rootPath, '..')
-
-  return rootPath
-}
-
-function buildPath (path: string) {
-  if (isAbsolute(path)) return path
-
-  return join(root(), path)
-}
-
-function getLowercaseExtension (filename: string) {
-  const ext = extname(filename) || ''
-
-  return ext.toLowerCase()
-}
-
-// ---------------------------------------------------------------------------
-
 // Consistent with .length, lodash truncate function is not
 function peertubeTruncate (str: string, options: { length: number, separator?: RegExp, omission?: string }) {
   const truncatedStr = truncate(str, options)
@@ -221,16 +192,6 @@ function parseSemVersion (s: string) {
 
 // ---------------------------------------------------------------------------
 
-function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
-  return createHash('sha256').update(str).digest(encoding)
-}
-
-function sha1 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
-  return createHash('sha1').update(str).digest(encoding)
-}
-
-// ---------------------------------------------------------------------------
-
 function execShell (command: string, options?: ExecOptions) {
   return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => {
     exec(command, options, (err, stdout, stderr) => {
@@ -298,9 +259,6 @@ export {
   objectConverter,
   mapToJSON,
 
-  root,
-  buildPath,
-  getLowercaseExtension,
   sanitizeUrl,
   sanitizeHost,
 
@@ -309,9 +267,6 @@ export {
   pageToStartAndCount,
   peertubeTruncate,
 
-  sha256,
-  sha1,
-
   promisify0,
   promisify1,
   promisify2,

server/helpers/custom-validators/misc.ts

@@ -2,7 +2,7 @@ import 'multer'
 import { UploadFilesForCheck } from 'express'
 import { sep } from 'path'
 import validator from 'validator'
-import { isShortUUID, shortToUUID } from '../uuid'
+import { isShortUUID, shortToUUID } from '@shared/core-utils/uuid'
 
 function exists (value: any) {
   return value !== undefined && value !== null

server/helpers/express-utils.ts

@@ -1,9 +1,9 @@
-import express from 'express'
+import express, { RequestHandler } from 'express'
 import multer, { diskStorage } from 'multer'
 import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
 import { CONFIG } from '../initializers/config'
 import { REMOTE_SCHEME } from '../initializers/constants'
-import { getLowercaseExtension } from './core-utils'
+import { getLowercaseExtension } from '@shared/core-utils'
 import { isArray } from './custom-validators/misc'
 import { logger } from './logger'
 import { deleteFileAndCatch, generateRandomString } from './utils'
@@ -69,7 +69,7 @@ function createReqFiles (
   fieldNames: string[],
   mimeTypes: { [id: string]: string | string[] },
   destinations: { [fieldName: string]: string }
-) {
+): RequestHandler {
   const storage = diskStorage({
     destination: (req, file, cb) => {
       cb(null, destinations[file.fieldname])

server/helpers/ffprobe-utils.ts

@@ -1,9 +1,22 @@
-import { ffprobe, FfprobeData } from 'fluent-ffmpeg'
+import { FfprobeData } from 'fluent-ffmpeg'
 import { getMaxBitrate } from '@shared/core-utils'
-import { VideoFileMetadata, VideoResolution, VideoTranscodingFPS } from '../../shared/models/videos'
+import { VideoResolution, VideoTranscodingFPS } from '../../shared/models/videos'
 import { CONFIG } from '../initializers/config'
 import { VIDEO_TRANSCODING_FPS } from '../initializers/constants'
 import { logger } from './logger'
+import {
+  canDoQuickAudioTranscode,
+  ffprobePromise,
+  getDurationFromVideoFile,
+  getAudioStream,
+  getMaxAudioBitrate,
+  getMetadataFromFile,
+  getVideoFileBitrate,
+  getVideoFileFPS,
+  getVideoFileResolution,
+  getVideoStreamFromFile,
+  getVideoStreamSize
+} from '@shared/extra-utils/ffprobe'
 
 /**
  *
@@ -11,79 +24,6 @@ import { logger } from './logger'
  *
  */
 
-function ffprobePromise (path: string) {
-  return new Promise<FfprobeData>((res, rej) => {
-    ffprobe(path, (err, data) => {
-      if (err) return rej(err)
-
-      return res(data)
-    })
-  })
-}
-
-async function getAudioStream (videoPath: string, existingProbe?: FfprobeData) {
-  // without position, ffprobe considers the last input only
-  // we make it consider the first input only
-  // if you pass a file path to pos, then ffprobe acts on that file directly
-  const data = existingProbe || await ffprobePromise(videoPath)
-
-  if (Array.isArray(data.streams)) {
-    const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio')
-
-    if (audioStream) {
-      return {
-        absolutePath: data.format.filename,
-        audioStream,
-        bitrate: parseInt(audioStream['bit_rate'] + '', 10)
-      }
-    }
-  }
-
-  return { absolutePath: data.format.filename }
-}
-
-function getMaxAudioBitrate (type: 'aac' | 'mp3' | string, bitrate: number) {
-  const maxKBitrate = 384
-  const kToBits = (kbits: number) => kbits * 1000
-
-  // If we did not manage to get the bitrate, use an average value
-  if (!bitrate) return 256
-
-  if (type === 'aac') {
-    switch (true) {
-      case bitrate > kToBits(maxKBitrate):
-        return maxKBitrate
-
-      default:
-        return -1 // we interpret it as a signal to copy the audio stream as is
-    }
-  }
-
-  /*
-    a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac.
-    That's why, when using aac, we can go to lower kbit/sec. The equivalences
-    made here are not made to be accurate, especially with good mp3 encoders.
-    */
-  switch (true) {
-    case bitrate <= kToBits(192):
-      return 128
-
-    case bitrate <= kToBits(384):
-      return 256
-
-    default:
-      return maxKBitrate
-  }
-}
-
-async function getVideoStreamSize (path: string, existingProbe?: FfprobeData): Promise<{ width: number, height: number }> {
-  const videoStream = await getVideoStreamFromFile(path, existingProbe)
-
-  return videoStream === null
-    ? { width: 0, height: 0 }
-    : { width: videoStream.width, height: videoStream.height }
-}
-
 async function getVideoStreamCodec (path: string) {
   const videoStream = await getVideoStreamFromFile(path)
 
@@ -143,69 +83,6 @@ async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) {
   return 'mp4a.40.2' // Fallback
 }
 
-async function getVideoFileResolution (path: string, existingProbe?: FfprobeData) {
-  const size = await getVideoStreamSize(path, existingProbe)
-
-  return {
-    width: size.width,
-    height: size.height,
-    ratio: Math.max(size.height, size.width) / Math.min(size.height, size.width),
-    resolution: Math.min(size.height, size.width),
-    isPortraitMode: size.height > size.width
-  }
-}
-
-async function getVideoFileFPS (path: string, existingProbe?: FfprobeData) {
-  const videoStream = await getVideoStreamFromFile(path, existingProbe)
-  if (videoStream === null) return 0
-
-  for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) {
-    const valuesText: string = videoStream[key]
-    if (!valuesText) continue
-
-    const [ frames, seconds ] = valuesText.split('/')
-    if (!frames || !seconds) continue
-
-    const result = parseInt(frames, 10) / parseInt(seconds, 10)
-    if (result > 0) return Math.round(result)
-  }
-
-  return 0
-}
-
-async function getMetadataFromFile (path: string, existingProbe?: FfprobeData) {
-  const metadata = existingProbe || await ffprobePromise(path)
-
-  return new VideoFileMetadata(metadata)
-}
-
-async function getVideoFileBitrate (path: string, existingProbe?: FfprobeData): Promise<number> {
-  const metadata = await getMetadataFromFile(path, existingProbe)
-
-  let bitrate = metadata.format.bit_rate as number
-  if (bitrate && !isNaN(bitrate)) return bitrate
-
-  const videoStream = await getVideoStreamFromFile(path, existingProbe)
-  if (!videoStream) return undefined
-
-  bitrate = videoStream?.bit_rate
-  if (bitrate && !isNaN(bitrate)) return bitrate
-
-  return undefined
-}
-
-async function getDurationFromVideoFile (path: string, existingProbe?: FfprobeData) {
-  const metadata = await getMetadataFromFile(path, existingProbe)
-
-  return Math.round(metadata.format.duration)
-}
-
-async function getVideoStreamFromFile (path: string, existingProbe?: FfprobeData) {
-  const metadata = await getMetadataFromFile(path, existingProbe)
-
-  return metadata.streams.find(s => s.codec_type === 'video') || null
-}
-
 function computeLowerResolutionsToTranscode (videoFileResolution: number, type: 'vod' | 'live') {
   const configResolutions = type === 'vod'
     ? CONFIG.TRANSCODING.RESOLUTIONS
@@ -263,26 +140,6 @@ async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Pro
   return true
 }
 
-async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise<boolean> {
-  const parsedAudio = await getAudioStream(path, probe)
-
-  if (!parsedAudio.audioStream) return true
-
-  if (parsedAudio.audioStream['codec_name'] !== 'aac') return false
-
-  const audioBitrate = parsedAudio.bitrate
-  if (!audioBitrate) return false
-
-  const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate)
-  if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false
-
-  const channelLayout = parsedAudio.audioStream['channel_layout']
-  // Causes playback issues with Chrome
-  if (!channelLayout || channelLayout === 'unknown') return false
-
-  return true
-}
-
 function getClosestFramerateStandard <K extends keyof Pick<VideoTranscodingFPS, 'HD_STANDARD' | 'STANDARD'>> (fps: number, type: K) {
   return VIDEO_TRANSCODING_FPS[type].slice(0)
                                     .sort((a, b) => fps % a - fps % b)[0]

server/helpers/image-utils.ts

@@ -1,9 +1,9 @@
 import { copy, readFile, remove, rename } from 'fs-extra'
 import Jimp, { read } from 'jimp'
-import { getLowercaseExtension } from './core-utils'
+import { getLowercaseExtension } from '@shared/core-utils'
 import { convertWebPToJPG, processGIF } from './ffmpeg-utils'
 import { logger } from './logger'
-import { buildUUID } from './uuid'
+import { buildUUID } from '@shared/core-utils/uuid'
 
 function generateImageFilename (extension = '.jpg') {
   return buildUUID() + extension

server/helpers/peertube-crypto.ts

@@ -4,7 +4,8 @@ import { Request } from 'express'
 import { cloneDeep } from 'lodash'
 import { BCRYPT_SALT_SIZE, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants'
 import { MActor } from '../types/models'
-import { createPrivateKey, getPublicKey, promisify1, promisify2, sha256 } from './core-utils'
+import { sha256 } from '@shared/core-utils/crypto'
+import { createPrivateKey, getPublicKey, promisify1, promisify2 } from './core-utils'
 import { jsonld } from './custom-jsonld-signature'
 import { logger } from './logger'
 

server/helpers/utils.ts

@@ -3,7 +3,8 @@ import { Instance as ParseTorrent } from 'parse-torrent'
 import { join } from 'path'
 import { ResultList } from '../../shared'
 import { CONFIG } from '../initializers/config'
-import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils'
+import { sha256 } from '@shared/core-utils/crypto'
+import { execPromise, execPromise2, randomBytesPromise } from './core-utils'
 import { logger } from './logger'
 
 function deleteFileAndCatch (path: string) {

server/helpers/webtorrent.ts

@@ -14,7 +14,8 @@ import { MVideo } from '@server/types/models/video/video'
 import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file'
 import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist'
 import { CONFIG } from '../initializers/config'
-import { promisify2, sha1 } from './core-utils'
+import { promisify2 } from './core-utils'
+import { sha1 } from '@shared/core-utils/crypto'
 import { logger } from './logger'
 import { generateVideoImportTmpPath } from './utils'
 import { extractVideo } from './video'

server/initializers/config.ts

@@ -6,7 +6,8 @@ import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-red
 import { BroadcastMessageLevel } from '@shared/models/server'
 import { VideoPrivacy, VideosRedundancyStrategy } from '../../shared/models'
 import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
-import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils'
+import { buildPath, root } from '../../shared/core-utils'
+import { parseBytes, parseDurationToMs } from '../helpers/core-utils'
 
 // Use a variable to reload the configuration if we need
 let config: IConfig = require('config')

server/initializers/constants.ts

@@ -18,8 +18,9 @@ import { FollowState } from '../../shared/models/actors'
 import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
 import { VideoPlaylistPrivacy } from '../../shared/models/videos/playlist/video-playlist-privacy.model'
 import { VideoPlaylistType } from '../../shared/models/videos/playlist/video-playlist-type.model'
+import { root } from '../../shared/core-utils'
 // Do not use barrels, remain constants as independent as possible
-import { isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
+import { isTestInstance, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
 import { CONFIG, registerConfigChangedHandler } from './config'
 
 // ---------------------------------------------------------------------------

server/initializers/migrations/0080-video-channels.ts

@@ -1,4 +1,4 @@
-import { buildUUID } from '@server/helpers/uuid'
+import { buildUUID } from '@shared/core-utils/uuid'
 import * as Sequelize from 'sequelize'
 
 async function up (utils: {

server/initializers/migrations/0345-video-playlists.ts

@@ -1,5 +1,5 @@
 import * as Sequelize from 'sequelize'
-import { buildUUID } from '@server/helpers/uuid'
+import { buildUUID } from '@shared/core-utils/uuid'
 import { VideoPlaylistPrivacy, VideoPlaylistType } from '../../../shared/models/videos'
 import { WEBSERVER } from '../constants'
 

server/initializers/migrations/0560-user-feed-token.ts

@@ -1,5 +1,5 @@
 import * as Sequelize from 'sequelize'
-import { buildUUID } from '@server/helpers/uuid'
+import { buildUUID } from '@shared/core-utils/uuid'
 
 async function up (utils: {
   transaction: Sequelize.Transaction

server/lib/activitypub/actors/shared/object-to-model-attributes.ts

@@ -1,6 +1,6 @@
-import { getLowercaseExtension } from '@server/helpers/core-utils'
+import { getLowercaseExtension } from '@shared/core-utils'
 import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
-import { buildUUID } from '@server/helpers/uuid'
+import { buildUUID } from '@shared/core-utils/uuid'
 import { MIMETYPES } from '@server/initializers/constants'
 import { ActorModel } from '@server/models/actor/actor'
 import { FilteredModelAttributes } from '@server/types'

server/lib/auth/oauth.ts

@@ -8,7 +8,8 @@ import {
   UnauthorizedClientError,
   UnsupportedGrantTypeError
 } from 'oauth2-server'
-import { randomBytesPromise, sha1 } from '@server/helpers/core-utils'
+import { sha1 } from '@shared/core-utils/crypto'
+import { randomBytesPromise } from '@server/helpers/core-utils'
 import { MOAuthClient } from '@server/types/models'
 import { OAUTH_LIFETIME } from '../../initializers/constants'
 import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model'

server/lib/client-html.ts

@@ -8,7 +8,8 @@ import { HTMLServerConfig } from '@shared/models'
 import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/core-utils/i18n/i18n'
 import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
 import { VideoPlaylistPrivacy, VideoPrivacy } from '../../shared/models/videos'
-import { isTestInstance, sha256 } from '../helpers/core-utils'
+import { isTestInstance } from '../helpers/core-utils'
+import { sha256 } from '@shared/core-utils/crypto'
 import { logger } from '../helpers/logger'
 import { mdToPlainText } from '../helpers/markdown'
 import { CONFIG } from '../initializers/config'

server/lib/emailer.ts

@@ -4,7 +4,8 @@ import { createTransport, Transporter } from 'nodemailer'
 import { join } from 'path'
 import { EmailPayload } from '@shared/models'
 import { SendEmailDefaultOptions } from '../../shared/models/server/emailer.model'
-import { isTestInstance, root } from '../helpers/core-utils'
+import { isTestInstance } from '../helpers/core-utils'
+import { root } from '@shared/core-utils'
 import { bunyanLogger, logger } from '../helpers/logger'
 import { CONFIG, isEmailEnabled } from '../initializers/config'
 import { WEBSERVER } from '../initializers/constants'

server/lib/hls.ts

@@ -2,7 +2,7 @@ import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat,
 import { flatten, uniq } from 'lodash'
 import { basename, dirname, join } from 'path'
 import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models'
-import { sha256 } from '../helpers/core-utils'
+import { sha256 } from '@shared/core-utils/crypto'
 import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
 import { logger } from '../helpers/logger'
 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'

server/lib/job-queue/handlers/video-file-import.ts

@@ -1,6 +1,6 @@
 import { Job } from 'bull'
 import { copy, stat } from 'fs-extra'
-import { getLowercaseExtension } from '@server/helpers/core-utils'
+import { getLowercaseExtension } from '@shared/core-utils'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { CONFIG } from '@server/initializers/config'
 import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'

server/lib/job-queue/handlers/video-import.ts

@@ -1,6 +1,6 @@
 import { Job } from 'bull'
 import { move, remove, stat } from 'fs-extra'
-import { getLowercaseExtension } from '@server/helpers/core-utils'
+import { getLowercaseExtension } from '@shared/core-utils'
 import { retryTransactionWrapper } from '@server/helpers/database-utils'
 import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
 import { isPostImportVideoAccepted } from '@server/lib/moderation'

server/lib/local-actor.ts

@@ -2,8 +2,8 @@ import 'multer'
 import { queue } from 'async'
 import LRUCache from 'lru-cache'
 import { join } from 'path'
-import { getLowercaseExtension } from '@server/helpers/core-utils'
-import { buildUUID } from '@server/helpers/uuid'
+import { getLowercaseExtension } from '@shared/core-utils'
+import { buildUUID } from '@shared/core-utils/uuid'
 import { ActorModel } from '@server/models/actor/actor'
 import { ActivityPubActorType, ActorImageType } from '@shared/models'
 import { retryTransactionWrapper } from '../helpers/database-utils'

server/lib/paths.ts

@@ -1,5 +1,5 @@
 import { join } from 'path'
-import { buildUUID } from '@server/helpers/uuid'
+import { buildUUID } from '@shared/core-utils/uuid'
 import { CONFIG } from '@server/initializers/config'
 import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
 import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'

server/lib/user.ts

@@ -1,5 +1,5 @@
 import { Transaction } from 'sequelize/types'
-import { buildUUID } from '@server/helpers/uuid'
+import { buildUUID } from '@shared/core-utils/uuid'
 import { UserModel } from '@server/models/user/user'
 import { MActorDefault } from '@server/types/models/actor'
 import { ActivityPubActorType } from '../../shared/models/activitypub'

server/lib/video-path-manager.ts

@@ -1,6 +1,6 @@
 import { remove } from 'fs-extra'
 import { extname, join } from 'path'
-import { buildUUID } from '@server/helpers/uuid'
+import { buildUUID } from '@shared/core-utils/uuid'
 import { extractVideo } from '@server/helpers/video'
 import { CONFIG } from '@server/initializers/config'
 import {

server/models/actor/actor.ts

@@ -16,9 +16,8 @@ import {
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
-import { getLowercaseExtension } from '@server/helpers/core-utils'
 import { ModelCache } from '@server/models/model-cache'
-import { AttributesOnly } from '@shared/core-utils'
+import { getLowercaseExtension, AttributesOnly } from '@shared/core-utils'
 import { ActivityIconObject, ActivityPubActorType } from '../../../shared/models/activitypub'
 import { ActorImage } from '../../../shared/models/actors/actor-image.model'
 import { activityPubContextify } from '../../helpers/activitypub'

server/models/user/user-notification.ts

@@ -1,6 +1,6 @@
 import { FindOptions, ModelIndexesOptions, Op, WhereOptions } from 'sequelize'
 import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Is, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript'
-import { uuidToShort } from '@server/helpers/uuid'
+import { uuidToShort } from '@shared/core-utils/uuid'
 import { UserNotificationIncludes, UserNotificationModelForApi } from '@server/types/models/user'
 import { AttributesOnly } from '@shared/core-utils'
 import { UserNotification, UserNotificationType } from '../../../shared'

server/models/video/formatter/video-format-utils.ts

@@ -1,4 +1,4 @@
-import { uuidToShort } from '@server/helpers/uuid'
+import { uuidToShort } from '@shared/core-utils/uuid'
 import { generateMagnetUri } from '@server/helpers/webtorrent'
 import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
 import { VideoViews } from '@server/lib/video-views'

server/models/video/video-caption.ts

@@ -15,7 +15,7 @@ import {
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
-import { buildUUID } from '@server/helpers/uuid'
+import { buildUUID } from '@shared/core-utils/uuid'
 import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo } from '@server/types/models'
 import { AttributesOnly } from '@shared/core-utils'
 import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model'

server/models/video/video-playlist.ts

@@ -17,7 +17,7 @@ import {
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
-import { buildUUID, uuidToShort } from '@server/helpers/uuid'
+import { buildUUID, uuidToShort } from '@shared/core-utils/uuid'
 import { MAccountId, MChannelId } from '@server/types/models'
 import { AttributesOnly, buildPlaylistEmbedPath, buildPlaylistWatchPath, pick } from '@shared/core-utils'
 import { ActivityIconObject } from '../../../shared/models/activitypub/objects'

server/models/video/video-streaming-playlist.ts

@@ -21,7 +21,7 @@ import { MStreamingPlaylist, MVideo } from '@server/types/models'
 import { AttributesOnly } from '@shared/core-utils'
 import { VideoStorage } from '@shared/models'
 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
-import { sha1 } from '../../helpers/core-utils'
+import { sha1 } from '@shared/core-utils/crypto'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
 import { isArrayOf } from '../../helpers/custom-validators/misc'
 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'

server/models/video/video.ts

@@ -25,7 +25,7 @@ import {
   UpdatedAt
 } from 'sequelize-typescript'
 import { buildNSFWFilter } from '@server/helpers/express-utils'
-import { uuidToShort } from '@server/helpers/uuid'
+import { uuidToShort } from '@shared/core-utils/uuid'
 import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video'
 import { LiveManager } from '@server/lib/live/live-manager'
 import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'

server/tests/api/activitypub/security.ts

@@ -7,7 +7,7 @@ import { buildDigest } from '@server/helpers/peertube-crypto'
 import { HTTP_SIGNATURE } from '@server/initializers/constants'
 import { buildGlobalHeaders } from '@server/lib/job-queue/handlers/utils/activitypub-http-utils'
 import { buildAbsoluteFixturePath, cleanupTests, createMultipleServers, killallServers, PeerTubeServer, wait } from '@shared/extra-utils'
-import { makeFollowRequest, makePOSTAPRequest } from '@shared/extra-utils/requests/activitypub'
+import { makeFollowRequest, makePOSTAPRequest } from '@server/tests/shared'
 import { HttpStatusCode } from '@shared/models'
 
 const expect = chai.expect

server/tests/api/notifications/moderation-notifications.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
 import 'mocha'
-import { buildUUID } from '@server/helpers/uuid'
+import { buildUUID } from '@shared/core-utils/uuid'
 import {
   checkAbuseStateChange,
   checkAutoInstanceFollowing,

server/tests/api/notifications/user-notifications.ts

@@ -2,7 +2,7 @@
 
 import 'mocha'
 import * as chai from 'chai'
-import { buildUUID } from '@server/helpers/uuid'
+import { buildUUID } from '@shared/core-utils/uuid'
 import {
   CheckerBaseParams,
   checkMyVideoImportIsFinished,

server/tests/api/server/follows.ts

@@ -4,7 +4,6 @@ import 'mocha'
 import * as chai from 'chai'
 import {
   cleanupTests,
-  completeVideoCheck,
   createMultipleServers,
   dateIsValid,
   expectAccountFollows,
@@ -15,6 +14,7 @@ import {
   waitJobs
 } from '@shared/extra-utils'
 import { VideoCreateResult, VideoPrivacy } from '@shared/models'
+import { completeVideoCheck } from '@server/tests/shared/video'
 
 const expect = chai.expect
 

server/tests/api/server/handle-down.ts

@@ -5,7 +5,6 @@ import * as chai from 'chai'
 import {
   cleanupTests,
   CommentsCommand,
-  completeVideoCheck,
   createMultipleServers,
   killallServers,
   PeerTubeServer,
@@ -14,6 +13,7 @@ import {
   waitJobs
 } from '@shared/extra-utils'
 import { HttpStatusCode, JobState, VideoCreateResult, VideoPrivacy } from '@shared/models'
+import { completeVideoCheck } from '@server/tests/shared/video'
 
 const expect = chai.expect
 

server/tests/api/videos/multiple-servers.ts

@@ -8,7 +8,6 @@ import {
   checkTmpIsEmpty,
   checkVideoFilesWereRemoved,
   cleanupTests,
-  completeVideoCheck,
   createMultipleServers,
   dateIsValid,
   doubleFollow,
@@ -21,6 +20,7 @@ import {
   webtorrentAdd
 } from '@shared/extra-utils'
 import { HttpStatusCode, VideoCommentThreadTree, VideoPrivacy } from '@shared/models'
+import { completeVideoCheck } from '@server/tests/shared/video'
 
 const expect = chai.expect
 

server/tests/api/videos/single-server.ts

@@ -5,7 +5,6 @@ import * as chai from 'chai'
 import {
   checkVideoFilesWereRemoved,
   cleanupTests,
-  completeVideoCheck,
   createSingleServer,
   PeerTubeServer,
   setAccessTokensToServers,
@@ -13,6 +12,7 @@ import {
   wait
 } from '@shared/extra-utils'
 import { Video, VideoPrivacy } from '@shared/models'
+import { completeVideoCheck } from '@server/tests/shared/video'
 
 const expect = chai.expect
 

server/tests/cli/prune-storage.ts

@@ -4,7 +4,7 @@ import 'mocha'
 import * as chai from 'chai'
 import { createFile, readdir } from 'fs-extra'
 import { join } from 'path'
-import { buildUUID } from '@server/helpers/uuid'
+import { buildUUID } from '@shared/core-utils/uuid'
 import {
   cleanupTests,
   CLICommand,

server/tests/shared/index.ts

@@ -0,0 +1,2 @@
+export * from './requests'
+export * from './video'

server/tests/shared/requests.ts

@@ -1,9 +1,9 @@
-import { activityPubContextify } from '../../../server/helpers/activitypub'
-import { doRequest } from '../../../server/helpers/requests'
-import { HTTP_SIGNATURE } from '../../../server/initializers/constants'
-import { buildGlobalHeaders } from '../../../server/lib/job-queue/handlers/utils/activitypub-http-utils'
+import { doRequest } from '@server/helpers/requests'
+import { activityPubContextify } from '@server/helpers/activitypub'
+import { HTTP_SIGNATURE } from '@server/initializers/constants'
+import { buildGlobalHeaders } from '@server/lib/job-queue/handlers/utils/activitypub-http-utils'
 
-function makePOSTAPRequest (url: string, body: any, httpSignature: any, headers: any) {
+export function makePOSTAPRequest (url: string, body: any, httpSignature: any, headers: any) {
   const options = {
     method: 'POST' as 'POST',
     json: body,
@@ -14,7 +14,7 @@ function makePOSTAPRequest (url: string, body: any, httpSignature: any, headers:
   return doRequest(url, options)
 }
 
-async function makeFollowRequest (to: { url: string }, by: { url: string, privateKey }) {
+export async function makeFollowRequest (to: { url: string }, by: { url: string, privateKey }) {
   const follow = {
     type: 'Follow',
     id: by.url + '/' + new Date().getTime(),
@@ -35,8 +35,3 @@ async function makeFollowRequest (to: { url: string }, by: { url: string, privat
 
   return makePOSTAPRequest(to.url + '/inbox', body, httpSignature, headers)
 }
-
-export {
-  makePOSTAPRequest,
-  makeFollowRequest
-}

server/tests/shared/video.ts

@@ -0,0 +1,148 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions */
+import { dateIsValid, makeRawRequest, PeerTubeServer, testImage, webtorrentAdd } from '@shared/extra-utils'
+import { expect } from 'chai'
+import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '@server/initializers/constants'
+import { getLowercaseExtension, uuidRegex } from '@shared/core-utils'
+
+export async function completeVideoCheck (
+  server: PeerTubeServer,
+  video: any,
+  attributes: {
+    name: string
+    category: number
+    licence: number
+    language: string
+    nsfw: boolean
+    commentsEnabled: boolean
+    downloadEnabled: boolean
+    description: string
+    publishedAt?: string
+    support: string
+    originallyPublishedAt?: string
+    account: {
+      name: string
+      host: string
+    }
+    isLocal: boolean
+    tags: string[]
+    privacy: number
+    likes?: number
+    dislikes?: number
+    duration: number
+    channel: {
+      displayName: string
+      name: string
+      description: string
+      isLocal: boolean
+    }
+    fixture: string
+    files: {
+      resolution: number
+      size: number
+    }[]
+    thumbnailfile?: string
+    previewfile?: string
+  }
+) {
+  if (!attributes.likes) attributes.likes = 0
+  if (!attributes.dislikes) attributes.dislikes = 0
+
+  const host = new URL(server.url).host
+  const originHost = attributes.account.host
+
+  expect(video.name).to.equal(attributes.name)
+  expect(video.category.id).to.equal(attributes.category)
+  expect(video.category.label).to.equal(attributes.category !== null ? VIDEO_CATEGORIES[attributes.category] : 'Misc')
+  expect(video.licence.id).to.equal(attributes.licence)
+  expect(video.licence.label).to.equal(attributes.licence !== null ? VIDEO_LICENCES[attributes.licence] : 'Unknown')
+  expect(video.language.id).to.equal(attributes.language)
+  expect(video.language.label).to.equal(attributes.language !== null ? VIDEO_LANGUAGES[attributes.language] : 'Unknown')
+  expect(video.privacy.id).to.deep.equal(attributes.privacy)
+  expect(video.privacy.label).to.deep.equal(VIDEO_PRIVACIES[attributes.privacy])
+  expect(video.nsfw).to.equal(attributes.nsfw)
+  expect(video.description).to.equal(attributes.description)
+  expect(video.account.id).to.be.a('number')
+  expect(video.account.host).to.equal(attributes.account.host)
+  expect(video.account.name).to.equal(attributes.account.name)
+  expect(video.channel.displayName).to.equal(attributes.channel.displayName)
+  expect(video.channel.name).to.equal(attributes.channel.name)
+  expect(video.likes).to.equal(attributes.likes)
+  expect(video.dislikes).to.equal(attributes.dislikes)
+  expect(video.isLocal).to.equal(attributes.isLocal)
+  expect(video.duration).to.equal(attributes.duration)
+  expect(video.url).to.contain(originHost)
+  expect(dateIsValid(video.createdAt)).to.be.true
+  expect(dateIsValid(video.publishedAt)).to.be.true
+  expect(dateIsValid(video.updatedAt)).to.be.true
+
+  if (attributes.publishedAt) {
+    expect(video.publishedAt).to.equal(attributes.publishedAt)
+  }
+
+  if (attributes.originallyPublishedAt) {
+    expect(video.originallyPublishedAt).to.equal(attributes.originallyPublishedAt)
+  } else {
+    expect(video.originallyPublishedAt).to.be.null
+  }
+
+  const videoDetails = await server.videos.get({ id: video.uuid })
+
+  expect(videoDetails.files).to.have.lengthOf(attributes.files.length)
+  expect(videoDetails.tags).to.deep.equal(attributes.tags)
+  expect(videoDetails.account.name).to.equal(attributes.account.name)
+  expect(videoDetails.account.host).to.equal(attributes.account.host)
+  expect(video.channel.displayName).to.equal(attributes.channel.displayName)
+  expect(video.channel.name).to.equal(attributes.channel.name)
+  expect(videoDetails.channel.host).to.equal(attributes.account.host)
+  expect(videoDetails.channel.isLocal).to.equal(attributes.channel.isLocal)
+  expect(dateIsValid(videoDetails.channel.createdAt.toString())).to.be.true
+  expect(dateIsValid(videoDetails.channel.updatedAt.toString())).to.be.true
+  expect(videoDetails.commentsEnabled).to.equal(attributes.commentsEnabled)
+  expect(videoDetails.downloadEnabled).to.equal(attributes.downloadEnabled)
+
+  for (const attributeFile of attributes.files) {
+    const file = videoDetails.files.find(f => f.resolution.id === attributeFile.resolution)
+    expect(file).not.to.be.undefined
+
+    let extension = getLowercaseExtension(attributes.fixture)
+    // Transcoding enabled: extension will always be .mp4
+    if (attributes.files.length > 1) extension = '.mp4'
+
+    expect(file.magnetUri).to.have.lengthOf.above(2)
+
+    expect(file.torrentDownloadUrl).to.match(new RegExp(`http://${host}/download/torrents/${uuidRegex}-${file.resolution.id}.torrent`))
+    expect(file.torrentUrl).to.match(new RegExp(`http://${host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}.torrent`))
+
+    expect(file.fileUrl).to.match(new RegExp(`http://${originHost}/static/webseed/${uuidRegex}-${file.resolution.id}${extension}`))
+    expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`))
+
+    await Promise.all([
+      makeRawRequest(file.torrentUrl, 200),
+      makeRawRequest(file.torrentDownloadUrl, 200),
+      makeRawRequest(file.metadataUrl, 200)
+    ])
+
+    expect(file.resolution.id).to.equal(attributeFile.resolution)
+    expect(file.resolution.label).to.equal(attributeFile.resolution + 'p')
+
+    const minSize = attributeFile.size - ((10 * attributeFile.size) / 100)
+    const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100)
+    expect(
+      file.size,
+      'File size for resolution ' + file.resolution.label + ' outside confidence interval (' + minSize + '> size <' + maxSize + ')'
+    ).to.be.above(minSize).and.below(maxSize)
+
+    const torrent = await webtorrentAdd(file.magnetUri, true)
+    expect(torrent.files).to.be.an('array')
+    expect(torrent.files.length).to.equal(1)
+    expect(torrent.files[0].path).to.exist.and.to.not.equal('')
+  }
+
+  expect(videoDetails.thumbnailPath).to.exist
+  await testImage(server.url, attributes.thumbnailfile || attributes.fixture, videoDetails.thumbnailPath)
+
+  if (attributes.previewfile) {
+    expect(videoDetails.previewPath).to.exist
+    await testImage(server.url, attributes.previewfile, videoDetails.previewPath)
+  }
+}

server/tools/cli.ts

@@ -5,7 +5,9 @@ import { createLogger, format, transports } from 'winston'
 import { PeerTubeServer } from '@shared/extra-utils'
 import { UserRole } from '@shared/models'
 import { VideoPrivacy } from '../../shared/models/videos'
-import { getAppNumber, isTestInstance, root } from '../helpers/core-utils'
+import { getAppNumber, isTestInstance } from '../helpers/core-utils'
+import { root } from '@shared/core-utils'
+import { loadLanguages } from '@server/initializers/constants'
 
 let configName = 'PeerTube/CLI'
 if (isTestInstance()) configName += `-${getAppNumber()}`
@@ -180,6 +182,7 @@ function getServerCredentials (program: Command) {
 }
 
 function buildServer (url: string) {
+  loadLanguages()
   return new PeerTubeServer({ url })
 }
 

server/tools/peertube-import-videos.ts

@@ -5,7 +5,7 @@ import { program } from 'commander'
 import { accessSync, constants } from 'fs'
 import { remove } from 'fs-extra'
 import { join } from 'path'
-import { sha256 } from '../helpers/core-utils'
+import { sha256 } from '@shared/core-utils/crypto'
 import { doRequestAndSaveToFile } from '../helpers/requests'
 import {
   assignToken,

server/tools/tsconfig.json

@@ -1,5 +1,11 @@
 {
   "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/server/tools"
+  },
   "include": [ ".", "../typings" ],
+  "references": [
+    { "path": "../" },
+  ],
   "exclude": [ ] // Overwrite exclude property
 }

server/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "../dist/server"
+  },
+  "references": [
+    { "path": "../shared" }
+  ],
+  "exclude": [
+    "tools/"
+  ]
+}

shared/core-utils/crypto.ts

@@ -0,0 +1,14 @@
+import { BinaryToTextEncoding, createHash } from 'crypto'
+
+function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
+  return createHash('sha256').update(str).digest(encoding)
+}
+
+function sha1 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
+  return createHash('sha1').update(str).digest(encoding)
+}
+
+export {
+  sha256,
+  sha1
+}

shared/core-utils/index.ts

@@ -1,8 +1,10 @@
 export * from './abuse'
 export * from './common'
 export * from './i18n'
+export * from './path'
 export * from './plugins'
 export * from './renderer'
 export * from './users'
 export * from './utils'
 export * from './videos'
+export * from './uuid'

shared/core-utils/path.ts

@@ -0,0 +1,34 @@
+import { basename, extname, isAbsolute, join, resolve } from 'path'
+
+let rootPath: string
+
+function root () {
+  if (rootPath) return rootPath
+
+  rootPath = __dirname
+
+  if (basename(rootPath) === 'core-utils') rootPath = resolve(rootPath, '..')
+  if (basename(rootPath) === 'shared') rootPath = resolve(rootPath, '..')
+  if (basename(rootPath) === 'server') rootPath = resolve(rootPath, '..')
+  if (basename(rootPath) === 'dist') rootPath = resolve(rootPath, '..')
+
+  return rootPath
+}
+
+function buildPath (path: string) {
+  if (isAbsolute(path)) return path
+
+  return join(root(), path)
+}
+
+function getLowercaseExtension (filename: string) {
+  const ext = extname(filename) || ''
+
+  return ext.toLowerCase()
+}
+
+export {
+  root,
+  buildPath,
+  getLowercaseExtension
+}

shared/extra-utils/ffprobe.ts

@@ -0,0 +1,180 @@
+import { ffprobe, FfprobeData } from 'fluent-ffmpeg'
+import { VideoFileMetadata } from '@shared/models/videos'
+
+/**
+ *
+ * Helpers to run ffprobe and extract data from the JSON output
+ *
+ */
+
+function ffprobePromise (path: string) {
+  return new Promise<FfprobeData>((res, rej) => {
+    ffprobe(path, (err, data) => {
+      if (err) return rej(err)
+
+      return res(data)
+    })
+  })
+}
+
+async function getAudioStream (videoPath: string, existingProbe?: FfprobeData) {
+  // without position, ffprobe considers the last input only
+  // we make it consider the first input only
+  // if you pass a file path to pos, then ffprobe acts on that file directly
+  const data = existingProbe || await ffprobePromise(videoPath)
+
+  if (Array.isArray(data.streams)) {
+    const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio')
+
+    if (audioStream) {
+      return {
+        absolutePath: data.format.filename,
+        audioStream,
+        bitrate: parseInt(audioStream['bit_rate'] + '', 10)
+      }
+    }
+  }
+
+  return { absolutePath: data.format.filename }
+}
+
+function getMaxAudioBitrate (type: 'aac' | 'mp3' | string, bitrate: number) {
+  const maxKBitrate = 384
+  const kToBits = (kbits: number) => kbits * 1000
+
+  // If we did not manage to get the bitrate, use an average value
+  if (!bitrate) return 256
+
+  if (type === 'aac') {
+    switch (true) {
+      case bitrate > kToBits(maxKBitrate):
+        return maxKBitrate
+
+      default:
+        return -1 // we interpret it as a signal to copy the audio stream as is
+    }
+  }
+
+  /*
+    a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac.
+    That's why, when using aac, we can go to lower kbit/sec. The equivalences
+    made here are not made to be accurate, especially with good mp3 encoders.
+    */
+  switch (true) {
+    case bitrate <= kToBits(192):
+      return 128
+
+    case bitrate <= kToBits(384):
+      return 256
+
+    default:
+      return maxKBitrate
+  }
+}
+
+async function getVideoStreamSize (path: string, existingProbe?: FfprobeData): Promise<{ width: number, height: number }> {
+  const videoStream = await getVideoStreamFromFile(path, existingProbe)
+
+  return videoStream === null
+    ? { width: 0, height: 0 }
+    : { width: videoStream.width, height: videoStream.height }
+}
+
+async function getVideoFileResolution (path: string, existingProbe?: FfprobeData) {
+  const size = await getVideoStreamSize(path, existingProbe)
+
+  return {
+    width: size.width,
+    height: size.height,
+    ratio: Math.max(size.height, size.width) / Math.min(size.height, size.width),
+    resolution: Math.min(size.height, size.width),
+    isPortraitMode: size.height > size.width
+  }
+}
+
+async function getVideoFileFPS (path: string, existingProbe?: FfprobeData) {
+  const videoStream = await getVideoStreamFromFile(path, existingProbe)
+  if (videoStream === null) return 0
+
+  for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) {
+    const valuesText: string = videoStream[key]
+    if (!valuesText) continue
+
+    const [ frames, seconds ] = valuesText.split('/')
+    if (!frames || !seconds) continue
+
+    const result = parseInt(frames, 10) / parseInt(seconds, 10)
+    if (result > 0) return Math.round(result)
+  }
+
+  return 0
+}
+
+async function getMetadataFromFile (path: string, existingProbe?: FfprobeData) {
+  const metadata = existingProbe || await ffprobePromise(path)
+
+  return new VideoFileMetadata(metadata)
+}
+
+async function getVideoFileBitrate (path: string, existingProbe?: FfprobeData): Promise<number> {
+  const metadata = await getMetadataFromFile(path, existingProbe)
+
+  let bitrate = metadata.format.bit_rate as number
+  if (bitrate && !isNaN(bitrate)) return bitrate
+
+  const videoStream = await getVideoStreamFromFile(path, existingProbe)
+  if (!videoStream) return undefined
+
+  bitrate = videoStream?.bit_rate
+  if (bitrate && !isNaN(bitrate)) return bitrate
+
+  return undefined
+}
+
+async function getDurationFromVideoFile (path: string, existingProbe?: FfprobeData) {
+  const metadata = await getMetadataFromFile(path, existingProbe)
+
+  return Math.round(metadata.format.duration)
+}
+
+async function getVideoStreamFromFile (path: string, existingProbe?: FfprobeData) {
+  const metadata = await getMetadataFromFile(path, existingProbe)
+
+  return metadata.streams.find(s => s.codec_type === 'video') || null
+}
+
+async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise<boolean> {
+  const parsedAudio = await getAudioStream(path, probe)
+
+  if (!parsedAudio.audioStream) return true
+
+  if (parsedAudio.audioStream['codec_name'] !== 'aac') return false
+
+  const audioBitrate = parsedAudio.bitrate
+  if (!audioBitrate) return false
+
+  const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate)
+  if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false
+
+  const channelLayout = parsedAudio.audioStream['channel_layout']
+  // Causes playback issues with Chrome
+  if (!channelLayout || channelLayout === 'unknown') return false
+
+  return true
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getVideoStreamSize,
+  getVideoFileResolution,
+  getMetadataFromFile,
+  getMaxAudioBitrate,
+  getVideoStreamFromFile,
+  getDurationFromVideoFile,
+  getAudioStream,
+  getVideoFileFPS,
+  ffprobePromise,
+  getVideoFileBitrate,
+  canDoQuickAudioTranscode
+}

shared/extra-utils/miscs/checks.ts

@@ -3,7 +3,7 @@
 import { expect } from 'chai'
 import { pathExists, readFile } from 'fs-extra'
 import { join } from 'path'
-import { root } from '@server/helpers/core-utils'
+import { root } from '@shared/core-utils'
 import { HttpStatusCode } from '@shared/models'
 import { makeGetRequest } from '../requests'
 import { PeerTubeServer } from '../server'

shared/extra-utils/miscs/generate.ts

@@ -2,7 +2,7 @@ import { expect } from 'chai'
 import ffmpeg from 'fluent-ffmpeg'
 import { ensureDir, pathExists } from 'fs-extra'
 import { dirname } from 'path'
-import { getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
+import { getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '@shared/extra-utils/ffprobe'
 import { getMaxBitrate } from '@shared/core-utils'
 import { buildAbsoluteFixturePath } from './tests'
 

shared/extra-utils/server/directories.ts

@@ -3,7 +3,7 @@
 import { expect } from 'chai'
 import { pathExists, readdir } from 'fs-extra'
 import { join } from 'path'
-import { root } from '@server/helpers/core-utils'
+import { root } from '@shared/core-utils'
 import { PeerTubeServer } from './server'
 
 async function checkTmpIsEmpty (server: PeerTubeServer) {

shared/extra-utils/server/plugins-command.ts

@@ -2,7 +2,7 @@
 
 import { readJSON, writeJSON } from 'fs-extra'
 import { join } from 'path'
-import { root } from '@server/helpers/core-utils'
+import { root } from '@shared/core-utils'
 import {
   HttpStatusCode,
   PeerTubePlugin,

shared/extra-utils/server/server.ts

@@ -1,8 +1,7 @@
 import { ChildProcess, fork } from 'child_process'
 import { copy } from 'fs-extra'
 import { join } from 'path'
-import { root } from '@server/helpers/core-utils'
-import { randomInt } from '@shared/core-utils'
+import { root, randomInt } from '@shared/core-utils'
 import { Video, VideoChannel, VideoCreateResult, VideoDetails } from '../../models/videos'
 import { BulkCommand } from '../bulk'
 import { CLICommand } from '../cli'

shared/extra-utils/server/servers-command.ts

@@ -1,7 +1,7 @@
 import { exec } from 'child_process'
 import { copy, ensureDir, readFile, remove } from 'fs-extra'
 import { basename, join } from 'path'
-import { root } from '@server/helpers/core-utils'
+import { root } from '@shared/core-utils'
 import { HttpStatusCode } from '@shared/models'
 import { getFileSize, isGithubCI, wait } from '../miscs'
 import { AbstractCommand, OverrideCommandOptions } from '../shared'

shared/extra-utils/server/tracker.ts

@@ -1,5 +1,5 @@
 import { expect } from 'chai'
-import { sha1 } from '@server/helpers/core-utils'
+import { sha1 } from '@shared/core-utils/crypto'
 import { makeGetRequest } from '../requests'
 
 async function hlsInfohashExist (serverUrl: string, masterPlaylistUrl: string, fileNumber: number) {

shared/extra-utils/users/actors.ts

@@ -3,7 +3,7 @@
 import { expect } from 'chai'
 import { pathExists, readdir } from 'fs-extra'
 import { join } from 'path'
-import { root } from '@server/helpers/core-utils'
+import { root } from '@shared/core-utils'
 import { Account, VideoChannel } from '@shared/models'
 import { PeerTubeServer } from '../server'
 

shared/extra-utils/videos/streaming-playlists.ts

@@ -1,6 +1,6 @@
 import { expect } from 'chai'
 import { basename } from 'path'
-import { sha256 } from '@server/helpers/core-utils'
+import { sha256 } from '@shared/core-utils/crypto'
 import { removeFragmentedMP4Ext } from '@shared/core-utils'
 import { HttpStatusCode, VideoStreamingPlaylist } from '@shared/models'
 import { PeerTubeServer } from '../server'

shared/extra-utils/videos/videos-command.ts

@@ -5,8 +5,7 @@ import { createReadStream, stat } from 'fs-extra'
 import got, { Response as GotResponse } from 'got'
 import { omit } from 'lodash'
 import validator from 'validator'
-import { buildUUID } from '@server/helpers/uuid'
-import { loadLanguages } from '@server/initializers/constants'
+import { buildUUID } from '@shared/core-utils/uuid'
 import { pick } from '@shared/core-utils'
 import {
   HttpStatusCode,
@@ -23,7 +22,7 @@ import {
 } from '@shared/models'
 import { buildAbsoluteFixturePath, wait } from '../miscs'
 import { unwrapBody } from '../requests'
-import { PeerTubeServer, waitJobs } from '../server'
+import { waitJobs } from '../server'
 import { AbstractCommand, OverrideCommandOptions } from '../shared'
 
 export type VideoEdit = Partial<Omit<VideoCreate, 'thumbnailfile' | 'previewfile'>> & {
@@ -33,13 +32,6 @@ export type VideoEdit = Partial<Omit<VideoCreate, 'thumbnailfile' | 'previewfile
 }
 
 export class VideosCommand extends AbstractCommand {
-
-  constructor (server: PeerTubeServer) {
-    super(server)
-
-    loadLanguages()
-  }
-
   getCategories (options: OverrideCommandOptions = {}) {
     const path = '/api/v1/videos/categories'
 

shared/extra-utils/videos/videos.ts

@@ -3,12 +3,7 @@
 import { expect } from 'chai'
 import { pathExists, readdir } from 'fs-extra'
 import { basename, join } from 'path'
-import { getLowercaseExtension } from '@server/helpers/core-utils'
-import { uuidRegex } from '@shared/core-utils'
 import { HttpStatusCode, VideoCaption, VideoDetails } from '@shared/models'
-import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants'
-import { dateIsValid, testImage, webtorrentAdd } from '../miscs'
-import { makeRawRequest } from '../requests/requests'
 import { waitJobs } from '../server'
 import { PeerTubeServer } from '../server/server'
 import { VideoEdit } from './videos-command'
@@ -85,150 +80,6 @@ function checkUploadVideoParam (
     : server.videos.buildResumeUpload({ token, attributes, expectedStatus })
 }
 
-async function completeVideoCheck (
-  server: PeerTubeServer,
-  video: any,
-  attributes: {
-    name: string
-    category: number
-    licence: number
-    language: string
-    nsfw: boolean
-    commentsEnabled: boolean
-    downloadEnabled: boolean
-    description: string
-    publishedAt?: string
-    support: string
-    originallyPublishedAt?: string
-    account: {
-      name: string
-      host: string
-    }
-    isLocal: boolean
-    tags: string[]
-    privacy: number
-    likes?: number
-    dislikes?: number
-    duration: number
-    channel: {
-      displayName: string
-      name: string
-      description: string
-      isLocal: boolean
-    }
-    fixture: string
-    files: {
-      resolution: number
-      size: number
-    }[]
-    thumbnailfile?: string
-    previewfile?: string
-  }
-) {
-  if (!attributes.likes) attributes.likes = 0
-  if (!attributes.dislikes) attributes.dislikes = 0
-
-  const host = new URL(server.url).host
-  const originHost = attributes.account.host
-
-  expect(video.name).to.equal(attributes.name)
-  expect(video.category.id).to.equal(attributes.category)
-  expect(video.category.label).to.equal(attributes.category !== null ? VIDEO_CATEGORIES[attributes.category] : 'Misc')
-  expect(video.licence.id).to.equal(attributes.licence)
-  expect(video.licence.label).to.equal(attributes.licence !== null ? VIDEO_LICENCES[attributes.licence] : 'Unknown')
-  expect(video.language.id).to.equal(attributes.language)
-  expect(video.language.label).to.equal(attributes.language !== null ? VIDEO_LANGUAGES[attributes.language] : 'Unknown')
-  expect(video.privacy.id).to.deep.equal(attributes.privacy)
-  expect(video.privacy.label).to.deep.equal(VIDEO_PRIVACIES[attributes.privacy])
-  expect(video.nsfw).to.equal(attributes.nsfw)
-  expect(video.description).to.equal(attributes.description)
-  expect(video.account.id).to.be.a('number')
-  expect(video.account.host).to.equal(attributes.account.host)
-  expect(video.account.name).to.equal(attributes.account.name)
-  expect(video.channel.displayName).to.equal(attributes.channel.displayName)
-  expect(video.channel.name).to.equal(attributes.channel.name)
-  expect(video.likes).to.equal(attributes.likes)
-  expect(video.dislikes).to.equal(attributes.dislikes)
-  expect(video.isLocal).to.equal(attributes.isLocal)
-  expect(video.duration).to.equal(attributes.duration)
-  expect(video.url).to.contain(originHost)
-  expect(dateIsValid(video.createdAt)).to.be.true
-  expect(dateIsValid(video.publishedAt)).to.be.true
-  expect(dateIsValid(video.updatedAt)).to.be.true
-
-  if (attributes.publishedAt) {
-    expect(video.publishedAt).to.equal(attributes.publishedAt)
-  }
-
-  if (attributes.originallyPublishedAt) {
-    expect(video.originallyPublishedAt).to.equal(attributes.originallyPublishedAt)
-  } else {
-    expect(video.originallyPublishedAt).to.be.null
-  }
-
-  const videoDetails = await server.videos.get({ id: video.uuid })
-
-  expect(videoDetails.files).to.have.lengthOf(attributes.files.length)
-  expect(videoDetails.tags).to.deep.equal(attributes.tags)
-  expect(videoDetails.account.name).to.equal(attributes.account.name)
-  expect(videoDetails.account.host).to.equal(attributes.account.host)
-  expect(video.channel.displayName).to.equal(attributes.channel.displayName)
-  expect(video.channel.name).to.equal(attributes.channel.name)
-  expect(videoDetails.channel.host).to.equal(attributes.account.host)
-  expect(videoDetails.channel.isLocal).to.equal(attributes.channel.isLocal)
-  expect(dateIsValid(videoDetails.channel.createdAt.toString())).to.be.true
-  expect(dateIsValid(videoDetails.channel.updatedAt.toString())).to.be.true
-  expect(videoDetails.commentsEnabled).to.equal(attributes.commentsEnabled)
-  expect(videoDetails.downloadEnabled).to.equal(attributes.downloadEnabled)
-
-  for (const attributeFile of attributes.files) {
-    const file = videoDetails.files.find(f => f.resolution.id === attributeFile.resolution)
-    expect(file).not.to.be.undefined
-
-    let extension = getLowercaseExtension(attributes.fixture)
-    // Transcoding enabled: extension will always be .mp4
-    if (attributes.files.length > 1) extension = '.mp4'
-
-    expect(file.magnetUri).to.have.lengthOf.above(2)
-
-    expect(file.torrentDownloadUrl).to.match(new RegExp(`http://${host}/download/torrents/${uuidRegex}-${file.resolution.id}.torrent`))
-    expect(file.torrentUrl).to.match(new RegExp(`http://${host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}.torrent`))
-
-    expect(file.fileUrl).to.match(new RegExp(`http://${originHost}/static/webseed/${uuidRegex}-${file.resolution.id}${extension}`))
-    expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`))
-
-    await Promise.all([
-      makeRawRequest(file.torrentUrl, 200),
-      makeRawRequest(file.torrentDownloadUrl, 200),
-      makeRawRequest(file.metadataUrl, 200)
-    ])
-
-    expect(file.resolution.id).to.equal(attributeFile.resolution)
-    expect(file.resolution.label).to.equal(attributeFile.resolution + 'p')
-
-    const minSize = attributeFile.size - ((10 * attributeFile.size) / 100)
-    const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100)
-    expect(
-      file.size,
-      'File size for resolution ' + file.resolution.label + ' outside confidence interval (' + minSize + '> size <' + maxSize + ')'
-    ).to.be.above(minSize).and.below(maxSize)
-
-    const torrent = await webtorrentAdd(file.magnetUri, true)
-    expect(torrent.files).to.be.an('array')
-    expect(torrent.files.length).to.equal(1)
-    expect(torrent.files[0].path).to.exist.and.to.not.equal('')
-    expect(torrent.files[0].name).to.equal(`${videoDetails.name} ${file.resolution.id}p${extension}`)
-  }
-
-  expect(videoDetails.thumbnailPath).to.exist
-  await testImage(server.url, attributes.thumbnailfile || attributes.fixture, videoDetails.thumbnailPath)
-
-  if (attributes.previewfile) {
-    expect(videoDetails.previewPath).to.exist
-    await testImage(server.url, attributes.previewfile, videoDetails.previewPath)
-  }
-}
-
 // serverNumber starts from 1
 async function uploadRandomVideoOnServers (
   servers: PeerTubeServer[],
@@ -247,7 +98,6 @@ async function uploadRandomVideoOnServers (
 
 export {
   checkUploadVideoParam,
-  completeVideoCheck,
   uploadRandomVideoOnServers,
   checkVideoFilesWereRemoved,
   saveVideoInServers

shared/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "../dist/shared"
+  }
+}

tsconfig.base.json

@@ -0,0 +1,35 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "target": "es2015",
+    "noImplicitAny": false,
+    "sourceMap": false,
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true,
+    "importHelpers": true,
+    "removeComments": true,
+    "strictBindCallApply": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "lib": [
+      "es2015",
+      "es2016",
+      "es2017",
+      "es2018",
+      "es2019"
+    ],
+    "typeRoots": [
+      "node_modules/@types",
+    ],
+    "baseUrl": "./",
+    "outDir": "./dist/",
+    "paths": {
+      "@server/*": [ "server/*" ],
+      "@shared/*": [ "shared/*" ]
+    },
+    "resolveJsonModule": true,
+    "strict": false,
+    "skipLibCheck": true,
+    "composite": true
+  }
+}

tsconfig.json

@@ -1,46 +1,9 @@
 {
-  "compilerOptions": {
-    "module": "commonjs",
-    "target": "es2015",
-    "noImplicitAny": false,
-    "sourceMap": false,
-    "experimentalDecorators": true,
-    "emitDecoratorMetadata": true,
-    "importHelpers": true,
-    "removeComments": true,
-    "strictBindCallApply": true,
-    "esModuleInterop": true,
-    "forceConsistentCasingInFileNames": true,
-    "outDir": "./dist",
-    "lib": [
-      "dom",
-      "es2015",
-      "es2016",
-      "es2017",
-      "es2018",
-      "es2019"
-    ],
-    "typeRoots": [
-      "node_modules/@types",
-      "server/typings"
-    ],
-    "baseUrl": "./",
-    "paths": {
-      "@server/*": [ "server/*" ],
-      "@shared/*": [ "shared/*" ]
-    }
-  },
-  "exclude": [
-    "server/tools/",
-    "node_modules",
-    "dist",
-    "storage",
-    "client",
-    "test1",
-    "test2",
-    "test3",
-    "test4",
-    "test5",
-    "test6"
-  ]
+  "extends": "./tsconfig.base.json",
+  "references": [
+    { "path": "./shared" },
+    { "path": "./server" },
+    { "path": "./scripts" }
+  ],
+  "files": ["server.ts"]
 }

tsconfig.types.json

@@ -0,0 +1,19 @@
+{
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {
+    "incremental": true,
+    "sourceMap": true,
+    "stripInternal": true,
+    "removeComments": false,
+    "declaration": true,
+    "declarationMap": true,
+    "emitDeclarationOnly": true
+  },
+  "references": [
+    { "path": "./shared/tsconfig.types.json" },
+    { "path": "./server/tsconfig.types.json" },
+    { "path": "./scripts/tsconfig.types.json" }
+  ],
+  "files": []
+}
+