PeerTube/server/helpers/utils.ts
kontrollanten f6d6e7f861
Resumable video uploads (#3933)
* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00

94 lines
2.6 KiB
TypeScript

import { remove } from 'fs-extra'
import { Instance as ParseTorrent } from 'parse-torrent'
import { join } from 'path'
import { ResultList } from '../../shared'
import { CONFIG } from '../initializers/config'
import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils'
import { logger } from './logger'
function deleteFileAndCatch (path: string) {
remove(path)
.catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err }))
}
async function generateRandomString (size: number) {
const raw = await randomBytesPromise(size)
return raw.toString('hex')
}
interface FormattableToJSON<U, V> {
toFormattedJSON (args?: U): V
}
function getFormattedObjects<U, V, T extends FormattableToJSON<U, V>> (objects: T[], objectsTotal: number, formattedArg?: U) {
const formattedObjects = objects.map(o => o.toFormattedJSON(formattedArg))
return {
total: objectsTotal,
data: formattedObjects
} as ResultList<V>
}
function generateVideoImportTmpPath (target: string | ParseTorrent, extension = '.mp4') {
const id = typeof target === 'string'
? target
: target.infoHash
const hash = sha256(id)
return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`)
}
function getSecureTorrentName (originalName: string) {
return sha256(originalName) + '.torrent'
}
async function getServerCommit () {
try {
const tag = await execPromise2(
'[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true',
{ stdio: [ 0, 1, 2 ] }
)
if (tag) return tag.replace(/^v/, '')
} catch (err) {
logger.debug('Cannot get version from git tags.', { err })
}
try {
const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD')
if (version) return version.toString().trim()
} catch (err) {
logger.debug('Cannot get version from git HEAD.', { err })
}
return ''
}
/**
* From a filename like "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3.mp4", returns
* only the "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3" part. If the filename does
* not contain a UUID, returns null.
*/
function getUUIDFromFilename (filename: string) {
const regex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
const result = filename.match(regex)
if (!result || Array.isArray(result) === false) return null
return result[0]
}
// ---------------------------------------------------------------------------
export {
deleteFileAndCatch,
generateRandomString,
getFormattedObjects,
getSecureTorrentName,
getServerCommit,
generateVideoImportTmpPath,
getUUIDFromFilename
}