PeerTube/server/lib/moderation.ts
kontrollanten f6d6e7f861
Resumable video uploads (#3933)
* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00

242 lines
6.9 KiB
TypeScript

import { VideoUploadFile } from 'express'
import { PathLike } from 'fs-extra'
import { Transaction } from 'sequelize/types'
import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger'
import { afterCommitIfTransaction } from '@server/helpers/database-utils'
import { logger } from '@server/helpers/logger'
import { AbuseModel } from '@server/models/abuse/abuse'
import { VideoAbuseModel } from '@server/models/abuse/video-abuse'
import { VideoCommentAbuseModel } from '@server/models/abuse/video-comment-abuse'
import { VideoFileModel } from '@server/models/video/video-file'
import { FilteredModelAttributes } from '@server/types'
import {
MAbuseFull,
MAccountDefault,
MAccountLight,
MCommentAbuseAccountVideo,
MCommentOwnerVideo,
MUser,
MVideoAbuseVideoFull,
MVideoAccountLightBlacklistAllFiles
} from '@server/types/models'
import { ActivityCreate } from '../../shared/models/activitypub'
import { VideoObject } from '../../shared/models/activitypub/objects'
import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object'
import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos'
import { VideoCommentCreate } from '../../shared/models/videos/video-comment.model'
import { UserModel } from '../models/account/user'
import { ActorModel } from '../models/activitypub/actor'
import { VideoModel } from '../models/video/video'
import { VideoCommentModel } from '../models/video/video-comment'
import { sendAbuse } from './activitypub/send/send-flag'
import { Notifier } from './notifier'
export type AcceptResult = {
accepted: boolean
errorMessage?: string
}
// Can be filtered by plugins
function isLocalVideoAccepted (object: {
videoBody: VideoCreate
videoFile: VideoUploadFile
user: UserModel
}): AcceptResult {
return { accepted: true }
}
function isLocalLiveVideoAccepted (object: {
liveVideoBody: LiveVideoCreate
user: UserModel
}): AcceptResult {
return { accepted: true }
}
function isLocalVideoThreadAccepted (_object: {
commentBody: VideoCommentCreate
video: VideoModel
user: UserModel
}): AcceptResult {
return { accepted: true }
}
function isLocalVideoCommentReplyAccepted (_object: {
commentBody: VideoCommentCreate
parentComment: VideoCommentModel
video: VideoModel
user: UserModel
}): AcceptResult {
return { accepted: true }
}
function isRemoteVideoAccepted (_object: {
activity: ActivityCreate
videoAP: VideoObject
byActor: ActorModel
}): AcceptResult {
return { accepted: true }
}
function isRemoteVideoCommentAccepted (_object: {
activity: ActivityCreate
commentAP: VideoCommentObject
byActor: ActorModel
}): AcceptResult {
return { accepted: true }
}
function isPreImportVideoAccepted (object: {
videoImportBody: VideoImportCreate
user: MUser
}): AcceptResult {
return { accepted: true }
}
function isPostImportVideoAccepted (object: {
videoFilePath: PathLike
videoFile: VideoFileModel
user: MUser
}): AcceptResult {
return { accepted: true }
}
async function createVideoAbuse (options: {
baseAbuse: FilteredModelAttributes<AbuseModel>
videoInstance: MVideoAccountLightBlacklistAllFiles
startAt: number
endAt: number
transaction: Transaction
reporterAccount: MAccountDefault
}) {
const { baseAbuse, videoInstance, startAt, endAt, transaction, reporterAccount } = options
const associateFun = async (abuseInstance: MAbuseFull) => {
const videoAbuseInstance: MVideoAbuseVideoFull = await VideoAbuseModel.create({
abuseId: abuseInstance.id,
videoId: videoInstance.id,
startAt: startAt,
endAt: endAt
}, { transaction })
videoAbuseInstance.Video = videoInstance
abuseInstance.VideoAbuse = videoAbuseInstance
return { isOwned: videoInstance.isOwned() }
}
return createAbuse({
base: baseAbuse,
reporterAccount,
flaggedAccount: videoInstance.VideoChannel.Account,
transaction,
associateFun
})
}
function createVideoCommentAbuse (options: {
baseAbuse: FilteredModelAttributes<AbuseModel>
commentInstance: MCommentOwnerVideo
transaction: Transaction
reporterAccount: MAccountDefault
}) {
const { baseAbuse, commentInstance, transaction, reporterAccount } = options
const associateFun = async (abuseInstance: MAbuseFull) => {
const commentAbuseInstance: MCommentAbuseAccountVideo = await VideoCommentAbuseModel.create({
abuseId: abuseInstance.id,
videoCommentId: commentInstance.id
}, { transaction })
commentAbuseInstance.VideoComment = commentInstance
abuseInstance.VideoCommentAbuse = commentAbuseInstance
return { isOwned: commentInstance.isOwned() }
}
return createAbuse({
base: baseAbuse,
reporterAccount,
flaggedAccount: commentInstance.Account,
transaction,
associateFun
})
}
function createAccountAbuse (options: {
baseAbuse: FilteredModelAttributes<AbuseModel>
accountInstance: MAccountDefault
transaction: Transaction
reporterAccount: MAccountDefault
}) {
const { baseAbuse, accountInstance, transaction, reporterAccount } = options
const associateFun = async () => {
return { isOwned: accountInstance.isOwned() }
}
return createAbuse({
base: baseAbuse,
reporterAccount,
flaggedAccount: accountInstance,
transaction,
associateFun
})
}
export {
isLocalLiveVideoAccepted,
isLocalVideoAccepted,
isLocalVideoThreadAccepted,
isRemoteVideoAccepted,
isRemoteVideoCommentAccepted,
isLocalVideoCommentReplyAccepted,
isPreImportVideoAccepted,
isPostImportVideoAccepted,
createAbuse,
createVideoAbuse,
createVideoCommentAbuse,
createAccountAbuse
}
// ---------------------------------------------------------------------------
async function createAbuse (options: {
base: FilteredModelAttributes<AbuseModel>
reporterAccount: MAccountDefault
flaggedAccount: MAccountLight
associateFun: (abuseInstance: MAbuseFull) => Promise<{ isOwned: boolean} >
transaction: Transaction
}) {
const { base, reporterAccount, flaggedAccount, associateFun, transaction } = options
const auditLogger = auditLoggerFactory('abuse')
const abuseAttributes = Object.assign({}, base, { flaggedAccountId: flaggedAccount.id })
const abuseInstance: MAbuseFull = await AbuseModel.create(abuseAttributes, { transaction })
abuseInstance.ReporterAccount = reporterAccount
abuseInstance.FlaggedAccount = flaggedAccount
const { isOwned } = await associateFun(abuseInstance)
if (isOwned === false) {
await sendAbuse(reporterAccount.Actor, abuseInstance, abuseInstance.FlaggedAccount, transaction)
}
const abuseJSON = abuseInstance.toFormattedAdminJSON()
auditLogger.create(reporterAccount.Actor.getIdentifier(), new AbuseAuditView(abuseJSON))
afterCommitIfTransaction(transaction, () => {
Notifier.Instance.notifyOnNewAbuse({
abuse: abuseJSON,
abuseInstance,
reporter: reporterAccount.Actor.getIdentifier()
})
})
logger.info('Abuse report %d created.', abuseInstance.id)
return abuseJSON
}