Implement auto tag on comments and videos

* Comments and videos can be automatically tagged using core rules or
   watched word lists
 * These tags can be used to automatically filter videos and comments
 * Introduce a new video comment policy where comments must be approved
   first
 * Comments may have to be approved if the user auto block them using
   core rules or watched word lists
 * Implement FEP-5624 to federate reply control policies
This commit is contained in:
Chocobozzz
2024-03-29 14:25:03 +01:00
committed by Chocobozzz
parent b3e39df59e
commit 29329d6c45
241 changed files with 8090 additions and 1399 deletions

View File

@@ -0,0 +1,45 @@
import { CommentAutomaticTagPoliciesUpdate } from '@peertube/peertube-models'
import { isStringArray } from '@server/helpers/custom-validators/search.js'
import { AutomaticTagger } from '@server/lib/automatic-tags/automatic-tagger.js'
import express from 'express'
import { body, param } from 'express-validator'
import { doesAccountNameWithHostExist } from './shared/accounts.js'
import { checkUserCanManageAccount } from './shared/users.js'
import { areValidationErrors } from './shared/utils.js'
export const manageAccountAutomaticTagsValidator = [
param('accountName')
.exists(),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
if (!await doesAccountNameWithHostExist(req.params.accountName, res)) return
if (!checkUserCanManageAccount({ user: res.locals.oauth.token.User, account: res.locals.account, specialRight: null, res })) return
return next()
}
]
export const updateAutomaticTagPoliciesValidator = [
...manageAccountAutomaticTagsValidator,
body('review')
.custom(isStringArray).withMessage('Should have a valid review array'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
const body = req.body as CommentAutomaticTagPoliciesUpdate
const tagsObj = await AutomaticTagger.getAutomaticTagAvailable(res.locals.account)
const available = new Set(tagsObj.available.map(({ name }) => name))
for (const name of body.review) {
if (!available.has(name)) {
return res.fail({ message: `${name} is not an available automatic tag` })
}
}
return next()
}
]

View File

@@ -17,7 +17,7 @@ const bulkRemoveCommentsOfValidator = [
const user = res.locals.oauth.token.User
const body = req.body as BulkRemoveCommentsOfBody
if (body.scope === 'instance' && user.hasRight(UserRight.REMOVE_ANY_VIDEO_COMMENT) !== true) {
if (body.scope === 'instance' && user.hasRight(UserRight.MANAGE_ANY_VIDEO_COMMENT) !== true) {
return res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message: 'User cannot remove any comments of this instance.'

View File

@@ -1,20 +1,20 @@
import express from 'express'
import { forceNumber } from '@peertube/peertube-core-utils'
import { HttpStatusCode, UserRightType } from '@peertube/peertube-models'
import { ActorModel } from '@server/models/actor/actor.js'
import { UserModel } from '@server/models/user/user.js'
import { MUserDefault } from '@server/types/models/index.js'
import { forceNumber } from '@peertube/peertube-core-utils'
import { HttpStatusCode } from '@peertube/peertube-models'
import { MAccountId, MUserAccountId, MUserDefault } from '@server/types/models/index.js'
import express from 'express'
function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) {
export function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) {
const id = forceNumber(idArg)
return checkUserExist(() => UserModel.loadByIdWithChannels(id, withStats), res)
}
function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) {
export function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) {
return checkUserExist(() => UserModel.loadByEmail(email), res, abortResponse)
}
async function checkUserNameOrEmailDoNotAlreadyExist (username: string, email: string, res: express.Response) {
export async function checkUserNameOrEmailDoNotAlreadyExist (username: string, email: string, res: express.Response) {
const user = await UserModel.loadByUsernameOrEmail(username, email)
if (user) {
@@ -37,7 +37,7 @@ async function checkUserNameOrEmailDoNotAlreadyExist (username: string, email: s
return true
}
async function checkUserExist (finder: () => Promise<MUserDefault>, res: express.Response, abortResponse = true) {
export async function checkUserExist (finder: () => Promise<MUserDefault>, res: express.Response, abortResponse = true) {
const user = await finder()
if (!user) {
@@ -55,9 +55,30 @@ async function checkUserExist (finder: () => Promise<MUserDefault>, res: express
return true
}
export {
checkUserIdExist,
checkUserEmailExist,
checkUserNameOrEmailDoNotAlreadyExist,
checkUserExist
export function checkUserCanManageAccount (options: {
user: MUserAccountId
account: MAccountId
specialRight: UserRightType
res: express.Response
}) {
const { user, account, specialRight, res } = options
if (account.id === user.Account.id) return true
if (specialRight && user.hasRight(specialRight) === true) return true
if (!specialRight) {
res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message: 'Only the owner of this account can manage this account resource.'
})
return false
}
res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message: 'Only a user with sufficient right can access this account resource.'
})
return false
}

View File

@@ -287,7 +287,6 @@ function assignVideoTokenIfNeeded (req: Request, res: Response, video: MVideoUUI
// ---------------------------------------------------------------------------
export function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRightType, res: Response, onlyOwned = true) {
// Retrieve the user who did the request
if (onlyOwned && video.isOwned() === false) {
res.fail({
status: HttpStatusCode.FORBIDDEN_403,
@@ -296,9 +295,6 @@ export function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight,
return false
}
// Check if the user can delete the video
// The user can delete it if he has the right
// Or if s/he is the video's account
const account = video.VideoChannel.Account
if (user.hasRight(right) === false && account.userId !== user.id) {
res.fail({

View File

@@ -30,6 +30,8 @@ export const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.
export const videoChannelSyncsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNEL_SYNCS)
export const videoPasswordsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PASSWORDS)
export const watchedWordsListsSortValidator = checkSortFactory(SORTABLE_COLUMNS.WATCHED_WORDS_LISTS)
export const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS)
export const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS)

View File

@@ -34,6 +34,7 @@ import {
checkUserEmailExist,
checkUserIdExist,
checkUserNameOrEmailDoNotAlreadyExist,
checkUserCanManageAccount,
doesVideoChannelIdExist,
doesVideoExist,
isValidVideoIdParam
@@ -421,12 +422,7 @@ const ensureAuthUserOwnsAccountValidator = [
(req: express.Request, res: express.Response, next: express.NextFunction) => {
const user = res.locals.oauth.token.User
if (res.locals.account.id !== user.Account.id) {
return res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message: 'Only owner of this account can access this resource.'
})
}
if (!checkUserCanManageAccount({ user, account: res.locals.account, specialRight: null, res })) return
return next()
}
@@ -436,16 +432,8 @@ const ensureCanManageChannelOrAccount = [
(req: express.Request, res: express.Response, next: express.NextFunction) => {
const user = res.locals.oauth.token.user
const account = res.locals.videoChannel?.Account ?? res.locals.account
const isUserOwner = account.userId === user.id
if (!isUserOwner && user.hasRight(UserRight.MANAGE_ANY_VIDEO_CHANNEL) === false) {
const message = `User ${user.username} does not have right this channel or account.`
return res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message
})
}
if (!checkUserCanManageAccount({ account, user, res, specialRight: UserRight.MANAGE_ANY_VIDEO_CHANNEL })) return
return next()
}
@@ -461,7 +449,7 @@ const ensureCanModerateUser = [
return res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message: 'A moderator can only manage users.'
message: 'Users can only be managed by moderators or admins.'
})
}
]

View File

@@ -1,8 +1,19 @@
import { arrayify } from '@peertube/peertube-core-utils'
import { HttpStatusCode, UserRight, VideoCommentPolicy } from '@peertube/peertube-models'
import { isStringArray } from '@server/helpers/custom-validators/search.js'
import { canVideoBeFederated } from '@server/lib/activitypub/videos/federate.js'
import { MUserAccountUrl } from '@server/types/models/index.js'
import express from 'express'
import { body, param, query } from 'express-validator'
import { MUserAccountUrl } from '@server/types/models/index.js'
import { HttpStatusCode, UserRight } from '@peertube/peertube-models'
import { exists, isBooleanValid, isIdValid, toBooleanOrNull } from '../../../helpers/custom-validators/misc.js'
import {
exists,
isBooleanValid,
isIdOrUUIDValid,
isIdValid,
toBooleanOrNull,
toCompleteUUID,
toIntOrNull
} from '../../../helpers/custom-validators/misc.js'
import { isValidVideoCommentText } from '../../../helpers/custom-validators/video-comments.js'
import { logger } from '../../../helpers/logger.js'
import { AcceptResult, isLocalVideoCommentReplyAccepted, isLocalVideoThreadAccepted } from '../../../lib/moderation.js'
@@ -11,15 +22,19 @@ import { MCommentOwnerVideoReply, MVideo, MVideoFullLight } from '../../../types
import {
areValidationErrors,
checkCanSeeVideo,
checkUserCanManageAccount,
checkUserCanManageVideo,
doesVideoChannelIdExist,
doesVideoCommentExist,
doesVideoCommentThreadExist,
doesVideoExist,
isValidVideoIdParam,
isValidVideoPasswordHeader
} from '../shared/index.js'
import { canVideoBeFederated } from '@server/lib/activitypub/videos/federate.js'
const listVideoCommentsValidator = [
export const listAllVideoCommentsForAdminValidator = [
...getCommonVideoCommentsValidators(),
query('isLocal')
.optional()
.customSanitizer(toBooleanOrNull)
@@ -32,26 +47,46 @@ const listVideoCommentsValidator = [
.custom(isBooleanValid)
.withMessage('Should have a valid onLocalVideo boolean'),
query('search')
.optional()
.custom(exists),
query('searchAccount')
.optional()
.custom(exists),
query('searchVideo')
.optional()
.custom(exists),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
if (req.query.videoId && !await doesVideoExist(req.query.videoId, res, 'unsafe-only-immutable-attributes')) return
if (req.query.videoChannelId && !await doesVideoChannelIdExist(req.query.videoChannelId, res)) return
return next()
}
]
const listVideoCommentThreadsValidator = [
export const listCommentsOnUserVideosValidator = [
...getCommonVideoCommentsValidators(),
query('isHeldForReview')
.optional()
.customSanitizer(toBooleanOrNull)
.custom(isBooleanValid)
.withMessage('Should have a valid isHeldForReview boolean'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
if (req.query.videoId && !await doesVideoExist(req.query.videoId, res, 'all')) return
if (req.query.videoChannelId && !await doesVideoChannelIdExist(req.query.videoChannelId, res)) return
const user = res.locals.oauth.token.User
const video = res.locals.videoAll
if (video && !checkUserCanManageVideo(user, video, UserRight.SEE_ALL_COMMENTS, res)) return
const channel = res.locals.videoChannel
if (channel && !checkUserCanManageAccount({ account: channel.Account, user, res, specialRight: UserRight.SEE_ALL_COMMENTS })) return
return next()
}
]
// ---------------------------------------------------------------------------
export const listVideoCommentThreadsValidator = [
isValidVideoIdParam('videoId'),
isValidVideoPasswordHeader(),
@@ -65,7 +100,7 @@ const listVideoCommentThreadsValidator = [
}
]
const listVideoThreadCommentsValidator = [
export const listVideoThreadCommentsValidator = [
isValidVideoIdParam('videoId'),
param('threadId')
@@ -83,7 +118,7 @@ const listVideoThreadCommentsValidator = [
}
]
const addVideoCommentThreadValidator = [
export const addVideoCommentThreadValidator = [
isValidVideoIdParam('videoId'),
body('text')
@@ -103,7 +138,7 @@ const addVideoCommentThreadValidator = [
}
]
const addVideoCommentReplyValidator = [
export const addVideoCommentReplyValidator = [
isValidVideoIdParam('videoId'),
param('commentId').custom(isIdValid),
@@ -125,7 +160,7 @@ const addVideoCommentReplyValidator = [
}
]
const videoCommentGetValidator = [
export const videoCommentGetValidator = [
isValidVideoIdParam('videoId'),
param('commentId')
@@ -143,7 +178,7 @@ const videoCommentGetValidator = [
}
]
const removeVideoCommentValidator = [
export const removeVideoCommentValidator = [
isValidVideoIdParam('videoId'),
param('commentId')
@@ -154,29 +189,35 @@ const removeVideoCommentValidator = [
if (!await doesVideoExist(req.params.videoId, res)) return
if (!await doesVideoCommentExist(req.params.commentId, res.locals.videoAll, res)) return
// Check if the user who did the request is able to delete the video
if (!checkUserCanDeleteVideoComment(res.locals.oauth.token.User, res.locals.videoCommentFull, res)) return
return next()
}
]
export const approveVideoCommentValidator = [
isValidVideoIdParam('videoId'),
param('commentId')
.custom(isIdValid),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
if (!await doesVideoExist(req.params.videoId, res)) return
if (!await doesVideoCommentExist(req.params.commentId, res.locals.videoAll, res)) return
if (!checkUserCanApproveVideoComment(res.locals.oauth.token.User, res.locals.videoCommentFull, res)) return
return next()
}
]
// ---------------------------------------------------------------------------
export {
listVideoCommentThreadsValidator,
listVideoThreadCommentsValidator,
addVideoCommentThreadValidator,
listVideoCommentsValidator,
addVideoCommentReplyValidator,
videoCommentGetValidator,
removeVideoCommentValidator
}
// Private
// ---------------------------------------------------------------------------
function isVideoCommentsEnabled (video: MVideo, res: express.Response) {
if (video.commentsEnabled !== true) {
if (video.commentsPolicy === VideoCommentPolicy.DISABLED) {
res.fail({
status: HttpStatusCode.CONFLICT_409,
message: 'Video comments are disabled for this video.'
@@ -196,10 +237,34 @@ function checkUserCanDeleteVideoComment (user: MUserAccountUrl, videoComment: MC
return false
}
return checkUserCanManageVideoComment(user, videoComment, res)
}
function checkUserCanApproveVideoComment (user: MUserAccountUrl, videoComment: MCommentOwnerVideoReply, res: express.Response) {
if (videoComment.isDeleted()) {
res.fail({
status: HttpStatusCode.CONFLICT_409,
message: 'This comment is deleted'
})
return false
}
if (videoComment.heldForReview !== true) {
res.fail({
status: HttpStatusCode.BAD_REQUEST_400,
message: 'This comment is not held for review'
})
return false
}
return checkUserCanManageVideoComment(user, videoComment, res)
}
function checkUserCanManageVideoComment (user: MUserAccountUrl, videoComment: MCommentOwnerVideoReply, res: express.Response) {
const userAccount = user.Account
if (
user.hasRight(UserRight.REMOVE_ANY_VIDEO_COMMENT) === false && // Not a moderator
user.hasRight(UserRight.MANAGE_ANY_VIDEO_COMMENT) === false && // Not a moderator
videoComment.accountId !== userAccount.id && // Not the comment owner
videoComment.Video.VideoChannel.accountId !== userAccount.id // Not the video owner
) {
@@ -251,3 +316,34 @@ async function isVideoCommentAccepted (req: express.Request, res: express.Respon
return true
}
function getCommonVideoCommentsValidators () {
return [
query('search')
.optional()
.custom(exists),
query('searchAccount')
.optional()
.custom(exists),
query('searchVideo')
.optional()
.custom(exists),
query('videoId')
.optional()
.custom(toCompleteUUID)
.custom(isIdOrUUIDValid),
query('videoChannelId')
.optional()
.customSanitizer(toIntOrNull)
.custom(isIdValid),
query('autoTagOneOf')
.optional()
.customSanitizer(arrayify)
.custom(isStringArray).withMessage('Should have a valid autoTagOneOf array')
]
}

View File

@@ -23,6 +23,7 @@ import {
isScheduleVideoUpdatePrivacyValid,
isValidPasswordProtectedPrivacy,
isVideoCategoryValid,
isVideoCommentsPolicyValid,
isVideoDescriptionValid,
isVideoImageValid,
isVideoIncludeValid,
@@ -375,10 +376,15 @@ function getCommonVideoEditAttributes () {
`Should have an array of up to ${CONSTRAINTS_FIELDS.VIDEOS.TAGS.max} tags between ` +
`${CONSTRAINTS_FIELDS.VIDEOS.TAG.min} and ${CONSTRAINTS_FIELDS.VIDEOS.TAG.max} characters each`
),
// TODO: remove, deprecated in PeerTube 6.2
body('commentsEnabled')
.optional()
.customSanitizer(toBooleanOrNull)
.custom(isBooleanValid).withMessage('Should have commentsEnabled boolean'),
.custom(isBooleanValid).withMessage('Should have valid commentsEnabled boolean'),
body('commentsPolicy')
.optional()
.custom(isVideoCommentsPolicyValid),
body('downloadEnabled')
.optional()
.customSanitizer(toBooleanOrNull)
@@ -462,6 +468,10 @@ const commonVideosFiltersValidator = [
.optional()
.customSanitizer(toBooleanOrNull)
.isBoolean().withMessage('Should be a valid excludeAlreadyWatched boolean'),
query('autoTagOneOf')
.optional()
.customSanitizer(arrayify)
.custom(isStringArray).withMessage('Should have a valid autoTagOneOf array'),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
@@ -469,10 +479,10 @@ const commonVideosFiltersValidator = [
const user = res.locals.oauth?.token.User
if ((!user || user.hasRight(UserRight.SEE_ALL_VIDEOS) !== true)) {
if (req.query.include || req.query.privacyOneOf) {
if (req.query.include || req.query.privacyOneOf || req.query.autoTagOneOf) {
return res.fail({
status: HttpStatusCode.UNAUTHORIZED_401,
message: 'You are not allowed to see all videos or specify a custom include.'
message: 'You are not allowed to see all videos, specify a custom include or auto tags filter.'
})
}
}

View File

@@ -0,0 +1,128 @@
import { HttpStatusCode } from '@peertube/peertube-models'
import { Awaitable } from '@peertube/peertube-typescript-utils'
import { isIdValid } from '@server/helpers/custom-validators/misc.js'
import { areWatchedWordsValid, isWatchedWordListNameValid } from '@server/helpers/custom-validators/watched-words.js'
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
import { WatchedWordsListModel } from '@server/models/watched-words/watched-words-list.js'
import { MAccountId, MWatchedWordsList } from '@server/types/models/index.js'
import express from 'express'
import { ValidationChain, body, param } from 'express-validator'
import { doesAccountNameWithHostExist } from './shared/accounts.js'
import { checkUserCanManageAccount } from './shared/users.js'
import { areValidationErrors } from './shared/utils.js'
export const manageAccountWatchedWordsListValidator = [
param('accountName')
.exists(),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
if (!await doesAccountNameWithHostExist(req.params.accountName, res)) return
if (!checkUserCanManageAccount({ user: res.locals.oauth.token.User, account: res.locals.account, specialRight: null, res })) return
return next()
}
]
export function getWatchedWordsListValidatorFactory (accountGetter: (res: express.Response) => Awaitable<MAccountId>) {
return [
param('listId')
.custom(isIdValid),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
const watchedWordsList = await WatchedWordsListModel.load({ id: +req.params.listId, accountId: (await accountGetter(res)).id })
if (!watchedWordsList) {
return res.fail({
status: HttpStatusCode.NOT_FOUND_404,
message: 'Unknown watched words list id for this account'
})
}
res.locals.watchedWordsList = watchedWordsList
return next()
}
]
}
function buildUpdateOrAddValidators ({ optional }: { optional: boolean }) {
const makeOptionalIfNeeded = (chain: ValidationChain) => {
if (optional) return chain.optional()
return chain
}
return [
makeOptionalIfNeeded(body('listName'))
.trim()
.custom(isWatchedWordListNameValid).withMessage(
`Should have a list name between ` +
`${CONSTRAINTS_FIELDS.WATCHED_WORDS.LIST_NAME.min} and ${CONSTRAINTS_FIELDS.WATCHED_WORDS.LIST_NAME.max} characters long`
),
makeOptionalIfNeeded(body('words'))
.custom(areWatchedWordsValid)
.withMessage(
`Should have an array of up to ${CONSTRAINTS_FIELDS.WATCHED_WORDS.WORDS.max} words between ` +
`${CONSTRAINTS_FIELDS.WATCHED_WORDS.WORD.min} and ${CONSTRAINTS_FIELDS.WATCHED_WORDS.WORD.max} characters each`
)
]
}
export function addWatchedWordsListValidatorFactory (accountGetter: (res: express.Response) => Awaitable<MAccountId>) {
return [
...buildUpdateOrAddValidators({ optional: false }),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
const listName = req.body.listName
if (!await checkListNameIsUnique({ accountId: (await accountGetter(res)).id, listName, res })) return
return next()
}
]
}
export function updateWatchedWordsListValidatorFactory (accountGetter: (res: express.Response) => Awaitable<MAccountId>) {
return [
...buildUpdateOrAddValidators({ optional: true }),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
const currentList = res.locals.watchedWordsList
const listName = req.body.listName
if (listName && !await checkListNameIsUnique({ accountId: (await accountGetter(res)).id, listName, currentList, res })) return
return next()
}
]
}
// ---------------------------------------------------------------------------
// Private
// ---------------------------------------------------------------------------
async function checkListNameIsUnique (options: {
accountId: number
listName: string
res: express.Response
currentList?: MWatchedWordsList
}) {
const { accountId, listName, currentList, res } = options
const existing = await WatchedWordsListModel.loadByListName({ accountId, listName })
if (existing && (!currentList || currentList.id !== existing.id)) {
res.fail({
status: HttpStatusCode.BAD_REQUEST_400,
message: `Watched words list with name ${listName} already exists`
})
return false
}
return true
}