IntenseWebs/PeerTube
3
0
Fork 0
mirror of https://github.com/Chocobozzz/PeerTube.git synced 2026-09-03 20:53:09 -05:00
Code Issues Packages Projects Releases Wiki Activity

Optimize listing videos with collaboration

Browse Source
This commit is contained in:
Chocobozzz
2026-03-23 16:02:42 +01:00
parent af8f5abce0
commit 1f63b98c28
4 changed files with 261 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/tests/src/api/videos/channel-collaborators.ts
+42 -10
View File
@@ -279,20 +279,52 @@ describe('Test channel collaborators', function () {
await servers[0].videos.quickUpload({ name: 'video collab 2', channelId: channelCollabId, privacy: VideoPrivacy.PRIVATE })
for (const token of [ collaborator1, collaborator2 ]) {
const videoSortableColumns = [
'name',
'duration',
'createdAt',
'publishedAt',
'originallyPublishedAt',
'views',
'likes',
'comments',
'trending',
'hot',
'best',
'localVideoFilesSize'
]
// Check sort correctly works with our UNION
for (const sortColumn of videoSortableColumns) {
for (const order of [ '', '-' ]) {
const sort = `${order}${sortColumn}`
{
const videos = await servers[0].videos.listMyVideos({ token, sort, includeCollaborations: true })
expect(videos.total).to.equal(2)
expect(videos.data).to.have.lengthOf(2)
expect(videos.data.map(v => v.name)).to.have.members([ 'video collab 1', 'video collab 2' ])
}
{
const videos = await servers[0].videos.listMyVideos({ token, sort, includeCollaborations: false })
expect(videos.total).to.equal(0)
expect(videos.data).to.have.lengthOf(0)
}
}
}
// Search
{
const videos = await servers[0].videos.listMyVideos({ token, includeCollaborations: true })
const videos = await servers[0].videos.listMyVideos({ token, sort: '-match', includeCollaborations: true, search: 'collab 1' })
expect(videos.total).to.equal(2)
expect(videos.data).to.have.lengthOf(2)
expect(videos.data[0].name).to.equal('video collab 2')
expect(videos.data[1].name).to.equal('video collab 1')
}
{
const videos = await servers[0].videos.listMyVideos({ token, includeCollaborations: false })
expect(videos.total).to.equal(0)
expect(videos.data).to.have.lengthOf(0)
expect(videos.data[0].name).to.equal('video collab 1')
expect(videos.data[1].name).to.equal('video collab 2')
}
}
scripts/dev/server.sh
+1 -1
View File
@@ -22,4 +22,4 @@ npm run resolve-tspaths:server
cp -r "./server/core/static" "./server/core/assets" ./dist/core
cp -r "./server/locales" ./dist
./node_modules/.bin/tsc-watch --build --preserveWatchOutput --verbose --onSuccess 'sh -c "npm run resolve-tspaths:server && NODE_ENV=dev node --inspect dist/server"' server/tsconfig.json
./node_modules/.bin/tsc-watch --build --preserveWatchOutput --verbose --onSuccess 'sh -c "npm run resolve-tspaths:server && NODE_ENV=dev node --inspect --enable-source-maps dist/server"' server/tsconfig.json
server/core/models/shared/abstract-run-query.ts
+1 -1
View File
@@ -8,7 +8,7 @@ import { Col } from 'sequelize/lib/utils'
export class AbstractRunQuery {
protected query: string
protected replacements: any = {}
protected replacements: Record<string, string | number | string[] | number[]> = {}
protected queryConfig = ''
server/core/models/video/sql/video/videos-id-list-query-builder.ts
+217 -126
View File
@@ -65,6 +65,8 @@ export type BuildVideosListQueryOptions = {
hasWebVideoFiles?: boolean
accountId?: number
onlyCollaborated?: boolean
includeCollaborations?: boolean
videoChannelId?: number
@@ -151,20 +153,68 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {
}
private buildIdsListQuery (options: BuildVideosListQueryOptions) {
this.attributes = options.attributes || [ '"video"."id"' ]
if (options.attributes) {
this.attributes = [ ...options.attributes ]
} else if (options.isCount === true) {
this.attributes = [ 'COUNT(*) as "total"' ]
} else {
this.attributes = [ '"video"."id"' ]
}
if (options.group) this.group = options.group
if (options.having) this.having = options.having
let sortColumn: string
let sortDirection: SortDirection
if (options.includeCollaborations) {
if (!options.accountId) throw new Error('accountId parameter is required when includeCollaborations is true')
if (options.sort) {
const { direction, field } = buildSortDirectionAndField(options.sort)
sortColumn = field
sortDirection = direction
this.buildUnionIdsListQuery(options)
} else {
this.buildSingleIdsListQuery(options)
}
}
private buildUnionIdsListQuery (options: BuildVideosListQueryOptions) {
const { sortColumn } = this.buildSortAndPagination(options)
if (sortColumn === 'match') this.attributes.push('"similarity"')
const unionAttributes = new Set([ '"video"."id"' ])
const videoAttributesForSort = this.updateQueryForComplexSort({ ...options, column: sortColumn })
for (const attr of videoAttributesForSort) {
unionAttributes.add(`"video".${attr}`)
}
const baseOptions = {
...options,
attributes: Array.from(unionAttributes),
isCount: false,
count: undefined,
start: undefined,
sort: undefined,
includeCollaborations: false
} satisfies BuildVideosListQueryOptions
const union = this.makeUnion([
{ ...baseOptions, onlyCollaborated: false },
{ ...baseOptions, onlyCollaborated: true }
])
this.query = 'SELECT ' + this.attributes.join(', ') + ' FROM ' +
'(' + union + ') AS "video" ' +
this.joins.join(' ') + ' ' +
this.group + ' ' +
this.having + ' ' +
this.sort + ' ' +
this.limit + ' ' +
this.offset
}
private buildSingleIdsListQuery (options: BuildVideosListQueryOptions) {
const { sortColumn } = this.buildSortAndPagination(options)
this.updateQueryForComplexSort({ ...options, column: sortColumn })
this.joins = this.joins.concat([
'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId"',
@@ -199,8 +249,10 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {
this.whereHost(options.host)
}
if (options.accountId) {
this.whereAccountId({ accountId: options.accountId, includeCollaborations: options.includeCollaborations })
if (options.onlyCollaborated) {
this.whereOnlyCollaborated(options.accountId)
} else if (options.accountId) {
this.whereAccountId(options.accountId)
}
if (options.videoChannelId) {
@@ -276,15 +328,6 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {
this.whereLanguageOneOf(options.languageOneOf)
}
// We don't exclude results in this so if we do a count we don't need to add this complex clause
if (options.isCount !== true) {
if (sortColumn === 'trending') {
this.groupForTrending(options.trendingDays)
} else if (sortColumn === 'hot' || sortColumn === 'best') {
this.addAttributeForHotOrBest(sortColumn, options.user)
}
}
if (options.historyOfUser) {
this.joinHistory(options.historyOfUser.id)
}
@@ -321,23 +364,7 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {
}
}
this.whereSearch(options.search)
if (options.isCount === true) {
this.setCountAttribute()
} else {
if (exists(sortColumn)) {
this.setSort(sortColumn, sortDirection)
}
if (exists(options.count)) {
this.setLimit(options.count)
}
if (exists(options.start)) {
this.setOffset(options.start)
}
}
this.whereSearch(options)
const cteString = this.cte.length !== 0
? `WITH ${this.cte.join(', ')} `
@@ -354,8 +381,48 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {
this.offset
}
private setCountAttribute () {
this.attributes = [ 'COUNT(*) as "total"' ]
private buildSortAndPagination (options: BuildVideosListQueryOptions) {
if (options.isCount) return {}
let sortColumn: string
let sortDirection: SortDirection
if (options.sort) {
const { direction, field } = buildSortDirectionAndField(options.sort)
sortColumn = field
sortDirection = direction
}
if (exists(sortColumn)) {
this.setSort(sortColumn, sortDirection)
}
if (exists(options.count)) {
this.setLimit(options.count)
}
if (exists(options.start)) {
this.setOffset(options.start)
}
return { sortColumn, sortDirection }
}
private makeUnion (queryOptions: BuildVideosListQueryOptions[]) {
const queries: string[] = []
for (const queryOption of queryOptions) {
const builder = new VideosIdListQueryBuilder(this.sequelize)
const { query, replacements, queryConfig } = builder.getQuery(queryOption)
queries.push('(' + query + ')')
this.replacements = { ...this.replacements, ...replacements }
this.queryConfig = this.queryConfig || queryConfig
}
return queries.join(' UNION')
}
private joinHistory (userId: number) {
@@ -435,26 +502,20 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {
this.replacements.host = host
}
private whereAccountId (options: {
accountId: number
includeCollaborations: boolean
}) {
if (options.includeCollaborations !== true) {
this.and.push('"account"."id" = :accountId')
this.replacements.accountId = options.accountId
return
}
private whereAccountId (accountId: number) {
this.and.push('"account"."id" = :accountId')
this.replacements.accountId = accountId
}
private whereOnlyCollaborated (accountId: number) {
this.joins.push(
'LEFT JOIN "videoChannelCollaborator" ON "videoChannelCollaborator"."channelId" = "videoChannel".id ' +
'INNER JOIN "videoChannelCollaborator" ON "videoChannelCollaborator"."channelId" = "videoChannel".id ' +
'AND "videoChannelCollaborator"."state" = :channelCollaboratorState ' +
// Ensure we join with max 1 collaborator to not duplicate rows
'AND "videoChannelCollaborator"."accountId" = :accountId'
)
this.and.push('("account"."id" = :accountId OR "videoChannelCollaborator"."accountId" = :accountId)')
this.replacements.accountId = options.accountId
this.replacements.accountId = accountId
this.replacements.channelCollaboratorState = VideoChannelCollaboratorState.ACCEPTED
}
@@ -713,9 +774,15 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {
)
}
private whereSearch (search?: string) {
private whereSearch (options: {
isCount?: boolean
search?: string
}) {
const { search, isCount } = options
if (!search) {
this.attributes.push('0 as similarity')
if (!isCount) this.attributes.push('0 as similarity')
return
}
@@ -756,7 +823,9 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {
let attribute = `COALESCE("trigramSearch"."similarity", 0)`
if (this.group) attribute = `AVG(${attribute})`
this.attributes.push(`${attribute} as similarity`)
if (!isCount) {
this.attributes.push(`${attribute} as similarity`)
}
}
private whereNotBlacklisted () {
@@ -805,88 +874,110 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {
this.replacements.excludeAlreadyWatchedUserId = userId
}
private groupForTrending (trendingDays: number) {
const viewsGteDate = new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays)
private updateQueryForComplexSort (options: {
column: string
trendingDays?: number
user?: MUserAccountId
}) {
const { column, trendingDays, user } = options
if (!column) return []
this.joins.push('LEFT JOIN "videoStat" ON "video"."id" = "videoStat"."videoId" AND "videoStat"."startDate" >= :viewsGteDate')
this.replacements.viewsGteDate = viewsGteDate
let videoAttributesForSort: string[] = []
this.attributes.push('COALESCE(SUM("videoStat"."views"), 0) AS "score"')
this.group = 'GROUP BY "video"."id"'
}
private addAttributeForHotOrBest (sort: string, user?: MUserAccountId) {
/**
* "Hotness" is a measure based on absolute view/comment/like/dislike numbers,
* with fixed weights only applied to their log values.
*
* This algorithm gives little chance for an old video to have a good score,
* for which recent spikes in interactions could be a sign of "hotness" and
* justify a better score. However there are multiple ways to achieve that
* goal, which is left for later. Yes, this is a TODO :)
*
* notes:
* - weights and base score are in number of half-days.
* - all comments are counted, regardless of being written by the video author or not
* see https://github.com/reddit-archive/reddit/blob/master/r2/r2/lib/db/_sorts.pyx#L47-L58
* - we have less interactions than on reddit, so multiply weights by an arbitrary factor
*/
const weights = {
like: 3 * 50,
dislike: -3 * 50,
view: Math.floor((1 / 3) * 50),
comment: 2 * 50, // a comment takes more time than a like to do, but can be done multiple times
history: -2 * 50
}
let attribute = `LOG(GREATEST(1, "video"."likes" - 1)) * ${weights.like} ` + // likes (+)
`+ LOG(GREATEST(1, "video"."dislikes" - 1)) * ${weights.dislike} ` + // dislikes (-)
`+ LOG("video"."views" + 1) * ${weights.view} ` + // views (+)
`+ LOG(GREATEST(1, "video"."comments")) * ${weights.comment} ` + // comments (+)
'+ (SELECT (EXTRACT(epoch FROM "video"."publishedAt") - 1446156582) / 47000) ' // base score (in number of half-days)
if (sort.endsWith('best') && user) {
this.joins.push(
'LEFT JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId" AND "userVideoHistory"."userId" = :bestUser'
)
this.replacements.bestUser = user.id
attribute += `+ POWER(CASE WHEN "userVideoHistory"."id" IS NULL THEN 0 ELSE 1 END, 2.0) * ${weights.history} `
}
attribute += 'AS "score"'
this.attributes.push(attribute)
}
private setSort (column: string, direction: 'ASC' | 'DESC') {
if (column === 'originallyPublishedAt') {
this.attributes.push('COALESCE("video"."originallyPublishedAt", "video"."publishedAt") AS "publishedAtForOrder"')
}
this.attributes.push(
`COALESCE("video"."originallyPublishedAt", "video"."publishedAt") AS "publishedAtForOrder"`
)
if (column === 'localVideoFilesSize') {
videoAttributesForSort.push('"originallyPublishedAt"', '"publishedAt"')
} else if (column === 'localVideoFilesSize') {
this.attributes.push(
'(' +
'CASE ' +
'WHEN "video"."remote" IS TRUE THEN 0 ' + // Consider remote videos with size of 0
'ELSE (' +
'(SELECT COALESCE(SUM(size), 0) FROM "videoFile" WHERE "videoFile"."videoId" = "video"."id")' +
' + ' +
'(' +
'SELECT COALESCE(SUM(size), 0) FROM "videoFile" ' +
'INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
'AND "videoStreamingPlaylist"."videoId" = "video"."id"' +
')' +
' + ' +
'(' +
'SELECT COALESCE(SUM(size), 0) FROM "videoSource" ' +
'WHERE "videoSource"."videoId" = "video"."id" AND "videoSource"."storage" IS NOT NULL' +
')' +
') END' +
` WHEN "video"."remote" IS TRUE THEN 0 ` + // Consider remote videos with size of 0
' ELSE (' +
` (SELECT COALESCE(SUM(size), 0) FROM "videoFile" WHERE "videoFile"."videoId" = "video"."id")` +
' + ' +
` (` +
` SELECT COALESCE(SUM(size), 0) FROM "videoFile" ` +
` INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ` +
` AND "videoStreamingPlaylist"."videoId" = "video"."id"` +
' )' +
' + ' +
' (' +
' SELECT COALESCE(SUM(size), 0) FROM "videoSource" ' +
` WHERE "videoSource"."videoId" = "video"."id" AND "videoSource"."storage" IS NOT NULL` +
' )' +
' ) END' +
') AS "localVideoFilesSize"'
)
videoAttributesForSort.push('"remote"')
} else if (column === 'trending') {
const viewsGteDate = new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays)
this.joins.push(
`LEFT JOIN "videoStat" ON "video"."id" = "videoStat"."videoId" AND "videoStat"."startDate" >= :viewsGteDate`
)
this.replacements.viewsGteDate = viewsGteDate
this.attributes.push('COALESCE(SUM("videoStat"."views"), 0) AS "score"')
videoAttributesForSort.push('"views"')
this.group = `GROUP BY "video"."id"`
} else if (column === 'hot' || column === 'best') {
/**
* "Hotness" is a measure based on absolute view/comment/like/dislike numbers,
* with fixed weights only applied to their log values.
*
* This algorithm gives little chance for an old video to have a good score,
* for which recent spikes in interactions could be a sign of "hotness" and
* justify a better score. However there are multiple ways to achieve that
* goal, which is left for later. Yes, this is a TODO :)
*
* notes:
* - weights and base score are in number of half-days.
* - all comments are counted, regardless of being written by the video author or not
* see https://github.com/reddit-archive/reddit/blob/master/r2/r2/lib/db/_sorts.pyx#L47-L58
* - we have less interactions than on reddit, so multiply weights by an arbitrary factor
*/
const weights = {
like: 3 * 50,
dislike: -3 * 50,
view: Math.floor((1 / 3) * 50),
comment: 2 * 50, // a comment takes more time than a like to do, but can be done multiple times
history: -2 * 50
}
let attribute = `LOG(GREATEST(1, "video"."likes" - 1)) * ${weights.like} ` + // likes (+)
`+ LOG(GREATEST(1, "video"."dislikes" - 1)) * ${weights.dislike} ` + // dislikes (-)
`+ LOG("video"."views" + 1) * ${weights.view} ` + // views (+)
`+ LOG(GREATEST(1, "video"."comments")) * ${weights.comment} ` + // comments (+)
`+ (SELECT (EXTRACT(epoch FROM "video"."publishedAt") - 1446156582) / 47000) ` // base score (in number of half-days)
if (column === 'best' && user) {
this.joins.push(
`LEFT JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId" ` +
`AND "userVideoHistory"."userId" = :bestUser`
)
this.replacements.bestUser = user.id
attribute += `+ POWER(CASE WHEN "userVideoHistory"."id" IS NULL THEN 0 ELSE 1 END, 2.0) * ${weights.history} `
}
attribute += 'AS "score"'
this.attributes.push(attribute)
videoAttributesForSort = [ '"likes"', '"dislikes"', '"views"', '"comments"', '"publishedAt"' ]
} else if (column === 'match') {
videoAttributesForSort = []
} else {
videoAttributesForSort = [ `"${column}"` ]
}
return videoAttributesForSort
}
private setSort (column: string, direction: 'ASC' | 'DESC') {
this.sort = this.buildOrder(column, direction)
}
@@ -897,7 +988,7 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {
if (column === 'total') return `ORDER BY "total" ${direction}`
if ([ 'trending', 'hot', 'best' ].includes(column)) { // Sort by aggregation
return `ORDER BY "score" ${direction}, "video"."views" ${direction}`
return `ORDER BY "score" ${direction}`
}
let firstSort: string