diff --git a/client/src/app/+videos/+video-edit/shared/caption/video-caption-edit-modal-content.component.ts b/client/src/app/+videos/+video-edit/shared/caption/video-caption-edit-modal-content.component.ts index 6bdfa266c..18efc34de 100644 --- a/client/src/app/+videos/+video-edit/shared/caption/video-caption-edit-modal-content.component.ts +++ b/client/src/app/+videos/+video-edit/shared/caption/video-caption-edit-modal-content.component.ts @@ -135,10 +135,10 @@ export class VideoCaptionEditModalContentComponent extends FormReactive implemen return } - const { captionPath } = this.videoCaption - if (!captionPath) return + const { fileUrl } = this.videoCaption + if (!fileUrl) return - this.videoCaptionService.getCaptionContent({ captionPath }) + this.videoCaptionService.getCaptionContent({ fileUrl }) .subscribe(content => { this.loadSegments(content) }) diff --git a/client/src/app/+videos/+video-edit/shared/video-edit.component.html b/client/src/app/+videos/+video-edit/shared/video-edit.component.html index deaadef79..b03c9129a 100644 --- a/client/src/app/+videos/+video-edit/shared/video-edit.component.html +++ b/client/src/app/+videos/+video-edit/shared/video-edit.component.html @@ -209,7 +209,7 @@ } @else { {{ getCaptionLabel(videoCaption) }}
Already uploaded on {{ videoCaption.updatedAt | ptDate }} ✔
diff --git a/client/src/app/+videos/+video-watch/shared/player-widgets/video-transcription.component.ts b/client/src/app/+videos/+video-watch/shared/player-widgets/video-transcription.component.ts index 971f911ed..0e543afec 100644 --- a/client/src/app/+videos/+video-watch/shared/player-widgets/video-transcription.component.ts +++ b/client/src/app/+videos/+video-watch/shared/player-widgets/video-transcription.component.ts @@ -155,7 +155,7 @@ export class VideoTranscriptionComponent implements OnInit, OnChanges { } private parseCurrentCaption () { - this.captionService.getCaptionContent({ captionPath: this.currentCaption.captionPath }) + this.captionService.getCaptionContent({ fileUrl: this.currentCaption.fileUrl }) .subscribe({ next: content => { try { diff --git a/client/src/app/+videos/+video-watch/video-watch.component.ts b/client/src/app/+videos/+video-watch/video-watch.component.ts index 3454b55b0..ca4956439 100644 --- a/client/src/app/+videos/+video-watch/video-watch.component.ts +++ b/client/src/app/+videos/+video-watch/video-watch.component.ts @@ -787,12 +787,12 @@ export class VideoWatchComponent implements OnInit, OnDestroy { label: c.language.label, language: c.language.id, automaticallyGenerated: c.automaticallyGenerated, - src: environment.apiUrl + c.captionPath + src: c.fileUrl })) const storyboard = storyboards.length !== 0 ? { - url: environment.apiUrl + storyboards[0].storyboardPath, + url: storyboards[0].fileUrl, height: storyboards[0].spriteHeight, width: storyboards[0].spriteWidth, interval: storyboards[0].spriteDuration diff --git a/client/src/app/shared/shared-main/account/actor.model.ts b/client/src/app/shared/shared-main/account/actor.model.ts index 1a1dc2ce6..5a7ff4087 100644 --- a/client/src/app/shared/shared-main/account/actor.model.ts +++ b/client/src/app/shared/shared-main/account/actor.model.ts @@ -17,7 +17,7 @@ export abstract class Actor implements ServerActor { isLocal: boolean - static GET_ACTOR_AVATAR_URL (actor: { avatars: { width: number, url?: string, path: string }[] }, size?: number) { + static GET_ACTOR_AVATAR_URL (actor: { avatars: { width: number, fileUrl?: string, url?: string, path: string }[] }, size?: number) { const avatarsAscWidth = actor.avatars.sort((a, b) => a.width - b.width) const avatar = size && avatarsAscWidth.length > 1 @@ -25,6 +25,7 @@ export abstract class Actor implements ServerActor { : avatarsAscWidth[avatarsAscWidth.length - 1] // Bigger one if (!avatar) return '' + if (avatar.fileUrl) return avatar.fileUrl if (avatar.url) return avatar.url const absoluteAPIUrl = getAbsoluteAPIUrl() diff --git a/client/src/app/shared/shared-main/channel/video-channel.model.ts b/client/src/app/shared/shared-main/channel/video-channel.model.ts index 6fae772d7..88cf87f6a 100644 --- a/client/src/app/shared/shared-main/channel/video-channel.model.ts +++ b/client/src/app/shared/shared-main/channel/video-channel.model.ts @@ -25,7 +25,12 @@ export class VideoChannel extends Actor implements ServerVideoChannel { viewsPerDay?: ViewsPerDate[] totalViews?: number - static GET_ACTOR_AVATAR_URL (actor: { avatars: { width: number, url?: string, path: string }[] }, size: number) { + static GET_ACTOR_AVATAR_URL ( + actor: { + avatars: { width: number, fileUrl?: string, url?: string, path: string }[] + }, + size: number + ) { return Actor.GET_ACTOR_AVATAR_URL(actor, size) } diff --git a/client/src/app/shared/shared-main/users/user-notification.model.ts b/client/src/app/shared/shared-main/users/user-notification.model.ts index bd459b2c3..b82c02c61 100644 --- a/client/src/app/shared/shared-main/users/user-notification.model.ts +++ b/client/src/app/shared/shared-main/users/user-notification.model.ts @@ -293,11 +293,17 @@ export class UserNotification implements UserNotificationServer { return [ this.buildVideoUrl(comment.video), { threadId: comment.threadId } ] } - private setAccountAvatarUrl (actor: { avatarUrl?: string, avatars: { width: number, url?: string, path: string }[] }) { + private setAccountAvatarUrl (actor: { + avatarUrl?: string + avatars: { width: number, fileUrl?: string, url?: string, path: string }[] + }) { actor.avatarUrl = VideoChannel.GET_ACTOR_AVATAR_URL(actor, 48) || Account.GET_DEFAULT_AVATAR_URL(48) } - private setVideoChannelAvatarUrl (actor: { avatarUrl?: string, avatars: { width: number, url?: string, path: string }[] }) { + private setVideoChannelAvatarUrl (actor: { + avatarUrl?: string + avatars: { width: number, fileUrl?: string, url?: string, path: string }[] + }) { actor.avatarUrl = VideoChannel.GET_ACTOR_AVATAR_URL(actor, 48) || VideoChannel.GET_DEFAULT_AVATAR_URL(48) } } diff --git a/client/src/app/shared/shared-main/video-caption/video-caption-edit.model.ts b/client/src/app/shared/shared-main/video-caption/video-caption-edit.model.ts index 08a9587d7..c6d8b7edc 100644 --- a/client/src/app/shared/shared-main/video-caption/video-caption-edit.model.ts +++ b/client/src/app/shared/shared-main/video-caption/video-caption-edit.model.ts @@ -11,4 +11,4 @@ export interface VideoCaptionEdit { updatedAt?: string } -export type VideoCaptionWithPathEdit = VideoCaptionEdit & { captionPath?: string } +export type VideoCaptionWithPathEdit = VideoCaptionEdit & { fileUrl?: string } diff --git a/client/src/app/shared/shared-main/video-caption/video-caption.service.ts b/client/src/app/shared/shared-main/video-caption/video-caption.service.ts index 210b73d9c..a7a288fc3 100644 --- a/client/src/app/shared/shared-main/video-caption/video-caption.service.ts +++ b/client/src/app/shared/shared-main/video-caption/video-caption.service.ts @@ -6,7 +6,6 @@ import { peertubeTranslate, sortBy } from '@peertube/peertube-core-utils' import { PeerTubeProblemDocument, ResultList, ServerErrorCode, Video, VideoCaption, VideoCaptionGenerate } from '@peertube/peertube-models' import { Observable, from, of, throwError } from 'rxjs' import { catchError, concatMap, map, switchMap, toArray } from 'rxjs/operators' -import { environment } from '../../../../environments/environment' import { VideoPasswordService } from '../video/video-password.service' import { VideoService } from '../video/video.service' import { VideoCaptionEdit } from './video-caption-edit.model' @@ -72,8 +71,8 @@ export class VideoCaptionService { return obs } - getCaptionContent ({ captionPath }: Pick) { - return this.authHttp.get(environment.originServerUrl + captionPath, { responseType: 'text' }) + getCaptionContent ({ fileUrl }: Pick) { + return this.authHttp.get(fileUrl, { responseType: 'text' }) } // --------------------------------------------------------------------------- diff --git a/client/src/app/shared/shared-video-miniature/download/subtitle-files-download.component.ts b/client/src/app/shared/shared-video-miniature/download/subtitle-files-download.component.ts index edfe02e12..089bb2362 100644 --- a/client/src/app/shared/shared-video-miniature/download/subtitle-files-download.component.ts +++ b/client/src/app/shared/shared-video-miniature/download/subtitle-files-download.component.ts @@ -60,6 +60,6 @@ export class SubtitleFilesDownloadComponent implements OnInit { const caption = this.getCaption() if (!caption) return '' - return window.location.origin + caption.captionPath + return caption.fileUrl } } diff --git a/client/src/standalone/videos/shared/player-options-builder.ts b/client/src/standalone/videos/shared/player-options-builder.ts index 94b103f50..4e3daec57 100644 --- a/client/src/standalone/videos/shared/player-options-builder.ts +++ b/client/src/standalone/videos/shared/player-options-builder.ts @@ -335,7 +335,7 @@ export class PlayerOptionsBuilder { if (!storyboards || storyboards.length === 0) return undefined return { - url: getBackendUrl() + storyboards[0].storyboardPath, + url: storyboards[0].fileUrl, height: storyboards[0].spriteHeight, width: storyboards[0].spriteWidth, interval: storyboards[0].spriteDuration @@ -428,7 +428,7 @@ export class PlayerOptionsBuilder { label: peertubeTranslate(c.language.label, translations), language: c.language.id, automaticallyGenerated: c.automaticallyGenerated, - src: getBackendUrl() + c.captionPath + src: c.fileUrl })) } diff --git a/config/default.yaml b/config/default.yaml index 3fca070eb..bbb023cac 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -260,6 +260,12 @@ object_storage: prefix: '' base_url: '' + # Video captions + captions: + bucket_name: 'captions' + prefix: '' + base_url: '' + log: level: 'info' # 'debug' | 'info' | 'warn' | 'error' diff --git a/config/production.yaml.example b/config/production.yaml.example index 85ceb5259..97e682c2a 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -258,6 +258,12 @@ object_storage: prefix: '' base_url: '' + # Video captions + captions: + bucket_name: 'captions' + prefix: '' + base_url: '' + log: level: 'info' # 'debug' | 'info' | 'warn' | 'error' diff --git a/packages/models/src/actors/actor-image.model.ts b/packages/models/src/actors/actor-image.model.ts index cfe44ac15..b2b684a53 100644 --- a/packages/models/src/actors/actor-image.model.ts +++ b/packages/models/src/actors/actor-image.model.ts @@ -1,9 +1,13 @@ export interface ActorImage { width: number - path: string + // TODO: remove, deprecated in 7.1 + path: string + // TODO: remove, deprecated in 7.1 url?: string + fileUrl: string + createdAt: Date | string updatedAt: Date | string } diff --git a/packages/models/src/server/job.model.ts b/packages/models/src/server/job.model.ts index a6534a1b7..26883a77c 100644 --- a/packages/models/src/server/job.model.ts +++ b/packages/models/src/server/job.model.ts @@ -204,12 +204,30 @@ export interface DeleteResumableUploadMetaFilePayload { filepath: string } -export interface MoveStoragePayload { +// --------------------------------------------------------------------------- + +export type MoveStoragePayload = MoveVideoStoragePayload | MoveCaptionPayload + +export interface MoveVideoStoragePayload { videoUUID: string isNewVideo: boolean previousVideoState: VideoStateType } +export interface MoveCaptionPayload { + captionId: number +} + +export function isMoveVideoStoragePayload (payload: any): payload is MoveVideoStoragePayload { + return 'videoUUID' in payload +} + +export function isMoveCaptionPayload (payload: any): payload is MoveCaptionPayload { + return 'captionId' in payload +} + +// --------------------------------------------------------------------------- + export type VideoStudioTaskCutPayload = VideoStudioTaskCut export type VideoStudioTaskIntroPayload = { diff --git a/packages/models/src/users/user-notification.model.ts b/packages/models/src/users/user-notification.model.ts index 678262589..cc4623255 100644 --- a/packages/models/src/users/user-notification.model.ts +++ b/packages/models/src/users/user-notification.model.ts @@ -58,7 +58,11 @@ export interface VideoInfo { export interface AvatarInfo { width: number + + // TODO: remove, deprecated in 7.1 path: string + + fileUrl: string } export interface ActorInfo { diff --git a/packages/models/src/videos/caption/video-caption.model.ts b/packages/models/src/videos/caption/video-caption.model.ts index b28a6a05d..f0777fb14 100644 --- a/packages/models/src/videos/caption/video-caption.model.ts +++ b/packages/models/src/videos/caption/video-caption.model.ts @@ -2,7 +2,12 @@ import { VideoConstant } from '../video-constant.model.js' export interface VideoCaption { language: VideoConstant + + // TODO: remove, deprecated in 7.1 captionPath: string + + fileUrl: string + automaticallyGenerated: boolean updatedAt: string } diff --git a/packages/models/src/videos/storyboard.model.ts b/packages/models/src/videos/storyboard.model.ts index c92c81f09..61274a87e 100644 --- a/packages/models/src/videos/storyboard.model.ts +++ b/packages/models/src/videos/storyboard.model.ts @@ -1,6 +1,9 @@ export interface Storyboard { + // TODO: remove, deprecated in 7.1 storyboardPath: string + fileUrl: string + totalHeight: number totalWidth: number diff --git a/packages/server-commands/src/cli/cli-command.ts b/packages/server-commands/src/cli/cli-command.ts index 8b9400c85..772027280 100644 --- a/packages/server-commands/src/cli/cli-command.ts +++ b/packages/server-commands/src/cli/cli-command.ts @@ -4,24 +4,26 @@ import { AbstractCommand } from '../shared/index.js' export class CLICommand extends AbstractCommand { static exec (command: string) { - return new Promise((res, rej) => { - exec(command, (err, stdout, _stderr) => { + return new Promise<{ stdout: string, stderr: string }>((res, rej) => { + exec(command, (err, stdout, stderr) => { if (err) return rej(err) - return res(stdout) + return res({ stdout, stderr }) }) }) } - getEnv () { - return `NODE_ENV=test NODE_APP_INSTANCE=${this.server.internalServerNumber}` + static getNodeConfigEnv (configOverride?: any) { + return configOverride + ? `NODE_CONFIG='${JSON.stringify(configOverride)}'` + : '' + } + + getEnv (configOverride?: any) { + return `NODE_ENV=test NODE_APP_INSTANCE=${this.server.internalServerNumber} ${CLICommand.getNodeConfigEnv(configOverride)}` } async execWithEnv (command: string, configOverride?: any) { - const prefix = configOverride - ? `NODE_CONFIG='${JSON.stringify(configOverride)}'` - : '' - - return CLICommand.exec(`${prefix} ${this.getEnv()} ${command}`) + return CLICommand.exec(`${this.getEnv(configOverride)} ${command}`) } } diff --git a/packages/server-commands/src/server/object-storage-command.ts b/packages/server-commands/src/server/object-storage-command.ts index 7959dceaa..37aeb109b 100644 --- a/packages/server-commands/src/server/object-storage-command.ts +++ b/packages/server-commands/src/server/object-storage-command.ts @@ -31,8 +31,9 @@ export class ObjectStorageCommand { getDefaultMockConfig (options: { storeLiveStreams?: boolean // default true + proxifyPrivateFiles?: boolean // default true } = {}) { - const { storeLiveStreams = true } = options + const { storeLiveStreams = true, proxifyPrivateFiles = true } = options return { object_storage: { @@ -58,6 +59,14 @@ export class ObjectStorageCommand { original_video_files: { bucket_name: this.getMockOriginalFileBucketName() + }, + + captions: { + bucket_name: this.getMockCaptionsBucketName() + }, + + proxy: { + proxify_private_files: proxifyPrivateFiles } } } @@ -79,9 +88,16 @@ export class ObjectStorageCommand { return `http://${this.getMockOriginalFileBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/` } + getMockCaptionFileBaseUrl () { + return `http://${this.getMockCaptionsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/` + } + async prepareDefaultMockBuckets () { await this.createMockBucket(this.getMockStreamingPlaylistsBucketName()) await this.createMockBucket(this.getMockWebVideosBucketName()) + await this.createMockBucket(this.getMockOriginalFileBucketName()) + await this.createMockBucket(this.getMockUserExportBucketName()) + await this.createMockBucket(this.getMockCaptionsBucketName()) } async createMockBucket (name: string) { @@ -124,6 +140,10 @@ export class ObjectStorageCommand { return this.getMockBucketName(name) } + getMockCaptionsBucketName (name = 'captions') { + return this.getMockBucketName(name) + } + getMockBucketName (name: string) { return `${this.seed}-${name}` } diff --git a/packages/tests/src/api/server/follows.ts b/packages/tests/src/api/server/follows.ts index 264f408c2..682c780c9 100644 --- a/packages/tests/src/api/server/follows.ts +++ b/packages/tests/src/api/server/follows.ts @@ -575,7 +575,7 @@ describe('Test follows', function () { expect(caption1.language.id).to.equal('ar') expect(caption1.language.label).to.equal('Arabic') expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/.+-ar.vtt$')) - await testCaptionFile(servers[0].url, caption1.captionPath, 'Subtitle good 2.') + await testCaptionFile(caption1.fileUrl, 'Subtitle good 2.') }) it('Should unfollow server 3 on server 1 and does not list server 3 videos', async function () { diff --git a/packages/tests/src/api/videos/multiple-servers.ts b/packages/tests/src/api/videos/multiple-servers.ts index 811e705f8..bcb84301a 100644 --- a/packages/tests/src/api/videos/multiple-servers.ts +++ b/packages/tests/src/api/videos/multiple-servers.ts @@ -141,7 +141,7 @@ describe('Test multiple servers', function () { await makeGetRequest({ url: server.url, - path: image.path, + path: image.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) } diff --git a/packages/tests/src/api/videos/video-captions.ts b/packages/tests/src/api/videos/video-captions.ts index 9913041ad..1d47e807e 100644 --- a/packages/tests/src/api/videos/video-captions.ts +++ b/packages/tests/src/api/videos/video-captions.ts @@ -1,17 +1,23 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { expect } from 'chai' import { wait } from '@peertube/peertube-core-utils' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' import { cleanupTests, createMultipleServers, doubleFollow, + makeRawRequest, + ObjectStorageCommand, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@peertube/peertube-server-commands' import { testCaptionFile } from '@tests/shared/captions.js' +import { expectStartWith } from '@tests/shared/checks.js' +import { checkDirectoryIsEmpty } from '@tests/shared/directories.js' import { checkVideoFilesWereRemoved } from '@tests/shared/videos.js' +import { expect } from 'chai' +import { HttpStatusCode } from '../../../../models/src/http/http-status-codes.js' describe('Test video captions', function () { const uuidRegex = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' @@ -35,154 +41,315 @@ describe('Test video captions', function () { await waitJobs(servers) }) - it('Should list the captions and return an empty list', async function () { - for (const server of servers) { - const body = await server.captions.list({ videoId: videoUUID }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - }) + describe('Common on filesystem', function () { - it('Should create two new captions', async function () { - this.timeout(30000) - - await servers[0].captions.add({ - language: 'ar', - videoId: videoUUID, - fixture: 'subtitle-good1.vtt' + it('Should list the captions and return an empty list', async function () { + for (const server of servers) { + const body = await server.captions.list({ videoId: videoUUID }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } }) - await servers[0].captions.add({ - language: 'zh', - videoId: videoUUID, - fixture: 'subtitle-good2.vtt', - mimeType: 'application/octet-stream' + it('Should create two new captions', async function () { + this.timeout(30000) + + await servers[0].captions.add({ + language: 'ar', + videoId: videoUUID, + fixture: 'subtitle-good1.vtt' + }) + + await servers[0].captions.add({ + language: 'zh', + videoId: videoUUID, + fixture: 'subtitle-good2.vtt', + mimeType: 'application/octet-stream' + }) + + await waitJobs(servers) }) - await waitJobs(servers) - }) + it('Should list these uploaded captions', async function () { + for (const server of servers) { + const body = await server.captions.list({ videoId: videoUUID }) + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) - it('Should list these uploaded captions', async function () { - for (const server of servers) { - const body = await server.captions.list({ videoId: videoUUID }) - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(2) + const caption1 = body.data[0] + expect(caption1.language.id).to.equal('ar') + expect(caption1.language.label).to.equal('Arabic') + expect(caption1.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/${uuidRegex}-ar.vtt$`)) + expect(caption1.fileUrl).to.match(new RegExp(`^${server.url}/lazy-static/video-captions/${uuidRegex}-ar.vtt$`)) + expect(caption1.automaticallyGenerated).to.be.false + await testCaptionFile(caption1.fileUrl, 'Subtitle good 1.') - const caption1 = body.data[0] - expect(caption1.language.id).to.equal('ar') - expect(caption1.language.label).to.equal('Arabic') - expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-ar.vtt$')) - expect(caption1.automaticallyGenerated).to.be.false - await testCaptionFile(server.url, caption1.captionPath, 'Subtitle good 1.') - - const caption2 = body.data[1] - expect(caption2.language.id).to.equal('zh') - expect(caption2.language.label).to.equal('Chinese') - expect(caption2.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-zh.vtt$')) - expect(caption1.automaticallyGenerated).to.be.false - await testCaptionFile(server.url, caption2.captionPath, 'Subtitle good 2.') - } - }) - - it('Should replace an existing caption', async function () { - this.timeout(30000) - - await servers[0].captions.add({ - language: 'ar', - videoId: videoUUID, - fixture: 'subtitle-good2.vtt' + const caption2 = body.data[1] + expect(caption2.language.id).to.equal('zh') + expect(caption2.language.label).to.equal('Chinese') + expect(caption2.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/${uuidRegex}-zh.vtt$`)) + expect(caption2.fileUrl).to.match(new RegExp(`^${server.url}/lazy-static/video-captions/${uuidRegex}-zh.vtt$`)) + expect(caption1.automaticallyGenerated).to.be.false + await testCaptionFile(caption2.fileUrl, 'Subtitle good 2.') + } }) - await waitJobs(servers) - }) + it('Should replace an existing caption', async function () { + this.timeout(30000) - it('Should have this caption updated', async function () { - for (const server of servers) { - const body = await server.captions.list({ videoId: videoUUID }) - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(2) + await servers[0].captions.add({ + language: 'ar', + videoId: videoUUID, + fixture: 'subtitle-good2.vtt' + }) - const caption1 = body.data[0] - expect(caption1.language.id).to.equal('ar') - expect(caption1.language.label).to.equal('Arabic') - expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-ar.vtt$')) - await testCaptionFile(server.url, caption1.captionPath, 'Subtitle good 2.') - } - }) - - it('Should replace an existing caption with a srt file and convert it', async function () { - this.timeout(30000) - - await servers[0].captions.add({ - language: 'ar', - videoId: videoUUID, - fixture: 'subtitle-good.srt' + await waitJobs(servers) }) - await waitJobs(servers) + it('Should have this caption updated', async function () { + for (const server of servers) { + const body = await server.captions.list({ videoId: videoUUID }) + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) - // Cache invalidation - await wait(3000) + const caption1 = body.data[0] + expect(caption1.language.id).to.equal('ar') + expect(caption1.language.label).to.equal('Arabic') + expect(caption1.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/${uuidRegex}-ar.vtt$`)) + expect(caption1.fileUrl).to.match(new RegExp(`^${server.url}/lazy-static/video-captions/${uuidRegex}-ar.vtt$`)) + await testCaptionFile(caption1.fileUrl, 'Subtitle good 2.') + } + }) + + it('Should replace an existing caption with a srt file and convert it', async function () { + this.timeout(30000) + + await servers[0].captions.add({ + language: 'ar', + videoId: videoUUID, + fixture: 'subtitle-good.srt' + }) + + await waitJobs(servers) + + // Cache invalidation + await wait(3000) + }) + + it('Should have this caption updated and converted', async function () { + for (const server of servers) { + const body = await server.captions.list({ videoId: videoUUID }) + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) + + const caption1 = body.data[0] + expect(caption1.language.id).to.equal('ar') + expect(caption1.language.label).to.equal('Arabic') + expect(caption1.fileUrl).to.match(new RegExp(`${server.url}/lazy-static/video-captions/${uuidRegex}-ar.vtt$`)) + + const expected = 'WEBVTT FILE\r\n' + + '\r\n' + + '1\r\n' + + '00:00:01.600 --> 00:00:04.200\r\n' + + 'English (US)\r\n' + + '\r\n' + + '2\r\n' + + '00:00:05.900 --> 00:00:07.999\r\n' + + 'This is a subtitle in American English\r\n' + + '\r\n' + + '3\r\n' + + '00:00:10.000 --> 00:00:14.000\r\n' + + 'Adding subtitles is very easy to do\r\n' + await testCaptionFile(caption1.fileUrl, expected) + } + }) + + it('Should remove one caption', async function () { + this.timeout(30000) + + await servers[0].captions.delete({ videoId: videoUUID, language: 'ar' }) + + await waitJobs(servers) + }) + + it('Should only list the caption that was not deleted', async function () { + for (const server of servers) { + const body = await server.captions.list({ videoId: videoUUID }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const caption = body.data[0] + + expect(caption.language.id).to.equal('zh') + expect(caption.language.label).to.equal('Chinese') + expect(caption.fileUrl).to.match(new RegExp(`^${server.url}/lazy-static/video-captions/${uuidRegex}-zh.vtt$`)) + await testCaptionFile(caption.fileUrl, 'Subtitle good 2.') + } + }) + + it('Should remove the video, and thus all video captions', async function () { + const video = await servers[0].videos.get({ id: videoUUID }) + const { data: captions } = await servers[0].captions.list({ videoId: videoUUID }) + + await servers[0].videos.remove({ id: videoUUID }) + + await checkVideoFilesWereRemoved({ server: servers[0], video, captions }) + }) }) - it('Should have this caption updated and converted', async function () { - for (const server of servers) { - const body = await server.captions.list({ videoId: videoUUID }) - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(2) + describe('On object storage', function () { + let videoUUID: string + let oldFileUrlsAr: string[] = [] + const oldFileUrlsZh: string[] = [] - const caption1 = body.data[0] - expect(caption1.language.id).to.equal('ar') - expect(caption1.language.label).to.equal('Arabic') - expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-ar.vtt$')) + if (areMockObjectStorageTestsDisabled()) return - const expected = 'WEBVTT FILE\r\n' + - '\r\n' + - '1\r\n' + - '00:00:01.600 --> 00:00:04.200\r\n' + - 'English (US)\r\n' + - '\r\n' + - '2\r\n' + - '00:00:05.900 --> 00:00:07.999\r\n' + - 'This is a subtitle in American English\r\n' + - '\r\n' + - '3\r\n' + - '00:00:10.000 --> 00:00:14.000\r\n' + - 'Adding subtitles is very easy to do\r\n' - await testCaptionFile(server.url, caption1.captionPath, expected) - } - }) + const objectStorage = new ObjectStorageCommand() - it('Should remove one caption', async function () { - this.timeout(30000) + before(async function () { + this.timeout(120000) - await servers[0].captions.delete({ videoId: videoUUID, language: 'ar' }) + const configOverride = objectStorage.getDefaultMockConfig() + await objectStorage.prepareDefaultMockBuckets() - await waitJobs(servers) - }) + await servers[0].kill() + await servers[0].run(configOverride) - it('Should only list the caption that was not deleted', async function () { - for (const server of servers) { - const body = await server.captions.list({ videoId: videoUUID }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) + const { uuid } = await servers[0].videos.quickUpload({ name: 'object storage' }) + videoUUID = uuid - const caption = body.data[0] + await waitJobs(servers) + }) - expect(caption.language.id).to.equal('zh') - expect(caption.language.label).to.equal('Chinese') - expect(caption.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-zh.vtt$')) - await testCaptionFile(server.url, caption.captionPath, 'Subtitle good 2.') - } - }) + it('Should create captions', async function () { + this.timeout(30000) - it('Should remove the video, and thus all video captions', async function () { - const video = await servers[0].videos.get({ id: videoUUID }) - const { data: captions } = await servers[0].captions.list({ videoId: videoUUID }) + await servers[0].captions.add({ + language: 'ar', + videoId: videoUUID, + fixture: 'subtitle-good1.vtt' + }) - await servers[0].videos.remove({ id: videoUUID }) + await servers[0].captions.add({ + language: 'zh', + videoId: videoUUID, + fixture: 'subtitle-good2.vtt', + mimeType: 'application/octet-stream' + }) - await checkVideoFilesWereRemoved({ server: servers[0], video, captions }) + await waitJobs(servers) + }) + + it('Should have these captions in object storage', async function () { + for (const server of servers) { + const body = await server.captions.list({ videoId: videoUUID }) + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) + + { + const caption1 = body.data[0] + expect(caption1.language.id).to.equal('ar') + + if (server === servers[0]) { + expectStartWith(caption1.fileUrl, objectStorage.getMockCaptionFileBaseUrl()) + expect(caption1.captionPath).to.be.null + + oldFileUrlsAr.push(caption1.fileUrl) + } else { + expect(caption1.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/${uuidRegex}-ar.vtt$`)) + expect(caption1.fileUrl).to.match(new RegExp(`^${server.url}/lazy-static/video-captions/${uuidRegex}-ar.vtt$`)) + } + + await testCaptionFile(caption1.fileUrl, 'Subtitle good 1.') + } + + { + const caption2 = body.data[1] + expect(caption2.language.id).to.equal('zh') + + if (server === servers[0]) { + expectStartWith(caption2.fileUrl, objectStorage.getMockCaptionFileBaseUrl()) + expect(caption2.captionPath).to.be.null + + oldFileUrlsZh.push(caption2.fileUrl) + } + + await testCaptionFile(caption2.fileUrl, 'Subtitle good 2.') + } + } + + await checkDirectoryIsEmpty(servers[0], 'captions') + }) + + it('Should replace an existing caption', async function () { + this.timeout(30000) + + await servers[0].captions.add({ + language: 'ar', + videoId: videoUUID, + fixture: 'subtitle-good.srt' + }) + + await waitJobs(servers) + // Cache invalidation + await wait(3000) + + for (const url of oldFileUrlsAr) { + await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + + await checkDirectoryIsEmpty(servers[0], 'captions') + + oldFileUrlsAr = [] + + for (const server of servers) { + const body = await server.captions.list({ videoId: videoUUID }) + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) + + const caption = body.data.find(c => c.language.id === 'ar') + + if (server === servers[0]) { + expectStartWith(caption.fileUrl, objectStorage.getMockCaptionFileBaseUrl()) + expect(caption.captionPath).to.be.null + + oldFileUrlsAr.push(caption.fileUrl) + } + + await testCaptionFile(caption.fileUrl, 'This is a subtitle in American English') + } + }) + + it('Should remove a caption', async function () { + this.timeout(30000) + + await servers[0].captions.delete({ videoId: videoUUID, language: 'ar' }) + await waitJobs(servers) + + await checkDirectoryIsEmpty(servers[0], 'captions') + + for (const url of oldFileUrlsAr) { + await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + + for (const url of oldFileUrlsZh) { + await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + + it('Should remove the video, and thus all video captions', async function () { + await servers[0].videos.remove({ id: videoUUID }) + + await waitJobs(servers) + + for (const url of oldFileUrlsZh) { + await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + }) + + after(async function () { + await objectStorage.cleanupMock() + }) }) after(async function () { diff --git a/packages/tests/src/api/videos/video-imports.ts b/packages/tests/src/api/videos/video-imports.ts index 1e3d89942..562bfe822 100644 --- a/packages/tests/src/api/videos/video-imports.ts +++ b/packages/tests/src/api/videos/video-imports.ts @@ -145,7 +145,7 @@ describe('Test video imports', function () { `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+English \\(US\\)[ \n]+` + `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+This is a subtitle in American English[ \n]+` + `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Adding subtitles is very easy to do` - await testCaptionFile(servers[0].url, enCaption.captionPath, new RegExp(regex)) + await testCaptionFile(enCaption.fileUrl, new RegExp(regex)) } { @@ -160,7 +160,7 @@ describe('Test video imports', function () { `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+C'est un sous-titre français[ \n]+` + `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Ajouter un sous-titre est vraiment facile` - await testCaptionFile(servers[0].url, frCaption.captionPath, new RegExp(regex)) + await testCaptionFile(frCaption.fileUrl, new RegExp(regex)) } }) @@ -510,7 +510,7 @@ describe('Test video imports', function () { `1\r?\n` + `00:00:04.000 --> 00:00:09.000\r?\n` + `January 1, 1994. The North American` - await testCaptionFile(server.url, captions[0].captionPath, new RegExp(str)) + await testCaptionFile(captions[0].fileUrl, new RegExp(str)) } } }) diff --git a/packages/tests/src/api/videos/video-storyboard.ts b/packages/tests/src/api/videos/video-storyboard.ts index eb2354210..2ede57a1e 100644 --- a/packages/tests/src/api/videos/video-storyboard.ts +++ b/packages/tests/src/api/videos/video-storyboard.ts @@ -11,6 +11,7 @@ import { createMultipleServers, doubleFollow, makeGetRequest, + makeRawRequest, PeerTubeServer, sendRTMPStream, setAccessTokensToServers, @@ -46,8 +47,15 @@ async function checkStoryboard (options: { expect(storyboard.totalHeight).to.equal(spriteHeight * Math.max((tilesCount / 11), 1)) } - const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) - expect(body.length).to.be.above(minSize) + { + const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) + expect(body.length).to.be.above(minSize) + } + + { + const { body } = await makeRawRequest({ url: storyboard.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + expect(body.length).to.be.above(minSize) + } } describe('Test video storyboard', function () { diff --git a/packages/tests/src/api/videos/video-transcription.ts b/packages/tests/src/api/videos/video-transcription.ts index 4d9b55039..fb330346f 100644 --- a/packages/tests/src/api/videos/video-transcription.ts +++ b/packages/tests/src/api/videos/video-transcription.ts @@ -1,7 +1,9 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { VideoPrivacy } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' import { + ObjectStorageCommand, PeerTubeServer, cleanupTests, createMultipleServers, @@ -36,200 +38,255 @@ describe('Test video transcription', function () { // --------------------------------------------------------------------------- - it('Should generate a transcription on request', async function () { - this.timeout(360000) + describe('Common on filesystem', function () { - await servers[0].config.disableTranscription() + it('Should generate a transcription on request', async function () { + this.timeout(360000) - const uuid = await uploadForTranscription(servers[0]) - await waitJobs(servers) - await checkLanguage(servers, uuid, null) + await servers[0].config.disableTranscription() - await servers[0].config.enableTranscription() + const uuid = await uploadForTranscription(servers[0]) + await waitJobs(servers) + await checkLanguage(servers, uuid, null) - await servers[0].captions.runGenerate({ videoId: uuid }) - await waitJobs(servers) - await checkLanguage(servers, uuid, 'en') + await servers[0].config.enableTranscription() - await checkAutoCaption(servers, uuid) - }) + await servers[0].captions.runGenerate({ videoId: uuid }) + await waitJobs(servers) + await checkLanguage(servers, uuid, 'en') - it('Should run transcription on upload by default', async function () { - this.timeout(360000) - - const uuid = await uploadForTranscription(servers[0]) - - await waitJobs(servers) - await checkAutoCaption(servers, uuid) - await checkLanguage(servers, uuid, 'en') - }) - - it('Should run transcription on import by default', async function () { - this.timeout(360000) - - const { video } = await servers[0].videoImports.importVideo({ - attributes: { - privacy: VideoPrivacy.PUBLIC, - targetUrl: FIXTURE_URLS.transcriptionVideo, - language: undefined - } + await checkAutoCaption({ servers, uuid }) }) - await waitJobs(servers) - await checkAutoCaption(servers, video.uuid) - await checkLanguage(servers, video.uuid, 'en') - }) + it('Should run transcription on upload by default', async function () { + this.timeout(360000) - it('Should run transcription when live ended', async function () { - this.timeout(360000) - - await servers[0].config.enableMinimumTranscoding() - await servers[0].config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' }) - - const { live, video } = await servers[0].live.quickCreate({ - saveReplay: true, - permanentLive: false, - privacy: VideoPrivacy.PUBLIC - }) - - const ffmpegCommand = sendRTMPStream({ - rtmpBaseUrl: live.rtmpUrl, - streamKey: live.streamKey, - fixtureName: join('transcription', 'videos', 'the_last_man_on_earth.mp4') - }) - await servers[0].live.waitUntilPublished({ videoId: video.id }) - - await stopFfmpeg(ffmpegCommand) - - await servers[0].live.waitUntilReplacedByReplay({ videoId: video.id }) - await waitJobs(servers) - await checkAutoCaption(servers, video.uuid, new RegExp('^WEBVTT\\n\\n00:\\d{2}.\\d{3} --> 00:')) - await checkLanguage(servers, video.uuid, 'en') - - await servers[0].config.enableLive({ allowReplay: false }) - await servers[0].config.disableTranscoding() - }) - - it('Should not run transcription if disabled by user', async function () { - this.timeout(120000) - - { - const uuid = await uploadForTranscription(servers[0], { generateTranscription: false }) + const uuid = await uploadForTranscription(servers[0]) await waitJobs(servers) - await checkNoCaption(servers, uuid) - await checkLanguage(servers, uuid, null) - } + await checkAutoCaption({ servers, uuid }) + await checkLanguage(servers, uuid, 'en') + }) + + it('Should run transcription on import by default', async function () { + this.timeout(360000) - { const { video } = await servers[0].videoImports.importVideo({ attributes: { privacy: VideoPrivacy.PUBLIC, targetUrl: FIXTURE_URLS.transcriptionVideo, - generateTranscription: false + language: undefined } }) await waitJobs(servers) - await checkNoCaption(servers, video.uuid) - await checkLanguage(servers, video.uuid, null) - } - }) - - it('Should not run a transcription if the video does not contain audio', async function () { - this.timeout(120000) - - const uuid = await uploadForTranscription(servers[0], { fixture: 'video_short_no_audio.mp4' }) - - await waitJobs(servers) - await checkNoCaption(servers, uuid) - await checkLanguage(servers, uuid, null) - }) - - it('Should not replace an existing caption', async function () { - const uuid = await uploadForTranscription(servers[0]) - - await servers[0].captions.add({ - language: 'en', - videoId: uuid, - fixture: 'subtitle-good1.vtt' + await checkAutoCaption({ servers, uuid: video.uuid }) + await checkLanguage(servers, video.uuid, 'en') }) - const contentBefore = await getCaptionContent(servers[0], uuid, 'en') - await waitJobs(servers) - const contentAter = await getCaptionContent(servers[0], uuid, 'en') + it('Should run transcription when live ended', async function () { + this.timeout(360000) - expect(contentBefore).to.equal(contentAter) - }) + await servers[0].config.enableMinimumTranscoding() + await servers[0].config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' }) - it('Should run transcription after a video edition', async function () { - this.timeout(120000) + const { live, video } = await servers[0].live.quickCreate({ + saveReplay: true, + permanentLive: false, + privacy: VideoPrivacy.PUBLIC + }) - await servers[0].config.enableMinimumTranscoding() - await servers[0].config.enableStudio() + const ffmpegCommand = sendRTMPStream({ + rtmpBaseUrl: live.rtmpUrl, + streamKey: live.streamKey, + fixtureName: join('transcription', 'videos', 'the_last_man_on_earth.mp4') + }) + await servers[0].live.waitUntilPublished({ videoId: video.id }) - const uuid = await uploadForTranscription(servers[0]) - await waitJobs(servers) + await stopFfmpeg(ffmpegCommand) - await checkAutoCaption(servers, uuid) - const oldContent = await getCaptionContent(servers[0], uuid, 'en') + await servers[0].live.waitUntilReplacedByReplay({ videoId: video.id }) + await waitJobs(servers) + await checkAutoCaption({ + servers, + uuid: video.uuid, + captionContains: new RegExp('^WEBVTT\\n\\n00:\\d{2}.\\d{3} --> 00:') + }) + await checkLanguage(servers, video.uuid, 'en') - await servers[0].videoStudio.createEditionTasks({ - videoId: uuid, - tasks: [ - { - name: 'cut' as 'cut', - options: { start: 1 } - } - ] + await servers[0].config.enableLive({ allowReplay: false }) + await servers[0].config.disableTranscoding() }) - await waitJobs(servers) - await checkAutoCaption(servers, uuid) + it('Should not run transcription if disabled by user', async function () { + this.timeout(120000) - const newContent = await getCaptionContent(servers[0], uuid, 'en') - expect(oldContent).to.not.equal(newContent) - }) + { + const uuid = await uploadForTranscription(servers[0], { generateTranscription: false }) - it('Should not run transcription after video edition if the subtitle has not been auto generated', async function () { - this.timeout(120000) + await waitJobs(servers) + await checkNoCaption(servers, uuid) + await checkLanguage(servers, uuid, null) + } - const uuid = await uploadForTranscription(servers[0], { language: 'en' }) - await waitJobs(servers) + { + const { video } = await servers[0].videoImports.importVideo({ + attributes: { + privacy: VideoPrivacy.PUBLIC, + targetUrl: FIXTURE_URLS.transcriptionVideo, + generateTranscription: false + } + }) - await servers[0].captions.add({ language: 'en', videoId: uuid, fixture: 'subtitle-good1.vtt' }) - const oldContent = await getCaptionContent(servers[0], uuid, 'en') - - await servers[0].videoStudio.createEditionTasks({ - videoId: uuid, - tasks: [ - { - name: 'cut' as 'cut', - options: { start: 1 } - } - ] + await waitJobs(servers) + await checkNoCaption(servers, video.uuid) + await checkLanguage(servers, video.uuid, null) + } }) - await waitJobs(servers) + it('Should not run a transcription if the video does not contain audio', async function () { + this.timeout(120000) - const newContent = await getCaptionContent(servers[0], uuid, 'en') - expect(oldContent).to.equal(newContent) + const uuid = await uploadForTranscription(servers[0], { fixture: 'video_short_no_audio.mp4' }) + + await waitJobs(servers) + await checkNoCaption(servers, uuid) + await checkLanguage(servers, uuid, null) + }) + + it('Should not replace an existing caption', async function () { + const uuid = await uploadForTranscription(servers[0]) + + await servers[0].captions.add({ + language: 'en', + videoId: uuid, + fixture: 'subtitle-good1.vtt' + }) + + const contentBefore = await getCaptionContent(servers[0], uuid, 'en') + await waitJobs(servers) + const contentAter = await getCaptionContent(servers[0], uuid, 'en') + + expect(contentBefore).to.equal(contentAter) + }) + + it('Should run transcription after a video edition', async function () { + this.timeout(120000) + + await servers[0].config.enableMinimumTranscoding() + await servers[0].config.enableStudio() + + const uuid = await uploadForTranscription(servers[0]) + await waitJobs(servers) + + await checkAutoCaption({ servers, uuid }) + const oldContent = await getCaptionContent(servers[0], uuid, 'en') + + await servers[0].videoStudio.createEditionTasks({ + videoId: uuid, + tasks: [ + { + name: 'cut' as 'cut', + options: { start: 1 } + } + ] + }) + + await waitJobs(servers) + await checkAutoCaption({ servers, uuid }) + + const newContent = await getCaptionContent(servers[0], uuid, 'en') + expect(oldContent).to.not.equal(newContent) + }) + + it('Should not run transcription after video edition if the subtitle has not been auto generated', async function () { + this.timeout(120000) + + const uuid = await uploadForTranscription(servers[0], { language: 'en' }) + await waitJobs(servers) + + await servers[0].captions.add({ language: 'en', videoId: uuid, fixture: 'subtitle-good1.vtt' }) + const oldContent = await getCaptionContent(servers[0], uuid, 'en') + + await servers[0].videoStudio.createEditionTasks({ + videoId: uuid, + tasks: [ + { + name: 'cut' as 'cut', + options: { start: 1 } + } + ] + }) + + await waitJobs(servers) + + const newContent = await getCaptionContent(servers[0], uuid, 'en') + expect(oldContent).to.equal(newContent) + }) + + it('Should run transcription with HLS only and audio splitted', async function () { + this.timeout(360000) + + await servers[0].config.enableMinimumTranscoding({ hls: true, webVideo: false, splitAudioAndVideo: true }) + + const uuid = await uploadForTranscription(servers[0], { generateTranscription: false }) + await waitJobs(servers) + await checkLanguage(servers, uuid, null) + + await servers[0].captions.runGenerate({ videoId: uuid }) + await waitJobs(servers) + + await checkAutoCaption({ servers, uuid }) + await checkLanguage(servers, uuid, 'en') + }) }) - it('Should run transcription with HLS only and audio splitted', async function () { - this.timeout(360000) + describe('On object storage', async function () { + if (areMockObjectStorageTestsDisabled()) return - await servers[0].config.enableMinimumTranscoding({ hls: true, webVideo: false, splitAudioAndVideo: true }) + const objectStorage = new ObjectStorageCommand() - const uuid = await uploadForTranscription(servers[0], { generateTranscription: false }) - await waitJobs(servers) - await checkLanguage(servers, uuid, null) + before(async function () { + this.timeout(120000) - await servers[0].captions.runGenerate({ videoId: uuid }) - await waitJobs(servers) + const configOverride = objectStorage.getDefaultMockConfig() + await objectStorage.prepareDefaultMockBuckets() - await checkAutoCaption(servers, uuid) - await checkLanguage(servers, uuid, 'en') + await servers[0].kill() + await servers[0].run(configOverride) + }) + + it('Should generate a transcription on request', async function () { + this.timeout(360000) + + await servers[0].config.disableTranscription() + + const uuid = await uploadForTranscription(servers[0]) + await waitJobs(servers) + await checkLanguage(servers, uuid, null) + + await servers[0].config.enableTranscription() + + await servers[0].captions.runGenerate({ videoId: uuid }) + await waitJobs(servers) + await checkLanguage(servers, uuid, 'en') + + await checkAutoCaption({ servers, uuid, objectStorageBaseUrl: objectStorage.getMockCaptionFileBaseUrl() }) + }) + + it('Should run transcription on upload by default', async function () { + this.timeout(360000) + + const uuid = await uploadForTranscription(servers[0]) + + await waitJobs(servers) + await checkAutoCaption({ servers, uuid, objectStorageBaseUrl: objectStorage.getMockCaptionFileBaseUrl() }) + await checkLanguage(servers, uuid, 'en') + }) + + after(async function () { + await objectStorage.cleanupMock() + }) }) after(async function () { diff --git a/packages/tests/src/cli/create-move-video-storage-job.ts b/packages/tests/src/cli/create-move-video-storage-job.ts index b0ccbf069..596f61a65 100644 --- a/packages/tests/src/cli/create-move-video-storage-job.ts +++ b/packages/tests/src/cli/create-move-video-storage-job.ts @@ -18,7 +18,12 @@ import { checkDirectoryIsEmpty } from '@tests/shared/directories.js' import { join } from 'path' import { expectStartWith } from '../shared/checks.js' -async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) { +async function checkFiles (options: { + origin: PeerTubeServer + video: VideoDetails + objectStorage?: ObjectStorageCommand +}) { + const { origin, video, objectStorage } = options // Web videos for (const file of video.files) { @@ -62,6 +67,21 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectSt expectStartWith(source.fileDownloadUrl, origin.url) } } + + // Captions + { + const start = objectStorage + ? objectStorage.getMockCaptionFileBaseUrl() + : origin.url + + const { data: captions } = await origin.captions.list({ videoId: video.uuid }) + + for (const caption of captions) { + expectStartWith(caption.fileUrl, start) + + await makeRawRequest({ url: caption.fileUrl, token: origin.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + } + } } describe('Test create move video storage job CLI', function () { @@ -86,6 +106,10 @@ describe('Test create move video storage job CLI', function () { for (let i = 0; i < 3; i++) { const { uuid } = await servers[0].videos.quickUpload({ name: 'video' + i }) + + await servers[0].captions.add({ language: 'ar', videoId: uuid, fixture: 'subtitle-good1.vtt' }) + await servers[0].captions.add({ language: 'zh', videoId: uuid, fixture: 'subtitle-good1.vtt' }) + uuids.push(uuid) } @@ -107,12 +131,12 @@ describe('Test create move video storage job CLI', function () { for (const server of servers) { const video = await server.videos.get({ id: uuids[1] }) - await checkFiles(servers[0], video, objectStorage) + await checkFiles({ origin: servers[0], video, objectStorage }) for (const id of [ uuids[0], uuids[2] ]) { const video = await server.videos.get({ id }) - await checkFiles(servers[0], video) + await checkFiles({ origin: servers[0], video }) } } }) @@ -128,7 +152,7 @@ describe('Test create move video storage job CLI', function () { for (const id of [ uuids[0], uuids[2] ]) { const video = await server.videos.get({ id }) - await checkFiles(servers[0], video, objectStorage) + await checkFiles({ origin: servers[0], video, objectStorage }) } } }) @@ -164,12 +188,12 @@ describe('Test create move video storage job CLI', function () { for (const server of servers) { const video = await server.videos.get({ id: uuids[1] }) - await checkFiles(servers[0], video) + await checkFiles({ origin: servers[0], video }) for (const id of [ uuids[0], uuids[2] ]) { const video = await server.videos.get({ id }) - await checkFiles(servers[0], video, objectStorage) + await checkFiles({ origin: servers[0], video, objectStorage }) } } }) @@ -185,7 +209,7 @@ describe('Test create move video storage job CLI', function () { for (const id of [ uuids[0], uuids[2] ]) { const video = await server.videos.get({ id }) - await checkFiles(servers[0], video) + await checkFiles({ origin: servers[0], video }) } } }) diff --git a/packages/tests/src/cli/peertube.ts b/packages/tests/src/cli/peertube.ts index 2c66b7a18..94cd57274 100644 --- a/packages/tests/src/cli/peertube.ts +++ b/packages/tests/src/cli/peertube.ts @@ -48,7 +48,7 @@ describe('Test CLI wrapper', function () { describe('Authentication and instance selection', function () { it('Should get an access token', async function () { - const stdout = await cliCommand.execWithEnv(`${cmd} token --url ${server.url} --username user_1 --password super_password`) + const { stdout } = await cliCommand.execWithEnv(`${cmd} token --url ${server.url} --username user_1 --password super_password`) const token = stdout.trim() const body = await server.users.getMyInfo({ token }) diff --git a/packages/tests/src/cli/prune-storage.ts b/packages/tests/src/cli/prune-storage.ts index da4a183f7..b6a0f2719 100644 --- a/packages/tests/src/cli/prune-storage.ts +++ b/packages/tests/src/cli/prune-storage.ts @@ -267,27 +267,46 @@ describe('Test prune storage CLI', function () { if (areMockObjectStorageTestsDisabled()) return const videos: string[] = [] + const objectStorage = new ObjectStorageCommand() + const videoFileUrls: { [ uuid: string ]: string[] } = {} + const sourceFileUrls: { [ uuid: string ]: string } = {} + const captionFileUrls: { [ uuid: string ]: { [ language: string ]: string } } = {} + let sqlCommand: SQLCommand let rootId: number + let captionVideoId: number + + async function execPruneStorage () { + const env = servers[0].cli.getEnv(objectStorage.getDefaultMockConfig({ proxifyPrivateFiles: false })) + + await servers[0].cli.execWithEnv(`${env} npm run prune-storage -- -y`) + } async function checkVideosFiles (uuids: string[], expectedStatus: HttpStatusCodeType) { for (const uuid of uuids) { - const video = await servers[0].videos.getWithToken({ id: uuid }) - - for (const file of getAllFiles(video)) { - await makeRawRequest({ url: file.fileUrl, token: servers[0].accessToken, expectedStatus }) + for (const url of videoFileUrls[uuid]) { + await makeRawRequest({ url, token: servers[0].accessToken, expectedStatus }) } - const source = await servers[0].videos.getSource({ id: uuid }) - await makeRawRequest({ url: source.fileDownloadUrl, redirects: 1, token: servers[0].accessToken, expectedStatus }) + await makeRawRequest({ url: sourceFileUrls[uuid], redirects: 1, token: servers[0].accessToken, expectedStatus }) + } + } + + async function checkCaptionFiles (uuids: string[], languages: string[], expectedStatus: HttpStatusCodeType) { + for (const uuid of uuids) { + for (const language of languages) { + await makeRawRequest({ url: captionFileUrls[uuid][language], token: servers[0].accessToken, expectedStatus }) + } } } async function checkUserExport (expectedStatus: HttpStatusCodeType) { - const { data } = await servers[0].userExports.list({ userId: rootId }) - await makeRawRequest({ url: data[0].privateDownloadUrl, redirects: 1, expectedStatus }) + const { data: userExports } = await servers[0].userExports.list({ userId: rootId }) + const userExportUrl = userExports[0].privateDownloadUrl + + await makeRawRequest({ url: userExportUrl, token: servers[0].accessToken, redirects: 1, expectedStatus }) } before(async function () { @@ -297,7 +316,7 @@ describe('Test prune storage CLI', function () { await objectStorage.prepareDefaultMockBuckets() - await servers[0].run(objectStorage.getDefaultMockConfig()) + await servers[0].run(objectStorage.getDefaultMockConfig({ proxifyPrivateFiles: false })) { const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 1', privacy: VideoPrivacy.PUBLIC }) @@ -310,7 +329,13 @@ describe('Test prune storage CLI', function () { } { - const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 3', privacy: VideoPrivacy.PRIVATE }) + const { id, uuid } = await servers[0].videos.quickUpload({ name: 's3 video 3', privacy: VideoPrivacy.PRIVATE }) + + await servers[0].captions.add({ language: 'ar', videoId: uuid, fixture: 'subtitle-good1.vtt' }) + + await servers[0].captions.add({ language: 'zh', videoId: uuid, fixture: 'subtitle-good1.vtt' }) + captionVideoId = id + videos.push(uuid) } @@ -321,33 +346,62 @@ describe('Test prune storage CLI', function () { await servers[0].userExports.request({ userId: rootId, withVideoFiles: false }) await waitJobs([ servers[0] ]) + + // Grab all file URLs + for (const uuid of videos) { + const video = await servers[0].videos.getWithToken({ id: uuid }) + + videoFileUrls[uuid] = getAllFiles(video).map(f => f.fileUrl) + + const source = await servers[0].videos.getSource({ id: uuid }) + sourceFileUrls[uuid] = source.fileDownloadUrl + + const { data: captions } = await servers[0].captions.list({ videoId: uuid, token: servers[0].accessToken }) + if (!captionFileUrls[uuid]) captionFileUrls[uuid] = {} + + for (const caption of captions) { + captionFileUrls[uuid][caption.language.id] = caption.fileUrl + } + } }) it('Should have the files on object storage', async function () { await checkVideosFiles(videos, HttpStatusCode.OK_200) await checkUserExport(HttpStatusCode.OK_200) + await checkCaptionFiles([ videos[2] ], [ 'ar', 'zh' ], HttpStatusCode.OK_200) }) it('Should run prune-storage script on videos', async function () { await sqlCommand.setVideoFileStorageOf(videos[1], FileStorage.FILE_SYSTEM) await sqlCommand.setVideoFileStorageOf(videos[2], FileStorage.FILE_SYSTEM) + await execPruneStorage() + await checkVideosFiles([ videos[1], videos[2] ], HttpStatusCode.NOT_FOUND_404) await checkVideosFiles([ videos[0] ], HttpStatusCode.OK_200) await checkUserExport(HttpStatusCode.OK_200) + await checkCaptionFiles([ videos[2] ], [ 'ar', 'zh' ], HttpStatusCode.OK_200) }) it('Should run prune-storage script on exports', async function () { await sqlCommand.setUserExportStorageOf(rootId, FileStorage.FILE_SYSTEM) - - await checkVideosFiles([ videos[1], videos[2] ], HttpStatusCode.NOT_FOUND_404) - await checkVideosFiles([ videos[0] ], HttpStatusCode.OK_200) + await execPruneStorage() await checkUserExport(HttpStatusCode.NOT_FOUND_404) + await checkCaptionFiles([ videos[2] ], [ 'ar', 'zh' ], HttpStatusCode.OK_200) + }) + + it('Should run prune-storage script on captions', async function () { + await sqlCommand.setCaptionStorageOf(captionVideoId, 'zh', FileStorage.FILE_SYSTEM) + await execPruneStorage() + + await checkCaptionFiles([ videos[2] ], [ 'ar' ], HttpStatusCode.OK_200) + await checkCaptionFiles([ videos[2] ], [ 'zh' ], HttpStatusCode.NOT_FOUND_404) }) after(async function () { + await objectStorage.cleanupMock() await sqlCommand.cleanup() }) }) diff --git a/packages/tests/src/cli/update-object-storage-url.ts b/packages/tests/src/cli/update-object-storage-url.ts index 522f7f60c..769210962 100644 --- a/packages/tests/src/cli/update-object-storage-url.ts +++ b/packages/tests/src/cli/update-object-storage-url.ts @@ -41,6 +41,9 @@ describe('Update object storage URL CLI', function () { const video = await server.videos.quickUpload({ name: 'video' }) uuid = video.uuid + await server.captions.add({ language: 'ar', videoId: uuid, fixture: 'subtitle-good1.vtt' }) + await server.captions.add({ language: 'zh', videoId: uuid, fixture: 'subtitle-good1.vtt' }) + await waitJobs([ server ]) }) @@ -99,6 +102,16 @@ describe('Update object storage URL CLI', function () { return [ source.fileUrl ] } }) + + await check({ + baseUrl: objectStorage.getMockCaptionFileBaseUrl(), + newBaseUrl: 'https://captions.example.com/', + urlGetter: async video => { + const { data } = await server.captions.list({ videoId: video.uuid }) + + return data.map(c => c.fileUrl) + } + }) }) it('Should update user export URLs', async function () { diff --git a/packages/tests/src/peertube-runner/video-transcription.ts b/packages/tests/src/peertube-runner/video-transcription.ts index fd9ee63ae..d264db0aa 100644 --- a/packages/tests/src/peertube-runner/video-transcription.ts +++ b/packages/tests/src/peertube-runner/video-transcription.ts @@ -2,7 +2,9 @@ import { wait } from '@peertube/peertube-core-utils' import { RunnerJobState } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' import { + ObjectStorageCommand, PeerTubeServer, cleanupTests, createMultipleServers, @@ -41,82 +43,115 @@ describe('Test transcription in peertube-runner program', function () { describe('Running transcription', function () { - it('Should run transcription on classic file', async function () { - this.timeout(360000) + describe('Common on filesystem', function () { - const uuid = await uploadForTranscription(servers[0]) - await waitJobs(servers, { runnerJobs: true }) + it('Should run transcription on classic file', async function () { + this.timeout(360000) - await checkAutoCaption(servers, uuid) - await checkLanguage(servers, uuid, 'en') + const uuid = await uploadForTranscription(servers[0]) + await waitJobs(servers, { runnerJobs: true }) + + await checkAutoCaption({ servers, uuid }) + await checkLanguage(servers, uuid, 'en') + }) + + it('Should run transcription on HLS with audio separated', async function () { + await servers[0].config.enableMinimumTranscoding({ hls: true, webVideo: false, splitAudioAndVideo: true }) + + const uuid = await uploadForTranscription(servers[0], { generateTranscription: false }) + await waitJobs(servers) + await checkLanguage(servers, uuid, null) + + await servers[0].captions.runGenerate({ videoId: uuid }) + await waitJobs(servers, { runnerJobs: true }) + + await checkAutoCaption({ servers, uuid }) + await checkLanguage(servers, uuid, 'en') + }) + + it('Should not run transcription on video without audio stream', async function () { + this.timeout(360000) + + const uuid = await uploadForTranscription(servers[0], { fixture: 'video_short_no_audio.mp4' }) + + await waitJobs(servers) + + let continueWhile = true + while (continueWhile) { + await wait(500) + + const { data } = await servers[0].runnerJobs.list({ stateOneOf: [ RunnerJobState.ERRORED ] }) + + continueWhile = !data.some(j => j.type === 'video-transcription') + } + + await checkNoCaption(servers, uuid) + await checkLanguage(servers, uuid, null) + }) }) - it('Should run transcription on HLS with audio separated', async function () { - await servers[0].config.enableMinimumTranscoding({ hls: true, webVideo: false, splitAudioAndVideo: true }) + describe('On object storage', function () { + if (areMockObjectStorageTestsDisabled()) return - const uuid = await uploadForTranscription(servers[0], { generateTranscription: false }) - await waitJobs(servers) - await checkLanguage(servers, uuid, null) + const objectStorage = new ObjectStorageCommand() - await servers[0].captions.runGenerate({ videoId: uuid }) - await waitJobs(servers, { runnerJobs: true }) + before(async function () { + this.timeout(120000) - await checkAutoCaption(servers, uuid) - await checkLanguage(servers, uuid, 'en') + const configOverride = objectStorage.getDefaultMockConfig() + await objectStorage.prepareDefaultMockBuckets() + + await servers[0].kill() + await servers[0].run(configOverride) + }) + + it('Should run transcription and upload it on object storage', async function () { + this.timeout(360000) + + const uuid = await uploadForTranscription(servers[0]) + await waitJobs(servers, { runnerJobs: true }) + + await checkAutoCaption({ servers, uuid, objectStorageBaseUrl: objectStorage.getMockCaptionFileBaseUrl() }) + await checkLanguage(servers, uuid, 'en') + }) + + after(async function () { + await objectStorage.cleanupMock() + }) }) - it('Should not run transcription on video without audio stream', async function () { - this.timeout(360000) + describe('When transcription is not enabled in runner', function () { - const uuid = await uploadForTranscription(servers[0], { fixture: 'video_short_no_audio.mp4' }) - - await waitJobs(servers) - - let continueWhile = true - while (continueWhile) { + before(async function () { + await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' }) + peertubeRunner.kill() await wait(500) - const { data } = await servers[0].runnerJobs.list({ stateOneOf: [ RunnerJobState.ERRORED ] }) + const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken() + await peertubeRunner.runServer({ jobType: 'live-rtmp-hls-transcoding' }) + await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' }) + }) - continueWhile = !data.some(j => j.type === 'video-transcription') - } + it('Should not run transcription', async function () { + this.timeout(60000) - await checkNoCaption(servers, uuid) - await checkLanguage(servers, uuid, null) - }) - }) + const uuid = await uploadForTranscription(servers[0]) + await waitJobs(servers) + await wait(2000) - describe('When transcription is not enabled in runner', function () { + const { data } = await servers[0].runnerJobs.list({ stateOneOf: [ RunnerJobState.PENDING ] }) + expect(data.some(j => j.type === 'video-transcription')).to.be.true - before(async function () { - await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' }) - peertubeRunner.kill() - await wait(500) - - const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken() - await peertubeRunner.runServer({ jobType: 'live-rtmp-hls-transcoding' }) - await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' }) + await checkNoCaption(servers, uuid) + await checkLanguage(servers, uuid, null) + }) }) - it('Should not run transcription', async function () { - this.timeout(60000) + describe('Check cleanup', function () { - const uuid = await uploadForTranscription(servers[0]) - await waitJobs(servers) - await wait(2000) - - const { data } = await servers[0].runnerJobs.list({ stateOneOf: [ RunnerJobState.PENDING ] }) - expect(data.some(j => j.type === 'video-transcription')).to.be.true - - await checkNoCaption(servers, uuid) - await checkLanguage(servers, uuid, null) - }) - }) - - describe('Check cleanup', function () { - - it('Should have an empty cache directory', async function () { - await checkPeerTubeRunnerCacheIsEmpty(peertubeRunner, 'transcription') + it('Should have an empty cache directory', async function () { + await checkPeerTubeRunnerCacheIsEmpty(peertubeRunner, 'transcription') + }) }) }) diff --git a/packages/tests/src/shared/captions.ts b/packages/tests/src/shared/captions.ts index 436cf8dcc..131c846d0 100644 --- a/packages/tests/src/shared/captions.ts +++ b/packages/tests/src/shared/captions.ts @@ -1,11 +1,9 @@ -import { expect } from 'chai' -import request from 'supertest' import { HttpStatusCode } from '@peertube/peertube-models' +import { expect } from 'chai' +import { makeRawRequest } from '../../../server-commands/src/requests/requests.js' -async function testCaptionFile (url: string, captionPath: string, toTest: RegExp | string) { - const res = await request(url) - .get(captionPath) - .expect(HttpStatusCode.OK_200) +export async function testCaptionFile (fileUrl: string, toTest: RegExp | string) { + const res = await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 }) if (toTest instanceof RegExp) { expect(res.text).to.match(toTest) @@ -13,9 +11,3 @@ async function testCaptionFile (url: string, captionPath: string, toTest: RegExp expect(res.text).to.contain(toTest) } } - -// --------------------------------------------------------------------------- - -export { - testCaptionFile -} diff --git a/packages/tests/src/shared/sql-command.ts b/packages/tests/src/shared/sql-command.ts index af8a10091..0e61e750a 100644 --- a/packages/tests/src/shared/sql-command.ts +++ b/packages/tests/src/shared/sql-command.ts @@ -63,7 +63,6 @@ export class SQLCommand { await this.updateQuery( `UPDATE "videoFile" SET storage = :storage ` + `WHERE "videoId" IN (SELECT id FROM "video" WHERE uuid = :uuid) OR ` + - // eslint-disable-next-line max-len `"videoStreamingPlaylistId" IN (` + `SELECT "videoStreamingPlaylist".id FROM "videoStreamingPlaylist" ` + `INNER JOIN video ON video.id = "videoStreamingPlaylist"."videoId" AND "video".uuid = :uuid` + @@ -71,6 +70,12 @@ export class SQLCommand { { storage, uuid } ) + await this.updateQuery( + `UPDATE "videoStreamingPlaylist" SET storage = :storage ` + + `WHERE "videoId" IN (SELECT id FROM "video" WHERE uuid = :uuid)`, + { storage, uuid } + ) + await this.updateQuery( `UPDATE "videoSource" SET storage = :storage WHERE "videoId" IN (SELECT id FROM "video" WHERE uuid = :uuid)`, { storage, uuid } @@ -81,6 +86,15 @@ export class SQLCommand { await this.updateQuery(`UPDATE "userExport" SET storage = :storage WHERE "userId" = :userId`, { storage, userId }) } + async setCaptionStorageOf (videoId: number, language: string, storage: FileStorageType) { + await this.updateQuery( + `UPDATE "videoCaption" SET storage = :storage WHERE "videoId" = :videoId AND language = :language`, + { storage, videoId, language } + ) + } + + // --------------------------------------------------------------------------- + async setUserEmail (username: string, email: string) { await this.updateQuery(`UPDATE "user" SET email = :email WHERE "username" = :username`, { email, username }) } diff --git a/packages/tests/src/shared/transcription.ts b/packages/tests/src/shared/transcription.ts index 6a9e8b4e8..3bcaf913d 100644 --- a/packages/tests/src/shared/transcription.ts +++ b/packages/tests/src/shared/transcription.ts @@ -9,6 +9,7 @@ import { ensureDir, pathExists } from 'fs-extra/esm' import { join } from 'path' import { testCaptionFile } from './captions.js' import { FIXTURE_URLS } from './fixture-urls.js' +import { expectStartWith } from './checks.js' type CustomModelName = 'tiny.pt' | 'faster-whisper-tiny' @@ -29,11 +30,23 @@ export function getCustomModelPath (modelName: CustomModelName) { // --------------------------------------------------------------------------- -export async function checkAutoCaption ( - servers: PeerTubeServer[], - uuid: string, - captionContains = new RegExp('^WEBVTT\\n\\n00:00.\\d{3} --> 00:') -) { +export async function checkAutoCaption (options: { + servers: PeerTubeServer[] + uuid: string + + captionContains?: RegExp + + rootServer?: PeerTubeServer + objectStorageBaseUrl?: string +}) { + const { + servers, + rootServer = servers[0], + uuid, + captionContains = new RegExp('^WEBVTT\\n\\n00:00.\\d{3} --> 00:'), + objectStorageBaseUrl + } = options + for (const server of servers) { const body = await server.captions.list({ videoId: uuid }) expect(body.total).to.equal(1) @@ -44,9 +57,11 @@ export async function checkAutoCaption ( expect(caption.language.label).to.equal('English') expect(caption.automaticallyGenerated).to.be.true - { - await testCaptionFile(server.url, caption.captionPath, captionContains) + if (objectStorageBaseUrl && server === rootServer) { + expectStartWith(caption.fileUrl, objectStorageBaseUrl) } + + await testCaptionFile(caption.fileUrl, captionContains) } } diff --git a/server/core/assets/email-templates/password-create/html.pug b/server/core/assets/email-templates/password-create/html.pug index afa30ae97..d36179208 100644 --- a/server/core/assets/email-templates/password-create/html.pug +++ b/server/core/assets/email-templates/password-create/html.pug @@ -6,5 +6,5 @@ block title block content p. Welcome to #[a(href=WEBSERVER.URL) #{instanceName}]. Your username is: #{username}. - Please set your password by following #[a(href=createPasswordUrl) this link]: #[a(href=createPasswordUrl) #{createPasswordUrl}] - (this link will expire within seven days). \ No newline at end of file + Please set your password by following #[a(href=createPasswordUrl) this link]: #[a(href=createPasswordUrl) #{createPasswordUrl}] + (this link will expire within seven days). diff --git a/server/core/controllers/api/videos/source.ts b/server/core/controllers/api/videos/source.ts index 66480949a..3c65357c9 100644 --- a/server/core/controllers/api/videos/source.ts +++ b/server/core/controllers/api/videos/source.ts @@ -7,7 +7,7 @@ import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js' import { setupUploadResumableRoutes } from '@server/lib/uploadx.js' import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js' import { buildNewFile, createVideoSource } from '@server/lib/video-file.js' -import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js' +import { buildMoveVideoJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' import { buildNextVideoState } from '@server/lib/video-state.js' import { openapiOperationDoc } from '@server/middlewares/doc.js' @@ -181,7 +181,7 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide ] if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { - jobs.push(await buildMoveJob({ video, isNewVideo: false, previousVideoState: undefined, type: 'move-to-object-storage' })) + jobs.push(await buildMoveVideoJob({ video, isNewVideo: false, previousVideoState: undefined, type: 'move-to-object-storage' })) } if (video.state === VideoState.TO_TRANSCODE) { diff --git a/server/core/initializers/checker-before-init.ts b/server/core/initializers/checker-before-init.ts index bbc58aa11..6720f65db 100644 --- a/server/core/initializers/checker-before-init.ts +++ b/server/core/initializers/checker-before-init.ts @@ -72,6 +72,7 @@ function checkMissedConfig () { 'object_storage.streaming_playlists.prefix', 'object_storage.streaming_playlists.base_url', 'object_storage.web_videos.bucket_name', 'object_storage.web_videos.prefix', 'object_storage.web_videos.base_url', 'object_storage.original_video_files.bucket_name', 'object_storage.original_video_files.prefix', 'object_storage.original_video_files.base_url', 'object_storage.max_request_attempts', + 'object_storage.captions.bucket_name', 'object_storage.captions.prefix', 'object_storage.captions.base_url', 'theme.default', 'feeds.videos.count', 'feeds.comments.count', 'geo_ip.enabled', 'geo_ip.country.database_url', 'geo_ip.city.database_url', diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.ts index 72fad9373..7c016fb65 100644 --- a/server/core/initializers/config.ts +++ b/server/core/initializers/config.ts @@ -170,6 +170,11 @@ const CONFIG = { BUCKET_NAME: config.get('object_storage.original_video_files.bucket_name'), PREFIX: config.get('object_storage.original_video_files.prefix'), BASE_URL: config.get('object_storage.original_video_files.base_url') + }, + CAPTIONS: { + BUCKET_NAME: config.get('object_storage.captions.bucket_name'), + PREFIX: config.get('object_storage.captions.prefix'), + BASE_URL: config.get('object_storage.captions.base_url') } }, WEBSERVER: { diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts index 87dce5e24..d748a1f1c 100644 --- a/server/core/initializers/constants.ts +++ b/server/core/initializers/constants.ts @@ -46,7 +46,7 @@ import { CONFIG, registerConfigChangedHandler } from './config.js' // --------------------------------------------------------------------------- -export const LAST_MIGRATION_VERSION = 870 +export const LAST_MIGRATION_VERSION = 875 // --------------------------------------------------------------------------- diff --git a/server/core/initializers/migrations/0875-caption-object-storage.ts b/server/core/initializers/migrations/0875-caption-object-storage.ts new file mode 100644 index 000000000..a9db3f50d --- /dev/null +++ b/server/core/initializers/migrations/0875-caption-object-storage.ts @@ -0,0 +1,32 @@ +import { FileStorage } from '@peertube/peertube-models' +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + const { transaction } = utils + + { + await utils.queryInterface.addColumn('videoCaption', 'storage', { + type: Sequelize.INTEGER, + allowNull: true, + defaultValue: FileStorage.FILE_SYSTEM + }, { transaction }) + + await utils.queryInterface.changeColumn('videoCaption', 'storage', { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: null + }, { transaction }) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + down, up +} diff --git a/server/core/lib/files-cache/video-captions-simple-file-cache.ts b/server/core/lib/files-cache/video-captions-simple-file-cache.ts index 5f7e5a158..7cef1d966 100644 --- a/server/core/lib/files-cache/video-captions-simple-file-cache.ts +++ b/server/core/lib/files-cache/video-captions-simple-file-cache.ts @@ -40,7 +40,7 @@ class VideoCaptionsSimpleFileCache extends AbstractSimpleFileCache { const video = await VideoModel.loadFull(videoCaption.videoId) if (!video) return undefined - const remoteUrl = videoCaption.getFileUrl(video) + const remoteUrl = videoCaption.getOriginFileUrl(video) const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.filename) try { diff --git a/server/core/lib/job-queue/handlers/move-to-file-system.ts b/server/core/lib/job-queue/handlers/move-to-file-system.ts index 57914564c..3b0b5ed43 100644 --- a/server/core/lib/job-queue/handlers/move-to-file-system.ts +++ b/server/core/lib/job-queue/handlers/move-to-file-system.ts @@ -1,11 +1,13 @@ -import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models' +import { FileStorage, isMoveCaptionPayload, isMoveVideoStoragePayload, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { updateTorrentMetadata } from '@server/helpers/webtorrent.js' import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js' import { + makeCaptionFileAvailable, makeHLSFileAvailable, makeOriginalFileAvailable, makeWebVideoFileAvailable, + removeCaptionObjectStorage, removeHLSFileObjectStorageByFilename, removeHLSObjectStorage, removeOriginalFileObjectStorage, @@ -14,36 +16,54 @@ import { import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js' -import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js' +import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js' import { MVideoSource } from '@server/types/models/video/video-source.js' import { Job } from 'bullmq' import { join } from 'path' -import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js' +import { moveCaptionToStorageJob } from './shared/move-caption.js' +import { moveVideoToStorageJob, onMoveVideoToStorageFailure } from './shared/move-video.js' const lTagsBase = loggerTagsFactory('move-file-system') export async function processMoveToFileSystem (job: Job) { const payload = job.data as MoveStoragePayload - logger.info('Moving video %s to file system in job %s.', payload.videoUUID, job.id) - await moveToJob({ - jobId: job.id, - videoUUID: payload.videoUUID, - loggerTags: lTagsBase().tags, + if (isMoveVideoStoragePayload(payload)) { // Move all video related files + logger.info('Moving video %s to file system in job %s.', payload.videoUUID, job.id) - moveWebVideoFiles, - moveHLSFiles, - moveVideoSourceFile, + await moveVideoToStorageJob({ + jobId: job.id, + videoUUID: payload.videoUUID, + loggerTags: lTagsBase().tags, - doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }), - moveToFailedState: moveToFailedMoveToFileSystemState - }) + moveWebVideoFiles, + moveHLSFiles, + moveVideoSourceFile, + moveCaptionFiles, + + doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }), + moveToFailedState: moveToFailedMoveToFileSystemState + }) + } else if (isMoveCaptionPayload(payload)) { // Only caption file + logger.info(`Moving video caption ${payload.captionId} to file system in job ${job.id}.`) + + await moveCaptionToStorageJob({ + jobId: job.id, + captionId: payload.captionId, + loggerTags: lTagsBase().tags, + moveCaptionFiles + }) + } else { + throw new Error('Unknown payload type') + } } export async function onMoveToFileSystemFailure (job: Job, err: any) { const payload = job.data as MoveStoragePayload - await onMoveToStorageFailure({ + if (!isMoveVideoStoragePayload(payload)) return + + await onMoveVideoToStorageFailure({ videoUUID: payload.videoUUID, err, lTags: lTagsBase(), @@ -130,6 +150,28 @@ async function onVideoFileMoved (options: { await objetStorageRemover() } +// --------------------------------------------------------------------------- + +async function moveCaptionFiles (captions: MVideoCaption[]) { + for (const caption of captions) { + if (caption.storage === FileStorage.FILE_SYSTEM) continue + + await makeCaptionFileAvailable(caption.filename, caption.getFSPath()) + + const oldFileUrl = caption.fileUrl + + caption.fileUrl = null + caption.storage = FileStorage.FILE_SYSTEM + await caption.save() + + logger.debug('Removing caption file %s because it\'s now on file system', oldFileUrl, lTagsBase()) + + await removeCaptionObjectStorage(caption) + } +} + +// --------------------------------------------------------------------------- + async function doAfterLastMove (options: { video: MVideoWithAllFiles previousVideoState: VideoStateType diff --git a/server/core/lib/job-queue/handlers/move-to-object-storage.ts b/server/core/lib/job-queue/handlers/move-to-object-storage.ts index a1748cc74..be1f71bdf 100644 --- a/server/core/lib/job-queue/handlers/move-to-object-storage.ts +++ b/server/core/lib/job-queue/handlers/move-to-object-storage.ts @@ -1,41 +1,63 @@ -import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models' +import { FileStorage, isMoveCaptionPayload, isMoveVideoStoragePayload, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { updateTorrentMetadata } from '@server/helpers/webtorrent.js' import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js' -import { storeHLSFileFromFilename, storeOriginalVideoFile, storeWebVideoFile } from '@server/lib/object-storage/index.js' +import { storeHLSFileFromFilename, storeOriginalVideoFile, storeVideoCaption, storeWebVideoFile } from '@server/lib/object-storage/index.js' import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js' -import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js' +import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js' import { MVideoSource } from '@server/types/models/video/video-source.js' import { Job } from 'bullmq' import { remove } from 'fs-extra/esm' import { join } from 'path' -import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js' +import { moveCaptionToStorageJob } from './shared/move-caption.js' +import { moveVideoToStorageJob, onMoveVideoToStorageFailure } from './shared/move-video.js' const lTagsBase = loggerTagsFactory('move-object-storage') export async function processMoveToObjectStorage (job: Job) { const payload = job.data as MoveStoragePayload - logger.info('Moving video %s to object storage in job %s.', payload.videoUUID, job.id) - await moveToJob({ - jobId: job.id, - videoUUID: payload.videoUUID, - loggerTags: lTagsBase().tags, + if (isMoveVideoStoragePayload(payload)) { // Move all video related files + logger.info('Moving video %s to object storage in job %s.', payload.videoUUID, job.id) - moveWebVideoFiles, - moveHLSFiles, - moveVideoSourceFile, - doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }), - moveToFailedState: moveToFailedMoveToObjectStorageState - }) + await moveVideoToStorageJob({ + jobId: job.id, + videoUUID: payload.videoUUID, + loggerTags: lTagsBase().tags, + + moveWebVideoFiles, + moveHLSFiles, + moveVideoSourceFile, + moveCaptionFiles, + + doAfterLastMove: video => { + return doAfterLastVideoMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }) + }, + + moveToFailedState: moveToFailedMoveToObjectStorageState + }) + } else if (isMoveCaptionPayload(payload)) { // Only caption file + logger.info(`Moving video caption ${payload.captionId} to object storage in job ${job.id}.`) + + await moveCaptionToStorageJob({ + jobId: job.id, + captionId: payload.captionId, + loggerTags: lTagsBase().tags, + moveCaptionFiles + }) + } else { + throw new Error('Unknown payload type') + } } export async function onMoveToObjectStorageFailure (job: Job, err: any) { const payload = job.data as MoveStoragePayload - await onMoveToStorageFailure({ + if (!isMoveVideoStoragePayload(payload)) return + + await onMoveVideoToStorageFailure({ videoUUID: payload.videoUUID, err, lTags: lTagsBase(), @@ -60,6 +82,27 @@ async function moveVideoSourceFile (source: MVideoSource) { await remove(sourcePath) } +// --------------------------------------------------------------------------- + +async function moveCaptionFiles (captions: MVideoCaption[]) { + for (const caption of captions) { + if (caption.storage !== FileStorage.FILE_SYSTEM) continue + + const captionPath = caption.getFSPath() + const fileUrl = await storeVideoCaption(captionPath, caption.filename) + + caption.storage = FileStorage.OBJECT_STORAGE + caption.fileUrl = fileUrl + await caption.save() + + logger.debug(`Removing video caption file ${captionPath} because it's now on object storage`, lTagsBase()) + + await remove(captionPath) + } +} + +// --------------------------------------------------------------------------- + async function moveWebVideoFiles (video: MVideoWithAllFiles) { for (const file of video.VideoFiles) { if (file.storage !== FileStorage.FILE_SYSTEM) continue @@ -110,7 +153,9 @@ async function onVideoFileMoved (options: { await remove(oldPath) } -async function doAfterLastMove (options: { +// --------------------------------------------------------------------------- + +async function doAfterLastVideoMove (options: { video: MVideoWithAllFiles previousVideoState: VideoStateType isNewVideo: boolean diff --git a/server/core/lib/job-queue/handlers/shared/move-caption.ts b/server/core/lib/job-queue/handlers/shared/move-caption.ts new file mode 100644 index 000000000..260a23f83 --- /dev/null +++ b/server/core/lib/job-queue/handlers/shared/move-caption.ts @@ -0,0 +1,48 @@ +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/federate.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { VideoCaptionModel } from '@server/models/video/video-caption.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideoCaption } from '@server/types/models/index.js' + +export async function moveCaptionToStorageJob (options: { + jobId: string + captionId: number + loggerTags: (number | string)[] + + moveCaptionFiles: (captions: MVideoCaption[]) => Promise +}) { + const { + jobId, + loggerTags, + captionId, + moveCaptionFiles + } = options + + const lTagsBase = loggerTagsFactory(...loggerTags) + + const caption = await VideoCaptionModel.loadWithVideo(captionId) + + if (!caption) { + logger.info(`Can't process job ${jobId}, caption does not exist anymore.`, lTagsBase()) + return + } + + const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(caption.Video.uuid) + + try { + await moveCaptionFiles([ caption ]) + + await retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async t => { + const videoFull = await VideoModel.loadFull(caption.Video.id, t) + + await federateVideoIfNeeded(videoFull, false, t) + }) + }) + } finally { + fileMutexReleaser() + } +} diff --git a/server/core/lib/job-queue/handlers/shared/move-video.ts b/server/core/lib/job-queue/handlers/shared/move-video.ts index acf774359..ceb73d689 100644 --- a/server/core/lib/job-queue/handlers/shared/move-video.ts +++ b/server/core/lib/job-queue/handlers/shared/move-video.ts @@ -1,12 +1,13 @@ import { LoggerTags, logger, loggerTagsFactory } from '@server/helpers/logger.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { VideoCaptionModel } from '@server/models/video/video-caption.js' import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' import { VideoSourceModel } from '@server/models/video/video-source.js' import { VideoModel } from '@server/models/video/video.js' -import { MVideoWithAllFiles } from '@server/types/models/index.js' +import { MVideoCaption, MVideoWithAllFiles } from '@server/types/models/index.js' import { MVideoSource } from '@server/types/models/video/video-source.js' -export async function moveToJob (options: { +export async function moveVideoToStorageJob (options: { jobId: string videoUUID: string loggerTags: (number | string)[] @@ -14,6 +15,8 @@ export async function moveToJob (options: { moveWebVideoFiles: (video: MVideoWithAllFiles) => Promise moveHLSFiles: (video: MVideoWithAllFiles) => Promise moveVideoSourceFile: (source: MVideoSource) => Promise + moveCaptionFiles: (captions: MVideoCaption[]) => Promise + moveToFailedState: (video: MVideoWithAllFiles) => Promise doAfterLastMove: (video: MVideoWithAllFiles) => Promise }) { @@ -24,6 +27,7 @@ export async function moveToJob (options: { moveVideoSourceFile, moveHLSFiles, moveWebVideoFiles, + moveCaptionFiles, moveToFailedState, doAfterLastMove } = options @@ -62,6 +66,13 @@ export async function moveToJob (options: { await moveHLSFiles(video) } + const captions = await VideoCaptionModel.listVideoCaptions(video.id) + if (captions.length !== 0) { + logger.debug('Moving captions of %s.', video.uuid, lTags) + + await moveCaptionFiles(captions) + } + const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') if (pendingMove === 0) { logger.info('Running cleanup after moving files (video %s in job %s)', video.uuid, jobId, lTags) @@ -69,7 +80,7 @@ export async function moveToJob (options: { await doAfterLastMove(video) } } catch (err) { - await onMoveToStorageFailure({ videoUUID, err, lTags, moveToFailedState }) + await onMoveVideoToStorageFailure({ videoUUID, err, lTags, moveToFailedState }) throw err } finally { @@ -77,7 +88,7 @@ export async function moveToJob (options: { } } -export async function onMoveToStorageFailure (options: { +export async function onMoveVideoToStorageFailure (options: { videoUUID: string err: any lTags: LoggerTags diff --git a/server/core/lib/job-queue/handlers/video-file-import.ts b/server/core/lib/job-queue/handlers/video-file-import.ts index ae876b355..cabaa1bef 100644 --- a/server/core/lib/job-queue/handlers/video-file-import.ts +++ b/server/core/lib/job-queue/handlers/video-file-import.ts @@ -10,7 +10,7 @@ import { MVideoFullLight } from '@server/types/models/index.js' import { getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg' import { logger } from '../../../helpers/logger.js' import { JobQueue } from '../job-queue.js' -import { buildMoveJob } from '@server/lib/video-jobs.js' +import { buildMoveVideoJob } from '@server/lib/video-jobs.js' import { buildNewFile } from '@server/lib/video-file.js' async function processVideoFileImport (job: Job) { @@ -27,7 +27,7 @@ async function processVideoFileImport (job: Job) { await updateVideoFile(video, payload.filePath) if (CONFIG.OBJECT_STORAGE.ENABLED) { - await JobQueue.Instance.createJob(await buildMoveJob({ video, previousVideoState: video.state, type: 'move-to-object-storage' })) + await JobQueue.Instance.createJob(await buildMoveVideoJob({ video, previousVideoState: video.state, type: 'move-to-object-storage' })) } else { await federateVideoIfNeeded(video, false) } diff --git a/server/core/lib/job-queue/handlers/video-import.ts b/server/core/lib/job-queue/handlers/video-import.ts index 15583f973..df08043f2 100644 --- a/server/core/lib/job-queue/handlers/video-import.ts +++ b/server/core/lib/job-queue/handlers/video-import.ts @@ -27,7 +27,7 @@ import { isUserQuotaValid } from '@server/lib/user.js' import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js' import { replaceChaptersIfNotExist } from '@server/lib/video-chapters.js' import { buildNewFile } from '@server/lib/video-file.js' -import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js' +import { buildMoveVideoJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' import { buildNextVideoState } from '@server/lib/video-state.js' import { VideoCaptionModel } from '@server/models/video/video-caption.js' @@ -313,7 +313,7 @@ async function afterImportSuccess (options: { if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { await JobQueue.Instance.createJob( - await buildMoveJob({ video, previousVideoState: VideoState.TO_IMPORT, type: 'move-to-object-storage' }) + await buildMoveVideoJob({ video, previousVideoState: VideoState.TO_IMPORT, type: 'move-to-object-storage' }) ) return } diff --git a/server/core/lib/object-storage/keys.ts b/server/core/lib/object-storage/keys.ts index 75e5d60d7..1c1e92d74 100644 --- a/server/core/lib/object-storage/keys.ts +++ b/server/core/lib/object-storage/keys.ts @@ -17,6 +17,10 @@ export function generateOriginalVideoObjectStorageKey (filename: string) { return filename } +export function generateCaptionObjectStorageKey (filename: string) { + return filename +} + export function generateUserExportObjectStorageKey (filename: string) { return filename } diff --git a/server/core/lib/object-storage/object-storage-helpers.ts b/server/core/lib/object-storage/object-storage-helpers.ts index 758d064eb..6886d726b 100644 --- a/server/core/lib/object-storage/object-storage-helpers.ts +++ b/server/core/lib/object-storage/object-storage-helpers.ts @@ -50,14 +50,16 @@ async function storeObject (options: { objectStorageKey: string bucketInfo: BucketInfo isPrivate: boolean + + contentType?: string }): Promise { - const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options + const { inputPath, objectStorageKey, bucketInfo, isPrivate, contentType } = options logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) const fileStream = createReadStream(inputPath) - return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate }) + return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate, contentType }) } async function storeContent (options: { @@ -65,12 +67,14 @@ async function storeContent (options: { objectStorageKey: string bucketInfo: BucketInfo isPrivate: boolean + + contentType?: string }): Promise { - const { content, objectStorageKey, bucketInfo, isPrivate } = options + const { content, objectStorageKey, bucketInfo, isPrivate, contentType } = options logger.debug('Uploading %s content to %s%s in bucket %s', content, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) - return uploadToStorage({ objectStorageKey, content, bucketInfo, isPrivate }) + return uploadToStorage({ objectStorageKey, content, bucketInfo, isPrivate, contentType }) } async function storeStream (options: { @@ -78,12 +82,14 @@ async function storeStream (options: { objectStorageKey: string bucketInfo: BucketInfo isPrivate: boolean + + contentType?: string }): Promise { - const { stream, objectStorageKey, bucketInfo, isPrivate } = options + const { stream, objectStorageKey, bucketInfo, isPrivate, contentType } = options logger.debug('Streaming file to %s%s in bucket %s', bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) - return uploadToStorage({ objectStorageKey, content: stream, bucketInfo, isPrivate }) + return uploadToStorage({ objectStorageKey, content: stream, bucketInfo, isPrivate, contentType }) } // --------------------------------------------------------------------------- @@ -296,13 +302,16 @@ async function uploadToStorage (options: { objectStorageKey: string bucketInfo: BucketInfo isPrivate: boolean + + contentType?: string }) { - const { content, objectStorageKey, bucketInfo, isPrivate } = options + const { content, objectStorageKey, bucketInfo, isPrivate, contentType } = options const input: PutObjectCommandInput = { Body: content, Bucket: bucketInfo.BUCKET_NAME, - Key: buildKey(objectStorageKey, bucketInfo) + Key: buildKey(objectStorageKey, bucketInfo), + ContentType: contentType } const acl = getACL(isPrivate) diff --git a/server/core/lib/object-storage/videos.ts b/server/core/lib/object-storage/videos.ts index 009949256..9e48757f5 100644 --- a/server/core/lib/object-storage/videos.ts +++ b/server/core/lib/object-storage/videos.ts @@ -1,11 +1,12 @@ import { logger } from '@server/helpers/logger.js' import { CONFIG } from '@server/initializers/config.js' -import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models/index.js' +import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile } from '@server/types/models/index.js' import { MVideoSource } from '@server/types/models/video/video-source.js' import { basename, join } from 'path' import { getHLSDirectory } from '../paths.js' import { VideoPathManager } from '../video-path-manager.js' import { + generateCaptionObjectStorageKey, generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateOriginalVideoObjectStorageKey, @@ -71,6 +72,18 @@ export function storeWebVideoFile (video: MVideo, file: MVideoFile) { // --------------------------------------------------------------------------- +export function storeVideoCaption (inputPath: string, filename: string) { + return storeObject({ + inputPath, + objectStorageKey: generateCaptionObjectStorageKey(filename), + bucketInfo: CONFIG.OBJECT_STORAGE.CAPTIONS, + isPrivate: false, + contentType: 'text/vtt' + }) +} + +// --------------------------------------------------------------------------- + export function storeOriginalVideoFile (inputPath: string, filename: string) { return storeObject({ inputPath, @@ -130,6 +143,12 @@ export function removeOriginalFileObjectStorage (videoSource: MVideoSource) { // --------------------------------------------------------------------------- +export function removeCaptionObjectStorage (videoCaption: MVideoCaption) { + return removeObject(generateCaptionObjectStorageKey(videoCaption.filename), CONFIG.OBJECT_STORAGE.CAPTIONS) +} + +// --------------------------------------------------------------------------- + export async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { const key = generateHLSObjectStorageKey(playlist, filename) @@ -172,6 +191,20 @@ export async function makeOriginalFileAvailable (keptOriginalFilename: string, d return destination } +export async function makeCaptionFileAvailable (filename: string, destination: string) { + const key = generateCaptionObjectStorageKey(filename) + + logger.info('Fetching Caption file %s from object storage to %s.', key, destination, lTags()) + + await makeAvailable({ + key, + destination, + bucketInfo: CONFIG.OBJECT_STORAGE.CAPTIONS + }) + + return destination +} + // --------------------------------------------------------------------------- export function getWebVideoFileReadStream (options: { diff --git a/server/core/lib/video-captions.ts b/server/core/lib/video-captions.ts index 66730de1c..71a1cef91 100644 --- a/server/core/lib/video-captions.ts +++ b/server/core/lib/video-captions.ts @@ -1,4 +1,4 @@ -import { VideoFileStream } from '@peertube/peertube-models' +import { FileStorage, VideoFileStream } from '@peertube/peertube-models' import { buildSUUID } from '@peertube/peertube-node-utils' import { AbstractTranscriber, TranscriptionModel, WhisperBuiltinModel, transcriberFactory } from '@peertube/peertube-transcription' import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils.js' @@ -34,6 +34,7 @@ export async function createLocalCaption (options: { const videoCaption = new VideoCaptionModel({ videoId: video.id, filename: VideoCaptionModel.generateCaptionName(language), + storage: FileStorage.FILE_SYSTEM, language, automaticallyGenerated }) as MVideoCaption @@ -46,6 +47,12 @@ export async function createLocalCaption (options: { }) }) + if (CONFIG.OBJECT_STORAGE.ENABLED) { + await JobQueue.Instance.createJob({ type: 'move-to-object-storage', payload: { captionId: videoCaption.id } }) + } + + logger.info(`Created/replaced caption ${videoCaption.filename} of ${language} of video ${video.uuid}`, lTags(video.uuid)) + return Object.assign(videoCaption, { Video: video }) } diff --git a/server/core/lib/video-jobs.ts b/server/core/lib/video-jobs.ts index 99b8bdc3c..3e2bcbe79 100644 --- a/server/core/lib/video-jobs.ts +++ b/server/core/lib/video-jobs.ts @@ -6,7 +6,7 @@ import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-q import { createTranscriptionTaskIfNeeded } from './video-captions.js' import { moveFilesIfPrivacyChanged } from './video-privacy.js' -export async function buildMoveJob (options: { +export async function buildMoveVideoJob (options: { video: MVideoUUID previousVideoState: VideoStateType type: 'move-to-object-storage' | 'move-to-file-system' @@ -92,7 +92,7 @@ export async function addVideoJobsAfterCreation (options: { ] if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { - jobs.push(await buildMoveJob({ video, previousVideoState: undefined, type: 'move-to-object-storage' })) + jobs.push(await buildMoveVideoJob({ video, previousVideoState: undefined, type: 'move-to-object-storage' })) } if (video.state === VideoState.TO_TRANSCODE) { diff --git a/server/core/lib/video-state.ts b/server/core/lib/video-state.ts index d360a49ca..1ea463002 100644 --- a/server/core/lib/video-state.ts +++ b/server/core/lib/video-state.ts @@ -10,7 +10,7 @@ import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models/index. import { federateVideoIfNeeded } from './activitypub/videos/index.js' import { JobQueue } from './job-queue/index.js' import { Notifier } from './notifier/index.js' -import { buildMoveJob } from './video-jobs.js' +import { buildMoveVideoJob } from './video-jobs.js' function buildNextVideoState (currentState?: VideoStateType) { if (currentState === VideoState.PUBLISHED) { @@ -94,7 +94,7 @@ async function moveToExternalStorageState (options: { logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] }) try { - await JobQueue.Instance.createJob(await buildMoveJob({ video, previousVideoState, isNewVideo, type: 'move-to-object-storage' })) + await JobQueue.Instance.createJob(await buildMoveVideoJob({ video, previousVideoState, isNewVideo, type: 'move-to-object-storage' })) return true } catch (err) { @@ -120,7 +120,7 @@ async function moveToFileSystemState (options: { logger.info('Creating move to file system job for video %s.', video.uuid, { tags: [ video.uuid ] }) try { - await JobQueue.Instance.createJob(await buildMoveJob({ video, previousVideoState, isNewVideo, type: 'move-to-file-system' })) + await JobQueue.Instance.createJob(await buildMoveVideoJob({ video, previousVideoState, isNewVideo, type: 'move-to-file-system' })) return true } catch (err) { diff --git a/server/core/models/actor/actor-image.ts b/server/core/models/actor/actor-image.ts index 6d349df7d..79c5fbc3c 100644 --- a/server/core/models/actor/actor-image.ts +++ b/server/core/models/actor/actor-image.ts @@ -1,6 +1,6 @@ import { ActivityIconObject, ActorImage, ActorImageType, type ActorImageType_Type } from '@peertube/peertube-models' import { getLowercaseExtension } from '@peertube/peertube-node-utils' -import { MActorId, MActorImage, MActorImageFormattable } from '@server/types/models/index.js' +import { MActorId, MActorImage, MActorImageFormattable, MActorImagePath } from '@server/types/models/index.js' import { remove } from 'fs-extra/esm' import { join } from 'path' import { Op } from 'sequelize' @@ -149,7 +149,7 @@ export class ActorImageModel extends SequelizeModel { }) } - static getImageUrl (image: MActorImage) { + static getImageUrl (image: MActorImagePath) { if (!image) return undefined return WEBSERVER.URL + image.getStaticPath() @@ -161,6 +161,7 @@ export class ActorImageModel extends SequelizeModel { return { width: this.width, path: this.getStaticPath(), + fileUrl: ActorImageModel.getImageUrl(this), createdAt: this.createdAt, updatedAt: this.updatedAt } @@ -178,7 +179,7 @@ export class ActorImageModel extends SequelizeModel { } } - getStaticPath () { + getStaticPath (this: MActorImagePath) { switch (this.type) { case ActorImageType.AVATAR: return join(LAZY_STATIC_PATHS.AVATARS, this.filename) diff --git a/server/core/models/user/user-notification.ts b/server/core/models/user/user-notification.ts index 2f742a2b1..52d72a5ba 100644 --- a/server/core/models/user/user-notification.ts +++ b/server/core/models/user/user-notification.ts @@ -21,6 +21,7 @@ import { VideoModel } from '../video/video.js' import { UserNotificationListQueryBuilder } from './sql/user-notitication-list-query-builder.js' import { UserRegistrationModel } from './user-registration.js' import { UserModel } from './user.js' +import { ActorImageModel } from '../actor/actor-image.js' @Table({ tableName: 'userNotification', @@ -552,13 +553,7 @@ export class UserNotificationModel extends SequelizeModel formatAvatar (a: UserNotificationIncludes.ActorImageInclude) { return { - path: a.getStaticPath(), - width: a.width - } - } - - formatVideoCaption (a: UserNotificationIncludes.ActorImageInclude) { - return { + fileUrl: ActorImageModel.getImageUrl(a), path: a.getStaticPath(), width: a.width } diff --git a/server/core/models/video/storyboard.ts b/server/core/models/video/storyboard.ts index 039b24b12..84a352a5f 100644 --- a/server/core/models/video/storyboard.ts +++ b/server/core/models/video/storyboard.ts @@ -141,6 +141,10 @@ export class StoryboardModel extends SequelizeModel { return this.fileUrl } + getFileUrl () { + return WEBSERVER.URL + this.getLocalStaticPath() + } + getLocalStaticPath () { return LAZY_STATIC_PATHS.STORYBOARDS + this.filename } @@ -155,6 +159,7 @@ export class StoryboardModel extends SequelizeModel { toFormattedJSON (this: MStoryboardVideo): Storyboard { return { + fileUrl: this.getFileUrl(), storyboardPath: this.getLocalStaticPath(), totalHeight: this.totalHeight, diff --git a/server/core/models/video/thumbnail.ts b/server/core/models/video/thumbnail.ts index 51a9971ef..2a1827096 100644 --- a/server/core/models/video/thumbnail.ts +++ b/server/core/models/video/thumbnail.ts @@ -191,7 +191,8 @@ export class ThumbnailModel extends SequelizeModel { getOriginFileUrl (videoOrPlaylist: MVideo | MVideoPlaylist) { const staticPath = ThumbnailModel.types[this.type].staticPath + this.filename - if (videoOrPlaylist.isOwned()) return WEBSERVER.URL + staticPath + // FIXME: typings + if ((videoOrPlaylist as MVideo).isOwned()) return WEBSERVER.URL + staticPath return this.fileUrl } diff --git a/server/core/models/video/video-caption.ts b/server/core/models/video/video-caption.ts index 1e6971845..7c71f1309 100644 --- a/server/core/models/video/video-caption.ts +++ b/server/core/models/video/video-caption.ts @@ -1,11 +1,14 @@ -import { VideoCaption, VideoCaptionObject } from '@peertube/peertube-models' +import { FileStorage, type FileStorageType, VideoCaption, VideoCaptionObject } from '@peertube/peertube-models' import { buildUUID } from '@peertube/peertube-node-utils' +import { getObjectStoragePublicFileUrl } from '@server/lib/object-storage/urls.js' +import { removeCaptionObjectStorage } from '@server/lib/object-storage/videos.js' import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionLanguageUrl, - MVideoCaptionVideo + MVideoCaptionVideo, + MVideoOwned } from '@server/types/models/index.js' import { remove } from 'fs-extra/esm' import { join } from 'path' @@ -17,6 +20,7 @@ import { Column, CreatedAt, DataType, + Default, ForeignKey, Is, Scopes, Table, @@ -26,7 +30,7 @@ import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/vid import { logger } from '../../helpers/logger.js' import { CONFIG } from '../../initializers/config.js' import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, VIDEO_LANGUAGES, WEBSERVER } from '../../initializers/constants.js' -import { SequelizeModel, buildWhereIdOrUUID, throwIfNotValid } from '../shared/index.js' +import { SequelizeModel, buildWhereIdOrUUID, doesExist, throwIfNotValid } from '../shared/index.js' import { VideoModel } from './video.js' export enum ScopeNames { @@ -79,6 +83,11 @@ export class VideoCaptionModel extends SequelizeModel { @Column filename: string + @AllowNull(false) + @Default(FileStorage.FILE_SYSTEM) + @Column + storage: FileStorageType + @AllowNull(true) @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max)) fileUrl: string @@ -127,8 +136,30 @@ export class VideoCaptionModel extends SequelizeModel { return caption.save({ transaction }) } + static async doesOwnedFileExist (filename: string, storage: FileStorageType) { + const query = 'SELECT 1 FROM "videoCaption" ' + + `WHERE "filename" = $filename AND "storage" = $storage LIMIT 1` + + return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } }) + } + // --------------------------------------------------------------------------- + static loadWithVideo (captionId: number, transaction?: Transaction): Promise { + const query = { + where: { id: captionId }, + include: [ + { + model: VideoModel.unscoped(), + attributes: videoAttributes + } + ], + transaction + } + + return VideoCaptionModel.findOne(query) + } + static loadByVideoIdAndLanguage (videoId: string | number, language: string, transaction?: Transaction): Promise { const videoInclude = { model: VideoModel.unscoped(), @@ -231,7 +262,13 @@ export class VideoCaptionModel extends SequelizeModel { label: VideoCaptionModel.getLanguageLabel(this.language) }, automaticallyGenerated: this.automaticallyGenerated, - captionPath: this.getCaptionStaticPath(), + + captionPath: this.Video.isOwned() && this.fileUrl + ? null // On object storage + : this.getCaptionStaticPath(), + + fileUrl: this.getFileUrl(this.Video), + updatedAt: this.updatedAt.toISOString() } } @@ -241,7 +278,7 @@ export class VideoCaptionModel extends SequelizeModel { identifier: this.language, name: VideoCaptionModel.getLanguageLabel(this.language), automaticallyGenerated: this.automaticallyGenerated, - url: this.getFileUrl(video) + url: this.getOriginFileUrl(video) } } @@ -260,15 +297,31 @@ export class VideoCaptionModel extends SequelizeModel { } removeCaptionFile (this: MVideoCaption) { + if (this.storage === FileStorage.OBJECT_STORAGE) { + return removeCaptionObjectStorage(this) + } + return remove(this.getFSPath()) } - getFileUrl (this: MVideoCaptionLanguageUrl, video: MVideo) { - if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath() + // --------------------------------------------------------------------------- + + getFileUrl (this: MVideoCaptionLanguageUrl, video: MVideoOwned) { + if (video.isOwned() && this.storage === FileStorage.OBJECT_STORAGE) { + return getObjectStoragePublicFileUrl(this.fileUrl, CONFIG.OBJECT_STORAGE.CAPTIONS) + } + + return WEBSERVER.URL + this.getCaptionStaticPath() + } + + getOriginFileUrl (this: MVideoCaptionLanguageUrl, video: MVideoOwned) { + if (video.isOwned()) return this.getFileUrl(video) return this.fileUrl } + // --------------------------------------------------------------------------- + isEqual (this: MVideoCaption, other: MVideoCaption) { if (this.fileUrl) return this.fileUrl === other.fileUrl diff --git a/server/core/models/video/video-file.ts b/server/core/models/video/video-file.ts index 101fa0607..9979078ec 100644 --- a/server/core/models/video/video-file.ts +++ b/server/core/models/video/video-file.ts @@ -278,7 +278,7 @@ export class VideoFileModel extends SequelizeModel { return doesExist({ sequelize: this.sequelize, query, bind: { filename } }) } - static async doesOwnedFileExist (filename: string, storage: FileStorageType) { + static async doesOwnedWebVideoFileExist (filename: string, storage: FileStorageType) { const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + `WHERE "filename" = $filename AND "storage" = $storage LIMIT 1` diff --git a/server/core/models/video/video.ts b/server/core/models/video/video.ts index cf8b39fa9..72c3cccf6 100644 --- a/server/core/models/video/video.ts +++ b/server/core/models/video/video.ts @@ -99,6 +99,7 @@ import { MVideoFullLight, MVideoId, MVideoImmutable, + MVideoOwned, MVideoThumbnail, MVideoThumbnailBlacklist, MVideoWithAllFiles, @@ -935,7 +936,7 @@ export class VideoModel extends SequelizeModel { }, include: [ { - attributes: [ 'filename', 'language', 'fileUrl' ], + attributes: [ 'filename', 'language', 'storage', 'fileUrl' ], model: VideoCaptionModel.unscoped(), required: false }, @@ -1845,7 +1846,7 @@ export class VideoModel extends SequelizeModel { // --------------------------------------------------------------------------- - isOwned () { + isOwned (this: MVideoOwned) { return this.remote === false } @@ -1922,7 +1923,7 @@ export class VideoModel extends SequelizeModel { if (isArray(videoAP.VideoCaptions)) return videoAP.VideoCaptions return this.$get('VideoCaptions', { - attributes: [ 'filename', 'language', 'fileUrl', 'automaticallyGenerated' ], + attributes: [ 'filename', 'language', 'fileUrl', 'storage', 'automaticallyGenerated' ], transaction }) as Promise } diff --git a/server/core/types/models/actor/actor-image.ts b/server/core/types/models/actor/actor-image.ts index de8a62216..7c9013a9e 100644 --- a/server/core/types/models/actor/actor-image.ts +++ b/server/core/types/models/actor/actor-image.ts @@ -5,8 +5,10 @@ export type MActorImage = ActorImageModel // ############################################################################ +export type MActorImagePath = Pick + // Format for API or AP object export type MActorImageFormattable = FunctionProperties & - Pick + Pick diff --git a/server/core/types/models/user/user-notification.ts b/server/core/types/models/user/user-notification.ts index 0e59c4754..f90ebdab2 100644 --- a/server/core/types/models/user/user-notification.ts +++ b/server/core/types/models/user/user-notification.ts @@ -23,7 +23,7 @@ type Use = PickWith + export type ActorImageInclude = Pick export type VideoInclude = Pick export type VideoIncludeChannel = diff --git a/server/core/types/models/video/video-caption.ts b/server/core/types/models/video/video-caption.ts index ebba1edb5..02f0ffdf1 100644 --- a/server/core/types/models/video/video-caption.ts +++ b/server/core/types/models/video/video-caption.ts @@ -1,6 +1,6 @@ import { PickWith } from '@peertube/peertube-typescript-utils' import { VideoCaptionModel } from '../../../models/video/video-caption.js' -import { MVideo, MVideoUUID } from './video.js' +import { MVideo, MVideoOwned, MVideoUUID } from './video.js' type Use = PickWith @@ -12,12 +12,12 @@ export type MVideoCaption = Omit export type MVideoCaptionLanguage = Pick export type MVideoCaptionLanguageUrl = - Pick + Pick export type MVideoCaptionVideo = MVideoCaption & - Use<'Video', Pick> + Use<'Video', Pick> // ############################################################################ @@ -26,4 +26,4 @@ export type MVideoCaptionVideo = export type MVideoCaptionFormattable = MVideoCaption & Pick & - Use<'Video', MVideoUUID> + Use<'Video', MVideoOwned & MVideoUUID> diff --git a/server/core/types/models/video/video.ts b/server/core/types/models/video/video.ts index e3ec16d9d..8dafbd467 100644 --- a/server/core/types/models/video/video.ts +++ b/server/core/types/models/video/video.ts @@ -44,6 +44,7 @@ export type MVideoUrl = Pick export type MVideoUUID = Pick export type MVideoImmutable = Pick +export type MVideoOwned = Pick export type MVideoIdUrl = MVideoId & MVideoUrl export type MVideoFeed = Pick diff --git a/server/scripts/create-move-video-storage-job.ts b/server/scripts/create-move-video-storage-job.ts index 2d17f602d..dec1efaf7 100644 --- a/server/scripts/create-move-video-storage-job.ts +++ b/server/scripts/create-move-video-storage-job.ts @@ -1,12 +1,15 @@ -import { program } from 'commander' +import { FileStorage, VideoState } from '@peertube/peertube-models' import { toCompleteUUID } from '@server/helpers/custom-validators/misc.js' import { CONFIG } from '@server/initializers/config.js' import { initDatabaseModels } from '@server/initializers/database.js' import { JobQueue } from '@server/lib/job-queue/index.js' import { moveToExternalStorageState, moveToFileSystemState } from '@server/lib/video-state.js' +import { VideoCaptionModel } from '@server/models/video/video-caption.js' +import { VideoSourceModel } from '@server/models/video/video-source.js' import { VideoModel } from '@server/models/video/video.js' -import { VideoState, FileStorage } from '@peertube/peertube-models' -import { MStreamingPlaylist, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' +import { MStreamingPlaylist, MVideoCaption, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' +import { MVideoSource } from '@server/types/models/video/video-source.js' +import { program } from 'commander' program .description('Move videos to another storage.') @@ -83,8 +86,13 @@ async function run () { await createMoveJobIfNeeded({ video: videoFull, type: 'to object storage', - canProcessVideo: (files, hls) => { - return files.some(f => f.storage === FileStorage.FILE_SYSTEM) || hls?.storage === FileStorage.FILE_SYSTEM + canProcessVideo: (options) => { + const { files, hls, source, captions } = options + + return files.some(f => f.storage === FileStorage.FILE_SYSTEM) || + hls?.storage === FileStorage.FILE_SYSTEM || + source?.storage === FileStorage.FILE_SYSTEM || + captions.some(c => c.storage === FileStorage.FILE_SYSTEM) }, handler: () => moveToExternalStorageState({ video: videoFull, isNewVideo: false, transaction: undefined }) }) @@ -97,9 +105,15 @@ async function run () { video: videoFull, type: 'to file system', - canProcessVideo: (files, hls) => { - return files.some(f => f.storage === FileStorage.OBJECT_STORAGE) || hls?.storage === FileStorage.OBJECT_STORAGE + canProcessVideo: options => { + const { files, hls, source, captions } = options + + return files.some(f => f.storage === FileStorage.OBJECT_STORAGE) || + hls?.storage === FileStorage.OBJECT_STORAGE || + source?.storage === FileStorage.OBJECT_STORAGE || + captions.some(c => c.storage === FileStorage.OBJECT_STORAGE) }, + handler: () => moveToFileSystemState({ video: videoFull, isNewVideo: false, transaction: undefined }) }) } @@ -110,7 +124,13 @@ async function createMoveJobIfNeeded (options: { video: MVideoFullLight type: 'to object storage' | 'to file system' - canProcessVideo: (files: MVideoFile[], hls: MStreamingPlaylist) => boolean + canProcessVideo: (options: { + files: MVideoFile[] + hls: MStreamingPlaylist + source: MVideoSource + captions: MVideoCaption[] + }) => boolean + handler: () => Promise }) { const { video, type, canProcessVideo, handler } = options @@ -118,7 +138,10 @@ async function createMoveJobIfNeeded (options: { const files = video.VideoFiles || [] const hls = video.getHLSPlaylist() - if (canProcessVideo(files, hls)) { + const source = await VideoSourceModel.loadLatest(video.id) + const captions = await VideoCaptionModel.listVideoCaptions(video.id) + + if (canProcessVideo({ files, hls, source, captions })) { console.log(`Moving ${type} video ${video.name}`) const success = await handler() diff --git a/server/scripts/house-keeping.ts b/server/scripts/house-keeping.ts index e8c0913e1..439d292c0 100644 --- a/server/scripts/house-keeping.ts +++ b/server/scripts/house-keeping.ts @@ -6,7 +6,7 @@ import Bluebird from 'bluebird' import { askConfirmation, displayPeerTubeMustBeStoppedWarning } from './shared/common.js' const program = createCommand() - .description('Remove unused objects from database or remote files') + .description('Remove remote files') .option('--delete-remote-files', 'Remove remote files (avatars, banners, thumbnails...)') .parse(process.argv) diff --git a/server/scripts/prune-storage.ts b/server/scripts/prune-storage.ts index 84a1fec37..ca0e6495b 100755 --- a/server/scripts/prune-storage.ts +++ b/server/scripts/prune-storage.ts @@ -1,3 +1,4 @@ +import { createCommand } from '@commander-js/extra-typings' import { uniqify } from '@peertube/peertube-core-utils' import { FileStorage, ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models' import { DIRECTORIES, USER_EXPORT_FILE_PREFIX } from '@server/initializers/constants.js' @@ -21,6 +22,13 @@ import { ThumbnailModel } from '../core/models/video/thumbnail.js' import { VideoModel } from '../core/models/video/video.js' import { askConfirmation, displayPeerTubeMustBeStoppedWarning } from './shared/common.js' +const program = createCommand() + .description('Remove unused local objects (video files, captions, user exports...) from object storage or file system') + .option('-y, --yes', 'Auto confirm files deletion') + .parse(process.argv) + +const options = program.opts() + run() .then(() => process.exit(0)) .catch(err => { @@ -56,6 +64,7 @@ class ObjectStoragePruner { await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, this.doesStreamingPlaylistFileExistFactory()) await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES, this.doesOriginalFileExistFactory()) await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.USER_EXPORTS, this.doesUserExportFileExistFactory()) + await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.CAPTIONS, this.doesCaptionFileExistFactory()) if (this.keysToDelete.length === 0) { console.log('No unknown object storage files to delete.') @@ -65,7 +74,7 @@ class ObjectStoragePruner { const formattedKeysToDelete = this.keysToDelete.map(({ bucket, key }) => ` In bucket ${bucket}: ${key}`).join('\n') console.log(`${this.keysToDelete.length} unknown files from object storage can be deleted:\n${formattedKeysToDelete}\n`) - const res = await askPruneConfirmation() + const res = await askPruneConfirmation(options.yes) if (res !== true) { console.log('Exiting without deleting object storage files.') return @@ -97,7 +106,7 @@ class ObjectStoragePruner { ? ` and prefix ${config.PREFIX}` : '' - console.error('Cannot find files to delete in bucket ' + config.BUCKET_NAME + prefixMessage) + console.error('Cannot find files to delete in bucket ' + config.BUCKET_NAME + prefixMessage, { err }) } } @@ -105,13 +114,14 @@ class ObjectStoragePruner { return (key: string) => { const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS) - return VideoFileModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE) + return VideoFileModel.doesOwnedWebVideoFileExist(filename, FileStorage.OBJECT_STORAGE) } } private doesStreamingPlaylistFileExistFactory () { return (key: string) => { - const uuid = basename(dirname(this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS))) + const sanitizedKey = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) + const uuid = dirname(sanitizedKey).replace(/^hls\//, '') return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(uuid, FileStorage.OBJECT_STORAGE) } @@ -133,6 +143,14 @@ class ObjectStoragePruner { } } + private doesCaptionFileExistFactory () { + return (key: string) => { + const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.CAPTIONS) + + return VideoCaptionModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE) + } + } + private sanitizeKey (key: string, config: { PREFIX: string }) { return key.replace(new RegExp(`^${config.PREFIX}`), '') } @@ -191,7 +209,7 @@ class FSPruner { const formattedKeysToDelete = this.pathsToDelete.map(p => ` ${p}`).join('\n') console.log(`${this.pathsToDelete.length} unknown files from filesystem can be deleted:\n${formattedKeysToDelete}\n`) - const res = await askPruneConfirmation() + const res = await askPruneConfirmation(options.yes) if (res !== true) { console.log('Exiting without deleting filesystem files.') return @@ -223,7 +241,7 @@ class FSPruner { // Don't delete private directory if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true - return VideoFileModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM) + return VideoFileModel.doesOwnedWebVideoFileExist(basename(filePath), FileStorage.FILE_SYSTEM) } } @@ -320,7 +338,9 @@ class FSPruner { } } -async function askPruneConfirmation () { +async function askPruneConfirmation (yes?: boolean) { + if (yes === true) return true + return askConfirmation( 'These unknown files can be deleted, but please check your backups first (bugs happen). ' + 'Can we delete these files?' diff --git a/server/scripts/update-object-storage-url.ts b/server/scripts/update-object-storage-url.ts index 76687c18f..b85075d78 100644 --- a/server/scripts/update-object-storage-url.ts +++ b/server/scripts/update-object-storage-url.ts @@ -38,7 +38,8 @@ async function run () { `SELECT COUNT(*) AS "c", 'videoStreamingPlaylist->playlistUrl: ' || COUNT(*) AS "t" FROM "videoStreamingPlaylist" WHERE "playlistUrl" ~ :fromRegexp AND "storage" = :storage`, `SELECT COUNT(*) AS "c", 'videoStreamingPlaylist->segmentsSha256Url: ' || COUNT(*) AS "t" FROM "videoStreamingPlaylist" WHERE "segmentsSha256Url" ~ :fromRegexp AND "storage" = :storage`, `SELECT COUNT(*) AS "c", 'userExport->fileUrl: ' || COUNT(*) AS "t" FROM "userExport" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`, - `SELECT COUNT(*) AS "c", 'videoSource->fileUrl: ' || COUNT(*) AS "t" FROM "videoSource" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage` + `SELECT COUNT(*) AS "c", 'videoSource->fileUrl: ' || COUNT(*) AS "t" FROM "videoSource" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`, + `SELECT COUNT(*) AS "c", 'videoCaption->fileUrl: ' || COUNT(*) AS "t" FROM "videoCaption" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage` ] let hasResults = false @@ -73,7 +74,8 @@ async function run () { `UPDATE "videoStreamingPlaylist" SET "playlistUrl" = regexp_replace("playlistUrl", :fromRegexp, :to) WHERE "storage" = :storage`, `UPDATE "videoStreamingPlaylist" SET "segmentsSha256Url" = regexp_replace("segmentsSha256Url", :fromRegexp, :to) WHERE "storage" = :storage`, `UPDATE "userExport" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`, - `UPDATE "videoSource" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage` + `UPDATE "videoSource" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`, + `UPDATE "videoCaption" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage` ] for (const query of queries) { diff --git a/support/docker/production/config/custom-environment-variables.yaml b/support/docker/production/config/custom-environment-variables.yaml index e9cb6026a..f63d114d5 100644 --- a/support/docker/production/config/custom-environment-variables.yaml +++ b/support/docker/production/config/custom-environment-variables.yaml @@ -112,6 +112,11 @@ object_storage: prefix: "PEERTUBE_OBJECT_STORAGE_ORIGINAL_VIDEO_FILES_PREFIX" base_url: "PEERTUBE_OBJECT_STORAGE_ORIGINAL_VIDEO_FILES_BASE_URL" + captions: + bucket_name: "PEERTUBE_OBJECT_STORAGE_CAPTIONS_BUCKET_NAME" + prefix: "PEERTUBE_OBJECT_STORAGE_CAPTIONS_PREFIX" + base_url: "PEERTUBE_OBJECT_STORAGE_CAPTIONS_BASE_URL" + webadmin: configuration: edition: