Add storyboard support

This commit is contained in:
Chocobozzz 2023-06-01 14:51:16 +02:00
parent 1fb7d09422
commit d8f39b126d
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
79 changed files with 1476 additions and 100 deletions

View File

@ -52,6 +52,20 @@
<div *ngIf="formErrors.cache.torrents.size" class="form-error">{{ formErrors.cache.torrents.size }}</div>
</div>
<div class="form-group" formGroupName="torrents">
<label i18n for="cacheTorrentsSize">Number of video storyboard images to keep in cache</label>
<div class="number-with-unit">
<input
type="number" min="0" id="cacheStoryboardsSize" class="form-control"
formControlName="size" [ngClass]="{ 'input-error': formErrors['cache.storyboards.size'] }"
>
<span i18n>{getCacheSize('storyboards'), plural, =1 {cached storyboard} other {cached storyboards}}</span>
</div>
<div *ngIf="formErrors.cache.storyboards.size" class="form-error">{{ formErrors.cache.storyboards.size }}</div>
</div>
</ng-container>
</div>

View File

@ -10,7 +10,7 @@ export class EditAdvancedConfigurationComponent {
@Input() form: FormGroup
@Input() formErrors: any
getCacheSize (type: 'captions' | 'previews' | 'torrents') {
getCacheSize (type: 'captions' | 'previews' | 'torrents' | 'storyboards') {
return this.form.value['cache'][type]['size']
}
}

View File

@ -9,8 +9,7 @@ import { Notifier } from '@app/core'
import { ServerService } from '@app/core/server/server.service'
import {
ADMIN_EMAIL_VALIDATOR,
CACHE_CAPTIONS_SIZE_VALIDATOR,
CACHE_PREVIEWS_SIZE_VALIDATOR,
CACHE_SIZE_VALIDATOR,
CONCURRENCY_VALIDATOR,
INDEX_URL_VALIDATOR,
INSTANCE_NAME_VALIDATOR,
@ -120,13 +119,16 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
},
cache: {
previews: {
size: CACHE_PREVIEWS_SIZE_VALIDATOR
size: CACHE_SIZE_VALIDATOR
},
captions: {
size: CACHE_CAPTIONS_SIZE_VALIDATOR
size: CACHE_SIZE_VALIDATOR
},
torrents: {
size: CACHE_CAPTIONS_SIZE_VALIDATOR
size: CACHE_SIZE_VALIDATOR
},
storyboards: {
size: CACHE_SIZE_VALIDATOR
}
},
signup: {

View File

@ -33,6 +33,7 @@ import {
LiveVideo,
PeerTubeProblemDocument,
ServerErrorCode,
Storyboard,
VideoCaption,
VideoPrivacy,
VideoState
@ -69,6 +70,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
videoCaptions: VideoCaption[] = []
liveVideo: LiveVideo
videoPassword: string
storyboards: Storyboard[] = []
playlistPosition: number
playlist: VideoPlaylist = null
@ -285,9 +287,10 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
forkJoin([
videoAndLiveObs,
this.videoCaptionService.listCaptions(videoId, videoPassword),
this.videoService.getStoryboards(videoId),
this.userService.getAnonymousOrLoggedUser()
]).subscribe({
next: ([ { video, live, videoFileToken }, captionsResult, loggedInOrAnonymousUser ]) => {
next: ([ { video, live, videoFileToken }, captionsResult, storyboards, loggedInOrAnonymousUser ]) => {
const queryParams = this.route.snapshot.queryParams
const urlOptions = {
@ -309,6 +312,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
video,
live,
videoCaptions: captionsResult.data,
storyboards,
videoFileToken,
videoPassword,
loggedInOrAnonymousUser,
@ -414,6 +418,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
video: VideoDetails
live: LiveVideo
videoCaptions: VideoCaption[]
storyboards: Storyboard[]
videoFileToken: string
videoPassword: string
@ -421,7 +426,17 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
loggedInOrAnonymousUser: User
forceAutoplay: boolean
}) {
const { video, live, videoCaptions, urlOptions, videoFileToken, videoPassword, loggedInOrAnonymousUser, forceAutoplay } = options
const {
video,
live,
videoCaptions,
storyboards,
urlOptions,
videoFileToken,
videoPassword,
loggedInOrAnonymousUser,
forceAutoplay
} = options
this.subscribeToLiveEventsIfNeeded(this.video, video)
@ -430,6 +445,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
this.liveVideo = live
this.videoFileToken = videoFileToken
this.videoPassword = videoPassword
this.storyboards = storyboards
// Re init attributes
this.playerPlaceholderImgSrc = undefined
@ -485,6 +501,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
const params = {
video: this.video,
videoCaptions: this.videoCaptions,
storyboards: this.storyboards,
liveVideo: this.liveVideo,
videoFileToken: this.videoFileToken,
videoPassword: this.videoPassword,
@ -636,6 +653,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
video: VideoDetails
liveVideo: LiveVideo
videoCaptions: VideoCaption[]
storyboards: Storyboard[]
videoFileToken: string
videoPassword: string
@ -646,7 +664,17 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
forceAutoplay: boolean
user?: AuthUser // Keep for plugins
}) {
const { video, liveVideo, videoCaptions, videoFileToken, videoPassword, urlOptions, loggedInOrAnonymousUser, forceAutoplay } = params
const {
video,
liveVideo,
videoCaptions,
storyboards,
videoFileToken,
videoPassword,
urlOptions,
loggedInOrAnonymousUser,
forceAutoplay
} = params
const getStartTime = () => {
const byUrl = urlOptions.startTime !== undefined
@ -673,6 +701,15 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
src: environment.apiUrl + c.captionPath
}))
const storyboard = storyboards.length !== 0
? {
url: environment.apiUrl + storyboards[0].storyboardPath,
height: storyboards[0].spriteHeight,
width: storyboards[0].spriteWidth,
interval: storyboards[0].spriteDuration
}
: undefined
const liveOptions = video.isLive
? { latencyMode: liveVideo.latencyMode }
: undefined
@ -734,6 +771,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
videoPassword: () => videoPassword,
videoCaptions: playerCaptions,
storyboard,
videoShortUUID: video.shortUUID,
videoUUID: video.uuid,
@ -767,6 +805,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
else mode = 'webtorrent'
}
// FIXME: remove, we don't support these old web browsers anymore
// p2p-media-loader needs TextEncoder, fallback on WebTorrent if not available
if (typeof TextEncoder === 'undefined') {
mode = 'webtorrent'

View File

@ -22,21 +22,12 @@ export const SERVICES_TWITTER_USERNAME_VALIDATOR: BuildFormValidator = {
}
}
export const CACHE_PREVIEWS_SIZE_VALIDATOR: BuildFormValidator = {
export const CACHE_SIZE_VALIDATOR: BuildFormValidator = {
VALIDATORS: [ Validators.required, Validators.min(1), Validators.pattern('[0-9]+') ],
MESSAGES: {
required: $localize`Previews cache size is required.`,
min: $localize`Previews cache size must be greater than 1.`,
pattern: $localize`Previews cache size must be a number.`
}
}
export const CACHE_CAPTIONS_SIZE_VALIDATOR: BuildFormValidator = {
VALIDATORS: [ Validators.required, Validators.min(1), Validators.pattern('[0-9]+') ],
MESSAGES: {
required: $localize`Captions cache size is required.`,
min: $localize`Captions cache size must be greater than 1.`,
pattern: $localize`Captions cache size must be a number.`
required: $localize`Cache size is required.`,
min: $localize`Cache size must be greater than 1.`,
pattern: $localize`Cache size must be a number.`
}
}

View File

@ -11,6 +11,7 @@ import {
FeedFormat,
NSFWPolicyType,
ResultList,
Storyboard,
UserVideoRate,
UserVideoRateType,
UserVideoRateUpdate,
@ -344,6 +345,25 @@ export class VideoService {
)
}
// ---------------------------------------------------------------------------
getStoryboards (videoId: string | number) {
return this.authHttp
.get<{ storyboards: Storyboard[] }>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/storyboards')
.pipe(
map(({ storyboards }) => storyboards),
catchError(err => {
if (err.status === 404) {
return of([])
}
this.restExtractor.handleError(err)
})
)
}
// ---------------------------------------------------------------------------
getSource (videoId: number) {
return this.authHttp
.get<{ source: VideoSource }>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/source')
@ -358,6 +378,8 @@ export class VideoService {
)
}
// ---------------------------------------------------------------------------
setVideoLike (id: string, videoPassword: string) {
return this.setVideoRate(id, 'like', videoPassword)
}
@ -370,6 +392,8 @@ export class VideoService {
return this.setVideoRate(id, 'none', videoPassword)
}
// ---------------------------------------------------------------------------
getUserVideoRating (id: string) {
const url = UserService.BASE_USERS_URL + 'me/videos/' + id + '/rating'

View File

@ -6,6 +6,7 @@ import './shared/stats/stats-plugin'
import './shared/bezels/bezels-plugin'
import './shared/peertube/peertube-plugin'
import './shared/resolutions/peertube-resolutions-plugin'
import './shared/control-bar/storyboard-plugin'
import './shared/control-bar/next-previous-video-button'
import './shared/control-bar/p2p-info-button'
import './shared/control-bar/peertube-link-button'
@ -42,6 +43,12 @@ CaptionsButton.prototype.controlText_ = 'Subtitles/CC'
// We just want to display 'Off' instead of 'captions off', keep a space so the variable == true (hacky I know)
CaptionsButton.prototype.label_ = ' '
// TODO: remove when https://github.com/videojs/video.js/pull/7598 is merged
const PlayProgressBar = videojs.getComponent('PlayProgressBar') as any
if (PlayProgressBar.prototype.options_.children.includes('timeTooltip') !== true) {
PlayProgressBar.prototype.options_.children.push('timeTooltip')
}
export class PeertubePlayerManager {
private static playerElementClassName: string
private static playerElementAttributes: { name: string, value: string }[] = []
@ -135,6 +142,10 @@ export class PeertubePlayerManager {
p2pEnabled: options.common.p2pEnabled
})
if (options.common.storyboard) {
player.storyboard(options.common.storyboard)
}
player.on('p2pInfo', (_, data: PlayerNetworkInfo) => {
if (data.source !== 'p2p-media-loader' || isNaN(data.bandwidthEstimate)) return

View File

@ -3,4 +3,5 @@ export * from './p2p-info-button'
export * from './peertube-link-button'
export * from './peertube-live-display'
export * from './peertube-load-progress-bar'
export * from './storyboard-plugin'
export * from './theater-button'

View File

@ -0,0 +1,184 @@
import videojs from 'video.js'
import { StoryboardOptions } from '../../types'
// Big thanks to this beautiful plugin: https://github.com/phloxic/videojs-sprite-thumbnails
// Adapted to respect peertube player style
const Plugin = videojs.getPlugin('plugin')
class StoryboardPlugin extends Plugin {
private url: string
private height: number
private width: number
private interval: number
private cached: boolean
private mouseTimeTooltip: videojs.MouseTimeDisplay
private seekBar: { el(): HTMLElement, mouseTimeDisplay: any, playProgressBar: any }
private progress: any
private spritePlaceholder: HTMLElement
private readonly sprites: { [id: string]: HTMLImageElement } = {}
private readonly boundedHijackMouseTooltip: typeof StoryboardPlugin.prototype.hijackMouseTooltip
constructor (player: videojs.Player, options: videojs.ComponentOptions & StoryboardOptions) {
super(player, options)
this.url = options.url
this.height = options.height
this.width = options.width
this.interval = options.interval
this.boundedHijackMouseTooltip = this.hijackMouseTooltip.bind(this)
this.player.ready(() => {
player.addClass('vjs-storyboard')
this.init()
})
}
init () {
const controls = this.player.controlBar as any
// default control bar component tree is expected
// https://docs.videojs.com/tutorial-components.html#default-component-tree
this.progress = controls?.progressControl
this.seekBar = this.progress?.seekBar
this.mouseTimeTooltip = this.seekBar?.mouseTimeDisplay?.timeTooltip
this.spritePlaceholder = videojs.dom.createEl('div', { className: 'vjs-storyboard-sprite-placeholder' }) as HTMLElement
this.seekBar?.el()?.appendChild(this.spritePlaceholder)
this.player.on([ 'ready', 'loadstart' ], evt => {
if (evt !== 'ready') {
const spriteSource = this.player.currentSources().find(source => {
return Object.prototype.hasOwnProperty.call(source, 'storyboard')
}) as any
const spriteOpts = spriteSource?.['storyboard'] as StoryboardOptions
if (spriteOpts) {
this.url = spriteOpts.url
this.height = spriteOpts.height
this.width = spriteOpts.width
this.interval = spriteOpts.interval
}
}
this.cached = !!this.sprites[this.url]
this.load()
})
}
private load () {
const spriteEvents = [ 'mousemove', 'touchmove' ]
if (this.isReady()) {
if (!this.cached) {
this.sprites[this.url] = videojs.dom.createEl('img', {
src: this.url
})
}
this.progress.on(spriteEvents, this.boundedHijackMouseTooltip)
} else {
this.progress.off(spriteEvents, this.boundedHijackMouseTooltip)
this.resetMouseTooltip()
}
}
private hijackMouseTooltip (evt: Event) {
const sprite = this.sprites[this.url]
const imgWidth = sprite.naturalWidth
const imgHeight = sprite.naturalHeight
const seekBarEl = this.seekBar.el()
if (!sprite.complete || !imgWidth || !imgHeight) {
this.resetMouseTooltip()
return
}
this.player.requestNamedAnimationFrame('StoryBoardPlugin#hijackMouseTooltip', () => {
const seekBarRect = videojs.dom.getBoundingClientRect(seekBarEl)
const playerRect = videojs.dom.getBoundingClientRect(this.player.el())
if (!seekBarRect || !playerRect) return
const seekBarX = videojs.dom.getPointerPosition(seekBarEl, evt).x
let position = seekBarX * this.player.duration()
const maxPosition = Math.round((imgHeight / this.height) * (imgWidth / this.width)) - 1
position = Math.min(position / this.interval, maxPosition)
const responsive = 600
const playerWidth = this.player.currentWidth()
const scaleFactor = responsive && playerWidth < responsive
? playerWidth / responsive
: 1
const columns = imgWidth / this.width
const scaledWidth = this.width * scaleFactor
const scaledHeight = this.height * scaleFactor
const cleft = Math.floor(position % columns) * -scaledWidth
const ctop = Math.floor(position / columns) * -scaledHeight
const bgSize = `${imgWidth * scaleFactor}px ${imgHeight * scaleFactor}px`
const topOffset = -scaledHeight - 60
const previewHalfSize = Math.round(scaledWidth / 2)
let left = seekBarRect.width * seekBarX - previewHalfSize
// Seek bar doesn't take all the player width, so we can add/minus a few more pixels
const minLeft = playerRect.left - seekBarRect.left
const maxLeft = seekBarRect.width - scaledWidth + (playerRect.right - seekBarRect.right)
if (left < minLeft) left = minLeft
if (left > maxLeft) left = maxLeft
const tooltipStyle: { [id: string]: string } = {
'background-image': `url("${this.url}")`,
'background-repeat': 'no-repeat',
'background-position': `${cleft}px ${ctop}px`,
'background-size': bgSize,
'color': '#fff',
'text-shadow': '1px 1px #000',
'position': 'relative',
'top': `${topOffset}px`,
'border': '1px solid #000',
// border should not overlay thumbnail area
'width': `${scaledWidth + 2}px`,
'height': `${scaledHeight + 2}px`
}
tooltipStyle.left = `${left}px`
for (const [ key, value ] of Object.entries(tooltipStyle)) {
this.spritePlaceholder.style.setProperty(key, value)
}
})
}
private resetMouseTooltip () {
if (this.spritePlaceholder) {
this.spritePlaceholder.style.cssText = ''
}
}
private isReady () {
return this.mouseTimeTooltip && this.width && this.height && this.url
}
}
videojs.registerPlugin('storyboard', StoryboardPlugin)
export { StoryboardPlugin }

View File

@ -1,6 +1,6 @@
import { PluginsManager } from '@root-helpers/plugins-manager'
import { LiveVideoLatencyMode, VideoFile } from '@shared/models'
import { PlaylistPluginOptions, VideoJSCaption } from './peertube-videojs-typings'
import { PlaylistPluginOptions, VideoJSCaption, VideoJSStoryboard } from './peertube-videojs-typings'
export type PlayerMode = 'webtorrent' | 'p2p-media-loader'
@ -78,6 +78,7 @@ export interface CommonOptions extends CustomizationOptions {
language?: string
videoCaptions: VideoJSCaption[]
storyboard: VideoJSStoryboard
videoUUID: string
videoShortUUID: string

View File

@ -49,6 +49,8 @@ declare module 'video.js' {
stats (options?: StatsCardOptions): StatsForNerdsPlugin
storyboard (options: StoryboardOptions): void
textTracks (): TextTrackList & {
tracks_: (TextTrack & { id: string, label: string, src: string })[]
}
@ -89,6 +91,13 @@ type VideoJSCaption = {
src: string
}
type VideoJSStoryboard = {
url: string
width: number
height: number
interval: number
}
type PeerTubePluginOptions = {
mode: PlayerMode
@ -118,6 +127,13 @@ type MetricsPluginOptions = {
videoUUID: string
}
type StoryboardOptions = {
url: string
width: number
height: number
interval: number
}
type PlaylistPluginOptions = {
elements: VideoPlaylistElement[]
@ -238,6 +254,7 @@ type PlaylistItemOptions = {
export {
PlayerNetworkInfo,
VideoJSStoryboard,
PlaylistItemOptions,
NextPreviousVideoButtonOptions,
ResolutionUpdateData,
@ -251,6 +268,7 @@ export {
PeerTubeResolution,
VideoJSPluginOptions,
LoadedQualityData,
StoryboardOptions,
PeerTubeLinkButtonOptions,
PeerTubeP2PInfoButtonOptions
}

View File

@ -3,6 +3,20 @@
@use '_mixins' as *;
@use './_player-variables' as *;
// Like the time tooltip
.video-js .vjs-progress-holder .vjs-storyboard-sprite-placeholder {
display: none;
}
.video-js .vjs-progress-control:hover .vjs-storyboard-sprite-placeholder,
.video-js .vjs-progress-control:hover .vjs-progress-holder:focus .vjs-storyboard-sprite-placeholder {
display: block;
// Ensure that we maintain a font-size of ~10px.
font-size: 0.6em;
visibility: visible;
}
.video-js.vjs-peertube-skin .vjs-control-bar {
z-index: 100;
@ -79,6 +93,7 @@
top: -0.3em;
}
// Only used on mobile
.vjs-time-tooltip {
display: none;
}

View File

@ -6,6 +6,31 @@
/* Special mobile style */
.video-js.vjs-peertube-skin.vjs-is-mobile {
// No hover means we can't display the storyboard/time tooltip on mouse hover
// Use the time tooltip in progress control instead
.vjs-mouse-display {
display: none !important;
}
.vjs-storyboard-sprite-placeholder {
display: none;
}
.vjs-progress-control .vjs-sliding {
.vjs-time-tooltip,
.vjs-storyboard-sprite-placeholder {
display: block !important;
visibility: visible !important;
}
.vjs-time-tooltip {
color: #fff;
background-color: rgba(0, 0, 0, 0.8);
}
}
.vjs-control-bar {
.vjs-progress-control .vjs-slider .vjs-play-progress {
// Always display the circle on mobile

View File

@ -136,6 +136,7 @@ storage:
logs: 'storage/logs/'
previews: 'storage/previews/'
thumbnails: 'storage/thumbnails/'
storyboards: 'storage/storyboards/'
torrents: 'storage/torrents/'
captions: 'storage/captions/'
cache: 'storage/cache/'
@ -396,6 +397,8 @@ cache:
size: 500 # Max number of video captions/subtitles you want to cache
torrents:
size: 500 # Max number of video torrents you want to cache
storyboards:
size: 500 # Max number of video storyboards you want to cache
admin:
# Used to generate the root user at first startup

View File

@ -134,6 +134,7 @@ storage:
logs: '/var/www/peertube/storage/logs/'
previews: '/var/www/peertube/storage/previews/'
thumbnails: '/var/www/peertube/storage/thumbnails/'
storyboards: '/var/www/peertube/storage/storyboards/'
torrents: '/var/www/peertube/storage/torrents/'
captions: '/var/www/peertube/storage/captions/'
cache: '/var/www/peertube/storage/cache/'
@ -406,6 +407,8 @@ cache:
size: 500 # Max number of video captions/subtitles you want to cache
torrents:
size: 500 # Max number of video torrents you want to cache
storyboards:
size: 500 # Max number of video storyboards you want to cache
admin:
# Used to generate the root user at first startup

View File

@ -19,6 +19,7 @@ storage:
logs: 'test1/logs/'
previews: 'test1/previews/'
thumbnails: 'test1/thumbnails/'
storyboards: 'test1/storyboards/'
torrents: 'test1/torrents/'
captions: 'test1/captions/'
cache: 'test1/cache/'

View File

@ -19,6 +19,7 @@ storage:
logs: 'test2/logs/'
previews: 'test2/previews/'
thumbnails: 'test2/thumbnails/'
storyboards: 'test2/storyboards/'
torrents: 'test2/torrents/'
captions: 'test2/captions/'
cache: 'test2/cache/'

View File

@ -19,6 +19,7 @@ storage:
logs: 'test3/logs/'
previews: 'test3/previews/'
thumbnails: 'test3/thumbnails/'
storyboards: 'test3/storyboards/'
torrents: 'test3/torrents/'
captions: 'test3/captions/'
cache: 'test3/cache/'

View File

@ -19,6 +19,7 @@ storage:
logs: 'test4/logs/'
previews: 'test4/previews/'
thumbnails: 'test4/thumbnails/'
storyboards: 'test4/storyboards/'
torrents: 'test4/torrents/'
captions: 'test4/captions/'
cache: 'test4/cache/'

View File

@ -19,6 +19,7 @@ storage:
logs: 'test5/logs/'
previews: 'test5/previews/'
thumbnails: 'test5/thumbnails/'
storyboards: 'test5/storyboards/'
torrents: 'test5/torrents/'
captions: 'test5/captions/'
cache: 'test5/cache/'

View File

@ -19,6 +19,7 @@ storage:
logs: 'test6/logs/'
previews: 'test6/previews/'
thumbnails: 'test6/thumbnails/'
storyboards: 'test6/storyboards/'
torrents: 'test6/torrents/'
captions: 'test6/captions/'
cache: 'test6/cache/'

View File

@ -73,6 +73,8 @@ cache:
size: 1
torrents:
size: 1
storyboards:
size: 1
signup:
enabled: true

View File

@ -101,7 +101,7 @@ loadLanguages()
import { installApplication } from './server/initializers/installer'
import { Emailer } from './server/lib/emailer'
import { JobQueue } from './server/lib/job-queue'
import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache'
import { VideosPreviewCache, VideosCaptionCache, VideosStoryboardCache } from './server/lib/files-cache'
import {
activityPubRouter,
apiRouter,
@ -316,6 +316,7 @@ async function startApplication () {
VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE)
VideosStoryboardCache.Instance.init(CONFIG.CACHE.STORYBOARDS.SIZE, FILES_CACHE.STORYBOARDS.MAX_AGE)
// Enable Schedulers
ActorFollowScheduler.Instance.enable()

View File

@ -33,7 +33,6 @@ import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '
import { AccountModel } from '../../models/account/account'
import { AccountVideoRateModel } from '../../models/account/account-video-rate'
import { ActorFollowModel } from '../../models/actor/actor-follow'
import { VideoCaptionModel } from '../../models/video/video-caption'
import { VideoCommentModel } from '../../models/video/video-comment'
import { VideoPlaylistModel } from '../../models/video/video-playlist'
import { VideoShareModel } from '../../models/video/video-share'
@ -242,14 +241,13 @@ async function videoController (req: express.Request, res: express.Response) {
if (redirectIfNotOwned(video.url, res)) return
// We need captions to render AP object
const captions = await VideoCaptionModel.listVideoCaptions(video.id)
const videoWithCaptions = Object.assign(video, { VideoCaptions: captions })
const videoAP = await video.lightAPToFullAP(undefined)
const audience = getAudience(videoWithCaptions.VideoChannel.Account.Actor, videoWithCaptions.privacy === VideoPrivacy.PUBLIC)
const videoObject = audiencify(await videoWithCaptions.toActivityPubObject(), audience)
const audience = getAudience(videoAP.VideoChannel.Account.Actor, videoAP.privacy === VideoPrivacy.PUBLIC)
const videoObject = audiencify(await videoAP.toActivityPubObject(), audience)
if (req.path.endsWith('/activity')) {
const data = buildCreateActivity(videoWithCaptions.url, video.VideoChannel.Account.Actor, videoObject, audience)
const data = buildCreateActivity(videoAP.url, video.VideoChannel.Account.Actor, videoObject, audience)
return activityPubResponse(activityPubContextify(data, 'Video'), res)
}

View File

@ -190,6 +190,9 @@ function customConfig (): CustomConfig {
},
torrents: {
size: CONFIG.CACHE.TORRENTS.SIZE
},
storyboards: {
size: CONFIG.CACHE.STORYBOARDS.SIZE
}
},
signup: {

View File

@ -41,6 +41,7 @@ import { liveRouter } from './live'
import { ownershipVideoRouter } from './ownership'
import { rateVideoRouter } from './rate'
import { statsRouter } from './stats'
import { storyboardRouter } from './storyboard'
import { studioRouter } from './studio'
import { tokenRouter } from './token'
import { transcodingRouter } from './transcoding'
@ -70,6 +71,7 @@ videosRouter.use('/', filesRouter)
videosRouter.use('/', transcodingRouter)
videosRouter.use('/', tokenRouter)
videosRouter.use('/', videoPasswordRouter)
videosRouter.use('/', storyboardRouter)
videosRouter.get('/categories',
openapiOperationDoc({ operationId: 'getCategories' }),

View File

@ -0,0 +1,29 @@
import express from 'express'
import { getVideoWithAttributes } from '@server/helpers/video'
import { StoryboardModel } from '@server/models/video/storyboard'
import { asyncMiddleware, videosGetValidator } from '../../../middlewares'
const storyboardRouter = express.Router()
storyboardRouter.get('/:id/storyboards',
asyncMiddleware(videosGetValidator),
asyncMiddleware(listStoryboards)
)
// ---------------------------------------------------------------------------
export {
storyboardRouter
}
// ---------------------------------------------------------------------------
async function listStoryboards (req: express.Request, res: express.Response) {
const video = getVideoWithAttributes(res)
const storyboards = await StoryboardModel.listStoryboardsOf(video)
return res.json({
storyboards: storyboards.map(s => s.toFormattedJSON())
})
}

View File

@ -234,6 +234,15 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
}
},
{
type: 'generate-video-storyboard' as 'generate-video-storyboard',
payload: {
videoUUID: video.uuid,
// No need to federate, we process these jobs sequentially
federate: false
}
},
{
type: 'notify',
payload: {

View File

@ -5,7 +5,7 @@ import { MActorImage } from '@server/types/models'
import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
import { logger } from '../helpers/logger'
import { ACTOR_IMAGES_SIZE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants'
import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache'
import { VideosCaptionCache, VideosPreviewCache, VideosStoryboardCache } from '../lib/files-cache'
import { actorImagePathUnsafeCache, downloadActorImageFromWorker } from '../lib/local-actor'
import { asyncMiddleware, handleStaticError } from '../middlewares'
import { ActorImageModel } from '../models/actor/actor-image'
@ -32,6 +32,12 @@ lazyStaticRouter.use(
handleStaticError
)
lazyStaticRouter.use(
LAZY_STATIC_PATHS.STORYBOARDS + ':filename',
asyncMiddleware(getStoryboard),
handleStaticError
)
lazyStaticRouter.use(
LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':filename',
asyncMiddleware(getVideoCaption),
@ -126,6 +132,13 @@ async function getPreview (req: express.Request, res: express.Response) {
return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
}
async function getStoryboard (req: express.Request, res: express.Response) {
const result = await VideosStoryboardCache.Instance.getFilePath(req.params.filename)
if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
}
async function getVideoCaption (req: express.Request, res: express.Response) {
const result = await VideosCaptionCache.Instance.getFilePath(req.params.filename)
if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()

View File

@ -1,6 +1,6 @@
import validator from 'validator'
import { logger } from '@server/helpers/logger'
import { ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject } from '@shared/models'
import { ActivityPubStoryboard, ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject, VideoObject } from '@shared/models'
import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos'
import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants'
import { peertubeTruncate } from '../../core-utils'
@ -48,6 +48,10 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
logger.debug('Video has invalid icons', { video })
return false
}
if (!setValidStoryboard(video)) {
logger.debug('Video has invalid preview (storyboard)', { video })
return false
}
// Default attributes
if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED
@ -201,3 +205,36 @@ function setRemoteVideoContent (video: any) {
return true
}
function setValidStoryboard (video: VideoObject) {
if (!video.preview) return true
if (!Array.isArray(video.preview)) return false
video.preview = video.preview.filter(p => isStorybordValid(p))
return true
}
function isStorybordValid (preview: ActivityPubStoryboard) {
if (!preview) return false
if (
preview.type !== 'Image' ||
!isArray(preview.rel) ||
!preview.rel.includes('storyboard')
) {
return false
}
preview.url = preview.url.filter(u => {
return u.mediaType === 'image/jpeg' &&
isActivityPubUrlValid(u.href) &&
validator.isInt(u.width + '', { min: 0 }) &&
validator.isInt(u.height + '', { min: 0 }) &&
validator.isInt(u.tileWidth + '', { min: 0 }) &&
validator.isInt(u.tileHeight + '', { min: 0 }) &&
isActivityPubVideoDurationValid(u.tileDuration)
})
return preview.url.length !== 0
}

View File

@ -29,7 +29,8 @@ function checkMissedConfig () {
'video_channels.max_per_user',
'csp.enabled', 'csp.report_only', 'csp.report_uri',
'security.frameguard.enabled', 'security.powered_by_header.enabled',
'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'admin.email', 'contact_form.enabled',
'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'cache.storyboards.size',
'admin.email', 'contact_form.enabled',
'signup.enabled', 'signup.limit', 'signup.requires_approval', 'signup.requires_email_verification', 'signup.minimum_age',
'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
'redundancy.videos.strategies', 'redundancy.videos.check_interval',

View File

@ -112,6 +112,7 @@ const CONFIG = {
STREAMING_PLAYLISTS_DIR: buildPath(config.get<string>('storage.streaming_playlists')),
REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
STORYBOARDS_DIR: buildPath(config.get<string>('storage.storyboards')),
PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
CAPTIONS_DIR: buildPath(config.get<string>('storage.captions')),
TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')),
@ -482,6 +483,9 @@ const CONFIG = {
},
TORRENTS: {
get SIZE () { return config.get<number>('cache.torrents.size') }
},
STORYBOARDS: {
get SIZE () { return config.get<number>('cache.storyboards.size') }
}
},
INSTANCE: {

View File

@ -174,6 +174,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
'after-video-channel-import': 1,
'move-to-object-storage': 3,
'transcoding-job-builder': 1,
'generate-video-storyboard': 1,
'notify': 1,
'federate-video': 1
}
@ -198,6 +199,7 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
'video-channel-import': 1,
'after-video-channel-import': 1,
'transcoding-job-builder': 1,
'generate-video-storyboard': 1,
'notify': 5,
'federate-video': 3
}
@ -218,6 +220,7 @@ const JOB_TTL: { [id in JobType]: number } = {
'activitypub-refresher': 60000 * 10, // 10 minutes
'video-redundancy': 1000 * 3600 * 3, // 3 hours
'video-live-ending': 1000 * 60 * 10, // 10 minutes
'generate-video-storyboard': 1000 * 60 * 10, // 10 minutes
'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
@ -766,7 +769,8 @@ const LAZY_STATIC_PATHS = {
AVATARS: '/lazy-static/avatars/',
PREVIEWS: '/lazy-static/previews/',
VIDEO_CAPTIONS: '/lazy-static/video-captions/',
TORRENTS: '/lazy-static/torrents/'
TORRENTS: '/lazy-static/torrents/',
STORYBOARDS: '/lazy-static/storyboards/'
}
const OBJECT_STORAGE_PROXY_PATHS = {
PRIVATE_WEBSEED: '/object-storage-proxy/webseed/private/',
@ -813,6 +817,14 @@ const ACTOR_IMAGES_SIZE: { [key in ActorImageType]: { width: number, height: num
]
}
const STORYBOARD = {
SPRITE_SIZE: {
width: 192,
height: 108
},
SPRITES_MAX_EDGE_COUNT: 10
}
const EMBED_SIZE = {
width: 560,
height: 315
@ -824,6 +836,10 @@ const FILES_CACHE = {
DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'),
MAX_AGE: 1000 * 3600 * 3 // 3 hours
},
STORYBOARDS: {
DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'storyboards'),
MAX_AGE: 1000 * 3600 * 24 // 24 hours
},
VIDEO_CAPTIONS: {
DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'video-captions'),
MAX_AGE: 1000 * 3600 * 3 // 3 hours
@ -1090,6 +1106,7 @@ export {
RESUMABLE_UPLOAD_SESSION_LIFETIME,
RUNNER_JOB_STATES,
P2P_MEDIA_LOADER_PEER_VERSION,
STORYBOARD,
ACTOR_IMAGES_SIZE,
ACCEPT_HEADERS,
BCRYPT_SALT_SIZE,

View File

@ -10,6 +10,7 @@ import { UserModel } from '@server/models/user/user'
import { UserNotificationModel } from '@server/models/user/user-notification'
import { UserRegistrationModel } from '@server/models/user/user-registration'
import { UserVideoHistoryModel } from '@server/models/user/user-video-history'
import { StoryboardModel } from '@server/models/video/storyboard'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
@ -167,7 +168,8 @@ async function initDatabaseModels (silent: boolean) {
VideoPasswordModel,
RunnerRegistrationTokenModel,
RunnerModel,
RunnerJobModel
RunnerJobModel,
StoryboardModel
])
// Check extensions exist in the database

View File

@ -46,6 +46,19 @@ const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string
Infohash: 'pt:Infohash',
tileWidth: {
'@type': 'sc:Number',
'@id': 'pt:tileWidth'
},
tileHeight: {
'@type': 'sc:Number',
'@id': 'pt:tileHeight'
},
tileDuration: {
'@type': 'sc:Number',
'@id': 'pt:tileDuration'
},
originallyPublishedAt: 'sc:datePublished',
views: {
'@type': 'sc:Number',

View File

@ -10,8 +10,7 @@ import {
MActor,
MActorLight,
MChannelDefault,
MVideoAP,
MVideoAPWithoutCaption,
MVideoAPLight,
MVideoPlaylistFull,
MVideoRedundancyVideo
} from '../../../types/models'
@ -20,10 +19,10 @@ import { getUpdateActivityPubUrl } from '../url'
import { getActorsInvolvedInVideo } from './shared'
import { broadcastToFollowers, sendVideoRelatedActivity } from './shared/send-utils'
async function sendUpdateVideo (videoArg: MVideoAPWithoutCaption, transaction: Transaction, overriddenByActor?: MActor) {
const video = videoArg as MVideoAP
async function sendUpdateVideo (videoArg: MVideoAPLight, transaction: Transaction, overriddenByActor?: MActor) {
if (!videoArg.hasPrivacyForFederation()) return undefined
if (!video.hasPrivacyForFederation()) return undefined
const video = await videoArg.lightAPToFullAP(transaction)
logger.info('Creating job to update video %s.', video.url)
@ -31,11 +30,6 @@ async function sendUpdateVideo (videoArg: MVideoAPWithoutCaption, transaction: T
const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString())
// Needed to build the AP object
if (!video.VideoCaptions) {
video.VideoCaptions = await video.$get('VideoCaptions', { transaction })
}
const videoObject = await video.toActivityPubObject()
const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)

View File

@ -1,10 +1,9 @@
import { Transaction } from 'sequelize/types'
import { isArray } from '@server/helpers/custom-validators/misc'
import { MVideoAP, MVideoAPWithoutCaption } from '@server/types/models'
import { MVideoAP, MVideoAPLight } from '@server/types/models'
import { sendCreateVideo, sendUpdateVideo } from '../send'
import { shareVideoByServerAndChannel } from '../share'
async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) {
async function federateVideoIfNeeded (videoArg: MVideoAPLight, isNewVideo: boolean, transaction?: Transaction) {
const video = videoArg as MVideoAP
if (
@ -13,13 +12,7 @@ async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVid
// Check the video is public/unlisted and published
video.hasPrivacyForFederation() && video.hasStateForFederation()
) {
// Fetch more attributes that we will need to serialize in AP object
if (isArray(video.VideoCaptions) === false) {
video.VideoCaptions = await video.$get('VideoCaptions', {
attributes: [ 'filename', 'language' ],
transaction
})
}
const video = await videoArg.lightAPToFullAP(transaction)
if (isNewVideo) {
// Now we'll add the video's meta data to our followers

View File

@ -3,6 +3,7 @@ import { deleteAllModels, filterNonExistingModels } from '@server/helpers/databa
import { logger, LoggerTagsFn } from '@server/helpers/logger'
import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
import { setVideoTags } from '@server/lib/video'
import { StoryboardModel } from '@server/models/video/storyboard'
import { VideoCaptionModel } from '@server/models/video/video-caption'
import { VideoFileModel } from '@server/models/video/video-file'
import { VideoLiveModel } from '@server/models/video/video-live'
@ -24,6 +25,7 @@ import {
getFileAttributesFromUrl,
getLiveAttributesFromObject,
getPreviewFromIcons,
getStoryboardAttributeFromObject,
getStreamingPlaylistAttributesFromObject,
getTagsFromObject,
getThumbnailFromIcons
@ -107,6 +109,16 @@ export abstract class APVideoAbstractBuilder {
}
}
protected async insertOrReplaceStoryboard (video: MVideoFullLight, t: Transaction) {
const existingStoryboard = await StoryboardModel.loadByVideo(video.id, t)
if (existingStoryboard) await existingStoryboard.destroy({ transaction: t })
const storyboardAttributes = getStoryboardAttributeFromObject(video, this.videoObject)
if (!storyboardAttributes) return
return StoryboardModel.create(storyboardAttributes, { transaction: t })
}
protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) {
const attributes = getLiveAttributesFromObject(video, this.videoObject)
const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true })

View File

@ -48,6 +48,7 @@ export class APVideoCreator extends APVideoAbstractBuilder {
await this.setTrackers(videoCreated, t)
await this.insertOrReplaceCaptions(videoCreated, t)
await this.insertOrReplaceLive(videoCreated, t)
await this.insertOrReplaceStoryboard(videoCreated, t)
// We added a video in this channel, set it as updated
await channel.setAsUpdated(t)

View File

@ -1,6 +1,6 @@
import { maxBy, minBy } from 'lodash'
import { decode as magnetUriDecode } from 'magnet-uri'
import { basename } from 'path'
import { basename, extname } from 'path'
import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos'
import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos'
import { logger } from '@server/helpers/logger'
@ -25,6 +25,9 @@ import {
VideoStreamingPlaylistType
} from '@shared/models'
import { getDurationFromActivityStream } from '../../activity'
import { isArray } from '@server/helpers/custom-validators/misc'
import { generateImageFilename } from '@server/helpers/image-utils'
import { arrayify } from '@shared/core-utils'
function getThumbnailFromIcons (videoObject: VideoObject) {
let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
@ -166,6 +169,26 @@ function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObje
}))
}
function getStoryboardAttributeFromObject (video: MVideoId, videoObject: VideoObject) {
if (!isArray(videoObject.preview)) return undefined
const storyboard = videoObject.preview.find(p => p.rel.includes('storyboard'))
if (!storyboard) return undefined
const url = arrayify(storyboard.url).find(u => u.mediaType === 'image/jpeg')
return {
filename: generateImageFilename(extname(url.href)),
totalHeight: url.height,
totalWidth: url.width,
spriteHeight: url.tileHeight,
spriteWidth: url.tileWidth,
spriteDuration: getDurationFromActivityStream(url.tileDuration),
fileUrl: url.href,
videoId: video.id
}
}
function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) {
const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
? VideoPrivacy.PUBLIC
@ -228,6 +251,7 @@ export {
getLiveAttributesFromObject,
getCaptionAttributesFromObject,
getStoryboardAttributeFromObject,
getVideoAttributesFromObject
}

View File

@ -57,6 +57,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
await Promise.all([
runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)),
runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)),
runInReadCommittedTransaction(t => this.setStoryboard(videoUpdated, t)),
this.setOrDeleteLive(videoUpdated),
this.setPreview(videoUpdated)
])
@ -138,6 +139,10 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
await this.insertOrReplaceCaptions(videoUpdated, t)
}
private async setStoryboard (videoUpdated: MVideoFullLight, t: Transaction) {
await this.insertOrReplaceStoryboard(videoUpdated, t)
}
private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) {
if (!this.video.isLive) return

View File

@ -1,3 +1,4 @@
export * from './videos-preview-cache'
export * from './videos-caption-cache'
export * from './videos-preview-cache'
export * from './videos-storyboard-cache'
export * from './videos-torrent-cache'

View File

@ -0,0 +1,53 @@
import { join } from 'path'
import { logger } from '@server/helpers/logger'
import { doRequestAndSaveToFile } from '@server/helpers/requests'
import { StoryboardModel } from '@server/models/video/storyboard'
import { FILES_CACHE } from '../../initializers/constants'
import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
class VideosStoryboardCache extends AbstractVideoStaticFileCache <string> {
private static instance: VideosStoryboardCache
private constructor () {
super()
}
static get Instance () {
return this.instance || (this.instance = new this())
}
async getFilePathImpl (filename: string) {
const storyboard = await StoryboardModel.loadWithVideoByFilename(filename)
if (!storyboard) return undefined
if (storyboard.Video.isOwned()) return { isOwned: true, path: storyboard.getPath() }
return this.loadRemoteFile(storyboard.filename)
}
// Key is the storyboard filename
protected async loadRemoteFile (key: string) {
const storyboard = await StoryboardModel.loadWithVideoByFilename(key)
if (!storyboard) return undefined
const destPath = join(FILES_CACHE.STORYBOARDS.DIRECTORY, storyboard.filename)
const remoteUrl = storyboard.getOriginFileUrl(storyboard.Video)
try {
await doRequestAndSaveToFile(remoteUrl, destPath)
logger.debug('Fetched remote storyboard %s to %s.', remoteUrl, destPath)
return { isOwned: false, path: destPath }
} catch (err) {
logger.info('Cannot fetch remote storyboard file %s.', remoteUrl, { err })
return undefined
}
}
}
export {
VideosStoryboardCache
}

View File

@ -0,0 +1,138 @@
import { Job } from 'bullmq'
import { join } from 'path'
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg'
import { generateImageFilename, getImageSize } from '@server/helpers/image-utils'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { CONFIG } from '@server/initializers/config'
import { STORYBOARD } from '@server/initializers/constants'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { StoryboardModel } from '@server/models/video/storyboard'
import { VideoModel } from '@server/models/video/video'
import { MVideo } from '@server/types/models'
import { FFmpegImage, isAudioFile } from '@shared/ffmpeg'
import { GenerateStoryboardPayload } from '@shared/models'
const lTagsBase = loggerTagsFactory('storyboard')
async function processGenerateStoryboard (job: Job): Promise<void> {
const payload = job.data as GenerateStoryboardPayload
const lTags = lTagsBase(payload.videoUUID)
logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags)
const video = await VideoModel.loadFull(payload.videoUUID)
if (!video) {
logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags)
return
}
const inputFile = video.getMaxQualityFile()
await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
const isAudio = await isAudioFile(videoPath)
if (isAudio) {
logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags)
return
}
const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
const filename = generateImageFilename()
const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
const totalSprites = buildTotalSprites(video)
const spriteDuration = Math.round(video.duration / totalSprites)
const spritesCount = findGridSize({
toFind: totalSprites,
maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
})
logger.debug(
'Generating storyboard from video of %s to %s', video.uuid, destination,
{ ...lTags, spritesCount, spriteDuration, videoDuration: video.duration }
)
await ffmpeg.generateStoryboardFromVideo({
destination,
path: videoPath,
sprites: {
size: STORYBOARD.SPRITE_SIZE,
count: spritesCount,
duration: spriteDuration
}
})
const imageSize = await getImageSize(destination)
const existing = await StoryboardModel.loadByVideo(video.id)
if (existing) await existing.destroy()
await StoryboardModel.create({
filename,
totalHeight: imageSize.height,
totalWidth: imageSize.width,
spriteHeight: STORYBOARD.SPRITE_SIZE.height,
spriteWidth: STORYBOARD.SPRITE_SIZE.width,
spriteDuration,
videoId: video.id
})
logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags)
})
if (payload.federate) {
await federateVideoIfNeeded(video, false)
}
}
// ---------------------------------------------------------------------------
export {
processGenerateStoryboard
}
function buildTotalSprites (video: MVideo) {
const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width
const totalSprites = Math.min(Math.ceil(video.duration), maxSprites)
// We can generate a single line
if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites
return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT)
}
function findGridSize (options: {
toFind: number
maxEdgeCount: number
}) {
const { toFind, maxEdgeCount } = options
for (let i = 1; i <= maxEdgeCount; i++) {
for (let j = i; j <= maxEdgeCount; j++) {
if (toFind === i * j) return { width: j, height: i }
}
}
throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)
}
function findGridFit (value: number, maxMultiplier: number) {
for (let i = value; i--; i > 0) {
if (!isPrimeWithin(i, maxMultiplier)) return i
}
throw new Error('Could not find prime number below ' + value)
}
function isPrimeWithin (value: number, maxMultiplier: number) {
if (value < 2) return false
for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) {
if (value % i === 0 && value / i <= maxMultiplier) return false
}
return true
}

View File

@ -306,6 +306,15 @@ async function afterImportSuccess (options: {
Notifier.Instance.notifyOnNewVideoIfNeeded(video)
}
// Generate the storyboard in the job queue, and don't forget to federate an update after
await JobQueue.Instance.createJob({
type: 'generate-video-storyboard' as 'generate-video-storyboard',
payload: {
videoUUID: video.uuid,
federate: true
}
})
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
await JobQueue.Instance.createJob(
await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT })

View File

@ -1,6 +1,8 @@
import { Job } from 'bullmq'
import { readdir, remove } from 'fs-extra'
import { join } from 'path'
import { peertubeTruncate } from '@server/helpers/core-utils'
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
@ -20,8 +22,7 @@ import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@serv
import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg'
import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
import { logger, loggerTagsFactory } from '../../../helpers/logger'
import { peertubeTruncate } from '@server/helpers/core-utils'
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
import { JobQueue } from '../job-queue'
const lTags = loggerTagsFactory('live', 'job')
@ -147,6 +148,8 @@ async function saveReplayToExternalVideo (options: {
}
await moveToNextState({ video: replayVideo, isNewVideo: true })
await createStoryboardJob(replayVideo)
}
async function replaceLiveByReplay (options: {
@ -186,6 +189,7 @@ async function replaceLiveByReplay (options: {
await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory })
// FIXME: should not happen in this function
if (permanentLive) { // Remove session replay
await remove(replayDirectory)
} else { // We won't stream again in this live, we can delete the base replay directory
@ -213,6 +217,8 @@ async function replaceLiveByReplay (options: {
// We consider this is a new video
await moveToNextState({ video: videoWithFiles, isNewVideo: true })
await createStoryboardJob(videoWithFiles)
}
async function assignReplayFilesToVideo (options: {
@ -277,3 +283,13 @@ async function cleanupLiveAndFederate (options: {
logger.warn('Cannot federate live after cleanup', { videoId: video.id, err })
}
}
function createStoryboardJob (video: MVideo) {
return JobQueue.Instance.createJob({
type: 'generate-video-storyboard' as 'generate-video-storyboard',
payload: {
videoUUID: video.uuid,
federate: true
}
})
}

View File

@ -25,6 +25,7 @@ import {
DeleteResumableUploadMetaFilePayload,
EmailPayload,
FederateVideoPayload,
GenerateStoryboardPayload,
JobState,
JobType,
ManageVideoTorrentPayload,
@ -65,6 +66,7 @@ import { processVideoLiveEnding } from './handlers/video-live-ending'
import { processVideoStudioEdition } from './handlers/video-studio-edition'
import { processVideoTranscoding } from './handlers/video-transcoding'
import { processVideosViewsStats } from './handlers/video-views-stats'
import { processGenerateStoryboard } from './handlers/generate-storyboard'
export type CreateJobArgument =
{ type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@ -91,7 +93,8 @@ export type CreateJobArgument =
{ type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
{ type: 'notify', payload: NotifyPayload } |
{ type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
{ type: 'federate-video', payload: FederateVideoPayload }
{ type: 'federate-video', payload: FederateVideoPayload } |
{ type: 'generate-video-storyboard', payload: GenerateStoryboardPayload }
export type CreateJobOptions = {
delay?: number
@ -122,7 +125,8 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
'video-redundancy': processVideoRedundancy,
'video-studio-edition': processVideoStudioEdition,
'video-transcoding': processVideoTranscoding,
'videos-views-stats': processVideosViewsStats
'videos-views-stats': processVideosViewsStats,
'generate-video-storyboard': processGenerateStoryboard
}
const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {
@ -141,10 +145,11 @@ const jobTypes: JobType[] = [
'after-video-channel-import',
'email',
'federate-video',
'transcoding-job-builder',
'generate-video-storyboard',
'manage-video-torrent',
'move-to-object-storage',
'notify',
'transcoding-job-builder',
'video-channel-import',
'video-file-import',
'video-import',

View File

@ -325,8 +325,8 @@ class Redis {
const value = await this.getValue('resumable-upload-' + uploadId)
return value
? JSON.parse(value)
: ''
? JSON.parse(value) as { video: { id: number, shortUUID: string, uuid: string } }
: undefined
}
deleteUploadSession (uploadId: string) {

View File

@ -9,6 +9,7 @@ import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVOD
import { VideoResolution, VideoStorage } from '@shared/models'
import { CONFIG } from '../../initializers/config'
import { VideoFileModel } from '../../models/video/video-file'
import { JobQueue } from '../job-queue'
import { generateWebTorrentVideoFilename } from '../paths'
import { buildFileMetadata } from '../video-file'
import { VideoPathManager } from '../video-path-manager'
@ -198,7 +199,8 @@ export async function mergeAudioVideofile (options: {
return onWebTorrentVideoFileTranscoding({
video,
videoFile: inputVideoFile,
videoOutputPath
videoOutputPath,
wasAudioFile: true
})
})
@ -212,8 +214,9 @@ export async function onWebTorrentVideoFileTranscoding (options: {
video: MVideoFullLight
videoFile: MVideoFile
videoOutputPath: string
wasAudioFile?: boolean // default false
}) {
const { video, videoFile, videoOutputPath } = options
const { video, videoFile, videoOutputPath, wasAudioFile } = options
const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
@ -242,6 +245,17 @@ export async function onWebTorrentVideoFileTranscoding (options: {
await VideoFileModel.customUpsert(videoFile, 'video', undefined)
video.VideoFiles = await video.$get('VideoFiles')
if (wasAudioFile) {
await JobQueue.Instance.createJob({
type: 'generate-video-storyboard' as 'generate-video-storyboard',
payload: {
videoUUID: video.uuid,
// No need to federate, we process these jobs sequentially
federate: false
}
})
}
return { video, videoFile }
} finally {
mutexReleaser()

View File

@ -25,6 +25,7 @@ const customConfigUpdateValidator = [
body('cache.previews.size').isInt(),
body('cache.captions.size').isInt(),
body('cache.torrents.size').isInt(),
body('cache.storyboards.size').isInt(),
body('signup.enabled').isBoolean(),
body('signup.limit').isInt(),

View File

@ -5,6 +5,7 @@ import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
import { VideoViewsManager } from '@server/lib/views/video-views-manager'
import { uuidToShort } from '@shared/extra-utils'
import {
ActivityPubStoryboard,
ActivityTagObject,
ActivityUrlObject,
Video,
@ -347,29 +348,17 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
name: t.name
}))
let language
if (video.language) {
language = {
identifier: video.language,
name: getLanguageLabel(video.language)
}
}
const language = video.language
? { identifier: video.language, name: getLanguageLabel(video.language) }
: undefined
let category
if (video.category) {
category = {
identifier: video.category + '',
name: getCategoryLabel(video.category)
}
}
const category = video.category
? { identifier: video.category + '', name: getCategoryLabel(video.category) }
: undefined
let licence
if (video.licence) {
licence = {
identifier: video.licence + '',
name: getLicenceLabel(video.licence)
}
}
const licence = video.licence
? { identifier: video.licence + '', name: getLicenceLabel(video.licence) }
: undefined
const url: ActivityUrlObject[] = [
// HTML url should be the first element in the array so Mastodon correctly displays the embed
@ -465,6 +454,8 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
height: i.height
})),
preview: buildPreviewAPAttribute(video),
url,
likes: getLocalVideoLikesActivityPubUrl(video),
@ -541,3 +532,30 @@ function buildLiveAPAttributes (video: MVideoAP) {
latencyMode: video.VideoLive.latencyMode
}
}
function buildPreviewAPAttribute (video: MVideoAP): ActivityPubStoryboard[] {
if (!video.Storyboard) return undefined
const storyboard = video.Storyboard
return [
{
type: 'Image',
rel: [ 'storyboard' ],
url: [
{
mediaType: 'image/jpeg',
href: storyboard.getOriginFileUrl(video),
width: storyboard.totalWidth,
height: storyboard.totalHeight,
tileWidth: storyboard.spriteWidth,
tileHeight: storyboard.spriteHeight,
tileDuration: getActivityStreamDuration(storyboard.spriteDuration)
}
]
}
]
}

View File

@ -0,0 +1,169 @@
import { remove } from 'fs-extra'
import { join } from 'path'
import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
import { CONFIG } from '@server/initializers/config'
import { MStoryboard, MStoryboardVideo, MVideo } from '@server/types/models'
import { Storyboard } from '@shared/models'
import { AttributesOnly } from '@shared/typescript-utils'
import { logger } from '../../helpers/logger'
import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
import { VideoModel } from './video'
import { Transaction } from 'sequelize'
@Table({
tableName: 'storyboard',
indexes: [
{
fields: [ 'videoId' ],
unique: true
},
{
fields: [ 'filename' ],
unique: true
}
]
})
export class StoryboardModel extends Model<Partial<AttributesOnly<StoryboardModel>>> {
@AllowNull(false)
@Column
filename: string
@AllowNull(false)
@Column
totalHeight: number
@AllowNull(false)
@Column
totalWidth: number
@AllowNull(false)
@Column
spriteHeight: number
@AllowNull(false)
@Column
spriteWidth: number
@AllowNull(false)
@Column
spriteDuration: number
@AllowNull(true)
@Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max))
fileUrl: string
@ForeignKey(() => VideoModel)
@Column
videoId: number
@BelongsTo(() => VideoModel, {
foreignKey: {
allowNull: true
},
onDelete: 'CASCADE'
})
Video: VideoModel
@CreatedAt
createdAt: Date
@UpdatedAt
updatedAt: Date
@AfterDestroy
static removeInstanceFile (instance: StoryboardModel) {
logger.info('Removing storyboard file %s.', instance.filename)
// Don't block the transaction
instance.removeFile()
.catch(err => logger.error('Cannot remove storyboard file %s.', instance.filename, { err }))
}
static loadByVideo (videoId: number, transaction?: Transaction): Promise<MStoryboard> {
const query = {
where: {
videoId
},
transaction
}
return StoryboardModel.findOne(query)
}
static loadByFilename (filename: string): Promise<MStoryboard> {
const query = {
where: {
filename
}
}
return StoryboardModel.findOne(query)
}
static loadWithVideoByFilename (filename: string): Promise<MStoryboardVideo> {
const query = {
where: {
filename
},
include: [
{
model: VideoModel.unscoped(),
required: true
}
]
}
return StoryboardModel.findOne(query)
}
// ---------------------------------------------------------------------------
static async listStoryboardsOf (video: MVideo): Promise<MStoryboardVideo[]> {
const query = {
where: {
videoId: video.id
}
}
const storyboards = await StoryboardModel.findAll<MStoryboard>(query)
return storyboards.map(s => Object.assign(s, { Video: video }))
}
// ---------------------------------------------------------------------------
getOriginFileUrl (video: MVideo) {
if (video.isOwned()) {
return WEBSERVER.URL + this.getLocalStaticPath()
}
return this.fileUrl
}
getLocalStaticPath () {
return LAZY_STATIC_PATHS.STORYBOARDS + this.filename
}
getPath () {
return join(CONFIG.STORAGE.STORYBOARDS_DIR, this.filename)
}
removeFile () {
return remove(this.getPath())
}
toFormattedJSON (this: MStoryboardVideo): Storyboard {
return {
storyboardPath: this.getLocalStaticPath(),
totalHeight: this.totalHeight,
totalWidth: this.totalWidth,
spriteWidth: this.spriteWidth,
spriteHeight: this.spriteHeight,
spriteDuration: this.spriteDuration
}
}
}

View File

@ -15,7 +15,7 @@ import {
Table,
UpdatedAt
} from 'sequelize-typescript'
import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo } from '@server/types/models'
import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionLanguageUrl, MVideoCaptionVideo } from '@server/types/models'
import { buildUUID } from '@shared/extra-utils'
import { AttributesOnly } from '@shared/typescript-utils'
import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model'
@ -225,7 +225,7 @@ export class VideoCaptionModel extends Model<Partial<AttributesOnly<VideoCaption
}
}
getCaptionStaticPath (this: MVideoCaption) {
getCaptionStaticPath (this: MVideoCaptionLanguageUrl) {
return join(LAZY_STATIC_PATHS.VIDEO_CAPTIONS, this.filename)
}
@ -233,9 +233,7 @@ export class VideoCaptionModel extends Model<Partial<AttributesOnly<VideoCaption
return remove(CONFIG.STORAGE.CAPTIONS_DIR + this.filename)
}
getFileUrl (video: MVideo) {
if (!this.Video) this.Video = video as VideoModel
getFileUrl (this: MVideoCaptionLanguageUrl, video: MVideo) {
if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath()
return this.fileUrl

View File

@ -58,7 +58,7 @@ import {
import { AttributesOnly } from '@shared/typescript-utils'
import { peertubeTruncate } from '../../helpers/core-utils'
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import { exists, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc'
import { exists, isArray, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc'
import {
isVideoDescriptionValid,
isVideoDurationValid,
@ -75,6 +75,7 @@ import {
MChannel,
MChannelAccountDefault,
MChannelId,
MStoryboard,
MStreamingPlaylist,
MStreamingPlaylistFilesVideo,
MUserAccountId,
@ -83,6 +84,8 @@ import {
MVideoAccountLight,
MVideoAccountLightBlacklistAllFiles,
MVideoAP,
MVideoAPLight,
MVideoCaptionLanguageUrl,
MVideoDetails,
MVideoFileVideo,
MVideoFormattable,
@ -126,6 +129,7 @@ import {
VideosIdListQueryBuilder,
VideosModelListQueryBuilder
} from './sql/video'
import { StoryboardModel } from './storyboard'
import { TagModel } from './tag'
import { ThumbnailModel } from './thumbnail'
import { VideoBlacklistModel } from './video-blacklist'
@ -753,6 +757,15 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
})
VideoJobInfo: VideoJobInfoModel
@HasOne(() => StoryboardModel, {
foreignKey: {
name: 'videoId',
allowNull: false
},
onDelete: 'cascade'
})
Storyboard: StoryboardModel
@AfterCreate
static notifyCreate (video: MVideo) {
InternalEventEmitter.Instance.emit('video-created', { video })
@ -903,6 +916,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
model: VideoCaptionModel.unscoped(),
required: false
},
{
model: StoryboardModel.unscoped(),
required: false
},
{
attributes: [ 'id', 'url' ],
model: VideoShareModel.unscoped(),
@ -1768,6 +1785,32 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
)
}
async lightAPToFullAP (this: MVideoAPLight, transaction: Transaction): Promise<MVideoAP> {
const videoAP = this as MVideoAP
const getCaptions = () => {
if (isArray(videoAP.VideoCaptions)) return videoAP.VideoCaptions
return this.$get('VideoCaptions', {
attributes: [ 'filename', 'language', 'fileUrl' ],
transaction
}) as Promise<MVideoCaptionLanguageUrl[]>
}
const getStoryboard = () => {
if (videoAP.Storyboard) return videoAP.Storyboard
return this.$get('Storyboard', { transaction }) as Promise<MStoryboard>
}
const [ captions, storyboard ] = await Promise.all([ getCaptions(), getStoryboard() ])
return Object.assign(this, {
VideoCaptions: captions,
Storyboard: storyboard
})
}
getTruncatedDescription () {
if (!this.description) return null

View File

@ -74,6 +74,9 @@ describe('Test config API validators', function () {
},
torrents: {
size: 4
},
storyboards: {
size: 5
}
},
signup: {

View File

@ -34,6 +34,7 @@ import './video-comments'
import './video-files'
import './video-imports'
import './video-playlists'
import './video-storyboards'
import './video-source'
import './video-studio'
import './video-token'

View File

@ -0,0 +1,45 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { HttpStatusCode, VideoPrivacy } from '@shared/models'
import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
describe('Test video storyboards API validator', function () {
let server: PeerTubeServer
let publicVideo: { uuid: string }
let privateVideo: { uuid: string }
// ---------------------------------------------------------------
before(async function () {
this.timeout(30000)
server = await createSingleServer(1)
await setAccessTokensToServers([ server ])
publicVideo = await server.videos.quickUpload({ name: 'public' })
privateVideo = await server.videos.quickUpload({ name: 'private', privacy: VideoPrivacy.PRIVATE })
})
it('Should fail without a valid uuid', async function () {
await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should receive 404 when passing a non existing video id', async function () {
await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should not get the private storyboard without the appropriate token', async function () {
await server.storyboard.list({ id: privateVideo.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null })
await server.storyboard.list({ id: publicVideo.uuid, expectedStatus: HttpStatusCode.OK_200, token: null })
})
it('Should succeed with the correct parameters', async function () {
await server.storyboard.list({ id: privateVideo.uuid })
await server.storyboard.list({ id: publicVideo.uuid })
})
after(async function () {
await cleanupTests([ server ])
})
})

View File

@ -2,7 +2,7 @@
import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands'
describe('Test videos overview', function () {
describe('Test videos overview API validator', function () {
let server: PeerTubeServer
// ---------------------------------------------------------------

View File

@ -46,6 +46,7 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) {
expect(data.cache.previews.size).to.equal(1)
expect(data.cache.captions.size).to.equal(1)
expect(data.cache.torrents.size).to.equal(1)
expect(data.cache.storyboards.size).to.equal(1)
expect(data.signup.enabled).to.be.true
expect(data.signup.limit).to.equal(4)
@ -154,6 +155,7 @@ function checkUpdatedConfig (data: CustomConfig) {
expect(data.cache.previews.size).to.equal(2)
expect(data.cache.captions.size).to.equal(3)
expect(data.cache.torrents.size).to.equal(4)
expect(data.cache.storyboards.size).to.equal(5)
expect(data.signup.enabled).to.be.false
expect(data.signup.limit).to.equal(5)
@ -290,6 +292,9 @@ const newCustomConfig: CustomConfig = {
},
torrents: {
size: 4
},
storyboards: {
size: 5
}
},
signup: {

View File

@ -20,3 +20,4 @@ import './videos-history'
import './videos-overview'
import './video-source'
import './video-static-file-privacy'
import './video-storyboard'

View File

@ -0,0 +1,184 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { expect } from 'chai'
import { FIXTURE_URLS } from '@server/tests/shared'
import { areHttpImportTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, VideoPrivacy } from '@shared/models'
import {
cleanupTests,
createMultipleServers,
doubleFollow,
makeGetRequest,
PeerTubeServer,
sendRTMPStream,
setAccessTokensToServers,
setDefaultVideoChannel,
stopFfmpeg,
waitJobs
} from '@shared/server-commands'
async function checkStoryboard (options: {
server: PeerTubeServer
uuid: string
tilesCount?: number
minSize?: number
}) {
const { server, uuid, tilesCount, minSize = 1000 } = options
const { storyboards } = await server.storyboard.list({ id: uuid })
expect(storyboards).to.have.lengthOf(1)
const storyboard = storyboards[0]
expect(storyboard.spriteDuration).to.equal(1)
expect(storyboard.spriteHeight).to.equal(108)
expect(storyboard.spriteWidth).to.equal(192)
expect(storyboard.storyboardPath).to.exist
if (tilesCount) {
expect(storyboard.totalWidth).to.equal(192 * Math.min(tilesCount, 10))
expect(storyboard.totalHeight).to.equal(108 * Math.max((tilesCount / 10), 1))
}
const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 })
expect(body.length).to.be.above(minSize)
}
describe('Test video storyboard', function () {
let servers: PeerTubeServer[]
before(async function () {
this.timeout(120000)
servers = await createMultipleServers(2)
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
await doubleFollow(servers[0], servers[1])
})
it('Should generate a storyboard after upload without transcoding', async function () {
this.timeout(60000)
// 5s video
const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' })
await waitJobs(servers)
for (const server of servers) {
await checkStoryboard({ server, uuid, tilesCount: 5 })
}
})
it('Should generate a storyboard after upload without transcoding with a long video', async function () {
this.timeout(60000)
// 124s video
const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_very_long_10p.mp4' })
await waitJobs(servers)
for (const server of servers) {
await checkStoryboard({ server, uuid, tilesCount: 100 })
}
})
it('Should generate a storyboard after upload with transcoding', async function () {
this.timeout(60000)
await servers[0].config.enableMinimumTranscoding()
// 5s video
const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' })
await waitJobs(servers)
for (const server of servers) {
await checkStoryboard({ server, uuid, tilesCount: 5 })
}
})
it('Should generate a storyboard after an audio upload', async function () {
this.timeout(60000)
// 6s audio
const attributes = { name: 'audio', fixture: 'sample.ogg' }
const { uuid } = await servers[0].videos.upload({ attributes, mode: 'legacy' })
await waitJobs(servers)
for (const server of servers) {
await checkStoryboard({ server, uuid, tilesCount: 6, minSize: 250 })
}
})
it('Should generate a storyboard after HTTP import', async function () {
this.timeout(60000)
if (areHttpImportTestsDisabled()) return
// 3s video
const { video } = await servers[0].imports.importVideo({
attributes: {
targetUrl: FIXTURE_URLS.goodVideo,
channelId: servers[0].store.channel.id,
privacy: VideoPrivacy.PUBLIC
}
})
await waitJobs(servers)
for (const server of servers) {
await checkStoryboard({ server, uuid: video.uuid, tilesCount: 3 })
}
})
it('Should generate a storyboard after torrent import', async function () {
this.timeout(60000)
if (areHttpImportTestsDisabled()) return
// 10s video
const { video } = await servers[0].imports.importVideo({
attributes: {
magnetUri: FIXTURE_URLS.magnet,
channelId: servers[0].store.channel.id,
privacy: VideoPrivacy.PUBLIC
}
})
await waitJobs(servers)
for (const server of servers) {
await checkStoryboard({ server, uuid: video.uuid, tilesCount: 10 })
}
})
it('Should generate a storyboard after a live', async function () {
this.timeout(240000)
await servers[0].config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' })
const { live, video } = await servers[0].live.quickCreate({
saveReplay: true,
permanentLive: false,
privacy: VideoPrivacy.PUBLIC
})
const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
await servers[0].live.waitUntilPublished({ videoId: video.id })
await stopFfmpeg(ffmpegCommand)
await servers[0].live.waitUntilReplacedByReplay({ videoId: video.id })
await waitJobs(servers)
for (const server of servers) {
await checkStoryboard({ server, uuid: video.uuid })
}
})
it('Should generate a storyboard with different video durations', async function () {
this.timeout(60000)
})
after(async function () {
await cleanupTests(servers)
})
})

Binary file not shown.

View File

@ -1,6 +1,7 @@
export * from './local-video-viewer-watch-section'
export * from './local-video-viewer-watch-section'
export * from './local-video-viewer'
export * from './storyboard'
export * from './schedule-video-update'
export * from './tag'
export * from './thumbnail'

View File

@ -0,0 +1,15 @@
import { StoryboardModel } from '@server/models/video/storyboard'
import { PickWith } from '@shared/typescript-utils'
import { MVideo } from './video'
type Use<K extends keyof StoryboardModel, M> = PickWith<StoryboardModel, K, M>
// ############################################################################
export type MStoryboard = Omit<StoryboardModel, 'Video'>
// ############################################################################
export type MStoryboardVideo =
MStoryboard &
Use<'Video', MVideo>

View File

@ -11,7 +11,7 @@ export type MVideoCaption = Omit<VideoCaptionModel, 'Video'>
// ############################################################################
export type MVideoCaptionLanguage = Pick<MVideoCaption, 'language'>
export type MVideoCaptionLanguageUrl = Pick<MVideoCaption, 'language' | 'fileUrl' | 'getFileUrl'>
export type MVideoCaptionLanguageUrl = Pick<MVideoCaption, 'language' | 'fileUrl' | 'filename' | 'getFileUrl' | 'getCaptionStaticPath'>
export type MVideoCaptionVideo =
MVideoCaption &

View File

@ -3,6 +3,7 @@ import { VideoModel } from '../../../models/video/video'
import { MTrackerUrl } from '../server/tracker'
import { MUserVideoHistoryTime } from '../user/user-video-history'
import { MScheduleVideoUpdate } from './schedule-video-update'
import { MStoryboard } from './storyboard'
import { MTag } from './tag'
import { MThumbnail } from './thumbnail'
import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist'
@ -32,7 +33,7 @@ type Use<K extends keyof VideoModel, M> = PickWith<VideoModel, K, M>
export type MVideo =
Omit<VideoModel, 'VideoChannel' | 'Tags' | 'Thumbnails' | 'VideoPlaylistElements' | 'VideoAbuses' |
'VideoFiles' | 'VideoStreamingPlaylists' | 'VideoShares' | 'AccountVideoRates' | 'VideoComments' | 'VideoViews' | 'UserVideoHistories' |
'ScheduleVideoUpdate' | 'VideoBlacklist' | 'VideoImport' | 'VideoCaptions' | 'VideoLive' | 'Trackers' | 'VideoPasswords'>
'ScheduleVideoUpdate' | 'VideoBlacklist' | 'VideoImport' | 'VideoCaptions' | 'VideoLive' | 'Trackers' | 'VideoPasswords' | 'Storyboard'>
// ############################################################################
@ -173,9 +174,10 @@ export type MVideoAP =
Use<'VideoBlacklist', MVideoBlacklistUnfederated> &
Use<'VideoFiles', MVideoFileRedundanciesOpt[]> &
Use<'Thumbnails', MThumbnail[]> &
Use<'VideoLive', MVideoLive>
Use<'VideoLive', MVideoLive> &
Use<'Storyboard', MStoryboard>
export type MVideoAPWithoutCaption = Omit<MVideoAP, 'VideoCaptions'>
export type MVideoAPLight = Omit<MVideoAP, 'VideoCaptions' | 'Storyboard'>
export type MVideoDetails =
MVideo &

View File

@ -56,4 +56,41 @@ export class FFmpegImage {
.thumbnail(thumbnailOptions)
})
}
async generateStoryboardFromVideo (options: {
path: string
destination: string
sprites: {
size: {
width: number
height: number
}
count: {
width: number
height: number
}
duration: number
}
}) {
const { path, destination, sprites } = options
const command = this.commandWrapper.buildCommand(path)
const filter = [
`setpts=N/round(FRAME_RATE)/TB`,
`select='not(mod(t,${options.sprites.duration}))'`,
`scale=${sprites.size.width}:${sprites.size.height}`,
`tile=layout=${sprites.count.width}x${sprites.count.height}`
].join(',')
command.outputOption('-filter_complex', filter)
command.outputOption('-frames:v', '1')
command.outputOption('-q:v', '2')
command.output(destination)
return this.commandWrapper.runCommand()
}
}

View File

@ -6,5 +6,5 @@ export * from './object.model'
export * from './playlist-element-object'
export * from './playlist-object'
export * from './video-comment-object'
export * from './video-torrent-object'
export * from './video-object'
export * from './watch-action-object'

View File

@ -51,6 +51,22 @@ export interface VideoObject {
attributedTo: ActivityPubAttributedTo[]
preview?: ActivityPubStoryboard[]
to?: string[]
cc?: string[]
}
export interface ActivityPubStoryboard {
type: 'Image'
rel: [ 'storyboard' ]
url: {
href: string
mediaType: string
width: number
height: number
tileWidth: number
tileHeight: number
tileDuration: string
}[]
}

View File

@ -78,6 +78,10 @@ export interface CustomConfig {
torrents: {
size: number
}
storyboards: {
size: number
}
}
signup: {

View File

@ -30,6 +30,7 @@ export type JobType =
| 'video-studio-edition'
| 'video-transcoding'
| 'videos-views-stats'
| 'generate-video-storyboard'
export interface Job {
id: number | string
@ -294,3 +295,10 @@ export interface TranscodingJobBuilderPayload {
priority?: number
}[][]
}
// ---------------------------------------------------------------------------
export interface GenerateStoryboardPayload {
videoUUID: string
federate: boolean
}

View File

@ -15,6 +15,7 @@ export * from './channel-sync'
export * from './nsfw-policy.type'
export * from './storyboard.model'
export * from './thumbnail.type'
export * from './video-constant.model'

View File

@ -0,0 +1,11 @@
export interface Storyboard {
storyboardPath: string
totalHeight: number
totalWidth: number
spriteHeight: number
spriteWidth: number
spriteDuration: number
}

View File

@ -159,6 +159,10 @@ export class ConfigCommand extends AbstractCommand {
newConfig: {
transcoding: {
enabled: true,
allowAudioFiles: true,
allowAdditionalExtensions: true,
resolutions: {
...ConfigCommand.getCustomConfigResolutions(false),
@ -368,6 +372,9 @@ export class ConfigCommand extends AbstractCommand {
},
torrents: {
size: 4
},
storyboards: {
size: 5
}
},
signup: {

View File

@ -33,6 +33,8 @@ async function waitJobs (
// Check if each server has pending request
for (const server of servers) {
if (process.env.DEBUG) console.log('Checking ' + server.url)
for (const state of states) {
const jobPromise = server.jobs.list({
@ -45,6 +47,10 @@ async function waitJobs (
.then(jobs => {
if (jobs.length !== 0) {
pendingRequests = true
if (process.env.DEBUG) {
console.log(jobs)
}
}
})
@ -55,6 +61,10 @@ async function waitJobs (
.then(obj => {
if (obj.activityPubMessagesWaiting !== 0) {
pendingRequests = true
if (process.env.DEBUG) {
console.log('AP messages waiting: ' + obj.activityPubMessagesWaiting)
}
}
})
tasks.push(debugPromise)
@ -65,12 +75,15 @@ async function waitJobs (
for (const job of data) {
if (job.state.id !== RunnerJobState.COMPLETED) {
pendingRequests = true
if (process.env.DEBUG) {
console.log(job)
}
}
}
})
tasks.push(runnerJobsPromise)
}
}
return tasks

View File

@ -35,6 +35,7 @@ import {
VideoPasswordsCommand,
PlaylistsCommand,
ServicesCommand,
StoryboardCommand,
StreamingPlaylistsCommand,
VideosCommand,
VideoStudioCommand,
@ -149,6 +150,8 @@ export class PeerTubeServer {
registrations?: RegistrationsCommand
videoPasswords?: VideoPasswordsCommand
storyboard?: StoryboardCommand
runners?: RunnersCommand
runnerRegistrationTokens?: RunnerRegistrationTokensCommand
runnerJobs?: RunnerJobsCommand
@ -436,6 +439,8 @@ export class PeerTubeServer {
this.videoToken = new VideoTokenCommand(this)
this.registrations = new RegistrationsCommand(this)
this.storyboard = new StoryboardCommand(this)
this.runners = new RunnersCommand(this)
this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this)
this.runnerJobs = new RunnerJobsCommand(this)

View File

@ -11,6 +11,7 @@ export * from './live-command'
export * from './live'
export * from './playlists-command'
export * from './services-command'
export * from './storyboard-command'
export * from './streaming-playlists-command'
export * from './comments-command'
export * from './video-studio-command'

View File

@ -0,0 +1,19 @@
import { HttpStatusCode, Storyboard } from '@shared/models'
import { AbstractCommand, OverrideCommandOptions } from '../shared'
export class StoryboardCommand extends AbstractCommand {
list (options: OverrideCommandOptions & {
id: number | string
}) {
const path = '/api/v1/videos/' + options.id + '/storyboards'
return this.getRequestBody<{ storyboards: Storyboard[] }>({
...options,
path,
implicitToken: true,
defaultExpectedStatus: HttpStatusCode.OK_200
})
}
}

View File

@ -3668,6 +3668,27 @@ paths:
items:
$ref: '#/components/schemas/VideoBlacklist'
/api/v1/videos/{id}/storyboards:
get:
summary: List storyboards of a video
operationId: listVideoStoryboards
tags:
- Video
parameters:
- $ref: '#/components/parameters/idOrUUID'
responses:
'200':
description: successful operation
content:
application/json:
schema:
type: object
properties:
storyboards:
type: array
items:
$ref: '#/components/schemas/Storyboard'
/api/v1/videos/{id}/captions:
get:
summary: List captions of a video
@ -7509,6 +7530,20 @@ components:
type: array
items:
$ref: '#/components/schemas/VideoCommentThreadTree'
Storyboard:
properties:
storyboardPath:
type: string
totalHeight:
type: integer
totalWidth:
type: integer
spriteHeight:
type: integer
spriteWidth:
type: integer
spriteDuration:
type: integer
VideoCaption:
properties:
language: