Add previews cache system between pods

This commit is contained in:
Chocobozzz 2017-07-12 11:56:02 +02:00
parent 075f16caac
commit f981dae861
27 changed files with 202 additions and 16 deletions

1
.gitignore vendored
View File

@ -12,6 +12,7 @@
/certs/
/logs/
/torrents/
/cache/
/config/production.yaml
/ffmpeg/
/*.sublime-project

View File

@ -22,6 +22,11 @@ storage:
previews: 'previews/'
thumbnails: 'thumbnails/'
torrents: 'torrents/'
cache: 'cache/'
cache:
previews:
size: 1 # Max number of previews you want to cache
admin:
email: 'admin@example.com'

View File

@ -23,6 +23,7 @@ storage:
previews: 'previews/'
thumbnails: 'thumbnails/'
torrents: 'torrents/'
cache: 'cache/'
admin:
email: 'admin@example.com'

View File

@ -13,8 +13,10 @@ storage:
certs: 'test1/certs/'
videos: 'test1/videos/'
logs: 'test1/logs/'
previews: 'test1/previews/'
thumbnails: 'test1/thumbnails/'
torrents: 'test1/torrents/'
cache: 'test1/cache/'
admin:
email: 'admin1@example.com'

View File

@ -13,8 +13,10 @@ storage:
certs: 'test2/certs/'
videos: 'test2/videos/'
logs: 'test2/logs/'
previews: 'test2/previews/'
thumbnails: 'test2/thumbnails/'
torrents: 'test2/torrents/'
cache: 'test2/cache/'
admin:
email: 'admin2@example.com'

View File

@ -13,8 +13,10 @@ storage:
certs: 'test3/certs/'
videos: 'test3/videos/'
logs: 'test3/logs/'
previews: 'test3/previews/'
thumbnails: 'test3/thumbnails/'
torrents: 'test3/torrents/'
cache: 'test3/cache/'
admin:
email: 'admin3@example.com'

View File

@ -13,8 +13,10 @@ storage:
certs: 'test4/certs/'
videos: 'test4/videos/'
logs: 'test4/logs/'
previews: 'test4/previews/'
thumbnails: 'test4/thumbnails/'
torrents: 'test4/torrents/'
cache: 'test4/cache/'
admin:
email: 'admin4@example.com'

View File

@ -13,8 +13,10 @@ storage:
certs: 'test5/certs/'
videos: 'test5/videos/'
logs: 'test5/logs/'
previews: 'test5/previews/'
thumbnails: 'test5/thumbnails/'
torrents: 'test5/torrents/'
cache: 'test5/cache/'
admin:
email: 'admin5@example.com'

View File

@ -13,8 +13,10 @@ storage:
certs: 'test6/certs/'
videos: 'test6/videos/'
logs: 'test6/logs/'
previews: 'test6/previews/'
thumbnails: 'test6/thumbnails/'
torrents: 'test6/torrents/'
cache: 'test6/cache/'
admin:
email: 'admin6@example.com'

View File

@ -47,6 +47,7 @@
},
"dependencies": {
"async": "^2.0.0",
"async-lru": "^1.1.1",
"bcrypt": "^1.0.2",
"bittorrent-tracker": "^9.0.0",
"bluebird": "^3.5.0",

View File

@ -47,7 +47,7 @@ if (errorMessage !== null) {
// ----------- PeerTube modules -----------
import { migrate, installApplication } from './server/initializers'
import { JobScheduler, activateSchedulers } from './server/lib'
import { JobScheduler, activateSchedulers, VideosPreviewCache } from './server/lib'
import * as customValidators from './server/helpers/custom-validators'
import { apiRouter, clientsRouter, staticRouter } from './server/controllers'
@ -147,6 +147,8 @@ function onDatabaseInitDone () {
// Activate job scheduler
JobScheduler.Instance.activate()
VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE)
logger.info('Server listening on port %d', port)
logger.info('Webserver: %s', CONFIG.WEBSERVER.URL)
})

View File

@ -6,6 +6,7 @@ import {
STATIC_MAX_AGE,
STATIC_PATHS
} from '../initializers'
import { VideosPreviewCache } from '../lib'
const staticRouter = express.Router()
@ -38,8 +39,8 @@ staticRouter.use(
// Video previews path for express
const previewsPhysicalPath = CONFIG.STORAGE.PREVIEWS_DIR
staticRouter.use(
STATIC_PATHS.PREVIEWS,
express.static(previewsPhysicalPath, { maxAge: STATIC_MAX_AGE })
STATIC_PATHS.PREVIEWS + ':uuid.jpg',
getPreview
)
// ---------------------------------------------------------------------------
@ -47,3 +48,14 @@ staticRouter.use(
export {
staticRouter
}
// ---------------------------------------------------------------------------
function getPreview (req: express.Request, res: express.Response, next: express.NextFunction) {
VideosPreviewCache.Instance.getPreviewPath(req.params.uuid)
.then(path => {
if (!path) return res.sendStatus(404)
return res.sendFile(path, { maxAge: STATIC_MAX_AGE })
})
}

View File

@ -16,6 +16,7 @@ import {
import * as mkdirp from 'mkdirp'
import * as bcrypt from 'bcrypt'
import * as createTorrent from 'create-torrent'
import * as rimraf from 'rimraf'
import * as openssl from 'openssl-wrapper'
import * as Promise from 'bluebird'
@ -83,6 +84,7 @@ const bcryptComparePromise = promisify2<any, string, boolean>(bcrypt.compare)
const bcryptGenSaltPromise = promisify1<number, string>(bcrypt.genSalt)
const bcryptHashPromise = promisify2<any, string|number, string>(bcrypt.hash)
const createTorrentPromise = promisify2<string, any, any>(createTorrent)
const rimrafPromise = promisify1WithVoid<string>(rimraf)
// ---------------------------------------------------------------------------
@ -105,5 +107,6 @@ export {
bcryptComparePromise,
bcryptGenSaltPromise,
bcryptHashPromise,
createTorrentPromise
createTorrentPromise,
rimrafPromise
}

View File

@ -61,7 +61,8 @@ const CONFIG = {
VIDEOS_DIR: join(root(), config.get<string>('storage.videos')),
THUMBNAILS_DIR: join(root(), config.get<string>('storage.thumbnails')),
PREVIEWS_DIR: join(root(), config.get<string>('storage.previews')),
TORRENTS_DIR: join(root(), config.get<string>('storage.torrents'))
TORRENTS_DIR: join(root(), config.get<string>('storage.torrents')),
CACHE_DIR: join(root(), config.get<string>('storage.cache'))
},
WEBSERVER: {
SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http',
@ -80,6 +81,11 @@ const CONFIG = {
TRANSCODING: {
ENABLED: config.get<boolean>('transcoding.enabled'),
THREADS: config.get<number>('transcoding.threads')
},
CACHE: {
PREVIEWS: {
SIZE: config.get<number>('cache.previews.size')
}
}
}
CONFIG.WEBSERVER.URL = CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
@ -278,6 +284,13 @@ let STATIC_MAX_AGE = '30d'
const THUMBNAILS_SIZE = '200x110'
const PREVIEWS_SIZE = '640x480'
// Subfolders of cache directory
const CACHE = {
DIRECTORIES: {
PREVIEWS: join(CONFIG.STORAGE.CACHE_DIR, 'previews')
}
}
// ---------------------------------------------------------------------------
const USER_ROLES: { [ id: string ]: UserRole } = {
@ -307,6 +320,7 @@ if (isTestInstance() === true) {
export {
API_VERSION,
BCRYPT_SALT_SIZE,
CACHE,
CONFIG,
CONSTRAINTS_FIELDS,
FRIEND_SCORE,

View File

@ -4,12 +4,13 @@ import * as passwordGenerator from 'password-generator'
import * as Promise from 'bluebird'
import { database as db } from './database'
import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION } from './constants'
import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION, CACHE } from './constants'
import { clientsExist, usersExist } from './checker'
import { logger, createCertsIfNotExist, root, mkdirpPromise } from '../helpers'
import { logger, createCertsIfNotExist, root, mkdirpPromise, rimrafPromise } from '../helpers'
function installApplication () {
return db.sequelize.sync()
.then(() => removeCacheDirectories())
.then(() => createDirectoriesIfNotExist())
.then(() => createCertsIfNotExist())
.then(() => createOAuthClientIfNotExist())
@ -24,13 +25,34 @@ export {
// ---------------------------------------------------------------------------
function removeCacheDirectories () {
const cacheDirectories = CACHE.DIRECTORIES
const tasks = []
// Cache directories
Object.keys(cacheDirectories).forEach(key => {
const dir = cacheDirectories[key]
tasks.push(rimrafPromise(dir))
})
return Promise.all(tasks)
}
function createDirectoriesIfNotExist () {
const storages = config.get('storage')
const storages = CONFIG.STORAGE
const cacheDirectories = CACHE.DIRECTORIES
const tasks = []
Object.keys(storages).forEach(key => {
const dir = storages[key]
tasks.push(mkdirpPromise(join(root(), dir)))
tasks.push(mkdirpPromise(dir))
})
// Cache directories
Object.keys(cacheDirectories).forEach(key => {
const dir = cacheDirectories[key]
tasks.push(mkdirpPromise(dir))
})
return Promise.all(tasks)

1
server/lib/cache/index.ts vendored Normal file
View File

@ -0,0 +1 @@
export * from './videos-preview-cache'

View File

@ -0,0 +1,74 @@
import * as request from 'request'
import * as asyncLRU from 'async-lru'
import { join } from 'path'
import { createWriteStream } from 'fs'
import * as Promise from 'bluebird'
import { database as db, CONFIG, CACHE } from '../../initializers'
import { logger, writeFilePromise, unlinkPromise } from '../../helpers'
import { VideoInstance } from '../../models'
import { fetchRemotePreview } from '../../lib'
class VideosPreviewCache {
private static instance: VideosPreviewCache
private lru
private constructor () { }
static get Instance () {
return this.instance || (this.instance = new this())
}
init (max: number) {
this.lru = new asyncLRU({
max,
load: (key, cb) => {
this.loadPreviews(key)
.then(res => cb(null, res))
.catch(err => cb(err))
}
})
this.lru.on('evict', (obj: { key: string, value: string }) => {
unlinkPromise(obj.value).then(() => logger.debug('%s evicted from VideosPreviewCache', obj.value))
})
}
getPreviewPath (key: string) {
return new Promise<string>((res, rej) => {
this.lru.get(key, (err, value) => {
err ? rej(err) : res(value)
})
})
}
private loadPreviews (key: string) {
return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(key)
.then(video => {
if (!video) return undefined
if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
return this.saveRemotePreviewAndReturnPath(video)
})
}
private saveRemotePreviewAndReturnPath (video: VideoInstance) {
const req = fetchRemotePreview(video.Author.Pod, video)
return new Promise<string>((res, rej) => {
const path = join(CACHE.DIRECTORIES.PREVIEWS, video.getPreviewName())
const stream = createWriteStream(path)
req.pipe(stream)
.on('finish', () => res(path))
.on('error', (err) => rej(err))
})
}
}
export {
VideosPreviewCache
}

View File

@ -1,6 +1,7 @@
import * as request from 'request'
import * as Sequelize from 'sequelize'
import * as Promise from 'bluebird'
import { join } from 'path'
import { database as db } from '../initializers/database'
import {
@ -9,7 +10,8 @@ import {
REQUESTS_IN_PARALLEL,
REQUEST_ENDPOINTS,
REQUEST_ENDPOINT_ACTIONS,
REMOTE_SCHEME
REMOTE_SCHEME,
STATIC_PATHS
} from '../initializers'
import {
logger,
@ -233,6 +235,13 @@ function sendOwnedVideosToPod (podId: number) {
})
}
function fetchRemotePreview (pod: PodInstance, video: VideoInstance) {
const host = video.Author.Pod.host
const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
return request.get(REMOTE_SCHEME.HTTP + '://' + host + path)
}
function getRequestScheduler () {
return requestScheduler
}
@ -263,7 +272,8 @@ export {
sendOwnedVideosToPod,
getRequestScheduler,
getRequestVideoQaduScheduler,
getRequestVideoEventScheduler
getRequestVideoEventScheduler,
fetchRemotePreview
}
// ---------------------------------------------------------------------------

View File

@ -1,3 +1,4 @@
export * from './cache'
export * from './jobs'
export * from './request'
export * from './friends'

View File

@ -35,6 +35,8 @@ export interface OAuthTokenAttributes {
refreshToken: string
refreshTokenExpiresAt: Date
userId?: number
oAuthClientId?: number
User?: UserModel
}

View File

@ -106,10 +106,10 @@ getByRefreshTokenAndPopulateClient = function (refreshToken: string) {
refreshToken: token.refreshToken,
refreshTokenExpiresAt: token.refreshTokenExpiresAt,
client: {
id: token['client'].id
id: token.oAuthClientId
},
user: {
id: token['user']
id: token.userId
}
}

View File

@ -451,6 +451,7 @@ toFormatedJSON = function (this: VideoInstance) {
dislikes: this.dislikes,
tags: map<TagInstance, string>(this.Tags, 'name'),
thumbnailPath: join(STATIC_PATHS.THUMBNAILS, this.getThumbnailName()),
previewPath: join(STATIC_PATHS.PREVIEWS, this.getPreviewName()),
createdAt: this.createdAt,
updatedAt: this.updatedAt
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 31 KiB

View File

@ -747,7 +747,7 @@ describe('Test multiple pods', function () {
expect(videos[0].name).not.to.equal(toRemove[1].name)
expect(videos[1].name).not.to.equal(toRemove[1].name)
videoUUID = videos[0].uuid
videoUUID = videos.find(video => video.name === 'my super name for pod 1').uuid
callback()
})
@ -781,6 +781,23 @@ describe('Test multiple pods', function () {
})
}, done)
})
it('Should get the preview from each pod', function (done) {
each(servers, function (server, callback) {
videosUtils.getVideo(server.url, videoUUID, function (err, res) {
if (err) throw err
const video = res.body
videosUtils.testVideoImage(server.url, 'video_short1-preview.webm', video.previewPath, function (err, test) {
if (err) throw err
expect(test).to.equal(true)
callback()
})
})
}, done)
})
})
after(function (done) {

View File

@ -195,7 +195,7 @@ function searchVideoWithSort (url, search, sort, end) {
.end(end)
}
function testVideoImage (url, videoName, imagePath, callback) {
function testVideoImage (url, imageName, imagePath, callback) {
// Don't test images if the node env is not set
// Because we need a special ffmpeg version for this test
if (process.env.NODE_TEST_IMAGE) {
@ -205,7 +205,7 @@ function testVideoImage (url, videoName, imagePath, callback) {
.end(function (err, res) {
if (err) return callback(err)
fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', videoName + '.jpg'), function (err, data) {
fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', imageName + '.jpg'), function (err, data) {
if (err) return callback(err)
callback(null, data.equals(res.body))

View File

@ -17,6 +17,7 @@ export interface Video {
podHost: string
tags: string[]
thumbnailPath: string
previewPath: string
views: number
likes: number
dislikes: number

View File

@ -285,6 +285,12 @@ async-each@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d"
async-lru@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/async-lru/-/async-lru-1.1.1.tgz#3edbf7e96484d5c2dd852a8bf9794fc07f5e7274"
dependencies:
lru "^3.1.0"
async@>=0.2.9, async@^2.0.0:
version "2.4.1"
resolved "https://registry.yarnpkg.com/async/-/async-2.4.1.tgz#62a56b279c98a11d0987096a01cc3eeb8eb7bbd7"