From f981dae8617271a2dc713bb683951730b306e0c5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 12 Jul 2017 11:56:02 +0200 Subject: Add previews cache system between pods --- server/controllers/static.ts | 16 ++++- server/helpers/core-utils.ts | 5 +- server/initializers/constants.ts | 16 ++++- server/initializers/installer.ts | 30 +++++++-- server/lib/cache/index.ts | 1 + server/lib/cache/videos-preview-cache.ts | 74 +++++++++++++++++++++ server/lib/friends.ts | 14 +++- server/lib/index.ts | 1 + server/models/oauth/oauth-token-interface.ts | 2 + server/models/oauth/oauth-token.ts | 4 +- server/models/video/video.ts | 1 + .../api/fixtures/video_short1-preview.webm.jpg | Bin 0 -> 31725 bytes server/tests/api/multiple-pods.js | 19 +++++- server/tests/utils/videos.js | 4 +- 14 files changed, 172 insertions(+), 15 deletions(-) create mode 100644 server/lib/cache/index.ts create mode 100644 server/lib/cache/videos-preview-cache.ts create mode 100644 server/tests/api/fixtures/video_short1-preview.webm.jpg (limited to 'server') diff --git a/server/controllers/static.ts b/server/controllers/static.ts index e65282339..2fd14131e 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -6,6 +6,7 @@ import { STATIC_MAX_AGE, STATIC_PATHS } from '../initializers' +import { VideosPreviewCache } from '../lib' const staticRouter = express.Router() @@ -38,8 +39,8 @@ staticRouter.use( // Video previews path for express const previewsPhysicalPath = CONFIG.STORAGE.PREVIEWS_DIR staticRouter.use( - STATIC_PATHS.PREVIEWS, - express.static(previewsPhysicalPath, { maxAge: STATIC_MAX_AGE }) + STATIC_PATHS.PREVIEWS + ':uuid.jpg', + getPreview ) // --------------------------------------------------------------------------- @@ -47,3 +48,14 @@ staticRouter.use( export { staticRouter } + +// --------------------------------------------------------------------------- + +function getPreview (req: express.Request, res: express.Response, next: express.NextFunction) { + VideosPreviewCache.Instance.getPreviewPath(req.params.uuid) + .then(path => { + if (!path) return res.sendStatus(404) + + return res.sendFile(path, { maxAge: STATIC_MAX_AGE }) + }) +} diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 1e92049f1..d28c97f09 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -16,6 +16,7 @@ import { import * as mkdirp from 'mkdirp' import * as bcrypt from 'bcrypt' import * as createTorrent from 'create-torrent' +import * as rimraf from 'rimraf' import * as openssl from 'openssl-wrapper' import * as Promise from 'bluebird' @@ -83,6 +84,7 @@ const bcryptComparePromise = promisify2(bcrypt.compare) const bcryptGenSaltPromise = promisify1(bcrypt.genSalt) const bcryptHashPromise = promisify2(bcrypt.hash) const createTorrentPromise = promisify2(createTorrent) +const rimrafPromise = promisify1WithVoid(rimraf) // --------------------------------------------------------------------------- @@ -105,5 +107,6 @@ export { bcryptComparePromise, bcryptGenSaltPromise, bcryptHashPromise, - createTorrentPromise + createTorrentPromise, + rimrafPromise } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index f087b7476..928a3f570 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -61,7 +61,8 @@ const CONFIG = { VIDEOS_DIR: join(root(), config.get('storage.videos')), THUMBNAILS_DIR: join(root(), config.get('storage.thumbnails')), PREVIEWS_DIR: join(root(), config.get('storage.previews')), - TORRENTS_DIR: join(root(), config.get('storage.torrents')) + TORRENTS_DIR: join(root(), config.get('storage.torrents')), + CACHE_DIR: join(root(), config.get('storage.cache')) }, WEBSERVER: { SCHEME: config.get('webserver.https') === true ? 'https' : 'http', @@ -80,6 +81,11 @@ const CONFIG = { TRANSCODING: { ENABLED: config.get('transcoding.enabled'), THREADS: config.get('transcoding.threads') + }, + CACHE: { + PREVIEWS: { + SIZE: config.get('cache.previews.size') + } } } CONFIG.WEBSERVER.URL = CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT @@ -278,6 +284,13 @@ let STATIC_MAX_AGE = '30d' const THUMBNAILS_SIZE = '200x110' const PREVIEWS_SIZE = '640x480' +// Subfolders of cache directory +const CACHE = { + DIRECTORIES: { + PREVIEWS: join(CONFIG.STORAGE.CACHE_DIR, 'previews') + } +} + // --------------------------------------------------------------------------- const USER_ROLES: { [ id: string ]: UserRole } = { @@ -307,6 +320,7 @@ if (isTestInstance() === true) { export { API_VERSION, BCRYPT_SALT_SIZE, + CACHE, CONFIG, CONSTRAINTS_FIELDS, FRIEND_SCORE, diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index 1ec24c4ad..3c5a77df9 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts @@ -4,12 +4,13 @@ import * as passwordGenerator from 'password-generator' import * as Promise from 'bluebird' import { database as db } from './database' -import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION } from './constants' +import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION, CACHE } from './constants' import { clientsExist, usersExist } from './checker' -import { logger, createCertsIfNotExist, root, mkdirpPromise } from '../helpers' +import { logger, createCertsIfNotExist, root, mkdirpPromise, rimrafPromise } from '../helpers' function installApplication () { return db.sequelize.sync() + .then(() => removeCacheDirectories()) .then(() => createDirectoriesIfNotExist()) .then(() => createCertsIfNotExist()) .then(() => createOAuthClientIfNotExist()) @@ -24,13 +25,34 @@ export { // --------------------------------------------------------------------------- +function removeCacheDirectories () { + const cacheDirectories = CACHE.DIRECTORIES + + const tasks = [] + + // Cache directories + Object.keys(cacheDirectories).forEach(key => { + const dir = cacheDirectories[key] + tasks.push(rimrafPromise(dir)) + }) + + return Promise.all(tasks) +} + function createDirectoriesIfNotExist () { - const storages = config.get('storage') + const storages = CONFIG.STORAGE + const cacheDirectories = CACHE.DIRECTORIES const tasks = [] Object.keys(storages).forEach(key => { const dir = storages[key] - tasks.push(mkdirpPromise(join(root(), dir))) + tasks.push(mkdirpPromise(dir)) + }) + + // Cache directories + Object.keys(cacheDirectories).forEach(key => { + const dir = cacheDirectories[key] + tasks.push(mkdirpPromise(dir)) }) return Promise.all(tasks) diff --git a/server/lib/cache/index.ts b/server/lib/cache/index.ts new file mode 100644 index 000000000..7bf63790a --- /dev/null +++ b/server/lib/cache/index.ts @@ -0,0 +1 @@ +export * from './videos-preview-cache' diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/cache/videos-preview-cache.ts new file mode 100644 index 000000000..9d365e496 --- /dev/null +++ b/server/lib/cache/videos-preview-cache.ts @@ -0,0 +1,74 @@ +import * as request from 'request' +import * as asyncLRU from 'async-lru' +import { join } from 'path' +import { createWriteStream } from 'fs' +import * as Promise from 'bluebird' + +import { database as db, CONFIG, CACHE } from '../../initializers' +import { logger, writeFilePromise, unlinkPromise } from '../../helpers' +import { VideoInstance } from '../../models' +import { fetchRemotePreview } from '../../lib' + +class VideosPreviewCache { + + private static instance: VideosPreviewCache + + private lru + + private constructor () { } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + init (max: number) { + this.lru = new asyncLRU({ + max, + load: (key, cb) => { + this.loadPreviews(key) + .then(res => cb(null, res)) + .catch(err => cb(err)) + } + }) + + this.lru.on('evict', (obj: { key: string, value: string }) => { + unlinkPromise(obj.value).then(() => logger.debug('%s evicted from VideosPreviewCache', obj.value)) + }) + } + + getPreviewPath (key: string) { + return new Promise((res, rej) => { + this.lru.get(key, (err, value) => { + err ? rej(err) : res(value) + }) + }) + } + + private loadPreviews (key: string) { + return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(key) + .then(video => { + if (!video) return undefined + + if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()) + + return this.saveRemotePreviewAndReturnPath(video) + }) + } + + private saveRemotePreviewAndReturnPath (video: VideoInstance) { + const req = fetchRemotePreview(video.Author.Pod, video) + + return new Promise((res, rej) => { + const path = join(CACHE.DIRECTORIES.PREVIEWS, video.getPreviewName()) + const stream = createWriteStream(path) + + req.pipe(stream) + .on('finish', () => res(path)) + .on('error', (err) => rej(err)) + }) + } +} + +export { + VideosPreviewCache +} diff --git a/server/lib/friends.ts b/server/lib/friends.ts index 6ed0da013..50355d5d1 100644 --- a/server/lib/friends.ts +++ b/server/lib/friends.ts @@ -1,6 +1,7 @@ import * as request from 'request' import * as Sequelize from 'sequelize' import * as Promise from 'bluebird' +import { join } from 'path' import { database as db } from '../initializers/database' import { @@ -9,7 +10,8 @@ import { REQUESTS_IN_PARALLEL, REQUEST_ENDPOINTS, REQUEST_ENDPOINT_ACTIONS, - REMOTE_SCHEME + REMOTE_SCHEME, + STATIC_PATHS } from '../initializers' import { logger, @@ -233,6 +235,13 @@ function sendOwnedVideosToPod (podId: number) { }) } +function fetchRemotePreview (pod: PodInstance, video: VideoInstance) { + const host = video.Author.Pod.host + const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName()) + + return request.get(REMOTE_SCHEME.HTTP + '://' + host + path) +} + function getRequestScheduler () { return requestScheduler } @@ -263,7 +272,8 @@ export { sendOwnedVideosToPod, getRequestScheduler, getRequestVideoQaduScheduler, - getRequestVideoEventScheduler + getRequestVideoEventScheduler, + fetchRemotePreview } // --------------------------------------------------------------------------- diff --git a/server/lib/index.ts b/server/lib/index.ts index b8697fb96..8628da4dd 100644 --- a/server/lib/index.ts +++ b/server/lib/index.ts @@ -1,3 +1,4 @@ +export * from './cache' export * from './jobs' export * from './request' export * from './friends' diff --git a/server/models/oauth/oauth-token-interface.ts b/server/models/oauth/oauth-token-interface.ts index f2ddafa54..97af3c815 100644 --- a/server/models/oauth/oauth-token-interface.ts +++ b/server/models/oauth/oauth-token-interface.ts @@ -35,6 +35,8 @@ export interface OAuthTokenAttributes { refreshToken: string refreshTokenExpiresAt: Date + userId?: number + oAuthClientId?: number User?: UserModel } diff --git a/server/models/oauth/oauth-token.ts b/server/models/oauth/oauth-token.ts index 5c3781394..e3de9468e 100644 --- a/server/models/oauth/oauth-token.ts +++ b/server/models/oauth/oauth-token.ts @@ -106,10 +106,10 @@ getByRefreshTokenAndPopulateClient = function (refreshToken: string) { refreshToken: token.refreshToken, refreshTokenExpiresAt: token.refreshTokenExpiresAt, client: { - id: token['client'].id + id: token.oAuthClientId }, user: { - id: token['user'] + id: token.userId } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 650025205..b7eb24c4a 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -451,6 +451,7 @@ toFormatedJSON = function (this: VideoInstance) { dislikes: this.dislikes, tags: map(this.Tags, 'name'), thumbnailPath: join(STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), + previewPath: join(STATIC_PATHS.PREVIEWS, this.getPreviewName()), createdAt: this.createdAt, updatedAt: this.updatedAt } diff --git a/server/tests/api/fixtures/video_short1-preview.webm.jpg b/server/tests/api/fixtures/video_short1-preview.webm.jpg new file mode 100644 index 000000000..69c100c4e Binary files /dev/null and b/server/tests/api/fixtures/video_short1-preview.webm.jpg differ diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index 1bc6157e8..7753e6f2d 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -747,7 +747,7 @@ describe('Test multiple pods', function () { expect(videos[0].name).not.to.equal(toRemove[1].name) expect(videos[1].name).not.to.equal(toRemove[1].name) - videoUUID = videos[0].uuid + videoUUID = videos.find(video => video.name === 'my super name for pod 1').uuid callback() }) @@ -781,6 +781,23 @@ describe('Test multiple pods', function () { }) }, done) }) + + it('Should get the preview from each pod', function (done) { + each(servers, function (server, callback) { + videosUtils.getVideo(server.url, videoUUID, function (err, res) { + if (err) throw err + + const video = res.body + + videosUtils.testVideoImage(server.url, 'video_short1-preview.webm', video.previewPath, function (err, test) { + if (err) throw err + expect(test).to.equal(true) + + callback() + }) + }) + }, done) + }) }) after(function (done) { diff --git a/server/tests/utils/videos.js b/server/tests/utils/videos.js index 6e7aabc5d..cb3be6897 100644 --- a/server/tests/utils/videos.js +++ b/server/tests/utils/videos.js @@ -195,7 +195,7 @@ function searchVideoWithSort (url, search, sort, end) { .end(end) } -function testVideoImage (url, videoName, imagePath, callback) { +function testVideoImage (url, imageName, imagePath, callback) { // Don't test images if the node env is not set // Because we need a special ffmpeg version for this test if (process.env.NODE_TEST_IMAGE) { @@ -205,7 +205,7 @@ function testVideoImage (url, videoName, imagePath, callback) { .end(function (err, res) { if (err) return callback(err) - fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', videoName + '.jpg'), function (err, data) { + fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', imageName + '.jpg'), function (err, data) { if (err) return callback(err) callback(null, data.equals(res.body)) -- cgit v1.2.3