From f981dae8617271a2dc713bb683951730b306e0c5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 12 Jul 2017 11:56:02 +0200 Subject: Add previews cache system between pods --- .gitignore | 1 + config/default.yaml | 5 ++ config/production.yaml.example | 1 + config/test-1.yaml | 2 + config/test-2.yaml | 2 + config/test-3.yaml | 2 + config/test-4.yaml | 2 + config/test-5.yaml | 2 + config/test-6.yaml | 2 + package.json | 1 + server.ts | 4 +- server/controllers/static.ts | 16 ++++- server/helpers/core-utils.ts | 5 +- server/initializers/constants.ts | 16 ++++- server/initializers/installer.ts | 30 +++++++-- server/lib/cache/index.ts | 1 + server/lib/cache/videos-preview-cache.ts | 74 +++++++++++++++++++++ server/lib/friends.ts | 14 +++- server/lib/index.ts | 1 + server/models/oauth/oauth-token-interface.ts | 2 + server/models/oauth/oauth-token.ts | 4 +- server/models/video/video.ts | 1 + .../api/fixtures/video_short1-preview.webm.jpg | Bin 0 -> 31725 bytes server/tests/api/multiple-pods.js | 19 +++++- server/tests/utils/videos.js | 4 +- shared/models/videos/video.model.ts | 1 + yarn.lock | 6 ++ 27 files changed, 202 insertions(+), 16 deletions(-) create mode 100644 server/lib/cache/index.ts create mode 100644 server/lib/cache/videos-preview-cache.ts create mode 100644 server/tests/api/fixtures/video_short1-preview.webm.jpg diff --git a/.gitignore b/.gitignore index 6caee2e4c..169027c36 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ /certs/ /logs/ /torrents/ +/cache/ /config/production.yaml /ffmpeg/ /*.sublime-project diff --git a/config/default.yaml b/config/default.yaml index e03bf1aea..b4e7606cf 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -22,6 +22,11 @@ storage: previews: 'previews/' thumbnails: 'thumbnails/' torrents: 'torrents/' + cache: 'cache/' + +cache: + previews: + size: 1 # Max number of previews you want to cache admin: email: 'admin@example.com' diff --git a/config/production.yaml.example b/config/production.yaml.example index c18457df6..0857aa3ca 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -23,6 +23,7 @@ storage: previews: 'previews/' thumbnails: 'thumbnails/' torrents: 'torrents/' + cache: 'cache/' admin: email: 'admin@example.com' diff --git a/config/test-1.yaml b/config/test-1.yaml index dbe408a8c..e244a8797 100644 --- a/config/test-1.yaml +++ b/config/test-1.yaml @@ -13,8 +13,10 @@ storage: certs: 'test1/certs/' videos: 'test1/videos/' logs: 'test1/logs/' + previews: 'test1/previews/' thumbnails: 'test1/thumbnails/' torrents: 'test1/torrents/' + cache: 'test1/cache/' admin: email: 'admin1@example.com' diff --git a/config/test-2.yaml b/config/test-2.yaml index c95b9c229..236dcb10d 100644 --- a/config/test-2.yaml +++ b/config/test-2.yaml @@ -13,8 +13,10 @@ storage: certs: 'test2/certs/' videos: 'test2/videos/' logs: 'test2/logs/' + previews: 'test2/previews/' thumbnails: 'test2/thumbnails/' torrents: 'test2/torrents/' + cache: 'test2/cache/' admin: email: 'admin2@example.com' diff --git a/config/test-3.yaml b/config/test-3.yaml index 2eb984692..a29225a44 100644 --- a/config/test-3.yaml +++ b/config/test-3.yaml @@ -13,8 +13,10 @@ storage: certs: 'test3/certs/' videos: 'test3/videos/' logs: 'test3/logs/' + previews: 'test3/previews/' thumbnails: 'test3/thumbnails/' torrents: 'test3/torrents/' + cache: 'test3/cache/' admin: email: 'admin3@example.com' diff --git a/config/test-4.yaml b/config/test-4.yaml index a0a9bde21..da93e128d 100644 --- a/config/test-4.yaml +++ b/config/test-4.yaml @@ -13,8 +13,10 @@ storage: certs: 'test4/certs/' videos: 'test4/videos/' logs: 'test4/logs/' + previews: 'test4/previews/' thumbnails: 'test4/thumbnails/' torrents: 'test4/torrents/' + cache: 'test4/cache/' admin: email: 'admin4@example.com' diff --git a/config/test-5.yaml b/config/test-5.yaml index af8654e14..f95e25eb8 100644 --- a/config/test-5.yaml +++ b/config/test-5.yaml @@ -13,8 +13,10 @@ storage: certs: 'test5/certs/' videos: 'test5/videos/' logs: 'test5/logs/' + previews: 'test5/previews/' thumbnails: 'test5/thumbnails/' torrents: 'test5/torrents/' + cache: 'test5/cache/' admin: email: 'admin5@example.com' diff --git a/config/test-6.yaml b/config/test-6.yaml index d74d3b052..87d054439 100644 --- a/config/test-6.yaml +++ b/config/test-6.yaml @@ -13,8 +13,10 @@ storage: certs: 'test6/certs/' videos: 'test6/videos/' logs: 'test6/logs/' + previews: 'test6/previews/' thumbnails: 'test6/thumbnails/' torrents: 'test6/torrents/' + cache: 'test6/cache/' admin: email: 'admin6@example.com' diff --git a/package.json b/package.json index b875f5c26..d6da61975 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ }, "dependencies": { "async": "^2.0.0", + "async-lru": "^1.1.1", "bcrypt": "^1.0.2", "bittorrent-tracker": "^9.0.0", "bluebird": "^3.5.0", diff --git a/server.ts b/server.ts index e7fa99c90..a6a9fcb02 100644 --- a/server.ts +++ b/server.ts @@ -47,7 +47,7 @@ if (errorMessage !== null) { // ----------- PeerTube modules ----------- import { migrate, installApplication } from './server/initializers' -import { JobScheduler, activateSchedulers } from './server/lib' +import { JobScheduler, activateSchedulers, VideosPreviewCache } from './server/lib' import * as customValidators from './server/helpers/custom-validators' import { apiRouter, clientsRouter, staticRouter } from './server/controllers' @@ -147,6 +147,8 @@ function onDatabaseInitDone () { // Activate job scheduler JobScheduler.Instance.activate() + VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE) + logger.info('Server listening on port %d', port) logger.info('Webserver: %s', CONFIG.WEBSERVER.URL) }) diff --git a/server/controllers/static.ts b/server/controllers/static.ts index e65282339..2fd14131e 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -6,6 +6,7 @@ import { STATIC_MAX_AGE, STATIC_PATHS } from '../initializers' +import { VideosPreviewCache } from '../lib' const staticRouter = express.Router() @@ -38,8 +39,8 @@ staticRouter.use( // Video previews path for express const previewsPhysicalPath = CONFIG.STORAGE.PREVIEWS_DIR staticRouter.use( - STATIC_PATHS.PREVIEWS, - express.static(previewsPhysicalPath, { maxAge: STATIC_MAX_AGE }) + STATIC_PATHS.PREVIEWS + ':uuid.jpg', + getPreview ) // --------------------------------------------------------------------------- @@ -47,3 +48,14 @@ staticRouter.use( export { staticRouter } + +// --------------------------------------------------------------------------- + +function getPreview (req: express.Request, res: express.Response, next: express.NextFunction) { + VideosPreviewCache.Instance.getPreviewPath(req.params.uuid) + .then(path => { + if (!path) return res.sendStatus(404) + + return res.sendFile(path, { maxAge: STATIC_MAX_AGE }) + }) +} diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 1e92049f1..d28c97f09 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -16,6 +16,7 @@ import { import * as mkdirp from 'mkdirp' import * as bcrypt from 'bcrypt' import * as createTorrent from 'create-torrent' +import * as rimraf from 'rimraf' import * as openssl from 'openssl-wrapper' import * as Promise from 'bluebird' @@ -83,6 +84,7 @@ const bcryptComparePromise = promisify2(bcrypt.compare) const bcryptGenSaltPromise = promisify1(bcrypt.genSalt) const bcryptHashPromise = promisify2(bcrypt.hash) const createTorrentPromise = promisify2(createTorrent) +const rimrafPromise = promisify1WithVoid(rimraf) // --------------------------------------------------------------------------- @@ -105,5 +107,6 @@ export { bcryptComparePromise, bcryptGenSaltPromise, bcryptHashPromise, - createTorrentPromise + createTorrentPromise, + rimrafPromise } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index f087b7476..928a3f570 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -61,7 +61,8 @@ const CONFIG = { VIDEOS_DIR: join(root(), config.get('storage.videos')), THUMBNAILS_DIR: join(root(), config.get('storage.thumbnails')), PREVIEWS_DIR: join(root(), config.get('storage.previews')), - TORRENTS_DIR: join(root(), config.get('storage.torrents')) + TORRENTS_DIR: join(root(), config.get('storage.torrents')), + CACHE_DIR: join(root(), config.get('storage.cache')) }, WEBSERVER: { SCHEME: config.get('webserver.https') === true ? 'https' : 'http', @@ -80,6 +81,11 @@ const CONFIG = { TRANSCODING: { ENABLED: config.get('transcoding.enabled'), THREADS: config.get('transcoding.threads') + }, + CACHE: { + PREVIEWS: { + SIZE: config.get('cache.previews.size') + } } } CONFIG.WEBSERVER.URL = CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT @@ -278,6 +284,13 @@ let STATIC_MAX_AGE = '30d' const THUMBNAILS_SIZE = '200x110' const PREVIEWS_SIZE = '640x480' +// Subfolders of cache directory +const CACHE = { + DIRECTORIES: { + PREVIEWS: join(CONFIG.STORAGE.CACHE_DIR, 'previews') + } +} + // --------------------------------------------------------------------------- const USER_ROLES: { [ id: string ]: UserRole } = { @@ -307,6 +320,7 @@ if (isTestInstance() === true) { export { API_VERSION, BCRYPT_SALT_SIZE, + CACHE, CONFIG, CONSTRAINTS_FIELDS, FRIEND_SCORE, diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index 1ec24c4ad..3c5a77df9 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts @@ -4,12 +4,13 @@ import * as passwordGenerator from 'password-generator' import * as Promise from 'bluebird' import { database as db } from './database' -import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION } from './constants' +import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION, CACHE } from './constants' import { clientsExist, usersExist } from './checker' -import { logger, createCertsIfNotExist, root, mkdirpPromise } from '../helpers' +import { logger, createCertsIfNotExist, root, mkdirpPromise, rimrafPromise } from '../helpers' function installApplication () { return db.sequelize.sync() + .then(() => removeCacheDirectories()) .then(() => createDirectoriesIfNotExist()) .then(() => createCertsIfNotExist()) .then(() => createOAuthClientIfNotExist()) @@ -24,13 +25,34 @@ export { // --------------------------------------------------------------------------- +function removeCacheDirectories () { + const cacheDirectories = CACHE.DIRECTORIES + + const tasks = [] + + // Cache directories + Object.keys(cacheDirectories).forEach(key => { + const dir = cacheDirectories[key] + tasks.push(rimrafPromise(dir)) + }) + + return Promise.all(tasks) +} + function createDirectoriesIfNotExist () { - const storages = config.get('storage') + const storages = CONFIG.STORAGE + const cacheDirectories = CACHE.DIRECTORIES const tasks = [] Object.keys(storages).forEach(key => { const dir = storages[key] - tasks.push(mkdirpPromise(join(root(), dir))) + tasks.push(mkdirpPromise(dir)) + }) + + // Cache directories + Object.keys(cacheDirectories).forEach(key => { + const dir = cacheDirectories[key] + tasks.push(mkdirpPromise(dir)) }) return Promise.all(tasks) diff --git a/server/lib/cache/index.ts b/server/lib/cache/index.ts new file mode 100644 index 000000000..7bf63790a --- /dev/null +++ b/server/lib/cache/index.ts @@ -0,0 +1 @@ +export * from './videos-preview-cache' diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/cache/videos-preview-cache.ts new file mode 100644 index 000000000..9d365e496 --- /dev/null +++ b/server/lib/cache/videos-preview-cache.ts @@ -0,0 +1,74 @@ +import * as request from 'request' +import * as asyncLRU from 'async-lru' +import { join } from 'path' +import { createWriteStream } from 'fs' +import * as Promise from 'bluebird' + +import { database as db, CONFIG, CACHE } from '../../initializers' +import { logger, writeFilePromise, unlinkPromise } from '../../helpers' +import { VideoInstance } from '../../models' +import { fetchRemotePreview } from '../../lib' + +class VideosPreviewCache { + + private static instance: VideosPreviewCache + + private lru + + private constructor () { } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + init (max: number) { + this.lru = new asyncLRU({ + max, + load: (key, cb) => { + this.loadPreviews(key) + .then(res => cb(null, res)) + .catch(err => cb(err)) + } + }) + + this.lru.on('evict', (obj: { key: string, value: string }) => { + unlinkPromise(obj.value).then(() => logger.debug('%s evicted from VideosPreviewCache', obj.value)) + }) + } + + getPreviewPath (key: string) { + return new Promise((res, rej) => { + this.lru.get(key, (err, value) => { + err ? rej(err) : res(value) + }) + }) + } + + private loadPreviews (key: string) { + return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(key) + .then(video => { + if (!video) return undefined + + if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()) + + return this.saveRemotePreviewAndReturnPath(video) + }) + } + + private saveRemotePreviewAndReturnPath (video: VideoInstance) { + const req = fetchRemotePreview(video.Author.Pod, video) + + return new Promise((res, rej) => { + const path = join(CACHE.DIRECTORIES.PREVIEWS, video.getPreviewName()) + const stream = createWriteStream(path) + + req.pipe(stream) + .on('finish', () => res(path)) + .on('error', (err) => rej(err)) + }) + } +} + +export { + VideosPreviewCache +} diff --git a/server/lib/friends.ts b/server/lib/friends.ts index 6ed0da013..50355d5d1 100644 --- a/server/lib/friends.ts +++ b/server/lib/friends.ts @@ -1,6 +1,7 @@ import * as request from 'request' import * as Sequelize from 'sequelize' import * as Promise from 'bluebird' +import { join } from 'path' import { database as db } from '../initializers/database' import { @@ -9,7 +10,8 @@ import { REQUESTS_IN_PARALLEL, REQUEST_ENDPOINTS, REQUEST_ENDPOINT_ACTIONS, - REMOTE_SCHEME + REMOTE_SCHEME, + STATIC_PATHS } from '../initializers' import { logger, @@ -233,6 +235,13 @@ function sendOwnedVideosToPod (podId: number) { }) } +function fetchRemotePreview (pod: PodInstance, video: VideoInstance) { + const host = video.Author.Pod.host + const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName()) + + return request.get(REMOTE_SCHEME.HTTP + '://' + host + path) +} + function getRequestScheduler () { return requestScheduler } @@ -263,7 +272,8 @@ export { sendOwnedVideosToPod, getRequestScheduler, getRequestVideoQaduScheduler, - getRequestVideoEventScheduler + getRequestVideoEventScheduler, + fetchRemotePreview } // --------------------------------------------------------------------------- diff --git a/server/lib/index.ts b/server/lib/index.ts index b8697fb96..8628da4dd 100644 --- a/server/lib/index.ts +++ b/server/lib/index.ts @@ -1,3 +1,4 @@ +export * from './cache' export * from './jobs' export * from './request' export * from './friends' diff --git a/server/models/oauth/oauth-token-interface.ts b/server/models/oauth/oauth-token-interface.ts index f2ddafa54..97af3c815 100644 --- a/server/models/oauth/oauth-token-interface.ts +++ b/server/models/oauth/oauth-token-interface.ts @@ -35,6 +35,8 @@ export interface OAuthTokenAttributes { refreshToken: string refreshTokenExpiresAt: Date + userId?: number + oAuthClientId?: number User?: UserModel } diff --git a/server/models/oauth/oauth-token.ts b/server/models/oauth/oauth-token.ts index 5c3781394..e3de9468e 100644 --- a/server/models/oauth/oauth-token.ts +++ b/server/models/oauth/oauth-token.ts @@ -106,10 +106,10 @@ getByRefreshTokenAndPopulateClient = function (refreshToken: string) { refreshToken: token.refreshToken, refreshTokenExpiresAt: token.refreshTokenExpiresAt, client: { - id: token['client'].id + id: token.oAuthClientId }, user: { - id: token['user'] + id: token.userId } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 650025205..b7eb24c4a 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -451,6 +451,7 @@ toFormatedJSON = function (this: VideoInstance) { dislikes: this.dislikes, tags: map(this.Tags, 'name'), thumbnailPath: join(STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), + previewPath: join(STATIC_PATHS.PREVIEWS, this.getPreviewName()), createdAt: this.createdAt, updatedAt: this.updatedAt } diff --git a/server/tests/api/fixtures/video_short1-preview.webm.jpg b/server/tests/api/fixtures/video_short1-preview.webm.jpg new file mode 100644 index 000000000..69c100c4e Binary files /dev/null and b/server/tests/api/fixtures/video_short1-preview.webm.jpg differ diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index 1bc6157e8..7753e6f2d 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -747,7 +747,7 @@ describe('Test multiple pods', function () { expect(videos[0].name).not.to.equal(toRemove[1].name) expect(videos[1].name).not.to.equal(toRemove[1].name) - videoUUID = videos[0].uuid + videoUUID = videos.find(video => video.name === 'my super name for pod 1').uuid callback() }) @@ -781,6 +781,23 @@ describe('Test multiple pods', function () { }) }, done) }) + + it('Should get the preview from each pod', function (done) { + each(servers, function (server, callback) { + videosUtils.getVideo(server.url, videoUUID, function (err, res) { + if (err) throw err + + const video = res.body + + videosUtils.testVideoImage(server.url, 'video_short1-preview.webm', video.previewPath, function (err, test) { + if (err) throw err + expect(test).to.equal(true) + + callback() + }) + }) + }, done) + }) }) after(function (done) { diff --git a/server/tests/utils/videos.js b/server/tests/utils/videos.js index 6e7aabc5d..cb3be6897 100644 --- a/server/tests/utils/videos.js +++ b/server/tests/utils/videos.js @@ -195,7 +195,7 @@ function searchVideoWithSort (url, search, sort, end) { .end(end) } -function testVideoImage (url, videoName, imagePath, callback) { +function testVideoImage (url, imageName, imagePath, callback) { // Don't test images if the node env is not set // Because we need a special ffmpeg version for this test if (process.env.NODE_TEST_IMAGE) { @@ -205,7 +205,7 @@ function testVideoImage (url, videoName, imagePath, callback) { .end(function (err, res) { if (err) return callback(err) - fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', videoName + '.jpg'), function (err, data) { + fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', imageName + '.jpg'), function (err, data) { if (err) return callback(err) callback(null, data.equals(res.body)) diff --git a/shared/models/videos/video.model.ts b/shared/models/videos/video.model.ts index d472cc8fb..8aa8ee448 100644 --- a/shared/models/videos/video.model.ts +++ b/shared/models/videos/video.model.ts @@ -17,6 +17,7 @@ export interface Video { podHost: string tags: string[] thumbnailPath: string + previewPath: string views: number likes: number dislikes: number diff --git a/yarn.lock b/yarn.lock index 5636db494..68187f684 100644 --- a/yarn.lock +++ b/yarn.lock @@ -285,6 +285,12 @@ async-each@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d" +async-lru@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/async-lru/-/async-lru-1.1.1.tgz#3edbf7e96484d5c2dd852a8bf9794fc07f5e7274" + dependencies: + lru "^3.1.0" + async@>=0.2.9, async@^2.0.0: version "2.4.1" resolved "https://registry.yarnpkg.com/async/-/async-2.4.1.tgz#62a56b279c98a11d0987096a01cc3eeb8eb7bbd7" -- cgit v1.2.3