aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <florian.bigard@gmail.com>2017-07-12 11:56:02 +0200
committerChocobozzz <florian.bigard@gmail.com>2017-07-12 11:56:02 +0200
commitf981dae8617271a2dc713bb683951730b306e0c5 (patch)
treeecee631766bc1b98c20a7836479fed40850c5a56 /server
parent075f16caac5236cb04c98ae7b3a989766d764bb3 (diff)
downloadPeerTube-f981dae8617271a2dc713bb683951730b306e0c5.tar.gz
PeerTube-f981dae8617271a2dc713bb683951730b306e0c5.tar.zst
PeerTube-f981dae8617271a2dc713bb683951730b306e0c5.zip
Add previews cache system between pods
Diffstat (limited to 'server')
-rw-r--r--server/controllers/static.ts16
-rw-r--r--server/helpers/core-utils.ts5
-rw-r--r--server/initializers/constants.ts16
-rw-r--r--server/initializers/installer.ts30
-rw-r--r--server/lib/cache/index.ts1
-rw-r--r--server/lib/cache/videos-preview-cache.ts74
-rw-r--r--server/lib/friends.ts14
-rw-r--r--server/lib/index.ts1
-rw-r--r--server/models/oauth/oauth-token-interface.ts2
-rw-r--r--server/models/oauth/oauth-token.ts4
-rw-r--r--server/models/video/video.ts1
-rw-r--r--server/tests/api/fixtures/video_short1-preview.webm.jpgbin0 -> 31725 bytes
-rw-r--r--server/tests/api/multiple-pods.js19
-rw-r--r--server/tests/utils/videos.js4
14 files changed, 172 insertions, 15 deletions
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index e65282339..2fd14131e 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -6,6 +6,7 @@ import {
6 STATIC_MAX_AGE, 6 STATIC_MAX_AGE,
7 STATIC_PATHS 7 STATIC_PATHS
8} from '../initializers' 8} from '../initializers'
9import { VideosPreviewCache } from '../lib'
9 10
10const staticRouter = express.Router() 11const staticRouter = express.Router()
11 12
@@ -38,8 +39,8 @@ staticRouter.use(
38// Video previews path for express 39// Video previews path for express
39const previewsPhysicalPath = CONFIG.STORAGE.PREVIEWS_DIR 40const previewsPhysicalPath = CONFIG.STORAGE.PREVIEWS_DIR
40staticRouter.use( 41staticRouter.use(
41 STATIC_PATHS.PREVIEWS, 42 STATIC_PATHS.PREVIEWS + ':uuid.jpg',
42 express.static(previewsPhysicalPath, { maxAge: STATIC_MAX_AGE }) 43 getPreview
43) 44)
44 45
45// --------------------------------------------------------------------------- 46// ---------------------------------------------------------------------------
@@ -47,3 +48,14 @@ staticRouter.use(
47export { 48export {
48 staticRouter 49 staticRouter
49} 50}
51
52// ---------------------------------------------------------------------------
53
54function getPreview (req: express.Request, res: express.Response, next: express.NextFunction) {
55 VideosPreviewCache.Instance.getPreviewPath(req.params.uuid)
56 .then(path => {
57 if (!path) return res.sendStatus(404)
58
59 return res.sendFile(path, { maxAge: STATIC_MAX_AGE })
60 })
61}
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 1e92049f1..d28c97f09 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -16,6 +16,7 @@ import {
16import * as mkdirp from 'mkdirp' 16import * as mkdirp from 'mkdirp'
17import * as bcrypt from 'bcrypt' 17import * as bcrypt from 'bcrypt'
18import * as createTorrent from 'create-torrent' 18import * as createTorrent from 'create-torrent'
19import * as rimraf from 'rimraf'
19import * as openssl from 'openssl-wrapper' 20import * as openssl from 'openssl-wrapper'
20import * as Promise from 'bluebird' 21import * as Promise from 'bluebird'
21 22
@@ -83,6 +84,7 @@ const bcryptComparePromise = promisify2<any, string, boolean>(bcrypt.compare)
83const bcryptGenSaltPromise = promisify1<number, string>(bcrypt.genSalt) 84const bcryptGenSaltPromise = promisify1<number, string>(bcrypt.genSalt)
84const bcryptHashPromise = promisify2<any, string|number, string>(bcrypt.hash) 85const bcryptHashPromise = promisify2<any, string|number, string>(bcrypt.hash)
85const createTorrentPromise = promisify2<string, any, any>(createTorrent) 86const createTorrentPromise = promisify2<string, any, any>(createTorrent)
87const rimrafPromise = promisify1WithVoid<string>(rimraf)
86 88
87// --------------------------------------------------------------------------- 89// ---------------------------------------------------------------------------
88 90
@@ -105,5 +107,6 @@ export {
105 bcryptComparePromise, 107 bcryptComparePromise,
106 bcryptGenSaltPromise, 108 bcryptGenSaltPromise,
107 bcryptHashPromise, 109 bcryptHashPromise,
108 createTorrentPromise 110 createTorrentPromise,
111 rimrafPromise
109} 112}
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index f087b7476..928a3f570 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -61,7 +61,8 @@ const CONFIG = {
61 VIDEOS_DIR: join(root(), config.get<string>('storage.videos')), 61 VIDEOS_DIR: join(root(), config.get<string>('storage.videos')),
62 THUMBNAILS_DIR: join(root(), config.get<string>('storage.thumbnails')), 62 THUMBNAILS_DIR: join(root(), config.get<string>('storage.thumbnails')),
63 PREVIEWS_DIR: join(root(), config.get<string>('storage.previews')), 63 PREVIEWS_DIR: join(root(), config.get<string>('storage.previews')),
64 TORRENTS_DIR: join(root(), config.get<string>('storage.torrents')) 64 TORRENTS_DIR: join(root(), config.get<string>('storage.torrents')),
65 CACHE_DIR: join(root(), config.get<string>('storage.cache'))
65 }, 66 },
66 WEBSERVER: { 67 WEBSERVER: {
67 SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http', 68 SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http',
@@ -80,6 +81,11 @@ const CONFIG = {
80 TRANSCODING: { 81 TRANSCODING: {
81 ENABLED: config.get<boolean>('transcoding.enabled'), 82 ENABLED: config.get<boolean>('transcoding.enabled'),
82 THREADS: config.get<number>('transcoding.threads') 83 THREADS: config.get<number>('transcoding.threads')
84 },
85 CACHE: {
86 PREVIEWS: {
87 SIZE: config.get<number>('cache.previews.size')
88 }
83 } 89 }
84} 90}
85CONFIG.WEBSERVER.URL = CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT 91CONFIG.WEBSERVER.URL = CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
@@ -278,6 +284,13 @@ let STATIC_MAX_AGE = '30d'
278const THUMBNAILS_SIZE = '200x110' 284const THUMBNAILS_SIZE = '200x110'
279const PREVIEWS_SIZE = '640x480' 285const PREVIEWS_SIZE = '640x480'
280 286
287// Subfolders of cache directory
288const CACHE = {
289 DIRECTORIES: {
290 PREVIEWS: join(CONFIG.STORAGE.CACHE_DIR, 'previews')
291 }
292}
293
281// --------------------------------------------------------------------------- 294// ---------------------------------------------------------------------------
282 295
283const USER_ROLES: { [ id: string ]: UserRole } = { 296const USER_ROLES: { [ id: string ]: UserRole } = {
@@ -307,6 +320,7 @@ if (isTestInstance() === true) {
307export { 320export {
308 API_VERSION, 321 API_VERSION,
309 BCRYPT_SALT_SIZE, 322 BCRYPT_SALT_SIZE,
323 CACHE,
310 CONFIG, 324 CONFIG,
311 CONSTRAINTS_FIELDS, 325 CONSTRAINTS_FIELDS,
312 FRIEND_SCORE, 326 FRIEND_SCORE,
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index 1ec24c4ad..3c5a77df9 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -4,12 +4,13 @@ import * as passwordGenerator from 'password-generator'
4import * as Promise from 'bluebird' 4import * as Promise from 'bluebird'
5 5
6import { database as db } from './database' 6import { database as db } from './database'
7import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION } from './constants' 7import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION, CACHE } from './constants'
8import { clientsExist, usersExist } from './checker' 8import { clientsExist, usersExist } from './checker'
9import { logger, createCertsIfNotExist, root, mkdirpPromise } from '../helpers' 9import { logger, createCertsIfNotExist, root, mkdirpPromise, rimrafPromise } from '../helpers'
10 10
11function installApplication () { 11function installApplication () {
12 return db.sequelize.sync() 12 return db.sequelize.sync()
13 .then(() => removeCacheDirectories())
13 .then(() => createDirectoriesIfNotExist()) 14 .then(() => createDirectoriesIfNotExist())
14 .then(() => createCertsIfNotExist()) 15 .then(() => createCertsIfNotExist())
15 .then(() => createOAuthClientIfNotExist()) 16 .then(() => createOAuthClientIfNotExist())
@@ -24,13 +25,34 @@ export {
24 25
25// --------------------------------------------------------------------------- 26// ---------------------------------------------------------------------------
26 27
28function removeCacheDirectories () {
29 const cacheDirectories = CACHE.DIRECTORIES
30
31 const tasks = []
32
33 // Cache directories
34 Object.keys(cacheDirectories).forEach(key => {
35 const dir = cacheDirectories[key]
36 tasks.push(rimrafPromise(dir))
37 })
38
39 return Promise.all(tasks)
40}
41
27function createDirectoriesIfNotExist () { 42function createDirectoriesIfNotExist () {
28 const storages = config.get('storage') 43 const storages = CONFIG.STORAGE
44 const cacheDirectories = CACHE.DIRECTORIES
29 45
30 const tasks = [] 46 const tasks = []
31 Object.keys(storages).forEach(key => { 47 Object.keys(storages).forEach(key => {
32 const dir = storages[key] 48 const dir = storages[key]
33 tasks.push(mkdirpPromise(join(root(), dir))) 49 tasks.push(mkdirpPromise(dir))
50 })
51
52 // Cache directories
53 Object.keys(cacheDirectories).forEach(key => {
54 const dir = cacheDirectories[key]
55 tasks.push(mkdirpPromise(dir))
34 }) 56 })
35 57
36 return Promise.all(tasks) 58 return Promise.all(tasks)
diff --git a/server/lib/cache/index.ts b/server/lib/cache/index.ts
new file mode 100644
index 000000000..7bf63790a
--- /dev/null
+++ b/server/lib/cache/index.ts
@@ -0,0 +1 @@
export * from './videos-preview-cache'
diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/cache/videos-preview-cache.ts
new file mode 100644
index 000000000..9d365e496
--- /dev/null
+++ b/server/lib/cache/videos-preview-cache.ts
@@ -0,0 +1,74 @@
1import * as request from 'request'
2import * as asyncLRU from 'async-lru'
3import { join } from 'path'
4import { createWriteStream } from 'fs'
5import * as Promise from 'bluebird'
6
7import { database as db, CONFIG, CACHE } from '../../initializers'
8import { logger, writeFilePromise, unlinkPromise } from '../../helpers'
9import { VideoInstance } from '../../models'
10import { fetchRemotePreview } from '../../lib'
11
12class VideosPreviewCache {
13
14 private static instance: VideosPreviewCache
15
16 private lru
17
18 private constructor () { }
19
20 static get Instance () {
21 return this.instance || (this.instance = new this())
22 }
23
24 init (max: number) {
25 this.lru = new asyncLRU({
26 max,
27 load: (key, cb) => {
28 this.loadPreviews(key)
29 .then(res => cb(null, res))
30 .catch(err => cb(err))
31 }
32 })
33
34 this.lru.on('evict', (obj: { key: string, value: string }) => {
35 unlinkPromise(obj.value).then(() => logger.debug('%s evicted from VideosPreviewCache', obj.value))
36 })
37 }
38
39 getPreviewPath (key: string) {
40 return new Promise<string>((res, rej) => {
41 this.lru.get(key, (err, value) => {
42 err ? rej(err) : res(value)
43 })
44 })
45 }
46
47 private loadPreviews (key: string) {
48 return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(key)
49 .then(video => {
50 if (!video) return undefined
51
52 if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
53
54 return this.saveRemotePreviewAndReturnPath(video)
55 })
56 }
57
58 private saveRemotePreviewAndReturnPath (video: VideoInstance) {
59 const req = fetchRemotePreview(video.Author.Pod, video)
60
61 return new Promise<string>((res, rej) => {
62 const path = join(CACHE.DIRECTORIES.PREVIEWS, video.getPreviewName())
63 const stream = createWriteStream(path)
64
65 req.pipe(stream)
66 .on('finish', () => res(path))
67 .on('error', (err) => rej(err))
68 })
69 }
70}
71
72export {
73 VideosPreviewCache
74}
diff --git a/server/lib/friends.ts b/server/lib/friends.ts
index 6ed0da013..50355d5d1 100644
--- a/server/lib/friends.ts
+++ b/server/lib/friends.ts
@@ -1,6 +1,7 @@
1import * as request from 'request' 1import * as request from 'request'
2import * as Sequelize from 'sequelize' 2import * as Sequelize from 'sequelize'
3import * as Promise from 'bluebird' 3import * as Promise from 'bluebird'
4import { join } from 'path'
4 5
5import { database as db } from '../initializers/database' 6import { database as db } from '../initializers/database'
6import { 7import {
@@ -9,7 +10,8 @@ import {
9 REQUESTS_IN_PARALLEL, 10 REQUESTS_IN_PARALLEL,
10 REQUEST_ENDPOINTS, 11 REQUEST_ENDPOINTS,
11 REQUEST_ENDPOINT_ACTIONS, 12 REQUEST_ENDPOINT_ACTIONS,
12 REMOTE_SCHEME 13 REMOTE_SCHEME,
14 STATIC_PATHS
13} from '../initializers' 15} from '../initializers'
14import { 16import {
15 logger, 17 logger,
@@ -233,6 +235,13 @@ function sendOwnedVideosToPod (podId: number) {
233 }) 235 })
234} 236}
235 237
238function fetchRemotePreview (pod: PodInstance, video: VideoInstance) {
239 const host = video.Author.Pod.host
240 const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
241
242 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path)
243}
244
236function getRequestScheduler () { 245function getRequestScheduler () {
237 return requestScheduler 246 return requestScheduler
238} 247}
@@ -263,7 +272,8 @@ export {
263 sendOwnedVideosToPod, 272 sendOwnedVideosToPod,
264 getRequestScheduler, 273 getRequestScheduler,
265 getRequestVideoQaduScheduler, 274 getRequestVideoQaduScheduler,
266 getRequestVideoEventScheduler 275 getRequestVideoEventScheduler,
276 fetchRemotePreview
267} 277}
268 278
269// --------------------------------------------------------------------------- 279// ---------------------------------------------------------------------------
diff --git a/server/lib/index.ts b/server/lib/index.ts
index b8697fb96..8628da4dd 100644
--- a/server/lib/index.ts
+++ b/server/lib/index.ts
@@ -1,3 +1,4 @@
1export * from './cache'
1export * from './jobs' 2export * from './jobs'
2export * from './request' 3export * from './request'
3export * from './friends' 4export * from './friends'
diff --git a/server/models/oauth/oauth-token-interface.ts b/server/models/oauth/oauth-token-interface.ts
index f2ddafa54..97af3c815 100644
--- a/server/models/oauth/oauth-token-interface.ts
+++ b/server/models/oauth/oauth-token-interface.ts
@@ -35,6 +35,8 @@ export interface OAuthTokenAttributes {
35 refreshToken: string 35 refreshToken: string
36 refreshTokenExpiresAt: Date 36 refreshTokenExpiresAt: Date
37 37
38 userId?: number
39 oAuthClientId?: number
38 User?: UserModel 40 User?: UserModel
39} 41}
40 42
diff --git a/server/models/oauth/oauth-token.ts b/server/models/oauth/oauth-token.ts
index 5c3781394..e3de9468e 100644
--- a/server/models/oauth/oauth-token.ts
+++ b/server/models/oauth/oauth-token.ts
@@ -106,10 +106,10 @@ getByRefreshTokenAndPopulateClient = function (refreshToken: string) {
106 refreshToken: token.refreshToken, 106 refreshToken: token.refreshToken,
107 refreshTokenExpiresAt: token.refreshTokenExpiresAt, 107 refreshTokenExpiresAt: token.refreshTokenExpiresAt,
108 client: { 108 client: {
109 id: token['client'].id 109 id: token.oAuthClientId
110 }, 110 },
111 user: { 111 user: {
112 id: token['user'] 112 id: token.userId
113 } 113 }
114 } 114 }
115 115
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 650025205..b7eb24c4a 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -451,6 +451,7 @@ toFormatedJSON = function (this: VideoInstance) {
451 dislikes: this.dislikes, 451 dislikes: this.dislikes,
452 tags: map<TagInstance, string>(this.Tags, 'name'), 452 tags: map<TagInstance, string>(this.Tags, 'name'),
453 thumbnailPath: join(STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), 453 thumbnailPath: join(STATIC_PATHS.THUMBNAILS, this.getThumbnailName()),
454 previewPath: join(STATIC_PATHS.PREVIEWS, this.getPreviewName()),
454 createdAt: this.createdAt, 455 createdAt: this.createdAt,
455 updatedAt: this.updatedAt 456 updatedAt: this.updatedAt
456 } 457 }
diff --git a/server/tests/api/fixtures/video_short1-preview.webm.jpg b/server/tests/api/fixtures/video_short1-preview.webm.jpg
new file mode 100644
index 000000000..69c100c4e
--- /dev/null
+++ b/server/tests/api/fixtures/video_short1-preview.webm.jpg
Binary files differ
diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js
index 1bc6157e8..7753e6f2d 100644
--- a/server/tests/api/multiple-pods.js
+++ b/server/tests/api/multiple-pods.js
@@ -747,7 +747,7 @@ describe('Test multiple pods', function () {
747 expect(videos[0].name).not.to.equal(toRemove[1].name) 747 expect(videos[0].name).not.to.equal(toRemove[1].name)
748 expect(videos[1].name).not.to.equal(toRemove[1].name) 748 expect(videos[1].name).not.to.equal(toRemove[1].name)
749 749
750 videoUUID = videos[0].uuid 750 videoUUID = videos.find(video => video.name === 'my super name for pod 1').uuid
751 751
752 callback() 752 callback()
753 }) 753 })
@@ -781,6 +781,23 @@ describe('Test multiple pods', function () {
781 }) 781 })
782 }, done) 782 }, done)
783 }) 783 })
784
785 it('Should get the preview from each pod', function (done) {
786 each(servers, function (server, callback) {
787 videosUtils.getVideo(server.url, videoUUID, function (err, res) {
788 if (err) throw err
789
790 const video = res.body
791
792 videosUtils.testVideoImage(server.url, 'video_short1-preview.webm', video.previewPath, function (err, test) {
793 if (err) throw err
794 expect(test).to.equal(true)
795
796 callback()
797 })
798 })
799 }, done)
800 })
784 }) 801 })
785 802
786 after(function (done) { 803 after(function (done) {
diff --git a/server/tests/utils/videos.js b/server/tests/utils/videos.js
index 6e7aabc5d..cb3be6897 100644
--- a/server/tests/utils/videos.js
+++ b/server/tests/utils/videos.js
@@ -195,7 +195,7 @@ function searchVideoWithSort (url, search, sort, end) {
195 .end(end) 195 .end(end)
196} 196}
197 197
198function testVideoImage (url, videoName, imagePath, callback) { 198function testVideoImage (url, imageName, imagePath, callback) {
199 // Don't test images if the node env is not set 199 // Don't test images if the node env is not set
200 // Because we need a special ffmpeg version for this test 200 // Because we need a special ffmpeg version for this test
201 if (process.env.NODE_TEST_IMAGE) { 201 if (process.env.NODE_TEST_IMAGE) {
@@ -205,7 +205,7 @@ function testVideoImage (url, videoName, imagePath, callback) {
205 .end(function (err, res) { 205 .end(function (err, res) {
206 if (err) return callback(err) 206 if (err) return callback(err)
207 207
208 fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', videoName + '.jpg'), function (err, data) { 208 fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', imageName + '.jpg'), function (err, data) {
209 if (err) return callback(err) 209 if (err) return callback(err)
210 210
211 callback(null, data.equals(res.body)) 211 callback(null, data.equals(res.body))