Add previews cache system between pods

14 files changed, 172 insertions, 15 deletions

diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index e65282339..2fd14131e 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -6,6 +6,7 @@ import {
   STATIC_MAX_AGE,
   STATIC_PATHS
 } from '../initializers'
+import { VideosPreviewCache } from '../lib'
 
 const staticRouter = express.Router()
 
@@ -38,8 +39,8 @@ staticRouter.use(
 // Video previews path for express
 const previewsPhysicalPath = CONFIG.STORAGE.PREVIEWS_DIR
 staticRouter.use(
-  STATIC_PATHS.PREVIEWS,
-  express.static(previewsPhysicalPath, { maxAge: STATIC_MAX_AGE })
+  STATIC_PATHS.PREVIEWS + ':uuid.jpg',
+  getPreview
 )
 
 // ---------------------------------------------------------------------------
@@ -47,3 +48,14 @@ staticRouter.use(
 export {
   staticRouter
 }
+
+// ---------------------------------------------------------------------------
+
+function getPreview (req: express.Request, res: express.Response, next: express.NextFunction) {
+  VideosPreviewCache.Instance.getPreviewPath(req.params.uuid)
+    .then(path => {
+      if (!path) return res.sendStatus(404)
+
+      return res.sendFile(path, { maxAge: STATIC_MAX_AGE })
+    })
+}
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 1e92049f1..d28c97f09 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -16,6 +16,7 @@ import {
 import * as mkdirp from 'mkdirp'
 import * as bcrypt from 'bcrypt'
 import * as createTorrent from 'create-torrent'
+import * as rimraf from 'rimraf'
 import * as openssl from 'openssl-wrapper'
 import * as Promise from 'bluebird'
 
@@ -83,6 +84,7 @@ const bcryptComparePromise = promisify2<any, string, boolean>(bcrypt.compare)
 const bcryptGenSaltPromise = promisify1<number, string>(bcrypt.genSalt)
 const bcryptHashPromise = promisify2<any, string|number, string>(bcrypt.hash)
 const createTorrentPromise = promisify2<string, any, any>(createTorrent)
+const rimrafPromise = promisify1WithVoid<string>(rimraf)
 
 // ---------------------------------------------------------------------------
 
@@ -105,5 +107,6 @@ export {
   bcryptComparePromise,
   bcryptGenSaltPromise,
   bcryptHashPromise,
-  createTorrentPromise
+  createTorrentPromise,
+  rimrafPromise
 }
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index f087b7476..928a3f570 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -61,7 +61,8 @@ const CONFIG = {
     VIDEOS_DIR: join(root(), config.get<string>('storage.videos')),
     THUMBNAILS_DIR: join(root(), config.get<string>('storage.thumbnails')),
     PREVIEWS_DIR: join(root(), config.get<string>('storage.previews')),
-    TORRENTS_DIR: join(root(), config.get<string>('storage.torrents'))
+    TORRENTS_DIR: join(root(), config.get<string>('storage.torrents')),
+    CACHE_DIR: join(root(), config.get<string>('storage.cache'))
   },
   WEBSERVER: {
     SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http',
@@ -80,6 +81,11 @@ const CONFIG = {
   TRANSCODING: {
     ENABLED: config.get<boolean>('transcoding.enabled'),
     THREADS: config.get<number>('transcoding.threads')
+  },
+  CACHE: {
+    PREVIEWS: {
+      SIZE: config.get<number>('cache.previews.size')
+    }
   }
 }
 CONFIG.WEBSERVER.URL = CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
@@ -278,6 +284,13 @@ let STATIC_MAX_AGE = '30d'
 const THUMBNAILS_SIZE = '200x110'
 const PREVIEWS_SIZE = '640x480'
 
+// Subfolders of cache directory
+const CACHE = {
+  DIRECTORIES: {
+    PREVIEWS: join(CONFIG.STORAGE.CACHE_DIR, 'previews')
+  }
+}
+
 // ---------------------------------------------------------------------------
 
 const USER_ROLES: { [ id: string ]: UserRole } = {
@@ -307,6 +320,7 @@ if (isTestInstance() === true) {
 export {
   API_VERSION,
   BCRYPT_SALT_SIZE,
+  CACHE,
   CONFIG,
   CONSTRAINTS_FIELDS,
   FRIEND_SCORE,
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index 1ec24c4ad..3c5a77df9 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -4,12 +4,13 @@ import * as passwordGenerator from 'password-generator'
 import * as Promise from 'bluebird'
 
 import { database as db } from './database'
-import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION } from './constants'
+import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION, CACHE } from './constants'
 import { clientsExist, usersExist } from './checker'
-import { logger, createCertsIfNotExist, root, mkdirpPromise } from '../helpers'
+import { logger, createCertsIfNotExist, root, mkdirpPromise, rimrafPromise } from '../helpers'
 
 function installApplication () {
   return db.sequelize.sync()
+    .then(() => removeCacheDirectories())
     .then(() => createDirectoriesIfNotExist())
     .then(() => createCertsIfNotExist())
     .then(() => createOAuthClientIfNotExist())
@@ -24,13 +25,34 @@ export {
 
 // ---------------------------------------------------------------------------
 
+function removeCacheDirectories () {
+  const cacheDirectories = CACHE.DIRECTORIES
+
+  const tasks = []
+
+  // Cache directories
+  Object.keys(cacheDirectories).forEach(key => {
+    const dir = cacheDirectories[key]
+    tasks.push(rimrafPromise(dir))
+  })
+
+  return Promise.all(tasks)
+}
+
 function createDirectoriesIfNotExist () {
-  const storages = config.get('storage')
+  const storages = CONFIG.STORAGE
+  const cacheDirectories = CACHE.DIRECTORIES
 
   const tasks = []
   Object.keys(storages).forEach(key => {
     const dir = storages[key]
-    tasks.push(mkdirpPromise(join(root(), dir)))
+    tasks.push(mkdirpPromise(dir))
+  })
+
+  // Cache directories
+  Object.keys(cacheDirectories).forEach(key => {
+    const dir = cacheDirectories[key]
+    tasks.push(mkdirpPromise(dir))
   })
 
   return Promise.all(tasks)
diff --git a/server/lib/cache/index.ts b/server/lib/cache/index.ts
new file mode 100644
index 000000000..7bf63790a
--- /dev/null
+++ b/server/lib/cache/index.ts
@@ -0,0 +1 @@
+export * from './videos-preview-cache'
diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/cache/videos-preview-cache.ts
new file mode 100644
index 000000000..9d365e496
--- /dev/null
+++ b/server/lib/cache/videos-preview-cache.ts
@@ -0,0 +1,74 @@
+import * as request from 'request'
+import * as asyncLRU from 'async-lru'
+import { join } from 'path'
+import { createWriteStream } from 'fs'
+import * as Promise from 'bluebird'
+
+import { database as db, CONFIG, CACHE } from '../../initializers'
+import { logger, writeFilePromise, unlinkPromise } from '../../helpers'
+import { VideoInstance } from '../../models'
+import { fetchRemotePreview } from '../../lib'
+
+class VideosPreviewCache {
+
+  private static instance: VideosPreviewCache
+
+  private lru
+
+  private constructor () { }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  init (max: number) {
+    this.lru = new asyncLRU({
+      max,
+      load: (key, cb) => {
+        this.loadPreviews(key)
+          .then(res => cb(null, res))
+          .catch(err => cb(err))
+      }
+    })
+
+    this.lru.on('evict', (obj: { key: string, value: string }) => {
+      unlinkPromise(obj.value).then(() => logger.debug('%s evicted from VideosPreviewCache', obj.value))
+    })
+  }
+
+  getPreviewPath (key: string) {
+    return new Promise<string>((res, rej) => {
+      this.lru.get(key, (err, value) => {
+        err ? rej(err) : res(value)
+      })
+    })
+  }
+
+  private loadPreviews (key: string) {
+    return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(key)
+      .then(video => {
+        if (!video) return undefined
+
+        if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
+
+        return this.saveRemotePreviewAndReturnPath(video)
+      })
+  }
+
+  private saveRemotePreviewAndReturnPath (video: VideoInstance) {
+    const req = fetchRemotePreview(video.Author.Pod, video)
+
+    return new Promise<string>((res, rej) => {
+      const path = join(CACHE.DIRECTORIES.PREVIEWS, video.getPreviewName())
+      const stream = createWriteStream(path)
+
+      req.pipe(stream)
+         .on('finish', () => res(path))
+         .on('error', (err) => rej(err))
+    })
+  }
+}
+
+export {
+  VideosPreviewCache
+}
diff --git a/server/lib/friends.ts b/server/lib/friends.ts
index 6ed0da013..50355d5d1 100644
--- a/server/lib/friends.ts
+++ b/server/lib/friends.ts
@@ -1,6 +1,7 @@
 import * as request from 'request'
 import * as Sequelize from 'sequelize'
 import * as Promise from 'bluebird'
+import { join } from 'path'
 
 import { database as db } from '../initializers/database'
 import {
@@ -9,7 +10,8 @@ import {
   REQUESTS_IN_PARALLEL,
   REQUEST_ENDPOINTS,
   REQUEST_ENDPOINT_ACTIONS,
-  REMOTE_SCHEME
+  REMOTE_SCHEME,
+  STATIC_PATHS
 } from '../initializers'
 import {
   logger,
@@ -233,6 +235,13 @@ function sendOwnedVideosToPod (podId: number) {
     })
 }
 
+function fetchRemotePreview (pod: PodInstance, video: VideoInstance) {
+  const host = video.Author.Pod.host
+  const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
+
+  return request.get(REMOTE_SCHEME.HTTP + '://' + host + path)
+}
+
 function getRequestScheduler () {
   return requestScheduler
 }
@@ -263,7 +272,8 @@ export {
   sendOwnedVideosToPod,
   getRequestScheduler,
   getRequestVideoQaduScheduler,
-  getRequestVideoEventScheduler
+  getRequestVideoEventScheduler,
+  fetchRemotePreview
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/lib/index.ts b/server/lib/index.ts
index b8697fb96..8628da4dd 100644
--- a/server/lib/index.ts
+++ b/server/lib/index.ts
@@ -1,3 +1,4 @@
+export * from './cache'
 export * from './jobs'
 export * from './request'
 export * from './friends'
diff --git a/server/models/oauth/oauth-token-interface.ts b/server/models/oauth/oauth-token-interface.ts
index f2ddafa54..97af3c815 100644
--- a/server/models/oauth/oauth-token-interface.ts
+++ b/server/models/oauth/oauth-token-interface.ts
@@ -35,6 +35,8 @@ export interface OAuthTokenAttributes {
   refreshToken: string
   refreshTokenExpiresAt: Date
 
+  userId?: number
+  oAuthClientId?: number
   User?: UserModel
 }
 
diff --git a/server/models/oauth/oauth-token.ts b/server/models/oauth/oauth-token.ts
index 5c3781394..e3de9468e 100644
--- a/server/models/oauth/oauth-token.ts
+++ b/server/models/oauth/oauth-token.ts
@@ -106,10 +106,10 @@ getByRefreshTokenAndPopulateClient = function (refreshToken: string) {
         refreshToken: token.refreshToken,
         refreshTokenExpiresAt: token.refreshTokenExpiresAt,
         client: {
-          id: token['client'].id
+          id: token.oAuthClientId
         },
         user: {
-          id: token['user']
+          id: token.userId
         }
       }
 
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 650025205..b7eb24c4a 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -451,6 +451,7 @@ toFormatedJSON = function (this: VideoInstance) {
     dislikes: this.dislikes,
     tags: map<TagInstance, string>(this.Tags, 'name'),
     thumbnailPath: join(STATIC_PATHS.THUMBNAILS, this.getThumbnailName()),
+    previewPath: join(STATIC_PATHS.PREVIEWS, this.getPreviewName()),
     createdAt: this.createdAt,
     updatedAt: this.updatedAt
   }
diff --git a/server/tests/api/fixtures/video_short1-preview.webm.jpg b/server/tests/api/fixtures/video_short1-preview.webm.jpg
new file mode 100644
index 000000000..69c100c4e
--- /dev/null
+++ b/server/tests/api/fixtures/video_short1-preview.webm.jpg
diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js
index 1bc6157e8..7753e6f2d 100644
--- a/server/tests/api/multiple-pods.js
+++ b/server/tests/api/multiple-pods.js
@@ -747,7 +747,7 @@ describe('Test multiple pods', function () {
           expect(videos[0].name).not.to.equal(toRemove[1].name)
           expect(videos[1].name).not.to.equal(toRemove[1].name)
 
-          videoUUID = videos[0].uuid
+          videoUUID = videos.find(video => video.name === 'my super name for pod 1').uuid
 
           callback()
         })
@@ -781,6 +781,23 @@ describe('Test multiple pods', function () {
         })
       }, done)
     })
+
+    it('Should get the preview from each pod', function (done) {
+      each(servers, function (server, callback) {
+        videosUtils.getVideo(server.url, videoUUID, function (err, res) {
+          if (err) throw err
+
+          const video = res.body
+
+          videosUtils.testVideoImage(server.url, 'video_short1-preview.webm', video.previewPath, function (err, test) {
+            if (err) throw err
+            expect(test).to.equal(true)
+
+            callback()
+          })
+        })
+      }, done)
+    })
   })
 
   after(function (done) {
diff --git a/server/tests/utils/videos.js b/server/tests/utils/videos.js
index 6e7aabc5d..cb3be6897 100644
--- a/server/tests/utils/videos.js
+++ b/server/tests/utils/videos.js
@@ -195,7 +195,7 @@ function searchVideoWithSort (url, search, sort, end) {
     .end(end)
 }
 
-function testVideoImage (url, videoName, imagePath, callback) {
+function testVideoImage (url, imageName, imagePath, callback) {
   // Don't test images if the node env is not set
   // Because we need a special ffmpeg version for this test
   if (process.env.NODE_TEST_IMAGE) {
@@ -205,7 +205,7 @@ function testVideoImage (url, videoName, imagePath, callback) {
       .end(function (err, res) {
         if (err) return callback(err)
 
-        fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', videoName + '.jpg'), function (err, data) {
+        fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', imageName + '.jpg'), function (err, data) {
           if (err) return callback(err)
 
           callback(null, data.equals(res.body))