Add lazy description on server

16 files changed, 217 insertions, 66 deletions

diff --git a/server/controllers/api/remote/videos.ts b/server/controllers/api/remote/videos.ts
index d0febdd4b..3ecc62ada 100644
--- a/server/controllers/api/remote/videos.ts
+++ b/server/controllers/api/remote/videos.ts
@@ -258,7 +258,7 @@ async function addRemoteVideo (videoToCreateData: RemoteVideoCreateData, fromPod
       licence: videoToCreateData.licence,
       language: videoToCreateData.language,
       nsfw: videoToCreateData.nsfw,
-      description: videoToCreateData.description,
+      description: videoToCreateData.truncatedDescription,
       channelId: videoChannel.id,
       duration: videoToCreateData.duration,
       createdAt: videoToCreateData.createdAt,
@@ -327,7 +327,7 @@ async function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData
       videoInstance.set('licence', videoAttributesToUpdate.licence)
       videoInstance.set('language', videoAttributesToUpdate.language)
       videoInstance.set('nsfw', videoAttributesToUpdate.nsfw)
-      videoInstance.set('description', videoAttributesToUpdate.description)
+      videoInstance.set('description', videoAttributesToUpdate.truncatedDescription)
       videoInstance.set('duration', videoAttributesToUpdate.duration)
       videoInstance.set('createdAt', videoAttributesToUpdate.createdAt)
       videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt)
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 9e233a8cc..49f0e4630 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -16,7 +16,8 @@ import {
   quickAndDirtyUpdateVideoToFriends,
   addVideoToFriends,
   updateVideoToFriends,
-  JobScheduler
+  JobScheduler,
+  fetchRemoteDescription
 } from '../../../lib'
 import {
   authenticate,
@@ -102,6 +103,11 @@ videosRouter.post('/upload',
   videosAddValidator,
   asyncMiddleware(addVideoRetryWrapper)
 )
+
+videosRouter.get('/:id/description',
+  videosGetValidator,
+  asyncMiddleware(getVideoDescription)
+)
 videosRouter.get('/:id',
   videosGetValidator,
   getVideo
@@ -328,6 +334,19 @@ function getVideo (req: express.Request, res: express.Response) {
   return res.json(videoInstance.toFormattedDetailsJSON())
 }
 
+async function getVideoDescription (req: express.Request, res: express.Response) {
+  const videoInstance = res.locals.video
+  let description = ''
+
+  if (videoInstance.isOwned()) {
+    description = videoInstance.description
+  } else {
+    description = await fetchRemoteDescription(videoInstance)
+  }
+
+  return res.json({ description })
+}
+
 async function listVideos (req: express.Request, res: express.Response, next: express.NextFunction) {
   const resultList = await db.Video.listForApi(req.query.start, req.query.count, req.query.sort)
 
diff --git a/server/helpers/custom-validators/remote/videos.ts b/server/helpers/custom-validators/remote/videos.ts
index a9ca36fe8..e0ffba679 100644
--- a/server/helpers/custom-validators/remote/videos.ts
+++ b/server/helpers/custom-validators/remote/videos.ts
@@ -19,7 +19,7 @@ import {
   isRemoteVideoLicenceValid,
   isRemoteVideoLanguageValid,
   isVideoNSFWValid,
-  isVideoDescriptionValid,
+  isVideoTruncatedDescriptionValid,
   isVideoDurationValid,
   isVideoFileInfoHashValid,
   isVideoNameValid,
@@ -112,7 +112,7 @@ function isCommonVideoAttributesValid (video: any) {
          isRemoteVideoLicenceValid(video.licence) &&
          isRemoteVideoLanguageValid(video.language) &&
          isVideoNSFWValid(video.nsfw) &&
-         isVideoDescriptionValid(video.description) &&
+         isVideoTruncatedDescriptionValid(video.truncatedDescription) &&
          isVideoDurationValid(video.duration) &&
          isVideoNameValid(video.name) &&
          isVideoTagsValid(video.tags) &&
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index 11b085b78..5b9102275 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -54,6 +54,10 @@ function isVideoNSFWValid (value: any) {
   return typeof value === 'boolean' || (typeof value === 'string' && validator.isBoolean(value))
 }
 
+function isVideoTruncatedDescriptionValid (value: string) {
+  return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.TRUNCATED_DESCRIPTION)
+}
+
 function isVideoDescriptionValid (value: string) {
   return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION)
 }
@@ -173,6 +177,7 @@ export {
   isVideoLicenceValid,
   isVideoLanguageValid,
   isVideoNSFWValid,
+  isVideoTruncatedDescriptionValid,
   isVideoDescriptionValid,
   isVideoDurationValid,
   isVideoFileInfoHashValid,
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 6dc9737d2..adccb9f41 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -15,7 +15,7 @@ import {
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 85
+const LAST_MIGRATION_VERSION = 90
 
 // ---------------------------------------------------------------------------
 
@@ -122,7 +122,8 @@ const CONSTRAINTS_FIELDS = {
   },
   VIDEOS: {
     NAME: { min: 3, max: 120 }, // Length
-    DESCRIPTION: { min: 3, max: 250 }, // Length
+    TRUNCATED_DESCRIPTION: { min: 3, max: 250 }, // Length
+    DESCRIPTION: { min: 3, max: 3000 }, // Length
     EXTNAME: [ '.mp4', '.ogv', '.webm' ],
     INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2
     DURATION: { min: 1, max: 7200 }, // Number
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index dfad01581..141566c3a 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -84,9 +84,14 @@ database.init = async (silent: boolean) => {
   const filePaths = await getModelFiles(modelDirectory)
 
   for (const filePath of filePaths) {
-    const model = sequelize.import(filePath)
+    try {
+      const model = sequelize.import(filePath)
 
-    database[model['name']] = model
+      database[model['name']] = model
+    } catch (err) {
+      logger.error('Cannot import database model %s.', filePath, err)
+      process.exit(0)
+    }
   }
 
   for (const modelName of Object.keys(database)) {
diff --git a/server/initializers/migrations/0090-videos-description.ts b/server/initializers/migrations/0090-videos-description.ts
new file mode 100644
index 000000000..6f98dcade
--- /dev/null
+++ b/server/initializers/migrations/0090-videos-description.ts
@@ -0,0 +1,25 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction,
+  queryInterface: Sequelize.QueryInterface,
+  sequelize: Sequelize.Sequelize,
+  db: any
+}): Promise<void> {
+  const q = utils.queryInterface
+
+  const data = {
+    type: Sequelize.STRING(3000),
+    allowNull: false
+  }
+  await q.changeColumn('Videos', 'description', data)
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}
diff --git a/server/lib/friends.ts b/server/lib/friends.ts
index 55cbb55b9..5c9baef47 100644
--- a/server/lib/friends.ts
+++ b/server/lib/friends.ts
@@ -349,6 +349,24 @@ function fetchRemotePreview (video: VideoInstance) {
   return request.get(REMOTE_SCHEME.HTTP + '://' + host + path)
 }
 
+function fetchRemoteDescription (video: VideoInstance) {
+  const host = video.VideoChannel.Author.Pod.host
+  const path = video.getDescriptionPath()
+
+  const requestOptions = {
+    url: REMOTE_SCHEME.HTTP + '://' + host + path,
+    json: true
+  }
+
+  return new Promise<string>((res, rej) => {
+    request.get(requestOptions, (err, response, body) => {
+      if (err) return rej(err)
+
+      return res(body.description ? body.description : '')
+    })
+  })
+}
+
 async function removeFriend (pod: PodInstance) {
   const requestParams = {
     method: 'POST' as 'POST',
@@ -407,6 +425,7 @@ export {
   getRequestVideoEventScheduler,
   fetchRemotePreview,
   addVideoChannelToFriends,
+  fetchRemoteDescription,
   updateVideoChannelToFriends,
   removeVideoChannelToFriends
 }
diff --git a/server/models/video/video-interface.ts b/server/models/video/video-interface.ts
index 2afbaf09e..3a7bc82a4 100644
--- a/server/models/video/video-interface.ts
+++ b/server/models/video/video-interface.ts
@@ -38,6 +38,8 @@ export namespace VideoMethods {
   export type GetEmbedPath = (this: VideoInstance) => string
   export type GetThumbnailPath = (this: VideoInstance) => string
   export type GetPreviewPath = (this: VideoInstance) => string
+  export type GetDescriptionPath = (this: VideoInstance) => string
+  export type GetTruncatedDescription = (this: VideoInstance) => string
 
   // Return thumbnail name
   export type GenerateThumbnailFromData = (video: VideoInstance, thumbnailData: string) => Promise<string>
@@ -135,6 +137,8 @@ export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.In
   transcodeOriginalVideofile: VideoMethods.TranscodeOriginalVideofile
   getOriginalFileHeight: VideoMethods.GetOriginalFileHeight
   getEmbedPath: VideoMethods.GetEmbedPath
+  getDescriptionPath: VideoMethods.GetDescriptionPath
+  getTruncatedDescription : VideoMethods.GetTruncatedDescription
 
   setTags: Sequelize.HasManySetAssociationsMixin<TagAttributes, string>
   addVideoFile: Sequelize.HasManyAddAssociationMixin<VideoFileAttributes, string>
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 27f59f3a9..1877c506a 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -6,7 +6,7 @@ import * as parseTorrent from 'parse-torrent'
 import { join } from 'path'
 import * as Sequelize from 'sequelize'
 import * as Promise from 'bluebird'
-import { maxBy } from 'lodash'
+import { maxBy, truncate } from 'lodash'
 
 import { TagInstance } from './tag-interface'
 import {
@@ -35,7 +35,10 @@ import {
   VIDEO_CATEGORIES,
   VIDEO_LICENCES,
   VIDEO_LANGUAGES,
-  THUMBNAILS_SIZE
+  THUMBNAILS_SIZE,
+  PREVIEWS_SIZE,
+  CONSTRAINTS_FIELDS,
+  API_VERSION
 } from '../../initializers'
 import { removeVideoToFriends } from '../../lib'
 import { VideoResolution } from '../../../shared'
@@ -48,7 +51,6 @@ import {
 
   VideoMethods
 } from './video-interface'
-import { PREVIEWS_SIZE } from '../../initializers/constants'
 
 let Video: Sequelize.Model<VideoInstance, VideoAttributes>
 let getOriginalFile: VideoMethods.GetOriginalFile
@@ -71,6 +73,8 @@ let getVideoFilePath: VideoMethods.GetVideoFilePath
 let createTorrentAndSetInfoHash: VideoMethods.CreateTorrentAndSetInfoHash
 let getOriginalFileHeight: VideoMethods.GetOriginalFileHeight
 let getEmbedPath: VideoMethods.GetEmbedPath
+let getDescriptionPath: VideoMethods.GetDescriptionPath
+let getTruncatedDescription: VideoMethods.GetTruncatedDescription
 
 let generateThumbnailFromData: VideoMethods.GenerateThumbnailFromData
 let list: VideoMethods.List
@@ -153,7 +157,7 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da
         }
       },
       description: {
-        type: DataTypes.STRING,
+        type: DataTypes.STRING(CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max),
         allowNull: false,
         validate: {
           descriptionValid: value => {
@@ -276,7 +280,9 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da
     optimizeOriginalVideofile,
     transcodeOriginalVideofile,
     getOriginalFileHeight,
-    getEmbedPath
+    getEmbedPath,
+    getTruncatedDescription,
+    getDescriptionPath
   ]
   addMethodsToModel(Video, classMethods, instanceMethods)
 
@@ -473,7 +479,7 @@ toFormattedJSON = function (this: VideoInstance) {
     language: this.language,
     languageLabel,
     nsfw: this.nsfw,
-    description: this.description,
+    description: this.getTruncatedDescription(),
     podHost,
     isLocal: this.isOwned(),
     author: this.VideoChannel.Author.name,
@@ -493,59 +499,17 @@ toFormattedJSON = function (this: VideoInstance) {
 }
 
 toFormattedDetailsJSON = function (this: VideoInstance) {
-  let podHost
-
-  if (this.VideoChannel.Author.Pod) {
-    podHost = this.VideoChannel.Author.Pod.host
-  } else {
-    // It means it's our video
-    podHost = CONFIG.WEBSERVER.HOST
-  }
+  const formattedJson = this.toFormattedJSON()
 
-  // Maybe our pod is not up to date and there are new categories since our version
-  let categoryLabel = VIDEO_CATEGORIES[this.category]
-  if (!categoryLabel) categoryLabel = 'Misc'
-
-  // Maybe our pod is not up to date and there are new licences since our version
-  let licenceLabel = VIDEO_LICENCES[this.licence]
-  if (!licenceLabel) licenceLabel = 'Unknown'
-
-  // Language is an optional attribute
-  let languageLabel = VIDEO_LANGUAGES[this.language]
-  if (!languageLabel) languageLabel = 'Unknown'
-
-  const json = {
-    id: this.id,
-    uuid: this.uuid,
-    name: this.name,
-    category: this.category,
-    categoryLabel,
-    licence: this.licence,
-    licenceLabel,
-    language: this.language,
-    languageLabel,
-    nsfw: this.nsfw,
-    description: this.description,
-    podHost,
-    isLocal: this.isOwned(),
-    author: this.VideoChannel.Author.name,
-    duration: this.duration,
-    views: this.views,
-    likes: this.likes,
-    dislikes: this.dislikes,
-    tags: map<TagInstance, string>(this.Tags, 'name'),
-    thumbnailPath: this.getThumbnailPath(),
-    previewPath: this.getPreviewPath(),
-    embedPath: this.getEmbedPath(),
-    createdAt: this.createdAt,
-    updatedAt: this.updatedAt,
+  const detailsJson = {
+    descriptionPath: this.getDescriptionPath(),
     channel: this.VideoChannel.toFormattedJSON(),
     files: []
   }
 
   // Format and sort video files
   const { baseUrlHttp, baseUrlWs } = getBaseUrls(this)
-  json.files = this.VideoFiles
+  detailsJson.files = this.VideoFiles
                    .map(videoFile => {
                      let resolutionLabel = videoFile.resolution + 'p'
 
@@ -566,7 +530,7 @@ toFormattedDetailsJSON = function (this: VideoInstance) {
                      return -1
                    })
 
-  return json
+  return Object.assign(formattedJson, detailsJson)
 }
 
 toAddRemoteJSON = function (this: VideoInstance) {
@@ -581,7 +545,7 @@ toAddRemoteJSON = function (this: VideoInstance) {
       licence: this.licence,
       language: this.language,
       nsfw: this.nsfw,
-      description: this.description,
+      truncatedDescription: this.getTruncatedDescription(),
       channelUUID: this.VideoChannel.uuid,
       duration: this.duration,
       thumbnailData: thumbnailData.toString('binary'),
@@ -615,7 +579,7 @@ toUpdateRemoteJSON = function (this: VideoInstance) {
     licence: this.licence,
     language: this.language,
     nsfw: this.nsfw,
-    description: this.description,
+    truncatedDescription: this.getTruncatedDescription(),
     duration: this.duration,
     tags: map<TagInstance, string>(this.Tags, 'name'),
     createdAt: this.createdAt,
@@ -638,6 +602,14 @@ toUpdateRemoteJSON = function (this: VideoInstance) {
   return json
 }
 
+getTruncatedDescription = function (this: VideoInstance) {
+  const options = {
+    length: CONSTRAINTS_FIELDS.VIDEOS.TRUNCATED_DESCRIPTION.max
+  }
+
+  return truncate(this.description, options)
+}
+
 optimizeOriginalVideofile = function (this: VideoInstance) {
   const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
   const newExtname = '.mp4'
@@ -730,6 +702,10 @@ getOriginalFileHeight = function (this: VideoInstance) {
   return getVideoFileHeight(originalFilePath)
 }
 
+getDescriptionPath = function (this: VideoInstance) {
+  return `/api/${API_VERSION}/videos/${this.uuid}/description`
+}
+
 removeThumbnail = function (this: VideoInstance) {
   const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName())
   return unlinkPromise(thumbnailPath)
diff --git a/server/tests/api/index.ts b/server/tests/api/index.ts
index e50e65049..2ff0ecf24 100644
--- a/server/tests/api/index.ts
+++ b/server/tests/api/index.ts
@@ -7,6 +7,7 @@ import './single-pod'
 import './video-abuse'
 import './video-blacklist'
 import './video-blacklist-management'
+import './video-description'
 import './multiple-pods'
 import './services'
 import './request-schedulers'
diff --git a/server/tests/api/video-description.ts b/server/tests/api/video-description.ts
new file mode 100644
index 000000000..f04c5f1f6
--- /dev/null
+++ b/server/tests/api/video-description.ts
@@ -0,0 +1,86 @@
+/* tslint:disable:no-unused-expression */
+
+import 'mocha'
+import * as chai from 'chai'
+
+import {
+  flushAndRunMultipleServers,
+  flushTests,
+  getVideo,
+  getVideosList,
+  killallServers,
+  makeFriends,
+  ServerInfo,
+  setAccessTokensToServers,
+  uploadVideo,
+  wait,
+  getVideoDescription
+} from '../utils'
+
+const expect = chai.expect
+
+describe('Test video description', function () {
+  let servers: ServerInfo[] = []
+  let videoUUID = ''
+  let longDescription = 'my super description for pod 1'.repeat(50)
+
+  before(async function () {
+    this.timeout(10000)
+
+    // Run servers
+    servers = await flushAndRunMultipleServers(2)
+
+    // Get the access tokens
+    await setAccessTokensToServers(servers)
+
+    // Pod 1 makes friend with pod 2
+    await makeFriends(servers[0].url, servers[0].accessToken)
+  })
+
+  it('Should upload video with long description', async function () {
+    this.timeout(15000)
+
+    const attributes = {
+      description: longDescription
+    }
+    await uploadVideo(servers[0].url, servers[0].accessToken, attributes)
+
+    await wait(11000)
+
+    const res = await getVideosList(servers[0].url)
+
+    videoUUID = res.body.data[0].uuid
+  })
+
+  it('Should have a truncated description on each pod', async function () {
+    for (const server of servers) {
+      const res = await getVideo(server.url, videoUUID)
+      const video = res.body
+
+      // 30 characters * 6 -> 240 characters
+      const truncatedDescription = 'my super description for pod 1'.repeat(8) +
+                                   'my supe...'
+
+      expect(video.description).to.equal(truncatedDescription)
+    }
+  })
+
+  it('Should fetch long description on each pod', async function () {
+    for (const server of servers) {
+      const res = await getVideo(server.url, videoUUID)
+      const video = res.body
+
+      const res2 = await getVideoDescription(server.url, video.descriptionPath)
+      expect(res2.body.description).to.equal(longDescription)
+    }
+  })
+
+  after(async function () {
+    killallServers(servers)
+
+    // Keep the logs if the test failed
+    if (this['ok']) {
+      await flushTests()
+    }
+  })
+})
diff --git a/server/tests/utils/videos.ts b/server/tests/utils/videos.ts
index 08fa48da6..2a5d00255 100644
--- a/server/tests/utils/videos.ts
+++ b/server/tests/utils/videos.ts
@@ -61,6 +61,14 @@ function getVideo (url: string, id: number | string) {
           .expect('Content-Type', /json/)
 }
 
+function getVideoDescription (url: string, descriptionPath: string) {
+  return request(url)
+    .get(descriptionPath)
+    .set('Accept', 'application/json')
+    .expect(200)
+    .expect('Content-Type', /json/)
+}
+
 function getVideosList (url: string) {
   const path = '/api/v1/videos'
 
@@ -263,6 +271,7 @@ function parseTorrentVideo (server: ServerInfo, videoUUID: string, resolution: n
 // ---------------------------------------------------------------------------
 
 export {
+  getVideoDescription,
   getVideoCategories,
   getVideoLicences,
   getVideoLanguages,
diff --git a/shared/models/pods/remote-video/remote-video-create-request.model.ts b/shared/models/pods/remote-video/remote-video-create-request.model.ts
index e00e81214..cb20dfa03 100644
--- a/shared/models/pods/remote-video/remote-video-create-request.model.ts
+++ b/shared/models/pods/remote-video/remote-video-create-request.model.ts
@@ -9,7 +9,7 @@ export interface RemoteVideoCreateData {
   licence: number
   language: number
   nsfw: boolean
-  description: string
+  truncatedDescription: string
   duration: number
   createdAt: Date
   updatedAt: Date
diff --git a/shared/models/pods/remote-video/remote-video-update-request.model.ts b/shared/models/pods/remote-video/remote-video-update-request.model.ts
index 90c42fc28..8439cfa24 100644
--- a/shared/models/pods/remote-video/remote-video-update-request.model.ts
+++ b/shared/models/pods/remote-video/remote-video-update-request.model.ts
@@ -8,7 +8,7 @@ export interface RemoteVideoUpdateData {
   licence: number
   language: number
   nsfw: boolean
-  description: string
+  truncatedDescription: string
   duration: number
   createdAt: Date
   updatedAt: Date
diff --git a/shared/models/videos/video.model.ts b/shared/models/videos/video.model.ts
index 32463933d..1490d345c 100644
--- a/shared/models/videos/video.model.ts
+++ b/shared/models/videos/video.model.ts
@@ -37,6 +37,7 @@ export interface Video {
 }
 
 export interface VideoDetails extends Video {
+  descriptionPath: string,
   channel: VideoChannel
   files: VideoFile[]
 }