From 9567011bf01f36c7f796ac1e0f1fb12c71635e53 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 30 Oct 2017 10:16:27 +0100 Subject: [PATCH] Add lazy description on server --- server/controllers/api/remote/videos.ts | 4 +- server/controllers/api/videos/index.ts | 21 ++++- .../custom-validators/remote/videos.ts | 4 +- server/helpers/custom-validators/videos.ts | 5 ++ server/initializers/constants.ts | 5 +- server/initializers/database.ts | 9 +- .../migrations/0090-videos-description.ts | 25 ++++++ server/lib/friends.ts | 19 ++++ server/models/video/video-interface.ts | 4 + server/models/video/video.ts | 86 +++++++------------ server/tests/api/index.ts | 1 + server/tests/api/video-description.ts | 86 +++++++++++++++++++ server/tests/utils/videos.ts | 9 ++ .../remote-video-create-request.model.ts | 2 +- .../remote-video-update-request.model.ts | 2 +- shared/models/videos/video.model.ts | 1 + 16 files changed, 217 insertions(+), 66 deletions(-) create mode 100644 server/initializers/migrations/0090-videos-description.ts create mode 100644 server/tests/api/video-description.ts diff --git a/server/controllers/api/remote/videos.ts b/server/controllers/api/remote/videos.ts index d0febdd4b..3ecc62ada 100644 --- a/server/controllers/api/remote/videos.ts +++ b/server/controllers/api/remote/videos.ts @@ -258,7 +258,7 @@ async function addRemoteVideo (videoToCreateData: RemoteVideoCreateData, fromPod licence: videoToCreateData.licence, language: videoToCreateData.language, nsfw: videoToCreateData.nsfw, - description: videoToCreateData.description, + description: videoToCreateData.truncatedDescription, channelId: videoChannel.id, duration: videoToCreateData.duration, createdAt: videoToCreateData.createdAt, @@ -327,7 +327,7 @@ async function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData videoInstance.set('licence', videoAttributesToUpdate.licence) videoInstance.set('language', videoAttributesToUpdate.language) videoInstance.set('nsfw', videoAttributesToUpdate.nsfw) - videoInstance.set('description', videoAttributesToUpdate.description) + videoInstance.set('description', videoAttributesToUpdate.truncatedDescription) videoInstance.set('duration', videoAttributesToUpdate.duration) videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 9e233a8cc..49f0e4630 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -16,7 +16,8 @@ import { quickAndDirtyUpdateVideoToFriends, addVideoToFriends, updateVideoToFriends, - JobScheduler + JobScheduler, + fetchRemoteDescription } from '../../../lib' import { authenticate, @@ -102,6 +103,11 @@ videosRouter.post('/upload', videosAddValidator, asyncMiddleware(addVideoRetryWrapper) ) + +videosRouter.get('/:id/description', + videosGetValidator, + asyncMiddleware(getVideoDescription) +) videosRouter.get('/:id', videosGetValidator, getVideo @@ -328,6 +334,19 @@ function getVideo (req: express.Request, res: express.Response) { return res.json(videoInstance.toFormattedDetailsJSON()) } +async function getVideoDescription (req: express.Request, res: express.Response) { + const videoInstance = res.locals.video + let description = '' + + if (videoInstance.isOwned()) { + description = videoInstance.description + } else { + description = await fetchRemoteDescription(videoInstance) + } + + return res.json({ description }) +} + async function listVideos (req: express.Request, res: express.Response, next: express.NextFunction) { const resultList = await db.Video.listForApi(req.query.start, req.query.count, req.query.sort) diff --git a/server/helpers/custom-validators/remote/videos.ts b/server/helpers/custom-validators/remote/videos.ts index a9ca36fe8..e0ffba679 100644 --- a/server/helpers/custom-validators/remote/videos.ts +++ b/server/helpers/custom-validators/remote/videos.ts @@ -19,7 +19,7 @@ import { isRemoteVideoLicenceValid, isRemoteVideoLanguageValid, isVideoNSFWValid, - isVideoDescriptionValid, + isVideoTruncatedDescriptionValid, isVideoDurationValid, isVideoFileInfoHashValid, isVideoNameValid, @@ -112,7 +112,7 @@ function isCommonVideoAttributesValid (video: any) { isRemoteVideoLicenceValid(video.licence) && isRemoteVideoLanguageValid(video.language) && isVideoNSFWValid(video.nsfw) && - isVideoDescriptionValid(video.description) && + isVideoTruncatedDescriptionValid(video.truncatedDescription) && isVideoDurationValid(video.duration) && isVideoNameValid(video.name) && isVideoTagsValid(video.tags) && diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index 11b085b78..5b9102275 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -54,6 +54,10 @@ function isVideoNSFWValid (value: any) { return typeof value === 'boolean' || (typeof value === 'string' && validator.isBoolean(value)) } +function isVideoTruncatedDescriptionValid (value: string) { + return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.TRUNCATED_DESCRIPTION) +} + function isVideoDescriptionValid (value: string) { return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION) } @@ -173,6 +177,7 @@ export { isVideoLicenceValid, isVideoLanguageValid, isVideoNSFWValid, + isVideoTruncatedDescriptionValid, isVideoDescriptionValid, isVideoDurationValid, isVideoFileInfoHashValid, diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6dc9737d2..adccb9f41 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -15,7 +15,7 @@ import { // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 85 +const LAST_MIGRATION_VERSION = 90 // --------------------------------------------------------------------------- @@ -122,7 +122,8 @@ const CONSTRAINTS_FIELDS = { }, VIDEOS: { NAME: { min: 3, max: 120 }, // Length - DESCRIPTION: { min: 3, max: 250 }, // Length + TRUNCATED_DESCRIPTION: { min: 3, max: 250 }, // Length + DESCRIPTION: { min: 3, max: 3000 }, // Length EXTNAME: [ '.mp4', '.ogv', '.webm' ], INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2 DURATION: { min: 1, max: 7200 }, // Number diff --git a/server/initializers/database.ts b/server/initializers/database.ts index dfad01581..141566c3a 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -84,9 +84,14 @@ database.init = async (silent: boolean) => { const filePaths = await getModelFiles(modelDirectory) for (const filePath of filePaths) { - const model = sequelize.import(filePath) + try { + const model = sequelize.import(filePath) - database[model['name']] = model + database[model['name']] = model + } catch (err) { + logger.error('Cannot import database model %s.', filePath, err) + process.exit(0) + } } for (const modelName of Object.keys(database)) { diff --git a/server/initializers/migrations/0090-videos-description.ts b/server/initializers/migrations/0090-videos-description.ts new file mode 100644 index 000000000..6f98dcade --- /dev/null +++ b/server/initializers/migrations/0090-videos-description.ts @@ -0,0 +1,25 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize, + db: any +}): Promise { + const q = utils.queryInterface + + const data = { + type: Sequelize.STRING(3000), + allowNull: false + } + await q.changeColumn('Videos', 'description', data) +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/friends.ts b/server/lib/friends.ts index 55cbb55b9..5c9baef47 100644 --- a/server/lib/friends.ts +++ b/server/lib/friends.ts @@ -349,6 +349,24 @@ function fetchRemotePreview (video: VideoInstance) { return request.get(REMOTE_SCHEME.HTTP + '://' + host + path) } +function fetchRemoteDescription (video: VideoInstance) { + const host = video.VideoChannel.Author.Pod.host + const path = video.getDescriptionPath() + + const requestOptions = { + url: REMOTE_SCHEME.HTTP + '://' + host + path, + json: true + } + + return new Promise((res, rej) => { + request.get(requestOptions, (err, response, body) => { + if (err) return rej(err) + + return res(body.description ? body.description : '') + }) + }) +} + async function removeFriend (pod: PodInstance) { const requestParams = { method: 'POST' as 'POST', @@ -407,6 +425,7 @@ export { getRequestVideoEventScheduler, fetchRemotePreview, addVideoChannelToFriends, + fetchRemoteDescription, updateVideoChannelToFriends, removeVideoChannelToFriends } diff --git a/server/models/video/video-interface.ts b/server/models/video/video-interface.ts index 2afbaf09e..3a7bc82a4 100644 --- a/server/models/video/video-interface.ts +++ b/server/models/video/video-interface.ts @@ -38,6 +38,8 @@ export namespace VideoMethods { export type GetEmbedPath = (this: VideoInstance) => string export type GetThumbnailPath = (this: VideoInstance) => string export type GetPreviewPath = (this: VideoInstance) => string + export type GetDescriptionPath = (this: VideoInstance) => string + export type GetTruncatedDescription = (this: VideoInstance) => string // Return thumbnail name export type GenerateThumbnailFromData = (video: VideoInstance, thumbnailData: string) => Promise @@ -135,6 +137,8 @@ export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.In transcodeOriginalVideofile: VideoMethods.TranscodeOriginalVideofile getOriginalFileHeight: VideoMethods.GetOriginalFileHeight getEmbedPath: VideoMethods.GetEmbedPath + getDescriptionPath: VideoMethods.GetDescriptionPath + getTruncatedDescription : VideoMethods.GetTruncatedDescription setTags: Sequelize.HasManySetAssociationsMixin addVideoFile: Sequelize.HasManyAddAssociationMixin diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 27f59f3a9..1877c506a 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -6,7 +6,7 @@ import * as parseTorrent from 'parse-torrent' import { join } from 'path' import * as Sequelize from 'sequelize' import * as Promise from 'bluebird' -import { maxBy } from 'lodash' +import { maxBy, truncate } from 'lodash' import { TagInstance } from './tag-interface' import { @@ -35,7 +35,10 @@ import { VIDEO_CATEGORIES, VIDEO_LICENCES, VIDEO_LANGUAGES, - THUMBNAILS_SIZE + THUMBNAILS_SIZE, + PREVIEWS_SIZE, + CONSTRAINTS_FIELDS, + API_VERSION } from '../../initializers' import { removeVideoToFriends } from '../../lib' import { VideoResolution } from '../../../shared' @@ -48,7 +51,6 @@ import { VideoMethods } from './video-interface' -import { PREVIEWS_SIZE } from '../../initializers/constants' let Video: Sequelize.Model let getOriginalFile: VideoMethods.GetOriginalFile @@ -71,6 +73,8 @@ let getVideoFilePath: VideoMethods.GetVideoFilePath let createTorrentAndSetInfoHash: VideoMethods.CreateTorrentAndSetInfoHash let getOriginalFileHeight: VideoMethods.GetOriginalFileHeight let getEmbedPath: VideoMethods.GetEmbedPath +let getDescriptionPath: VideoMethods.GetDescriptionPath +let getTruncatedDescription: VideoMethods.GetTruncatedDescription let generateThumbnailFromData: VideoMethods.GenerateThumbnailFromData let list: VideoMethods.List @@ -153,7 +157,7 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da } }, description: { - type: DataTypes.STRING, + type: DataTypes.STRING(CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max), allowNull: false, validate: { descriptionValid: value => { @@ -276,7 +280,9 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da optimizeOriginalVideofile, transcodeOriginalVideofile, getOriginalFileHeight, - getEmbedPath + getEmbedPath, + getTruncatedDescription, + getDescriptionPath ] addMethodsToModel(Video, classMethods, instanceMethods) @@ -473,7 +479,7 @@ toFormattedJSON = function (this: VideoInstance) { language: this.language, languageLabel, nsfw: this.nsfw, - description: this.description, + description: this.getTruncatedDescription(), podHost, isLocal: this.isOwned(), author: this.VideoChannel.Author.name, @@ -493,59 +499,17 @@ toFormattedJSON = function (this: VideoInstance) { } toFormattedDetailsJSON = function (this: VideoInstance) { - let podHost - - if (this.VideoChannel.Author.Pod) { - podHost = this.VideoChannel.Author.Pod.host - } else { - // It means it's our video - podHost = CONFIG.WEBSERVER.HOST - } + const formattedJson = this.toFormattedJSON() - // Maybe our pod is not up to date and there are new categories since our version - let categoryLabel = VIDEO_CATEGORIES[this.category] - if (!categoryLabel) categoryLabel = 'Misc' - - // Maybe our pod is not up to date and there are new licences since our version - let licenceLabel = VIDEO_LICENCES[this.licence] - if (!licenceLabel) licenceLabel = 'Unknown' - - // Language is an optional attribute - let languageLabel = VIDEO_LANGUAGES[this.language] - if (!languageLabel) languageLabel = 'Unknown' - - const json = { - id: this.id, - uuid: this.uuid, - name: this.name, - category: this.category, - categoryLabel, - licence: this.licence, - licenceLabel, - language: this.language, - languageLabel, - nsfw: this.nsfw, - description: this.description, - podHost, - isLocal: this.isOwned(), - author: this.VideoChannel.Author.name, - duration: this.duration, - views: this.views, - likes: this.likes, - dislikes: this.dislikes, - tags: map(this.Tags, 'name'), - thumbnailPath: this.getThumbnailPath(), - previewPath: this.getPreviewPath(), - embedPath: this.getEmbedPath(), - createdAt: this.createdAt, - updatedAt: this.updatedAt, + const detailsJson = { + descriptionPath: this.getDescriptionPath(), channel: this.VideoChannel.toFormattedJSON(), files: [] } // Format and sort video files const { baseUrlHttp, baseUrlWs } = getBaseUrls(this) - json.files = this.VideoFiles + detailsJson.files = this.VideoFiles .map(videoFile => { let resolutionLabel = videoFile.resolution + 'p' @@ -566,7 +530,7 @@ toFormattedDetailsJSON = function (this: VideoInstance) { return -1 }) - return json + return Object.assign(formattedJson, detailsJson) } toAddRemoteJSON = function (this: VideoInstance) { @@ -581,7 +545,7 @@ toAddRemoteJSON = function (this: VideoInstance) { licence: this.licence, language: this.language, nsfw: this.nsfw, - description: this.description, + truncatedDescription: this.getTruncatedDescription(), channelUUID: this.VideoChannel.uuid, duration: this.duration, thumbnailData: thumbnailData.toString('binary'), @@ -615,7 +579,7 @@ toUpdateRemoteJSON = function (this: VideoInstance) { licence: this.licence, language: this.language, nsfw: this.nsfw, - description: this.description, + truncatedDescription: this.getTruncatedDescription(), duration: this.duration, tags: map(this.Tags, 'name'), createdAt: this.createdAt, @@ -638,6 +602,14 @@ toUpdateRemoteJSON = function (this: VideoInstance) { return json } +getTruncatedDescription = function (this: VideoInstance) { + const options = { + length: CONSTRAINTS_FIELDS.VIDEOS.TRUNCATED_DESCRIPTION.max + } + + return truncate(this.description, options) +} + optimizeOriginalVideofile = function (this: VideoInstance) { const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR const newExtname = '.mp4' @@ -730,6 +702,10 @@ getOriginalFileHeight = function (this: VideoInstance) { return getVideoFileHeight(originalFilePath) } +getDescriptionPath = function (this: VideoInstance) { + return `/api/${API_VERSION}/videos/${this.uuid}/description` +} + removeThumbnail = function (this: VideoInstance) { const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) return unlinkPromise(thumbnailPath) diff --git a/server/tests/api/index.ts b/server/tests/api/index.ts index e50e65049..2ff0ecf24 100644 --- a/server/tests/api/index.ts +++ b/server/tests/api/index.ts @@ -7,6 +7,7 @@ import './single-pod' import './video-abuse' import './video-blacklist' import './video-blacklist-management' +import './video-description' import './multiple-pods' import './services' import './request-schedulers' diff --git a/server/tests/api/video-description.ts b/server/tests/api/video-description.ts new file mode 100644 index 000000000..f04c5f1f6 --- /dev/null +++ b/server/tests/api/video-description.ts @@ -0,0 +1,86 @@ +/* tslint:disable:no-unused-expression */ + +import 'mocha' +import * as chai from 'chai' + +import { + flushAndRunMultipleServers, + flushTests, + getVideo, + getVideosList, + killallServers, + makeFriends, + ServerInfo, + setAccessTokensToServers, + uploadVideo, + wait, + getVideoDescription +} from '../utils' + +const expect = chai.expect + +describe('Test video description', function () { + let servers: ServerInfo[] = [] + let videoUUID = '' + let longDescription = 'my super description for pod 1'.repeat(50) + + before(async function () { + this.timeout(10000) + + // Run servers + servers = await flushAndRunMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + + // Pod 1 makes friend with pod 2 + await makeFriends(servers[0].url, servers[0].accessToken) + }) + + it('Should upload video with long description', async function () { + this.timeout(15000) + + const attributes = { + description: longDescription + } + await uploadVideo(servers[0].url, servers[0].accessToken, attributes) + + await wait(11000) + + const res = await getVideosList(servers[0].url) + + videoUUID = res.body.data[0].uuid + }) + + it('Should have a truncated description on each pod', async function () { + for (const server of servers) { + const res = await getVideo(server.url, videoUUID) + const video = res.body + + // 30 characters * 6 -> 240 characters + const truncatedDescription = 'my super description for pod 1'.repeat(8) + + 'my supe...' + + expect(video.description).to.equal(truncatedDescription) + } + }) + + it('Should fetch long description on each pod', async function () { + for (const server of servers) { + const res = await getVideo(server.url, videoUUID) + const video = res.body + + const res2 = await getVideoDescription(server.url, video.descriptionPath) + expect(res2.body.description).to.equal(longDescription) + } + }) + + after(async function () { + killallServers(servers) + + // Keep the logs if the test failed + if (this['ok']) { + await flushTests() + } + }) +}) diff --git a/server/tests/utils/videos.ts b/server/tests/utils/videos.ts index 08fa48da6..2a5d00255 100644 --- a/server/tests/utils/videos.ts +++ b/server/tests/utils/videos.ts @@ -61,6 +61,14 @@ function getVideo (url: string, id: number | string) { .expect('Content-Type', /json/) } +function getVideoDescription (url: string, descriptionPath: string) { + return request(url) + .get(descriptionPath) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) +} + function getVideosList (url: string) { const path = '/api/v1/videos' @@ -263,6 +271,7 @@ function parseTorrentVideo (server: ServerInfo, videoUUID: string, resolution: n // --------------------------------------------------------------------------- export { + getVideoDescription, getVideoCategories, getVideoLicences, getVideoLanguages, diff --git a/shared/models/pods/remote-video/remote-video-create-request.model.ts b/shared/models/pods/remote-video/remote-video-create-request.model.ts index e00e81214..cb20dfa03 100644 --- a/shared/models/pods/remote-video/remote-video-create-request.model.ts +++ b/shared/models/pods/remote-video/remote-video-create-request.model.ts @@ -9,7 +9,7 @@ export interface RemoteVideoCreateData { licence: number language: number nsfw: boolean - description: string + truncatedDescription: string duration: number createdAt: Date updatedAt: Date diff --git a/shared/models/pods/remote-video/remote-video-update-request.model.ts b/shared/models/pods/remote-video/remote-video-update-request.model.ts index 90c42fc28..8439cfa24 100644 --- a/shared/models/pods/remote-video/remote-video-update-request.model.ts +++ b/shared/models/pods/remote-video/remote-video-update-request.model.ts @@ -8,7 +8,7 @@ export interface RemoteVideoUpdateData { licence: number language: number nsfw: boolean - description: string + truncatedDescription: string duration: number createdAt: Date updatedAt: Date diff --git a/shared/models/videos/video.model.ts b/shared/models/videos/video.model.ts index 32463933d..1490d345c 100644 --- a/shared/models/videos/video.model.ts +++ b/shared/models/videos/video.model.ts @@ -37,6 +37,7 @@ export interface Video { } export interface VideoDetails extends Video { + descriptionPath: string, channel: VideoChannel files: VideoFile[] } -- 2.41.0