aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <florian.bigard@gmail.com>2017-10-02 12:20:26 +0200
committerChocobozzz <florian.bigard@gmail.com>2017-10-03 15:31:26 +0200
commit40298b02546e8225dd21bf6048fe7f224aefc32a (patch)
tree0a0b981dbeb2af47810adff6553a0df995a03734 /server
parentf0adb2701c1cf404ff63095f71e542bfe6d025ae (diff)
downloadPeerTube-40298b02546e8225dd21bf6048fe7f224aefc32a.tar.gz
PeerTube-40298b02546e8225dd21bf6048fe7f224aefc32a.tar.zst
PeerTube-40298b02546e8225dd21bf6048fe7f224aefc32a.zip
Implement video transcoding on server side
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/videos/index.ts9
-rw-r--r--server/helpers/core-utils.ts8
-rw-r--r--server/helpers/utils.ts27
-rw-r--r--server/initializers/constants.ts30
-rw-r--r--server/lib/jobs/handlers/index.ts6
-rw-r--r--server/lib/jobs/handlers/video-file-optimizer.ts78
-rw-r--r--server/lib/jobs/handlers/video-file-transcoder.ts (renamed from server/lib/jobs/handlers/video-transcoder.ts)17
-rw-r--r--server/models/pod/pod.ts2
-rw-r--r--server/models/user/user.ts3
-rw-r--r--server/models/video/video-interface.ts63
-rw-r--r--server/models/video/video.ts105
-rw-r--r--server/tests/api/multiple-pods.ts65
-rw-r--r--server/tests/api/video-transcoder.ts4
-rw-r--r--server/tests/cli/update-host.ts30
-rw-r--r--server/tests/utils/videos.ts5
15 files changed, 327 insertions, 125 deletions
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 6fa84c801..14c969ec3 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -39,13 +39,12 @@ import {
39 getFormattedObjects, 39 getFormattedObjects,
40 renamePromise 40 renamePromise
41} from '../../../helpers' 41} from '../../../helpers'
42import { TagInstance } from '../../../models' 42import { TagInstance, VideoInstance } from '../../../models'
43import { VideoCreate, VideoUpdate } from '../../../../shared' 43import { VideoCreate, VideoUpdate, VideoResolution } from '../../../../shared'
44 44
45import { abuseVideoRouter } from './abuse' 45import { abuseVideoRouter } from './abuse'
46import { blacklistRouter } from './blacklist' 46import { blacklistRouter } from './blacklist'
47import { rateVideoRouter } from './rate' 47import { rateVideoRouter } from './rate'
48import { VideoInstance } from '../../../models/video/video-interface'
49 48
50const videosRouter = express.Router() 49const videosRouter = express.Router()
51 50
@@ -195,7 +194,7 @@ function addVideo (req: express.Request, res: express.Response, videoPhysicalFil
195 .then(({ author, tagInstances, video }) => { 194 .then(({ author, tagInstances, video }) => {
196 const videoFileData = { 195 const videoFileData = {
197 extname: extname(videoPhysicalFile.filename), 196 extname: extname(videoPhysicalFile.filename),
198 resolution: 0, // TODO: improve readability, 197 resolution: VideoResolution.ORIGINAL,
199 size: videoPhysicalFile.size 198 size: videoPhysicalFile.size
200 } 199 }
201 200
@@ -230,7 +229,7 @@ function addVideo (req: express.Request, res: express.Response, videoPhysicalFil
230 } 229 }
231 230
232 tasks.push( 231 tasks.push(
233 JobScheduler.Instance.createJob(t, 'videoTranscoder', dataInput) 232 JobScheduler.Instance.createJob(t, 'videoFileOptimizer', dataInput)
234 ) 233 )
235 } 234 }
236 235
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 2ec7e6515..3118dc500 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -11,7 +11,9 @@ import {
11 rename, 11 rename,
12 unlink, 12 unlink,
13 writeFile, 13 writeFile,
14 access 14 access,
15 stat,
16 Stats
15} from 'fs' 17} from 'fs'
16import * as mkdirp from 'mkdirp' 18import * as mkdirp from 'mkdirp'
17import * as bcrypt from 'bcrypt' 19import * as bcrypt from 'bcrypt'
@@ -92,6 +94,7 @@ const bcryptGenSaltPromise = promisify1<number, string>(bcrypt.genSalt)
92const bcryptHashPromise = promisify2<any, string|number, string>(bcrypt.hash) 94const bcryptHashPromise = promisify2<any, string|number, string>(bcrypt.hash)
93const createTorrentPromise = promisify2<string, any, any>(createTorrent) 95const createTorrentPromise = promisify2<string, any, any>(createTorrent)
94const rimrafPromise = promisify1WithVoid<string>(rimraf) 96const rimrafPromise = promisify1WithVoid<string>(rimraf)
97const statPromise = promisify1<string, Stats>(stat)
95 98
96// --------------------------------------------------------------------------- 99// ---------------------------------------------------------------------------
97 100
@@ -115,5 +118,6 @@ export {
115 bcryptGenSaltPromise, 118 bcryptGenSaltPromise,
116 bcryptHashPromise, 119 bcryptHashPromise,
117 createTorrentPromise, 120 createTorrentPromise,
118 rimrafPromise 121 rimrafPromise,
122 statPromise
119} 123}
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index ce07ceff9..b74442ab0 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -4,6 +4,7 @@ import * as Promise from 'bluebird'
4import { pseudoRandomBytesPromise } from './core-utils' 4import { pseudoRandomBytesPromise } from './core-utils'
5import { CONFIG, database as db } from '../initializers' 5import { CONFIG, database as db } from '../initializers'
6import { ResultList } from '../../shared' 6import { ResultList } from '../../shared'
7import { VideoResolution } from '../../shared/models/videos/video-resolution.enum'
7 8
8function badRequest (req: express.Request, res: express.Response, next: express.NextFunction) { 9function badRequest (req: express.Request, res: express.Response, next: express.NextFunction) {
9 res.type('json').status(400).end() 10 res.type('json').status(400).end()
@@ -13,11 +14,11 @@ function generateRandomString (size: number) {
13 return pseudoRandomBytesPromise(size).then(raw => raw.toString('hex')) 14 return pseudoRandomBytesPromise(size).then(raw => raw.toString('hex'))
14} 15}
15 16
16interface FormatableToJSON { 17interface FormattableToJSON {
17 toFormattedJSON () 18 toFormattedJSON ()
18} 19}
19 20
20function getFormattedObjects<U, T extends FormatableToJSON> (objects: T[], objectsTotal: number) { 21function getFormattedObjects<U, T extends FormattableToJSON> (objects: T[], objectsTotal: number) {
21 const formattedObjects: U[] = [] 22 const formattedObjects: U[] = []
22 23
23 objects.forEach(object => { 24 objects.forEach(object => {
@@ -47,6 +48,27 @@ function isSignupAllowed () {
47 }) 48 })
48} 49}
49 50
51function computeResolutionsToTranscode (videoFileHeight: number) {
52 const resolutionsEnabled: number[] = []
53 const configResolutions = CONFIG.TRANSCODING.RESOLUTIONS
54
55 const resolutions = [
56 VideoResolution.H_240P,
57 VideoResolution.H_360P,
58 VideoResolution.H_480P,
59 VideoResolution.H_720P,
60 VideoResolution.H_1080P
61 ]
62
63 for (const resolution of resolutions) {
64 if (configResolutions[resolution.toString()] === true && videoFileHeight >= resolution) {
65 resolutionsEnabled.push(resolution)
66 }
67 }
68
69 return resolutionsEnabled
70}
71
50type SortType = { sortModel: any, sortValue: string } 72type SortType = { sortModel: any, sortValue: string }
51 73
52// --------------------------------------------------------------------------- 74// ---------------------------------------------------------------------------
@@ -56,5 +78,6 @@ export {
56 generateRandomString, 78 generateRandomString,
57 getFormattedObjects, 79 getFormattedObjects,
58 isSignupAllowed, 80 isSignupAllowed,
81 computeResolutionsToTranscode,
59 SortType 82 SortType
60} 83}
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 54e91d35d..073fabd27 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -10,7 +10,8 @@ import {
10 RequestEndpoint, 10 RequestEndpoint,
11 RequestVideoEventType, 11 RequestVideoEventType,
12 RequestVideoQaduType, 12 RequestVideoQaduType,
13 JobState 13 JobState,
14 VideoResolution
14} from '../../shared/models' 15} from '../../shared/models'
15 16
16// --------------------------------------------------------------------------- 17// ---------------------------------------------------------------------------
@@ -85,7 +86,14 @@ const CONFIG = {
85 }, 86 },
86 TRANSCODING: { 87 TRANSCODING: {
87 ENABLED: config.get<boolean>('transcoding.enabled'), 88 ENABLED: config.get<boolean>('transcoding.enabled'),
88 THREADS: config.get<number>('transcoding.threads') 89 THREADS: config.get<number>('transcoding.threads'),
90 RESOLUTIONS: {
91 '240' : config.get<boolean>('transcoding.resolutions.240p'),
92 '360': config.get<boolean>('transcoding.resolutions.360p'),
93 '480': config.get<boolean>('transcoding.resolutions.480p'),
94 '720': config.get<boolean>('transcoding.resolutions.720p'),
95 '1080': config.get<boolean>('transcoding.resolutions.1080p')
96 }
89 }, 97 },
90 CACHE: { 98 CACHE: {
91 PREVIEWS: { 99 PREVIEWS: {
@@ -144,7 +152,7 @@ const VIDEO_CATEGORIES = {
144 9: 'Comedy', 152 9: 'Comedy',
145 10: 'Entertainment', 153 10: 'Entertainment',
146 11: 'News', 154 11: 'News',
147 12: 'Howto', 155 12: 'How To',
148 13: 'Education', 156 13: 'Education',
149 14: 'Activism', 157 14: 'Activism',
150 15: 'Science & Technology', 158 15: 'Science & Technology',
@@ -179,15 +187,17 @@ const VIDEO_LANGUAGES = {
179 11: 'German', 187 11: 'German',
180 12: 'Korean', 188 12: 'Korean',
181 13: 'French', 189 13: 'French',
182 14: 'Italien' 190 14: 'Italian'
183} 191}
184 192
185const VIDEO_FILE_RESOLUTIONS = { 193// TODO: use VideoResolution when https://github.com/Microsoft/TypeScript/issues/13042 is fixed
194const VIDEO_FILE_RESOLUTIONS: { [ id: number ]: string } = {
186 0: 'original', 195 0: 'original',
187 1: '360p', 196 240: '240p',
188 2: '480p', 197 360: '360p',
189 3: '720p', 198 480: '480p',
190 4: '1080p' 199 720: '720p',
200 1080: '1080p'
191} 201}
192 202
193// --------------------------------------------------------------------------- 203// ---------------------------------------------------------------------------
@@ -202,7 +212,7 @@ const FRIEND_SCORE = {
202 212
203// Number of points we add/remove from a friend after a successful/bad request 213// Number of points we add/remove from a friend after a successful/bad request
204const PODS_SCORE = { 214const PODS_SCORE = {
205 MALUS: -10, 215 PENALTY: -10,
206 BONUS: 10 216 BONUS: 10
207} 217}
208 218
diff --git a/server/lib/jobs/handlers/index.ts b/server/lib/jobs/handlers/index.ts
index 8abddae35..5941427a1 100644
--- a/server/lib/jobs/handlers/index.ts
+++ b/server/lib/jobs/handlers/index.ts
@@ -1,4 +1,5 @@
1import * as videoTranscoder from './video-transcoder' 1import * as videoFileOptimizer from './video-file-optimizer'
2import * as videoFileTranscoder from './video-file-transcoder'
2 3
3export interface JobHandler<T> { 4export interface JobHandler<T> {
4 process (data: object): T 5 process (data: object): T
@@ -7,7 +8,8 @@ export interface JobHandler<T> {
7} 8}
8 9
9const jobHandlers: { [ handlerName: string ]: JobHandler<any> } = { 10const jobHandlers: { [ handlerName: string ]: JobHandler<any> } = {
10 videoTranscoder 11 videoFileOptimizer,
12 videoFileTranscoder
11} 13}
12 14
13export { 15export {
diff --git a/server/lib/jobs/handlers/video-file-optimizer.ts b/server/lib/jobs/handlers/video-file-optimizer.ts
new file mode 100644
index 000000000..a87ce52dc
--- /dev/null
+++ b/server/lib/jobs/handlers/video-file-optimizer.ts
@@ -0,0 +1,78 @@
1import * as Promise from 'bluebird'
2
3import { database as db } from '../../../initializers/database'
4import { logger, computeResolutionsToTranscode } from '../../../helpers'
5import { VideoInstance } from '../../../models'
6import { addVideoToFriends } from '../../friends'
7import { JobScheduler } from '../job-scheduler'
8
9function process (data: { videoUUID: string }) {
10 return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID).then(video => {
11 return video.optimizeOriginalVideofile().then(() => video)
12 })
13}
14
15function onError (err: Error, jobId: number) {
16 logger.error('Error when optimized video file in job %d.', jobId, err)
17 return Promise.resolve()
18}
19
20function onSuccess (jobId: number, video: VideoInstance) {
21 logger.info('Job %d is a success.', jobId)
22
23 video.toAddRemoteJSON()
24 .then(remoteVideo => {
25 // Now we'll add the video's meta data to our friends
26 return addVideoToFriends(remoteVideo, null)
27 })
28 .then(() => {
29 return video.getOriginalFileHeight()
30 })
31 .then(originalFileHeight => {
32 // Create transcoding jobs if there are enabled resolutions
33 const resolutionsEnabled = computeResolutionsToTranscode(originalFileHeight)
34 logger.info(
35 'Resolutions computed for video %s and origin file height of %d.', video.uuid, originalFileHeight,
36 { resolutions: resolutionsEnabled }
37 )
38
39 if (resolutionsEnabled.length === 0) return undefined
40
41 return db.sequelize.transaction(t => {
42 const tasks: Promise<any>[] = []
43
44 resolutionsEnabled.forEach(resolution => {
45 const dataInput = {
46 videoUUID: video.uuid,
47 resolution
48 }
49
50 const p = JobScheduler.Instance.createJob(t, 'videoFileTranscoder', dataInput)
51 tasks.push(p)
52 })
53
54 return Promise.all(tasks).then(() => resolutionsEnabled)
55 })
56 })
57 .then(resolutionsEnabled => {
58 if (resolutionsEnabled === undefined) {
59 logger.info('No transcoding jobs created for video %s (no resolutions enabled).')
60 return undefined
61 }
62
63 logger.info('Transcoding jobs created for uuid %s.', video.uuid, { resolutionsEnabled })
64 })
65 .catch((err: Error) => {
66 logger.debug('Cannot transcode the video.', err)
67 throw err
68 })
69
70}
71
72// ---------------------------------------------------------------------------
73
74export {
75 process,
76 onError,
77 onSuccess
78}
diff --git a/server/lib/jobs/handlers/video-transcoder.ts b/server/lib/jobs/handlers/video-file-transcoder.ts
index 87d8ffa6a..0e45b4dca 100644
--- a/server/lib/jobs/handlers/video-transcoder.ts
+++ b/server/lib/jobs/handlers/video-file-transcoder.ts
@@ -1,13 +1,12 @@
1import { database as db } from '../../../initializers/database' 1import { database as db } from '../../../initializers/database'
2import { updateVideoToFriends } from '../../friends'
2import { logger } from '../../../helpers' 3import { logger } from '../../../helpers'
3import { addVideoToFriends } from '../../../lib'
4import { VideoInstance } from '../../../models' 4import { VideoInstance } from '../../../models'
5import { VideoResolution } from '../../../../shared'
5 6
6function process (data: { videoUUID: string }) { 7function process (data: { videoUUID: string, resolution: VideoResolution }) {
7 return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID).then(video => { 8 return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID).then(video => {
8 // TODO: handle multiple resolutions 9 return video.transcodeOriginalVideofile(data.resolution).then(() => video)
9 const videoFile = video.VideoFiles[0]
10 return video.transcodeVideofile(videoFile).then(() => video)
11 }) 10 })
12} 11}
13 12
@@ -19,10 +18,10 @@ function onError (err: Error, jobId: number) {
19function onSuccess (jobId: number, video: VideoInstance) { 18function onSuccess (jobId: number, video: VideoInstance) {
20 logger.info('Job %d is a success.', jobId) 19 logger.info('Job %d is a success.', jobId)
21 20
22 video.toAddRemoteJSON().then(remoteVideo => { 21 const remoteVideo = video.toUpdateRemoteJSON()
23 // Now we'll add the video's meta data to our friends 22
24 return addVideoToFriends(remoteVideo, null) 23 // Now we'll add the video's meta data to our friends
25 }) 24 return updateVideoToFriends(remoteVideo, null)
26} 25}
27 26
28// --------------------------------------------------------------------------- 27// ---------------------------------------------------------------------------
diff --git a/server/models/pod/pod.ts b/server/models/pod/pod.ts
index df6412721..1440ac9b4 100644
--- a/server/models/pod/pod.ts
+++ b/server/models/pod/pod.ts
@@ -219,7 +219,7 @@ updatePodsScore = function (goodPods: number[], badPods: number[]) {
219 } 219 }
220 220
221 if (badPods.length !== 0) { 221 if (badPods.length !== 0) {
222 incrementScores(badPods, PODS_SCORE.MALUS) 222 incrementScores(badPods, PODS_SCORE.PENALTY)
223 .then(() => removeBadPods()) 223 .then(() => removeBadPods())
224 .catch(err => { 224 .catch(err => {
225 if (err) logger.error('Cannot decrement scores of bad pods.', err) 225 if (err) logger.error('Cannot decrement scores of bad pods.', err)
diff --git a/server/models/user/user.ts b/server/models/user/user.ts
index 79a595528..7a21dbefa 100644
--- a/server/models/user/user.ts
+++ b/server/models/user/user.ts
@@ -12,6 +12,7 @@ import {
12 isUserDisplayNSFWValid, 12 isUserDisplayNSFWValid,
13 isUserVideoQuotaValid 13 isUserVideoQuotaValid
14} from '../../helpers' 14} from '../../helpers'
15import { VideoResolution } from '../../../shared'
15 16
16import { addMethodsToModel } from '../utils' 17import { addMethodsToModel } from '../utils'
17import { 18import {
@@ -245,7 +246,7 @@ function getOriginalVideoFileTotalFromUser (user: UserInstance) {
245 // attributes = [] because we don't want other fields than the sum 246 // attributes = [] because we don't want other fields than the sum
246 const query = { 247 const query = {
247 where: { 248 where: {
248 resolution: 0 // Original, TODO: improve readability 249 resolution: VideoResolution.ORIGINAL
249 }, 250 },
250 include: [ 251 include: [
251 { 252 {
diff --git a/server/models/video/video-interface.ts b/server/models/video/video-interface.ts
index fb31c6a8f..340426f45 100644
--- a/server/models/video/video-interface.ts
+++ b/server/models/video/video-interface.ts
@@ -7,60 +7,17 @@ import { VideoFileAttributes, VideoFileInstance } from './video-file-interface'
7 7
8// Don't use barrel, import just what we need 8// Don't use barrel, import just what we need
9import { Video as FormattedVideo } from '../../../shared/models/videos/video.model' 9import { Video as FormattedVideo } from '../../../shared/models/videos/video.model'
10import { RemoteVideoUpdateData } from '../../../shared/models/pods/remote-video/remote-video-update-request.model'
11import { RemoteVideoCreateData } from '../../../shared/models/pods/remote-video/remote-video-create-request.model'
10import { ResultList } from '../../../shared/models/result-list.model' 12import { ResultList } from '../../../shared/models/result-list.model'
11 13
12export type FormattedRemoteVideoFile = {
13 infoHash: string
14 resolution: number
15 extname: string
16 size: number
17}
18
19export type FormattedAddRemoteVideo = {
20 uuid: string
21 name: string
22 category: number
23 licence: number
24 language: number
25 nsfw: boolean
26 description: string
27 author: string
28 duration: number
29 thumbnailData: string
30 tags: string[]
31 createdAt: Date
32 updatedAt: Date
33 views: number
34 likes: number
35 dislikes: number
36 files: FormattedRemoteVideoFile[]
37}
38
39export type FormattedUpdateRemoteVideo = {
40 uuid: string
41 name: string
42 category: number
43 licence: number
44 language: number
45 nsfw: boolean
46 description: string
47 author: string
48 duration: number
49 tags: string[]
50 createdAt: Date
51 updatedAt: Date
52 views: number
53 likes: number
54 dislikes: number
55 files: FormattedRemoteVideoFile[]
56}
57
58export namespace VideoMethods { 14export namespace VideoMethods {
59 export type GetThumbnailName = (this: VideoInstance) => string 15 export type GetThumbnailName = (this: VideoInstance) => string
60 export type GetPreviewName = (this: VideoInstance) => string 16 export type GetPreviewName = (this: VideoInstance) => string
61 export type IsOwned = (this: VideoInstance) => boolean 17 export type IsOwned = (this: VideoInstance) => boolean
62 export type ToFormattedJSON = (this: VideoInstance) => FormattedVideo 18 export type ToFormattedJSON = (this: VideoInstance) => FormattedVideo
63 19
20 export type GetOriginalFile = (this: VideoInstance) => VideoFileInstance
64 export type GenerateMagnetUri = (this: VideoInstance, videoFile: VideoFileInstance) => string 21 export type GenerateMagnetUri = (this: VideoInstance, videoFile: VideoFileInstance) => string
65 export type GetTorrentFileName = (this: VideoInstance, videoFile: VideoFileInstance) => string 22 export type GetTorrentFileName = (this: VideoInstance, videoFile: VideoFileInstance) => string
66 export type GetVideoFilename = (this: VideoInstance, videoFile: VideoFileInstance) => string 23 export type GetVideoFilename = (this: VideoInstance, videoFile: VideoFileInstance) => string
@@ -69,10 +26,12 @@ export namespace VideoMethods {
69 export type GetVideoFilePath = (this: VideoInstance, videoFile: VideoFileInstance) => string 26 export type GetVideoFilePath = (this: VideoInstance, videoFile: VideoFileInstance) => string
70 export type CreateTorrentAndSetInfoHash = (this: VideoInstance, videoFile: VideoFileInstance) => Promise<void> 27 export type CreateTorrentAndSetInfoHash = (this: VideoInstance, videoFile: VideoFileInstance) => Promise<void>
71 28
72 export type ToAddRemoteJSON = (this: VideoInstance) => Promise<FormattedAddRemoteVideo> 29 export type ToAddRemoteJSON = (this: VideoInstance) => Promise<RemoteVideoCreateData>
73 export type ToUpdateRemoteJSON = (this: VideoInstance) => FormattedUpdateRemoteVideo 30 export type ToUpdateRemoteJSON = (this: VideoInstance) => RemoteVideoUpdateData
74 31
75 export type TranscodeVideofile = (this: VideoInstance, inputVideoFile: VideoFileInstance) => Promise<void> 32 export type OptimizeOriginalVideofile = (this: VideoInstance) => Promise<void>
33 export type TranscodeOriginalVideofile = (this: VideoInstance, resolution: number) => Promise<void>
34 export type GetOriginalFileHeight = (this: VideoInstance) => Promise<number>
76 35
77 // Return thumbnail name 36 // Return thumbnail name
78 export type GenerateThumbnailFromData = (video: VideoInstance, thumbnailData: string) => Promise<string> 37 export type GenerateThumbnailFromData = (video: VideoInstance, thumbnailData: string) => Promise<string>
@@ -147,6 +106,7 @@ export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.In
147 createPreview: VideoMethods.CreatePreview 106 createPreview: VideoMethods.CreatePreview
148 createThumbnail: VideoMethods.CreateThumbnail 107 createThumbnail: VideoMethods.CreateThumbnail
149 createTorrentAndSetInfoHash: VideoMethods.CreateTorrentAndSetInfoHash 108 createTorrentAndSetInfoHash: VideoMethods.CreateTorrentAndSetInfoHash
109 getOriginalFile: VideoMethods.GetOriginalFile
150 generateMagnetUri: VideoMethods.GenerateMagnetUri 110 generateMagnetUri: VideoMethods.GenerateMagnetUri
151 getPreviewName: VideoMethods.GetPreviewName 111 getPreviewName: VideoMethods.GetPreviewName
152 getThumbnailName: VideoMethods.GetThumbnailName 112 getThumbnailName: VideoMethods.GetThumbnailName
@@ -161,9 +121,12 @@ export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.In
161 toAddRemoteJSON: VideoMethods.ToAddRemoteJSON 121 toAddRemoteJSON: VideoMethods.ToAddRemoteJSON
162 toFormattedJSON: VideoMethods.ToFormattedJSON 122 toFormattedJSON: VideoMethods.ToFormattedJSON
163 toUpdateRemoteJSON: VideoMethods.ToUpdateRemoteJSON 123 toUpdateRemoteJSON: VideoMethods.ToUpdateRemoteJSON
164 transcodeVideofile: VideoMethods.TranscodeVideofile 124 optimizeOriginalVideofile: VideoMethods.OptimizeOriginalVideofile
125 transcodeOriginalVideofile: VideoMethods.TranscodeOriginalVideofile
126 getOriginalFileHeight: VideoMethods.GetOriginalFileHeight
165 127
166 setTags: Sequelize.HasManySetAssociationsMixin<TagAttributes, string> 128 setTags: Sequelize.HasManySetAssociationsMixin<TagAttributes, string>
129 addVideoFile: Sequelize.HasManyAddAssociationMixin<VideoFileAttributes, string>
167 setVideoFiles: Sequelize.HasManySetAssociationsMixin<VideoFileAttributes, string> 130 setVideoFiles: Sequelize.HasManySetAssociationsMixin<VideoFileAttributes, string>
168} 131}
169 132
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index e011c3b4d..28df91a7b 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -22,7 +22,8 @@ import {
22 unlinkPromise, 22 unlinkPromise,
23 renamePromise, 23 renamePromise,
24 writeFilePromise, 24 writeFilePromise,
25 createTorrentPromise 25 createTorrentPromise,
26 statPromise
26} from '../../helpers' 27} from '../../helpers'
27import { 28import {
28 CONFIG, 29 CONFIG,
@@ -35,7 +36,8 @@ import {
35 VIDEO_FILE_RESOLUTIONS 36 VIDEO_FILE_RESOLUTIONS
36} from '../../initializers' 37} from '../../initializers'
37import { removeVideoToFriends } from '../../lib' 38import { removeVideoToFriends } from '../../lib'
38import { VideoFileInstance } from './video-file-interface' 39import { VideoResolution } from '../../../shared'
40import { VideoFileInstance, VideoFileModel } from './video-file-interface'
39 41
40import { addMethodsToModel, getSort } from '../utils' 42import { addMethodsToModel, getSort } from '../utils'
41import { 43import {
@@ -46,6 +48,7 @@ import {
46} from './video-interface' 48} from './video-interface'
47 49
48let Video: Sequelize.Model<VideoInstance, VideoAttributes> 50let Video: Sequelize.Model<VideoInstance, VideoAttributes>
51let getOriginalFile: VideoMethods.GetOriginalFile
49let generateMagnetUri: VideoMethods.GenerateMagnetUri 52let generateMagnetUri: VideoMethods.GenerateMagnetUri
50let getVideoFilename: VideoMethods.GetVideoFilename 53let getVideoFilename: VideoMethods.GetVideoFilename
51let getThumbnailName: VideoMethods.GetThumbnailName 54let getThumbnailName: VideoMethods.GetThumbnailName
@@ -55,11 +58,13 @@ let isOwned: VideoMethods.IsOwned
55let toFormattedJSON: VideoMethods.ToFormattedJSON 58let toFormattedJSON: VideoMethods.ToFormattedJSON
56let toAddRemoteJSON: VideoMethods.ToAddRemoteJSON 59let toAddRemoteJSON: VideoMethods.ToAddRemoteJSON
57let toUpdateRemoteJSON: VideoMethods.ToUpdateRemoteJSON 60let toUpdateRemoteJSON: VideoMethods.ToUpdateRemoteJSON
58let transcodeVideofile: VideoMethods.TranscodeVideofile 61let optimizeOriginalVideofile: VideoMethods.OptimizeOriginalVideofile
62let transcodeOriginalVideofile: VideoMethods.TranscodeOriginalVideofile
59let createPreview: VideoMethods.CreatePreview 63let createPreview: VideoMethods.CreatePreview
60let createThumbnail: VideoMethods.CreateThumbnail 64let createThumbnail: VideoMethods.CreateThumbnail
61let getVideoFilePath: VideoMethods.GetVideoFilePath 65let getVideoFilePath: VideoMethods.GetVideoFilePath
62let createTorrentAndSetInfoHash: VideoMethods.CreateTorrentAndSetInfoHash 66let createTorrentAndSetInfoHash: VideoMethods.CreateTorrentAndSetInfoHash
67let getOriginalFileHeight: VideoMethods.GetOriginalFileHeight
63 68
64let generateThumbnailFromData: VideoMethods.GenerateThumbnailFromData 69let generateThumbnailFromData: VideoMethods.GenerateThumbnailFromData
65let getDurationFromFile: VideoMethods.GetDurationFromFile 70let getDurationFromFile: VideoMethods.GetDurationFromFile
@@ -251,6 +256,7 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da
251 getTorrentFileName, 256 getTorrentFileName,
252 getVideoFilename, 257 getVideoFilename,
253 getVideoFilePath, 258 getVideoFilePath,
259 getOriginalFile,
254 isOwned, 260 isOwned,
255 removeFile, 261 removeFile,
256 removePreview, 262 removePreview,
@@ -259,7 +265,9 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da
259 toAddRemoteJSON, 265 toAddRemoteJSON,
260 toFormattedJSON, 266 toFormattedJSON,
261 toUpdateRemoteJSON, 267 toUpdateRemoteJSON,
262 transcodeVideofile 268 optimizeOriginalVideofile,
269 transcodeOriginalVideofile,
270 getOriginalFileHeight
263 ] 271 ]
264 addMethodsToModel(Video, classMethods, instanceMethods) 272 addMethodsToModel(Video, classMethods, instanceMethods)
265 273
@@ -327,9 +335,14 @@ function afterDestroy (video: VideoInstance, options: { transaction: Sequelize.T
327 return Promise.all(tasks) 335 return Promise.all(tasks)
328} 336}
329 337
338getOriginalFile = function (this: VideoInstance) {
339 if (Array.isArray(this.VideoFiles) === false) return undefined
340
341 return this.VideoFiles.find(file => file.resolution === VideoResolution.ORIGINAL)
342}
343
330getVideoFilename = function (this: VideoInstance, videoFile: VideoFileInstance) { 344getVideoFilename = function (this: VideoInstance, videoFile: VideoFileInstance) {
331 // return this.uuid + '-' + VIDEO_FILE_RESOLUTIONS[videoFile.resolution] + videoFile.extname 345 return this.uuid + '-' + VIDEO_FILE_RESOLUTIONS[videoFile.resolution] + videoFile.extname
332 return this.uuid + videoFile.extname
333} 346}
334 347
335getThumbnailName = function (this: VideoInstance) { 348getThumbnailName = function (this: VideoInstance) {
@@ -345,8 +358,7 @@ getPreviewName = function (this: VideoInstance) {
345 358
346getTorrentFileName = function (this: VideoInstance, videoFile: VideoFileInstance) { 359getTorrentFileName = function (this: VideoInstance, videoFile: VideoFileInstance) {
347 const extension = '.torrent' 360 const extension = '.torrent'
348 // return this.uuid + '-' + VIDEO_FILE_RESOLUTIONS[videoFile.resolution] + extension 361 return this.uuid + '-' + VIDEO_FILE_RESOLUTIONS[videoFile.resolution] + extension
349 return this.uuid + extension
350} 362}
351 363
352isOwned = function (this: VideoInstance) { 364isOwned = function (this: VideoInstance) {
@@ -552,9 +564,10 @@ toUpdateRemoteJSON = function (this: VideoInstance) {
552 return json 564 return json
553} 565}
554 566
555transcodeVideofile = function (this: VideoInstance, inputVideoFile: VideoFileInstance) { 567optimizeOriginalVideofile = function (this: VideoInstance) {
556 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR 568 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
557 const newExtname = '.mp4' 569 const newExtname = '.mp4'
570 const inputVideoFile = this.getOriginalFile()
558 const videoInputPath = join(videosDirectory, this.getVideoFilename(inputVideoFile)) 571 const videoInputPath = join(videosDirectory, this.getVideoFilename(inputVideoFile))
559 const videoOutputPath = join(videosDirectory, this.id + '-transcoded' + newExtname) 572 const videoOutputPath = join(videosDirectory, this.id + '-transcoded' + newExtname)
560 573
@@ -575,6 +588,12 @@ transcodeVideofile = function (this: VideoInstance, inputVideoFile: VideoFileIns
575 return renamePromise(videoOutputPath, this.getVideoFilePath(inputVideoFile)) 588 return renamePromise(videoOutputPath, this.getVideoFilePath(inputVideoFile))
576 }) 589 })
577 .then(() => { 590 .then(() => {
591 return statPromise(this.getVideoFilePath(inputVideoFile))
592 })
593 .then(stats => {
594 return inputVideoFile.set('size', stats.size)
595 })
596 .then(() => {
578 return this.createTorrentAndSetInfoHash(inputVideoFile) 597 return this.createTorrentAndSetInfoHash(inputVideoFile)
579 }) 598 })
580 .then(() => { 599 .then(() => {
@@ -594,6 +613,74 @@ transcodeVideofile = function (this: VideoInstance, inputVideoFile: VideoFileIns
594 }) 613 })
595} 614}
596 615
616transcodeOriginalVideofile = function (this: VideoInstance, resolution: VideoResolution) {
617 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
618 const extname = '.mp4'
619
620 // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed
621 const videoInputPath = join(videosDirectory, this.getVideoFilename(this.getOriginalFile()))
622
623 const newVideoFile = (Video['sequelize'].models.VideoFile as VideoFileModel).build({
624 resolution,
625 extname,
626 size: 0,
627 videoId: this.id
628 })
629 const videoOutputPath = join(videosDirectory, this.getVideoFilename(newVideoFile))
630 const resolutionWidthSizes = {
631 1: '240x?',
632 2: '360x?',
633 3: '480x?',
634 4: '720x?',
635 5: '1080x?'
636 }
637
638 return new Promise<void>((res, rej) => {
639 ffmpeg(videoInputPath)
640 .output(videoOutputPath)
641 .videoCodec('libx264')
642 .size(resolutionWidthSizes[resolution])
643 .outputOption('-threads ' + CONFIG.TRANSCODING.THREADS)
644 .outputOption('-movflags faststart')
645 .on('error', rej)
646 .on('end', () => {
647 return statPromise(videoOutputPath)
648 .then(stats => {
649 newVideoFile.set('size', stats.size)
650
651 return undefined
652 })
653 .then(() => {
654 return this.createTorrentAndSetInfoHash(newVideoFile)
655 })
656 .then(() => {
657 return newVideoFile.save()
658 })
659 .then(() => {
660 return this.VideoFiles.push(newVideoFile)
661 })
662 .then(() => {
663 return res()
664 })
665 .catch(rej)
666 })
667 .run()
668 })
669}
670
671getOriginalFileHeight = function (this: VideoInstance) {
672 const originalFilePath = this.getVideoFilePath(this.getOriginalFile())
673
674 return new Promise<number>((res, rej) => {
675 ffmpeg.ffprobe(originalFilePath, (err, metadata) => {
676 if (err) return rej(err)
677
678 const videoStream = metadata.streams.find(s => s.codec_type === 'video')
679 return res(videoStream.height)
680 })
681 })
682}
683
597removeThumbnail = function (this: VideoInstance) { 684removeThumbnail = function (this: VideoInstance) {
598 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) 685 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName())
599 return unlinkPromise(thumbnailPath) 686 return unlinkPromise(thumbnailPath)
diff --git a/server/tests/api/multiple-pods.ts b/server/tests/api/multiple-pods.ts
index 7117ab290..9860935e5 100644
--- a/server/tests/api/multiple-pods.ts
+++ b/server/tests/api/multiple-pods.ts
@@ -129,7 +129,7 @@ describe('Test multiple pods', function () {
129 }) 129 })
130 130
131 it('Should upload the video on pod 2 and propagate on each pod', async function () { 131 it('Should upload the video on pod 2 and propagate on each pod', async function () {
132 this.timeout(60000) 132 this.timeout(120000)
133 133
134 const videoAttributes = { 134 const videoAttributes = {
135 name: 'my super name for pod 2', 135 name: 'my super name for pod 2',
@@ -143,12 +143,12 @@ describe('Test multiple pods', function () {
143 } 143 }
144 await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes) 144 await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes)
145 145
146 // Transcoding, so wait more that 22 seconds 146 // Transcoding, so wait more than 22000
147 await wait(42000) 147 await wait(60000)
148 148
149 // All pods should have this video 149 // All pods should have this video
150 for (const server of servers) { 150 for (const server of servers) {
151 let baseMagnet = null 151 let baseMagnet = {}
152 152
153 const res = await getVideosList(server.url) 153 const res = await getVideosList(server.url)
154 154
@@ -172,28 +172,51 @@ describe('Test multiple pods', function () {
172 expect(dateIsValid(video.updatedAt)).to.be.true 172 expect(dateIsValid(video.updatedAt)).to.be.true
173 expect(video.author).to.equal('root') 173 expect(video.author).to.equal('root')
174 174
175 expect(video.files).to.have.lengthOf(1) 175 expect(video.files).to.have.lengthOf(5)
176 176
177 const file = video.files[0] 177 // Check common attributes
178 const magnetUri = file.magnetUri 178 for (const file of video.files) {
179 expect(file.magnetUri).to.have.lengthOf.above(2) 179 expect(file.magnetUri).to.have.lengthOf.above(2)
180 expect(file.resolution).to.equal(0)
181 expect(file.resolutionLabel).to.equal('original')
182 expect(file.size).to.equal(942961)
183 180
184 if (server.url !== 'http://localhost:9002') { 181 if (server.url !== 'http://localhost:9002') {
185 expect(video.isLocal).to.be.false 182 expect(video.isLocal).to.be.false
186 } else { 183 } else {
187 expect(video.isLocal).to.be.true 184 expect(video.isLocal).to.be.true
188 } 185 }
189 186
190 // All pods should have the same magnet Uri 187 // All pods should have the same magnet Uri
191 if (baseMagnet === null) { 188 if (baseMagnet[file.resolution] === undefined) {
192 baseMagnet = magnetUri 189 baseMagnet[file.resolution] = file.magnet
193 } else { 190 } else {
194 expect(baseMagnet).to.equal(magnetUri) 191 expect(baseMagnet[file.resolution]).to.equal(file.magnet)
192 }
195 } 193 }
196 194
195 const originalFile = video.files.find(f => f.resolution === 0)
196 expect(originalFile).not.to.be.undefined
197 expect(originalFile.resolutionLabel).to.equal('original')
198 expect(originalFile.size).to.equal(711327)
199
200 const file240p = video.files.find(f => f.resolution === 1)
201 expect(file240p).not.to.be.undefined
202 expect(file240p.resolutionLabel).to.equal('240p')
203 expect(file240p.size).to.equal(139953)
204
205 const file360p = video.files.find(f => f.resolution === 2)
206 expect(file360p).not.to.be.undefined
207 expect(file360p.resolutionLabel).to.equal('360p')
208 expect(file360p.size).to.equal(169926)
209
210 const file480p = video.files.find(f => f.resolution === 3)
211 expect(file480p).not.to.be.undefined
212 expect(file480p.resolutionLabel).to.equal('480p')
213 expect(file480p.size).to.equal(206758)
214
215 const file720p = video.files.find(f => f.resolution === 4)
216 expect(file720p).not.to.be.undefined
217 expect(file720p.resolutionLabel).to.equal('720p')
218 expect(file720p.size).to.equal(314913)
219
197 const test = await testVideoImage(server.url, 'video_short2.webm', video.thumbnailPath) 220 const test = await testVideoImage(server.url, 'video_short2.webm', video.thumbnailPath)
198 expect(test).to.equal(true) 221 expect(test).to.equal(true)
199 } 222 }
diff --git a/server/tests/api/video-transcoder.ts b/server/tests/api/video-transcoder.ts
index c6d4c61f5..b5d84d9e7 100644
--- a/server/tests/api/video-transcoder.ts
+++ b/server/tests/api/video-transcoder.ts
@@ -42,6 +42,8 @@ describe('Test video transcoding', function () {
42 42
43 const res = await getVideosList(servers[0].url) 43 const res = await getVideosList(servers[0].url)
44 const video = res.body.data[0] 44 const video = res.body.data[0]
45 expect(video.files).to.have.lengthOf(1)
46
45 const magnetUri = video.files[0].magnetUri 47 const magnetUri = video.files[0].magnetUri
46 expect(magnetUri).to.match(/\.webm/) 48 expect(magnetUri).to.match(/\.webm/)
47 49
@@ -66,6 +68,8 @@ describe('Test video transcoding', function () {
66 const res = await getVideosList(servers[1].url) 68 const res = await getVideosList(servers[1].url)
67 69
68 const video = res.body.data[0] 70 const video = res.body.data[0]
71 expect(video.files).to.have.lengthOf(5)
72
69 const magnetUri = video.files[0].magnetUri 73 const magnetUri = video.files[0].magnetUri
70 expect(magnetUri).to.match(/\.mp4/) 74 expect(magnetUri).to.match(/\.mp4/)
71 75
diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts
index 644b3807e..e31a84156 100644
--- a/server/tests/cli/update-host.ts
+++ b/server/tests/cli/update-host.ts
@@ -12,14 +12,15 @@ import {
12 runServer, 12 runServer,
13 ServerInfo, 13 ServerInfo,
14 setAccessTokensToServers, 14 setAccessTokensToServers,
15 uploadVideo 15 uploadVideo,
16 wait
16} from '../utils' 17} from '../utils'
17 18
18describe('Test update host scripts', function () { 19describe('Test update host scripts', function () {
19 let server: ServerInfo 20 let server: ServerInfo
20 21
21 before(async function () { 22 before(async function () {
22 this.timeout(30000) 23 this.timeout(60000)
23 24
24 await flushTests() 25 await flushTests()
25 26
@@ -28,36 +29,43 @@ describe('Test update host scripts', function () {
28 port: 9256 29 port: 9256
29 } 30 }
30 } 31 }
31 server = await runServer(1, overrideConfig) 32 // Run server 2 to have transcoding enabled
33 server = await runServer(2, overrideConfig)
32 await setAccessTokensToServers([ server ]) 34 await setAccessTokensToServers([ server ])
33 35
34 // Upload two videos for our needs 36 // Upload two videos for our needs
35 const videoAttributes = {} 37 const videoAttributes = {}
36 await uploadVideo(server.url, server.accessToken, videoAttributes) 38 await uploadVideo(server.url, server.accessToken, videoAttributes)
37 await uploadVideo(server.url, server.accessToken, videoAttributes) 39 await uploadVideo(server.url, server.accessToken, videoAttributes)
40 await wait(30000)
38 }) 41 })
39 42
40 it('Should update torrent hosts', async function () { 43 it('Should update torrent hosts', async function () {
41 this.timeout(30000) 44 this.timeout(30000)
42 45
43 killallServers([ server ]) 46 killallServers([ server ])
44 server = await runServer(1) 47 // Run server with standard configuration
48 server = await runServer(2)
45 49
46 const env = getEnvCli(server) 50 const env = getEnvCli(server)
47 await execCLI(`${env} npm run update-host`) 51 await execCLI(`${env} npm run update-host`)
48 52
49 const res = await getVideosList(server.url) 53 const res = await getVideosList(server.url)
50 const videos = res.body.data 54 const videos = res.body.data
55 expect(videos).to.have.lengthOf(2)
51 56
52 expect(videos[0].files[0].magnetUri).to.contain('localhost%3A9001%2Ftracker%2Fsocket') 57 for (const video of videos) {
53 expect(videos[0].files[0].magnetUri).to.contain('localhost%3A9001%2Fstatic%2Fwebseed%2F') 58 expect(video.files).to.have.lengthOf(5)
54 59
55 expect(videos[1].files[0].magnetUri).to.contain('localhost%3A9001%2Ftracker%2Fsocket') 60 for (const file of video.files) {
56 expect(videos[1].files[0].magnetUri).to.contain('localhost%3A9001%2Fstatic%2Fwebseed%2F') 61 expect(file.magnetUri).to.contain('localhost%3A9002%2Ftracker%2Fsocket')
62 expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2Fwebseed%2F')
57 63
58 const torrent = await parseTorrentVideo(server, videos[0].uuid) 64 const torrent = await parseTorrentVideo(server, video.uuid, file.resolutionLabel)
59 expect(torrent.announce[0]).to.equal('ws://localhost:9001/tracker/socket') 65 expect(torrent.announce[0]).to.equal('ws://localhost:9002/tracker/socket')
60 expect(torrent.urlList[0]).to.contain('http://localhost:9001/static/webseed') 66 expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/webseed')
67 }
68 }
61 }) 69 })
62 70
63 after(async function () { 71 after(async function () {
diff --git a/server/tests/utils/videos.ts b/server/tests/utils/videos.ts
index 0de506cd9..7f8bd39c0 100644
--- a/server/tests/utils/videos.ts
+++ b/server/tests/utils/videos.ts
@@ -238,9 +238,10 @@ function rateVideo (url: string, accessToken: string, id: number, rating: string
238 .expect(specialStatus) 238 .expect(specialStatus)
239} 239}
240 240
241function parseTorrentVideo (server: ServerInfo, videoUUID: string) { 241function parseTorrentVideo (server: ServerInfo, videoUUID: string, resolutionLabel: string) {
242 return new Promise<any>((res, rej) => { 242 return new Promise<any>((res, rej) => {
243 const torrentPath = join(__dirname, '..', '..', '..', 'test' + server.serverNumber, 'torrents', videoUUID + '.torrent') 243 const torrentName = videoUUID + '-' + resolutionLabel + '.torrent'
244 const torrentPath = join(__dirname, '..', '..', '..', 'test' + server.serverNumber, 'torrents', torrentName)
244 readFile(torrentPath, (err, data) => { 245 readFile(torrentPath, (err, data) => {
245 if (err) return rej(err) 246 if (err) return rej(err)
246 247