aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-06-07 09:43:18 +0200
committerChocobozzz <me@florianbigard.com>2018-06-07 09:43:18 +0200
commit28be89161aab245526d64f6fb7dd29391a97fe0a (patch)
tree1ee8bece60eea295f218cb5936141848f3a470fc /server
parent0138af9237b77dd7d3a49260d164193b4048de84 (diff)
downloadPeerTube-28be89161aab245526d64f6fb7dd29391a97fe0a.tar.gz
PeerTube-28be89161aab245526d64f6fb7dd29391a97fe0a.tar.zst
PeerTube-28be89161aab245526d64f6fb7dd29391a97fe0a.zip
Improve create import file job
Fix federation of .ogv videos
Diffstat (limited to 'server')
-rw-r--r--server/helpers/custom-validators/videos.ts4
-rw-r--r--server/initializers/constants.ts3
-rw-r--r--server/lib/job-queue/handlers/video-file.ts10
-rw-r--r--server/lib/job-queue/job-queue.ts6
-rw-r--r--server/models/video/video.ts27
-rw-r--r--server/tests/cli/create-import-video-file-job.ts31
6 files changed, 47 insertions, 34 deletions
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index 0c268a684..f365df985 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -7,8 +7,8 @@ import { UserRight, VideoRateType } from '../../../shared'
7import { 7import {
8 CONSTRAINTS_FIELDS, 8 CONSTRAINTS_FIELDS,
9 VIDEO_CATEGORIES, 9 VIDEO_CATEGORIES,
10 VIDEO_LANGUAGES, 10 VIDEO_LICENCES,
11 VIDEO_LICENCES, VIDEO_MIMETYPE_EXT, 11 VIDEO_MIMETYPE_EXT,
12 VIDEO_PRIVACIES, 12 VIDEO_PRIVACIES,
13 VIDEO_RATE_TYPES 13 VIDEO_RATE_TYPES
14} from '../../initializers' 14} from '../../initializers'
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 482db2d5c..54f22c95a 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -7,6 +7,7 @@ import { VideoPrivacy } from '../../shared/models/videos'
7// Do not use barrels, remain constants as independent as possible 7// Do not use barrels, remain constants as independent as possible
8import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' 8import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
9import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' 9import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
10import { invert } from 'lodash'
10 11
11// Use a variable to reload the configuration if we need 12// Use a variable to reload the configuration if we need
12let config: IConfig = require('config') 13let config: IConfig = require('config')
@@ -330,6 +331,7 @@ const VIDEO_MIMETYPE_EXT = {
330 'video/ogg': '.ogv', 331 'video/ogg': '.ogv',
331 'video/mp4': '.mp4' 332 'video/mp4': '.mp4'
332} 333}
334const VIDEO_EXT_MIMETYPE = invert(VIDEO_MIMETYPE_EXT)
333 335
334const IMAGE_MIMETYPE_EXT = { 336const IMAGE_MIMETYPE_EXT = {
335 'image/png': '.png', 337 'image/png': '.png',
@@ -501,6 +503,7 @@ export {
501 SCHEDULER_INTERVAL, 503 SCHEDULER_INTERVAL,
502 STATIC_DOWNLOAD_PATHS, 504 STATIC_DOWNLOAD_PATHS,
503 RATES_LIMIT, 505 RATES_LIMIT,
506 VIDEO_EXT_MIMETYPE,
504 JOB_COMPLETED_LIFETIME, 507 JOB_COMPLETED_LIFETIME,
505 VIDEO_VIEW_LIFETIME, 508 VIDEO_VIEW_LIFETIME,
506 buildLanguages 509 buildLanguages
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts
index 38eb3511c..a6fce4279 100644
--- a/server/lib/job-queue/handlers/video-file.ts
+++ b/server/lib/job-queue/handlers/video-file.ts
@@ -16,14 +16,14 @@ export type VideoFilePayload = {
16 isPortraitMode?: boolean 16 isPortraitMode?: boolean
17} 17}
18 18
19export type VideoImportPayload = { 19export type VideoFileImportPayload = {
20 videoUUID: string, 20 videoUUID: string,
21 filePath: string 21 filePath: string
22} 22}
23 23
24async function processVideoImport (job: kue.Job) { 24async function processVideoFileImport (job: kue.Job) {
25 const payload = job.data as VideoImportPayload 25 const payload = job.data as VideoFileImportPayload
26 logger.info('Processing video import in job %d.', job.id) 26 logger.info('Processing video file import in job %d.', job.id)
27 27
28 const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID) 28 const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID)
29 // No video, maybe deleted? 29 // No video, maybe deleted?
@@ -132,5 +132,5 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole
132 132
133export { 133export {
134 processVideoFile, 134 processVideoFile,
135 processVideoImport 135 processVideoFileImport
136} 136}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 69335acf0..bdfa19b61 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -7,7 +7,7 @@ import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from
7import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' 7import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher'
8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' 8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast'
9import { EmailPayload, processEmail } from './handlers/email' 9import { EmailPayload, processEmail } from './handlers/email'
10import { processVideoFile, processVideoImport, VideoFilePayload, VideoImportPayload } from './handlers/video-file' 10import { processVideoFile, processVideoFileImport, VideoFilePayload, VideoFileImportPayload } from './handlers/video-file'
11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' 11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow'
12 12
13type CreateJobArgument = 13type CreateJobArgument =
@@ -15,7 +15,7 @@ type CreateJobArgument =
15 { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | 15 { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } |
16 { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | 16 { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } |
17 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | 17 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } |
18 { type: 'video-file-import', payload: VideoImportPayload } | 18 { type: 'video-file-import', payload: VideoFileImportPayload } |
19 { type: 'video-file', payload: VideoFilePayload } | 19 { type: 'video-file', payload: VideoFilePayload } |
20 { type: 'email', payload: EmailPayload } 20 { type: 'email', payload: EmailPayload }
21 21
@@ -24,7 +24,7 @@ const handlers: { [ id in JobType ]: (job: kue.Job) => Promise<any>} = {
24 'activitypub-http-unicast': processActivityPubHttpUnicast, 24 'activitypub-http-unicast': processActivityPubHttpUnicast,
25 'activitypub-http-fetcher': processActivityPubHttpFetcher, 25 'activitypub-http-fetcher': processActivityPubHttpFetcher,
26 'activitypub-follow': processActivityPubFollow, 26 'activitypub-follow': processActivityPubFollow,
27 'video-file-import': processVideoImport, 27 'video-file-import': processVideoFileImport,
28 'video-file': processVideoFile, 28 'video-file': processVideoFile,
29 'email': processEmail 29 'email': processEmail
30} 30}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 2875e6685..faa6f3f9b 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -2,7 +2,7 @@ import * as Bluebird from 'bluebird'
2import { map, maxBy } from 'lodash' 2import { map, maxBy } from 'lodash'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import * as parseTorrent from 'parse-torrent' 4import * as parseTorrent from 'parse-torrent'
5import { join, extname } from 'path' 5import { extname, join } from 'path'
6import * as Sequelize from 'sequelize' 6import * as Sequelize from 'sequelize'
7import { 7import {
8 AllowNull, 8 AllowNull,
@@ -30,9 +30,9 @@ import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
30import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' 30import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
31import { VideoFilter } from '../../../shared/models/videos/video-query.type' 31import { VideoFilter } from '../../../shared/models/videos/video-query.type'
32import { 32import {
33 copyFilePromise,
33 createTorrentPromise, 34 createTorrentPromise,
34 peertubeTruncate, 35 peertubeTruncate,
35 copyFilePromise,
36 renamePromise, 36 renamePromise,
37 statPromise, 37 statPromise,
38 unlinkPromise, 38 unlinkPromise,
@@ -63,6 +63,7 @@ import {
63 STATIC_PATHS, 63 STATIC_PATHS,
64 THUMBNAILS_SIZE, 64 THUMBNAILS_SIZE,
65 VIDEO_CATEGORIES, 65 VIDEO_CATEGORIES,
66 VIDEO_EXT_MIMETYPE,
66 VIDEO_LANGUAGES, 67 VIDEO_LANGUAGES,
67 VIDEO_LICENCES, 68 VIDEO_LICENCES,
68 VIDEO_PRIVACIES 69 VIDEO_PRIVACIES
@@ -1166,7 +1167,7 @@ export class VideoModel extends Model<VideoModel> {
1166 for (const file of this.VideoFiles) { 1167 for (const file of this.VideoFiles) {
1167 url.push({ 1168 url.push({
1168 type: 'Link', 1169 type: 'Link',
1169 mimeType: 'video/' + file.extname.replace('.', ''), 1170 mimeType: VIDEO_EXT_MIMETYPE[file.extname],
1170 href: this.getVideoFileUrl(file, baseUrlHttp), 1171 href: this.getVideoFileUrl(file, baseUrlHttp),
1171 width: file.resolution, 1172 width: file.resolution,
1172 size: file.size 1173 size: file.size
@@ -1328,14 +1329,18 @@ export class VideoModel extends Model<VideoModel> {
1328 await copyFilePromise(inputFilePath, outputPath) 1329 await copyFilePromise(inputFilePath, outputPath)
1329 1330
1330 const currentVideoFile = this.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution) 1331 const currentVideoFile = this.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution)
1331 const isNewVideoFile = !currentVideoFile
1332 1332
1333 if (!isNewVideoFile) { 1333 if (currentVideoFile) {
1334 if (currentVideoFile.extname !== updatedVideoFile.extname) { 1334 // Remove old file and old torrent
1335 await this.removeFile(currentVideoFile) 1335 await this.removeFile(currentVideoFile)
1336 currentVideoFile.set('extname', updatedVideoFile.extname) 1336 await this.removeTorrent(currentVideoFile)
1337 } 1337 // Remove the old video file from the array
1338 this.VideoFiles = this.VideoFiles.filter(f => f !== currentVideoFile)
1339
1340 // Update the database
1341 currentVideoFile.set('extname', updatedVideoFile.extname)
1338 currentVideoFile.set('size', updatedVideoFile.size) 1342 currentVideoFile.set('size', updatedVideoFile.size)
1343
1339 updatedVideoFile = currentVideoFile 1344 updatedVideoFile = currentVideoFile
1340 } 1345 }
1341 1346
@@ -1343,9 +1348,7 @@ export class VideoModel extends Model<VideoModel> {
1343 1348
1344 await updatedVideoFile.save() 1349 await updatedVideoFile.save()
1345 1350
1346 if (isNewVideoFile) { 1351 this.VideoFiles.push(updatedVideoFile)
1347 this.VideoFiles.push(updatedVideoFile)
1348 }
1349 } 1352 }
1350 1353
1351 getOriginalFileResolution () { 1354 getOriginalFileResolution () {
diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts
index d486db600..251088882 100644
--- a/server/tests/cli/create-import-video-file-job.ts
+++ b/server/tests/cli/create-import-video-file-job.ts
@@ -3,29 +3,31 @@
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { VideoDetails, VideoFile } from '../../../shared/models/videos' 5import { VideoDetails, VideoFile } from '../../../shared/models/videos'
6const expect = chai.expect
7
8import { 6import {
7 doubleFollow,
9 execCLI, 8 execCLI,
9 flushAndRunMultipleServers,
10 flushTests, 10 flushTests,
11 getEnvCli, 11 getEnvCli,
12 getVideo,
12 getVideosList, 13 getVideosList,
13 killallServers, 14 killallServers,
14 parseTorrentVideo,
15 runServer,
16 ServerInfo, 15 ServerInfo,
17 setAccessTokensToServers, 16 setAccessTokensToServers,
18 uploadVideo, 17 uploadVideo,
19 wait, 18 wait
20 getVideo, flushAndRunMultipleServers, doubleFollow
21} from '../utils' 19} from '../utils'
22 20
23function assertVideoProperties (video: VideoFile, resolution: number, extname: string) { 21const expect = chai.expect
22
23function assertVideoProperties (video: VideoFile, resolution: number, extname: string, size?: number) {
24 expect(video).to.have.nested.property('resolution.id', resolution) 24 expect(video).to.have.nested.property('resolution.id', resolution)
25 expect(video).to.have.property('magnetUri').that.includes(`.${extname}`) 25 expect(video).to.have.property('magnetUri').that.includes(`.${extname}`)
26 expect(video).to.have.property('torrentUrl').that.includes(`-${resolution}.torrent`) 26 expect(video).to.have.property('torrentUrl').that.includes(`-${resolution}.torrent`)
27 expect(video).to.have.property('fileUrl').that.includes(`.${extname}`) 27 expect(video).to.have.property('fileUrl').that.includes(`.${extname}`)
28 expect(video).to.have.property('size').that.is.above(0) 28 expect(video).to.have.property('size').that.is.above(0)
29
30 if (size) expect(video.size).to.equal(size)
29} 31}
30 32
31describe('Test create import video jobs', function () { 33describe('Test create import video jobs', function () {
@@ -51,6 +53,7 @@ describe('Test create import video jobs', function () {
51 const res2 = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video2' }) 53 const res2 = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video2' })
52 video2UUID = res2.body.video.uuid 54 video2UUID = res2.body.video.uuid
53 55
56 // Transcoding
54 await wait(40000) 57 await wait(40000)
55 }) 58 })
56 59
@@ -60,12 +63,11 @@ describe('Test create import video jobs', function () {
60 63
61 await wait(30000) 64 await wait(30000)
62 65
66 let magnetUri: string
63 for (const server of servers) { 67 for (const server of servers) {
64 const { data: videos } = (await getVideosList(server.url)).body 68 const { data: videos } = (await getVideosList(server.url)).body
65 expect(videos).to.have.lengthOf(2) 69 expect(videos).to.have.lengthOf(2)
66 70
67 let infoHashes: { [ id: number ]: string } = {}
68
69 const video = videos.find(({ uuid }) => uuid === video1UUID) 71 const video = videos.find(({ uuid }) => uuid === video1UUID)
70 const videoDetail: VideoDetails = (await getVideo(server.url, video.uuid)).body 72 const videoDetail: VideoDetails = (await getVideo(server.url, video.uuid)).body
71 73
@@ -73,6 +75,9 @@ describe('Test create import video jobs', function () {
73 const [originalVideo, transcodedVideo] = videoDetail.files 75 const [originalVideo, transcodedVideo] = videoDetail.files
74 assertVideoProperties(originalVideo, 720, 'webm') 76 assertVideoProperties(originalVideo, 720, 'webm')
75 assertVideoProperties(transcodedVideo, 480, 'webm') 77 assertVideoProperties(transcodedVideo, 480, 'webm')
78
79 if (!magnetUri) magnetUri = transcodedVideo.magnetUri
80 else expect(transcodedVideo.magnetUri).to.equal(magnetUri)
76 } 81 }
77 }) 82 })
78 83
@@ -82,21 +87,23 @@ describe('Test create import video jobs', function () {
82 87
83 await wait(30000) 88 await wait(30000)
84 89
90 let magnetUri: string
85 for (const server of servers.reverse()) { 91 for (const server of servers.reverse()) {
86 const { data: videos } = (await getVideosList(server.url)).body 92 const { data: videos } = (await getVideosList(server.url)).body
87 expect(videos).to.have.lengthOf(2) 93 expect(videos).to.have.lengthOf(2)
88 94
89 let infoHashes: { [ id: number ]: string }
90
91 const video = videos.find(({ uuid }) => uuid === video2UUID) 95 const video = videos.find(({ uuid }) => uuid === video2UUID)
92 const videoDetail: VideoDetails = (await getVideo(server.url, video.uuid)).body 96 const videoDetail: VideoDetails = (await getVideo(server.url, video.uuid)).body
93 97
94 expect(videoDetail.files).to.have.lengthOf(4) 98 expect(videoDetail.files).to.have.lengthOf(4)
95 const [originalVideo, transcodedVideo420, transcodedVideo320, transcodedVideo240] = videoDetail.files 99 const [originalVideo, transcodedVideo420, transcodedVideo320, transcodedVideo240] = videoDetail.files
96 assertVideoProperties(originalVideo, 720, 'ogv') 100 assertVideoProperties(originalVideo, 720, 'ogv', 140849)
97 assertVideoProperties(transcodedVideo420, 480, 'mp4') 101 assertVideoProperties(transcodedVideo420, 480, 'mp4')
98 assertVideoProperties(transcodedVideo320, 360, 'mp4') 102 assertVideoProperties(transcodedVideo320, 360, 'mp4')
99 assertVideoProperties(transcodedVideo240, 240, 'mp4') 103 assertVideoProperties(transcodedVideo240, 240, 'mp4')
104
105 if (!magnetUri) magnetUri = originalVideo.magnetUri
106 else expect(originalVideo.magnetUri).to.equal(magnetUri)
100 } 107 }
101 }) 108 })
102 109