aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorFlorent Fayolle <florent.fayolle69@gmail.com>2018-06-02 21:39:41 +0200
committerChocobozzz <me@florianbigard.com>2018-06-07 08:57:48 +0200
commit0138af9237b77dd7d3a49260d164193b4048de84 (patch)
tree5a377c7577cd99d5b324967187e10cd9fcc81383
parent157b62b1f4450c32bb7383ccacbea555ec607013 (diff)
downloadPeerTube-0138af9237b77dd7d3a49260d164193b4048de84.tar.gz
PeerTube-0138af9237b77dd7d3a49260d164193b4048de84.tar.zst
PeerTube-0138af9237b77dd7d3a49260d164193b4048de84.zip
Add create-import-video-file-job command
-rw-r--r--package.json1
-rw-r--r--scripts/create-import-video-file-job.ts39
-rw-r--r--server/helpers/core-utils.ts4
-rw-r--r--server/initializers/constants.ts2
-rw-r--r--server/lib/job-queue/handlers/video-file.ts29
-rw-r--r--server/lib/job-queue/job-queue.ts4
-rw-r--r--server/models/video/video.ts35
-rw-r--r--server/tests/api/fixtures/video_short-480.webmbin0 -> 69217 bytes
-rw-r--r--server/tests/cli/create-import-video-file-job.ts111
-rw-r--r--server/tests/cli/index.ts1
-rw-r--r--shared/models/server/job.model.ts1
11 files changed, 221 insertions, 6 deletions
diff --git a/package.json b/package.json
index 707579af3..4daeecb88 100644
--- a/package.json
+++ b/package.json
@@ -40,6 +40,7 @@
40 "start": "node dist/server", 40 "start": "node dist/server",
41 "update-host": "node ./dist/scripts/update-host.js", 41 "update-host": "node ./dist/scripts/update-host.js",
42 "create-transcoding-job": "node ./dist/scripts/create-transcoding-job.js", 42 "create-transcoding-job": "node ./dist/scripts/create-transcoding-job.js",
43 "create-import-video-file-job": "node ./dist/scripts/create-import-video-file-job.js",
43 "test": "scripty", 44 "test": "scripty",
44 "help": "scripty", 45 "help": "scripty",
45 "generate-api-doc": "scripty", 46 "generate-api-doc": "scripty",
diff --git a/scripts/create-import-video-file-job.ts b/scripts/create-import-video-file-job.ts
new file mode 100644
index 000000000..a2f4f38f2
--- /dev/null
+++ b/scripts/create-import-video-file-job.ts
@@ -0,0 +1,39 @@
1import * as program from 'commander'
2import { resolve } from 'path'
3import { VideoModel } from '../server/models/video/video'
4import { initDatabaseModels } from '../server/initializers'
5import { JobQueue } from '../server/lib/job-queue'
6
7program
8 .option('-v, --video [videoUUID]', 'Video UUID')
9 .option('-i, --import [videoFile]', 'Video file')
10 .description('Import a video file to replace an already uploaded file or to add a new resolution')
11 .parse(process.argv)
12
13if (program['video'] === undefined || program['import'] === undefined) {
14 console.error('All parameters are mandatory.')
15 process.exit(-1)
16}
17
18run()
19 .then(() => process.exit(0))
20 .catch(err => {
21 console.error(err)
22 process.exit(-1)
23 })
24
25async function run () {
26 await initDatabaseModels(true)
27
28 const video = await VideoModel.loadByUUID(program['video'])
29 if (!video) throw new Error('Video not found.')
30
31 const dataInput = {
32 videoUUID: video.uuid,
33 filePath: resolve(program['import'])
34 }
35
36 await JobQueue.Instance.init()
37 await JobQueue.Instance.createJob({ type: 'video-file-import', payload: dataInput })
38 console.log('Import job for video %s created.', video.uuid)
39}
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index a3dfe27b5..c560222d3 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -6,7 +6,7 @@
6import * as bcrypt from 'bcrypt' 6import * as bcrypt from 'bcrypt'
7import * as createTorrent from 'create-torrent' 7import * as createTorrent from 'create-torrent'
8import { pseudoRandomBytes } from 'crypto' 8import { pseudoRandomBytes } from 'crypto'
9import { readdir, readFile, rename, stat, Stats, unlink, writeFile } from 'fs' 9import { copyFile, readdir, readFile, rename, stat, Stats, unlink, writeFile } from 'fs'
10import * as mkdirp from 'mkdirp' 10import * as mkdirp from 'mkdirp'
11import { isAbsolute, join } from 'path' 11import { isAbsolute, join } from 'path'
12import * as pem from 'pem' 12import * as pem from 'pem'
@@ -136,6 +136,7 @@ function promisify2WithVoid<T, U> (func: (arg1: T, arg2: U, cb: (err: any) => vo
136 } 136 }
137} 137}
138 138
139const copyFilePromise = promisify2WithVoid<string, string>(copyFile)
139const readFileBufferPromise = promisify1<string, Buffer>(readFile) 140const readFileBufferPromise = promisify1<string, Buffer>(readFile)
140const unlinkPromise = promisify1WithVoid<string>(unlink) 141const unlinkPromise = promisify1WithVoid<string>(unlink)
141const renamePromise = promisify2WithVoid<string, string>(rename) 142const renamePromise = promisify2WithVoid<string, string>(rename)
@@ -167,6 +168,7 @@ export {
167 promisify0, 168 promisify0,
168 promisify1, 169 promisify1,
169 170
171 copyFilePromise,
170 readdirPromise, 172 readdirPromise,
171 readFileBufferPromise, 173 readFileBufferPromise,
172 unlinkPromise, 174 unlinkPromise,
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 9b459c241..482db2d5c 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -74,6 +74,7 @@ const JOB_ATTEMPTS: { [ id in JobType ]: number } = {
74 'activitypub-http-unicast': 5, 74 'activitypub-http-unicast': 5,
75 'activitypub-http-fetcher': 5, 75 'activitypub-http-fetcher': 5,
76 'activitypub-follow': 5, 76 'activitypub-follow': 5,
77 'video-file-import': 1,
77 'video-file': 1, 78 'video-file': 1,
78 'email': 5 79 'email': 5
79} 80}
@@ -82,6 +83,7 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
82 'activitypub-http-unicast': 5, 83 'activitypub-http-unicast': 5,
83 'activitypub-http-fetcher': 1, 84 'activitypub-http-fetcher': 1,
84 'activitypub-follow': 3, 85 'activitypub-follow': 3,
86 'video-file-import': 1,
85 'video-file': 1, 87 'video-file': 1,
86 'email': 5 88 'email': 5
87} 89}
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts
index 93f9e9fe7..38eb3511c 100644
--- a/server/lib/job-queue/handlers/video-file.ts
+++ b/server/lib/job-queue/handlers/video-file.ts
@@ -16,6 +16,28 @@ export type VideoFilePayload = {
16 isPortraitMode?: boolean 16 isPortraitMode?: boolean
17} 17}
18 18
19export type VideoImportPayload = {
20 videoUUID: string,
21 filePath: string
22}
23
24async function processVideoImport (job: kue.Job) {
25 const payload = job.data as VideoImportPayload
26 logger.info('Processing video import in job %d.', job.id)
27
28 const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID)
29 // No video, maybe deleted?
30 if (!video) {
31 logger.info('Do not process job %d, video does not exist.', job.id, { videoUUID: video.uuid })
32 return undefined
33 }
34
35 await video.importVideoFile(payload.filePath)
36
37 await onVideoFileTranscoderOrImportSuccess(video)
38 return video
39}
40
19async function processVideoFile (job: kue.Job) { 41async function processVideoFile (job: kue.Job) {
20 const payload = job.data as VideoFilePayload 42 const payload = job.data as VideoFilePayload
21 logger.info('Processing video file in job %d.', job.id) 43 logger.info('Processing video file in job %d.', job.id)
@@ -30,7 +52,7 @@ async function processVideoFile (job: kue.Job) {
30 // Transcoding in other resolution 52 // Transcoding in other resolution
31 if (payload.resolution) { 53 if (payload.resolution) {
32 await video.transcodeOriginalVideofile(payload.resolution, payload.isPortraitMode) 54 await video.transcodeOriginalVideofile(payload.resolution, payload.isPortraitMode)
33 await onVideoFileTranscoderSuccess(video) 55 await onVideoFileTranscoderOrImportSuccess(video)
34 } else { 56 } else {
35 await video.optimizeOriginalVideofile() 57 await video.optimizeOriginalVideofile()
36 await onVideoFileOptimizerSuccess(video, payload.isNewVideo) 58 await onVideoFileOptimizerSuccess(video, payload.isNewVideo)
@@ -39,7 +61,7 @@ async function processVideoFile (job: kue.Job) {
39 return video 61 return video
40} 62}
41 63
42async function onVideoFileTranscoderSuccess (video: VideoModel) { 64async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) {
43 if (video === undefined) return undefined 65 if (video === undefined) return undefined
44 66
45 // Maybe the video changed in database, refresh it 67 // Maybe the video changed in database, refresh it
@@ -109,5 +131,6 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole
109// --------------------------------------------------------------------------- 131// ---------------------------------------------------------------------------
110 132
111export { 133export {
112 processVideoFile 134 processVideoFile,
135 processVideoImport
113} 136}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 0333464bd..69335acf0 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -7,7 +7,7 @@ import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from
7import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' 7import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher'
8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' 8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast'
9import { EmailPayload, processEmail } from './handlers/email' 9import { EmailPayload, processEmail } from './handlers/email'
10import { processVideoFile, VideoFilePayload } from './handlers/video-file' 10import { processVideoFile, processVideoImport, VideoFilePayload, VideoImportPayload } from './handlers/video-file'
11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' 11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow'
12 12
13type CreateJobArgument = 13type CreateJobArgument =
@@ -15,6 +15,7 @@ type CreateJobArgument =
15 { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | 15 { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } |
16 { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | 16 { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } |
17 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | 17 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } |
18 { type: 'video-file-import', payload: VideoImportPayload } |
18 { type: 'video-file', payload: VideoFilePayload } | 19 { type: 'video-file', payload: VideoFilePayload } |
19 { type: 'email', payload: EmailPayload } 20 { type: 'email', payload: EmailPayload }
20 21
@@ -23,6 +24,7 @@ const handlers: { [ id in JobType ]: (job: kue.Job) => Promise<any>} = {
23 'activitypub-http-unicast': processActivityPubHttpUnicast, 24 'activitypub-http-unicast': processActivityPubHttpUnicast,
24 'activitypub-http-fetcher': processActivityPubHttpFetcher, 25 'activitypub-http-fetcher': processActivityPubHttpFetcher,
25 'activitypub-follow': processActivityPubFollow, 26 'activitypub-follow': processActivityPubFollow,
27 'video-file-import': processVideoImport,
26 'video-file': processVideoFile, 28 'video-file': processVideoFile,
27 'email': processEmail 29 'email': processEmail
28} 30}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 5821ea397..2875e6685 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -2,7 +2,7 @@ import * as Bluebird from 'bluebird'
2import { map, maxBy } from 'lodash' 2import { map, maxBy } from 'lodash'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import * as parseTorrent from 'parse-torrent' 4import * as parseTorrent from 'parse-torrent'
5import { join } from 'path' 5import { join, extname } from 'path'
6import * as Sequelize from 'sequelize' 6import * as Sequelize from 'sequelize'
7import { 7import {
8 AllowNull, 8 AllowNull,
@@ -32,6 +32,7 @@ import { VideoFilter } from '../../../shared/models/videos/video-query.type'
32import { 32import {
33 createTorrentPromise, 33 createTorrentPromise,
34 peertubeTruncate, 34 peertubeTruncate,
35 copyFilePromise,
35 renamePromise, 36 renamePromise,
36 statPromise, 37 statPromise,
37 unlinkPromise, 38 unlinkPromise,
@@ -1315,6 +1316,38 @@ export class VideoModel extends Model<VideoModel> {
1315 this.VideoFiles.push(newVideoFile) 1316 this.VideoFiles.push(newVideoFile)
1316 } 1317 }
1317 1318
1319 async importVideoFile (inputFilePath: string) {
1320 let updatedVideoFile = new VideoFileModel({
1321 resolution: (await getVideoFileResolution(inputFilePath)).videoFileResolution,
1322 extname: extname(inputFilePath),
1323 size: (await statPromise(inputFilePath)).size,
1324 videoId: this.id
1325 })
1326
1327 const outputPath = this.getVideoFilePath(updatedVideoFile)
1328 await copyFilePromise(inputFilePath, outputPath)
1329
1330 const currentVideoFile = this.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution)
1331 const isNewVideoFile = !currentVideoFile
1332
1333 if (!isNewVideoFile) {
1334 if (currentVideoFile.extname !== updatedVideoFile.extname) {
1335 await this.removeFile(currentVideoFile)
1336 currentVideoFile.set('extname', updatedVideoFile.extname)
1337 }
1338 currentVideoFile.set('size', updatedVideoFile.size)
1339 updatedVideoFile = currentVideoFile
1340 }
1341
1342 await this.createTorrentAndSetInfoHash(updatedVideoFile)
1343
1344 await updatedVideoFile.save()
1345
1346 if (isNewVideoFile) {
1347 this.VideoFiles.push(updatedVideoFile)
1348 }
1349 }
1350
1318 getOriginalFileResolution () { 1351 getOriginalFileResolution () {
1319 const originalFilePath = this.getVideoFilePath(this.getOriginalFile()) 1352 const originalFilePath = this.getVideoFilePath(this.getOriginalFile())
1320 1353
diff --git a/server/tests/api/fixtures/video_short-480.webm b/server/tests/api/fixtures/video_short-480.webm
new file mode 100644
index 000000000..3145105e1
--- /dev/null
+++ b/server/tests/api/fixtures/video_short-480.webm
Binary files differ
diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts
new file mode 100644
index 000000000..d486db600
--- /dev/null
+++ b/server/tests/cli/create-import-video-file-job.ts
@@ -0,0 +1,111 @@
1/* tslint:disable:no-unused-expression */
2
3import 'mocha'
4import * as chai from 'chai'
5import { VideoDetails, VideoFile } from '../../../shared/models/videos'
6const expect = chai.expect
7
8import {
9 execCLI,
10 flushTests,
11 getEnvCli,
12 getVideosList,
13 killallServers,
14 parseTorrentVideo,
15 runServer,
16 ServerInfo,
17 setAccessTokensToServers,
18 uploadVideo,
19 wait,
20 getVideo, flushAndRunMultipleServers, doubleFollow
21} from '../utils'
22
23function assertVideoProperties (video: VideoFile, resolution: number, extname: string) {
24 expect(video).to.have.nested.property('resolution.id', resolution)
25 expect(video).to.have.property('magnetUri').that.includes(`.${extname}`)
26 expect(video).to.have.property('torrentUrl').that.includes(`-${resolution}.torrent`)
27 expect(video).to.have.property('fileUrl').that.includes(`.${extname}`)
28 expect(video).to.have.property('size').that.is.above(0)
29}
30
31describe('Test create import video jobs', function () {
32 this.timeout(60000)
33
34 let servers: ServerInfo[] = []
35 let video1UUID: string
36 let video2UUID: string
37
38 before(async function () {
39 this.timeout(90000)
40 await flushTests()
41
42 // Run server 2 to have transcoding enabled
43 servers = await flushAndRunMultipleServers(2)
44 await setAccessTokensToServers(servers)
45
46 await doubleFollow(servers[0], servers[1])
47
48 // Upload two videos for our needs
49 const res1 = await uploadVideo(servers[0].url, servers[0].accessToken, { name: 'video1' })
50 video1UUID = res1.body.video.uuid
51 const res2 = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video2' })
52 video2UUID = res2.body.video.uuid
53
54 await wait(40000)
55 })
56
57 it('Should run a import job on video 1 with a lower resolution', async function () {
58 const env = getEnvCli(servers[0])
59 await execCLI(`${env} npm run create-import-video-file-job -- -v ${video1UUID} -i server/tests/api/fixtures/video_short-480.webm`)
60
61 await wait(30000)
62
63 for (const server of servers) {
64 const { data: videos } = (await getVideosList(server.url)).body
65 expect(videos).to.have.lengthOf(2)
66
67 let infoHashes: { [ id: number ]: string } = {}
68
69 const video = videos.find(({ uuid }) => uuid === video1UUID)
70 const videoDetail: VideoDetails = (await getVideo(server.url, video.uuid)).body
71
72 expect(videoDetail.files).to.have.lengthOf(2)
73 const [originalVideo, transcodedVideo] = videoDetail.files
74 assertVideoProperties(originalVideo, 720, 'webm')
75 assertVideoProperties(transcodedVideo, 480, 'webm')
76 }
77 })
78
79 it('Should run a import job on video 2 with the same resolution', async function () {
80 const env = getEnvCli(servers[1])
81 await execCLI(`${env} npm run create-import-video-file-job -- -v ${video2UUID} -i server/tests/api/fixtures/video_short.ogv`)
82
83 await wait(30000)
84
85 for (const server of servers.reverse()) {
86 const { data: videos } = (await getVideosList(server.url)).body
87 expect(videos).to.have.lengthOf(2)
88
89 let infoHashes: { [ id: number ]: string }
90
91 const video = videos.find(({ uuid }) => uuid === video2UUID)
92 const videoDetail: VideoDetails = (await getVideo(server.url, video.uuid)).body
93
94 expect(videoDetail.files).to.have.lengthOf(4)
95 const [originalVideo, transcodedVideo420, transcodedVideo320, transcodedVideo240] = videoDetail.files
96 assertVideoProperties(originalVideo, 720, 'ogv')
97 assertVideoProperties(transcodedVideo420, 480, 'mp4')
98 assertVideoProperties(transcodedVideo320, 360, 'mp4')
99 assertVideoProperties(transcodedVideo240, 240, 'mp4')
100 }
101 })
102
103 after(async function () {
104 killallServers(servers)
105
106 // Keep the logs if the test failed
107 if (this['ok']) {
108 await flushTests()
109 }
110 })
111})
diff --git a/server/tests/cli/index.ts b/server/tests/cli/index.ts
index f0317aac0..f99eafe03 100644
--- a/server/tests/cli/index.ts
+++ b/server/tests/cli/index.ts
@@ -1,4 +1,5 @@
1// Order of the tests we want to execute 1// Order of the tests we want to execute
2import './create-transcoding-job' 2import './create-transcoding-job'
3import './create-import-video-file-job'
3import './reset-password' 4import './reset-password'
4import './update-host' 5import './update-host'
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index 0fa36820e..7d8d39a19 100644
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -4,6 +4,7 @@ export type JobType = 'activitypub-http-unicast' |
4 'activitypub-http-broadcast' | 4 'activitypub-http-broadcast' |
5 'activitypub-http-fetcher' | 5 'activitypub-http-fetcher' |
6 'activitypub-follow' | 6 'activitypub-follow' |
7 'video-file-import' |
7 'video-file' | 8 'video-file' |
8 'email' 9 'email'
9 10