diff options
-rw-r--r-- | scripts/create-import-video-file-job.ts | 1 | ||||
-rw-r--r-- | server/helpers/custom-validators/videos.ts | 4 | ||||
-rw-r--r-- | server/initializers/constants.ts | 3 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/video-file.ts | 10 | ||||
-rw-r--r-- | server/lib/job-queue/job-queue.ts | 6 | ||||
-rw-r--r-- | server/models/video/video.ts | 27 | ||||
-rw-r--r-- | server/tests/cli/create-import-video-file-job.ts | 31 |
7 files changed, 48 insertions, 34 deletions
diff --git a/scripts/create-import-video-file-job.ts b/scripts/create-import-video-file-job.ts index a2f4f38f2..2b636014a 100644 --- a/scripts/create-import-video-file-job.ts +++ b/scripts/create-import-video-file-job.ts | |||
@@ -27,6 +27,7 @@ async function run () { | |||
27 | 27 | ||
28 | const video = await VideoModel.loadByUUID(program['video']) | 28 | const video = await VideoModel.loadByUUID(program['video']) |
29 | if (!video) throw new Error('Video not found.') | 29 | if (!video) throw new Error('Video not found.') |
30 | if (video.isOwned() === false) throw new Error('Cannot import files of a non owned video.') | ||
30 | 31 | ||
31 | const dataInput = { | 32 | const dataInput = { |
32 | videoUUID: video.uuid, | 33 | videoUUID: video.uuid, |
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index 0c268a684..f365df985 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts | |||
@@ -7,8 +7,8 @@ import { UserRight, VideoRateType } from '../../../shared' | |||
7 | import { | 7 | import { |
8 | CONSTRAINTS_FIELDS, | 8 | CONSTRAINTS_FIELDS, |
9 | VIDEO_CATEGORIES, | 9 | VIDEO_CATEGORIES, |
10 | VIDEO_LANGUAGES, | 10 | VIDEO_LICENCES, |
11 | VIDEO_LICENCES, VIDEO_MIMETYPE_EXT, | 11 | VIDEO_MIMETYPE_EXT, |
12 | VIDEO_PRIVACIES, | 12 | VIDEO_PRIVACIES, |
13 | VIDEO_RATE_TYPES | 13 | VIDEO_RATE_TYPES |
14 | } from '../../initializers' | 14 | } from '../../initializers' |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 482db2d5c..54f22c95a 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -7,6 +7,7 @@ import { VideoPrivacy } from '../../shared/models/videos' | |||
7 | // Do not use barrels, remain constants as independent as possible | 7 | // Do not use barrels, remain constants as independent as possible |
8 | import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' | 8 | import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' |
9 | import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' | 9 | import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' |
10 | import { invert } from 'lodash' | ||
10 | 11 | ||
11 | // Use a variable to reload the configuration if we need | 12 | // Use a variable to reload the configuration if we need |
12 | let config: IConfig = require('config') | 13 | let config: IConfig = require('config') |
@@ -330,6 +331,7 @@ const VIDEO_MIMETYPE_EXT = { | |||
330 | 'video/ogg': '.ogv', | 331 | 'video/ogg': '.ogv', |
331 | 'video/mp4': '.mp4' | 332 | 'video/mp4': '.mp4' |
332 | } | 333 | } |
334 | const VIDEO_EXT_MIMETYPE = invert(VIDEO_MIMETYPE_EXT) | ||
333 | 335 | ||
334 | const IMAGE_MIMETYPE_EXT = { | 336 | const IMAGE_MIMETYPE_EXT = { |
335 | 'image/png': '.png', | 337 | 'image/png': '.png', |
@@ -501,6 +503,7 @@ export { | |||
501 | SCHEDULER_INTERVAL, | 503 | SCHEDULER_INTERVAL, |
502 | STATIC_DOWNLOAD_PATHS, | 504 | STATIC_DOWNLOAD_PATHS, |
503 | RATES_LIMIT, | 505 | RATES_LIMIT, |
506 | VIDEO_EXT_MIMETYPE, | ||
504 | JOB_COMPLETED_LIFETIME, | 507 | JOB_COMPLETED_LIFETIME, |
505 | VIDEO_VIEW_LIFETIME, | 508 | VIDEO_VIEW_LIFETIME, |
506 | buildLanguages | 509 | buildLanguages |
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 38eb3511c..a6fce4279 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts | |||
@@ -16,14 +16,14 @@ export type VideoFilePayload = { | |||
16 | isPortraitMode?: boolean | 16 | isPortraitMode?: boolean |
17 | } | 17 | } |
18 | 18 | ||
19 | export type VideoImportPayload = { | 19 | export type VideoFileImportPayload = { |
20 | videoUUID: string, | 20 | videoUUID: string, |
21 | filePath: string | 21 | filePath: string |
22 | } | 22 | } |
23 | 23 | ||
24 | async function processVideoImport (job: kue.Job) { | 24 | async function processVideoFileImport (job: kue.Job) { |
25 | const payload = job.data as VideoImportPayload | 25 | const payload = job.data as VideoFileImportPayload |
26 | logger.info('Processing video import in job %d.', job.id) | 26 | logger.info('Processing video file import in job %d.', job.id) |
27 | 27 | ||
28 | const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID) | 28 | const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID) |
29 | // No video, maybe deleted? | 29 | // No video, maybe deleted? |
@@ -132,5 +132,5 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole | |||
132 | 132 | ||
133 | export { | 133 | export { |
134 | processVideoFile, | 134 | processVideoFile, |
135 | processVideoImport | 135 | processVideoFileImport |
136 | } | 136 | } |
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 69335acf0..bdfa19b61 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -7,7 +7,7 @@ import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from | |||
7 | import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' | 7 | import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' |
8 | import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' | 8 | import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' |
9 | import { EmailPayload, processEmail } from './handlers/email' | 9 | import { EmailPayload, processEmail } from './handlers/email' |
10 | import { processVideoFile, processVideoImport, VideoFilePayload, VideoImportPayload } from './handlers/video-file' | 10 | import { processVideoFile, processVideoFileImport, VideoFilePayload, VideoFileImportPayload } from './handlers/video-file' |
11 | import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' | 11 | import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' |
12 | 12 | ||
13 | type CreateJobArgument = | 13 | type CreateJobArgument = |
@@ -15,7 +15,7 @@ type CreateJobArgument = | |||
15 | { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | | 15 | { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | |
16 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | | 16 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | |
17 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | | 17 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | |
18 | { type: 'video-file-import', payload: VideoImportPayload } | | 18 | { type: 'video-file-import', payload: VideoFileImportPayload } | |
19 | { type: 'video-file', payload: VideoFilePayload } | | 19 | { type: 'video-file', payload: VideoFilePayload } | |
20 | { type: 'email', payload: EmailPayload } | 20 | { type: 'email', payload: EmailPayload } |
21 | 21 | ||
@@ -24,7 +24,7 @@ const handlers: { [ id in JobType ]: (job: kue.Job) => Promise<any>} = { | |||
24 | 'activitypub-http-unicast': processActivityPubHttpUnicast, | 24 | 'activitypub-http-unicast': processActivityPubHttpUnicast, |
25 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, | 25 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, |
26 | 'activitypub-follow': processActivityPubFollow, | 26 | 'activitypub-follow': processActivityPubFollow, |
27 | 'video-file-import': processVideoImport, | 27 | 'video-file-import': processVideoFileImport, |
28 | 'video-file': processVideoFile, | 28 | 'video-file': processVideoFile, |
29 | 'email': processEmail | 29 | 'email': processEmail |
30 | } | 30 | } |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 2875e6685..faa6f3f9b 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -2,7 +2,7 @@ import * as Bluebird from 'bluebird' | |||
2 | import { map, maxBy } from 'lodash' | 2 | import { map, maxBy } from 'lodash' |
3 | import * as magnetUtil from 'magnet-uri' | 3 | import * as magnetUtil from 'magnet-uri' |
4 | import * as parseTorrent from 'parse-torrent' | 4 | import * as parseTorrent from 'parse-torrent' |
5 | import { join, extname } from 'path' | 5 | import { extname, join } from 'path' |
6 | import * as Sequelize from 'sequelize' | 6 | import * as Sequelize from 'sequelize' |
7 | import { | 7 | import { |
8 | AllowNull, | 8 | AllowNull, |
@@ -30,9 +30,9 @@ import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | |||
30 | import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' | 30 | import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' |
31 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' | 31 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' |
32 | import { | 32 | import { |
33 | copyFilePromise, | ||
33 | createTorrentPromise, | 34 | createTorrentPromise, |
34 | peertubeTruncate, | 35 | peertubeTruncate, |
35 | copyFilePromise, | ||
36 | renamePromise, | 36 | renamePromise, |
37 | statPromise, | 37 | statPromise, |
38 | unlinkPromise, | 38 | unlinkPromise, |
@@ -63,6 +63,7 @@ import { | |||
63 | STATIC_PATHS, | 63 | STATIC_PATHS, |
64 | THUMBNAILS_SIZE, | 64 | THUMBNAILS_SIZE, |
65 | VIDEO_CATEGORIES, | 65 | VIDEO_CATEGORIES, |
66 | VIDEO_EXT_MIMETYPE, | ||
66 | VIDEO_LANGUAGES, | 67 | VIDEO_LANGUAGES, |
67 | VIDEO_LICENCES, | 68 | VIDEO_LICENCES, |
68 | VIDEO_PRIVACIES | 69 | VIDEO_PRIVACIES |
@@ -1166,7 +1167,7 @@ export class VideoModel extends Model<VideoModel> { | |||
1166 | for (const file of this.VideoFiles) { | 1167 | for (const file of this.VideoFiles) { |
1167 | url.push({ | 1168 | url.push({ |
1168 | type: 'Link', | 1169 | type: 'Link', |
1169 | mimeType: 'video/' + file.extname.replace('.', ''), | 1170 | mimeType: VIDEO_EXT_MIMETYPE[file.extname], |
1170 | href: this.getVideoFileUrl(file, baseUrlHttp), | 1171 | href: this.getVideoFileUrl(file, baseUrlHttp), |
1171 | width: file.resolution, | 1172 | width: file.resolution, |
1172 | size: file.size | 1173 | size: file.size |
@@ -1328,14 +1329,18 @@ export class VideoModel extends Model<VideoModel> { | |||
1328 | await copyFilePromise(inputFilePath, outputPath) | 1329 | await copyFilePromise(inputFilePath, outputPath) |
1329 | 1330 | ||
1330 | const currentVideoFile = this.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution) | 1331 | const currentVideoFile = this.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution) |
1331 | const isNewVideoFile = !currentVideoFile | ||
1332 | 1332 | ||
1333 | if (!isNewVideoFile) { | 1333 | if (currentVideoFile) { |
1334 | if (currentVideoFile.extname !== updatedVideoFile.extname) { | 1334 | // Remove old file and old torrent |
1335 | await this.removeFile(currentVideoFile) | 1335 | await this.removeFile(currentVideoFile) |
1336 | currentVideoFile.set('extname', updatedVideoFile.extname) | 1336 | await this.removeTorrent(currentVideoFile) |
1337 | } | 1337 | // Remove the old video file from the array |
1338 | this.VideoFiles = this.VideoFiles.filter(f => f !== currentVideoFile) | ||
1339 | |||
1340 | // Update the database | ||
1341 | currentVideoFile.set('extname', updatedVideoFile.extname) | ||
1338 | currentVideoFile.set('size', updatedVideoFile.size) | 1342 | currentVideoFile.set('size', updatedVideoFile.size) |
1343 | |||
1339 | updatedVideoFile = currentVideoFile | 1344 | updatedVideoFile = currentVideoFile |
1340 | } | 1345 | } |
1341 | 1346 | ||
@@ -1343,9 +1348,7 @@ export class VideoModel extends Model<VideoModel> { | |||
1343 | 1348 | ||
1344 | await updatedVideoFile.save() | 1349 | await updatedVideoFile.save() |
1345 | 1350 | ||
1346 | if (isNewVideoFile) { | 1351 | this.VideoFiles.push(updatedVideoFile) |
1347 | this.VideoFiles.push(updatedVideoFile) | ||
1348 | } | ||
1349 | } | 1352 | } |
1350 | 1353 | ||
1351 | getOriginalFileResolution () { | 1354 | getOriginalFileResolution () { |
diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts index d486db600..251088882 100644 --- a/server/tests/cli/create-import-video-file-job.ts +++ b/server/tests/cli/create-import-video-file-job.ts | |||
@@ -3,29 +3,31 @@ | |||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import { VideoDetails, VideoFile } from '../../../shared/models/videos' | 5 | import { VideoDetails, VideoFile } from '../../../shared/models/videos' |
6 | const expect = chai.expect | ||
7 | |||
8 | import { | 6 | import { |
7 | doubleFollow, | ||
9 | execCLI, | 8 | execCLI, |
9 | flushAndRunMultipleServers, | ||
10 | flushTests, | 10 | flushTests, |
11 | getEnvCli, | 11 | getEnvCli, |
12 | getVideo, | ||
12 | getVideosList, | 13 | getVideosList, |
13 | killallServers, | 14 | killallServers, |
14 | parseTorrentVideo, | ||
15 | runServer, | ||
16 | ServerInfo, | 15 | ServerInfo, |
17 | setAccessTokensToServers, | 16 | setAccessTokensToServers, |
18 | uploadVideo, | 17 | uploadVideo, |
19 | wait, | 18 | wait |
20 | getVideo, flushAndRunMultipleServers, doubleFollow | ||
21 | } from '../utils' | 19 | } from '../utils' |
22 | 20 | ||
23 | function assertVideoProperties (video: VideoFile, resolution: number, extname: string) { | 21 | const expect = chai.expect |
22 | |||
23 | function assertVideoProperties (video: VideoFile, resolution: number, extname: string, size?: number) { | ||
24 | expect(video).to.have.nested.property('resolution.id', resolution) | 24 | expect(video).to.have.nested.property('resolution.id', resolution) |
25 | expect(video).to.have.property('magnetUri').that.includes(`.${extname}`) | 25 | expect(video).to.have.property('magnetUri').that.includes(`.${extname}`) |
26 | expect(video).to.have.property('torrentUrl').that.includes(`-${resolution}.torrent`) | 26 | expect(video).to.have.property('torrentUrl').that.includes(`-${resolution}.torrent`) |
27 | expect(video).to.have.property('fileUrl').that.includes(`.${extname}`) | 27 | expect(video).to.have.property('fileUrl').that.includes(`.${extname}`) |
28 | expect(video).to.have.property('size').that.is.above(0) | 28 | expect(video).to.have.property('size').that.is.above(0) |
29 | |||
30 | if (size) expect(video.size).to.equal(size) | ||
29 | } | 31 | } |
30 | 32 | ||
31 | describe('Test create import video jobs', function () { | 33 | describe('Test create import video jobs', function () { |
@@ -51,6 +53,7 @@ describe('Test create import video jobs', function () { | |||
51 | const res2 = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video2' }) | 53 | const res2 = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video2' }) |
52 | video2UUID = res2.body.video.uuid | 54 | video2UUID = res2.body.video.uuid |
53 | 55 | ||
56 | // Transcoding | ||
54 | await wait(40000) | 57 | await wait(40000) |
55 | }) | 58 | }) |
56 | 59 | ||
@@ -60,12 +63,11 @@ describe('Test create import video jobs', function () { | |||
60 | 63 | ||
61 | await wait(30000) | 64 | await wait(30000) |
62 | 65 | ||
66 | let magnetUri: string | ||
63 | for (const server of servers) { | 67 | for (const server of servers) { |
64 | const { data: videos } = (await getVideosList(server.url)).body | 68 | const { data: videos } = (await getVideosList(server.url)).body |
65 | expect(videos).to.have.lengthOf(2) | 69 | expect(videos).to.have.lengthOf(2) |
66 | 70 | ||
67 | let infoHashes: { [ id: number ]: string } = {} | ||
68 | |||
69 | const video = videos.find(({ uuid }) => uuid === video1UUID) | 71 | const video = videos.find(({ uuid }) => uuid === video1UUID) |
70 | const videoDetail: VideoDetails = (await getVideo(server.url, video.uuid)).body | 72 | const videoDetail: VideoDetails = (await getVideo(server.url, video.uuid)).body |
71 | 73 | ||
@@ -73,6 +75,9 @@ describe('Test create import video jobs', function () { | |||
73 | const [originalVideo, transcodedVideo] = videoDetail.files | 75 | const [originalVideo, transcodedVideo] = videoDetail.files |
74 | assertVideoProperties(originalVideo, 720, 'webm') | 76 | assertVideoProperties(originalVideo, 720, 'webm') |
75 | assertVideoProperties(transcodedVideo, 480, 'webm') | 77 | assertVideoProperties(transcodedVideo, 480, 'webm') |
78 | |||
79 | if (!magnetUri) magnetUri = transcodedVideo.magnetUri | ||
80 | else expect(transcodedVideo.magnetUri).to.equal(magnetUri) | ||
76 | } | 81 | } |
77 | }) | 82 | }) |
78 | 83 | ||
@@ -82,21 +87,23 @@ describe('Test create import video jobs', function () { | |||
82 | 87 | ||
83 | await wait(30000) | 88 | await wait(30000) |
84 | 89 | ||
90 | let magnetUri: string | ||
85 | for (const server of servers.reverse()) { | 91 | for (const server of servers.reverse()) { |
86 | const { data: videos } = (await getVideosList(server.url)).body | 92 | const { data: videos } = (await getVideosList(server.url)).body |
87 | expect(videos).to.have.lengthOf(2) | 93 | expect(videos).to.have.lengthOf(2) |
88 | 94 | ||
89 | let infoHashes: { [ id: number ]: string } | ||
90 | |||
91 | const video = videos.find(({ uuid }) => uuid === video2UUID) | 95 | const video = videos.find(({ uuid }) => uuid === video2UUID) |
92 | const videoDetail: VideoDetails = (await getVideo(server.url, video.uuid)).body | 96 | const videoDetail: VideoDetails = (await getVideo(server.url, video.uuid)).body |
93 | 97 | ||
94 | expect(videoDetail.files).to.have.lengthOf(4) | 98 | expect(videoDetail.files).to.have.lengthOf(4) |
95 | const [originalVideo, transcodedVideo420, transcodedVideo320, transcodedVideo240] = videoDetail.files | 99 | const [originalVideo, transcodedVideo420, transcodedVideo320, transcodedVideo240] = videoDetail.files |
96 | assertVideoProperties(originalVideo, 720, 'ogv') | 100 | assertVideoProperties(originalVideo, 720, 'ogv', 140849) |
97 | assertVideoProperties(transcodedVideo420, 480, 'mp4') | 101 | assertVideoProperties(transcodedVideo420, 480, 'mp4') |
98 | assertVideoProperties(transcodedVideo320, 360, 'mp4') | 102 | assertVideoProperties(transcodedVideo320, 360, 'mp4') |
99 | assertVideoProperties(transcodedVideo240, 240, 'mp4') | 103 | assertVideoProperties(transcodedVideo240, 240, 'mp4') |
104 | |||
105 | if (!magnetUri) magnetUri = originalVideo.magnetUri | ||
106 | else expect(originalVideo.magnetUri).to.equal(magnetUri) | ||
100 | } | 107 | } |
101 | }) | 108 | }) |
102 | 109 | ||