aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/job-queue
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-08-02 15:34:09 +0200
committerChocobozzz <me@florianbigard.com>2018-08-06 11:19:16 +0200
commitfbad87b0472f574409f7aa3ae7f8b54927d0cdd6 (patch)
tree197b4209e75d57dabae7cdd6f2da5f765e427023 /server/lib/job-queue
parent5e319fb7898fd0482c399cc3ae9dcfc20d274a58 (diff)
downloadPeerTube-fbad87b0472f574409f7aa3ae7f8b54927d0cdd6.tar.gz
PeerTube-fbad87b0472f574409f7aa3ae7f8b54927d0cdd6.tar.zst
PeerTube-fbad87b0472f574409f7aa3ae7f8b54927d0cdd6.zip
Add ability to import video with youtube-dl
Diffstat (limited to 'server/lib/job-queue')
-rw-r--r--server/lib/job-queue/handlers/video-import.ts129
-rw-r--r--server/lib/job-queue/job-queue.ts10
2 files changed, 136 insertions, 3 deletions
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
new file mode 100644
index 000000000..2f219e986
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -0,0 +1,129 @@
1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger'
3import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
4import { VideoImportModel } from '../../../models/video/video-import'
5import { VideoImportState } from '../../../../shared/models/videos'
6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
7import { extname, join } from 'path'
8import { VideoFileModel } from '../../../models/video/video-file'
9import { renamePromise, statPromise, unlinkPromise } from '../../../helpers/core-utils'
10import { CONFIG, sequelizeTypescript } from '../../../initializers'
11import { doRequestAndSaveToFile } from '../../../helpers/requests'
12import { VideoState } from '../../../../shared'
13import { JobQueue } from '../index'
14import { federateVideoIfNeeded } from '../../activitypub'
15
16export type VideoImportPayload = {
17 type: 'youtube-dl'
18 videoImportId: number
19 thumbnailUrl: string
20 downloadThumbnail: boolean
21 downloadPreview: boolean
22}
23
24async function processVideoImport (job: Bull.Job) {
25 const payload = job.data as VideoImportPayload
26 logger.info('Processing video import in job %d.', job.id)
27
28 const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId)
29 if (!videoImport) throw new Error('Cannot import video %s: the video import entry does not exist anymore.')
30
31 let tempVideoPath: string
32 try {
33 // Download video from youtubeDL
34 tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl)
35
36 // Get information about this video
37 const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
38 const fps = await getVideoFileFPS(tempVideoPath)
39 const stats = await statPromise(tempVideoPath)
40 const duration = await getDurationFromVideoFile(tempVideoPath)
41
42 // Create video file object in database
43 const videoFileData = {
44 extname: extname(tempVideoPath),
45 resolution: videoFileResolution,
46 size: stats.size,
47 fps,
48 videoId: videoImport.videoId
49 }
50 const videoFile = new VideoFileModel(videoFileData)
51
52 // Move file
53 const destination = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile))
54 await renamePromise(tempVideoPath, destination)
55
56 // Process thumbnail
57 if (payload.downloadThumbnail) {
58 if (payload.thumbnailUrl) {
59 const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
60 await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath)
61 } else {
62 await videoImport.Video.createThumbnail(videoFile)
63 }
64 }
65
66 // Process preview
67 if (payload.downloadPreview) {
68 if (payload.thumbnailUrl) {
69 const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
70 await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath)
71 } else {
72 await videoImport.Video.createPreview(videoFile)
73 }
74 }
75
76 // Create torrent
77 await videoImport.Video.createTorrentAndSetInfoHash(videoFile)
78
79 const videoImportUpdated: VideoImportModel = await sequelizeTypescript.transaction(async t => {
80 await videoFile.save({ transaction: t })
81
82 // Update video DB object
83 videoImport.Video.duration = duration
84 videoImport.Video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
85 const videoUpdated = await videoImport.Video.save({ transaction: t })
86
87 // Now we can federate the video
88 await federateVideoIfNeeded(videoImport.Video, true, t)
89
90 // Update video import object
91 videoImport.state = VideoImportState.SUCCESS
92 const videoImportUpdated = await videoImport.save({ transaction: t })
93
94 logger.info('Video %s imported.', videoImport.targetUrl)
95
96 videoImportUpdated.Video = videoUpdated
97 return videoImportUpdated
98 })
99
100 // Create transcoding jobs?
101 if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) {
102 // Put uuid because we don't have id auto incremented for now
103 const dataInput = {
104 videoUUID: videoImportUpdated.Video.uuid,
105 isNewVideo: true
106 }
107
108 await JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput })
109 }
110
111 } catch (err) {
112 try {
113 if (tempVideoPath) await unlinkPromise(tempVideoPath)
114 } catch (errUnlink) {
115 logger.error('Cannot cleanup files after a video import error.', { err: errUnlink })
116 }
117
118 videoImport.state = VideoImportState.FAILED
119 await videoImport.save()
120
121 throw err
122 }
123}
124
125// ---------------------------------------------------------------------------
126
127export {
128 processVideoImport
129}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 8ff0c169e..2e14867f2 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -9,6 +9,7 @@ import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './
9import { EmailPayload, processEmail } from './handlers/email' 9import { EmailPayload, processEmail } from './handlers/email'
10import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' 10import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file'
11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' 11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow'
12import { processVideoImport, VideoImportPayload } from './handlers/video-import'
12 13
13type CreateJobArgument = 14type CreateJobArgument =
14 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 15 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -17,7 +18,8 @@ type CreateJobArgument =
17 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | 18 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } |
18 { type: 'video-file-import', payload: VideoFileImportPayload } | 19 { type: 'video-file-import', payload: VideoFileImportPayload } |
19 { type: 'video-file', payload: VideoFilePayload } | 20 { type: 'video-file', payload: VideoFilePayload } |
20 { type: 'email', payload: EmailPayload } 21 { type: 'email', payload: EmailPayload } |
22 { type: 'video-import', payload: VideoImportPayload }
21 23
22const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { 24const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
23 'activitypub-http-broadcast': processActivityPubHttpBroadcast, 25 'activitypub-http-broadcast': processActivityPubHttpBroadcast,
@@ -26,7 +28,8 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
26 'activitypub-follow': processActivityPubFollow, 28 'activitypub-follow': processActivityPubFollow,
27 'video-file-import': processVideoFileImport, 29 'video-file-import': processVideoFileImport,
28 'video-file': processVideoFile, 30 'video-file': processVideoFile,
29 'email': processEmail 31 'email': processEmail,
32 'video-import': processVideoImport
30} 33}
31 34
32const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = { 35const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = {
@@ -43,7 +46,8 @@ const jobTypes: JobType[] = [
43 'activitypub-http-unicast', 46 'activitypub-http-unicast',
44 'email', 47 'email',
45 'video-file', 48 'video-file',
46 'video-file-import' 49 'video-file-import',
50 'video-import'
47] 51]
48 52
49class JobQueue { 53class JobQueue {