aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/job-queue
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-08-06 17:13:39 +0200
committerChocobozzz <me@florianbigard.com>2018-08-08 09:30:31 +0200
commitce33919c24e7402d92d81f3cd8e545df52d98240 (patch)
tree7e131a2f8df649899d0a71294665cf386ffb50d4 /server/lib/job-queue
parent788487140c500abeb69ca44daf3a9e26efa8d36f (diff)
downloadPeerTube-ce33919c24e7402d92d81f3cd8e545df52d98240.tar.gz
PeerTube-ce33919c24e7402d92d81f3cd8e545df52d98240.tar.zst
PeerTube-ce33919c24e7402d92d81f3cd8e545df52d98240.zip
Import magnets with webtorrent
Diffstat (limited to 'server/lib/job-queue')
-rw-r--r--server/lib/job-queue/handlers/video-import.ts99
-rw-r--r--server/lib/job-queue/job-queue.ts7
2 files changed, 83 insertions, 23 deletions
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index cdfe412cc..c457b71fc 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -13,30 +13,99 @@ import { VideoState } from '../../../../shared'
13import { JobQueue } from '../index' 13import { JobQueue } from '../index'
14import { federateVideoIfNeeded } from '../../activitypub' 14import { federateVideoIfNeeded } from '../../activitypub'
15import { VideoModel } from '../../../models/video/video' 15import { VideoModel } from '../../../models/video/video'
16import { downloadWebTorrentVideo } from '../../../helpers/webtorrent'
16 17
17export type VideoImportPayload = { 18type VideoImportYoutubeDLPayload = {
18 type: 'youtube-dl' 19 type: 'youtube-dl'
19 videoImportId: number 20 videoImportId: number
21
20 thumbnailUrl: string 22 thumbnailUrl: string
21 downloadThumbnail: boolean 23 downloadThumbnail: boolean
22 downloadPreview: boolean 24 downloadPreview: boolean
23} 25}
24 26
27type VideoImportTorrentPayload = {
28 type: 'magnet-uri'
29 videoImportId: number
30}
31
32export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
33
25async function processVideoImport (job: Bull.Job) { 34async function processVideoImport (job: Bull.Job) {
26 const payload = job.data as VideoImportPayload 35 const payload = job.data as VideoImportPayload
27 logger.info('Processing video import in job %d.', job.id)
28 36
29 const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId) 37 if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload)
38 if (payload.type === 'magnet-uri') return processTorrentImport(job, payload)
39}
40
41// ---------------------------------------------------------------------------
42
43export {
44 processVideoImport
45}
46
47// ---------------------------------------------------------------------------
48
49async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentPayload) {
50 logger.info('Processing torrent video import in job %d.', job.id)
51
52 const videoImport = await getVideoImportOrDie(payload.videoImportId)
53 const options = {
54 videoImportId: payload.videoImportId,
55
56 downloadThumbnail: false,
57 downloadPreview: false,
58
59 generateThumbnail: true,
60 generatePreview: true
61 }
62 return processFile(() => downloadWebTorrentVideo(videoImport.magnetUri), videoImport, options)
63}
64
65async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) {
66 logger.info('Processing youtubeDL video import in job %d.', job.id)
67
68 const videoImport = await getVideoImportOrDie(payload.videoImportId)
69 const options = {
70 videoImportId: videoImport.id,
71
72 downloadThumbnail: payload.downloadThumbnail,
73 downloadPreview: payload.downloadPreview,
74 thumbnailUrl: payload.thumbnailUrl,
75
76 generateThumbnail: false,
77 generatePreview: false
78 }
79
80 return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl), videoImport, options)
81}
82
83async function getVideoImportOrDie (videoImportId: number) {
84 const videoImport = await VideoImportModel.loadAndPopulateVideo(videoImportId)
30 if (!videoImport || !videoImport.Video) { 85 if (!videoImport || !videoImport.Video) {
31 throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.') 86 throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.')
32 } 87 }
33 88
89 return videoImport
90}
91
92type ProcessFileOptions = {
93 videoImportId: number
94
95 downloadThumbnail: boolean
96 downloadPreview: boolean
97 thumbnailUrl?: string
98
99 generateThumbnail: boolean
100 generatePreview: boolean
101}
102async function processFile (downloader: () => Promise<string>, videoImport: VideoImportModel, options: ProcessFileOptions) {
34 let tempVideoPath: string 103 let tempVideoPath: string
35 let videoDestFile: string 104 let videoDestFile: string
36 let videoFile: VideoFileModel 105 let videoFile: VideoFileModel
37 try { 106 try {
38 // Download video from youtubeDL 107 // Download video from youtubeDL
39 tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl) 108 tempVideoPath = await downloader()
40 109
41 // Get information about this video 110 // Get information about this video
42 const { videoFileResolution } = await getVideoFileResolution(tempVideoPath) 111 const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
@@ -62,23 +131,27 @@ async function processVideoImport (job: Bull.Job) {
62 tempVideoPath = null // This path is not used anymore 131 tempVideoPath = null // This path is not used anymore
63 132
64 // Process thumbnail 133 // Process thumbnail
65 if (payload.downloadThumbnail) { 134 if (options.downloadThumbnail) {
66 if (payload.thumbnailUrl) { 135 if (options.thumbnailUrl) {
67 const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) 136 const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
68 await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath) 137 await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destThumbnailPath)
69 } else { 138 } else {
70 await videoImport.Video.createThumbnail(videoFile) 139 await videoImport.Video.createThumbnail(videoFile)
71 } 140 }
141 } else if (options.generateThumbnail) {
142 await videoImport.Video.createThumbnail(videoFile)
72 } 143 }
73 144
74 // Process preview 145 // Process preview
75 if (payload.downloadPreview) { 146 if (options.downloadPreview) {
76 if (payload.thumbnailUrl) { 147 if (options.thumbnailUrl) {
77 const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) 148 const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
78 await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath) 149 await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destPreviewPath)
79 } else { 150 } else {
80 await videoImport.Video.createPreview(videoFile) 151 await videoImport.Video.createPreview(videoFile)
81 } 152 }
153 } else if (options.generatePreview) {
154 await videoImport.Video.createPreview(videoFile)
82 } 155 }
83 156
84 // Create torrent 157 // Create torrent
@@ -137,9 +210,3 @@ async function processVideoImport (job: Bull.Job) {
137 throw err 210 throw err
138 } 211 }
139} 212}
140
141// ---------------------------------------------------------------------------
142
143export {
144 processVideoImport
145}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 8a24604e1..ddb357db5 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -32,13 +32,6 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
32 'video-import': processVideoImport 32 'video-import': processVideoImport
33} 33}
34 34
35const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = {
36 'activitypub-http-broadcast': true,
37 'activitypub-http-unicast': true,
38 'activitypub-http-fetcher': true,
39 'activitypub-follow': true
40}
41
42const jobTypes: JobType[] = [ 35const jobTypes: JobType[] = [
43 'activitypub-follow', 36 'activitypub-follow',
44 'activitypub-http-broadcast', 37 'activitypub-http-broadcast',