aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-10-01 12:00:05 +0200
committerChocobozzz <me@florianbigard.com>2018-10-01 12:00:05 +0200
commitcf9166cf2fb7b51a1137a259eed9338798c73500 (patch)
tree7cf4b5995d4a29cc335e1fd72c1572510f8432ad
parent2a27c451f78922107c3f056e7506be8a79b31e03 (diff)
downloadPeerTube-cf9166cf2fb7b51a1137a259eed9338798c73500.tar.gz
PeerTube-cf9166cf2fb7b51a1137a259eed9338798c73500.tar.zst
PeerTube-cf9166cf2fb7b51a1137a259eed9338798c73500.zip
Add timeout on youtube dl to cleaup files
-rw-r--r--server/helpers/webtorrent.ts12
-rw-r--r--server/helpers/youtube-dl.ts20
-rw-r--r--server/initializers/constants.ts4
-rw-r--r--server/lib/job-queue/handlers/video-import.ts6
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts5
5 files changed, 30 insertions, 17 deletions
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 924d630e7..ce35b87da 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -5,7 +5,7 @@ import { createWriteStream, ensureDir, remove } from 'fs-extra'
5import { CONFIG } from '../initializers' 5import { CONFIG } from '../initializers'
6import { dirname, join } from 'path' 6import { dirname, join } from 'path'
7 7
8async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout?: number) { 8async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout: number) {
9 const id = target.magnetUri || target.torrentName 9 const id = target.magnetUri || target.torrentName
10 let timer 10 let timer
11 11
@@ -50,12 +50,10 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
50 50
51 torrent.on('error', err => rej(err)) 51 torrent.on('error', err => rej(err))
52 52
53 if (timeout) { 53 timer = setTimeout(async () => {
54 timer = setTimeout(async () => { 54 return safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName)
55 return safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName) 55 .then(() => rej(new Error('Webtorrent download timeout.')))
56 .then(() => rej(new Error('Webtorrent download timeout.'))) 56 }, timeout)
57 }, timeout)
58 }
59 }) 57 })
60} 58}
61 59
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 748c67e6c..70b4e1b78 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -4,7 +4,7 @@ import { logger } from './logger'
4import { generateVideoTmpPath } from './utils' 4import { generateVideoTmpPath } from './utils'
5import { join } from 'path' 5import { join } from 'path'
6import { root } from './core-utils' 6import { root } from './core-utils'
7import { ensureDir, writeFile } from 'fs-extra' 7import { ensureDir, writeFile, remove } from 'fs-extra'
8import * as request from 'request' 8import * as request from 'request'
9import { createWriteStream } from 'fs' 9import { createWriteStream } from 'fs'
10 10
@@ -39,8 +39,9 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
39 }) 39 })
40} 40}
41 41
42function downloadYoutubeDLVideo (url: string) { 42function downloadYoutubeDLVideo (url: string, timeout: number) {
43 const path = generateVideoTmpPath(url) 43 const path = generateVideoTmpPath(url)
44 let timer
44 45
45 logger.info('Importing youtubeDL video %s', url) 46 logger.info('Importing youtubeDL video %s', url)
46 47
@@ -49,10 +50,23 @@ function downloadYoutubeDLVideo (url: string) {
49 return new Promise<string>(async (res, rej) => { 50 return new Promise<string>(async (res, rej) => {
50 const youtubeDL = await safeGetYoutubeDL() 51 const youtubeDL = await safeGetYoutubeDL()
51 youtubeDL.exec(url, options, processOptions, err => { 52 youtubeDL.exec(url, options, processOptions, err => {
52 if (err) return rej(err) 53 clearTimeout(timer)
54
55 if (err) {
56 remove(path)
57 .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
58
59 return rej(err)
60 }
53 61
54 return res(path) 62 return res(path)
55 }) 63 })
64
65 timer = setTimeout(async () => {
66 await remove(path)
67
68 return rej(new Error('YoutubeDL download timeout.'))
69 }, timeout)
56 }) 70 })
57} 71}
58 72
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 09ecedfa5..1a3b52015 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -119,7 +119,7 @@ const JOB_TTL: { [ id in JobType ]: number } = {
119 'activitypub-follow': 60000 * 10, // 10 minutes 119 'activitypub-follow': 60000 * 10, // 10 minutes
120 'video-file-import': 1000 * 3600, // 1 hour 120 'video-file-import': 1000 * 3600, // 1 hour
121 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long 121 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long
122 'video-import': 1000 * 3600, // 1 hour 122 'video-import': 1000 * 3600 * 2, // hours
123 'email': 60000 * 10, // 10 minutes 123 'email': 60000 * 10, // 10 minutes
124 'videos-views': undefined // Unlimited 124 'videos-views': undefined // Unlimited
125} 125}
@@ -133,6 +133,7 @@ const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activ
133const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...) 133const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
134const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds 134const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds
135const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days 135const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
136const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour
136 137
137// 1 hour 138// 1 hour
138let SCHEDULER_INTERVALS_MS = { 139let SCHEDULER_INTERVALS_MS = {
@@ -700,6 +701,7 @@ export {
700 TORRENT_MIMETYPE_EXT, 701 TORRENT_MIMETYPE_EXT,
701 STATIC_MAX_AGE, 702 STATIC_MAX_AGE,
702 STATIC_PATHS, 703 STATIC_PATHS,
704 VIDEO_IMPORT_TIMEOUT,
703 ACTIVITY_PUB, 705 ACTIVITY_PUB,
704 ACTIVITY_PUB_ACTOR_TYPES, 706 ACTIVITY_PUB_ACTOR_TYPES,
705 THUMBNAILS_SIZE, 707 THUMBNAILS_SIZE,
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 8f237dd91..e3f2a276c 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -6,7 +6,7 @@ import { VideoImportState } from '../../../../shared/models/videos'
6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' 6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
7import { extname, join } from 'path' 7import { extname, join } from 'path'
8import { VideoFileModel } from '../../../models/video/video-file' 8import { VideoFileModel } from '../../../models/video/video-file'
9import { CONFIG, sequelizeTypescript } from '../../../initializers' 9import { CONFIG, sequelizeTypescript, VIDEO_IMPORT_TIMEOUT } from '../../../initializers'
10import { doRequestAndSaveToFile } from '../../../helpers/requests' 10import { doRequestAndSaveToFile } from '../../../helpers/requests'
11import { VideoState } from '../../../../shared' 11import { VideoState } from '../../../../shared'
12import { JobQueue } from '../index' 12import { JobQueue } from '../index'
@@ -65,7 +65,7 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP
65 torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, 65 torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined,
66 magnetUri: videoImport.magnetUri 66 magnetUri: videoImport.magnetUri
67 } 67 }
68 return processFile(() => downloadWebTorrentVideo(target), videoImport, options) 68 return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options)
69} 69}
70 70
71async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) { 71async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) {
@@ -83,7 +83,7 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
83 generatePreview: false 83 generatePreview: false
84 } 84 }
85 85
86 return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl), videoImport, options) 86 return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl, VIDEO_IMPORT_TIMEOUT), videoImport, options)
87} 87}
88 88
89async function getVideoImportOrDie (videoImportId: number) { 89async function getVideoImportOrDie (videoImportId: number) {
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 11ee05a53..432bade1f 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,5 +1,5 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { CONFIG, JOB_TTL, REDUNDANCY } from '../../initializers' 2import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideosRedundancy } from '../../../shared/models/redundancy' 4import { VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
@@ -9,7 +9,6 @@ import { join } from 'path'
9import { rename } from 'fs-extra' 9import { rename } from 'fs-extra'
10import { getServerActor } from '../../helpers/utils' 10import { getServerActor } from '../../helpers/utils'
11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
12import { VideoModel } from '../../models/video/video'
13import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' 12import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
14import { removeVideoRedundancy } from '../redundancy' 13import { removeVideoRedundancy } from '../redundancy'
15import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' 14import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
@@ -142,7 +141,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
142 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 141 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
143 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) 142 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
144 143
145 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, JOB_TTL['video-import']) 144 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
146 145
147 const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file)) 146 const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file))
148 await rename(tmpPath, destPath) 147 await rename(tmpPath, destPath)