aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-09-06 09:29:25 +0200
committerChocobozzz <me@florianbigard.com>2021-09-06 16:19:49 +0200
commit02b286f89088e07cac7e7068e884d3be0fd0098b (patch)
treec98e159e6ee4512bb7026d052f565fcecc840124
parent70430c2796b6c0455a863edc62760a3d45951fc5 (diff)
downloadPeerTube-02b286f89088e07cac7e7068e884d3be0fd0098b.tar.gz
PeerTube-02b286f89088e07cac7e7068e884d3be0fd0098b.tar.zst
PeerTube-02b286f89088e07cac7e7068e884d3be0fd0098b.zip
More robust webtorrent redundancy download
Avoid issues with inconsistencies between magnet infohash and torrent infohash, blocking webtorrent upload that will timeout
-rw-r--r--client/src/app/shared/shared-video-miniature/video-filters.model.ts2
-rw-r--r--server/helpers/webtorrent.ts6
-rw-r--r--server/lib/job-queue/handlers/video-import.ts2
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts8
-rw-r--r--server/models/video/video-file.ts2
5 files changed, 8 insertions, 12 deletions
diff --git a/client/src/app/shared/shared-video-miniature/video-filters.model.ts b/client/src/app/shared/shared-video-miniature/video-filters.model.ts
index f5095b85b..920dc826c 100644
--- a/client/src/app/shared/shared-video-miniature/video-filters.model.ts
+++ b/client/src/app/shared/shared-video-miniature/video-filters.model.ts
@@ -74,8 +74,6 @@ export class VideoFilters {
74 } 74 }
75 75
76 setNSFWPolicy (nsfwPolicy: NSFWPolicyType) { 76 setNSFWPolicy (nsfwPolicy: NSFWPolicyType) {
77 console.log(nsfwPolicy)
78
79 this.updateDefaultNSFW(nsfwPolicy) 77 this.updateDefaultNSFW(nsfwPolicy)
80 } 78 }
81 79
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 83b46e085..5e1ea6198 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -21,8 +21,8 @@ import { extractVideo } from './video'
21 21
22const createTorrentPromise = promisify2<string, any, any>(createTorrent) 22const createTorrentPromise = promisify2<string, any, any>(createTorrent)
23 23
24async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout: number) { 24async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) {
25 const id = target.magnetUri || target.torrentName 25 const id = target.uri || target.torrentName
26 let timer 26 let timer
27 27
28 const path = generateVideoImportTmpPath(id) 28 const path = generateVideoImportTmpPath(id)
@@ -35,7 +35,7 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
35 const webtorrent = new WebTorrent() 35 const webtorrent = new WebTorrent()
36 let file: TorrentFile 36 let file: TorrentFile
37 37
38 const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) 38 const torrentId = target.uri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName)
39 39
40 const options = { path: directoryPath } 40 const options = { path: directoryPath }
41 const torrent = webtorrent.add(torrentId, options, torrent => { 41 const torrent = webtorrent.add(torrentId, options, torrent => {
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index bdbf07a06..8313c2561 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -63,7 +63,7 @@ async function processTorrentImport (job: Job, payload: VideoImportTorrentPayloa
63 } 63 }
64 const target = { 64 const target = {
65 torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, 65 torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined,
66 magnetUri: videoImport.magnetUri 66 uri: videoImport.magnetUri
67 } 67 }
68 return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options) 68 return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options)
69} 69}
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 633cbcf6c..155d43343 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,7 +1,6 @@
1import { move } from 'fs-extra' 1import { move } from 'fs-extra'
2import { join } from 'path' 2import { join } from 'path'
3import { getServerActor } from '@server/models/application/application' 3import { getServerActor } from '@server/models/application/application'
4import { TrackerModel } from '@server/models/server/tracker'
5import { VideoModel } from '@server/models/video/video' 4import { VideoModel } from '@server/models/video/video'
6import { 5import {
7 MStreamingPlaylistFiles, 6 MStreamingPlaylistFiles,
@@ -15,7 +14,7 @@ import {
15} from '@server/types/models' 14} from '@server/types/models'
16import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' 15import { VideosRedundancyStrategy } from '../../../shared/models/redundancy'
17import { logger, loggerTagsFactory } from '../../helpers/logger' 16import { logger, loggerTagsFactory } from '../../helpers/logger'
18import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' 17import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
19import { CONFIG } from '../../initializers/config' 18import { CONFIG } from '../../initializers/config'
20import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' 19import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants'
21import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 20import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
@@ -232,10 +231,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
232 231
233 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid)) 232 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid))
234 233
235 const trackerUrls = await TrackerModel.listUrlsByVideoId(video.id) 234 const tmpPath = await downloadWebTorrentVideo({ uri: file.torrentUrl }, VIDEO_IMPORT_TIMEOUT)
236 const magnetUri = generateMagnetUri(video, file, trackerUrls)
237
238 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
239 235
240 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) 236 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename)
241 await move(tmpPath, destPath, { overwrite: true }) 237 await move(tmpPath, destPath, { overwrite: true })
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 5e8d29d0a..106f9602b 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -194,6 +194,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
194 @Column 194 @Column
195 metadataUrl: string 195 metadataUrl: string
196 196
197 // Could be null for remote files
197 @AllowNull(true) 198 @AllowNull(true)
198 @Column 199 @Column
199 fileUrl: string 200 fileUrl: string
@@ -203,6 +204,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
203 @Column 204 @Column
204 filename: string 205 filename: string
205 206
207 // Could be null for remote files
206 @AllowNull(true) 208 @AllowNull(true)
207 @Column 209 @Column
208 torrentUrl: string 210 torrentUrl: string