diff options
author | Chocobozzz <me@florianbigard.com> | 2021-09-06 09:29:25 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2021-09-06 16:19:49 +0200 |
commit | 02b286f89088e07cac7e7068e884d3be0fd0098b (patch) | |
tree | c98e159e6ee4512bb7026d052f565fcecc840124 | |
parent | 70430c2796b6c0455a863edc62760a3d45951fc5 (diff) | |
download | PeerTube-02b286f89088e07cac7e7068e884d3be0fd0098b.tar.gz PeerTube-02b286f89088e07cac7e7068e884d3be0fd0098b.tar.zst PeerTube-02b286f89088e07cac7e7068e884d3be0fd0098b.zip |
More robust webtorrent redundancy download
Avoid issues with inconsistencies between magnet infohash and torrent
infohash, blocking webtorrent upload that will timeout
-rw-r--r-- | client/src/app/shared/shared-video-miniature/video-filters.model.ts | 2 | ||||
-rw-r--r-- | server/helpers/webtorrent.ts | 6 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/video-import.ts | 2 | ||||
-rw-r--r-- | server/lib/schedulers/videos-redundancy-scheduler.ts | 8 | ||||
-rw-r--r-- | server/models/video/video-file.ts | 2 |
5 files changed, 8 insertions, 12 deletions
diff --git a/client/src/app/shared/shared-video-miniature/video-filters.model.ts b/client/src/app/shared/shared-video-miniature/video-filters.model.ts index f5095b85b..920dc826c 100644 --- a/client/src/app/shared/shared-video-miniature/video-filters.model.ts +++ b/client/src/app/shared/shared-video-miniature/video-filters.model.ts | |||
@@ -74,8 +74,6 @@ export class VideoFilters { | |||
74 | } | 74 | } |
75 | 75 | ||
76 | setNSFWPolicy (nsfwPolicy: NSFWPolicyType) { | 76 | setNSFWPolicy (nsfwPolicy: NSFWPolicyType) { |
77 | console.log(nsfwPolicy) | ||
78 | |||
79 | this.updateDefaultNSFW(nsfwPolicy) | 77 | this.updateDefaultNSFW(nsfwPolicy) |
80 | } | 78 | } |
81 | 79 | ||
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 83b46e085..5e1ea6198 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts | |||
@@ -21,8 +21,8 @@ import { extractVideo } from './video' | |||
21 | 21 | ||
22 | const createTorrentPromise = promisify2<string, any, any>(createTorrent) | 22 | const createTorrentPromise = promisify2<string, any, any>(createTorrent) |
23 | 23 | ||
24 | async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout: number) { | 24 | async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) { |
25 | const id = target.magnetUri || target.torrentName | 25 | const id = target.uri || target.torrentName |
26 | let timer | 26 | let timer |
27 | 27 | ||
28 | const path = generateVideoImportTmpPath(id) | 28 | const path = generateVideoImportTmpPath(id) |
@@ -35,7 +35,7 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName | |||
35 | const webtorrent = new WebTorrent() | 35 | const webtorrent = new WebTorrent() |
36 | let file: TorrentFile | 36 | let file: TorrentFile |
37 | 37 | ||
38 | const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) | 38 | const torrentId = target.uri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) |
39 | 39 | ||
40 | const options = { path: directoryPath } | 40 | const options = { path: directoryPath } |
41 | const torrent = webtorrent.add(torrentId, options, torrent => { | 41 | const torrent = webtorrent.add(torrentId, options, torrent => { |
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index bdbf07a06..8313c2561 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -63,7 +63,7 @@ async function processTorrentImport (job: Job, payload: VideoImportTorrentPayloa | |||
63 | } | 63 | } |
64 | const target = { | 64 | const target = { |
65 | torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, | 65 | torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, |
66 | magnetUri: videoImport.magnetUri | 66 | uri: videoImport.magnetUri |
67 | } | 67 | } |
68 | return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options) | 68 | return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options) |
69 | } | 69 | } |
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 633cbcf6c..155d43343 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -1,7 +1,6 @@ | |||
1 | import { move } from 'fs-extra' | 1 | import { move } from 'fs-extra' |
2 | import { join } from 'path' | 2 | import { join } from 'path' |
3 | import { getServerActor } from '@server/models/application/application' | 3 | import { getServerActor } from '@server/models/application/application' |
4 | import { TrackerModel } from '@server/models/server/tracker' | ||
5 | import { VideoModel } from '@server/models/video/video' | 4 | import { VideoModel } from '@server/models/video/video' |
6 | import { | 5 | import { |
7 | MStreamingPlaylistFiles, | 6 | MStreamingPlaylistFiles, |
@@ -15,7 +14,7 @@ import { | |||
15 | } from '@server/types/models' | 14 | } from '@server/types/models' |
16 | import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' | 15 | import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' |
17 | import { logger, loggerTagsFactory } from '../../helpers/logger' | 16 | import { logger, loggerTagsFactory } from '../../helpers/logger' |
18 | import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' | 17 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' |
19 | import { CONFIG } from '../../initializers/config' | 18 | import { CONFIG } from '../../initializers/config' |
20 | import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' | 19 | import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' |
21 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 20 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
@@ -232,10 +231,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
232 | 231 | ||
233 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid)) | 232 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid)) |
234 | 233 | ||
235 | const trackerUrls = await TrackerModel.listUrlsByVideoId(video.id) | 234 | const tmpPath = await downloadWebTorrentVideo({ uri: file.torrentUrl }, VIDEO_IMPORT_TIMEOUT) |
236 | const magnetUri = generateMagnetUri(video, file, trackerUrls) | ||
237 | |||
238 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) | ||
239 | 235 | ||
240 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) | 236 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) |
241 | await move(tmpPath, destPath, { overwrite: true }) | 237 | await move(tmpPath, destPath, { overwrite: true }) |
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 5e8d29d0a..106f9602b 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts | |||
@@ -194,6 +194,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
194 | @Column | 194 | @Column |
195 | metadataUrl: string | 195 | metadataUrl: string |
196 | 196 | ||
197 | // Could be null for remote files | ||
197 | @AllowNull(true) | 198 | @AllowNull(true) |
198 | @Column | 199 | @Column |
199 | fileUrl: string | 200 | fileUrl: string |
@@ -203,6 +204,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
203 | @Column | 204 | @Column |
204 | filename: string | 205 | filename: string |
205 | 206 | ||
207 | // Could be null for remote files | ||
206 | @AllowNull(true) | 208 | @AllowNull(true) |
207 | @Column | 209 | @Column |
208 | torrentUrl: string | 210 | torrentUrl: string |