From 02b286f89088e07cac7e7068e884d3be0fd0098b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 6 Sep 2021 09:29:25 +0200 Subject: More robust webtorrent redundancy download Avoid issues with inconsistencies between magnet infohash and torrent infohash, blocking webtorrent upload that will timeout --- .../src/app/shared/shared-video-miniature/video-filters.model.ts | 2 -- server/helpers/webtorrent.ts | 6 +++--- server/lib/job-queue/handlers/video-import.ts | 2 +- server/lib/schedulers/videos-redundancy-scheduler.ts | 8 ++------ server/models/video/video-file.ts | 2 ++ 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/client/src/app/shared/shared-video-miniature/video-filters.model.ts b/client/src/app/shared/shared-video-miniature/video-filters.model.ts index f5095b85b..920dc826c 100644 --- a/client/src/app/shared/shared-video-miniature/video-filters.model.ts +++ b/client/src/app/shared/shared-video-miniature/video-filters.model.ts @@ -74,8 +74,6 @@ export class VideoFilters { } setNSFWPolicy (nsfwPolicy: NSFWPolicyType) { - console.log(nsfwPolicy) - this.updateDefaultNSFW(nsfwPolicy) } diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 83b46e085..5e1ea6198 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -21,8 +21,8 @@ import { extractVideo } from './video' const createTorrentPromise = promisify2(createTorrent) -async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout: number) { - const id = target.magnetUri || target.torrentName +async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) { + const id = target.uri || target.torrentName let timer const path = generateVideoImportTmpPath(id) @@ -35,7 +35,7 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName const webtorrent = new WebTorrent() let file: TorrentFile - const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) + const torrentId = target.uri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) const options = { path: directoryPath } const torrent = webtorrent.add(torrentId, options, torrent => { diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index bdbf07a06..8313c2561 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -63,7 +63,7 @@ async function processTorrentImport (job: Job, payload: VideoImportTorrentPayloa } const target = { torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, - magnetUri: videoImport.magnetUri + uri: videoImport.magnetUri } return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options) } diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 633cbcf6c..155d43343 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,7 +1,6 @@ import { move } from 'fs-extra' import { join } from 'path' import { getServerActor } from '@server/models/application/application' -import { TrackerModel } from '@server/models/server/tracker' import { VideoModel } from '@server/models/video/video' import { MStreamingPlaylistFiles, @@ -15,7 +14,7 @@ import { } from '@server/types/models' import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' import { logger, loggerTagsFactory } from '../../helpers/logger' -import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' +import { downloadWebTorrentVideo } from '../../helpers/webtorrent' import { CONFIG } from '../../initializers/config' import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' @@ -232,10 +231,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid)) - const trackerUrls = await TrackerModel.listUrlsByVideoId(video.id) - const magnetUri = generateMagnetUri(video, file, trackerUrls) - - const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) + const tmpPath = await downloadWebTorrentVideo({ uri: file.torrentUrl }, VIDEO_IMPORT_TIMEOUT) const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) await move(tmpPath, destPath, { overwrite: true }) diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 5e8d29d0a..106f9602b 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -194,6 +194,7 @@ export class VideoFileModel extends Model @Column metadataUrl: string + // Could be null for remote files @AllowNull(true) @Column fileUrl: string @@ -203,6 +204,7 @@ export class VideoFileModel extends Model @Column filename: string + // Could be null for remote files @AllowNull(true) @Column torrentUrl: string -- cgit v1.2.3