From 1f6125be8b6306ba34b5ad9df985df462ef9759c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 18 Aug 2021 09:14:51 +0200 Subject: Optimize torrent URL update --- server/helpers/webtorrent.ts | 61 +++++++++++++++++----- .../job-queue/handlers/move-to-object-storage.ts | 4 +- server/tests/api/object-storage/videos.ts | 7 +++ 3 files changed, 57 insertions(+), 15 deletions(-) (limited to 'server') diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index c84376304..012918468 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -1,5 +1,6 @@ +import * as bencode from 'bencode' import * as createTorrent from 'create-torrent' -import { createWriteStream, ensureDir, remove, writeFile } from 'fs-extra' +import { createWriteStream, ensureDir, readFile, remove, writeFile } from 'fs-extra' import * as magnetUtil from 'magnet-uri' import * as parseTorrent from 'parse-torrent' import { dirname, join } from 'path' @@ -79,43 +80,65 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName }) } -function createTorrentAndSetInfoHash ( - videoOrPlaylist: MVideo | MStreamingPlaylistVideo, - videoFile: MVideoFile -) { +function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { const video = extractVideo(videoOrPlaylist) const options = { // Keep the extname, it's used by the client to stream the file inside a web browser name: `${video.name} ${videoFile.resolution}p${videoFile.extname}`, createdBy: 'PeerTube', - announceList: [ - [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ], - [ WEBSERVER.URL + '/tracker/announce' ] - ], - urlList: [ videoFile.getFileUrl(video) ] + announceList: buildAnnounceList(), + urlList: buildUrlList(video, videoFile) } return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => { - const torrent = await createTorrentPromise(videoPath, options) + const torrentContent = await createTorrentPromise(videoPath, options) const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) logger.info('Creating torrent %s.', torrentPath) - await writeFile(torrentPath, torrent) + await writeFile(torrentPath, torrentContent) // Remove old torrent file if it existed if (videoFile.hasTorrent()) { await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) } - const parsedTorrent = parseTorrent(torrent) + const parsedTorrent = parseTorrent(torrentContent) videoFile.infoHash = parsedTorrent.infoHash videoFile.torrentFilename = torrentFilename }) } +async function updateTorrentUrls (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { + const video = extractVideo(videoOrPlaylist) + + const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) + + const torrentContent = await readFile(oldTorrentPath) + const decoded = bencode.decode(torrentContent) + + decoded['announce-list'] = buildAnnounceList() + decoded.announce = decoded['announce-list'][0][0] + + decoded['url-list'] = buildUrlList(video, videoFile) + + const newTorrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) + const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, newTorrentFilename) + + logger.info('Updating torrent URLs %s.', newTorrentPath) + + await writeFile(newTorrentPath, bencode.encode(decoded)) + + // Remove old torrent file if it existed + if (videoFile.hasTorrent()) { + await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) + } + + videoFile.torrentFilename = newTorrentFilename +} + function generateMagnetUri ( video: MVideo, videoFile: MVideoFileRedundanciesOpt, @@ -143,6 +166,7 @@ function generateMagnetUri ( export { createTorrentPromise, + updateTorrentUrls, createTorrentAndSetInfoHash, generateMagnetUri, downloadWebTorrentVideo @@ -186,3 +210,14 @@ function deleteDownloadedFile (downloadedFile: { directoryPath: string, filepath remove(toRemovePath) .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', toRemovePath, { err })) } + +function buildAnnounceList () { + return [ + [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ], + [ WEBSERVER.URL + '/tracker/announce' ] + ] +} + +function buildUrlList (video: MVideo, videoFile: MVideoFile) { + return [ videoFile.getFileUrl(video) ] +} diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts index a0c58d211..f3b8726eb 100644 --- a/server/lib/job-queue/handlers/move-to-object-storage.ts +++ b/server/lib/job-queue/handlers/move-to-object-storage.ts @@ -2,7 +2,7 @@ import * as Bull from 'bull' import { remove } from 'fs-extra' import { join } from 'path' import { logger } from '@server/helpers/logger' -import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' +import { updateTorrentUrls } from '@server/helpers/webtorrent' import { CONFIG } from '@server/initializers/config' import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage' import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' @@ -106,7 +106,7 @@ async function onFileMoved (options: { file.fileUrl = fileUrl file.storage = VideoStorage.OBJECT_STORAGE - await createTorrentAndSetInfoHash(videoOrPlaylist, file) + await updateTorrentUrls(videoOrPlaylist, file) await file.save() logger.debug('Removing %s because it\'s now on object storage', oldPath) diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts index 3958bd3d7..6c9c224eb 100644 --- a/server/tests/api/object-storage/videos.ts +++ b/server/tests/api/object-storage/videos.ts @@ -10,6 +10,7 @@ import { createMultipleServers, createSingleServer, doubleFollow, + expectLogDoesNotContain, expectStartWith, killallServers, makeRawRequest, @@ -235,6 +236,12 @@ function runTestSuite (options: { } }) + it('Should not have downloaded files from object storage', async function () { + for (const server of servers) { + await expectLogDoesNotContain(server, 'from object storage') + } + }) + after(async function () { mockObjectStorage.terminate() -- cgit v1.2.3