From 1f6125be8b6306ba34b5ad9df985df462ef9759c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 18 Aug 2021 09:14:51 +0200 Subject: Optimize torrent URL update --- .github/workflows/test.yml | 2 +- package.json | 1 + scripts/update-host.ts | 6 +-- server/helpers/webtorrent.ts | 61 +++++++++++++++++----- .../job-queue/handlers/move-to-object-storage.ts | 4 +- server/tests/api/object-storage/videos.ts | 7 +++ shared/extra-utils/miscs/checks.ts | 7 +++ shared/extra-utils/server/servers-command.ts | 6 ++- 8 files changed, 74 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 35c91bf85..f38b07bc6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,7 +32,7 @@ jobs: - 10389:10389 s3ninja: - image: scireum/s3-ninja + image: chocobozzz/s3-ninja ports: - 9444:9000 diff --git a/package.json b/package.json index 32f4e7c31..18bce1123 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,7 @@ "async": "^3.0.1", "async-lru": "^1.1.1", "bcrypt": "5.0.1", + "bencode": "^2.0.2", "bittorrent-tracker": "^9.0.0", "bluebird": "^3.5.0", "bull": "^3.4.2", diff --git a/scripts/update-host.ts b/scripts/update-host.ts index 9e8dd41ca..5d81a8d74 100755 --- a/scripts/update-host.ts +++ b/scripts/update-host.ts @@ -17,7 +17,7 @@ import { VideoCommentModel } from '../server/models/video/video-comment' import { AccountModel } from '../server/models/account/account' import { VideoChannelModel } from '../server/models/video/video-channel' import { initDatabaseModels } from '../server/initializers/database' -import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' +import { updateTorrentUrls } from '@server/helpers/webtorrent' import { getServerActor } from '@server/models/application/application' run() @@ -126,7 +126,7 @@ async function run () { for (const file of video.VideoFiles) { console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid) - await createTorrentAndSetInfoHash(video, file) + await updateTorrentUrls(video, file) await file.save() } @@ -135,7 +135,7 @@ async function run () { for (const file of (playlist?.VideoFiles || [])) { console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid) - await createTorrentAndSetInfoHash(video, file) + await updateTorrentUrls(video, file) await file.save() } diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index c84376304..012918468 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -1,5 +1,6 @@ +import * as bencode from 'bencode' import * as createTorrent from 'create-torrent' -import { createWriteStream, ensureDir, remove, writeFile } from 'fs-extra' +import { createWriteStream, ensureDir, readFile, remove, writeFile } from 'fs-extra' import * as magnetUtil from 'magnet-uri' import * as parseTorrent from 'parse-torrent' import { dirname, join } from 'path' @@ -79,43 +80,65 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName }) } -function createTorrentAndSetInfoHash ( - videoOrPlaylist: MVideo | MStreamingPlaylistVideo, - videoFile: MVideoFile -) { +function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { const video = extractVideo(videoOrPlaylist) const options = { // Keep the extname, it's used by the client to stream the file inside a web browser name: `${video.name} ${videoFile.resolution}p${videoFile.extname}`, createdBy: 'PeerTube', - announceList: [ - [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ], - [ WEBSERVER.URL + '/tracker/announce' ] - ], - urlList: [ videoFile.getFileUrl(video) ] + announceList: buildAnnounceList(), + urlList: buildUrlList(video, videoFile) } return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => { - const torrent = await createTorrentPromise(videoPath, options) + const torrentContent = await createTorrentPromise(videoPath, options) const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) logger.info('Creating torrent %s.', torrentPath) - await writeFile(torrentPath, torrent) + await writeFile(torrentPath, torrentContent) // Remove old torrent file if it existed if (videoFile.hasTorrent()) { await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) } - const parsedTorrent = parseTorrent(torrent) + const parsedTorrent = parseTorrent(torrentContent) videoFile.infoHash = parsedTorrent.infoHash videoFile.torrentFilename = torrentFilename }) } +async function updateTorrentUrls (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { + const video = extractVideo(videoOrPlaylist) + + const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) + + const torrentContent = await readFile(oldTorrentPath) + const decoded = bencode.decode(torrentContent) + + decoded['announce-list'] = buildAnnounceList() + decoded.announce = decoded['announce-list'][0][0] + + decoded['url-list'] = buildUrlList(video, videoFile) + + const newTorrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) + const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, newTorrentFilename) + + logger.info('Updating torrent URLs %s.', newTorrentPath) + + await writeFile(newTorrentPath, bencode.encode(decoded)) + + // Remove old torrent file if it existed + if (videoFile.hasTorrent()) { + await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) + } + + videoFile.torrentFilename = newTorrentFilename +} + function generateMagnetUri ( video: MVideo, videoFile: MVideoFileRedundanciesOpt, @@ -143,6 +166,7 @@ function generateMagnetUri ( export { createTorrentPromise, + updateTorrentUrls, createTorrentAndSetInfoHash, generateMagnetUri, downloadWebTorrentVideo @@ -186,3 +210,14 @@ function deleteDownloadedFile (downloadedFile: { directoryPath: string, filepath remove(toRemovePath) .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', toRemovePath, { err })) } + +function buildAnnounceList () { + return [ + [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ], + [ WEBSERVER.URL + '/tracker/announce' ] + ] +} + +function buildUrlList (video: MVideo, videoFile: MVideoFile) { + return [ videoFile.getFileUrl(video) ] +} diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts index a0c58d211..f3b8726eb 100644 --- a/server/lib/job-queue/handlers/move-to-object-storage.ts +++ b/server/lib/job-queue/handlers/move-to-object-storage.ts @@ -2,7 +2,7 @@ import * as Bull from 'bull' import { remove } from 'fs-extra' import { join } from 'path' import { logger } from '@server/helpers/logger' -import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' +import { updateTorrentUrls } from '@server/helpers/webtorrent' import { CONFIG } from '@server/initializers/config' import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage' import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' @@ -106,7 +106,7 @@ async function onFileMoved (options: { file.fileUrl = fileUrl file.storage = VideoStorage.OBJECT_STORAGE - await createTorrentAndSetInfoHash(videoOrPlaylist, file) + await updateTorrentUrls(videoOrPlaylist, file) await file.save() logger.debug('Removing %s because it\'s now on object storage', oldPath) diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts index 3958bd3d7..6c9c224eb 100644 --- a/server/tests/api/object-storage/videos.ts +++ b/server/tests/api/object-storage/videos.ts @@ -10,6 +10,7 @@ import { createMultipleServers, createSingleServer, doubleFollow, + expectLogDoesNotContain, expectStartWith, killallServers, makeRawRequest, @@ -235,6 +236,12 @@ function runTestSuite (options: { } }) + it('Should not have downloaded files from object storage', async function () { + for (const server of servers) { + await expectLogDoesNotContain(server, 'from object storage') + } + }) + after(async function () { mockObjectStorage.terminate() diff --git a/shared/extra-utils/miscs/checks.ts b/shared/extra-utils/miscs/checks.ts index aa2c8e8fa..b1be214b1 100644 --- a/shared/extra-utils/miscs/checks.ts +++ b/shared/extra-utils/miscs/checks.ts @@ -20,6 +20,12 @@ function expectStartWith (str: string, start: string) { expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true } +async function expectLogDoesNotContain (server: PeerTubeServer, str: string) { + const content = await server.servers.getLogContent() + + expect(content.toString()).to.not.contain(str) +} + async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { const res = await makeGetRequest({ url, @@ -46,6 +52,7 @@ async function testFileExistsOrNot (server: PeerTubeServer, directory: string, f export { dateIsValid, testImage, + expectLogDoesNotContain, testFileExistsOrNot, expectStartWith } diff --git a/shared/extra-utils/server/servers-command.ts b/shared/extra-utils/server/servers-command.ts index 40a11e8d7..776d2123c 100644 --- a/shared/extra-utils/server/servers-command.ts +++ b/shared/extra-utils/server/servers-command.ts @@ -55,7 +55,7 @@ export class ServersCommand extends AbstractCommand { } async waitUntilLog (str: string, count = 1, strictCount = true) { - const logfile = this.server.servers.buildDirectory('logs/peertube.log') + const logfile = this.buildDirectory('logs/peertube.log') while (true) { const buf = await readFile(logfile) @@ -80,6 +80,10 @@ export class ServersCommand extends AbstractCommand { return this.buildDirectory(join('streaming-playlists', 'hls', videoUUID, basename(fileUrl))) } + getLogContent () { + return readFile(this.buildDirectory('logs/peertube.log')) + } + async getServerFileSize (subPath: string) { const path = this.server.servers.buildDirectory(subPath) -- cgit v1.2.3