aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-08-18 09:14:51 +0200
committerChocobozzz <me@florianbigard.com>2021-08-18 09:14:51 +0200
commit1f6125be8b6306ba34b5ad9df985df462ef9759c (patch)
tree9894ab5a6b239f7797303c24f21e79019b942238
parent9df52d660feb722404be00a50f3c8a612bec1c15 (diff)
downloadPeerTube-1f6125be8b6306ba34b5ad9df985df462ef9759c.tar.gz
PeerTube-1f6125be8b6306ba34b5ad9df985df462ef9759c.tar.zst
PeerTube-1f6125be8b6306ba34b5ad9df985df462ef9759c.zip
Optimize torrent URL update
-rw-r--r--.github/workflows/test.yml2
-rw-r--r--package.json1
-rwxr-xr-xscripts/update-host.ts6
-rw-r--r--server/helpers/webtorrent.ts61
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts4
-rw-r--r--server/tests/api/object-storage/videos.ts7
-rw-r--r--shared/extra-utils/miscs/checks.ts7
-rw-r--r--shared/extra-utils/server/servers-command.ts6
8 files changed, 74 insertions, 20 deletions
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 35c91bf85..f38b07bc6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,7 +32,7 @@ jobs:
32 - 10389:10389 32 - 10389:10389
33 33
34 s3ninja: 34 s3ninja:
35 image: scireum/s3-ninja 35 image: chocobozzz/s3-ninja
36 ports: 36 ports:
37 - 9444:9000 37 - 9444:9000
38 38
diff --git a/package.json b/package.json
index 32f4e7c31..18bce1123 100644
--- a/package.json
+++ b/package.json
@@ -77,6 +77,7 @@
77 "async": "^3.0.1", 77 "async": "^3.0.1",
78 "async-lru": "^1.1.1", 78 "async-lru": "^1.1.1",
79 "bcrypt": "5.0.1", 79 "bcrypt": "5.0.1",
80 "bencode": "^2.0.2",
80 "bittorrent-tracker": "^9.0.0", 81 "bittorrent-tracker": "^9.0.0",
81 "bluebird": "^3.5.0", 82 "bluebird": "^3.5.0",
82 "bull": "^3.4.2", 83 "bull": "^3.4.2",
diff --git a/scripts/update-host.ts b/scripts/update-host.ts
index 9e8dd41ca..5d81a8d74 100755
--- a/scripts/update-host.ts
+++ b/scripts/update-host.ts
@@ -17,7 +17,7 @@ import { VideoCommentModel } from '../server/models/video/video-comment'
17import { AccountModel } from '../server/models/account/account' 17import { AccountModel } from '../server/models/account/account'
18import { VideoChannelModel } from '../server/models/video/video-channel' 18import { VideoChannelModel } from '../server/models/video/video-channel'
19import { initDatabaseModels } from '../server/initializers/database' 19import { initDatabaseModels } from '../server/initializers/database'
20import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 20import { updateTorrentUrls } from '@server/helpers/webtorrent'
21import { getServerActor } from '@server/models/application/application' 21import { getServerActor } from '@server/models/application/application'
22 22
23run() 23run()
@@ -126,7 +126,7 @@ async function run () {
126 126
127 for (const file of video.VideoFiles) { 127 for (const file of video.VideoFiles) {
128 console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid) 128 console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid)
129 await createTorrentAndSetInfoHash(video, file) 129 await updateTorrentUrls(video, file)
130 130
131 await file.save() 131 await file.save()
132 } 132 }
@@ -135,7 +135,7 @@ async function run () {
135 for (const file of (playlist?.VideoFiles || [])) { 135 for (const file of (playlist?.VideoFiles || [])) {
136 console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid) 136 console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid)
137 137
138 await createTorrentAndSetInfoHash(video, file) 138 await updateTorrentUrls(video, file)
139 139
140 await file.save() 140 await file.save()
141 } 141 }
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index c84376304..012918468 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -1,5 +1,6 @@
1import * as bencode from 'bencode'
1import * as createTorrent from 'create-torrent' 2import * as createTorrent from 'create-torrent'
2import { createWriteStream, ensureDir, remove, writeFile } from 'fs-extra' 3import { createWriteStream, ensureDir, readFile, remove, writeFile } from 'fs-extra'
3import * as magnetUtil from 'magnet-uri' 4import * as magnetUtil from 'magnet-uri'
4import * as parseTorrent from 'parse-torrent' 5import * as parseTorrent from 'parse-torrent'
5import { dirname, join } from 'path' 6import { dirname, join } from 'path'
@@ -79,43 +80,65 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
79 }) 80 })
80} 81}
81 82
82function createTorrentAndSetInfoHash ( 83function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
83 videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
84 videoFile: MVideoFile
85) {
86 const video = extractVideo(videoOrPlaylist) 84 const video = extractVideo(videoOrPlaylist)
87 85
88 const options = { 86 const options = {
89 // Keep the extname, it's used by the client to stream the file inside a web browser 87 // Keep the extname, it's used by the client to stream the file inside a web browser
90 name: `${video.name} ${videoFile.resolution}p${videoFile.extname}`, 88 name: `${video.name} ${videoFile.resolution}p${videoFile.extname}`,
91 createdBy: 'PeerTube', 89 createdBy: 'PeerTube',
92 announceList: [ 90 announceList: buildAnnounceList(),
93 [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ], 91 urlList: buildUrlList(video, videoFile)
94 [ WEBSERVER.URL + '/tracker/announce' ]
95 ],
96 urlList: [ videoFile.getFileUrl(video) ]
97 } 92 }
98 93
99 return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => { 94 return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => {
100 const torrent = await createTorrentPromise(videoPath, options) 95 const torrentContent = await createTorrentPromise(videoPath, options)
101 96
102 const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) 97 const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
103 const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) 98 const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename)
104 logger.info('Creating torrent %s.', torrentPath) 99 logger.info('Creating torrent %s.', torrentPath)
105 100
106 await writeFile(torrentPath, torrent) 101 await writeFile(torrentPath, torrentContent)
107 102
108 // Remove old torrent file if it existed 103 // Remove old torrent file if it existed
109 if (videoFile.hasTorrent()) { 104 if (videoFile.hasTorrent()) {
110 await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) 105 await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
111 } 106 }
112 107
113 const parsedTorrent = parseTorrent(torrent) 108 const parsedTorrent = parseTorrent(torrentContent)
114 videoFile.infoHash = parsedTorrent.infoHash 109 videoFile.infoHash = parsedTorrent.infoHash
115 videoFile.torrentFilename = torrentFilename 110 videoFile.torrentFilename = torrentFilename
116 }) 111 })
117} 112}
118 113
114async function updateTorrentUrls (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
115 const video = extractVideo(videoOrPlaylist)
116
117 const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)
118
119 const torrentContent = await readFile(oldTorrentPath)
120 const decoded = bencode.decode(torrentContent)
121
122 decoded['announce-list'] = buildAnnounceList()
123 decoded.announce = decoded['announce-list'][0][0]
124
125 decoded['url-list'] = buildUrlList(video, videoFile)
126
127 const newTorrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
128 const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, newTorrentFilename)
129
130 logger.info('Updating torrent URLs %s.', newTorrentPath)
131
132 await writeFile(newTorrentPath, bencode.encode(decoded))
133
134 // Remove old torrent file if it existed
135 if (videoFile.hasTorrent()) {
136 await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
137 }
138
139 videoFile.torrentFilename = newTorrentFilename
140}
141
119function generateMagnetUri ( 142function generateMagnetUri (
120 video: MVideo, 143 video: MVideo,
121 videoFile: MVideoFileRedundanciesOpt, 144 videoFile: MVideoFileRedundanciesOpt,
@@ -143,6 +166,7 @@ function generateMagnetUri (
143 166
144export { 167export {
145 createTorrentPromise, 168 createTorrentPromise,
169 updateTorrentUrls,
146 createTorrentAndSetInfoHash, 170 createTorrentAndSetInfoHash,
147 generateMagnetUri, 171 generateMagnetUri,
148 downloadWebTorrentVideo 172 downloadWebTorrentVideo
@@ -186,3 +210,14 @@ function deleteDownloadedFile (downloadedFile: { directoryPath: string, filepath
186 remove(toRemovePath) 210 remove(toRemovePath)
187 .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', toRemovePath, { err })) 211 .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', toRemovePath, { err }))
188} 212}
213
214function buildAnnounceList () {
215 return [
216 [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ],
217 [ WEBSERVER.URL + '/tracker/announce' ]
218 ]
219}
220
221function buildUrlList (video: MVideo, videoFile: MVideoFile) {
222 return [ videoFile.getFileUrl(video) ]
223}
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index a0c58d211..f3b8726eb 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -2,7 +2,7 @@ import * as Bull from 'bull'
2import { remove } from 'fs-extra' 2import { remove } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import { logger } from '@server/helpers/logger' 4import { logger } from '@server/helpers/logger'
5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 5import { updateTorrentUrls } from '@server/helpers/webtorrent'
6import { CONFIG } from '@server/initializers/config' 6import { CONFIG } from '@server/initializers/config'
7import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage' 7import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage'
8import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' 8import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
@@ -106,7 +106,7 @@ async function onFileMoved (options: {
106 file.fileUrl = fileUrl 106 file.fileUrl = fileUrl
107 file.storage = VideoStorage.OBJECT_STORAGE 107 file.storage = VideoStorage.OBJECT_STORAGE
108 108
109 await createTorrentAndSetInfoHash(videoOrPlaylist, file) 109 await updateTorrentUrls(videoOrPlaylist, file)
110 await file.save() 110 await file.save()
111 111
112 logger.debug('Removing %s because it\'s now on object storage', oldPath) 112 logger.debug('Removing %s because it\'s now on object storage', oldPath)
diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts
index 3958bd3d7..6c9c224eb 100644
--- a/server/tests/api/object-storage/videos.ts
+++ b/server/tests/api/object-storage/videos.ts
@@ -10,6 +10,7 @@ import {
10 createMultipleServers, 10 createMultipleServers,
11 createSingleServer, 11 createSingleServer,
12 doubleFollow, 12 doubleFollow,
13 expectLogDoesNotContain,
13 expectStartWith, 14 expectStartWith,
14 killallServers, 15 killallServers,
15 makeRawRequest, 16 makeRawRequest,
@@ -235,6 +236,12 @@ function runTestSuite (options: {
235 } 236 }
236 }) 237 })
237 238
239 it('Should not have downloaded files from object storage', async function () {
240 for (const server of servers) {
241 await expectLogDoesNotContain(server, 'from object storage')
242 }
243 })
244
238 after(async function () { 245 after(async function () {
239 mockObjectStorage.terminate() 246 mockObjectStorage.terminate()
240 247
diff --git a/shared/extra-utils/miscs/checks.ts b/shared/extra-utils/miscs/checks.ts
index aa2c8e8fa..b1be214b1 100644
--- a/shared/extra-utils/miscs/checks.ts
+++ b/shared/extra-utils/miscs/checks.ts
@@ -20,6 +20,12 @@ function expectStartWith (str: string, start: string) {
20 expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true 20 expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true
21} 21}
22 22
23async function expectLogDoesNotContain (server: PeerTubeServer, str: string) {
24 const content = await server.servers.getLogContent()
25
26 expect(content.toString()).to.not.contain(str)
27}
28
23async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { 29async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') {
24 const res = await makeGetRequest({ 30 const res = await makeGetRequest({
25 url, 31 url,
@@ -46,6 +52,7 @@ async function testFileExistsOrNot (server: PeerTubeServer, directory: string, f
46export { 52export {
47 dateIsValid, 53 dateIsValid,
48 testImage, 54 testImage,
55 expectLogDoesNotContain,
49 testFileExistsOrNot, 56 testFileExistsOrNot,
50 expectStartWith 57 expectStartWith
51} 58}
diff --git a/shared/extra-utils/server/servers-command.ts b/shared/extra-utils/server/servers-command.ts
index 40a11e8d7..776d2123c 100644
--- a/shared/extra-utils/server/servers-command.ts
+++ b/shared/extra-utils/server/servers-command.ts
@@ -55,7 +55,7 @@ export class ServersCommand extends AbstractCommand {
55 } 55 }
56 56
57 async waitUntilLog (str: string, count = 1, strictCount = true) { 57 async waitUntilLog (str: string, count = 1, strictCount = true) {
58 const logfile = this.server.servers.buildDirectory('logs/peertube.log') 58 const logfile = this.buildDirectory('logs/peertube.log')
59 59
60 while (true) { 60 while (true) {
61 const buf = await readFile(logfile) 61 const buf = await readFile(logfile)
@@ -80,6 +80,10 @@ export class ServersCommand extends AbstractCommand {
80 return this.buildDirectory(join('streaming-playlists', 'hls', videoUUID, basename(fileUrl))) 80 return this.buildDirectory(join('streaming-playlists', 'hls', videoUUID, basename(fileUrl)))
81 } 81 }
82 82
83 getLogContent () {
84 return readFile(this.buildDirectory('logs/peertube.log'))
85 }
86
83 async getServerFileSize (subPath: string) { 87 async getServerFileSize (subPath: string) {
84 const path = this.server.servers.buildDirectory(subPath) 88 const path = this.server.servers.buildDirectory(subPath)
85 89