aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-08-17 10:33:36 +0200
committerChocobozzz <me@florianbigard.com>2021-08-17 10:33:36 +0200
commit66a7fc947321b923fa1e7182202e03d0e21ffb3e (patch)
tree5cb913531d3b84d0f2d4bbcc0d34f0a9a6435bc5
parent0305db28c98fd6cf43a3c50ba92c76215e99d512 (diff)
downloadPeerTube-66a7fc947321b923fa1e7182202e03d0e21ffb3e.tar.gz
PeerTube-66a7fc947321b923fa1e7182202e03d0e21ffb3e.tar.zst
PeerTube-66a7fc947321b923fa1e7182202e03d0e21ffb3e.zip
Remove optimize old videos script
It is not compatible with HLS and I don't have time to maintain it
-rw-r--r--package.json1
-rwxr-xr-xscripts/help.sh1
-rw-r--r--scripts/optimize-old-videos.ts92
-rw-r--r--server/lib/object-storage/keys.ts2
-rw-r--r--server/tests/cli/index.ts1
-rw-r--r--server/tests/cli/optimize-old-videos.ts96
-rw-r--r--support/doc/tools.md18
7 files changed, 1 insertions, 210 deletions
diff --git a/package.json b/package.json
index d2b1e1245..32f4e7c31 100644
--- a/package.json
+++ b/package.json
@@ -53,7 +53,6 @@
53 "generate-cli-doc": "bash ./scripts/generate-cli-doc.sh", 53 "generate-cli-doc": "bash ./scripts/generate-cli-doc.sh",
54 "parse-log": "node ./dist/scripts/parse-log.js", 54 "parse-log": "node ./dist/scripts/parse-log.js",
55 "prune-storage": "node ./dist/scripts/prune-storage.js", 55 "prune-storage": "node ./dist/scripts/prune-storage.js",
56 "optimize-old-videos": "node ./dist/scripts/optimize-old-videos.js",
57 "postinstall": "test -n \"$NOCLIENT\" || (cd client && yarn install --pure-lockfile)", 56 "postinstall": "test -n \"$NOCLIENT\" || (cd client && yarn install --pure-lockfile)",
58 "tsc": "tsc", 57 "tsc": "tsc",
59 "commander": "commander", 58 "commander": "commander",
diff --git a/scripts/help.sh b/scripts/help.sh
index da127092d..b58f52fa0 100755
--- a/scripts/help.sh
+++ b/scripts/help.sh
@@ -13,7 +13,6 @@ printf " reset-password -- -u [user] -> Reset the password of user [user]\n"
13printf " create-transcoding-job -- -v [video UUID] \n" 13printf " create-transcoding-job -- -v [video UUID] \n"
14printf " -> Create a transcoding job for a particular video\n" 14printf " -> Create a transcoding job for a particular video\n"
15printf " prune-storage -> Delete (after confirmation) unknown video files/thumbnails/previews... (due to a bad video deletion, transcoding job not finished...)\n" 15printf " prune-storage -> Delete (after confirmation) unknown video files/thumbnails/previews... (due to a bad video deletion, transcoding job not finished...)\n"
16printf " optimize-old-videos -> Re-transcode videos that have a high bitrate, to make them suitable for streaming over slow connections"
17printf " dev -> Watch, run the livereload and run the server so that you can develop the application\n" 16printf " dev -> Watch, run the livereload and run the server so that you can develop the application\n"
18printf " start -> Run the server\n" 17printf " start -> Run the server\n"
19printf " update-host -> Upgrade scheme/host in torrent files according to the webserver configuration (config/ folder)\n" 18printf " update-host -> Upgrade scheme/host in torrent files according to the webserver configuration (config/ folder)\n"
diff --git a/scripts/optimize-old-videos.ts b/scripts/optimize-old-videos.ts
deleted file mode 100644
index 245e4cf28..000000000
--- a/scripts/optimize-old-videos.ts
+++ /dev/null
@@ -1,92 +0,0 @@
1import { registerTSPaths } from '../server/helpers/register-ts-paths'
2registerTSPaths()
3
4import { copy, move, remove } from 'fs-extra'
5import { basename, dirname } from 'path'
6import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
7import { CONFIG } from '@server/initializers/config'
8import { processMoveToObjectStorage } from '@server/lib/job-queue/handlers/move-to-object-storage'
9import { VideoPathManager } from '@server/lib/video-path-manager'
10import { getMaxBitrate } from '@shared/core-utils'
11import { MoveObjectStoragePayload } from '@shared/models'
12import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffprobe-utils'
13import { initDatabaseModels } from '../server/initializers/database'
14import { optimizeOriginalVideofile } from '../server/lib/transcoding/video-transcoding'
15import { VideoModel } from '../server/models/video/video'
16
17run()
18 .then(() => process.exit(0))
19 .catch(err => {
20 console.error(err)
21 process.exit(-1)
22 })
23
24let currentVideoId: string
25let currentFilePath: string
26
27process.on('SIGINT', async function () {
28 console.log('Cleaning up temp files')
29 await remove(`${currentFilePath}_backup`)
30 await remove(`${dirname(currentFilePath)}/${currentVideoId}-transcoded.mp4`)
31 process.exit(0)
32})
33
34async function run () {
35 await initDatabaseModels(true)
36
37 const localVideos = await VideoModel.listLocal()
38
39 for (const localVideo of localVideos) {
40 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(localVideo.id)
41
42 currentVideoId = video.id
43
44 for (const file of video.VideoFiles) {
45 await VideoPathManager.Instance.makeAvailableVideoFile(video, file, async path => {
46 currentFilePath = path
47
48 const [ videoBitrate, fps, dataResolution ] = await Promise.all([
49 getVideoFileBitrate(currentFilePath),
50 getVideoFileFPS(currentFilePath),
51 getVideoFileResolution(currentFilePath)
52 ])
53
54 const maxBitrate = getMaxBitrate({ ...dataResolution, fps })
55 const isMaxBitrateExceeded = videoBitrate > maxBitrate
56 if (isMaxBitrateExceeded) {
57 console.log(
58 'Optimizing video file %s with bitrate %s kbps (max: %s kbps)',
59 basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000
60 )
61
62 const backupFile = `${currentFilePath}_backup`
63 await copy(currentFilePath, backupFile)
64
65 await optimizeOriginalVideofile(video, file)
66 // Update file path, the video filename changed
67 currentFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
68
69 const originalDuration = await getDurationFromVideoFile(backupFile)
70 const newDuration = await getDurationFromVideoFile(currentFilePath)
71
72 if (originalDuration === newDuration) {
73 console.log('Finished optimizing %s', basename(currentFilePath))
74 await remove(backupFile)
75 return
76 }
77
78 console.log('Failed to optimize %s, restoring original', basename(currentFilePath))
79 await move(backupFile, currentFilePath, { overwrite: true })
80 await createTorrentAndSetInfoHash(video, file)
81 await file.save()
82 }
83 })
84 }
85
86 if (CONFIG.OBJECT_STORAGE.ENABLED === true) {
87 await processMoveToObjectStorage({ data: { videoUUID: video.uuid } as MoveObjectStoragePayload } as any)
88 }
89 }
90
91 console.log('Finished optimizing videos')
92}
diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts
index 519474775..12acb3aec 100644
--- a/server/lib/object-storage/keys.ts
+++ b/server/lib/object-storage/keys.ts
@@ -6,7 +6,7 @@ function generateHLSObjectStorageKey (playlist: MStreamingPlaylist, video: MVide
6} 6}
7 7
8function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID) { 8function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID) {
9 return playlist.getStringType() + '_' + video.uuid 9 return join(playlist.getStringType(), video.uuid)
10} 10}
11 11
12function generateWebTorrentObjectStorageKey (filename: string) { 12function generateWebTorrentObjectStorageKey (filename: string) {
diff --git a/server/tests/cli/index.ts b/server/tests/cli/index.ts
index 7e6eebd17..c6dd0581a 100644
--- a/server/tests/cli/index.ts
+++ b/server/tests/cli/index.ts
@@ -1,7 +1,6 @@
1// Order of the tests we want to execute 1// Order of the tests we want to execute
2import './create-import-video-file-job' 2import './create-import-video-file-job'
3import './create-transcoding-job' 3import './create-transcoding-job'
4import './optimize-old-videos'
5import './peertube' 4import './peertube'
6import './plugins' 5import './plugins'
7import './print-transcode-command' 6import './print-transcode-command'
diff --git a/server/tests/cli/optimize-old-videos.ts b/server/tests/cli/optimize-old-videos.ts
deleted file mode 100644
index 9b75ae164..000000000
--- a/server/tests/cli/optimize-old-videos.ts
+++ /dev/null
@@ -1,96 +0,0 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import { getMaxBitrate } from '@shared/core-utils'
6import {
7 cleanupTests,
8 createMultipleServers,
9 doubleFollow,
10 generateHighBitrateVideo,
11 PeerTubeServer,
12 setAccessTokensToServers,
13 wait,
14 waitJobs
15} from '@shared/extra-utils'
16import { getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../../helpers/ffprobe-utils'
17
18const expect = chai.expect
19
20describe('Test optimize old videos', function () {
21 let servers: PeerTubeServer[] = []
22
23 before(async function () {
24 this.timeout(200000)
25
26 // Run server 2 to have transcoding enabled
27 servers = await createMultipleServers(2)
28 await setAccessTokensToServers(servers)
29
30 await doubleFollow(servers[0], servers[1])
31
32 const tempFixturePath = await generateHighBitrateVideo()
33
34 // Upload two videos for our needs
35 await servers[0].videos.upload({ attributes: { name: 'video1', fixture: tempFixturePath } })
36 await servers[0].videos.upload({ attributes: { name: 'video2', fixture: tempFixturePath } })
37
38 await waitJobs(servers)
39 })
40
41 it('Should have two video files on each server', async function () {
42 this.timeout(30000)
43
44 for (const server of servers) {
45 const { data } = await server.videos.list()
46 expect(data).to.have.lengthOf(2)
47
48 for (const video of data) {
49 const videoDetails = await server.videos.get({ id: video.uuid })
50 expect(videoDetails.files).to.have.lengthOf(1)
51 }
52 }
53 })
54
55 it('Should run optimize script', async function () {
56 this.timeout(200000)
57
58 await servers[0].cli.execWithEnv('npm run optimize-old-videos')
59 await waitJobs(servers)
60
61 for (const server of servers) {
62 const { data } = await server.videos.list()
63 expect(data).to.have.lengthOf(2)
64
65 for (const video of data) {
66 await server.videos.view({ id: video.uuid })
67
68 // Refresh video
69 await waitJobs(servers)
70 await wait(5000)
71 await waitJobs(servers)
72
73 const videoDetails = await server.videos.get({ id: video.uuid })
74
75 expect(videoDetails.files).to.have.lengthOf(1)
76 const file = videoDetails.files[0]
77
78 expect(file.size).to.be.below(8000000)
79
80 const path = servers[0].servers.buildWebTorrentFilePath(file.fileUrl)
81 const bitrate = await getVideoFileBitrate(path)
82 const fps = await getVideoFileFPS(path)
83 const data = await getVideoFileResolution(path)
84
85 expect(data.resolution).to.equal(file.resolution.id)
86
87 const maxBitrate = getMaxBitrate({ ...data, fps })
88 expect(bitrate).to.be.below(maxBitrate)
89 }
90 }
91 })
92
93 after(async function () {
94 await cleanupTests(servers)
95 })
96})
diff --git a/support/doc/tools.md b/support/doc/tools.md
index 1956dc572..433aaeff7 100644
--- a/support/doc/tools.md
+++ b/support/doc/tools.md
@@ -19,7 +19,6 @@
19 - [create-transcoding-job.js](#create-transcoding-jobjs) 19 - [create-transcoding-job.js](#create-transcoding-jobjs)
20 - [create-import-video-file-job.js](#create-import-video-file-jobjs) 20 - [create-import-video-file-job.js](#create-import-video-file-jobjs)
21 - [prune-storage.js](#prune-storagejs) 21 - [prune-storage.js](#prune-storagejs)
22 - [optimize-old-videos.js](#optimize-old-videosjs)
23 - [update-host.js](#update-hostjs) 22 - [update-host.js](#update-hostjs)
24 - [reset-password.js](#reset-passwordjs) 23 - [reset-password.js](#reset-passwordjs)
25 - [plugin install/uninstall](#plugin-installuninstall) 24 - [plugin install/uninstall](#plugin-installuninstall)
@@ -344,23 +343,6 @@ $ cd /var/www/peertube/peertube-latest
344$ sudo systemctl stop peertube && sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run prune-storage 343$ sudo systemctl stop peertube && sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run prune-storage
345``` 344```
346 345
347### optimize-old-videos.js
348
349Before version v1.0.0-beta.16, Peertube did not specify a bitrate for the
350transcoding of uploaded videos. This means that videos might be encoded into
351very large files that are too large for streaming. This script re-transcodes
352these videos so that they can be watched properly, even on slow connections.
353
354```bash
355$ # Basic installation
356$ cd /var/www/peertube/peertube-latest
357$ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run optimize-old-videos
358
359$ # Docker installation
360$ cd /var/www/peertube-docker
361$ docker-compose exec -u peertube peertube npm run optimize-old-videos
362```
363
364 346
365### update-host.js 347### update-host.js
366 348