aboutsummaryrefslogtreecommitdiffhomepage
path: root/scripts
diff options
context:
space:
mode:
authorkontrollanten <6680299+kontrollanten@users.noreply.github.com>2021-11-09 11:05:35 +0100
committerGitHub <noreply@github.com>2021-11-09 11:05:35 +0100
commite1ab52d7ec7370a6f9f5937192d6003206af1ac0 (patch)
treeaecc8b696b0021e073fd205dd6e126fb4f178e8f /scripts
parentc49c366ac320fe5ac3dc08f5891fe5898c1b34e3 (diff)
downloadPeerTube-e1ab52d7ec7370a6f9f5937192d6003206af1ac0.tar.gz
PeerTube-e1ab52d7ec7370a6f9f5937192d6003206af1ac0.tar.zst
PeerTube-e1ab52d7ec7370a6f9f5937192d6003206af1ac0.zip
Add migrate-to-object-storage script (#4481)
* add migrate-to-object-storage-script closes #4467 * add migrate-to-unique-playlist-filenames script * fix(migrate-to-unique-playlist-filenames): update master/segments256 run updateMasterHLSPlaylist and updateSha256VODSegments after file rename. * Improve move to object storage scripts * PR remarks Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'scripts')
-rw-r--r--scripts/create-import-video-file-job.ts2
-rw-r--r--scripts/create-move-video-storage-job.ts86
-rwxr-xr-xscripts/create-transcoding-job.ts2
-rw-r--r--scripts/migrations/peertube-4.0.ts107
-rw-r--r--scripts/regenerate-thumbnails.ts13
-rwxr-xr-xscripts/update-host.ts6
6 files changed, 204 insertions, 12 deletions
diff --git a/scripts/create-import-video-file-job.ts b/scripts/create-import-video-file-job.ts
index 726f51ccf..071d36df4 100644
--- a/scripts/create-import-video-file-job.ts
+++ b/scripts/create-import-video-file-job.ts
@@ -47,7 +47,7 @@ async function run () {
47 filePath: resolve(options.import) 47 filePath: resolve(options.import)
48 } 48 }
49 49
50 JobQueue.Instance.init() 50 JobQueue.Instance.init(true)
51 await JobQueue.Instance.createJobWithPromise({ type: 'video-file-import', payload: dataInput }) 51 await JobQueue.Instance.createJobWithPromise({ type: 'video-file-import', payload: dataInput })
52 console.log('Import job for video %s created.', video.uuid) 52 console.log('Import job for video %s created.', video.uuid)
53} 53}
diff --git a/scripts/create-move-video-storage-job.ts b/scripts/create-move-video-storage-job.ts
new file mode 100644
index 000000000..505bbd61b
--- /dev/null
+++ b/scripts/create-move-video-storage-job.ts
@@ -0,0 +1,86 @@
1import { registerTSPaths } from '../server/helpers/register-ts-paths'
2registerTSPaths()
3
4import { program } from 'commander'
5import { VideoModel } from '@server/models/video/video'
6import { initDatabaseModels } from '@server/initializers/database'
7import { VideoStorage } from '@shared/models'
8import { moveToExternalStorageState } from '@server/lib/video-state'
9import { JobQueue } from '@server/lib/job-queue'
10import { CONFIG } from '@server/initializers/config'
11
12program
13 .description('Move videos to another storage.')
14 .option('-o, --to-object-storage', 'Move videos in object storage')
15 .option('-v, --video [videoUUID]', 'Move a specific video')
16 .option('-a, --all-videos', 'Migrate all videos')
17 .parse(process.argv)
18
19const options = program.opts()
20
21if (!options['toObjectStorage']) {
22 console.error('You need to choose where to send video files.')
23 process.exit(-1)
24}
25
26if (!options['video'] && !options['allVideos']) {
27 console.error('You need to choose which videos to move.')
28 process.exit(-1)
29}
30
31if (options['toObjectStorage'] && !CONFIG.OBJECT_STORAGE.ENABLED) {
32 console.error('Object storage is not enabled on this instance.')
33 process.exit(-1)
34}
35
36run()
37 .then(() => process.exit(0))
38 .catch(err => console.error(err))
39
40async function run () {
41 await initDatabaseModels(true)
42
43 JobQueue.Instance.init(true)
44
45 let ids: number[] = []
46
47 if (options['video']) {
48 const video = await VideoModel.load(options['video'])
49
50 if (!video) {
51 console.error('Unknown video ' + options['video'])
52 process.exit(-1)
53 }
54
55 if (video.remote === true) {
56 console.error('Cannot process a remote video')
57 process.exit(-1)
58 }
59
60 ids.push(video.id)
61 } else {
62 ids = await VideoModel.listLocalIds()
63 }
64
65 for (const id of ids) {
66 const videoFull = await VideoModel.loadAndPopulateAccountAndServerAndTags(id)
67
68 const files = videoFull.VideoFiles || []
69 const hls = videoFull.getHLSPlaylist()
70
71 if (files.some(f => f.storage === VideoStorage.FILE_SYSTEM) || hls?.storage === VideoStorage.FILE_SYSTEM) {
72 console.log('Processing video %s.', videoFull.name)
73
74 const success = await moveToExternalStorageState(videoFull, false, undefined)
75
76 if (!success) {
77 console.error(
78 'Cannot create move job for %s: job creation may have failed or there may be pending transcoding jobs for this video',
79 videoFull.name
80 )
81 }
82 }
83
84 console.log(`Created move-to-object-storage job for ${videoFull.name}.`)
85 }
86}
diff --git a/scripts/create-transcoding-job.ts b/scripts/create-transcoding-job.ts
index fe3bd26de..29c398822 100755
--- a/scripts/create-transcoding-job.ts
+++ b/scripts/create-transcoding-job.ts
@@ -91,7 +91,7 @@ async function run () {
91 } 91 }
92 } 92 }
93 93
94 JobQueue.Instance.init() 94 JobQueue.Instance.init(true)
95 95
96 video.state = VideoState.TO_TRANSCODE 96 video.state = VideoState.TO_TRANSCODE
97 await video.save() 97 await video.save()
diff --git a/scripts/migrations/peertube-4.0.ts b/scripts/migrations/peertube-4.0.ts
new file mode 100644
index 000000000..387f6dc9c
--- /dev/null
+++ b/scripts/migrations/peertube-4.0.ts
@@ -0,0 +1,107 @@
1import { registerTSPaths } from '../../server/helpers/register-ts-paths'
2registerTSPaths()
3
4import { join } from 'path'
5import { JobQueue } from '@server/lib/job-queue'
6import { initDatabaseModels } from '../../server/initializers/database'
7import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
8import { VideoPathManager } from '@server/lib/video-path-manager'
9import { VideoModel } from '@server/models/video/video'
10import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
11import { move, readFile, writeFile } from 'fs-extra'
12import Bluebird from 'bluebird'
13import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
14
15run()
16 .then(() => process.exit(0))
17 .catch(err => {
18 console.error(err)
19 process.exit(-1)
20 })
21
22async function run () {
23 console.log('Migrate old HLS paths to new format.')
24
25 await initDatabaseModels(true)
26
27 JobQueue.Instance.init(true)
28
29 const ids = await VideoModel.listLocalIds()
30
31 await Bluebird.map(ids, async id => {
32 try {
33 await processVideo(id)
34 } catch (err) {
35 console.error('Cannot process video %s.', { err })
36 }
37 }, { concurrency: 5 })
38
39 console.log('Migration finished!')
40}
41
42async function processVideo (videoId: number) {
43 const video = await VideoModel.loadWithFiles(videoId)
44
45 const hls = video.getHLSPlaylist()
46 if (!hls || hls.playlistFilename !== 'master.m3u8' || hls.VideoFiles.length === 0) {
47 return
48 }
49
50 console.log(`Renaming HLS playlist files of video ${video.name}.`)
51
52 const playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
53 const hlsDirPath = VideoPathManager.Instance.getFSHLSOutputPath(video)
54
55 const masterPlaylistPath = join(hlsDirPath, playlist.playlistFilename)
56 let masterPlaylistContent = await readFile(masterPlaylistPath, 'utf8')
57
58 for (const videoFile of hls.VideoFiles) {
59 const srcName = `${videoFile.resolution}.m3u8`
60 const dstName = getHlsResolutionPlaylistFilename(videoFile.filename)
61
62 const src = join(hlsDirPath, srcName)
63 const dst = join(hlsDirPath, dstName)
64
65 try {
66 await move(src, dst)
67
68 masterPlaylistContent = masterPlaylistContent.replace(new RegExp('^' + srcName + '$', 'm'), dstName)
69 } catch (err) {
70 console.error('Cannot move video file %s to %s.', src, dst, err)
71 }
72 }
73
74 await writeFile(masterPlaylistPath, masterPlaylistContent)
75
76 if (playlist.segmentsSha256Filename === 'segments-sha256.json') {
77 try {
78 const newName = generateHlsSha256SegmentsFilename(video.isLive)
79
80 const dst = join(hlsDirPath, newName)
81 await move(join(hlsDirPath, playlist.segmentsSha256Filename), dst)
82 playlist.segmentsSha256Filename = newName
83 } catch (err) {
84 console.error(`Cannot rename ${video.name} segments-sha256.json file to a new name`, err)
85 }
86 }
87
88 if (playlist.playlistFilename === 'master.m3u8') {
89 try {
90 const newName = generateHLSMasterPlaylistFilename(video.isLive)
91
92 const dst = join(hlsDirPath, newName)
93 await move(join(hlsDirPath, playlist.playlistFilename), dst)
94 playlist.playlistFilename = newName
95 } catch (err) {
96 console.error(`Cannot rename ${video.name} master.m3u8 file to a new name`, err)
97 }
98 }
99
100 // Everything worked, we can save the playlist now
101 await playlist.save()
102
103 const allVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.id)
104 await federateVideoIfNeeded(allVideo, false)
105
106 console.log(`Successfully moved HLS files of ${video.name}.`)
107}
diff --git a/scripts/regenerate-thumbnails.ts b/scripts/regenerate-thumbnails.ts
index 8075f90ba..50d06f6fd 100644
--- a/scripts/regenerate-thumbnails.ts
+++ b/scripts/regenerate-thumbnails.ts
@@ -7,7 +7,6 @@ import { pathExists, remove } from 'fs-extra'
7import { generateImageFilename, processImage } from '@server/helpers/image-utils' 7import { generateImageFilename, processImage } from '@server/helpers/image-utils'
8import { THUMBNAILS_SIZE } from '@server/initializers/constants' 8import { THUMBNAILS_SIZE } from '@server/initializers/constants'
9import { VideoModel } from '@server/models/video/video' 9import { VideoModel } from '@server/models/video/video'
10import { MVideo } from '@server/types/models'
11import { initDatabaseModels } from '@server/initializers/database' 10import { initDatabaseModels } from '@server/initializers/database'
12 11
13program 12program
@@ -21,16 +20,16 @@ run()
21async function run () { 20async function run () {
22 await initDatabaseModels(true) 21 await initDatabaseModels(true)
23 22
24 const videos = await VideoModel.listLocal() 23 const ids = await VideoModel.listLocalIds()
25 24
26 await map(videos, v => { 25 await map(ids, id => {
27 return processVideo(v) 26 return processVideo(id)
28 .catch(err => console.error('Cannot process video %s.', v.url, err)) 27 .catch(err => console.error('Cannot process video %d.', id, err))
29 }, { concurrency: 20 }) 28 }, { concurrency: 20 })
30} 29}
31 30
32async function processVideo (videoArg: MVideo) { 31async function processVideo (id: number) {
33 const video = await VideoModel.loadWithFiles(videoArg.id) 32 const video = await VideoModel.loadWithFiles(id)
34 33
35 console.log('Processing video %s.', video.name) 34 console.log('Processing video %s.', video.name)
36 35
diff --git a/scripts/update-host.ts b/scripts/update-host.ts
index 5d81a8d74..c6eb9d533 100755
--- a/scripts/update-host.ts
+++ b/scripts/update-host.ts
@@ -115,9 +115,9 @@ async function run () {
115 115
116 console.log('Updating video and torrent files.') 116 console.log('Updating video and torrent files.')
117 117
118 const localVideos = await VideoModel.listLocal() 118 const ids = await VideoModel.listLocalIds()
119 for (const localVideo of localVideos) { 119 for (const id of ids) {
120 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(localVideo.id) 120 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(id)
121 121
122 console.log('Updating video ' + video.uuid) 122 console.log('Updating video ' + video.uuid)
123 123