From e1ab52d7ec7370a6f9f5937192d6003206af1ac0 Mon Sep 17 00:00:00 2001
From: kontrollanten <6680299+kontrollanten@users.noreply.github.com>
Date: Tue, 9 Nov 2021 11:05:35 +0100
Subject: Add migrate-to-object-storage script (#4481)
* add migrate-to-object-storage-script
closes #4467
* add migrate-to-unique-playlist-filenames script
* fix(migrate-to-unique-playlist-filenames): update master/segments256
run updateMasterHLSPlaylist and updateSha256VODSegments after
file rename.
* Improve move to object storage scripts
* PR remarks
Co-authored-by: Chocobozzz <me@florianbigard.com>
---
scripts/create-import-video-file-job.ts | 2 +-
scripts/create-move-video-storage-job.ts | 86 +++++++++++++++++++++++++
scripts/create-transcoding-job.ts | 2 +-
scripts/migrations/peertube-4.0.ts | 107 +++++++++++++++++++++++++++++++
scripts/regenerate-thumbnails.ts | 13 ++--
scripts/update-host.ts | 6 +-
6 files changed, 204 insertions(+), 12 deletions(-)
create mode 100644 scripts/create-move-video-storage-job.ts
create mode 100644 scripts/migrations/peertube-4.0.ts
(limited to 'scripts')
diff --git a/scripts/create-import-video-file-job.ts b/scripts/create-import-video-file-job.ts
index 726f51ccf..071d36df4 100644
--- a/scripts/create-import-video-file-job.ts
+++ b/scripts/create-import-video-file-job.ts
@@ -47,7 +47,7 @@ async function run () {
filePath: resolve(options.import)
}
- JobQueue.Instance.init()
+ JobQueue.Instance.init(true)
await JobQueue.Instance.createJobWithPromise({ type: 'video-file-import', payload: dataInput })
console.log('Import job for video %s created.', video.uuid)
}
diff --git a/scripts/create-move-video-storage-job.ts b/scripts/create-move-video-storage-job.ts
new file mode 100644
index 000000000..505bbd61b
--- /dev/null
+++ b/scripts/create-move-video-storage-job.ts
@@ -0,0 +1,86 @@
+import { registerTSPaths } from '../server/helpers/register-ts-paths'
+registerTSPaths()
+
+import { program } from 'commander'
+import { VideoModel } from '@server/models/video/video'
+import { initDatabaseModels } from '@server/initializers/database'
+import { VideoStorage } from '@shared/models'
+import { moveToExternalStorageState } from '@server/lib/video-state'
+import { JobQueue } from '@server/lib/job-queue'
+import { CONFIG } from '@server/initializers/config'
+
+program
+ .description('Move videos to another storage.')
+ .option('-o, --to-object-storage', 'Move videos in object storage')
+ .option('-v, --video [videoUUID]', 'Move a specific video')
+ .option('-a, --all-videos', 'Migrate all videos')
+ .parse(process.argv)
+
+const options = program.opts()
+
+if (!options['toObjectStorage']) {
+ console.error('You need to choose where to send video files.')
+ process.exit(-1)
+}
+
+if (!options['video'] && !options['allVideos']) {
+ console.error('You need to choose which videos to move.')
+ process.exit(-1)
+}
+
+if (options['toObjectStorage'] && !CONFIG.OBJECT_STORAGE.ENABLED) {
+ console.error('Object storage is not enabled on this instance.')
+ process.exit(-1)
+}
+
+run()
+ .then(() => process.exit(0))
+ .catch(err => console.error(err))
+
+async function run () {
+ await initDatabaseModels(true)
+
+ JobQueue.Instance.init(true)
+
+ let ids: number[] = []
+
+ if (options['video']) {
+ const video = await VideoModel.load(options['video'])
+
+ if (!video) {
+ console.error('Unknown video ' + options['video'])
+ process.exit(-1)
+ }
+
+ if (video.remote === true) {
+ console.error('Cannot process a remote video')
+ process.exit(-1)
+ }
+
+ ids.push(video.id)
+ } else {
+ ids = await VideoModel.listLocalIds()
+ }
+
+ for (const id of ids) {
+ const videoFull = await VideoModel.loadAndPopulateAccountAndServerAndTags(id)
+
+ const files = videoFull.VideoFiles || []
+ const hls = videoFull.getHLSPlaylist()
+
+ if (files.some(f => f.storage === VideoStorage.FILE_SYSTEM) || hls?.storage === VideoStorage.FILE_SYSTEM) {
+ console.log('Processing video %s.', videoFull.name)
+
+ const success = await moveToExternalStorageState(videoFull, false, undefined)
+
+ if (!success) {
+ console.error(
+ 'Cannot create move job for %s: job creation may have failed or there may be pending transcoding jobs for this video',
+ videoFull.name
+ )
+ }
+ }
+
+ console.log(`Created move-to-object-storage job for ${videoFull.name}.`)
+ }
+}
diff --git a/scripts/create-transcoding-job.ts b/scripts/create-transcoding-job.ts
index fe3bd26de..29c398822 100755
--- a/scripts/create-transcoding-job.ts
+++ b/scripts/create-transcoding-job.ts
@@ -91,7 +91,7 @@ async function run () {
}
}
- JobQueue.Instance.init()
+ JobQueue.Instance.init(true)
video.state = VideoState.TO_TRANSCODE
await video.save()
diff --git a/scripts/migrations/peertube-4.0.ts b/scripts/migrations/peertube-4.0.ts
new file mode 100644
index 000000000..387f6dc9c
--- /dev/null
+++ b/scripts/migrations/peertube-4.0.ts
@@ -0,0 +1,107 @@
+import { registerTSPaths } from '../../server/helpers/register-ts-paths'
+registerTSPaths()
+
+import { join } from 'path'
+import { JobQueue } from '@server/lib/job-queue'
+import { initDatabaseModels } from '../../server/initializers/database'
+import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
+import { VideoPathManager } from '@server/lib/video-path-manager'
+import { VideoModel } from '@server/models/video/video'
+import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
+import { move, readFile, writeFile } from 'fs-extra'
+import Bluebird from 'bluebird'
+import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
+
+run()
+ .then(() => process.exit(0))
+ .catch(err => {
+ console.error(err)
+ process.exit(-1)
+ })
+
+async function run () {
+ console.log('Migrate old HLS paths to new format.')
+
+ await initDatabaseModels(true)
+
+ JobQueue.Instance.init(true)
+
+ const ids = await VideoModel.listLocalIds()
+
+ await Bluebird.map(ids, async id => {
+ try {
+ await processVideo(id)
+ } catch (err) {
+ console.error('Cannot process video %s.', { err })
+ }
+ }, { concurrency: 5 })
+
+ console.log('Migration finished!')
+}
+
+async function processVideo (videoId: number) {
+ const video = await VideoModel.loadWithFiles(videoId)
+
+ const hls = video.getHLSPlaylist()
+ if (!hls || hls.playlistFilename !== 'master.m3u8' || hls.VideoFiles.length === 0) {
+ return
+ }
+
+ console.log(`Renaming HLS playlist files of video ${video.name}.`)
+
+ const playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
+ const hlsDirPath = VideoPathManager.Instance.getFSHLSOutputPath(video)
+
+ const masterPlaylistPath = join(hlsDirPath, playlist.playlistFilename)
+ let masterPlaylistContent = await readFile(masterPlaylistPath, 'utf8')
+
+ for (const videoFile of hls.VideoFiles) {
+ const srcName = `${videoFile.resolution}.m3u8`
+ const dstName = getHlsResolutionPlaylistFilename(videoFile.filename)
+
+ const src = join(hlsDirPath, srcName)
+ const dst = join(hlsDirPath, dstName)
+
+ try {
+ await move(src, dst)
+
+ masterPlaylistContent = masterPlaylistContent.replace(new RegExp('^' + srcName + '$', 'm'), dstName)
+ } catch (err) {
+ console.error('Cannot move video file %s to %s.', src, dst, err)
+ }
+ }
+
+ await writeFile(masterPlaylistPath, masterPlaylistContent)
+
+ if (playlist.segmentsSha256Filename === 'segments-sha256.json') {
+ try {
+ const newName = generateHlsSha256SegmentsFilename(video.isLive)
+
+ const dst = join(hlsDirPath, newName)
+ await move(join(hlsDirPath, playlist.segmentsSha256Filename), dst)
+ playlist.segmentsSha256Filename = newName
+ } catch (err) {
+ console.error(`Cannot rename ${video.name} segments-sha256.json file to a new name`, err)
+ }
+ }
+
+ if (playlist.playlistFilename === 'master.m3u8') {
+ try {
+ const newName = generateHLSMasterPlaylistFilename(video.isLive)
+
+ const dst = join(hlsDirPath, newName)
+ await move(join(hlsDirPath, playlist.playlistFilename), dst)
+ playlist.playlistFilename = newName
+ } catch (err) {
+ console.error(`Cannot rename ${video.name} master.m3u8 file to a new name`, err)
+ }
+ }
+
+ // Everything worked, we can save the playlist now
+ await playlist.save()
+
+ const allVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.id)
+ await federateVideoIfNeeded(allVideo, false)
+
+ console.log(`Successfully moved HLS files of ${video.name}.`)
+}
diff --git a/scripts/regenerate-thumbnails.ts b/scripts/regenerate-thumbnails.ts
index 8075f90ba..50d06f6fd 100644
--- a/scripts/regenerate-thumbnails.ts
+++ b/scripts/regenerate-thumbnails.ts
@@ -7,7 +7,6 @@ import { pathExists, remove } from 'fs-extra'
import { generateImageFilename, processImage } from '@server/helpers/image-utils'
import { THUMBNAILS_SIZE } from '@server/initializers/constants'
import { VideoModel } from '@server/models/video/video'
-import { MVideo } from '@server/types/models'
import { initDatabaseModels } from '@server/initializers/database'
program
@@ -21,16 +20,16 @@ run()
async function run () {
await initDatabaseModels(true)
- const videos = await VideoModel.listLocal()
+ const ids = await VideoModel.listLocalIds()
- await map(videos, v => {
- return processVideo(v)
- .catch(err => console.error('Cannot process video %s.', v.url, err))
+ await map(ids, id => {
+ return processVideo(id)
+ .catch(err => console.error('Cannot process video %d.', id, err))
}, { concurrency: 20 })
}
-async function processVideo (videoArg: MVideo) {
- const video = await VideoModel.loadWithFiles(videoArg.id)
+async function processVideo (id: number) {
+ const video = await VideoModel.loadWithFiles(id)
console.log('Processing video %s.', video.name)
diff --git a/scripts/update-host.ts b/scripts/update-host.ts
index 5d81a8d74..c6eb9d533 100755
--- a/scripts/update-host.ts
+++ b/scripts/update-host.ts
@@ -115,9 +115,9 @@ async function run () {
console.log('Updating video and torrent files.')
- const localVideos = await VideoModel.listLocal()
- for (const localVideo of localVideos) {
- const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(localVideo.id)
+ const ids = await VideoModel.listLocalIds()
+ for (const id of ids) {
+ const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(id)
console.log('Updating video ' + video.uuid)
--
cgit v1.2.3