aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorkontrollanten <6680299+kontrollanten@users.noreply.github.com>2021-11-09 11:05:35 +0100
committerGitHub <noreply@github.com>2021-11-09 11:05:35 +0100
commite1ab52d7ec7370a6f9f5937192d6003206af1ac0 (patch)
treeaecc8b696b0021e073fd205dd6e126fb4f178e8f
parentc49c366ac320fe5ac3dc08f5891fe5898c1b34e3 (diff)
downloadPeerTube-e1ab52d7ec7370a6f9f5937192d6003206af1ac0.tar.gz
PeerTube-e1ab52d7ec7370a6f9f5937192d6003206af1ac0.tar.zst
PeerTube-e1ab52d7ec7370a6f9f5937192d6003206af1ac0.zip
Add migrate-to-object-storage script (#4481)
* add migrate-to-object-storage-script closes #4467 * add migrate-to-unique-playlist-filenames script * fix(migrate-to-unique-playlist-filenames): update master/segments256 run updateMasterHLSPlaylist and updateSha256VODSegments after file rename. * Improve move to object storage scripts * PR remarks Co-authored-by: Chocobozzz <me@florianbigard.com>
-rw-r--r--package.json1
-rw-r--r--scripts/create-import-video-file-job.ts2
-rw-r--r--scripts/create-move-video-storage-job.ts86
-rwxr-xr-xscripts/create-transcoding-job.ts2
-rw-r--r--scripts/migrations/peertube-4.0.ts107
-rw-r--r--scripts/regenerate-thumbnails.ts13
-rwxr-xr-xscripts/update-host.ts6
-rw-r--r--server/lib/hls.ts6
-rw-r--r--server/lib/job-queue/job-queue.ts8
-rw-r--r--server/lib/video-state.ts38
-rw-r--r--server/models/video/video.ts9
-rw-r--r--server/tests/cli/create-move-video-storage-job.ts114
-rw-r--r--server/tests/cli/index.ts1
-rw-r--r--shared/extra-utils/cli/cli-command.ts8
-rw-r--r--support/doc/tools.md28
15 files changed, 394 insertions, 35 deletions
diff --git a/package.json b/package.json
index 10076d7b2..0d139044f 100644
--- a/package.json
+++ b/package.json
@@ -47,6 +47,7 @@
47 "create-transcoding-job": "node ./dist/scripts/create-transcoding-job.js", 47 "create-transcoding-job": "node ./dist/scripts/create-transcoding-job.js",
48 "regenerate-thumbnails": "node ./dist/scripts/regenerate-thumbnails.js", 48 "regenerate-thumbnails": "node ./dist/scripts/regenerate-thumbnails.js",
49 "create-import-video-file-job": "node ./dist/scripts/create-import-video-file-job.js", 49 "create-import-video-file-job": "node ./dist/scripts/create-import-video-file-job.js",
50 "create-move-video-storage-job": "node ./dist/scripts/create-move-video-storage-job.js",
50 "print-transcode-command": "node ./dist/scripts/print-transcode-command.js", 51 "print-transcode-command": "node ./dist/scripts/print-transcode-command.js",
51 "test": "bash ./scripts/test.sh", 52 "test": "bash ./scripts/test.sh",
52 "help": "bash ./scripts/help.sh", 53 "help": "bash ./scripts/help.sh",
diff --git a/scripts/create-import-video-file-job.ts b/scripts/create-import-video-file-job.ts
index 726f51ccf..071d36df4 100644
--- a/scripts/create-import-video-file-job.ts
+++ b/scripts/create-import-video-file-job.ts
@@ -47,7 +47,7 @@ async function run () {
47 filePath: resolve(options.import) 47 filePath: resolve(options.import)
48 } 48 }
49 49
50 JobQueue.Instance.init() 50 JobQueue.Instance.init(true)
51 await JobQueue.Instance.createJobWithPromise({ type: 'video-file-import', payload: dataInput }) 51 await JobQueue.Instance.createJobWithPromise({ type: 'video-file-import', payload: dataInput })
52 console.log('Import job for video %s created.', video.uuid) 52 console.log('Import job for video %s created.', video.uuid)
53} 53}
diff --git a/scripts/create-move-video-storage-job.ts b/scripts/create-move-video-storage-job.ts
new file mode 100644
index 000000000..505bbd61b
--- /dev/null
+++ b/scripts/create-move-video-storage-job.ts
@@ -0,0 +1,86 @@
1import { registerTSPaths } from '../server/helpers/register-ts-paths'
2registerTSPaths()
3
4import { program } from 'commander'
5import { VideoModel } from '@server/models/video/video'
6import { initDatabaseModels } from '@server/initializers/database'
7import { VideoStorage } from '@shared/models'
8import { moveToExternalStorageState } from '@server/lib/video-state'
9import { JobQueue } from '@server/lib/job-queue'
10import { CONFIG } from '@server/initializers/config'
11
12program
13 .description('Move videos to another storage.')
14 .option('-o, --to-object-storage', 'Move videos in object storage')
15 .option('-v, --video [videoUUID]', 'Move a specific video')
16 .option('-a, --all-videos', 'Migrate all videos')
17 .parse(process.argv)
18
19const options = program.opts()
20
21if (!options['toObjectStorage']) {
22 console.error('You need to choose where to send video files.')
23 process.exit(-1)
24}
25
26if (!options['video'] && !options['allVideos']) {
27 console.error('You need to choose which videos to move.')
28 process.exit(-1)
29}
30
31if (options['toObjectStorage'] && !CONFIG.OBJECT_STORAGE.ENABLED) {
32 console.error('Object storage is not enabled on this instance.')
33 process.exit(-1)
34}
35
36run()
37 .then(() => process.exit(0))
38 .catch(err => console.error(err))
39
40async function run () {
41 await initDatabaseModels(true)
42
43 JobQueue.Instance.init(true)
44
45 let ids: number[] = []
46
47 if (options['video']) {
48 const video = await VideoModel.load(options['video'])
49
50 if (!video) {
51 console.error('Unknown video ' + options['video'])
52 process.exit(-1)
53 }
54
55 if (video.remote === true) {
56 console.error('Cannot process a remote video')
57 process.exit(-1)
58 }
59
60 ids.push(video.id)
61 } else {
62 ids = await VideoModel.listLocalIds()
63 }
64
65 for (const id of ids) {
66 const videoFull = await VideoModel.loadAndPopulateAccountAndServerAndTags(id)
67
68 const files = videoFull.VideoFiles || []
69 const hls = videoFull.getHLSPlaylist()
70
71 if (files.some(f => f.storage === VideoStorage.FILE_SYSTEM) || hls?.storage === VideoStorage.FILE_SYSTEM) {
72 console.log('Processing video %s.', videoFull.name)
73
74 const success = await moveToExternalStorageState(videoFull, false, undefined)
75
76 if (!success) {
77 console.error(
78 'Cannot create move job for %s: job creation may have failed or there may be pending transcoding jobs for this video',
79 videoFull.name
80 )
81 }
82 }
83
84 console.log(`Created move-to-object-storage job for ${videoFull.name}.`)
85 }
86}
diff --git a/scripts/create-transcoding-job.ts b/scripts/create-transcoding-job.ts
index fe3bd26de..29c398822 100755
--- a/scripts/create-transcoding-job.ts
+++ b/scripts/create-transcoding-job.ts
@@ -91,7 +91,7 @@ async function run () {
91 } 91 }
92 } 92 }
93 93
94 JobQueue.Instance.init() 94 JobQueue.Instance.init(true)
95 95
96 video.state = VideoState.TO_TRANSCODE 96 video.state = VideoState.TO_TRANSCODE
97 await video.save() 97 await video.save()
diff --git a/scripts/migrations/peertube-4.0.ts b/scripts/migrations/peertube-4.0.ts
new file mode 100644
index 000000000..387f6dc9c
--- /dev/null
+++ b/scripts/migrations/peertube-4.0.ts
@@ -0,0 +1,107 @@
1import { registerTSPaths } from '../../server/helpers/register-ts-paths'
2registerTSPaths()
3
4import { join } from 'path'
5import { JobQueue } from '@server/lib/job-queue'
6import { initDatabaseModels } from '../../server/initializers/database'
7import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
8import { VideoPathManager } from '@server/lib/video-path-manager'
9import { VideoModel } from '@server/models/video/video'
10import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
11import { move, readFile, writeFile } from 'fs-extra'
12import Bluebird from 'bluebird'
13import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
14
15run()
16 .then(() => process.exit(0))
17 .catch(err => {
18 console.error(err)
19 process.exit(-1)
20 })
21
22async function run () {
23 console.log('Migrate old HLS paths to new format.')
24
25 await initDatabaseModels(true)
26
27 JobQueue.Instance.init(true)
28
29 const ids = await VideoModel.listLocalIds()
30
31 await Bluebird.map(ids, async id => {
32 try {
33 await processVideo(id)
34 } catch (err) {
35 console.error('Cannot process video %s.', { err })
36 }
37 }, { concurrency: 5 })
38
39 console.log('Migration finished!')
40}
41
42async function processVideo (videoId: number) {
43 const video = await VideoModel.loadWithFiles(videoId)
44
45 const hls = video.getHLSPlaylist()
46 if (!hls || hls.playlistFilename !== 'master.m3u8' || hls.VideoFiles.length === 0) {
47 return
48 }
49
50 console.log(`Renaming HLS playlist files of video ${video.name}.`)
51
52 const playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
53 const hlsDirPath = VideoPathManager.Instance.getFSHLSOutputPath(video)
54
55 const masterPlaylistPath = join(hlsDirPath, playlist.playlistFilename)
56 let masterPlaylistContent = await readFile(masterPlaylistPath, 'utf8')
57
58 for (const videoFile of hls.VideoFiles) {
59 const srcName = `${videoFile.resolution}.m3u8`
60 const dstName = getHlsResolutionPlaylistFilename(videoFile.filename)
61
62 const src = join(hlsDirPath, srcName)
63 const dst = join(hlsDirPath, dstName)
64
65 try {
66 await move(src, dst)
67
68 masterPlaylistContent = masterPlaylistContent.replace(new RegExp('^' + srcName + '$', 'm'), dstName)
69 } catch (err) {
70 console.error('Cannot move video file %s to %s.', src, dst, err)
71 }
72 }
73
74 await writeFile(masterPlaylistPath, masterPlaylistContent)
75
76 if (playlist.segmentsSha256Filename === 'segments-sha256.json') {
77 try {
78 const newName = generateHlsSha256SegmentsFilename(video.isLive)
79
80 const dst = join(hlsDirPath, newName)
81 await move(join(hlsDirPath, playlist.segmentsSha256Filename), dst)
82 playlist.segmentsSha256Filename = newName
83 } catch (err) {
84 console.error(`Cannot rename ${video.name} segments-sha256.json file to a new name`, err)
85 }
86 }
87
88 if (playlist.playlistFilename === 'master.m3u8') {
89 try {
90 const newName = generateHLSMasterPlaylistFilename(video.isLive)
91
92 const dst = join(hlsDirPath, newName)
93 await move(join(hlsDirPath, playlist.playlistFilename), dst)
94 playlist.playlistFilename = newName
95 } catch (err) {
96 console.error(`Cannot rename ${video.name} master.m3u8 file to a new name`, err)
97 }
98 }
99
100 // Everything worked, we can save the playlist now
101 await playlist.save()
102
103 const allVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.id)
104 await federateVideoIfNeeded(allVideo, false)
105
106 console.log(`Successfully moved HLS files of ${video.name}.`)
107}
diff --git a/scripts/regenerate-thumbnails.ts b/scripts/regenerate-thumbnails.ts
index 8075f90ba..50d06f6fd 100644
--- a/scripts/regenerate-thumbnails.ts
+++ b/scripts/regenerate-thumbnails.ts
@@ -7,7 +7,6 @@ import { pathExists, remove } from 'fs-extra'
7import { generateImageFilename, processImage } from '@server/helpers/image-utils' 7import { generateImageFilename, processImage } from '@server/helpers/image-utils'
8import { THUMBNAILS_SIZE } from '@server/initializers/constants' 8import { THUMBNAILS_SIZE } from '@server/initializers/constants'
9import { VideoModel } from '@server/models/video/video' 9import { VideoModel } from '@server/models/video/video'
10import { MVideo } from '@server/types/models'
11import { initDatabaseModels } from '@server/initializers/database' 10import { initDatabaseModels } from '@server/initializers/database'
12 11
13program 12program
@@ -21,16 +20,16 @@ run()
21async function run () { 20async function run () {
22 await initDatabaseModels(true) 21 await initDatabaseModels(true)
23 22
24 const videos = await VideoModel.listLocal() 23 const ids = await VideoModel.listLocalIds()
25 24
26 await map(videos, v => { 25 await map(ids, id => {
27 return processVideo(v) 26 return processVideo(id)
28 .catch(err => console.error('Cannot process video %s.', v.url, err)) 27 .catch(err => console.error('Cannot process video %d.', id, err))
29 }, { concurrency: 20 }) 28 }, { concurrency: 20 })
30} 29}
31 30
32async function processVideo (videoArg: MVideo) { 31async function processVideo (id: number) {
33 const video = await VideoModel.loadWithFiles(videoArg.id) 32 const video = await VideoModel.loadWithFiles(id)
34 33
35 console.log('Processing video %s.', video.name) 34 console.log('Processing video %s.', video.name)
36 35
diff --git a/scripts/update-host.ts b/scripts/update-host.ts
index 5d81a8d74..c6eb9d533 100755
--- a/scripts/update-host.ts
+++ b/scripts/update-host.ts
@@ -115,9 +115,9 @@ async function run () {
115 115
116 console.log('Updating video and torrent files.') 116 console.log('Updating video and torrent files.')
117 117
118 const localVideos = await VideoModel.listLocal() 118 const ids = await VideoModel.listLocalIds()
119 for (const localVideo of localVideos) { 119 for (const id of ids) {
120 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(localVideo.id) 120 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(id)
121 121
122 console.log('Updating video ' + video.uuid) 122 console.log('Updating video ' + video.uuid)
123 123
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 0828a2d0f..8160e7949 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -1,7 +1,7 @@
1import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' 1import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra'
2import { flatten, uniq } from 'lodash' 2import { flatten, uniq } from 'lodash'
3import { basename, dirname, join } from 'path' 3import { basename, dirname, join } from 'path'
4import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' 4import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models'
5import { sha256 } from '../helpers/core-utils' 5import { sha256 } from '../helpers/core-utils'
6import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' 6import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
7import { logger } from '../helpers/logger' 7import { logger } from '../helpers/logger'
@@ -31,7 +31,7 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () {
31 } 31 }
32} 32}
33 33
34async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { 34async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlaylistFilesVideo) {
35 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] 35 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
36 36
37 for (const file of playlist.VideoFiles) { 37 for (const file of playlist.VideoFiles) {
@@ -63,7 +63,7 @@ async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStream
63 }) 63 })
64} 64}
65 65
66async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { 66async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingPlaylistFilesVideo) {
67 const json: { [filename: string]: { [range: string]: string } } = {} 67 const json: { [filename: string]: { [range: string]: string } } = {}
68 68
69 // For all the resolutions available for this video 69 // For all the resolutions available for this video
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 53d6b6a9c..0eab720d9 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -108,7 +108,7 @@ class JobQueue {
108 private constructor () { 108 private constructor () {
109 } 109 }
110 110
111 init () { 111 init (produceOnly = false) {
112 // Already initialized 112 // Already initialized
113 if (this.initialized === true) return 113 if (this.initialized === true) return
114 this.initialized = true 114 this.initialized = true
@@ -124,6 +124,12 @@ class JobQueue {
124 124
125 for (const handlerName of (Object.keys(handlers) as JobType[])) { 125 for (const handlerName of (Object.keys(handlers) as JobType[])) {
126 const queue = new Bull(handlerName, queueOptions) 126 const queue = new Bull(handlerName, queueOptions)
127
128 if (produceOnly) {
129 queue.pause(true)
130 .catch(err => logger.error('Cannot pause queue %s in produced only job queue', handlerName, { err }))
131 }
132
127 const handler = handlers[handlerName] 133 const handler = handlers[handlerName]
128 134
129 queue.process(this.getJobConcurrency(handlerName), handler) 135 queue.process(this.getJobConcurrency(handlerName), handler)
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts
index 9352a67d1..d5bbbec43 100644
--- a/server/lib/video-state.ts
+++ b/server/lib/video-state.ts
@@ -57,10 +57,33 @@ function moveToNextState (video: MVideoUUID, isNewVideo = true) {
57 }) 57 })
58} 58}
59 59
60async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) {
61 const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction)
62 const pendingTranscode = videoJobInfo?.pendingTranscode || 0
63
64 // We want to wait all transcoding jobs before moving the video on an external storage
65 if (pendingTranscode !== 0) return false
66
67 await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, isNewVideo, transaction)
68
69 logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] })
70
71 try {
72 await addMoveToObjectStorageJob(video, isNewVideo)
73
74 return true
75 } catch (err) {
76 logger.error('Cannot add move to object storage job', { err })
77
78 return false
79 }
80}
81
60// --------------------------------------------------------------------------- 82// ---------------------------------------------------------------------------
61 83
62export { 84export {
63 buildNextVideoState, 85 buildNextVideoState,
86 moveToExternalStorageState,
64 moveToNextState 87 moveToNextState
65} 88}
66 89
@@ -82,18 +105,3 @@ async function moveToPublishedState (video: MVideoFullLight, isNewVideo: boolean
82 Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video) 105 Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video)
83 } 106 }
84} 107}
85
86async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) {
87 const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction)
88 const pendingTranscode = videoJobInfo?.pendingTranscode || 0
89
90 // We want to wait all transcoding jobs before moving the video on an external storage
91 if (pendingTranscode !== 0) return
92
93 await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, isNewVideo, transaction)
94
95 logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] })
96
97 addMoveToObjectStorageJob(video, isNewVideo)
98 .catch(err => logger.error('Cannot add move to object storage job', { err }))
99}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index aef4fd20a..3eed1b58d 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -805,14 +805,17 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
805 await Promise.all(tasks) 805 await Promise.all(tasks)
806 } 806 }
807 807
808 static listLocal (): Promise<MVideo[]> { 808 static listLocalIds (): Promise<number[]> {
809 const query = { 809 const query = {
810 attributes: [ 'id' ],
811 raw: true,
810 where: { 812 where: {
811 remote: false 813 remote: false
812 } 814 }
813 } 815 }
814 816
815 return VideoModel.findAll(query) 817 return VideoModel.findAll(query)
818 .then(rows => rows.map(r => r.id))
816 } 819 }
817 820
818 static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { 821 static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) {
@@ -1674,6 +1677,8 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1674 if (!this.VideoStreamingPlaylists) return undefined 1677 if (!this.VideoStreamingPlaylists) return undefined
1675 1678
1676 const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) 1679 const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
1680 if (!playlist) return undefined
1681
1677 playlist.Video = this 1682 playlist.Video = this
1678 1683
1679 return playlist 1684 return playlist
@@ -1785,7 +1790,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1785 await this.save({ transaction }) 1790 await this.save({ transaction })
1786 } 1791 }
1787 1792
1788 getBandwidthBits (videoFile: MVideoFile) { 1793 getBandwidthBits (this: MVideo, videoFile: MVideoFile) {
1789 return Math.ceil((videoFile.size * 8) / this.duration) 1794 return Math.ceil((videoFile.size * 8) / this.duration)
1790 } 1795 }
1791 1796
diff --git a/server/tests/cli/create-move-video-storage-job.ts b/server/tests/cli/create-move-video-storage-job.ts
new file mode 100644
index 000000000..b598c8359
--- /dev/null
+++ b/server/tests/cli/create-move-video-storage-job.ts
@@ -0,0 +1,114 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4
5import {
6 areObjectStorageTestsDisabled,
7 cleanupTests,
8 createMultipleServers,
9 doubleFollow,
10 expectStartWith,
11 makeRawRequest,
12 ObjectStorageCommand,
13 PeerTubeServer,
14 setAccessTokensToServers,
15 waitJobs
16} from '@shared/extra-utils'
17import { HttpStatusCode, VideoDetails } from '@shared/models'
18
19async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObjectStorage: boolean) {
20 for (const file of video.files) {
21 const start = inObjectStorage
22 ? ObjectStorageCommand.getWebTorrentBaseUrl()
23 : origin.url
24
25 expectStartWith(file.fileUrl, start)
26
27 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
28 }
29
30 const start = inObjectStorage
31 ? ObjectStorageCommand.getPlaylistBaseUrl()
32 : origin.url
33
34 const hls = video.streamingPlaylists[0]
35 expectStartWith(hls.playlistUrl, start)
36 expectStartWith(hls.segmentsSha256Url, start)
37
38 for (const file of hls.files) {
39 expectStartWith(file.fileUrl, start)
40
41 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
42 }
43}
44
45describe('Test create move video storage job', function () {
46 if (areObjectStorageTestsDisabled()) return
47
48 let servers: PeerTubeServer[] = []
49 const uuids: string[] = []
50
51 before(async function () {
52 this.timeout(360000)
53
54 // Run server 2 to have transcoding enabled
55 servers = await createMultipleServers(2)
56 await setAccessTokensToServers(servers)
57
58 await doubleFollow(servers[0], servers[1])
59
60 await ObjectStorageCommand.prepareDefaultBuckets()
61
62 await servers[0].config.enableTranscoding()
63
64 for (let i = 0; i < 3; i++) {
65 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } })
66 uuids.push(uuid)
67 }
68
69 await waitJobs(servers)
70
71 await servers[0].kill()
72 await servers[0].run(ObjectStorageCommand.getDefaultConfig())
73 })
74
75 it('Should move only one file', async function () {
76 this.timeout(120000)
77
78 const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}`
79 await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig())
80 await waitJobs(servers)
81
82 for (const server of servers) {
83 const video = await server.videos.get({ id: uuids[1] })
84
85 await checkFiles(servers[0], video, true)
86
87 for (const id of [ uuids[0], uuids[2] ]) {
88 const video = await server.videos.get({ id })
89
90 await checkFiles(servers[0], video, false)
91 }
92 }
93 })
94
95 it('Should move all files', async function () {
96 this.timeout(120000)
97
98 const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos`
99 await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig())
100 await waitJobs(servers)
101
102 for (const server of servers) {
103 for (const id of [ uuids[0], uuids[2] ]) {
104 const video = await server.videos.get({ id })
105
106 await checkFiles(servers[0], video, true)
107 }
108 }
109 })
110
111 after(async function () {
112 await cleanupTests(servers)
113 })
114})
diff --git a/server/tests/cli/index.ts b/server/tests/cli/index.ts
index c6dd0581a..6e0cbe58b 100644
--- a/server/tests/cli/index.ts
+++ b/server/tests/cli/index.ts
@@ -1,6 +1,7 @@
1// Order of the tests we want to execute 1// Order of the tests we want to execute
2import './create-import-video-file-job' 2import './create-import-video-file-job'
3import './create-transcoding-job' 3import './create-transcoding-job'
4import './create-move-video-storage-job'
4import './peertube' 5import './peertube'
5import './plugins' 6import './plugins'
6import './print-transcode-command' 7import './print-transcode-command'
diff --git a/shared/extra-utils/cli/cli-command.ts b/shared/extra-utils/cli/cli-command.ts
index bc1dddc68..ab9738174 100644
--- a/shared/extra-utils/cli/cli-command.ts
+++ b/shared/extra-utils/cli/cli-command.ts
@@ -17,7 +17,11 @@ export class CLICommand extends AbstractCommand {
17 return `NODE_ENV=test NODE_APP_INSTANCE=${this.server.internalServerNumber}` 17 return `NODE_ENV=test NODE_APP_INSTANCE=${this.server.internalServerNumber}`
18 } 18 }
19 19
20 async execWithEnv (command: string) { 20 async execWithEnv (command: string, configOverride?: any) {
21 return CLICommand.exec(`${this.getEnv()} ${command}`) 21 const prefix = configOverride
22 ? `NODE_CONFIG='${JSON.stringify(configOverride)}'`
23 : ''
24
25 return CLICommand.exec(`${prefix} ${this.getEnv()} ${command}`)
22 } 26 }
23} 27}
diff --git a/support/doc/tools.md b/support/doc/tools.md
index 78ace1344..c8cc2d1d7 100644
--- a/support/doc/tools.md
+++ b/support/doc/tools.md
@@ -17,6 +17,7 @@
17 - [regenerate-thumbnails.js](#regenerate-thumbnailsjs) 17 - [regenerate-thumbnails.js](#regenerate-thumbnailsjs)
18 - [create-transcoding-job.js](#create-transcoding-jobjs) 18 - [create-transcoding-job.js](#create-transcoding-jobjs)
19 - [create-import-video-file-job.js](#create-import-video-file-jobjs) 19 - [create-import-video-file-job.js](#create-import-video-file-jobjs)
20 - [create-move-video-storage-job.js](#create-move-video-storage-jobjs)
20 - [prune-storage.js](#prune-storagejs) 21 - [prune-storage.js](#prune-storagejs)
21 - [update-host.js](#update-hostjs) 22 - [update-host.js](#update-hostjs)
22 - [reset-password.js](#reset-passwordjs) 23 - [reset-password.js](#reset-passwordjs)
@@ -303,6 +304,33 @@ $ cd /var/www/peertube-docker
303$ docker-compose exec -u peertube peertube npm run create-import-video-file-job -- -v [videoUUID] -i [videoFile] 304$ docker-compose exec -u peertube peertube npm run create-import-video-file-job -- -v [videoUUID] -i [videoFile]
304``` 305```
305 306
307### create-move-video-storage-job.js
308
309Use this script to move all video files or a specific video file to object storage.
310
311```bash
312$ # Basic installation
313$ cd /var/www/peertube/peertube-latest
314$ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run create-move-video-storage-job -- --to-object-storage -v [videoUUID]
315
316$ # Docker installation
317$ cd /var/www/peertube-docker
318$ docker-compose exec -u peertube peertube npm run create-move-video-storage-job -- --to-object-storage -v [videoUUID]
319```
320
321The script can also move all video files that are not already in object storage:
322
323```bash
324$ # Basic installation
325$ cd /var/www/peertube/peertube-latest
326$ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run create-move-video-storage-job -- --to-object-storage --all-videos
327
328$ # Docker installation
329$ cd /var/www/peertube-docker
330$ docker-compose exec -u peertube peertube npm run create-move-video-storage-job -- --to-object-storage --all-videos
331```
332
333
306### prune-storage.js 334### prune-storage.js
307 335
308Some transcoded videos or shutdown at a bad time can leave some unused files on your storage. 336Some transcoded videos or shutdown at a bad time can leave some unused files on your storage.