diff options
-rw-r--r-- | package.json | 1 | ||||
-rw-r--r-- | scripts/create-import-video-file-job.ts | 2 | ||||
-rw-r--r-- | scripts/create-move-video-storage-job.ts | 86 | ||||
-rwxr-xr-x | scripts/create-transcoding-job.ts | 2 | ||||
-rw-r--r-- | scripts/migrations/peertube-4.0.ts | 107 | ||||
-rw-r--r-- | scripts/regenerate-thumbnails.ts | 13 | ||||
-rwxr-xr-x | scripts/update-host.ts | 6 | ||||
-rw-r--r-- | server/lib/hls.ts | 6 | ||||
-rw-r--r-- | server/lib/job-queue/job-queue.ts | 8 | ||||
-rw-r--r-- | server/lib/video-state.ts | 38 | ||||
-rw-r--r-- | server/models/video/video.ts | 9 | ||||
-rw-r--r-- | server/tests/cli/create-move-video-storage-job.ts | 114 | ||||
-rw-r--r-- | server/tests/cli/index.ts | 1 | ||||
-rw-r--r-- | shared/extra-utils/cli/cli-command.ts | 8 | ||||
-rw-r--r-- | support/doc/tools.md | 28 |
15 files changed, 394 insertions, 35 deletions
diff --git a/package.json b/package.json index 10076d7b2..0d139044f 100644 --- a/package.json +++ b/package.json | |||
@@ -47,6 +47,7 @@ | |||
47 | "create-transcoding-job": "node ./dist/scripts/create-transcoding-job.js", | 47 | "create-transcoding-job": "node ./dist/scripts/create-transcoding-job.js", |
48 | "regenerate-thumbnails": "node ./dist/scripts/regenerate-thumbnails.js", | 48 | "regenerate-thumbnails": "node ./dist/scripts/regenerate-thumbnails.js", |
49 | "create-import-video-file-job": "node ./dist/scripts/create-import-video-file-job.js", | 49 | "create-import-video-file-job": "node ./dist/scripts/create-import-video-file-job.js", |
50 | "create-move-video-storage-job": "node ./dist/scripts/create-move-video-storage-job.js", | ||
50 | "print-transcode-command": "node ./dist/scripts/print-transcode-command.js", | 51 | "print-transcode-command": "node ./dist/scripts/print-transcode-command.js", |
51 | "test": "bash ./scripts/test.sh", | 52 | "test": "bash ./scripts/test.sh", |
52 | "help": "bash ./scripts/help.sh", | 53 | "help": "bash ./scripts/help.sh", |
diff --git a/scripts/create-import-video-file-job.ts b/scripts/create-import-video-file-job.ts index 726f51ccf..071d36df4 100644 --- a/scripts/create-import-video-file-job.ts +++ b/scripts/create-import-video-file-job.ts | |||
@@ -47,7 +47,7 @@ async function run () { | |||
47 | filePath: resolve(options.import) | 47 | filePath: resolve(options.import) |
48 | } | 48 | } |
49 | 49 | ||
50 | JobQueue.Instance.init() | 50 | JobQueue.Instance.init(true) |
51 | await JobQueue.Instance.createJobWithPromise({ type: 'video-file-import', payload: dataInput }) | 51 | await JobQueue.Instance.createJobWithPromise({ type: 'video-file-import', payload: dataInput }) |
52 | console.log('Import job for video %s created.', video.uuid) | 52 | console.log('Import job for video %s created.', video.uuid) |
53 | } | 53 | } |
diff --git a/scripts/create-move-video-storage-job.ts b/scripts/create-move-video-storage-job.ts new file mode 100644 index 000000000..505bbd61b --- /dev/null +++ b/scripts/create-move-video-storage-job.ts | |||
@@ -0,0 +1,86 @@ | |||
1 | import { registerTSPaths } from '../server/helpers/register-ts-paths' | ||
2 | registerTSPaths() | ||
3 | |||
4 | import { program } from 'commander' | ||
5 | import { VideoModel } from '@server/models/video/video' | ||
6 | import { initDatabaseModels } from '@server/initializers/database' | ||
7 | import { VideoStorage } from '@shared/models' | ||
8 | import { moveToExternalStorageState } from '@server/lib/video-state' | ||
9 | import { JobQueue } from '@server/lib/job-queue' | ||
10 | import { CONFIG } from '@server/initializers/config' | ||
11 | |||
12 | program | ||
13 | .description('Move videos to another storage.') | ||
14 | .option('-o, --to-object-storage', 'Move videos in object storage') | ||
15 | .option('-v, --video [videoUUID]', 'Move a specific video') | ||
16 | .option('-a, --all-videos', 'Migrate all videos') | ||
17 | .parse(process.argv) | ||
18 | |||
19 | const options = program.opts() | ||
20 | |||
21 | if (!options['toObjectStorage']) { | ||
22 | console.error('You need to choose where to send video files.') | ||
23 | process.exit(-1) | ||
24 | } | ||
25 | |||
26 | if (!options['video'] && !options['allVideos']) { | ||
27 | console.error('You need to choose which videos to move.') | ||
28 | process.exit(-1) | ||
29 | } | ||
30 | |||
31 | if (options['toObjectStorage'] && !CONFIG.OBJECT_STORAGE.ENABLED) { | ||
32 | console.error('Object storage is not enabled on this instance.') | ||
33 | process.exit(-1) | ||
34 | } | ||
35 | |||
36 | run() | ||
37 | .then(() => process.exit(0)) | ||
38 | .catch(err => console.error(err)) | ||
39 | |||
40 | async function run () { | ||
41 | await initDatabaseModels(true) | ||
42 | |||
43 | JobQueue.Instance.init(true) | ||
44 | |||
45 | let ids: number[] = [] | ||
46 | |||
47 | if (options['video']) { | ||
48 | const video = await VideoModel.load(options['video']) | ||
49 | |||
50 | if (!video) { | ||
51 | console.error('Unknown video ' + options['video']) | ||
52 | process.exit(-1) | ||
53 | } | ||
54 | |||
55 | if (video.remote === true) { | ||
56 | console.error('Cannot process a remote video') | ||
57 | process.exit(-1) | ||
58 | } | ||
59 | |||
60 | ids.push(video.id) | ||
61 | } else { | ||
62 | ids = await VideoModel.listLocalIds() | ||
63 | } | ||
64 | |||
65 | for (const id of ids) { | ||
66 | const videoFull = await VideoModel.loadAndPopulateAccountAndServerAndTags(id) | ||
67 | |||
68 | const files = videoFull.VideoFiles || [] | ||
69 | const hls = videoFull.getHLSPlaylist() | ||
70 | |||
71 | if (files.some(f => f.storage === VideoStorage.FILE_SYSTEM) || hls?.storage === VideoStorage.FILE_SYSTEM) { | ||
72 | console.log('Processing video %s.', videoFull.name) | ||
73 | |||
74 | const success = await moveToExternalStorageState(videoFull, false, undefined) | ||
75 | |||
76 | if (!success) { | ||
77 | console.error( | ||
78 | 'Cannot create move job for %s: job creation may have failed or there may be pending transcoding jobs for this video', | ||
79 | videoFull.name | ||
80 | ) | ||
81 | } | ||
82 | } | ||
83 | |||
84 | console.log(`Created move-to-object-storage job for ${videoFull.name}.`) | ||
85 | } | ||
86 | } | ||
diff --git a/scripts/create-transcoding-job.ts b/scripts/create-transcoding-job.ts index fe3bd26de..29c398822 100755 --- a/scripts/create-transcoding-job.ts +++ b/scripts/create-transcoding-job.ts | |||
@@ -91,7 +91,7 @@ async function run () { | |||
91 | } | 91 | } |
92 | } | 92 | } |
93 | 93 | ||
94 | JobQueue.Instance.init() | 94 | JobQueue.Instance.init(true) |
95 | 95 | ||
96 | video.state = VideoState.TO_TRANSCODE | 96 | video.state = VideoState.TO_TRANSCODE |
97 | await video.save() | 97 | await video.save() |
diff --git a/scripts/migrations/peertube-4.0.ts b/scripts/migrations/peertube-4.0.ts new file mode 100644 index 000000000..387f6dc9c --- /dev/null +++ b/scripts/migrations/peertube-4.0.ts | |||
@@ -0,0 +1,107 @@ | |||
1 | import { registerTSPaths } from '../../server/helpers/register-ts-paths' | ||
2 | registerTSPaths() | ||
3 | |||
4 | import { join } from 'path' | ||
5 | import { JobQueue } from '@server/lib/job-queue' | ||
6 | import { initDatabaseModels } from '../../server/initializers/database' | ||
7 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from '@server/lib/paths' | ||
8 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
9 | import { VideoModel } from '@server/models/video/video' | ||
10 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | ||
11 | import { move, readFile, writeFile } from 'fs-extra' | ||
12 | import Bluebird from 'bluebird' | ||
13 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | ||
14 | |||
15 | run() | ||
16 | .then(() => process.exit(0)) | ||
17 | .catch(err => { | ||
18 | console.error(err) | ||
19 | process.exit(-1) | ||
20 | }) | ||
21 | |||
22 | async function run () { | ||
23 | console.log('Migrate old HLS paths to new format.') | ||
24 | |||
25 | await initDatabaseModels(true) | ||
26 | |||
27 | JobQueue.Instance.init(true) | ||
28 | |||
29 | const ids = await VideoModel.listLocalIds() | ||
30 | |||
31 | await Bluebird.map(ids, async id => { | ||
32 | try { | ||
33 | await processVideo(id) | ||
34 | } catch (err) { | ||
35 | console.error('Cannot process video %s.', { err }) | ||
36 | } | ||
37 | }, { concurrency: 5 }) | ||
38 | |||
39 | console.log('Migration finished!') | ||
40 | } | ||
41 | |||
42 | async function processVideo (videoId: number) { | ||
43 | const video = await VideoModel.loadWithFiles(videoId) | ||
44 | |||
45 | const hls = video.getHLSPlaylist() | ||
46 | if (!hls || hls.playlistFilename !== 'master.m3u8' || hls.VideoFiles.length === 0) { | ||
47 | return | ||
48 | } | ||
49 | |||
50 | console.log(`Renaming HLS playlist files of video ${video.name}.`) | ||
51 | |||
52 | const playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) | ||
53 | const hlsDirPath = VideoPathManager.Instance.getFSHLSOutputPath(video) | ||
54 | |||
55 | const masterPlaylistPath = join(hlsDirPath, playlist.playlistFilename) | ||
56 | let masterPlaylistContent = await readFile(masterPlaylistPath, 'utf8') | ||
57 | |||
58 | for (const videoFile of hls.VideoFiles) { | ||
59 | const srcName = `${videoFile.resolution}.m3u8` | ||
60 | const dstName = getHlsResolutionPlaylistFilename(videoFile.filename) | ||
61 | |||
62 | const src = join(hlsDirPath, srcName) | ||
63 | const dst = join(hlsDirPath, dstName) | ||
64 | |||
65 | try { | ||
66 | await move(src, dst) | ||
67 | |||
68 | masterPlaylistContent = masterPlaylistContent.replace(new RegExp('^' + srcName + '$', 'm'), dstName) | ||
69 | } catch (err) { | ||
70 | console.error('Cannot move video file %s to %s.', src, dst, err) | ||
71 | } | ||
72 | } | ||
73 | |||
74 | await writeFile(masterPlaylistPath, masterPlaylistContent) | ||
75 | |||
76 | if (playlist.segmentsSha256Filename === 'segments-sha256.json') { | ||
77 | try { | ||
78 | const newName = generateHlsSha256SegmentsFilename(video.isLive) | ||
79 | |||
80 | const dst = join(hlsDirPath, newName) | ||
81 | await move(join(hlsDirPath, playlist.segmentsSha256Filename), dst) | ||
82 | playlist.segmentsSha256Filename = newName | ||
83 | } catch (err) { | ||
84 | console.error(`Cannot rename ${video.name} segments-sha256.json file to a new name`, err) | ||
85 | } | ||
86 | } | ||
87 | |||
88 | if (playlist.playlistFilename === 'master.m3u8') { | ||
89 | try { | ||
90 | const newName = generateHLSMasterPlaylistFilename(video.isLive) | ||
91 | |||
92 | const dst = join(hlsDirPath, newName) | ||
93 | await move(join(hlsDirPath, playlist.playlistFilename), dst) | ||
94 | playlist.playlistFilename = newName | ||
95 | } catch (err) { | ||
96 | console.error(`Cannot rename ${video.name} master.m3u8 file to a new name`, err) | ||
97 | } | ||
98 | } | ||
99 | |||
100 | // Everything worked, we can save the playlist now | ||
101 | await playlist.save() | ||
102 | |||
103 | const allVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.id) | ||
104 | await federateVideoIfNeeded(allVideo, false) | ||
105 | |||
106 | console.log(`Successfully moved HLS files of ${video.name}.`) | ||
107 | } | ||
diff --git a/scripts/regenerate-thumbnails.ts b/scripts/regenerate-thumbnails.ts index 8075f90ba..50d06f6fd 100644 --- a/scripts/regenerate-thumbnails.ts +++ b/scripts/regenerate-thumbnails.ts | |||
@@ -7,7 +7,6 @@ import { pathExists, remove } from 'fs-extra' | |||
7 | import { generateImageFilename, processImage } from '@server/helpers/image-utils' | 7 | import { generateImageFilename, processImage } from '@server/helpers/image-utils' |
8 | import { THUMBNAILS_SIZE } from '@server/initializers/constants' | 8 | import { THUMBNAILS_SIZE } from '@server/initializers/constants' |
9 | import { VideoModel } from '@server/models/video/video' | 9 | import { VideoModel } from '@server/models/video/video' |
10 | import { MVideo } from '@server/types/models' | ||
11 | import { initDatabaseModels } from '@server/initializers/database' | 10 | import { initDatabaseModels } from '@server/initializers/database' |
12 | 11 | ||
13 | program | 12 | program |
@@ -21,16 +20,16 @@ run() | |||
21 | async function run () { | 20 | async function run () { |
22 | await initDatabaseModels(true) | 21 | await initDatabaseModels(true) |
23 | 22 | ||
24 | const videos = await VideoModel.listLocal() | 23 | const ids = await VideoModel.listLocalIds() |
25 | 24 | ||
26 | await map(videos, v => { | 25 | await map(ids, id => { |
27 | return processVideo(v) | 26 | return processVideo(id) |
28 | .catch(err => console.error('Cannot process video %s.', v.url, err)) | 27 | .catch(err => console.error('Cannot process video %d.', id, err)) |
29 | }, { concurrency: 20 }) | 28 | }, { concurrency: 20 }) |
30 | } | 29 | } |
31 | 30 | ||
32 | async function processVideo (videoArg: MVideo) { | 31 | async function processVideo (id: number) { |
33 | const video = await VideoModel.loadWithFiles(videoArg.id) | 32 | const video = await VideoModel.loadWithFiles(id) |
34 | 33 | ||
35 | console.log('Processing video %s.', video.name) | 34 | console.log('Processing video %s.', video.name) |
36 | 35 | ||
diff --git a/scripts/update-host.ts b/scripts/update-host.ts index 5d81a8d74..c6eb9d533 100755 --- a/scripts/update-host.ts +++ b/scripts/update-host.ts | |||
@@ -115,9 +115,9 @@ async function run () { | |||
115 | 115 | ||
116 | console.log('Updating video and torrent files.') | 116 | console.log('Updating video and torrent files.') |
117 | 117 | ||
118 | const localVideos = await VideoModel.listLocal() | 118 | const ids = await VideoModel.listLocalIds() |
119 | for (const localVideo of localVideos) { | 119 | for (const id of ids) { |
120 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(localVideo.id) | 120 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(id) |
121 | 121 | ||
122 | console.log('Updating video ' + video.uuid) | 122 | console.log('Updating video ' + video.uuid) |
123 | 123 | ||
diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 0828a2d0f..8160e7949 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' | 1 | import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' |
2 | import { flatten, uniq } from 'lodash' | 2 | import { flatten, uniq } from 'lodash' |
3 | import { basename, dirname, join } from 'path' | 3 | import { basename, dirname, join } from 'path' |
4 | import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' | 4 | import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models' |
5 | import { sha256 } from '../helpers/core-utils' | 5 | import { sha256 } from '../helpers/core-utils' |
6 | import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' | 6 | import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' |
7 | import { logger } from '../helpers/logger' | 7 | import { logger } from '../helpers/logger' |
@@ -31,7 +31,7 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () { | |||
31 | } | 31 | } |
32 | } | 32 | } |
33 | 33 | ||
34 | async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { | 34 | async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlaylistFilesVideo) { |
35 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] | 35 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] |
36 | 36 | ||
37 | for (const file of playlist.VideoFiles) { | 37 | for (const file of playlist.VideoFiles) { |
@@ -63,7 +63,7 @@ async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStream | |||
63 | }) | 63 | }) |
64 | } | 64 | } |
65 | 65 | ||
66 | async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { | 66 | async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingPlaylistFilesVideo) { |
67 | const json: { [filename: string]: { [range: string]: string } } = {} | 67 | const json: { [filename: string]: { [range: string]: string } } = {} |
68 | 68 | ||
69 | // For all the resolutions available for this video | 69 | // For all the resolutions available for this video |
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 53d6b6a9c..0eab720d9 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -108,7 +108,7 @@ class JobQueue { | |||
108 | private constructor () { | 108 | private constructor () { |
109 | } | 109 | } |
110 | 110 | ||
111 | init () { | 111 | init (produceOnly = false) { |
112 | // Already initialized | 112 | // Already initialized |
113 | if (this.initialized === true) return | 113 | if (this.initialized === true) return |
114 | this.initialized = true | 114 | this.initialized = true |
@@ -124,6 +124,12 @@ class JobQueue { | |||
124 | 124 | ||
125 | for (const handlerName of (Object.keys(handlers) as JobType[])) { | 125 | for (const handlerName of (Object.keys(handlers) as JobType[])) { |
126 | const queue = new Bull(handlerName, queueOptions) | 126 | const queue = new Bull(handlerName, queueOptions) |
127 | |||
128 | if (produceOnly) { | ||
129 | queue.pause(true) | ||
130 | .catch(err => logger.error('Cannot pause queue %s in produced only job queue', handlerName, { err })) | ||
131 | } | ||
132 | |||
127 | const handler = handlers[handlerName] | 133 | const handler = handlers[handlerName] |
128 | 134 | ||
129 | queue.process(this.getJobConcurrency(handlerName), handler) | 135 | queue.process(this.getJobConcurrency(handlerName), handler) |
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts index 9352a67d1..d5bbbec43 100644 --- a/server/lib/video-state.ts +++ b/server/lib/video-state.ts | |||
@@ -57,10 +57,33 @@ function moveToNextState (video: MVideoUUID, isNewVideo = true) { | |||
57 | }) | 57 | }) |
58 | } | 58 | } |
59 | 59 | ||
60 | async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) { | ||
61 | const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction) | ||
62 | const pendingTranscode = videoJobInfo?.pendingTranscode || 0 | ||
63 | |||
64 | // We want to wait all transcoding jobs before moving the video on an external storage | ||
65 | if (pendingTranscode !== 0) return false | ||
66 | |||
67 | await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, isNewVideo, transaction) | ||
68 | |||
69 | logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] }) | ||
70 | |||
71 | try { | ||
72 | await addMoveToObjectStorageJob(video, isNewVideo) | ||
73 | |||
74 | return true | ||
75 | } catch (err) { | ||
76 | logger.error('Cannot add move to object storage job', { err }) | ||
77 | |||
78 | return false | ||
79 | } | ||
80 | } | ||
81 | |||
60 | // --------------------------------------------------------------------------- | 82 | // --------------------------------------------------------------------------- |
61 | 83 | ||
62 | export { | 84 | export { |
63 | buildNextVideoState, | 85 | buildNextVideoState, |
86 | moveToExternalStorageState, | ||
64 | moveToNextState | 87 | moveToNextState |
65 | } | 88 | } |
66 | 89 | ||
@@ -82,18 +105,3 @@ async function moveToPublishedState (video: MVideoFullLight, isNewVideo: boolean | |||
82 | Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video) | 105 | Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video) |
83 | } | 106 | } |
84 | } | 107 | } |
85 | |||
86 | async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) { | ||
87 | const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction) | ||
88 | const pendingTranscode = videoJobInfo?.pendingTranscode || 0 | ||
89 | |||
90 | // We want to wait all transcoding jobs before moving the video on an external storage | ||
91 | if (pendingTranscode !== 0) return | ||
92 | |||
93 | await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, isNewVideo, transaction) | ||
94 | |||
95 | logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] }) | ||
96 | |||
97 | addMoveToObjectStorageJob(video, isNewVideo) | ||
98 | .catch(err => logger.error('Cannot add move to object storage job', { err })) | ||
99 | } | ||
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index aef4fd20a..3eed1b58d 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -805,14 +805,17 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
805 | await Promise.all(tasks) | 805 | await Promise.all(tasks) |
806 | } | 806 | } |
807 | 807 | ||
808 | static listLocal (): Promise<MVideo[]> { | 808 | static listLocalIds (): Promise<number[]> { |
809 | const query = { | 809 | const query = { |
810 | attributes: [ 'id' ], | ||
811 | raw: true, | ||
810 | where: { | 812 | where: { |
811 | remote: false | 813 | remote: false |
812 | } | 814 | } |
813 | } | 815 | } |
814 | 816 | ||
815 | return VideoModel.findAll(query) | 817 | return VideoModel.findAll(query) |
818 | .then(rows => rows.map(r => r.id)) | ||
816 | } | 819 | } |
817 | 820 | ||
818 | static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { | 821 | static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { |
@@ -1674,6 +1677,8 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1674 | if (!this.VideoStreamingPlaylists) return undefined | 1677 | if (!this.VideoStreamingPlaylists) return undefined |
1675 | 1678 | ||
1676 | const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) | 1679 | const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) |
1680 | if (!playlist) return undefined | ||
1681 | |||
1677 | playlist.Video = this | 1682 | playlist.Video = this |
1678 | 1683 | ||
1679 | return playlist | 1684 | return playlist |
@@ -1785,7 +1790,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1785 | await this.save({ transaction }) | 1790 | await this.save({ transaction }) |
1786 | } | 1791 | } |
1787 | 1792 | ||
1788 | getBandwidthBits (videoFile: MVideoFile) { | 1793 | getBandwidthBits (this: MVideo, videoFile: MVideoFile) { |
1789 | return Math.ceil((videoFile.size * 8) / this.duration) | 1794 | return Math.ceil((videoFile.size * 8) / this.duration) |
1790 | } | 1795 | } |
1791 | 1796 | ||
diff --git a/server/tests/cli/create-move-video-storage-job.ts b/server/tests/cli/create-move-video-storage-job.ts new file mode 100644 index 000000000..b598c8359 --- /dev/null +++ b/server/tests/cli/create-move-video-storage-job.ts | |||
@@ -0,0 +1,114 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | |||
5 | import { | ||
6 | areObjectStorageTestsDisabled, | ||
7 | cleanupTests, | ||
8 | createMultipleServers, | ||
9 | doubleFollow, | ||
10 | expectStartWith, | ||
11 | makeRawRequest, | ||
12 | ObjectStorageCommand, | ||
13 | PeerTubeServer, | ||
14 | setAccessTokensToServers, | ||
15 | waitJobs | ||
16 | } from '@shared/extra-utils' | ||
17 | import { HttpStatusCode, VideoDetails } from '@shared/models' | ||
18 | |||
19 | async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObjectStorage: boolean) { | ||
20 | for (const file of video.files) { | ||
21 | const start = inObjectStorage | ||
22 | ? ObjectStorageCommand.getWebTorrentBaseUrl() | ||
23 | : origin.url | ||
24 | |||
25 | expectStartWith(file.fileUrl, start) | ||
26 | |||
27 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
28 | } | ||
29 | |||
30 | const start = inObjectStorage | ||
31 | ? ObjectStorageCommand.getPlaylistBaseUrl() | ||
32 | : origin.url | ||
33 | |||
34 | const hls = video.streamingPlaylists[0] | ||
35 | expectStartWith(hls.playlistUrl, start) | ||
36 | expectStartWith(hls.segmentsSha256Url, start) | ||
37 | |||
38 | for (const file of hls.files) { | ||
39 | expectStartWith(file.fileUrl, start) | ||
40 | |||
41 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
42 | } | ||
43 | } | ||
44 | |||
45 | describe('Test create move video storage job', function () { | ||
46 | if (areObjectStorageTestsDisabled()) return | ||
47 | |||
48 | let servers: PeerTubeServer[] = [] | ||
49 | const uuids: string[] = [] | ||
50 | |||
51 | before(async function () { | ||
52 | this.timeout(360000) | ||
53 | |||
54 | // Run server 2 to have transcoding enabled | ||
55 | servers = await createMultipleServers(2) | ||
56 | await setAccessTokensToServers(servers) | ||
57 | |||
58 | await doubleFollow(servers[0], servers[1]) | ||
59 | |||
60 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
61 | |||
62 | await servers[0].config.enableTranscoding() | ||
63 | |||
64 | for (let i = 0; i < 3; i++) { | ||
65 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) | ||
66 | uuids.push(uuid) | ||
67 | } | ||
68 | |||
69 | await waitJobs(servers) | ||
70 | |||
71 | await servers[0].kill() | ||
72 | await servers[0].run(ObjectStorageCommand.getDefaultConfig()) | ||
73 | }) | ||
74 | |||
75 | it('Should move only one file', async function () { | ||
76 | this.timeout(120000) | ||
77 | |||
78 | const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}` | ||
79 | await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig()) | ||
80 | await waitJobs(servers) | ||
81 | |||
82 | for (const server of servers) { | ||
83 | const video = await server.videos.get({ id: uuids[1] }) | ||
84 | |||
85 | await checkFiles(servers[0], video, true) | ||
86 | |||
87 | for (const id of [ uuids[0], uuids[2] ]) { | ||
88 | const video = await server.videos.get({ id }) | ||
89 | |||
90 | await checkFiles(servers[0], video, false) | ||
91 | } | ||
92 | } | ||
93 | }) | ||
94 | |||
95 | it('Should move all files', async function () { | ||
96 | this.timeout(120000) | ||
97 | |||
98 | const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos` | ||
99 | await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig()) | ||
100 | await waitJobs(servers) | ||
101 | |||
102 | for (const server of servers) { | ||
103 | for (const id of [ uuids[0], uuids[2] ]) { | ||
104 | const video = await server.videos.get({ id }) | ||
105 | |||
106 | await checkFiles(servers[0], video, true) | ||
107 | } | ||
108 | } | ||
109 | }) | ||
110 | |||
111 | after(async function () { | ||
112 | await cleanupTests(servers) | ||
113 | }) | ||
114 | }) | ||
diff --git a/server/tests/cli/index.ts b/server/tests/cli/index.ts index c6dd0581a..6e0cbe58b 100644 --- a/server/tests/cli/index.ts +++ b/server/tests/cli/index.ts | |||
@@ -1,6 +1,7 @@ | |||
1 | // Order of the tests we want to execute | 1 | // Order of the tests we want to execute |
2 | import './create-import-video-file-job' | 2 | import './create-import-video-file-job' |
3 | import './create-transcoding-job' | 3 | import './create-transcoding-job' |
4 | import './create-move-video-storage-job' | ||
4 | import './peertube' | 5 | import './peertube' |
5 | import './plugins' | 6 | import './plugins' |
6 | import './print-transcode-command' | 7 | import './print-transcode-command' |
diff --git a/shared/extra-utils/cli/cli-command.ts b/shared/extra-utils/cli/cli-command.ts index bc1dddc68..ab9738174 100644 --- a/shared/extra-utils/cli/cli-command.ts +++ b/shared/extra-utils/cli/cli-command.ts | |||
@@ -17,7 +17,11 @@ export class CLICommand extends AbstractCommand { | |||
17 | return `NODE_ENV=test NODE_APP_INSTANCE=${this.server.internalServerNumber}` | 17 | return `NODE_ENV=test NODE_APP_INSTANCE=${this.server.internalServerNumber}` |
18 | } | 18 | } |
19 | 19 | ||
20 | async execWithEnv (command: string) { | 20 | async execWithEnv (command: string, configOverride?: any) { |
21 | return CLICommand.exec(`${this.getEnv()} ${command}`) | 21 | const prefix = configOverride |
22 | ? `NODE_CONFIG='${JSON.stringify(configOverride)}'` | ||
23 | : '' | ||
24 | |||
25 | return CLICommand.exec(`${prefix} ${this.getEnv()} ${command}`) | ||
22 | } | 26 | } |
23 | } | 27 | } |
diff --git a/support/doc/tools.md b/support/doc/tools.md index 78ace1344..c8cc2d1d7 100644 --- a/support/doc/tools.md +++ b/support/doc/tools.md | |||
@@ -17,6 +17,7 @@ | |||
17 | - [regenerate-thumbnails.js](#regenerate-thumbnailsjs) | 17 | - [regenerate-thumbnails.js](#regenerate-thumbnailsjs) |
18 | - [create-transcoding-job.js](#create-transcoding-jobjs) | 18 | - [create-transcoding-job.js](#create-transcoding-jobjs) |
19 | - [create-import-video-file-job.js](#create-import-video-file-jobjs) | 19 | - [create-import-video-file-job.js](#create-import-video-file-jobjs) |
20 | - [create-move-video-storage-job.js](#create-move-video-storage-jobjs) | ||
20 | - [prune-storage.js](#prune-storagejs) | 21 | - [prune-storage.js](#prune-storagejs) |
21 | - [update-host.js](#update-hostjs) | 22 | - [update-host.js](#update-hostjs) |
22 | - [reset-password.js](#reset-passwordjs) | 23 | - [reset-password.js](#reset-passwordjs) |
@@ -303,6 +304,33 @@ $ cd /var/www/peertube-docker | |||
303 | $ docker-compose exec -u peertube peertube npm run create-import-video-file-job -- -v [videoUUID] -i [videoFile] | 304 | $ docker-compose exec -u peertube peertube npm run create-import-video-file-job -- -v [videoUUID] -i [videoFile] |
304 | ``` | 305 | ``` |
305 | 306 | ||
307 | ### create-move-video-storage-job.js | ||
308 | |||
309 | Use this script to move all video files or a specific video file to object storage. | ||
310 | |||
311 | ```bash | ||
312 | $ # Basic installation | ||
313 | $ cd /var/www/peertube/peertube-latest | ||
314 | $ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run create-move-video-storage-job -- --to-object-storage -v [videoUUID] | ||
315 | |||
316 | $ # Docker installation | ||
317 | $ cd /var/www/peertube-docker | ||
318 | $ docker-compose exec -u peertube peertube npm run create-move-video-storage-job -- --to-object-storage -v [videoUUID] | ||
319 | ``` | ||
320 | |||
321 | The script can also move all video files that are not already in object storage: | ||
322 | |||
323 | ```bash | ||
324 | $ # Basic installation | ||
325 | $ cd /var/www/peertube/peertube-latest | ||
326 | $ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run create-move-video-storage-job -- --to-object-storage --all-videos | ||
327 | |||
328 | $ # Docker installation | ||
329 | $ cd /var/www/peertube-docker | ||
330 | $ docker-compose exec -u peertube peertube npm run create-move-video-storage-job -- --to-object-storage --all-videos | ||
331 | ``` | ||
332 | |||
333 | |||
306 | ### prune-storage.js | 334 | ### prune-storage.js |
307 | 335 | ||
308 | Some transcoded videos or shutdown at a bad time can leave some unused files on your storage. | 336 | Some transcoded videos or shutdown at a bad time can leave some unused files on your storage. |