diff options
Diffstat (limited to 'server')
53 files changed, 2342 insertions, 506 deletions
diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 89f50714d..5c740c041 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts | |||
@@ -1,12 +1,21 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { move } from 'fs-extra' | 2 | import { move } from 'fs-extra' |
3 | import { basename } from 'path' | ||
3 | import { getLowercaseExtension } from '@server/helpers/core-utils' | 4 | import { getLowercaseExtension } from '@server/helpers/core-utils' |
4 | import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' | 5 | import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' |
5 | import { uuidToShort } from '@server/helpers/uuid' | 6 | import { uuidToShort } from '@server/helpers/uuid' |
6 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 7 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
7 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' | 8 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' |
8 | import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' | 9 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' |
9 | import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' | 10 | import { |
11 | addMoveToObjectStorageJob, | ||
12 | addOptimizeOrMergeAudioJob, | ||
13 | buildLocalVideoFromReq, | ||
14 | buildVideoThumbnailsFromReq, | ||
15 | setVideoTags | ||
16 | } from '@server/lib/video' | ||
17 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
18 | import { buildNextVideoState } from '@server/lib/video-state' | ||
10 | import { openapiOperationDoc } from '@server/middlewares/doc' | 19 | import { openapiOperationDoc } from '@server/middlewares/doc' |
11 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 20 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
12 | import { uploadx } from '@uploadx/core' | 21 | import { uploadx } from '@uploadx/core' |
@@ -139,23 +148,20 @@ async function addVideo (options: { | |||
139 | 148 | ||
140 | const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) | 149 | const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) |
141 | 150 | ||
142 | videoData.state = CONFIG.TRANSCODING.ENABLED | 151 | videoData.state = buildNextVideoState() |
143 | ? VideoState.TO_TRANSCODE | ||
144 | : VideoState.PUBLISHED | ||
145 | |||
146 | videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware | 152 | videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware |
147 | 153 | ||
148 | const video = new VideoModel(videoData) as MVideoFullLight | 154 | const video = new VideoModel(videoData) as MVideoFullLight |
149 | video.VideoChannel = videoChannel | 155 | video.VideoChannel = videoChannel |
150 | video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object | 156 | video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object |
151 | 157 | ||
152 | const videoFile = await buildNewFile(video, videoPhysicalFile) | 158 | const videoFile = await buildNewFile(videoPhysicalFile) |
153 | 159 | ||
154 | // Move physical file | 160 | // Move physical file |
155 | const destination = getVideoFilePath(video, videoFile) | 161 | const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile) |
156 | await move(videoPhysicalFile.path, destination) | 162 | await move(videoPhysicalFile.path, destination) |
157 | // This is important in case if there is another attempt in the retry process | 163 | // This is important in case if there is another attempt in the retry process |
158 | videoPhysicalFile.filename = getVideoFilePath(video, videoFile) | 164 | videoPhysicalFile.filename = basename(destination) |
159 | videoPhysicalFile.path = destination | 165 | videoPhysicalFile.path = destination |
160 | 166 | ||
161 | const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ | 167 | const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ |
@@ -210,9 +216,13 @@ async function addVideo (options: { | |||
210 | 216 | ||
211 | createTorrentFederate(video, videoFile) | 217 | createTorrentFederate(video, videoFile) |
212 | .then(() => { | 218 | .then(() => { |
213 | if (video.state !== VideoState.TO_TRANSCODE) return | 219 | if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { |
220 | return addMoveToObjectStorageJob(video) | ||
221 | } | ||
214 | 222 | ||
215 | return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) | 223 | if (video.state === VideoState.TO_TRANSCODE) { |
224 | return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) | ||
225 | } | ||
216 | }) | 226 | }) |
217 | .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) })) | 227 | .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) })) |
218 | 228 | ||
@@ -227,7 +237,7 @@ async function addVideo (options: { | |||
227 | }) | 237 | }) |
228 | } | 238 | } |
229 | 239 | ||
230 | async function buildNewFile (video: MVideo, videoPhysicalFile: express.VideoUploadFile) { | 240 | async function buildNewFile (videoPhysicalFile: express.VideoUploadFile) { |
231 | const videoFile = new VideoFileModel({ | 241 | const videoFile = new VideoFileModel({ |
232 | extname: getLowercaseExtension(videoPhysicalFile.filename), | 242 | extname: getLowercaseExtension(videoPhysicalFile.filename), |
233 | size: videoPhysicalFile.size, | 243 | size: videoPhysicalFile.size, |
diff --git a/server/controllers/download.ts b/server/controllers/download.ts index ddacc1b68..ffe40d57e 100644 --- a/server/controllers/download.ts +++ b/server/controllers/download.ts | |||
@@ -3,9 +3,9 @@ import * as express from 'express' | |||
3 | import { logger } from '@server/helpers/logger' | 3 | import { logger } from '@server/helpers/logger' |
4 | import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' | 4 | import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' |
5 | import { Hooks } from '@server/lib/plugins/hooks' | 5 | import { Hooks } from '@server/lib/plugins/hooks' |
6 | import { getVideoFilePath } from '@server/lib/video-paths' | 6 | import { VideoPathManager } from '@server/lib/video-path-manager' |
7 | import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 7 | import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
8 | import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models' | 8 | import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models' |
9 | import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' | 9 | import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' |
10 | import { asyncMiddleware, videosDownloadValidator } from '../middlewares' | 10 | import { asyncMiddleware, videosDownloadValidator } from '../middlewares' |
11 | 11 | ||
@@ -81,7 +81,15 @@ async function downloadVideoFile (req: express.Request, res: express.Response) { | |||
81 | 81 | ||
82 | if (!checkAllowResult(res, allowParameters, allowedResult)) return | 82 | if (!checkAllowResult(res, allowParameters, allowedResult)) return |
83 | 83 | ||
84 | return res.download(getVideoFilePath(video, videoFile), `${video.name}-${videoFile.resolution}p${videoFile.extname}`) | 84 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { |
85 | return res.redirect(videoFile.getObjectStorageUrl()) | ||
86 | } | ||
87 | |||
88 | await VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, path => { | ||
89 | const filename = `${video.name}-${videoFile.resolution}p${videoFile.extname}` | ||
90 | |||
91 | return res.download(path, filename) | ||
92 | }) | ||
85 | } | 93 | } |
86 | 94 | ||
87 | async function downloadHLSVideoFile (req: express.Request, res: express.Response) { | 95 | async function downloadHLSVideoFile (req: express.Request, res: express.Response) { |
@@ -107,8 +115,15 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response | |||
107 | 115 | ||
108 | if (!checkAllowResult(res, allowParameters, allowedResult)) return | 116 | if (!checkAllowResult(res, allowParameters, allowedResult)) return |
109 | 117 | ||
110 | const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}` | 118 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { |
111 | return res.download(getVideoFilePath(streamingPlaylist, videoFile), filename) | 119 | return res.redirect(videoFile.getObjectStorageUrl()) |
120 | } | ||
121 | |||
122 | await VideoPathManager.Instance.makeAvailableVideoFile(streamingPlaylist, videoFile, path => { | ||
123 | const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}` | ||
124 | |||
125 | return res.download(path, filename) | ||
126 | }) | ||
112 | } | 127 | } |
113 | 128 | ||
114 | function getVideoFile (req: express.Request, files: MVideoFile[]) { | 129 | function getVideoFile (req: express.Request, files: MVideoFile[]) { |
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index ecf63e93e..c84376304 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts | |||
@@ -6,7 +6,8 @@ import { dirname, join } from 'path' | |||
6 | import * as WebTorrent from 'webtorrent' | 6 | import * as WebTorrent from 'webtorrent' |
7 | import { isArray } from '@server/helpers/custom-validators/misc' | 7 | import { isArray } from '@server/helpers/custom-validators/misc' |
8 | import { WEBSERVER } from '@server/initializers/constants' | 8 | import { WEBSERVER } from '@server/initializers/constants' |
9 | import { generateTorrentFileName, getVideoFilePath } from '@server/lib/video-paths' | 9 | import { generateTorrentFileName } from '@server/lib/paths' |
10 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
10 | import { MVideo } from '@server/types/models/video/video' | 11 | import { MVideo } from '@server/types/models/video/video' |
11 | import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' | 12 | import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' |
12 | import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' | 13 | import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' |
@@ -78,7 +79,7 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName | |||
78 | }) | 79 | }) |
79 | } | 80 | } |
80 | 81 | ||
81 | async function createTorrentAndSetInfoHash ( | 82 | function createTorrentAndSetInfoHash ( |
82 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo, | 83 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo, |
83 | videoFile: MVideoFile | 84 | videoFile: MVideoFile |
84 | ) { | 85 | ) { |
@@ -95,22 +96,24 @@ async function createTorrentAndSetInfoHash ( | |||
95 | urlList: [ videoFile.getFileUrl(video) ] | 96 | urlList: [ videoFile.getFileUrl(video) ] |
96 | } | 97 | } |
97 | 98 | ||
98 | const torrent = await createTorrentPromise(getVideoFilePath(videoOrPlaylist, videoFile), options) | 99 | return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => { |
100 | const torrent = await createTorrentPromise(videoPath, options) | ||
99 | 101 | ||
100 | const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) | 102 | const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) |
101 | const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) | 103 | const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) |
102 | logger.info('Creating torrent %s.', torrentPath) | 104 | logger.info('Creating torrent %s.', torrentPath) |
103 | 105 | ||
104 | await writeFile(torrentPath, torrent) | 106 | await writeFile(torrentPath, torrent) |
105 | 107 | ||
106 | // Remove old torrent file if it existed | 108 | // Remove old torrent file if it existed |
107 | if (videoFile.hasTorrent()) { | 109 | if (videoFile.hasTorrent()) { |
108 | await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) | 110 | await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) |
109 | } | 111 | } |
110 | 112 | ||
111 | const parsedTorrent = parseTorrent(torrent) | 113 | const parsedTorrent = parseTorrent(torrent) |
112 | videoFile.infoHash = parsedTorrent.infoHash | 114 | videoFile.infoHash = parsedTorrent.infoHash |
113 | videoFile.torrentFilename = torrentFilename | 115 | videoFile.torrentFilename = torrentFilename |
116 | }) | ||
114 | } | 117 | } |
115 | 118 | ||
116 | function generateMagnetUri ( | 119 | function generateMagnetUri ( |
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index 911734fa0..09f587274 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts | |||
@@ -153,6 +153,29 @@ function checkConfig () { | |||
153 | } | 153 | } |
154 | } | 154 | } |
155 | 155 | ||
156 | // Object storage | ||
157 | if (CONFIG.OBJECT_STORAGE.ENABLED === true) { | ||
158 | |||
159 | if (!CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME) { | ||
160 | return 'videos_bucket should be set when object storage support is enabled.' | ||
161 | } | ||
162 | |||
163 | if (!CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME) { | ||
164 | return 'streaming_playlists_bucket should be set when object storage support is enabled.' | ||
165 | } | ||
166 | |||
167 | if ( | ||
168 | CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME && | ||
169 | CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX | ||
170 | ) { | ||
171 | if (CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === '') { | ||
172 | return 'Object storage bucket prefixes should be set when the same bucket is used for both types of video.' | ||
173 | } else { | ||
174 | return 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.' | ||
175 | } | ||
176 | } | ||
177 | } | ||
178 | |||
156 | return null | 179 | return null |
157 | } | 180 | } |
158 | 181 | ||
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 30a9823b9..0e684eef8 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -73,6 +73,26 @@ const CONFIG = { | |||
73 | PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')), | 73 | PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')), |
74 | CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')) | 74 | CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')) |
75 | }, | 75 | }, |
76 | OBJECT_STORAGE: { | ||
77 | ENABLED: config.get<boolean>('object_storage.enabled'), | ||
78 | MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')), | ||
79 | ENDPOINT: config.get<string>('object_storage.endpoint'), | ||
80 | REGION: config.get<string>('object_storage.region'), | ||
81 | CREDENTIALS: { | ||
82 | ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'), | ||
83 | SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key') | ||
84 | }, | ||
85 | VIDEOS: { | ||
86 | BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'), | ||
87 | PREFIX: config.get<string>('object_storage.videos.prefix'), | ||
88 | BASE_URL: config.get<string>('object_storage.videos.base_url') | ||
89 | }, | ||
90 | STREAMING_PLAYLISTS: { | ||
91 | BUCKET_NAME: config.get<string>('object_storage.streaming_playlists.bucket_name'), | ||
92 | PREFIX: config.get<string>('object_storage.streaming_playlists.prefix'), | ||
93 | BASE_URL: config.get<string>('object_storage.streaming_playlists.base_url') | ||
94 | } | ||
95 | }, | ||
76 | WEBSERVER: { | 96 | WEBSERVER: { |
77 | SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http', | 97 | SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http', |
78 | WS: config.get<boolean>('webserver.https') === true ? 'wss' : 'ws', | 98 | WS: config.get<boolean>('webserver.https') === true ? 'wss' : 'ws', |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 5f121d9a4..8a1526ae8 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' | |||
24 | 24 | ||
25 | // --------------------------------------------------------------------------- | 25 | // --------------------------------------------------------------------------- |
26 | 26 | ||
27 | const LAST_MIGRATION_VERSION = 655 | 27 | const LAST_MIGRATION_VERSION = 660 |
28 | 28 | ||
29 | // --------------------------------------------------------------------------- | 29 | // --------------------------------------------------------------------------- |
30 | 30 | ||
@@ -147,7 +147,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { | |||
147 | 'videos-views': 1, | 147 | 'videos-views': 1, |
148 | 'activitypub-refresher': 1, | 148 | 'activitypub-refresher': 1, |
149 | 'video-redundancy': 1, | 149 | 'video-redundancy': 1, |
150 | 'video-live-ending': 1 | 150 | 'video-live-ending': 1, |
151 | 'move-to-object-storage': 3 | ||
151 | } | 152 | } |
152 | // Excluded keys are jobs that can be configured by admins | 153 | // Excluded keys are jobs that can be configured by admins |
153 | const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = { | 154 | const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = { |
@@ -162,7 +163,8 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im | |||
162 | 'videos-views': 1, | 163 | 'videos-views': 1, |
163 | 'activitypub-refresher': 1, | 164 | 'activitypub-refresher': 1, |
164 | 'video-redundancy': 1, | 165 | 'video-redundancy': 1, |
165 | 'video-live-ending': 10 | 166 | 'video-live-ending': 10, |
167 | 'move-to-object-storage': 1 | ||
166 | } | 168 | } |
167 | const JOB_TTL: { [id in JobType]: number } = { | 169 | const JOB_TTL: { [id in JobType]: number } = { |
168 | 'activitypub-http-broadcast': 60000 * 10, // 10 minutes | 170 | 'activitypub-http-broadcast': 60000 * 10, // 10 minutes |
@@ -178,7 +180,8 @@ const JOB_TTL: { [id in JobType]: number } = { | |||
178 | 'videos-views': undefined, // Unlimited | 180 | 'videos-views': undefined, // Unlimited |
179 | 'activitypub-refresher': 60000 * 10, // 10 minutes | 181 | 'activitypub-refresher': 60000 * 10, // 10 minutes |
180 | 'video-redundancy': 1000 * 3600 * 3, // 3 hours | 182 | 'video-redundancy': 1000 * 3600 * 3, // 3 hours |
181 | 'video-live-ending': 1000 * 60 * 10 // 10 minutes | 183 | 'video-live-ending': 1000 * 60 * 10, // 10 minutes |
184 | 'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours | ||
182 | } | 185 | } |
183 | const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { | 186 | const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { |
184 | 'videos-views': { | 187 | 'videos-views': { |
@@ -412,7 +415,8 @@ const VIDEO_STATES: { [ id in VideoState ]: string } = { | |||
412 | [VideoState.TO_TRANSCODE]: 'To transcode', | 415 | [VideoState.TO_TRANSCODE]: 'To transcode', |
413 | [VideoState.TO_IMPORT]: 'To import', | 416 | [VideoState.TO_IMPORT]: 'To import', |
414 | [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream', | 417 | [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream', |
415 | [VideoState.LIVE_ENDED]: 'Livestream ended' | 418 | [VideoState.LIVE_ENDED]: 'Livestream ended', |
419 | [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage' | ||
416 | } | 420 | } |
417 | 421 | ||
418 | const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = { | 422 | const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = { |
diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 38e7a76d0..0e690f6ae 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts | |||
@@ -45,6 +45,7 @@ import { VideoTagModel } from '../models/video/video-tag' | |||
45 | import { VideoViewModel } from '../models/video/video-view' | 45 | import { VideoViewModel } from '../models/video/video-view' |
46 | import { CONFIG } from './config' | 46 | import { CONFIG } from './config' |
47 | import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' | 47 | import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' |
48 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
48 | 49 | ||
49 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string | 50 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string |
50 | 51 | ||
@@ -143,7 +144,8 @@ async function initDatabaseModels (silent: boolean) { | |||
143 | TrackerModel, | 144 | TrackerModel, |
144 | VideoTrackerModel, | 145 | VideoTrackerModel, |
145 | PluginModel, | 146 | PluginModel, |
146 | ActorCustomPageModel | 147 | ActorCustomPageModel, |
148 | VideoJobInfoModel | ||
147 | ]) | 149 | ]) |
148 | 150 | ||
149 | // Check extensions exist in the database | 151 | // Check extensions exist in the database |
diff --git a/server/initializers/migrations/0065-video-file-size.ts b/server/initializers/migrations/0065-video-file-size.ts index 1aeb27f2d..ac952a98c 100644 --- a/server/initializers/migrations/0065-video-file-size.ts +++ b/server/initializers/migrations/0065-video-file-size.ts | |||
@@ -1,7 +1,4 @@ | |||
1 | import * as Sequelize from 'sequelize' | 1 | import * as Sequelize from 'sequelize' |
2 | import { stat } from 'fs-extra' | ||
3 | import { VideoModel } from '../../models/video/video' | ||
4 | import { getVideoFilePath } from '@server/lib/video-paths' | ||
5 | 2 | ||
6 | function up (utils: { | 3 | function up (utils: { |
7 | transaction: Sequelize.Transaction | 4 | transaction: Sequelize.Transaction |
@@ -9,30 +6,7 @@ function up (utils: { | |||
9 | sequelize: Sequelize.Sequelize | 6 | sequelize: Sequelize.Sequelize |
10 | db: any | 7 | db: any |
11 | }): Promise<void> { | 8 | }): Promise<void> { |
12 | return utils.db.Video.listOwnedAndPopulateAuthorAndTags() | 9 | throw new Error('Removed, please upgrade from a previous version first.') |
13 | .then((videos: VideoModel[]) => { | ||
14 | const tasks: Promise<any>[] = [] | ||
15 | |||
16 | videos.forEach(video => { | ||
17 | video.VideoFiles.forEach(videoFile => { | ||
18 | const p = new Promise((res, rej) => { | ||
19 | stat(getVideoFilePath(video, videoFile), (err, stats) => { | ||
20 | if (err) return rej(err) | ||
21 | |||
22 | videoFile.size = stats.size | ||
23 | videoFile.save().then(res).catch(rej) | ||
24 | }) | ||
25 | }) | ||
26 | |||
27 | tasks.push(p) | ||
28 | }) | ||
29 | }) | ||
30 | |||
31 | return tasks | ||
32 | }) | ||
33 | .then((tasks: Promise<any>[]) => { | ||
34 | return Promise.all(tasks) | ||
35 | }) | ||
36 | } | 10 | } |
37 | 11 | ||
38 | function down (options) { | 12 | function down (options) { |
diff --git a/server/initializers/migrations/0660-object-storage.ts b/server/initializers/migrations/0660-object-storage.ts new file mode 100644 index 000000000..c815c71c6 --- /dev/null +++ b/server/initializers/migrations/0660-object-storage.ts | |||
@@ -0,0 +1,58 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | import { VideoStorage } from '@shared/models' | ||
3 | |||
4 | async function up (utils: { | ||
5 | transaction: Sequelize.Transaction | ||
6 | queryInterface: Sequelize.QueryInterface | ||
7 | sequelize: Sequelize.Sequelize | ||
8 | db: any | ||
9 | }): Promise<void> { | ||
10 | { | ||
11 | const query = ` | ||
12 | CREATE TABLE IF NOT EXISTS "videoJobInfo" ( | ||
13 | "id" serial, | ||
14 | "pendingMove" INTEGER NOT NULL, | ||
15 | "pendingTranscode" INTEGER NOT NULL, | ||
16 | "videoId" serial UNIQUE NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, | ||
17 | "createdAt" timestamp WITH time zone NOT NULL, | ||
18 | "updatedAt" timestamp WITH time zone NOT NULL, | ||
19 | PRIMARY KEY ("id") | ||
20 | ); | ||
21 | ` | ||
22 | |||
23 | await utils.sequelize.query(query) | ||
24 | } | ||
25 | |||
26 | { | ||
27 | await utils.queryInterface.addColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: true }) | ||
28 | } | ||
29 | { | ||
30 | await utils.sequelize.query( | ||
31 | `UPDATE "videoFile" SET "storage" = ${VideoStorage.FILE_SYSTEM}` | ||
32 | ) | ||
33 | } | ||
34 | { | ||
35 | await utils.queryInterface.changeColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: false }) | ||
36 | } | ||
37 | |||
38 | { | ||
39 | await utils.queryInterface.addColumn('videoStreamingPlaylist', 'storage', { type: Sequelize.INTEGER, allowNull: true }) | ||
40 | } | ||
41 | { | ||
42 | await utils.sequelize.query( | ||
43 | `UPDATE "videoStreamingPlaylist" SET "storage" = ${VideoStorage.FILE_SYSTEM}` | ||
44 | ) | ||
45 | } | ||
46 | { | ||
47 | await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'storage', { type: Sequelize.INTEGER, allowNull: false }) | ||
48 | } | ||
49 | } | ||
50 | |||
51 | function down (options) { | ||
52 | throw new Error('Not implemented.') | ||
53 | } | ||
54 | |||
55 | export { | ||
56 | up, | ||
57 | down | ||
58 | } | ||
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts index 1fa16295d..bd9ed45a9 100644 --- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts | |||
@@ -6,7 +6,7 @@ import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/vide | |||
6 | import { logger } from '@server/helpers/logger' | 6 | import { logger } from '@server/helpers/logger' |
7 | import { getExtFromMimetype } from '@server/helpers/video' | 7 | import { getExtFromMimetype } from '@server/helpers/video' |
8 | import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' | 8 | import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' |
9 | import { generateTorrentFileName } from '@server/lib/video-paths' | 9 | import { generateTorrentFileName } from '@server/lib/paths' |
10 | import { VideoCaptionModel } from '@server/models/video/video-caption' | 10 | import { VideoCaptionModel } from '@server/models/video/video-caption' |
11 | import { VideoFileModel } from '@server/models/video/video-file' | 11 | import { VideoFileModel } from '@server/models/video/video-file' |
12 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | 12 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' |
diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 0e77ab9fa..0828a2d0f 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, stat, writeFile } from 'fs-extra' | 1 | import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' |
2 | import { flatten, uniq } from 'lodash' | 2 | import { flatten, uniq } from 'lodash' |
3 | import { basename, dirname, join } from 'path' | 3 | import { basename, dirname, join } from 'path' |
4 | import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' | 4 | import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' |
@@ -8,11 +8,12 @@ import { logger } from '../helpers/logger' | |||
8 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' | 8 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' |
9 | import { generateRandomString } from '../helpers/utils' | 9 | import { generateRandomString } from '../helpers/utils' |
10 | import { CONFIG } from '../initializers/config' | 10 | import { CONFIG } from '../initializers/config' |
11 | import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' | 11 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' |
12 | import { sequelizeTypescript } from '../initializers/database' | 12 | import { sequelizeTypescript } from '../initializers/database' |
13 | import { VideoFileModel } from '../models/video/video-file' | 13 | import { VideoFileModel } from '../models/video/video-file' |
14 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | 14 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' |
15 | import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths' | 15 | import { getHlsResolutionPlaylistFilename } from './paths' |
16 | import { VideoPathManager } from './video-path-manager' | ||
16 | 17 | ||
17 | async function updateStreamingPlaylistsInfohashesIfNeeded () { | 18 | async function updateStreamingPlaylistsInfohashesIfNeeded () { |
18 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() | 19 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() |
@@ -31,75 +32,66 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () { | |||
31 | } | 32 | } |
32 | 33 | ||
33 | async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { | 34 | async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { |
34 | const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | ||
35 | |||
36 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] | 35 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] |
37 | 36 | ||
38 | const masterPlaylistPath = join(directory, playlist.playlistFilename) | ||
39 | |||
40 | for (const file of playlist.VideoFiles) { | 37 | for (const file of playlist.VideoFiles) { |
41 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | 38 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) |
42 | 39 | ||
43 | // If we did not generated a playlist for this resolution, skip | 40 | await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, async videoFilePath => { |
44 | const filePlaylistPath = join(directory, playlistFilename) | 41 | const size = await getVideoStreamSize(videoFilePath) |
45 | if (await pathExists(filePlaylistPath) === false) continue | ||
46 | |||
47 | const videoFilePath = getVideoFilePath(playlist, file) | ||
48 | 42 | ||
49 | const size = await getVideoStreamSize(videoFilePath) | 43 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) |
44 | const resolution = `RESOLUTION=${size.width}x${size.height}` | ||
50 | 45 | ||
51 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | 46 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` |
52 | const resolution = `RESOLUTION=${size.width}x${size.height}` | 47 | if (file.fps) line += ',FRAME-RATE=' + file.fps |
53 | 48 | ||
54 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | 49 | const codecs = await Promise.all([ |
55 | if (file.fps) line += ',FRAME-RATE=' + file.fps | 50 | getVideoStreamCodec(videoFilePath), |
51 | getAudioStreamCodec(videoFilePath) | ||
52 | ]) | ||
56 | 53 | ||
57 | const codecs = await Promise.all([ | 54 | line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` |
58 | getVideoStreamCodec(videoFilePath), | ||
59 | getAudioStreamCodec(videoFilePath) | ||
60 | ]) | ||
61 | 55 | ||
62 | line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` | 56 | masterPlaylists.push(line) |
63 | 57 | masterPlaylists.push(playlistFilename) | |
64 | masterPlaylists.push(line) | 58 | }) |
65 | masterPlaylists.push(playlistFilename) | ||
66 | } | 59 | } |
67 | 60 | ||
68 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | 61 | await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => { |
62 | return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | ||
63 | }) | ||
69 | } | 64 | } |
70 | 65 | ||
71 | async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { | 66 | async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { |
72 | const json: { [filename: string]: { [range: string]: string } } = {} | 67 | const json: { [filename: string]: { [range: string]: string } } = {} |
73 | 68 | ||
74 | const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | ||
75 | |||
76 | // For all the resolutions available for this video | 69 | // For all the resolutions available for this video |
77 | for (const file of playlist.VideoFiles) { | 70 | for (const file of playlist.VideoFiles) { |
78 | const rangeHashes: { [range: string]: string } = {} | 71 | const rangeHashes: { [range: string]: string } = {} |
79 | 72 | ||
80 | const videoPath = getVideoFilePath(playlist, file) | 73 | await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, videoPath => { |
81 | const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename)) | ||
82 | |||
83 | // Maybe the playlist is not generated for this resolution yet | ||
84 | if (!await pathExists(resolutionPlaylistPath)) continue | ||
85 | 74 | ||
86 | const playlistContent = await readFile(resolutionPlaylistPath) | 75 | return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(playlist, file, async resolutionPlaylistPath => { |
87 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | 76 | const playlistContent = await readFile(resolutionPlaylistPath) |
77 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | ||
88 | 78 | ||
89 | const fd = await open(videoPath, 'r') | 79 | const fd = await open(videoPath, 'r') |
90 | for (const range of ranges) { | 80 | for (const range of ranges) { |
91 | const buf = Buffer.alloc(range.length) | 81 | const buf = Buffer.alloc(range.length) |
92 | await read(fd, buf, 0, range.length, range.offset) | 82 | await read(fd, buf, 0, range.length, range.offset) |
93 | 83 | ||
94 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) | 84 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) |
95 | } | 85 | } |
96 | await close(fd) | 86 | await close(fd) |
97 | 87 | ||
98 | const videoFilename = file.filename | 88 | const videoFilename = file.filename |
99 | json[videoFilename] = rangeHashes | 89 | json[videoFilename] = rangeHashes |
90 | }) | ||
91 | }) | ||
100 | } | 92 | } |
101 | 93 | ||
102 | const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename) | 94 | const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) |
103 | await outputJSON(outputPath, json) | 95 | await outputJSON(outputPath, json) |
104 | } | 96 | } |
105 | 97 | ||
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts new file mode 100644 index 000000000..a0c58d211 --- /dev/null +++ b/server/lib/job-queue/handlers/move-to-object-storage.ts | |||
@@ -0,0 +1,114 @@ | |||
1 | import * as Bull from 'bull' | ||
2 | import { remove } from 'fs-extra' | ||
3 | import { join } from 'path' | ||
4 | import { logger } from '@server/helpers/logger' | ||
5 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | ||
6 | import { CONFIG } from '@server/initializers/config' | ||
7 | import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage' | ||
8 | import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' | ||
9 | import { moveToNextState } from '@server/lib/video-state' | ||
10 | import { VideoModel } from '@server/models/video/video' | ||
11 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
12 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models' | ||
13 | import { MoveObjectStoragePayload, VideoStorage } from '../../../../shared' | ||
14 | |||
15 | export async function processMoveToObjectStorage (job: Bull.Job) { | ||
16 | const payload = job.data as MoveObjectStoragePayload | ||
17 | logger.info('Moving video %s in job %d.', payload.videoUUID, job.id) | ||
18 | |||
19 | const video = await VideoModel.loadWithFiles(payload.videoUUID) | ||
20 | // No video, maybe deleted? | ||
21 | if (!video) { | ||
22 | logger.info('Can\'t process job %d, video does not exist.', job.id) | ||
23 | return undefined | ||
24 | } | ||
25 | |||
26 | if (video.VideoFiles) { | ||
27 | await moveWebTorrentFiles(video) | ||
28 | } | ||
29 | |||
30 | if (video.VideoStreamingPlaylists) { | ||
31 | await moveHLSFiles(video) | ||
32 | } | ||
33 | |||
34 | const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') | ||
35 | if (pendingMove === 0) { | ||
36 | logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id) | ||
37 | await doAfterLastJob(video, payload.isNewVideo) | ||
38 | } | ||
39 | |||
40 | return payload.videoUUID | ||
41 | } | ||
42 | |||
43 | // --------------------------------------------------------------------------- | ||
44 | |||
45 | async function moveWebTorrentFiles (video: MVideoWithAllFiles) { | ||
46 | for (const file of video.VideoFiles) { | ||
47 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue | ||
48 | |||
49 | const fileUrl = await storeWebTorrentFile(file.filename) | ||
50 | |||
51 | const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename) | ||
52 | await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) | ||
53 | } | ||
54 | } | ||
55 | |||
56 | async function moveHLSFiles (video: MVideoWithAllFiles) { | ||
57 | for (const playlist of video.VideoStreamingPlaylists) { | ||
58 | |||
59 | for (const file of playlist.VideoFiles) { | ||
60 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue | ||
61 | |||
62 | // Resolution playlist | ||
63 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | ||
64 | await storeHLSFile(playlist, video, playlistFilename) | ||
65 | |||
66 | // Resolution fragmented file | ||
67 | const fileUrl = await storeHLSFile(playlist, video, file.filename) | ||
68 | |||
69 | const oldPath = join(getHLSDirectory(video), file.filename) | ||
70 | |||
71 | await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath }) | ||
72 | } | ||
73 | } | ||
74 | } | ||
75 | |||
76 | async function doAfterLastJob (video: MVideoWithAllFiles, isNewVideo: boolean) { | ||
77 | for (const playlist of video.VideoStreamingPlaylists) { | ||
78 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue | ||
79 | |||
80 | // Master playlist | ||
81 | playlist.playlistUrl = await storeHLSFile(playlist, video, playlist.playlistFilename) | ||
82 | // Sha256 segments file | ||
83 | playlist.segmentsSha256Url = await storeHLSFile(playlist, video, playlist.segmentsSha256Filename) | ||
84 | |||
85 | playlist.storage = VideoStorage.OBJECT_STORAGE | ||
86 | |||
87 | await playlist.save() | ||
88 | } | ||
89 | |||
90 | // Remove empty hls video directory | ||
91 | if (video.VideoStreamingPlaylists) { | ||
92 | await remove(getHLSDirectory(video)) | ||
93 | } | ||
94 | |||
95 | await moveToNextState(video, isNewVideo) | ||
96 | } | ||
97 | |||
98 | async function onFileMoved (options: { | ||
99 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo | ||
100 | file: MVideoFile | ||
101 | fileUrl: string | ||
102 | oldPath: string | ||
103 | }) { | ||
104 | const { videoOrPlaylist, file, fileUrl, oldPath } = options | ||
105 | |||
106 | file.fileUrl = fileUrl | ||
107 | file.storage = VideoStorage.OBJECT_STORAGE | ||
108 | |||
109 | await createTorrentAndSetInfoHash(videoOrPlaylist, file) | ||
110 | await file.save() | ||
111 | |||
112 | logger.debug('Removing %s because it\'s now on object storage', oldPath) | ||
113 | await remove(oldPath) | ||
114 | } | ||
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index 2f4abf730..e8ee1f759 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts | |||
@@ -2,15 +2,19 @@ import * as Bull from 'bull' | |||
2 | import { copy, stat } from 'fs-extra' | 2 | import { copy, stat } from 'fs-extra' |
3 | import { getLowercaseExtension } from '@server/helpers/core-utils' | 3 | import { getLowercaseExtension } from '@server/helpers/core-utils' |
4 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 4 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
5 | import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' | 5 | import { CONFIG } from '@server/initializers/config' |
6 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | ||
7 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' | ||
8 | import { addMoveToObjectStorageJob } from '@server/lib/video' | ||
9 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
6 | import { UserModel } from '@server/models/user/user' | 10 | import { UserModel } from '@server/models/user/user' |
7 | import { MVideoFullLight } from '@server/types/models' | 11 | import { MVideoFullLight } from '@server/types/models' |
8 | import { VideoFileImportPayload } from '@shared/models' | 12 | import { VideoFileImportPayload, VideoStorage } from '@shared/models' |
9 | import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' | 13 | import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' |
10 | import { logger } from '../../../helpers/logger' | 14 | import { logger } from '../../../helpers/logger' |
11 | import { VideoModel } from '../../../models/video/video' | 15 | import { VideoModel } from '../../../models/video/video' |
12 | import { VideoFileModel } from '../../../models/video/video-file' | 16 | import { VideoFileModel } from '../../../models/video/video-file' |
13 | import { onNewWebTorrentFileResolution } from './video-transcoding' | 17 | import { createHlsJobIfEnabled } from './video-transcoding' |
14 | 18 | ||
15 | async function processVideoFileImport (job: Bull.Job) { | 19 | async function processVideoFileImport (job: Bull.Job) { |
16 | const payload = job.data as VideoFileImportPayload | 20 | const payload = job.data as VideoFileImportPayload |
@@ -29,15 +33,19 @@ async function processVideoFileImport (job: Bull.Job) { | |||
29 | 33 | ||
30 | const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId) | 34 | const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId) |
31 | 35 | ||
32 | const newResolutionPayload = { | 36 | await createHlsJobIfEnabled(user, { |
33 | type: 'new-resolution-to-webtorrent' as 'new-resolution-to-webtorrent', | ||
34 | videoUUID: video.uuid, | 37 | videoUUID: video.uuid, |
35 | resolution: data.resolution, | 38 | resolution: data.resolution, |
36 | isPortraitMode: data.isPortraitMode, | 39 | isPortraitMode: data.isPortraitMode, |
37 | copyCodecs: false, | 40 | copyCodecs: true, |
38 | isNewVideo: false | 41 | isMaxQuality: false |
42 | }) | ||
43 | |||
44 | if (CONFIG.OBJECT_STORAGE.ENABLED) { | ||
45 | await addMoveToObjectStorageJob(video) | ||
46 | } else { | ||
47 | await federateVideoIfNeeded(video, false) | ||
39 | } | 48 | } |
40 | await onNewWebTorrentFileResolution(video, user, newResolutionPayload) | ||
41 | 49 | ||
42 | return video | 50 | return video |
43 | } | 51 | } |
@@ -72,12 +80,13 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { | |||
72 | resolution, | 80 | resolution, |
73 | extname: fileExt, | 81 | extname: fileExt, |
74 | filename: generateWebTorrentVideoFilename(resolution, fileExt), | 82 | filename: generateWebTorrentVideoFilename(resolution, fileExt), |
83 | storage: VideoStorage.FILE_SYSTEM, | ||
75 | size, | 84 | size, |
76 | fps, | 85 | fps, |
77 | videoId: video.id | 86 | videoId: video.id |
78 | }) | 87 | }) |
79 | 88 | ||
80 | const outputPath = getVideoFilePath(video, newVideoFile) | 89 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) |
81 | await copy(inputFilePath, outputPath) | 90 | await copy(inputFilePath, outputPath) |
82 | 91 | ||
83 | video.VideoFiles.push(newVideoFile) | 92 | video.VideoFiles.push(newVideoFile) |
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index fec553f2b..a5fa204f5 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -4,11 +4,13 @@ import { getLowercaseExtension } from '@server/helpers/core-utils' | |||
4 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | 4 | import { retryTransactionWrapper } from '@server/helpers/database-utils' |
5 | import { YoutubeDL } from '@server/helpers/youtube-dl' | 5 | import { YoutubeDL } from '@server/helpers/youtube-dl' |
6 | import { isPostImportVideoAccepted } from '@server/lib/moderation' | 6 | import { isPostImportVideoAccepted } from '@server/lib/moderation' |
7 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' | ||
7 | import { Hooks } from '@server/lib/plugins/hooks' | 8 | import { Hooks } from '@server/lib/plugins/hooks' |
8 | import { ServerConfigManager } from '@server/lib/server-config-manager' | 9 | import { ServerConfigManager } from '@server/lib/server-config-manager' |
9 | import { isAbleToUploadVideo } from '@server/lib/user' | 10 | import { isAbleToUploadVideo } from '@server/lib/user' |
10 | import { addOptimizeOrMergeAudioJob } from '@server/lib/video' | 11 | import { addMoveToObjectStorageJob, addOptimizeOrMergeAudioJob } from '@server/lib/video' |
11 | import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' | 12 | import { VideoPathManager } from '@server/lib/video-path-manager' |
13 | import { buildNextVideoState } from '@server/lib/video-state' | ||
12 | import { ThumbnailModel } from '@server/models/video/thumbnail' | 14 | import { ThumbnailModel } from '@server/models/video/thumbnail' |
13 | import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' | 15 | import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' |
14 | import { | 16 | import { |
@@ -25,7 +27,6 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro | |||
25 | import { logger } from '../../../helpers/logger' | 27 | import { logger } from '../../../helpers/logger' |
26 | import { getSecureTorrentName } from '../../../helpers/utils' | 28 | import { getSecureTorrentName } from '../../../helpers/utils' |
27 | import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' | 29 | import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' |
28 | import { CONFIG } from '../../../initializers/config' | ||
29 | import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' | 30 | import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' |
30 | import { sequelizeTypescript } from '../../../initializers/database' | 31 | import { sequelizeTypescript } from '../../../initializers/database' |
31 | import { VideoModel } from '../../../models/video/video' | 32 | import { VideoModel } from '../../../models/video/video' |
@@ -100,7 +101,6 @@ type ProcessFileOptions = { | |||
100 | } | 101 | } |
101 | async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) { | 102 | async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) { |
102 | let tempVideoPath: string | 103 | let tempVideoPath: string |
103 | let videoDestFile: string | ||
104 | let videoFile: VideoFileModel | 104 | let videoFile: VideoFileModel |
105 | 105 | ||
106 | try { | 106 | try { |
@@ -159,7 +159,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
159 | const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) | 159 | const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) |
160 | 160 | ||
161 | // Move file | 161 | // Move file |
162 | videoDestFile = getVideoFilePath(videoImportWithFiles.Video, videoFile) | 162 | const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile) |
163 | await move(tempVideoPath, videoDestFile) | 163 | await move(tempVideoPath, videoDestFile) |
164 | tempVideoPath = null // This path is not used anymore | 164 | tempVideoPath = null // This path is not used anymore |
165 | 165 | ||
@@ -204,7 +204,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
204 | 204 | ||
205 | // Update video DB object | 205 | // Update video DB object |
206 | video.duration = duration | 206 | video.duration = duration |
207 | video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED | 207 | video.state = buildNextVideoState(video.state) |
208 | await video.save({ transaction: t }) | 208 | await video.save({ transaction: t }) |
209 | 209 | ||
210 | if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) | 210 | if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) |
@@ -245,6 +245,10 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
245 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) | 245 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) |
246 | } | 246 | } |
247 | 247 | ||
248 | if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { | ||
249 | return addMoveToObjectStorageJob(videoImportUpdated.Video) | ||
250 | } | ||
251 | |||
248 | // Create transcoding jobs? | 252 | // Create transcoding jobs? |
249 | if (video.state === VideoState.TO_TRANSCODE) { | 253 | if (video.state === VideoState.TO_TRANSCODE) { |
250 | await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User) | 254 | await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User) |
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index aa5bd573a..9ccf724c2 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts | |||
@@ -4,10 +4,11 @@ import { join } from 'path' | |||
4 | import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' | 4 | import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' |
5 | import { VIDEO_LIVE } from '@server/initializers/constants' | 5 | import { VIDEO_LIVE } from '@server/initializers/constants' |
6 | import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live' | 6 | import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live' |
7 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveDirectory } from '@server/lib/paths' | ||
7 | import { generateVideoMiniature } from '@server/lib/thumbnail' | 8 | import { generateVideoMiniature } from '@server/lib/thumbnail' |
8 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' | 9 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' |
9 | import { publishAndFederateIfNeeded } from '@server/lib/video' | 10 | import { VideoPathManager } from '@server/lib/video-path-manager' |
10 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHLSDirectory } from '@server/lib/video-paths' | 11 | import { moveToNextState } from '@server/lib/video-state' |
11 | import { VideoModel } from '@server/models/video/video' | 12 | import { VideoModel } from '@server/models/video/video' |
12 | import { VideoFileModel } from '@server/models/video/video-file' | 13 | import { VideoFileModel } from '@server/models/video/video-file' |
13 | import { VideoLiveModel } from '@server/models/video/video-live' | 14 | import { VideoLiveModel } from '@server/models/video/video-live' |
@@ -55,16 +56,15 @@ export { | |||
55 | // --------------------------------------------------------------------------- | 56 | // --------------------------------------------------------------------------- |
56 | 57 | ||
57 | async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) { | 58 | async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) { |
58 | const hlsDirectory = getHLSDirectory(video, false) | 59 | const replayDirectory = VideoPathManager.Instance.getFSHLSOutputPath(video, VIDEO_LIVE.REPLAY_DIRECTORY) |
59 | const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY) | ||
60 | 60 | ||
61 | const rootFiles = await readdir(hlsDirectory) | 61 | const rootFiles = await readdir(getLiveDirectory(video)) |
62 | 62 | ||
63 | const playlistFiles = rootFiles.filter(file => { | 63 | const playlistFiles = rootFiles.filter(file => { |
64 | return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename | 64 | return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename |
65 | }) | 65 | }) |
66 | 66 | ||
67 | await cleanupLiveFiles(hlsDirectory) | 67 | await cleanupTMPLiveFiles(getLiveDirectory(video)) |
68 | 68 | ||
69 | await live.destroy() | 69 | await live.destroy() |
70 | 70 | ||
@@ -98,7 +98,7 @@ async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MSt | |||
98 | 98 | ||
99 | const { resolution, isPortraitMode } = await getVideoFileResolution(concatenatedTsFilePath, probe) | 99 | const { resolution, isPortraitMode } = await getVideoFileResolution(concatenatedTsFilePath, probe) |
100 | 100 | ||
101 | const outputPath = await generateHlsPlaylistResolutionFromTS({ | 101 | const { resolutionPlaylistPath: outputPath } = await generateHlsPlaylistResolutionFromTS({ |
102 | video: videoWithFiles, | 102 | video: videoWithFiles, |
103 | concatenatedTsFilePath, | 103 | concatenatedTsFilePath, |
104 | resolution, | 104 | resolution, |
@@ -133,10 +133,10 @@ async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MSt | |||
133 | }) | 133 | }) |
134 | } | 134 | } |
135 | 135 | ||
136 | await publishAndFederateIfNeeded(videoWithFiles, true) | 136 | await moveToNextState(videoWithFiles, false) |
137 | } | 137 | } |
138 | 138 | ||
139 | async function cleanupLiveFiles (hlsDirectory: string) { | 139 | async function cleanupTMPLiveFiles (hlsDirectory: string) { |
140 | if (!await pathExists(hlsDirectory)) return | 140 | if (!await pathExists(hlsDirectory)) return |
141 | 141 | ||
142 | const files = await readdir(hlsDirectory) | 142 | const files = await readdir(hlsDirectory) |
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 876d1460c..b3149dde8 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts | |||
@@ -1,9 +1,11 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils' | 2 | import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils' |
3 | import { getTranscodingJobPriority, publishAndFederateIfNeeded } from '@server/lib/video' | 3 | import { addTranscodingJob, getTranscodingJobPriority } from '@server/lib/video' |
4 | import { getVideoFilePath } from '@server/lib/video-paths' | 4 | import { VideoPathManager } from '@server/lib/video-path-manager' |
5 | import { moveToNextState } from '@server/lib/video-state' | ||
5 | import { UserModel } from '@server/models/user/user' | 6 | import { UserModel } from '@server/models/user/user' |
6 | import { MUser, MUserId, MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/types/models' | 7 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' |
8 | import { MUser, MUserId, MVideo, MVideoFullLight, MVideoWithFile } from '@server/types/models' | ||
7 | import { | 9 | import { |
8 | HLSTranscodingPayload, | 10 | HLSTranscodingPayload, |
9 | MergeAudioTranscodingPayload, | 11 | MergeAudioTranscodingPayload, |
@@ -16,17 +18,14 @@ import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils' | |||
16 | import { logger } from '../../../helpers/logger' | 18 | import { logger } from '../../../helpers/logger' |
17 | import { CONFIG } from '../../../initializers/config' | 19 | import { CONFIG } from '../../../initializers/config' |
18 | import { VideoModel } from '../../../models/video/video' | 20 | import { VideoModel } from '../../../models/video/video' |
19 | import { federateVideoIfNeeded } from '../../activitypub/videos' | ||
20 | import { Notifier } from '../../notifier' | ||
21 | import { | 21 | import { |
22 | generateHlsPlaylistResolution, | 22 | generateHlsPlaylistResolution, |
23 | mergeAudioVideofile, | 23 | mergeAudioVideofile, |
24 | optimizeOriginalVideofile, | 24 | optimizeOriginalVideofile, |
25 | transcodeNewWebTorrentResolution | 25 | transcodeNewWebTorrentResolution |
26 | } from '../../transcoding/video-transcoding' | 26 | } from '../../transcoding/video-transcoding' |
27 | import { JobQueue } from '../job-queue' | ||
28 | 27 | ||
29 | type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<any> | 28 | type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void> |
30 | 29 | ||
31 | const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { | 30 | const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { |
32 | 'new-resolution-to-hls': handleHLSJob, | 31 | 'new-resolution-to-hls': handleHLSJob, |
@@ -69,15 +68,16 @@ async function handleHLSJob (job: Bull.Job, payload: HLSTranscodingPayload, vide | |||
69 | : video.getMaxQualityFile() | 68 | : video.getMaxQualityFile() |
70 | 69 | ||
71 | const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() | 70 | const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() |
72 | const videoInputPath = getVideoFilePath(videoOrStreamingPlaylist, videoFileInput) | ||
73 | 71 | ||
74 | await generateHlsPlaylistResolution({ | 72 | await VideoPathManager.Instance.makeAvailableVideoFile(videoOrStreamingPlaylist, videoFileInput, videoInputPath => { |
75 | video, | 73 | return generateHlsPlaylistResolution({ |
76 | videoInputPath, | 74 | video, |
77 | resolution: payload.resolution, | 75 | videoInputPath, |
78 | copyCodecs: payload.copyCodecs, | 76 | resolution: payload.resolution, |
79 | isPortraitMode: payload.isPortraitMode || false, | 77 | copyCodecs: payload.copyCodecs, |
80 | job | 78 | isPortraitMode: payload.isPortraitMode || false, |
79 | job | ||
80 | }) | ||
81 | }) | 81 | }) |
82 | 82 | ||
83 | await retryTransactionWrapper(onHlsPlaylistGeneration, video, user, payload) | 83 | await retryTransactionWrapper(onHlsPlaylistGeneration, video, user, payload) |
@@ -101,7 +101,7 @@ async function handleWebTorrentMergeAudioJob (job: Bull.Job, payload: MergeAudio | |||
101 | } | 101 | } |
102 | 102 | ||
103 | async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { | 103 | async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { |
104 | const transcodeType = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job) | 104 | const { transcodeType } = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job) |
105 | 105 | ||
106 | await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType, user) | 106 | await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType, user) |
107 | } | 107 | } |
@@ -121,10 +121,18 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay | |||
121 | video.VideoFiles = [] | 121 | video.VideoFiles = [] |
122 | 122 | ||
123 | // Create HLS new resolution jobs | 123 | // Create HLS new resolution jobs |
124 | await createLowerResolutionsJobs(video, user, payload.resolution, payload.isPortraitMode, 'hls') | 124 | await createLowerResolutionsJobs({ |
125 | video, | ||
126 | user, | ||
127 | videoFileResolution: payload.resolution, | ||
128 | isPortraitMode: payload.isPortraitMode, | ||
129 | isNewVideo: payload.isNewVideo ?? true, | ||
130 | type: 'hls' | ||
131 | }) | ||
125 | } | 132 | } |
126 | 133 | ||
127 | return publishAndFederateIfNeeded(video) | 134 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') |
135 | await moveToNextState(video, payload.isNewVideo) | ||
128 | } | 136 | } |
129 | 137 | ||
130 | async function onVideoFileOptimizer ( | 138 | async function onVideoFileOptimizer ( |
@@ -143,58 +151,54 @@ async function onVideoFileOptimizer ( | |||
143 | // Video does not exist anymore | 151 | // Video does not exist anymore |
144 | if (!videoDatabase) return undefined | 152 | if (!videoDatabase) return undefined |
145 | 153 | ||
146 | let videoPublished = false | ||
147 | |||
148 | // Generate HLS version of the original file | 154 | // Generate HLS version of the original file |
149 | const originalFileHLSPayload = Object.assign({}, payload, { | 155 | const originalFileHLSPayload = { |
156 | ...payload, | ||
157 | |||
150 | isPortraitMode, | 158 | isPortraitMode, |
151 | resolution: videoDatabase.getMaxQualityFile().resolution, | 159 | resolution: videoDatabase.getMaxQualityFile().resolution, |
152 | // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues | 160 | // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues |
153 | copyCodecs: transcodeType !== 'quick-transcode', | 161 | copyCodecs: transcodeType !== 'quick-transcode', |
154 | isMaxQuality: true | 162 | isMaxQuality: true |
155 | }) | 163 | } |
156 | const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) | 164 | const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) |
165 | const hasNewResolutions = await createLowerResolutionsJobs({ | ||
166 | video: videoDatabase, | ||
167 | user, | ||
168 | videoFileResolution: resolution, | ||
169 | isPortraitMode, | ||
170 | type: 'webtorrent', | ||
171 | isNewVideo: payload.isNewVideo ?? true | ||
172 | }) | ||
157 | 173 | ||
158 | const hasNewResolutions = await createLowerResolutionsJobs(videoDatabase, user, resolution, isPortraitMode, 'webtorrent') | 174 | await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode') |
159 | 175 | ||
176 | // Move to next state if there are no other resolutions to generate | ||
160 | if (!hasHls && !hasNewResolutions) { | 177 | if (!hasHls && !hasNewResolutions) { |
161 | // No transcoding to do, it's now published | 178 | await moveToNextState(videoDatabase, payload.isNewVideo) |
162 | videoPublished = await videoDatabase.publishIfNeededAndSave(undefined) | ||
163 | } | 179 | } |
164 | |||
165 | await federateVideoIfNeeded(videoDatabase, payload.isNewVideo) | ||
166 | |||
167 | if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) | ||
168 | if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) | ||
169 | } | 180 | } |
170 | 181 | ||
171 | async function onNewWebTorrentFileResolution ( | 182 | async function onNewWebTorrentFileResolution ( |
172 | video: MVideoUUID, | 183 | video: MVideo, |
173 | user: MUserId, | 184 | user: MUserId, |
174 | payload: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload | 185 | payload: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload |
175 | ) { | 186 | ) { |
176 | await publishAndFederateIfNeeded(video) | 187 | await createHlsJobIfEnabled(user, { ...payload, copyCodecs: true, isMaxQuality: false }) |
188 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') | ||
177 | 189 | ||
178 | await createHlsJobIfEnabled(user, Object.assign({}, payload, { copyCodecs: true, isMaxQuality: false })) | 190 | await moveToNextState(video, payload.isNewVideo) |
179 | } | 191 | } |
180 | 192 | ||
181 | // --------------------------------------------------------------------------- | ||
182 | |||
183 | export { | ||
184 | processVideoTranscoding, | ||
185 | onNewWebTorrentFileResolution | ||
186 | } | ||
187 | |||
188 | // --------------------------------------------------------------------------- | ||
189 | |||
190 | async function createHlsJobIfEnabled (user: MUserId, payload: { | 193 | async function createHlsJobIfEnabled (user: MUserId, payload: { |
191 | videoUUID: string | 194 | videoUUID: string |
192 | resolution: number | 195 | resolution: number |
193 | isPortraitMode?: boolean | 196 | isPortraitMode?: boolean |
194 | copyCodecs: boolean | 197 | copyCodecs: boolean |
195 | isMaxQuality: boolean | 198 | isMaxQuality: boolean |
199 | isNewVideo?: boolean | ||
196 | }) { | 200 | }) { |
197 | if (!payload || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false | 201 | if (!payload || CONFIG.TRANSCODING.ENABLED !== true || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false |
198 | 202 | ||
199 | const jobOptions = { | 203 | const jobOptions = { |
200 | priority: await getTranscodingJobPriority(user) | 204 | priority: await getTranscodingJobPriority(user) |
@@ -206,21 +210,35 @@ async function createHlsJobIfEnabled (user: MUserId, payload: { | |||
206 | resolution: payload.resolution, | 210 | resolution: payload.resolution, |
207 | isPortraitMode: payload.isPortraitMode, | 211 | isPortraitMode: payload.isPortraitMode, |
208 | copyCodecs: payload.copyCodecs, | 212 | copyCodecs: payload.copyCodecs, |
209 | isMaxQuality: payload.isMaxQuality | 213 | isMaxQuality: payload.isMaxQuality, |
214 | isNewVideo: payload.isNewVideo | ||
210 | } | 215 | } |
211 | 216 | ||
212 | JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload }, jobOptions) | 217 | await addTranscodingJob(hlsTranscodingPayload, jobOptions) |
213 | 218 | ||
214 | return true | 219 | return true |
215 | } | 220 | } |
216 | 221 | ||
217 | async function createLowerResolutionsJobs ( | 222 | // --------------------------------------------------------------------------- |
218 | video: MVideoFullLight, | 223 | |
219 | user: MUserId, | 224 | export { |
220 | videoFileResolution: number, | 225 | processVideoTranscoding, |
221 | isPortraitMode: boolean, | 226 | createHlsJobIfEnabled, |
227 | onNewWebTorrentFileResolution | ||
228 | } | ||
229 | |||
230 | // --------------------------------------------------------------------------- | ||
231 | |||
232 | async function createLowerResolutionsJobs (options: { | ||
233 | video: MVideoFullLight | ||
234 | user: MUserId | ||
235 | videoFileResolution: number | ||
236 | isPortraitMode: boolean | ||
237 | isNewVideo: boolean | ||
222 | type: 'hls' | 'webtorrent' | 238 | type: 'hls' | 'webtorrent' |
223 | ) { | 239 | }) { |
240 | const { video, user, videoFileResolution, isPortraitMode, isNewVideo, type } = options | ||
241 | |||
224 | // Create transcoding jobs if there are enabled resolutions | 242 | // Create transcoding jobs if there are enabled resolutions |
225 | const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod') | 243 | const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod') |
226 | const resolutionCreated: number[] = [] | 244 | const resolutionCreated: number[] = [] |
@@ -234,7 +252,8 @@ async function createLowerResolutionsJobs ( | |||
234 | type: 'new-resolution-to-webtorrent', | 252 | type: 'new-resolution-to-webtorrent', |
235 | videoUUID: video.uuid, | 253 | videoUUID: video.uuid, |
236 | resolution, | 254 | resolution, |
237 | isPortraitMode | 255 | isPortraitMode, |
256 | isNewVideo | ||
238 | } | 257 | } |
239 | } | 258 | } |
240 | 259 | ||
@@ -245,7 +264,8 @@ async function createLowerResolutionsJobs ( | |||
245 | resolution, | 264 | resolution, |
246 | isPortraitMode, | 265 | isPortraitMode, |
247 | copyCodecs: false, | 266 | copyCodecs: false, |
248 | isMaxQuality: false | 267 | isMaxQuality: false, |
268 | isNewVideo | ||
249 | } | 269 | } |
250 | } | 270 | } |
251 | 271 | ||
@@ -257,7 +277,7 @@ async function createLowerResolutionsJobs ( | |||
257 | priority: await getTranscodingJobPriority(user) | 277 | priority: await getTranscodingJobPriority(user) |
258 | } | 278 | } |
259 | 279 | ||
260 | JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput }, jobOptions) | 280 | await addTranscodingJob(dataInput, jobOptions) |
261 | } | 281 | } |
262 | 282 | ||
263 | if (resolutionCreated.length === 0) { | 283 | if (resolutionCreated.length === 0) { |
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 42e8347b1..7a3a1bf82 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -11,6 +11,7 @@ import { | |||
11 | EmailPayload, | 11 | EmailPayload, |
12 | JobState, | 12 | JobState, |
13 | JobType, | 13 | JobType, |
14 | MoveObjectStoragePayload, | ||
14 | RefreshPayload, | 15 | RefreshPayload, |
15 | VideoFileImportPayload, | 16 | VideoFileImportPayload, |
16 | VideoImportPayload, | 17 | VideoImportPayload, |
@@ -34,6 +35,7 @@ import { processVideoImport } from './handlers/video-import' | |||
34 | import { processVideoLiveEnding } from './handlers/video-live-ending' | 35 | import { processVideoLiveEnding } from './handlers/video-live-ending' |
35 | import { processVideoTranscoding } from './handlers/video-transcoding' | 36 | import { processVideoTranscoding } from './handlers/video-transcoding' |
36 | import { processVideosViews } from './handlers/video-views' | 37 | import { processVideosViews } from './handlers/video-views' |
38 | import { processMoveToObjectStorage } from './handlers/move-to-object-storage' | ||
37 | 39 | ||
38 | type CreateJobArgument = | 40 | type CreateJobArgument = |
39 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | 41 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | |
@@ -49,9 +51,10 @@ type CreateJobArgument = | |||
49 | { type: 'videos-views', payload: {} } | | 51 | { type: 'videos-views', payload: {} } | |
50 | { type: 'video-live-ending', payload: VideoLiveEndingPayload } | | 52 | { type: 'video-live-ending', payload: VideoLiveEndingPayload } | |
51 | { type: 'actor-keys', payload: ActorKeysPayload } | | 53 | { type: 'actor-keys', payload: ActorKeysPayload } | |
52 | { type: 'video-redundancy', payload: VideoRedundancyPayload } | 54 | { type: 'video-redundancy', payload: VideoRedundancyPayload } | |
55 | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | ||
53 | 56 | ||
54 | type CreateJobOptions = { | 57 | export type CreateJobOptions = { |
55 | delay?: number | 58 | delay?: number |
56 | priority?: number | 59 | priority?: number |
57 | } | 60 | } |
@@ -70,7 +73,8 @@ const handlers: { [id in JobType]: (job: Bull.Job) => Promise<any> } = { | |||
70 | 'activitypub-refresher': refreshAPObject, | 73 | 'activitypub-refresher': refreshAPObject, |
71 | 'video-live-ending': processVideoLiveEnding, | 74 | 'video-live-ending': processVideoLiveEnding, |
72 | 'actor-keys': processActorKeys, | 75 | 'actor-keys': processActorKeys, |
73 | 'video-redundancy': processVideoRedundancy | 76 | 'video-redundancy': processVideoRedundancy, |
77 | 'move-to-object-storage': processMoveToObjectStorage | ||
74 | } | 78 | } |
75 | 79 | ||
76 | const jobTypes: JobType[] = [ | 80 | const jobTypes: JobType[] = [ |
@@ -87,7 +91,8 @@ const jobTypes: JobType[] = [ | |||
87 | 'activitypub-refresher', | 91 | 'activitypub-refresher', |
88 | 'video-redundancy', | 92 | 'video-redundancy', |
89 | 'actor-keys', | 93 | 'actor-keys', |
90 | 'video-live-ending' | 94 | 'video-live-ending', |
95 | 'move-to-object-storage' | ||
91 | ] | 96 | ] |
92 | 97 | ||
93 | class JobQueue { | 98 | class JobQueue { |
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts index 2a429fb33..d7dc841d9 100644 --- a/server/lib/live/live-manager.ts +++ b/server/lib/live/live-manager.ts | |||
@@ -20,7 +20,7 @@ import { VideoState, VideoStreamingPlaylistType } from '@shared/models' | |||
20 | import { federateVideoIfNeeded } from '../activitypub/videos' | 20 | import { federateVideoIfNeeded } from '../activitypub/videos' |
21 | import { JobQueue } from '../job-queue' | 21 | import { JobQueue } from '../job-queue' |
22 | import { PeerTubeSocket } from '../peertube-socket' | 22 | import { PeerTubeSocket } from '../peertube-socket' |
23 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../video-paths' | 23 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../paths' |
24 | import { LiveQuotaStore } from './live-quota-store' | 24 | import { LiveQuotaStore } from './live-quota-store' |
25 | import { LiveSegmentShaStore } from './live-segment-sha-store' | 25 | import { LiveSegmentShaStore } from './live-segment-sha-store' |
26 | import { cleanupLive } from './live-utils' | 26 | import { cleanupLive } from './live-utils' |
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts index e4526c7a5..3bf723b98 100644 --- a/server/lib/live/live-utils.ts +++ b/server/lib/live/live-utils.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { remove } from 'fs-extra' | 1 | import { remove } from 'fs-extra' |
2 | import { basename } from 'path' | 2 | import { basename } from 'path' |
3 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | 3 | import { MStreamingPlaylist, MVideo } from '@server/types/models' |
4 | import { getHLSDirectory } from '../video-paths' | 4 | import { getLiveDirectory } from '../paths' |
5 | 5 | ||
6 | function buildConcatenatedName (segmentOrPlaylistPath: string) { | 6 | function buildConcatenatedName (segmentOrPlaylistPath: string) { |
7 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) | 7 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) |
@@ -10,7 +10,7 @@ function buildConcatenatedName (segmentOrPlaylistPath: string) { | |||
10 | } | 10 | } |
11 | 11 | ||
12 | async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | 12 | async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { |
13 | const hlsDirectory = getHLSDirectory(video) | 13 | const hlsDirectory = getLiveDirectory(video) |
14 | 14 | ||
15 | await remove(hlsDirectory) | 15 | await remove(hlsDirectory) |
16 | 16 | ||
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts index a80abc843..9b5b6c4fc 100644 --- a/server/lib/live/shared/muxing-session.ts +++ b/server/lib/live/shared/muxing-session.ts | |||
@@ -11,9 +11,9 @@ import { CONFIG } from '@server/initializers/config' | |||
11 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' | 11 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' |
12 | import { VideoFileModel } from '@server/models/video/video-file' | 12 | import { VideoFileModel } from '@server/models/video/video-file' |
13 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' | 13 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' |
14 | import { getLiveDirectory } from '../../paths' | ||
14 | import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles' | 15 | import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles' |
15 | import { isAbleToUploadVideo } from '../../user' | 16 | import { isAbleToUploadVideo } from '../../user' |
16 | import { getHLSDirectory } from '../../video-paths' | ||
17 | import { LiveQuotaStore } from '../live-quota-store' | 17 | import { LiveQuotaStore } from '../live-quota-store' |
18 | import { LiveSegmentShaStore } from '../live-segment-sha-store' | 18 | import { LiveSegmentShaStore } from '../live-segment-sha-store' |
19 | import { buildConcatenatedName } from '../live-utils' | 19 | import { buildConcatenatedName } from '../live-utils' |
@@ -282,7 +282,7 @@ class MuxingSession extends EventEmitter { | |||
282 | } | 282 | } |
283 | 283 | ||
284 | private async prepareDirectories () { | 284 | private async prepareDirectories () { |
285 | const outPath = getHLSDirectory(this.videoLive.Video) | 285 | const outPath = getLiveDirectory(this.videoLive.Video) |
286 | await ensureDir(outPath) | 286 | await ensureDir(outPath) |
287 | 287 | ||
288 | const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY) | 288 | const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY) |
diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts new file mode 100644 index 000000000..8b413a40e --- /dev/null +++ b/server/lib/object-storage/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './keys' | ||
2 | export * from './urls' | ||
3 | export * from './videos' | ||
diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts new file mode 100644 index 000000000..519474775 --- /dev/null +++ b/server/lib/object-storage/keys.ts | |||
@@ -0,0 +1,20 @@ | |||
1 | import { join } from 'path' | ||
2 | import { MStreamingPlaylist, MVideoUUID } from '@server/types/models' | ||
3 | |||
4 | function generateHLSObjectStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) { | ||
5 | return join(generateHLSObjectBaseStorageKey(playlist, video), filename) | ||
6 | } | ||
7 | |||
8 | function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID) { | ||
9 | return playlist.getStringType() + '_' + video.uuid | ||
10 | } | ||
11 | |||
12 | function generateWebTorrentObjectStorageKey (filename: string) { | ||
13 | return filename | ||
14 | } | ||
15 | |||
16 | export { | ||
17 | generateHLSObjectStorageKey, | ||
18 | generateHLSObjectBaseStorageKey, | ||
19 | generateWebTorrentObjectStorageKey | ||
20 | } | ||
diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts new file mode 100644 index 000000000..c9a614593 --- /dev/null +++ b/server/lib/object-storage/shared/client.ts | |||
@@ -0,0 +1,56 @@ | |||
1 | import { S3Client } from '@aws-sdk/client-s3' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { lTags } from './logger' | ||
5 | |||
6 | let endpointParsed: URL | ||
7 | function getEndpointParsed () { | ||
8 | if (endpointParsed) return endpointParsed | ||
9 | |||
10 | endpointParsed = new URL(getEndpoint()) | ||
11 | |||
12 | return endpointParsed | ||
13 | } | ||
14 | |||
15 | let s3Client: S3Client | ||
16 | function getClient () { | ||
17 | if (s3Client) return s3Client | ||
18 | |||
19 | const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE | ||
20 | |||
21 | s3Client = new S3Client({ | ||
22 | endpoint: getEndpoint(), | ||
23 | region: OBJECT_STORAGE.REGION, | ||
24 | credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID | ||
25 | ? { | ||
26 | accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID, | ||
27 | secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY | ||
28 | } | ||
29 | : undefined | ||
30 | }) | ||
31 | |||
32 | logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags()) | ||
33 | |||
34 | return s3Client | ||
35 | } | ||
36 | |||
37 | // --------------------------------------------------------------------------- | ||
38 | |||
39 | export { | ||
40 | getEndpointParsed, | ||
41 | getClient | ||
42 | } | ||
43 | |||
44 | // --------------------------------------------------------------------------- | ||
45 | |||
46 | let endpoint: string | ||
47 | function getEndpoint () { | ||
48 | if (endpoint) return endpoint | ||
49 | |||
50 | const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT | ||
51 | endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://') | ||
52 | ? CONFIG.OBJECT_STORAGE.ENDPOINT | ||
53 | : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT | ||
54 | |||
55 | return endpoint | ||
56 | } | ||
diff --git a/server/lib/object-storage/shared/index.ts b/server/lib/object-storage/shared/index.ts new file mode 100644 index 000000000..11e10aa9f --- /dev/null +++ b/server/lib/object-storage/shared/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './client' | ||
2 | export * from './logger' | ||
3 | export * from './object-storage-helpers' | ||
diff --git a/server/lib/object-storage/shared/logger.ts b/server/lib/object-storage/shared/logger.ts new file mode 100644 index 000000000..8ab7cbd71 --- /dev/null +++ b/server/lib/object-storage/shared/logger.ts | |||
@@ -0,0 +1,7 @@ | |||
1 | import { loggerTagsFactory } from '@server/helpers/logger' | ||
2 | |||
3 | const lTags = loggerTagsFactory('object-storage') | ||
4 | |||
5 | export { | ||
6 | lTags | ||
7 | } | ||
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts new file mode 100644 index 000000000..e23216907 --- /dev/null +++ b/server/lib/object-storage/shared/object-storage-helpers.ts | |||
@@ -0,0 +1,229 @@ | |||
1 | import { close, createReadStream, createWriteStream, ensureDir, open, ReadStream, stat } from 'fs-extra' | ||
2 | import { min } from 'lodash' | ||
3 | import { dirname } from 'path' | ||
4 | import { Readable } from 'stream' | ||
5 | import { | ||
6 | CompletedPart, | ||
7 | CompleteMultipartUploadCommand, | ||
8 | CreateMultipartUploadCommand, | ||
9 | DeleteObjectCommand, | ||
10 | GetObjectCommand, | ||
11 | ListObjectsV2Command, | ||
12 | PutObjectCommand, | ||
13 | UploadPartCommand | ||
14 | } from '@aws-sdk/client-s3' | ||
15 | import { pipelinePromise } from '@server/helpers/core-utils' | ||
16 | import { isArray } from '@server/helpers/custom-validators/misc' | ||
17 | import { logger } from '@server/helpers/logger' | ||
18 | import { CONFIG } from '@server/initializers/config' | ||
19 | import { getPrivateUrl } from '../urls' | ||
20 | import { getClient } from './client' | ||
21 | import { lTags } from './logger' | ||
22 | |||
23 | type BucketInfo = { | ||
24 | BUCKET_NAME: string | ||
25 | PREFIX?: string | ||
26 | } | ||
27 | |||
28 | async function storeObject (options: { | ||
29 | inputPath: string | ||
30 | objectStorageKey: string | ||
31 | bucketInfo: BucketInfo | ||
32 | }): Promise<string> { | ||
33 | const { inputPath, objectStorageKey, bucketInfo } = options | ||
34 | |||
35 | logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) | ||
36 | |||
37 | const stats = await stat(inputPath) | ||
38 | |||
39 | // If bigger than max allowed size we do a multipart upload | ||
40 | if (stats.size > CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART) { | ||
41 | return multiPartUpload({ inputPath, objectStorageKey, bucketInfo }) | ||
42 | } | ||
43 | |||
44 | const fileStream = createReadStream(inputPath) | ||
45 | return objectStoragePut({ objectStorageKey, content: fileStream, bucketInfo }) | ||
46 | } | ||
47 | |||
48 | async function removeObject (filename: string, bucketInfo: BucketInfo) { | ||
49 | const command = new DeleteObjectCommand({ | ||
50 | Bucket: bucketInfo.BUCKET_NAME, | ||
51 | Key: buildKey(filename, bucketInfo) | ||
52 | }) | ||
53 | |||
54 | return getClient().send(command) | ||
55 | } | ||
56 | |||
57 | async function removePrefix (prefix: string, bucketInfo: BucketInfo) { | ||
58 | const s3Client = getClient() | ||
59 | |||
60 | const commandPrefix = bucketInfo.PREFIX + prefix | ||
61 | const listCommand = new ListObjectsV2Command({ | ||
62 | Bucket: bucketInfo.BUCKET_NAME, | ||
63 | Prefix: commandPrefix | ||
64 | }) | ||
65 | |||
66 | const listedObjects = await s3Client.send(listCommand) | ||
67 | |||
68 | // FIXME: use bulk delete when s3ninja will support this operation | ||
69 | // const deleteParams = { | ||
70 | // Bucket: bucketInfo.BUCKET_NAME, | ||
71 | // Delete: { Objects: [] } | ||
72 | // } | ||
73 | |||
74 | if (isArray(listedObjects.Contents) !== true) { | ||
75 | const message = `Cannot remove ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` | ||
76 | |||
77 | logger.error(message, { response: listedObjects, ...lTags() }) | ||
78 | throw new Error(message) | ||
79 | } | ||
80 | |||
81 | for (const object of listedObjects.Contents) { | ||
82 | const command = new DeleteObjectCommand({ | ||
83 | Bucket: bucketInfo.BUCKET_NAME, | ||
84 | Key: object.Key | ||
85 | }) | ||
86 | |||
87 | await s3Client.send(command) | ||
88 | |||
89 | // FIXME: use bulk delete when s3ninja will support this operation | ||
90 | // deleteParams.Delete.Objects.push({ Key: object.Key }) | ||
91 | } | ||
92 | |||
93 | // FIXME: use bulk delete when s3ninja will support this operation | ||
94 | // const deleteCommand = new DeleteObjectsCommand(deleteParams) | ||
95 | // await s3Client.send(deleteCommand) | ||
96 | |||
97 | // Repeat if not all objects could be listed at once (limit of 1000?) | ||
98 | if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo) | ||
99 | } | ||
100 | |||
101 | async function makeAvailable (options: { | ||
102 | key: string | ||
103 | destination: string | ||
104 | bucketInfo: BucketInfo | ||
105 | }) { | ||
106 | const { key, destination, bucketInfo } = options | ||
107 | |||
108 | await ensureDir(dirname(options.destination)) | ||
109 | |||
110 | const command = new GetObjectCommand({ | ||
111 | Bucket: bucketInfo.BUCKET_NAME, | ||
112 | Key: buildKey(key, bucketInfo) | ||
113 | }) | ||
114 | const response = await getClient().send(command) | ||
115 | |||
116 | const file = createWriteStream(destination) | ||
117 | await pipelinePromise(response.Body as Readable, file) | ||
118 | |||
119 | file.close() | ||
120 | } | ||
121 | |||
122 | function buildKey (key: string, bucketInfo: BucketInfo) { | ||
123 | return bucketInfo.PREFIX + key | ||
124 | } | ||
125 | |||
126 | // --------------------------------------------------------------------------- | ||
127 | |||
128 | export { | ||
129 | BucketInfo, | ||
130 | buildKey, | ||
131 | storeObject, | ||
132 | removeObject, | ||
133 | removePrefix, | ||
134 | makeAvailable | ||
135 | } | ||
136 | |||
137 | // --------------------------------------------------------------------------- | ||
138 | |||
139 | async function objectStoragePut (options: { | ||
140 | objectStorageKey: string | ||
141 | content: ReadStream | ||
142 | bucketInfo: BucketInfo | ||
143 | }) { | ||
144 | const { objectStorageKey, content, bucketInfo } = options | ||
145 | |||
146 | const command = new PutObjectCommand({ | ||
147 | Bucket: bucketInfo.BUCKET_NAME, | ||
148 | Key: buildKey(objectStorageKey, bucketInfo), | ||
149 | Body: content | ||
150 | }) | ||
151 | |||
152 | await getClient().send(command) | ||
153 | |||
154 | return getPrivateUrl(bucketInfo, objectStorageKey) | ||
155 | } | ||
156 | |||
157 | async function multiPartUpload (options: { | ||
158 | inputPath: string | ||
159 | objectStorageKey: string | ||
160 | bucketInfo: BucketInfo | ||
161 | }) { | ||
162 | const { objectStorageKey, inputPath, bucketInfo } = options | ||
163 | |||
164 | const key = buildKey(objectStorageKey, bucketInfo) | ||
165 | const s3Client = getClient() | ||
166 | |||
167 | const statResult = await stat(inputPath) | ||
168 | |||
169 | const createMultipartCommand = new CreateMultipartUploadCommand({ | ||
170 | Bucket: bucketInfo.BUCKET_NAME, | ||
171 | Key: key | ||
172 | }) | ||
173 | const createResponse = await s3Client.send(createMultipartCommand) | ||
174 | |||
175 | const fd = await open(inputPath, 'r') | ||
176 | let partNumber = 1 | ||
177 | const parts: CompletedPart[] = [] | ||
178 | const partSize = CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART | ||
179 | |||
180 | for (let start = 0; start < statResult.size; start += partSize) { | ||
181 | logger.debug( | ||
182 | 'Uploading part %d of file to %s%s in bucket %s', | ||
183 | partNumber, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags() | ||
184 | ) | ||
185 | |||
186 | // FIXME: Remove when https://github.com/aws/aws-sdk-js-v3/pull/2637 is released | ||
187 | // The s3 sdk needs to know the length of the http body beforehand, but doesn't support | ||
188 | // streams with start and end set, so it just tries to stat the file in stream.path. | ||
189 | // This fails for us because we only want to send part of the file. The stream type | ||
190 | // is modified so we can set the byteLength here, which s3 detects because array buffers | ||
191 | // have this field set | ||
192 | const stream: ReadStream & { byteLength: number } = | ||
193 | createReadStream( | ||
194 | inputPath, | ||
195 | { fd, autoClose: false, start, end: (start + partSize) - 1 } | ||
196 | ) as ReadStream & { byteLength: number } | ||
197 | |||
198 | // Calculate if the part size is more than what's left over, and in that case use left over bytes for byteLength | ||
199 | stream.byteLength = min([ statResult.size - start, partSize ]) | ||
200 | |||
201 | const uploadPartCommand = new UploadPartCommand({ | ||
202 | Bucket: bucketInfo.BUCKET_NAME, | ||
203 | Key: key, | ||
204 | UploadId: createResponse.UploadId, | ||
205 | PartNumber: partNumber, | ||
206 | Body: stream | ||
207 | }) | ||
208 | const uploadResponse = await s3Client.send(uploadPartCommand) | ||
209 | |||
210 | parts.push({ ETag: uploadResponse.ETag, PartNumber: partNumber }) | ||
211 | partNumber += 1 | ||
212 | } | ||
213 | await close(fd) | ||
214 | |||
215 | const completeUploadCommand = new CompleteMultipartUploadCommand({ | ||
216 | Bucket: bucketInfo.BUCKET_NAME, | ||
217 | Key: objectStorageKey, | ||
218 | UploadId: createResponse.UploadId, | ||
219 | MultipartUpload: { Parts: parts } | ||
220 | }) | ||
221 | await s3Client.send(completeUploadCommand) | ||
222 | |||
223 | logger.debug( | ||
224 | 'Completed %s%s in bucket %s in %d parts', | ||
225 | bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, partNumber - 1, lTags() | ||
226 | ) | ||
227 | |||
228 | return getPrivateUrl(bucketInfo, objectStorageKey) | ||
229 | } | ||
diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts new file mode 100644 index 000000000..2a889190b --- /dev/null +++ b/server/lib/object-storage/urls.ts | |||
@@ -0,0 +1,40 @@ | |||
1 | import { CONFIG } from '@server/initializers/config' | ||
2 | import { BucketInfo, buildKey, getEndpointParsed } from './shared' | ||
3 | |||
4 | function getPrivateUrl (config: BucketInfo, keyWithoutPrefix: string) { | ||
5 | return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) | ||
6 | } | ||
7 | |||
8 | function getWebTorrentPublicFileUrl (fileUrl: string) { | ||
9 | const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL | ||
10 | if (!baseUrl) return fileUrl | ||
11 | |||
12 | return replaceByBaseUrl(fileUrl, baseUrl) | ||
13 | } | ||
14 | |||
15 | function getHLSPublicFileUrl (fileUrl: string) { | ||
16 | const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL | ||
17 | if (!baseUrl) return fileUrl | ||
18 | |||
19 | return replaceByBaseUrl(fileUrl, baseUrl) | ||
20 | } | ||
21 | |||
22 | export { | ||
23 | getPrivateUrl, | ||
24 | getWebTorrentPublicFileUrl, | ||
25 | replaceByBaseUrl, | ||
26 | getHLSPublicFileUrl | ||
27 | } | ||
28 | |||
29 | // --------------------------------------------------------------------------- | ||
30 | |||
31 | function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) { | ||
32 | if (baseUrl) return baseUrl | ||
33 | |||
34 | return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/` | ||
35 | } | ||
36 | |||
37 | const regex = new RegExp('https?://[^/]+') | ||
38 | function replaceByBaseUrl (fileUrl: string, baseUrl: string) { | ||
39 | return fileUrl.replace(regex, baseUrl) | ||
40 | } | ||
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts new file mode 100644 index 000000000..15b8f58d5 --- /dev/null +++ b/server/lib/object-storage/videos.ts | |||
@@ -0,0 +1,72 @@ | |||
1 | import { join } from 'path' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { MStreamingPlaylist, MVideoFile, MVideoUUID } from '@server/types/models' | ||
5 | import { getHLSDirectory } from '../paths' | ||
6 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' | ||
7 | import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' | ||
8 | |||
9 | function storeHLSFile (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) { | ||
10 | const baseHlsDirectory = getHLSDirectory(video) | ||
11 | |||
12 | return storeObject({ | ||
13 | inputPath: join(baseHlsDirectory, filename), | ||
14 | objectStorageKey: generateHLSObjectStorageKey(playlist, video, filename), | ||
15 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS | ||
16 | }) | ||
17 | } | ||
18 | |||
19 | function storeWebTorrentFile (filename: string) { | ||
20 | return storeObject({ | ||
21 | inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename), | ||
22 | objectStorageKey: generateWebTorrentObjectStorageKey(filename), | ||
23 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS | ||
24 | }) | ||
25 | } | ||
26 | |||
27 | function removeHLSObjectStorage (playlist: MStreamingPlaylist, video: MVideoUUID) { | ||
28 | return removePrefix(generateHLSObjectBaseStorageKey(playlist, video), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | ||
29 | } | ||
30 | |||
31 | function removeWebTorrentObjectStorage (videoFile: MVideoFile) { | ||
32 | return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) | ||
33 | } | ||
34 | |||
35 | async function makeHLSFileAvailable (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string, destination: string) { | ||
36 | const key = generateHLSObjectStorageKey(playlist, video, filename) | ||
37 | |||
38 | logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags()) | ||
39 | |||
40 | await makeAvailable({ | ||
41 | key, | ||
42 | destination, | ||
43 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS | ||
44 | }) | ||
45 | |||
46 | return destination | ||
47 | } | ||
48 | |||
49 | async function makeWebTorrentFileAvailable (filename: string, destination: string) { | ||
50 | const key = generateWebTorrentObjectStorageKey(filename) | ||
51 | |||
52 | logger.info('Fetching WebTorrent file %s from object storage to %s.', key, destination, lTags()) | ||
53 | |||
54 | await makeAvailable({ | ||
55 | key, | ||
56 | destination, | ||
57 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS | ||
58 | }) | ||
59 | |||
60 | return destination | ||
61 | } | ||
62 | |||
63 | export { | ||
64 | storeWebTorrentFile, | ||
65 | storeHLSFile, | ||
66 | |||
67 | removeHLSObjectStorage, | ||
68 | removeWebTorrentObjectStorage, | ||
69 | |||
70 | makeWebTorrentFileAvailable, | ||
71 | makeHLSFileAvailable | ||
72 | } | ||
diff --git a/server/lib/video-paths.ts b/server/lib/paths.ts index 1e4382108..434e637c6 100644 --- a/server/lib/video-paths.ts +++ b/server/lib/paths.ts | |||
@@ -1,9 +1,8 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | import { extractVideo } from '@server/helpers/video' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' | ||
5 | import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
6 | import { buildUUID } from '@server/helpers/uuid' | 2 | import { buildUUID } from '@server/helpers/uuid' |
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants' | ||
5 | import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
7 | import { removeFragmentedMP4Ext } from '@shared/core-utils' | 6 | import { removeFragmentedMP4Ext } from '@shared/core-utils' |
8 | 7 | ||
9 | // ################## Video file name ################## | 8 | // ################## Video file name ################## |
@@ -16,39 +15,18 @@ function generateHLSVideoFilename (resolution: number) { | |||
16 | return `${buildUUID()}-${resolution}-fragmented.mp4` | 15 | return `${buildUUID()}-${resolution}-fragmented.mp4` |
17 | } | 16 | } |
18 | 17 | ||
19 | function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) { | 18 | // ################## Streaming playlist ################## |
20 | if (videoFile.isHLS()) { | ||
21 | const video = extractVideo(videoOrPlaylist) | ||
22 | |||
23 | return join(getHLSDirectory(video), videoFile.filename) | ||
24 | } | ||
25 | |||
26 | const baseDir = isRedundancy | ||
27 | ? CONFIG.STORAGE.REDUNDANCY_DIR | ||
28 | : CONFIG.STORAGE.VIDEOS_DIR | ||
29 | |||
30 | return join(baseDir, videoFile.filename) | ||
31 | } | ||
32 | |||
33 | // ################## Redundancy ################## | ||
34 | 19 | ||
35 | function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { | 20 | function getLiveDirectory (video: MVideoUUID) { |
36 | // Base URL used by our HLS player | 21 | return getHLSDirectory(video) |
37 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid | ||
38 | } | 22 | } |
39 | 23 | ||
40 | function generateWebTorrentRedundancyUrl (file: MVideoFile) { | 24 | function getHLSDirectory (video: MVideoUUID) { |
41 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename | 25 | return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) |
42 | } | 26 | } |
43 | 27 | ||
44 | // ################## Streaming playlist ################## | 28 | function getHLSRedundancyDirectory (video: MVideoUUID) { |
45 | 29 | return join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | |
46 | function getHLSDirectory (video: MVideoUUID, isRedundancy = false) { | ||
47 | const baseDir = isRedundancy | ||
48 | ? HLS_REDUNDANCY_DIRECTORY | ||
49 | : HLS_STREAMING_PLAYLIST_DIRECTORY | ||
50 | |||
51 | return join(baseDir, video.uuid) | ||
52 | } | 30 | } |
53 | 31 | ||
54 | function getHlsResolutionPlaylistFilename (videoFilename: string) { | 32 | function getHlsResolutionPlaylistFilename (videoFilename: string) { |
@@ -81,36 +59,24 @@ function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVi | |||
81 | return uuid + '-' + resolution + extension | 59 | return uuid + '-' + resolution + extension |
82 | } | 60 | } |
83 | 61 | ||
84 | function getTorrentFilePath (videoFile: MVideoFile) { | 62 | function getFSTorrentFilePath (videoFile: MVideoFile) { |
85 | return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) | 63 | return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) |
86 | } | 64 | } |
87 | 65 | ||
88 | // ################## Meta data ################## | ||
89 | |||
90 | function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { | ||
91 | const path = '/api/v1/videos/' | ||
92 | |||
93 | return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id | ||
94 | } | ||
95 | |||
96 | // --------------------------------------------------------------------------- | 66 | // --------------------------------------------------------------------------- |
97 | 67 | ||
98 | export { | 68 | export { |
99 | generateHLSVideoFilename, | 69 | generateHLSVideoFilename, |
100 | generateWebTorrentVideoFilename, | 70 | generateWebTorrentVideoFilename, |
101 | 71 | ||
102 | getVideoFilePath, | ||
103 | |||
104 | generateTorrentFileName, | 72 | generateTorrentFileName, |
105 | getTorrentFilePath, | 73 | getFSTorrentFilePath, |
106 | 74 | ||
107 | getHLSDirectory, | 75 | getHLSDirectory, |
76 | getLiveDirectory, | ||
77 | getHLSRedundancyDirectory, | ||
78 | |||
108 | generateHLSMasterPlaylistFilename, | 79 | generateHLSMasterPlaylistFilename, |
109 | generateHlsSha256SegmentsFilename, | 80 | generateHlsSha256SegmentsFilename, |
110 | getHlsResolutionPlaylistFilename, | 81 | getHlsResolutionPlaylistFilename |
111 | |||
112 | getLocalVideoFileMetadataUrl, | ||
113 | |||
114 | generateWebTorrentRedundancyUrl, | ||
115 | generateHLSRedundancyUrl | ||
116 | } | 82 | } |
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 137ae53a0..ebfd015b5 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -24,7 +24,7 @@ import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlayli | |||
24 | import { getOrCreateAPVideo } from '../activitypub/videos' | 24 | import { getOrCreateAPVideo } from '../activitypub/videos' |
25 | import { downloadPlaylistSegments } from '../hls' | 25 | import { downloadPlaylistSegments } from '../hls' |
26 | import { removeVideoRedundancy } from '../redundancy' | 26 | import { removeVideoRedundancy } from '../redundancy' |
27 | import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' | 27 | import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-urls' |
28 | import { AbstractScheduler } from './abstract-scheduler' | 28 | import { AbstractScheduler } from './abstract-scheduler' |
29 | 29 | ||
30 | type CandidateToDuplicate = { | 30 | type CandidateToDuplicate = { |
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts index c08523988..d2384f53c 100644 --- a/server/lib/thumbnail.ts +++ b/server/lib/thumbnail.ts | |||
@@ -1,5 +1,4 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | |||
3 | import { ThumbnailType } from '../../shared/models/videos/thumbnail.type' | 2 | import { ThumbnailType } from '../../shared/models/videos/thumbnail.type' |
4 | import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils' | 3 | import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils' |
5 | import { generateImageFilename, processImage } from '../helpers/image-utils' | 4 | import { generateImageFilename, processImage } from '../helpers/image-utils' |
@@ -10,7 +9,7 @@ import { ThumbnailModel } from '../models/video/thumbnail' | |||
10 | import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' | 9 | import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' |
11 | import { MThumbnail } from '../types/models/video/thumbnail' | 10 | import { MThumbnail } from '../types/models/video/thumbnail' |
12 | import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' | 11 | import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' |
13 | import { getVideoFilePath } from './video-paths' | 12 | import { VideoPathManager } from './video-path-manager' |
14 | 13 | ||
15 | type ImageSize = { height?: number, width?: number } | 14 | type ImageSize = { height?: number, width?: number } |
16 | 15 | ||
@@ -116,21 +115,22 @@ function generateVideoMiniature (options: { | |||
116 | }) { | 115 | }) { |
117 | const { video, videoFile, type } = options | 116 | const { video, videoFile, type } = options |
118 | 117 | ||
119 | const input = getVideoFilePath(video, videoFile) | 118 | return VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, input => { |
119 | const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) | ||
120 | 120 | ||
121 | const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) | 121 | const thumbnailCreator = videoFile.isAudio() |
122 | const thumbnailCreator = videoFile.isAudio() | 122 | ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) |
123 | ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) | 123 | : () => generateImageFromVideoFile(input, basePath, filename, { height, width }) |
124 | : () => generateImageFromVideoFile(input, basePath, filename, { height, width }) | ||
125 | 124 | ||
126 | return updateThumbnailFromFunction({ | 125 | return updateThumbnailFromFunction({ |
127 | thumbnailCreator, | 126 | thumbnailCreator, |
128 | filename, | 127 | filename, |
129 | height, | 128 | height, |
130 | width, | 129 | width, |
131 | type, | 130 | type, |
132 | automaticallyGenerated: true, | 131 | automaticallyGenerated: true, |
133 | existingThumbnail | 132 | existingThumbnail |
133 | }) | ||
134 | }) | 134 | }) |
135 | } | 135 | } |
136 | 136 | ||
diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts index d2a556360..ee228c011 100644 --- a/server/lib/transcoding/video-transcoding.ts +++ b/server/lib/transcoding/video-transcoding.ts | |||
@@ -4,13 +4,13 @@ import { basename, extname as extnameUtil, join } from 'path' | |||
4 | import { toEven } from '@server/helpers/core-utils' | 4 | import { toEven } from '@server/helpers/core-utils' |
5 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 5 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
6 | import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 6 | import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
7 | import { VideoResolution } from '../../../shared/models/videos' | 7 | import { VideoResolution, VideoStorage } from '../../../shared/models/videos' |
8 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | 8 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' |
9 | import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils' | 9 | import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils' |
10 | import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' | 10 | import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' |
11 | import { logger } from '../../helpers/logger' | 11 | import { logger } from '../../helpers/logger' |
12 | import { CONFIG } from '../../initializers/config' | 12 | import { CONFIG } from '../../initializers/config' |
13 | import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' | 13 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' |
14 | import { VideoFileModel } from '../../models/video/video-file' | 14 | import { VideoFileModel } from '../../models/video/video-file' |
15 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | 15 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' |
16 | import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' | 16 | import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' |
@@ -19,9 +19,9 @@ import { | |||
19 | generateHlsSha256SegmentsFilename, | 19 | generateHlsSha256SegmentsFilename, |
20 | generateHLSVideoFilename, | 20 | generateHLSVideoFilename, |
21 | generateWebTorrentVideoFilename, | 21 | generateWebTorrentVideoFilename, |
22 | getHlsResolutionPlaylistFilename, | 22 | getHlsResolutionPlaylistFilename |
23 | getVideoFilePath | 23 | } from '../paths' |
24 | } from '../video-paths' | 24 | import { VideoPathManager } from '../video-path-manager' |
25 | import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' | 25 | import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' |
26 | 26 | ||
27 | /** | 27 | /** |
@@ -32,159 +32,162 @@ import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' | |||
32 | */ | 32 | */ |
33 | 33 | ||
34 | // Optimize the original video file and replace it. The resolution is not changed. | 34 | // Optimize the original video file and replace it. The resolution is not changed. |
35 | async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) { | 35 | function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) { |
36 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | 36 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
37 | const newExtname = '.mp4' | 37 | const newExtname = '.mp4' |
38 | 38 | ||
39 | const videoInputPath = getVideoFilePath(video, inputVideoFile) | 39 | return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async videoInputPath => { |
40 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | 40 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) |
41 | 41 | ||
42 | const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) | 42 | const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) |
43 | ? 'quick-transcode' | 43 | ? 'quick-transcode' |
44 | : 'video' | 44 | : 'video' |
45 | 45 | ||
46 | const resolution = toEven(inputVideoFile.resolution) | 46 | const resolution = toEven(inputVideoFile.resolution) |
47 | 47 | ||
48 | const transcodeOptions: TranscodeOptions = { | 48 | const transcodeOptions: TranscodeOptions = { |
49 | type: transcodeType, | 49 | type: transcodeType, |
50 | 50 | ||
51 | inputPath: videoInputPath, | 51 | inputPath: videoInputPath, |
52 | outputPath: videoTranscodedPath, | 52 | outputPath: videoTranscodedPath, |
53 | 53 | ||
54 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 54 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
55 | profile: CONFIG.TRANSCODING.PROFILE, | 55 | profile: CONFIG.TRANSCODING.PROFILE, |
56 | 56 | ||
57 | resolution, | 57 | resolution, |
58 | 58 | ||
59 | job | 59 | job |
60 | } | 60 | } |
61 | 61 | ||
62 | // Could be very long! | 62 | // Could be very long! |
63 | await transcode(transcodeOptions) | 63 | await transcode(transcodeOptions) |
64 | 64 | ||
65 | try { | 65 | try { |
66 | await remove(videoInputPath) | 66 | await remove(videoInputPath) |
67 | 67 | ||
68 | // Important to do this before getVideoFilename() to take in account the new filename | 68 | // Important to do this before getVideoFilename() to take in account the new filename |
69 | inputVideoFile.extname = newExtname | 69 | inputVideoFile.extname = newExtname |
70 | inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) | 70 | inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) |
71 | inputVideoFile.storage = VideoStorage.FILE_SYSTEM | ||
71 | 72 | ||
72 | const videoOutputPath = getVideoFilePath(video, inputVideoFile) | 73 | const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) |
73 | 74 | ||
74 | await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) | 75 | const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) |
75 | 76 | ||
76 | return transcodeType | 77 | return { transcodeType, videoFile } |
77 | } catch (err) { | 78 | } catch (err) { |
78 | // Auto destruction... | 79 | // Auto destruction... |
79 | video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) | 80 | video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) |
80 | 81 | ||
81 | throw err | 82 | throw err |
82 | } | 83 | } |
84 | }) | ||
83 | } | 85 | } |
84 | 86 | ||
85 | // Transcode the original video file to a lower resolution. | 87 | // Transcode the original video file to a lower resolution |
86 | async function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) { | 88 | // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed |
89 | function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) { | ||
87 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | 90 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
88 | const extname = '.mp4' | 91 | const extname = '.mp4' |
89 | 92 | ||
90 | // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed | 93 | return VideoPathManager.Instance.makeAvailableVideoFile(video, video.getMaxQualityFile(), async videoInputPath => { |
91 | const videoInputPath = getVideoFilePath(video, video.getMaxQualityFile()) | 94 | const newVideoFile = new VideoFileModel({ |
95 | resolution, | ||
96 | extname, | ||
97 | filename: generateWebTorrentVideoFilename(resolution, extname), | ||
98 | size: 0, | ||
99 | videoId: video.id | ||
100 | }) | ||
92 | 101 | ||
93 | const newVideoFile = new VideoFileModel({ | 102 | const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) |
94 | resolution, | 103 | const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) |
95 | extname, | ||
96 | filename: generateWebTorrentVideoFilename(resolution, extname), | ||
97 | size: 0, | ||
98 | videoId: video.id | ||
99 | }) | ||
100 | 104 | ||
101 | const videoOutputPath = getVideoFilePath(video, newVideoFile) | 105 | const transcodeOptions = resolution === VideoResolution.H_NOVIDEO |
102 | const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) | 106 | ? { |
107 | type: 'only-audio' as 'only-audio', | ||
103 | 108 | ||
104 | const transcodeOptions = resolution === VideoResolution.H_NOVIDEO | 109 | inputPath: videoInputPath, |
105 | ? { | 110 | outputPath: videoTranscodedPath, |
106 | type: 'only-audio' as 'only-audio', | ||
107 | 111 | ||
108 | inputPath: videoInputPath, | 112 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
109 | outputPath: videoTranscodedPath, | 113 | profile: CONFIG.TRANSCODING.PROFILE, |
110 | 114 | ||
111 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 115 | resolution, |
112 | profile: CONFIG.TRANSCODING.PROFILE, | ||
113 | 116 | ||
114 | resolution, | 117 | job |
118 | } | ||
119 | : { | ||
120 | type: 'video' as 'video', | ||
121 | inputPath: videoInputPath, | ||
122 | outputPath: videoTranscodedPath, | ||
115 | 123 | ||
116 | job | 124 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
117 | } | 125 | profile: CONFIG.TRANSCODING.PROFILE, |
118 | : { | ||
119 | type: 'video' as 'video', | ||
120 | inputPath: videoInputPath, | ||
121 | outputPath: videoTranscodedPath, | ||
122 | 126 | ||
123 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 127 | resolution, |
124 | profile: CONFIG.TRANSCODING.PROFILE, | 128 | isPortraitMode: isPortrait, |
125 | 129 | ||
126 | resolution, | 130 | job |
127 | isPortraitMode: isPortrait, | 131 | } |
128 | 132 | ||
129 | job | 133 | await transcode(transcodeOptions) |
130 | } | ||
131 | |||
132 | await transcode(transcodeOptions) | ||
133 | 134 | ||
134 | return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) | 135 | return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) |
136 | }) | ||
135 | } | 137 | } |
136 | 138 | ||
137 | // Merge an image with an audio file to create a video | 139 | // Merge an image with an audio file to create a video |
138 | async function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) { | 140 | function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) { |
139 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | 141 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
140 | const newExtname = '.mp4' | 142 | const newExtname = '.mp4' |
141 | 143 | ||
142 | const inputVideoFile = video.getMinQualityFile() | 144 | const inputVideoFile = video.getMinQualityFile() |
143 | 145 | ||
144 | const audioInputPath = getVideoFilePath(video, inputVideoFile) | 146 | return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async audioInputPath => { |
145 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | 147 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) |
146 | 148 | ||
147 | // If the user updates the video preview during transcoding | 149 | // If the user updates the video preview during transcoding |
148 | const previewPath = video.getPreview().getPath() | 150 | const previewPath = video.getPreview().getPath() |
149 | const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) | 151 | const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) |
150 | await copyFile(previewPath, tmpPreviewPath) | 152 | await copyFile(previewPath, tmpPreviewPath) |
151 | 153 | ||
152 | const transcodeOptions = { | 154 | const transcodeOptions = { |
153 | type: 'merge-audio' as 'merge-audio', | 155 | type: 'merge-audio' as 'merge-audio', |
154 | 156 | ||
155 | inputPath: tmpPreviewPath, | 157 | inputPath: tmpPreviewPath, |
156 | outputPath: videoTranscodedPath, | 158 | outputPath: videoTranscodedPath, |
157 | 159 | ||
158 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 160 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
159 | profile: CONFIG.TRANSCODING.PROFILE, | 161 | profile: CONFIG.TRANSCODING.PROFILE, |
160 | 162 | ||
161 | audioPath: audioInputPath, | 163 | audioPath: audioInputPath, |
162 | resolution, | 164 | resolution, |
163 | 165 | ||
164 | job | 166 | job |
165 | } | 167 | } |
166 | 168 | ||
167 | try { | 169 | try { |
168 | await transcode(transcodeOptions) | 170 | await transcode(transcodeOptions) |
169 | 171 | ||
170 | await remove(audioInputPath) | 172 | await remove(audioInputPath) |
171 | await remove(tmpPreviewPath) | 173 | await remove(tmpPreviewPath) |
172 | } catch (err) { | 174 | } catch (err) { |
173 | await remove(tmpPreviewPath) | 175 | await remove(tmpPreviewPath) |
174 | throw err | 176 | throw err |
175 | } | 177 | } |
176 | 178 | ||
177 | // Important to do this before getVideoFilename() to take in account the new file extension | 179 | // Important to do this before getVideoFilename() to take in account the new file extension |
178 | inputVideoFile.extname = newExtname | 180 | inputVideoFile.extname = newExtname |
179 | inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) | 181 | inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) |
180 | 182 | ||
181 | const videoOutputPath = getVideoFilePath(video, inputVideoFile) | 183 | const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) |
182 | // ffmpeg generated a new video file, so update the video duration | 184 | // ffmpeg generated a new video file, so update the video duration |
183 | // See https://trac.ffmpeg.org/ticket/5456 | 185 | // See https://trac.ffmpeg.org/ticket/5456 |
184 | video.duration = await getDurationFromVideoFile(videoTranscodedPath) | 186 | video.duration = await getDurationFromVideoFile(videoTranscodedPath) |
185 | await video.save() | 187 | await video.save() |
186 | 188 | ||
187 | return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) | 189 | return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) |
190 | }) | ||
188 | } | 191 | } |
189 | 192 | ||
190 | // Concat TS segments from a live video to a fragmented mp4 HLS playlist | 193 | // Concat TS segments from a live video to a fragmented mp4 HLS playlist |
@@ -258,7 +261,7 @@ async function onWebTorrentVideoFileTranscoding ( | |||
258 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) | 261 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) |
259 | video.VideoFiles = await video.$get('VideoFiles') | 262 | video.VideoFiles = await video.$get('VideoFiles') |
260 | 263 | ||
261 | return video | 264 | return { video, videoFile } |
262 | } | 265 | } |
263 | 266 | ||
264 | async function generateHlsPlaylistCommon (options: { | 267 | async function generateHlsPlaylistCommon (options: { |
@@ -335,14 +338,13 @@ async function generateHlsPlaylistCommon (options: { | |||
335 | videoStreamingPlaylistId: playlist.id | 338 | videoStreamingPlaylistId: playlist.id |
336 | }) | 339 | }) |
337 | 340 | ||
338 | const videoFilePath = getVideoFilePath(playlist, newVideoFile) | 341 | const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) |
339 | 342 | ||
340 | // Move files from tmp transcoded directory to the appropriate place | 343 | // Move files from tmp transcoded directory to the appropriate place |
341 | const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | 344 | await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) |
342 | await ensureDir(baseHlsDirectory) | ||
343 | 345 | ||
344 | // Move playlist file | 346 | // Move playlist file |
345 | const resolutionPlaylistPath = join(baseHlsDirectory, resolutionPlaylistFilename) | 347 | const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename) |
346 | await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) | 348 | await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) |
347 | // Move video file | 349 | // Move video file |
348 | await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) | 350 | await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) |
@@ -355,7 +357,7 @@ async function generateHlsPlaylistCommon (options: { | |||
355 | 357 | ||
356 | await createTorrentAndSetInfoHash(playlist, newVideoFile) | 358 | await createTorrentAndSetInfoHash(playlist, newVideoFile) |
357 | 359 | ||
358 | await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) | 360 | const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) |
359 | 361 | ||
360 | const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo | 362 | const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo |
361 | playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') | 363 | playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') |
@@ -368,5 +370,5 @@ async function generateHlsPlaylistCommon (options: { | |||
368 | await updateMasterHLSPlaylist(video, playlistWithFiles) | 370 | await updateMasterHLSPlaylist(video, playlistWithFiles) |
369 | await updateSha256VODSegments(video, playlistWithFiles) | 371 | await updateSha256VODSegments(video, playlistWithFiles) |
370 | 372 | ||
371 | return resolutionPlaylistPath | 373 | return { resolutionPlaylistPath, videoFile: savedVideoFile } |
372 | } | 374 | } |
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts new file mode 100644 index 000000000..4c5d0c89d --- /dev/null +++ b/server/lib/video-path-manager.ts | |||
@@ -0,0 +1,139 @@ | |||
1 | import { remove } from 'fs-extra' | ||
2 | import { extname, join } from 'path' | ||
3 | import { buildUUID } from '@server/helpers/uuid' | ||
4 | import { extractVideo } from '@server/helpers/video' | ||
5 | import { CONFIG } from '@server/initializers/config' | ||
6 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
7 | import { VideoStorage } from '@shared/models' | ||
8 | import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage' | ||
9 | import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' | ||
10 | |||
11 | type MakeAvailableCB <T> = (path: string) => Promise<T> | T | ||
12 | |||
13 | class VideoPathManager { | ||
14 | |||
15 | private static instance: VideoPathManager | ||
16 | |||
17 | private constructor () {} | ||
18 | |||
19 | getFSHLSOutputPath (video: MVideoUUID, filename?: string) { | ||
20 | const base = getHLSDirectory(video) | ||
21 | if (!filename) return base | ||
22 | |||
23 | return join(base, filename) | ||
24 | } | ||
25 | |||
26 | getFSRedundancyVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { | ||
27 | if (videoFile.isHLS()) { | ||
28 | const video = extractVideo(videoOrPlaylist) | ||
29 | |||
30 | return join(getHLSRedundancyDirectory(video), videoFile.filename) | ||
31 | } | ||
32 | |||
33 | return join(CONFIG.STORAGE.REDUNDANCY_DIR, videoFile.filename) | ||
34 | } | ||
35 | |||
36 | getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { | ||
37 | if (videoFile.isHLS()) { | ||
38 | const video = extractVideo(videoOrPlaylist) | ||
39 | |||
40 | return join(getHLSDirectory(video), videoFile.filename) | ||
41 | } | ||
42 | |||
43 | return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename) | ||
44 | } | ||
45 | |||
46 | async makeAvailableVideoFile <T> (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) { | ||
47 | if (videoFile.storage === VideoStorage.FILE_SYSTEM) { | ||
48 | return this.makeAvailableFactory( | ||
49 | () => this.getFSVideoFileOutputPath(videoOrPlaylist, videoFile), | ||
50 | false, | ||
51 | cb | ||
52 | ) | ||
53 | } | ||
54 | |||
55 | const destination = this.buildTMPDestination(videoFile.filename) | ||
56 | |||
57 | if (videoFile.isHLS()) { | ||
58 | const video = extractVideo(videoOrPlaylist) | ||
59 | |||
60 | return this.makeAvailableFactory( | ||
61 | () => makeHLSFileAvailable(videoOrPlaylist as MStreamingPlaylistVideo, video, videoFile.filename, destination), | ||
62 | true, | ||
63 | cb | ||
64 | ) | ||
65 | } | ||
66 | |||
67 | return this.makeAvailableFactory( | ||
68 | () => makeWebTorrentFileAvailable(videoFile.filename, destination), | ||
69 | true, | ||
70 | cb | ||
71 | ) | ||
72 | } | ||
73 | |||
74 | async makeAvailableResolutionPlaylistFile <T> (playlist: MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) { | ||
75 | const filename = getHlsResolutionPlaylistFilename(videoFile.filename) | ||
76 | |||
77 | if (videoFile.storage === VideoStorage.FILE_SYSTEM) { | ||
78 | return this.makeAvailableFactory( | ||
79 | () => join(getHLSDirectory(playlist.Video), filename), | ||
80 | false, | ||
81 | cb | ||
82 | ) | ||
83 | } | ||
84 | |||
85 | return this.makeAvailableFactory( | ||
86 | () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)), | ||
87 | true, | ||
88 | cb | ||
89 | ) | ||
90 | } | ||
91 | |||
92 | async makeAvailablePlaylistFile <T> (playlist: MStreamingPlaylistVideo, filename: string, cb: MakeAvailableCB<T>) { | ||
93 | if (playlist.storage === VideoStorage.FILE_SYSTEM) { | ||
94 | return this.makeAvailableFactory( | ||
95 | () => join(getHLSDirectory(playlist.Video), filename), | ||
96 | false, | ||
97 | cb | ||
98 | ) | ||
99 | } | ||
100 | |||
101 | return this.makeAvailableFactory( | ||
102 | () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)), | ||
103 | true, | ||
104 | cb | ||
105 | ) | ||
106 | } | ||
107 | |||
108 | private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) { | ||
109 | let result: T | ||
110 | |||
111 | const destination = await method() | ||
112 | |||
113 | try { | ||
114 | result = await cb(destination) | ||
115 | } catch (err) { | ||
116 | if (destination && clean) await remove(destination) | ||
117 | throw err | ||
118 | } | ||
119 | |||
120 | if (clean) await remove(destination) | ||
121 | |||
122 | return result | ||
123 | } | ||
124 | |||
125 | private buildTMPDestination (filename: string) { | ||
126 | return join(CONFIG.STORAGE.TMP_DIR, buildUUID() + extname(filename)) | ||
127 | |||
128 | } | ||
129 | |||
130 | static get Instance () { | ||
131 | return this.instance || (this.instance = new this()) | ||
132 | } | ||
133 | } | ||
134 | |||
135 | // --------------------------------------------------------------------------- | ||
136 | |||
137 | export { | ||
138 | VideoPathManager | ||
139 | } | ||
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts new file mode 100644 index 000000000..0613d94bf --- /dev/null +++ b/server/lib/video-state.ts | |||
@@ -0,0 +1,99 @@ | |||
1 | import { Transaction } from 'sequelize' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { sequelizeTypescript } from '@server/initializers/database' | ||
5 | import { VideoModel } from '@server/models/video/video' | ||
6 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
7 | import { MVideoFullLight, MVideoUUID } from '@server/types/models' | ||
8 | import { VideoState } from '@shared/models' | ||
9 | import { federateVideoIfNeeded } from './activitypub/videos' | ||
10 | import { Notifier } from './notifier' | ||
11 | import { addMoveToObjectStorageJob } from './video' | ||
12 | |||
13 | function buildNextVideoState (currentState?: VideoState) { | ||
14 | if (currentState === VideoState.PUBLISHED) { | ||
15 | throw new Error('Video is already in its final state') | ||
16 | } | ||
17 | |||
18 | if ( | ||
19 | currentState !== VideoState.TO_TRANSCODE && | ||
20 | currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && | ||
21 | CONFIG.TRANSCODING.ENABLED | ||
22 | ) { | ||
23 | return VideoState.TO_TRANSCODE | ||
24 | } | ||
25 | |||
26 | if ( | ||
27 | currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && | ||
28 | CONFIG.OBJECT_STORAGE.ENABLED | ||
29 | ) { | ||
30 | return VideoState.TO_MOVE_TO_EXTERNAL_STORAGE | ||
31 | } | ||
32 | |||
33 | return VideoState.PUBLISHED | ||
34 | } | ||
35 | |||
36 | function moveToNextState (video: MVideoUUID, isNewVideo = true) { | ||
37 | return sequelizeTypescript.transaction(async t => { | ||
38 | // Maybe the video changed in database, refresh it | ||
39 | const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | ||
40 | // Video does not exist anymore | ||
41 | if (!videoDatabase) return undefined | ||
42 | |||
43 | // Already in its final state | ||
44 | if (videoDatabase.state === VideoState.PUBLISHED) { | ||
45 | return federateVideoIfNeeded(videoDatabase, false, t) | ||
46 | } | ||
47 | |||
48 | const newState = buildNextVideoState(videoDatabase.state) | ||
49 | |||
50 | if (newState === VideoState.PUBLISHED) { | ||
51 | return moveToPublishedState(videoDatabase, isNewVideo, t) | ||
52 | } | ||
53 | |||
54 | if (newState === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { | ||
55 | return moveToExternalStorageState(videoDatabase, isNewVideo, t) | ||
56 | } | ||
57 | }) | ||
58 | } | ||
59 | |||
60 | // --------------------------------------------------------------------------- | ||
61 | |||
62 | export { | ||
63 | buildNextVideoState, | ||
64 | moveToNextState | ||
65 | } | ||
66 | |||
67 | // --------------------------------------------------------------------------- | ||
68 | |||
69 | async function moveToPublishedState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) { | ||
70 | logger.info('Publishing video %s.', video.uuid, { tags: [ video.uuid ] }) | ||
71 | |||
72 | const previousState = video.state | ||
73 | await video.setNewState(VideoState.PUBLISHED, transaction) | ||
74 | |||
75 | // If the video was not published, we consider it is a new one for other instances | ||
76 | // Live videos are always federated, so it's not a new video | ||
77 | await federateVideoIfNeeded(video, isNewVideo, transaction) | ||
78 | |||
79 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) | ||
80 | |||
81 | if (previousState === VideoState.TO_TRANSCODE) { | ||
82 | Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video) | ||
83 | } | ||
84 | } | ||
85 | |||
86 | async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) { | ||
87 | const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction) | ||
88 | const pendingTranscode = videoJobInfo?.pendingTranscode || 0 | ||
89 | |||
90 | // We want to wait all transcoding jobs before moving the video on an external storage | ||
91 | if (pendingTranscode !== 0) return | ||
92 | |||
93 | await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, transaction) | ||
94 | |||
95 | logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] }) | ||
96 | |||
97 | addMoveToObjectStorageJob(video, isNewVideo) | ||
98 | .catch(err => logger.error('Cannot add move to object storage job', { err })) | ||
99 | } | ||
diff --git a/server/lib/video-urls.ts b/server/lib/video-urls.ts new file mode 100644 index 000000000..64c2c9bf9 --- /dev/null +++ b/server/lib/video-urls.ts | |||
@@ -0,0 +1,31 @@ | |||
1 | |||
2 | import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' | ||
3 | import { MStreamingPlaylist, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
4 | |||
5 | // ################## Redundancy ################## | ||
6 | |||
7 | function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { | ||
8 | // Base URL used by our HLS player | ||
9 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid | ||
10 | } | ||
11 | |||
12 | function generateWebTorrentRedundancyUrl (file: MVideoFile) { | ||
13 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename | ||
14 | } | ||
15 | |||
16 | // ################## Meta data ################## | ||
17 | |||
18 | function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { | ||
19 | const path = '/api/v1/videos/' | ||
20 | |||
21 | return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id | ||
22 | } | ||
23 | |||
24 | // --------------------------------------------------------------------------- | ||
25 | |||
26 | export { | ||
27 | getLocalVideoFileMetadataUrl, | ||
28 | |||
29 | generateWebTorrentRedundancyUrl, | ||
30 | generateHLSRedundancyUrl | ||
31 | } | ||
diff --git a/server/lib/video.ts b/server/lib/video.ts index 61fee4949..0a2b93cc0 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts | |||
@@ -1,15 +1,13 @@ | |||
1 | import { UploadFiles } from 'express' | 1 | import { UploadFiles } from 'express' |
2 | import { Transaction } from 'sequelize/types' | 2 | import { Transaction } from 'sequelize/types' |
3 | import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' | 3 | import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' |
4 | import { sequelizeTypescript } from '@server/initializers/database' | ||
5 | import { TagModel } from '@server/models/video/tag' | 4 | import { TagModel } from '@server/models/video/tag' |
6 | import { VideoModel } from '@server/models/video/video' | 5 | import { VideoModel } from '@server/models/video/video' |
6 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
7 | import { FilteredModelAttributes } from '@server/types' | 7 | import { FilteredModelAttributes } from '@server/types' |
8 | import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' | 8 | import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' |
9 | import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models' | 9 | import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models' |
10 | import { federateVideoIfNeeded } from './activitypub/videos' | 10 | import { CreateJobOptions, JobQueue } from './job-queue/job-queue' |
11 | import { JobQueue } from './job-queue/job-queue' | ||
12 | import { Notifier } from './notifier' | ||
13 | import { updateVideoMiniatureFromExisting } from './thumbnail' | 11 | import { updateVideoMiniatureFromExisting } from './thumbnail' |
14 | 12 | ||
15 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { | 13 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { |
@@ -82,29 +80,6 @@ async function setVideoTags (options: { | |||
82 | video.Tags = tagInstances | 80 | video.Tags = tagInstances |
83 | } | 81 | } |
84 | 82 | ||
85 | async function publishAndFederateIfNeeded (video: MVideoUUID, wasLive = false) { | ||
86 | const result = await sequelizeTypescript.transaction(async t => { | ||
87 | // Maybe the video changed in database, refresh it | ||
88 | const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | ||
89 | // Video does not exist anymore | ||
90 | if (!videoDatabase) return undefined | ||
91 | |||
92 | // We transcoded the video file in another format, now we can publish it | ||
93 | const videoPublished = await videoDatabase.publishIfNeededAndSave(t) | ||
94 | |||
95 | // If the video was not published, we consider it is a new one for other instances | ||
96 | // Live videos are always federated, so it's not a new video | ||
97 | await federateVideoIfNeeded(videoDatabase, !wasLive && videoPublished, t) | ||
98 | |||
99 | return { videoDatabase, videoPublished } | ||
100 | }) | ||
101 | |||
102 | if (result?.videoPublished) { | ||
103 | Notifier.Instance.notifyOnNewVideoIfNeeded(result.videoDatabase) | ||
104 | Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(result.videoDatabase) | ||
105 | } | ||
106 | } | ||
107 | |||
108 | async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) { | 83 | async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) { |
109 | let dataInput: VideoTranscodingPayload | 84 | let dataInput: VideoTranscodingPayload |
110 | 85 | ||
@@ -127,7 +102,20 @@ async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoF | |||
127 | priority: await getTranscodingJobPriority(user) | 102 | priority: await getTranscodingJobPriority(user) |
128 | } | 103 | } |
129 | 104 | ||
130 | return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput }, jobOptions) | 105 | return addTranscodingJob(dataInput, jobOptions) |
106 | } | ||
107 | |||
108 | async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions) { | ||
109 | await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') | ||
110 | |||
111 | return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: payload }, options) | ||
112 | } | ||
113 | |||
114 | async function addMoveToObjectStorageJob (video: MVideoUUID, isNewVideo = true) { | ||
115 | await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove') | ||
116 | |||
117 | const dataInput = { videoUUID: video.uuid, isNewVideo } | ||
118 | return JobQueue.Instance.createJobWithPromise({ type: 'move-to-object-storage', payload: dataInput }) | ||
131 | } | 119 | } |
132 | 120 | ||
133 | async function getTranscodingJobPriority (user: MUserId) { | 121 | async function getTranscodingJobPriority (user: MUserId) { |
@@ -143,9 +131,10 @@ async function getTranscodingJobPriority (user: MUserId) { | |||
143 | 131 | ||
144 | export { | 132 | export { |
145 | buildLocalVideoFromReq, | 133 | buildLocalVideoFromReq, |
146 | publishAndFederateIfNeeded, | ||
147 | buildVideoThumbnailsFromReq, | 134 | buildVideoThumbnailsFromReq, |
148 | setVideoTags, | 135 | setVideoTags, |
149 | addOptimizeOrMergeAudioJob, | 136 | addOptimizeOrMergeAudioJob, |
137 | addTranscodingJob, | ||
138 | addMoveToObjectStorageJob, | ||
150 | getTranscodingJobPriority | 139 | getTranscodingJobPriority |
151 | } | 140 | } |
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts index 8a54de3b0..b3c4f390d 100644 --- a/server/models/video/formatter/video-format-utils.ts +++ b/server/models/video/formatter/video-format-utils.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { uuidToShort } from '@server/helpers/uuid' | 1 | import { uuidToShort } from '@server/helpers/uuid' |
2 | import { generateMagnetUri } from '@server/helpers/webtorrent' | 2 | import { generateMagnetUri } from '@server/helpers/webtorrent' |
3 | import { getLocalVideoFileMetadataUrl } from '@server/lib/video-paths' | 3 | import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls' |
4 | import { VideoFile } from '@shared/models/videos/video-file.model' | 4 | import { VideoFile } from '@shared/models/videos/video-file.model' |
5 | import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects' | 5 | import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects' |
6 | import { Video, VideoDetails } from '../../../../shared/models/videos' | 6 | import { Video, VideoDetails } from '../../../../shared/models/videos' |
diff --git a/server/models/video/sql/shared/video-tables.ts b/server/models/video/sql/shared/video-tables.ts index 742d19099..75823864d 100644 --- a/server/models/video/sql/shared/video-tables.ts +++ b/server/models/video/sql/shared/video-tables.ts | |||
@@ -87,7 +87,8 @@ export class VideoTables { | |||
87 | 'fps', | 87 | 'fps', |
88 | 'metadataUrl', | 88 | 'metadataUrl', |
89 | 'videoStreamingPlaylistId', | 89 | 'videoStreamingPlaylistId', |
90 | 'videoId' | 90 | 'videoId', |
91 | 'storage' | ||
91 | ] | 92 | ] |
92 | } | 93 | } |
93 | 94 | ||
@@ -102,7 +103,8 @@ export class VideoTables { | |||
102 | 'segmentsSha256Url', | 103 | 'segmentsSha256Url', |
103 | 'videoId', | 104 | 'videoId', |
104 | 'createdAt', | 105 | 'createdAt', |
105 | 'updatedAt' | 106 | 'updatedAt', |
107 | 'storage' | ||
106 | ]) | 108 | ]) |
107 | } | 109 | } |
108 | 110 | ||
@@ -258,7 +260,8 @@ export class VideoTables { | |||
258 | 'originallyPublishedAt', | 260 | 'originallyPublishedAt', |
259 | 'channelId', | 261 | 'channelId', |
260 | 'createdAt', | 262 | 'createdAt', |
261 | 'updatedAt' | 263 | 'updatedAt', |
264 | 'moveJobsRunning' | ||
262 | ] | 265 | ] |
263 | } | 266 | } |
264 | } | 267 | } |
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 09fc5288b..627c95763 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts | |||
@@ -23,9 +23,11 @@ import validator from 'validator' | |||
23 | import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' | 23 | import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' |
24 | import { logger } from '@server/helpers/logger' | 24 | import { logger } from '@server/helpers/logger' |
25 | import { extractVideo } from '@server/helpers/video' | 25 | import { extractVideo } from '@server/helpers/video' |
26 | import { getTorrentFilePath } from '@server/lib/video-paths' | 26 | import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage' |
27 | import { getFSTorrentFilePath } from '@server/lib/paths' | ||
27 | import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' | 28 | import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' |
28 | import { AttributesOnly } from '@shared/core-utils' | 29 | import { AttributesOnly } from '@shared/core-utils' |
30 | import { VideoStorage } from '@shared/models' | ||
29 | import { | 31 | import { |
30 | isVideoFileExtnameValid, | 32 | isVideoFileExtnameValid, |
31 | isVideoFileInfoHashValid, | 33 | isVideoFileInfoHashValid, |
@@ -214,6 +216,11 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
214 | @Column | 216 | @Column |
215 | videoId: number | 217 | videoId: number |
216 | 218 | ||
219 | @AllowNull(false) | ||
220 | @Default(VideoStorage.FILE_SYSTEM) | ||
221 | @Column | ||
222 | storage: VideoStorage | ||
223 | |||
217 | @BelongsTo(() => VideoModel, { | 224 | @BelongsTo(() => VideoModel, { |
218 | foreignKey: { | 225 | foreignKey: { |
219 | allowNull: true | 226 | allowNull: true |
@@ -273,7 +280,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
273 | 280 | ||
274 | static async doesOwnedWebTorrentVideoFileExist (filename: string) { | 281 | static async doesOwnedWebTorrentVideoFileExist (filename: string) { |
275 | const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + | 282 | const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + |
276 | 'WHERE "filename" = $filename LIMIT 1' | 283 | `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1` |
277 | 284 | ||
278 | return doesExist(query, { filename }) | 285 | return doesExist(query, { filename }) |
279 | } | 286 | } |
@@ -450,9 +457,20 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
450 | return !!this.videoStreamingPlaylistId | 457 | return !!this.videoStreamingPlaylistId |
451 | } | 458 | } |
452 | 459 | ||
460 | getObjectStorageUrl () { | ||
461 | if (this.isHLS()) { | ||
462 | return getHLSPublicFileUrl(this.fileUrl) | ||
463 | } | ||
464 | |||
465 | return getWebTorrentPublicFileUrl(this.fileUrl) | ||
466 | } | ||
467 | |||
453 | getFileUrl (video: MVideo) { | 468 | getFileUrl (video: MVideo) { |
454 | if (!this.Video) this.Video = video as VideoModel | 469 | if (this.storage === VideoStorage.OBJECT_STORAGE) { |
470 | return this.getObjectStorageUrl() | ||
471 | } | ||
455 | 472 | ||
473 | if (!this.Video) this.Video = video as VideoModel | ||
456 | if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) | 474 | if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) |
457 | 475 | ||
458 | return this.fileUrl | 476 | return this.fileUrl |
@@ -503,7 +521,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
503 | removeTorrent () { | 521 | removeTorrent () { |
504 | if (!this.torrentFilename) return null | 522 | if (!this.torrentFilename) return null |
505 | 523 | ||
506 | const torrentPath = getTorrentFilePath(this) | 524 | const torrentPath = getFSTorrentFilePath(this) |
507 | return remove(torrentPath) | 525 | return remove(torrentPath) |
508 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) | 526 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) |
509 | } | 527 | } |
diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts new file mode 100644 index 000000000..7c1fe6734 --- /dev/null +++ b/server/models/video/video-job-info.ts | |||
@@ -0,0 +1,100 @@ | |||
1 | import { Op, QueryTypes, Transaction } from 'sequelize' | ||
2 | import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Model, Table, Unique, UpdatedAt } from 'sequelize-typescript' | ||
3 | import { AttributesOnly } from '@shared/core-utils' | ||
4 | import { VideoModel } from './video' | ||
5 | |||
6 | @Table({ | ||
7 | tableName: 'videoJobInfo', | ||
8 | indexes: [ | ||
9 | { | ||
10 | fields: [ 'videoId' ], | ||
11 | where: { | ||
12 | videoId: { | ||
13 | [Op.ne]: null | ||
14 | } | ||
15 | } | ||
16 | } | ||
17 | ] | ||
18 | }) | ||
19 | |||
20 | export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfoModel>>> { | ||
21 | @CreatedAt | ||
22 | createdAt: Date | ||
23 | |||
24 | @UpdatedAt | ||
25 | updatedAt: Date | ||
26 | |||
27 | @AllowNull(false) | ||
28 | @Default(0) | ||
29 | @IsInt | ||
30 | @Column | ||
31 | pendingMove: number | ||
32 | |||
33 | @AllowNull(false) | ||
34 | @Default(0) | ||
35 | @IsInt | ||
36 | @Column | ||
37 | pendingTranscode: number | ||
38 | |||
39 | @ForeignKey(() => VideoModel) | ||
40 | @Unique | ||
41 | @Column | ||
42 | videoId: number | ||
43 | |||
44 | @BelongsTo(() => VideoModel, { | ||
45 | foreignKey: { | ||
46 | allowNull: false | ||
47 | }, | ||
48 | onDelete: 'cascade' | ||
49 | }) | ||
50 | Video: VideoModel | ||
51 | |||
52 | static load (videoId: number, transaction: Transaction) { | ||
53 | const where = { | ||
54 | videoId | ||
55 | } | ||
56 | |||
57 | return VideoJobInfoModel.findOne({ where, transaction }) | ||
58 | } | ||
59 | |||
60 | static async increaseOrCreate (videoUUID: string, column: 'pendingMove' | 'pendingTranscode'): Promise<number> { | ||
61 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } | ||
62 | |||
63 | const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` | ||
64 | INSERT INTO "videoJobInfo" ("videoId", "${column}", "createdAt", "updatedAt") | ||
65 | SELECT | ||
66 | "video"."id" AS "videoId", 1, NOW(), NOW() | ||
67 | FROM | ||
68 | "video" | ||
69 | WHERE | ||
70 | "video"."uuid" = $videoUUID | ||
71 | ON CONFLICT ("videoId") DO UPDATE | ||
72 | SET | ||
73 | "${column}" = "videoJobInfo"."${column}" + 1, | ||
74 | "updatedAt" = NOW() | ||
75 | RETURNING | ||
76 | "${column}" | ||
77 | `, options) | ||
78 | |||
79 | return pendingMove | ||
80 | } | ||
81 | |||
82 | static async decrease (videoUUID: string, column: 'pendingMove' | 'pendingTranscode'): Promise<number> { | ||
83 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } | ||
84 | |||
85 | const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` | ||
86 | UPDATE | ||
87 | "videoJobInfo" | ||
88 | SET | ||
89 | "${column}" = "videoJobInfo"."${column}" - 1, | ||
90 | "updatedAt" = NOW() | ||
91 | FROM "video" | ||
92 | WHERE | ||
93 | "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID | ||
94 | RETURNING | ||
95 | "${column}"; | ||
96 | `, options) | ||
97 | |||
98 | return pendingMove | ||
99 | } | ||
100 | } | ||
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index d591a3134..3e9fd97c7 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts | |||
@@ -1,10 +1,25 @@ | |||
1 | import * as memoizee from 'memoizee' | 1 | import * as memoizee from 'memoizee' |
2 | import { join } from 'path' | 2 | import { join } from 'path' |
3 | import { Op } from 'sequelize' | 3 | import { Op } from 'sequelize' |
4 | import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' | 4 | import { |
5 | AllowNull, | ||
6 | BelongsTo, | ||
7 | Column, | ||
8 | CreatedAt, | ||
9 | DataType, | ||
10 | Default, | ||
11 | ForeignKey, | ||
12 | HasMany, | ||
13 | Is, | ||
14 | Model, | ||
15 | Table, | ||
16 | UpdatedAt | ||
17 | } from 'sequelize-typescript' | ||
18 | import { getHLSPublicFileUrl } from '@server/lib/object-storage' | ||
5 | import { VideoFileModel } from '@server/models/video/video-file' | 19 | import { VideoFileModel } from '@server/models/video/video-file' |
6 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | 20 | import { MStreamingPlaylist, MVideo } from '@server/types/models' |
7 | import { AttributesOnly } from '@shared/core-utils' | 21 | import { AttributesOnly } from '@shared/core-utils' |
22 | import { VideoStorage } from '@shared/models' | ||
8 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | 23 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' |
9 | import { sha1 } from '../../helpers/core-utils' | 24 | import { sha1 } from '../../helpers/core-utils' |
10 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | 25 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' |
@@ -81,6 +96,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi | |||
81 | @Column | 96 | @Column |
82 | videoId: number | 97 | videoId: number |
83 | 98 | ||
99 | @AllowNull(false) | ||
100 | @Default(VideoStorage.FILE_SYSTEM) | ||
101 | @Column | ||
102 | storage: VideoStorage | ||
103 | |||
84 | @BelongsTo(() => VideoModel, { | 104 | @BelongsTo(() => VideoModel, { |
85 | foreignKey: { | 105 | foreignKey: { |
86 | allowNull: false | 106 | allowNull: false |
@@ -185,12 +205,20 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi | |||
185 | } | 205 | } |
186 | 206 | ||
187 | getMasterPlaylistUrl (video: MVideo) { | 207 | getMasterPlaylistUrl (video: MVideo) { |
208 | if (this.storage === VideoStorage.OBJECT_STORAGE) { | ||
209 | return getHLSPublicFileUrl(this.playlistUrl) | ||
210 | } | ||
211 | |||
188 | if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) | 212 | if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) |
189 | 213 | ||
190 | return this.playlistUrl | 214 | return this.playlistUrl |
191 | } | 215 | } |
192 | 216 | ||
193 | getSha256SegmentsUrl (video: MVideo) { | 217 | getSha256SegmentsUrl (video: MVideo) { |
218 | if (this.storage === VideoStorage.OBJECT_STORAGE) { | ||
219 | return getHLSPublicFileUrl(this.segmentsSha256Url) | ||
220 | } | ||
221 | |||
194 | if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive) | 222 | if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive) |
195 | 223 | ||
196 | return this.segmentsSha256Url | 224 | return this.segmentsSha256Url |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 56a5b0e18..874ad168a 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -28,14 +28,16 @@ import { buildNSFWFilter } from '@server/helpers/express-utils' | |||
28 | import { uuidToShort } from '@server/helpers/uuid' | 28 | import { uuidToShort } from '@server/helpers/uuid' |
29 | import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' | 29 | import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' |
30 | import { LiveManager } from '@server/lib/live/live-manager' | 30 | import { LiveManager } from '@server/lib/live/live-manager' |
31 | import { getHLSDirectory, getVideoFilePath } from '@server/lib/video-paths' | 31 | import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' |
32 | import { getHLSDirectory, getHLSRedundancyDirectory } from '@server/lib/paths' | ||
33 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
32 | import { getServerActor } from '@server/models/application/application' | 34 | import { getServerActor } from '@server/models/application/application' |
33 | import { ModelCache } from '@server/models/model-cache' | 35 | import { ModelCache } from '@server/models/model-cache' |
34 | import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' | 36 | import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' |
35 | import { VideoFile } from '@shared/models/videos/video-file.model' | 37 | import { VideoFile } from '@shared/models/videos/video-file.model' |
36 | import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared' | 38 | import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared' |
37 | import { VideoObject } from '../../../shared/models/activitypub/objects' | 39 | import { VideoObject } from '../../../shared/models/activitypub/objects' |
38 | import { Video, VideoDetails, VideoRateType } from '../../../shared/models/videos' | 40 | import { Video, VideoDetails, VideoRateType, VideoStorage } from '../../../shared/models/videos' |
39 | import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' | 41 | import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' |
40 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' | 42 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' |
41 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | 43 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' |
@@ -114,6 +116,7 @@ import { ScopeNames as VideoChannelScopeNames, SummaryOptions, VideoChannelModel | |||
114 | import { VideoCommentModel } from './video-comment' | 116 | import { VideoCommentModel } from './video-comment' |
115 | import { VideoFileModel } from './video-file' | 117 | import { VideoFileModel } from './video-file' |
116 | import { VideoImportModel } from './video-import' | 118 | import { VideoImportModel } from './video-import' |
119 | import { VideoJobInfoModel } from './video-job-info' | ||
117 | import { VideoLiveModel } from './video-live' | 120 | import { VideoLiveModel } from './video-live' |
118 | import { VideoPlaylistElementModel } from './video-playlist-element' | 121 | import { VideoPlaylistElementModel } from './video-playlist-element' |
119 | import { VideoShareModel } from './video-share' | 122 | import { VideoShareModel } from './video-share' |
@@ -732,6 +735,15 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
732 | }) | 735 | }) |
733 | VideoCaptions: VideoCaptionModel[] | 736 | VideoCaptions: VideoCaptionModel[] |
734 | 737 | ||
738 | @HasOne(() => VideoJobInfoModel, { | ||
739 | foreignKey: { | ||
740 | name: 'videoId', | ||
741 | allowNull: false | ||
742 | }, | ||
743 | onDelete: 'cascade' | ||
744 | }) | ||
745 | VideoJobInfo: VideoJobInfoModel | ||
746 | |||
735 | @BeforeDestroy | 747 | @BeforeDestroy |
736 | static async sendDelete (instance: MVideoAccountLight, options) { | 748 | static async sendDelete (instance: MVideoAccountLight, options) { |
737 | if (!instance.isOwned()) return undefined | 749 | if (!instance.isOwned()) return undefined |
@@ -1641,9 +1653,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1641 | getMaxQualityResolution () { | 1653 | getMaxQualityResolution () { |
1642 | const file = this.getMaxQualityFile() | 1654 | const file = this.getMaxQualityFile() |
1643 | const videoOrPlaylist = file.getVideoOrStreamingPlaylist() | 1655 | const videoOrPlaylist = file.getVideoOrStreamingPlaylist() |
1644 | const originalFilePath = getVideoFilePath(videoOrPlaylist, file) | ||
1645 | 1656 | ||
1646 | return getVideoFileResolution(originalFilePath) | 1657 | return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, file, originalFilePath => { |
1658 | return getVideoFileResolution(originalFilePath) | ||
1659 | }) | ||
1647 | } | 1660 | } |
1648 | 1661 | ||
1649 | getDescriptionAPIPath () { | 1662 | getDescriptionAPIPath () { |
@@ -1673,16 +1686,24 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1673 | } | 1686 | } |
1674 | 1687 | ||
1675 | removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) { | 1688 | removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) { |
1676 | const filePath = getVideoFilePath(this, videoFile, isRedundancy) | 1689 | const filePath = isRedundancy |
1690 | ? VideoPathManager.Instance.getFSRedundancyVideoFilePath(this, videoFile) | ||
1691 | : VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile) | ||
1677 | 1692 | ||
1678 | const promises: Promise<any>[] = [ remove(filePath) ] | 1693 | const promises: Promise<any>[] = [ remove(filePath) ] |
1679 | if (!isRedundancy) promises.push(videoFile.removeTorrent()) | 1694 | if (!isRedundancy) promises.push(videoFile.removeTorrent()) |
1680 | 1695 | ||
1696 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { | ||
1697 | promises.push(removeWebTorrentObjectStorage(videoFile)) | ||
1698 | } | ||
1699 | |||
1681 | return Promise.all(promises) | 1700 | return Promise.all(promises) |
1682 | } | 1701 | } |
1683 | 1702 | ||
1684 | async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) { | 1703 | async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) { |
1685 | const directoryPath = getHLSDirectory(this, isRedundancy) | 1704 | const directoryPath = isRedundancy |
1705 | ? getHLSRedundancyDirectory(this) | ||
1706 | : getHLSDirectory(this) | ||
1686 | 1707 | ||
1687 | await remove(directoryPath) | 1708 | await remove(directoryPath) |
1688 | 1709 | ||
@@ -1698,6 +1719,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1698 | await Promise.all( | 1719 | await Promise.all( |
1699 | streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent()) | 1720 | streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent()) |
1700 | ) | 1721 | ) |
1722 | |||
1723 | if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | ||
1724 | await removeHLSObjectStorage(streamingPlaylist, this) | ||
1725 | } | ||
1701 | } | 1726 | } |
1702 | } | 1727 | } |
1703 | 1728 | ||
@@ -1741,16 +1766,16 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1741 | this.privacy === VideoPrivacy.INTERNAL | 1766 | this.privacy === VideoPrivacy.INTERNAL |
1742 | } | 1767 | } |
1743 | 1768 | ||
1744 | async publishIfNeededAndSave (t: Transaction) { | 1769 | async setNewState (newState: VideoState, transaction: Transaction) { |
1745 | if (this.state !== VideoState.PUBLISHED) { | 1770 | if (this.state === newState) throw new Error('Cannot use same state ' + newState) |
1746 | this.state = VideoState.PUBLISHED | 1771 | |
1747 | this.publishedAt = new Date() | 1772 | this.state = newState |
1748 | await this.save({ transaction: t }) | ||
1749 | 1773 | ||
1750 | return true | 1774 | if (this.state === VideoState.PUBLISHED) { |
1775 | this.publishedAt = new Date() | ||
1751 | } | 1776 | } |
1752 | 1777 | ||
1753 | return false | 1778 | await this.save({ transaction }) |
1754 | } | 1779 | } |
1755 | 1780 | ||
1756 | getBandwidthBits (videoFile: MVideoFile) { | 1781 | getBandwidthBits (videoFile: MVideoFile) { |
diff --git a/server/tests/api/index.ts b/server/tests/api/index.ts index b62e2f5f7..19301c0b9 100644 --- a/server/tests/api/index.ts +++ b/server/tests/api/index.ts | |||
@@ -2,6 +2,7 @@ | |||
2 | import './activitypub' | 2 | import './activitypub' |
3 | import './check-params' | 3 | import './check-params' |
4 | import './moderation' | 4 | import './moderation' |
5 | import './object-storage' | ||
5 | import './notifications' | 6 | import './notifications' |
6 | import './redundancy' | 7 | import './redundancy' |
7 | import './search' | 8 | import './search' |
diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts index 8f1fb78a5..6c4ea90ca 100644 --- a/server/tests/api/live/live-save-replay.ts +++ b/server/tests/api/live/live-save-replay.ts | |||
@@ -15,7 +15,9 @@ import { | |||
15 | stopFfmpeg, | 15 | stopFfmpeg, |
16 | testFfmpegStreamError, | 16 | testFfmpegStreamError, |
17 | wait, | 17 | wait, |
18 | waitJobs | 18 | waitJobs, |
19 | waitUntilLivePublishedOnAllServers, | ||
20 | waitUntilLiveSavedOnAllServers | ||
19 | } from '@shared/extra-utils' | 21 | } from '@shared/extra-utils' |
20 | import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' | 22 | import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' |
21 | 23 | ||
@@ -66,18 +68,6 @@ describe('Save replay setting', function () { | |||
66 | } | 68 | } |
67 | } | 69 | } |
68 | 70 | ||
69 | async function waitUntilLivePublishedOnAllServers (videoId: string) { | ||
70 | for (const server of servers) { | ||
71 | await server.live.waitUntilPublished({ videoId }) | ||
72 | } | ||
73 | } | ||
74 | |||
75 | async function waitUntilLiveSavedOnAllServers (videoId: string) { | ||
76 | for (const server of servers) { | ||
77 | await server.live.waitUntilSaved({ videoId }) | ||
78 | } | ||
79 | } | ||
80 | |||
81 | before(async function () { | 71 | before(async function () { |
82 | this.timeout(120000) | 72 | this.timeout(120000) |
83 | 73 | ||
@@ -127,7 +117,7 @@ describe('Save replay setting', function () { | |||
127 | 117 | ||
128 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 118 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
129 | 119 | ||
130 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 120 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
131 | 121 | ||
132 | await waitJobs(servers) | 122 | await waitJobs(servers) |
133 | 123 | ||
@@ -160,7 +150,7 @@ describe('Save replay setting', function () { | |||
160 | 150 | ||
161 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 151 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
162 | 152 | ||
163 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 153 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
164 | 154 | ||
165 | await waitJobs(servers) | 155 | await waitJobs(servers) |
166 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | 156 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
@@ -189,7 +179,7 @@ describe('Save replay setting', function () { | |||
189 | 179 | ||
190 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 180 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
191 | 181 | ||
192 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 182 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
193 | 183 | ||
194 | await waitJobs(servers) | 184 | await waitJobs(servers) |
195 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | 185 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
@@ -224,7 +214,7 @@ describe('Save replay setting', function () { | |||
224 | this.timeout(20000) | 214 | this.timeout(20000) |
225 | 215 | ||
226 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 216 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
227 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 217 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
228 | 218 | ||
229 | await waitJobs(servers) | 219 | await waitJobs(servers) |
230 | 220 | ||
@@ -237,7 +227,7 @@ describe('Save replay setting', function () { | |||
237 | 227 | ||
238 | await stopFfmpeg(ffmpegCommand) | 228 | await stopFfmpeg(ffmpegCommand) |
239 | 229 | ||
240 | await waitUntilLiveSavedOnAllServers(liveVideoUUID) | 230 | await waitUntilLiveSavedOnAllServers(servers, liveVideoUUID) |
241 | await waitJobs(servers) | 231 | await waitJobs(servers) |
242 | 232 | ||
243 | // Live has been transcoded | 233 | // Live has been transcoded |
@@ -268,7 +258,7 @@ describe('Save replay setting', function () { | |||
268 | liveVideoUUID = await createLiveWrapper(true) | 258 | liveVideoUUID = await createLiveWrapper(true) |
269 | 259 | ||
270 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 260 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
271 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 261 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
272 | 262 | ||
273 | await waitJobs(servers) | 263 | await waitJobs(servers) |
274 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | 264 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
@@ -296,7 +286,7 @@ describe('Save replay setting', function () { | |||
296 | liveVideoUUID = await createLiveWrapper(true) | 286 | liveVideoUUID = await createLiveWrapper(true) |
297 | 287 | ||
298 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 288 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
299 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 289 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
300 | 290 | ||
301 | await waitJobs(servers) | 291 | await waitJobs(servers) |
302 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | 292 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
diff --git a/server/tests/api/object-storage/index.ts b/server/tests/api/object-storage/index.ts new file mode 100644 index 000000000..f319d6ef5 --- /dev/null +++ b/server/tests/api/object-storage/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './live' | ||
2 | export * from './video-imports' | ||
3 | export * from './videos' | ||
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts new file mode 100644 index 000000000..d3e6777f2 --- /dev/null +++ b/server/tests/api/object-storage/live.ts | |||
@@ -0,0 +1,136 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
6 | import { | ||
7 | areObjectStorageTestsDisabled, | ||
8 | createMultipleServers, | ||
9 | doubleFollow, | ||
10 | expectStartWith, | ||
11 | killallServers, | ||
12 | makeRawRequest, | ||
13 | ObjectStorageCommand, | ||
14 | PeerTubeServer, | ||
15 | setAccessTokensToServers, | ||
16 | setDefaultVideoChannel, | ||
17 | stopFfmpeg, | ||
18 | waitJobs, | ||
19 | waitUntilLivePublishedOnAllServers, | ||
20 | waitUntilLiveSavedOnAllServers | ||
21 | } from '@shared/extra-utils' | ||
22 | import { HttpStatusCode, LiveVideoCreate, VideoFile, VideoPrivacy } from '@shared/models' | ||
23 | |||
24 | const expect = chai.expect | ||
25 | |||
26 | async function createLive (server: PeerTubeServer) { | ||
27 | const attributes: LiveVideoCreate = { | ||
28 | channelId: server.store.channel.id, | ||
29 | privacy: VideoPrivacy.PUBLIC, | ||
30 | name: 'my super live', | ||
31 | saveReplay: true | ||
32 | } | ||
33 | |||
34 | const { uuid } = await server.live.create({ fields: attributes }) | ||
35 | |||
36 | return uuid | ||
37 | } | ||
38 | |||
39 | async function checkFiles (files: VideoFile[]) { | ||
40 | for (const file of files) { | ||
41 | expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) | ||
42 | |||
43 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
44 | } | ||
45 | } | ||
46 | |||
47 | describe('Object storage for lives', function () { | ||
48 | if (areObjectStorageTestsDisabled()) return | ||
49 | |||
50 | let ffmpegCommand: FfmpegCommand | ||
51 | let servers: PeerTubeServer[] | ||
52 | let videoUUID: string | ||
53 | |||
54 | before(async function () { | ||
55 | this.timeout(120000) | ||
56 | |||
57 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
58 | |||
59 | servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultConfig()) | ||
60 | |||
61 | await setAccessTokensToServers(servers) | ||
62 | await setDefaultVideoChannel(servers) | ||
63 | await doubleFollow(servers[0], servers[1]) | ||
64 | |||
65 | await servers[0].config.enableTranscoding() | ||
66 | }) | ||
67 | |||
68 | describe('Without live transcoding', async function () { | ||
69 | |||
70 | before(async function () { | ||
71 | await servers[0].config.enableLive({ transcoding: false }) | ||
72 | |||
73 | videoUUID = await createLive(servers[0]) | ||
74 | }) | ||
75 | |||
76 | it('Should create a live and save the replay on object storage', async function () { | ||
77 | this.timeout(220000) | ||
78 | |||
79 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) | ||
80 | await waitUntilLivePublishedOnAllServers(servers, videoUUID) | ||
81 | |||
82 | await stopFfmpeg(ffmpegCommand) | ||
83 | |||
84 | await waitUntilLiveSavedOnAllServers(servers, videoUUID) | ||
85 | await waitJobs(servers) | ||
86 | |||
87 | for (const server of servers) { | ||
88 | const video = await server.videos.get({ id: videoUUID }) | ||
89 | |||
90 | expect(video.files).to.have.lengthOf(0) | ||
91 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
92 | |||
93 | const files = video.streamingPlaylists[0].files | ||
94 | |||
95 | await checkFiles(files) | ||
96 | } | ||
97 | }) | ||
98 | }) | ||
99 | |||
100 | describe('With live transcoding', async function () { | ||
101 | |||
102 | before(async function () { | ||
103 | await servers[0].config.enableLive({ transcoding: true }) | ||
104 | |||
105 | videoUUID = await createLive(servers[0]) | ||
106 | }) | ||
107 | |||
108 | it('Should import a video and have sent it to object storage', async function () { | ||
109 | this.timeout(240000) | ||
110 | |||
111 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) | ||
112 | await waitUntilLivePublishedOnAllServers(servers, videoUUID) | ||
113 | |||
114 | await stopFfmpeg(ffmpegCommand) | ||
115 | |||
116 | await waitUntilLiveSavedOnAllServers(servers, videoUUID) | ||
117 | await waitJobs(servers) | ||
118 | |||
119 | for (const server of servers) { | ||
120 | const video = await server.videos.get({ id: videoUUID }) | ||
121 | |||
122 | expect(video.files).to.have.lengthOf(0) | ||
123 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
124 | |||
125 | const files = video.streamingPlaylists[0].files | ||
126 | expect(files).to.have.lengthOf(4) | ||
127 | |||
128 | await checkFiles(files) | ||
129 | } | ||
130 | }) | ||
131 | }) | ||
132 | |||
133 | after(async function () { | ||
134 | await killallServers(servers) | ||
135 | }) | ||
136 | }) | ||
diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts new file mode 100644 index 000000000..efc01f550 --- /dev/null +++ b/server/tests/api/object-storage/video-imports.ts | |||
@@ -0,0 +1,112 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { | ||
6 | areObjectStorageTestsDisabled, | ||
7 | createSingleServer, | ||
8 | expectStartWith, | ||
9 | FIXTURE_URLS, | ||
10 | killallServers, | ||
11 | makeRawRequest, | ||
12 | ObjectStorageCommand, | ||
13 | PeerTubeServer, | ||
14 | setAccessTokensToServers, | ||
15 | setDefaultVideoChannel, | ||
16 | waitJobs | ||
17 | } from '@shared/extra-utils' | ||
18 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' | ||
19 | |||
20 | const expect = chai.expect | ||
21 | |||
22 | async function importVideo (server: PeerTubeServer) { | ||
23 | const attributes = { | ||
24 | name: 'import 2', | ||
25 | privacy: VideoPrivacy.PUBLIC, | ||
26 | channelId: server.store.channel.id, | ||
27 | targetUrl: FIXTURE_URLS.goodVideo720 | ||
28 | } | ||
29 | |||
30 | const { video: { uuid } } = await server.imports.importVideo({ attributes }) | ||
31 | |||
32 | return uuid | ||
33 | } | ||
34 | |||
35 | describe('Object storage for video import', function () { | ||
36 | if (areObjectStorageTestsDisabled()) return | ||
37 | |||
38 | let server: PeerTubeServer | ||
39 | |||
40 | before(async function () { | ||
41 | this.timeout(120000) | ||
42 | |||
43 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
44 | |||
45 | server = await createSingleServer(1, ObjectStorageCommand.getDefaultConfig()) | ||
46 | |||
47 | await setAccessTokensToServers([ server ]) | ||
48 | await setDefaultVideoChannel([ server ]) | ||
49 | |||
50 | await server.config.enableImports() | ||
51 | }) | ||
52 | |||
53 | describe('Without transcoding', async function () { | ||
54 | |||
55 | before(async function () { | ||
56 | await server.config.disableTranscoding() | ||
57 | }) | ||
58 | |||
59 | it('Should import a video and have sent it to object storage', async function () { | ||
60 | this.timeout(120000) | ||
61 | |||
62 | const uuid = await importVideo(server) | ||
63 | await waitJobs(server) | ||
64 | |||
65 | const video = await server.videos.get({ id: uuid }) | ||
66 | |||
67 | expect(video.files).to.have.lengthOf(1) | ||
68 | expect(video.streamingPlaylists).to.have.lengthOf(0) | ||
69 | |||
70 | const fileUrl = video.files[0].fileUrl | ||
71 | expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | ||
72 | |||
73 | await makeRawRequest(fileUrl, HttpStatusCode.OK_200) | ||
74 | }) | ||
75 | }) | ||
76 | |||
77 | describe('With transcoding', async function () { | ||
78 | |||
79 | before(async function () { | ||
80 | await server.config.enableTranscoding() | ||
81 | }) | ||
82 | |||
83 | it('Should import a video and have sent it to object storage', async function () { | ||
84 | this.timeout(120000) | ||
85 | |||
86 | const uuid = await importVideo(server) | ||
87 | await waitJobs(server) | ||
88 | |||
89 | const video = await server.videos.get({ id: uuid }) | ||
90 | |||
91 | expect(video.files).to.have.lengthOf(4) | ||
92 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
93 | expect(video.streamingPlaylists[0].files).to.have.lengthOf(4) | ||
94 | |||
95 | for (const file of video.files) { | ||
96 | expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | ||
97 | |||
98 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
99 | } | ||
100 | |||
101 | for (const file of video.streamingPlaylists[0].files) { | ||
102 | expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) | ||
103 | |||
104 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
105 | } | ||
106 | }) | ||
107 | }) | ||
108 | |||
109 | after(async function () { | ||
110 | await killallServers([ server ]) | ||
111 | }) | ||
112 | }) | ||
diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts new file mode 100644 index 000000000..3958bd3d7 --- /dev/null +++ b/server/tests/api/object-storage/videos.ts | |||
@@ -0,0 +1,391 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { merge } from 'lodash' | ||
6 | import { | ||
7 | areObjectStorageTestsDisabled, | ||
8 | checkTmpIsEmpty, | ||
9 | cleanupTests, | ||
10 | createMultipleServers, | ||
11 | createSingleServer, | ||
12 | doubleFollow, | ||
13 | expectStartWith, | ||
14 | killallServers, | ||
15 | makeRawRequest, | ||
16 | MockObjectStorage, | ||
17 | ObjectStorageCommand, | ||
18 | PeerTubeServer, | ||
19 | setAccessTokensToServers, | ||
20 | waitJobs, | ||
21 | webtorrentAdd | ||
22 | } from '@shared/extra-utils' | ||
23 | import { HttpStatusCode, VideoDetails } from '@shared/models' | ||
24 | |||
25 | const expect = chai.expect | ||
26 | |||
27 | async function checkFiles (options: { | ||
28 | video: VideoDetails | ||
29 | |||
30 | baseMockUrl?: string | ||
31 | |||
32 | playlistBucket: string | ||
33 | playlistPrefix?: string | ||
34 | |||
35 | webtorrentBucket: string | ||
36 | webtorrentPrefix?: string | ||
37 | }) { | ||
38 | const { | ||
39 | video, | ||
40 | playlistBucket, | ||
41 | webtorrentBucket, | ||
42 | baseMockUrl, | ||
43 | playlistPrefix, | ||
44 | webtorrentPrefix | ||
45 | } = options | ||
46 | |||
47 | let allFiles = video.files | ||
48 | |||
49 | for (const file of video.files) { | ||
50 | const baseUrl = baseMockUrl | ||
51 | ? `${baseMockUrl}/${webtorrentBucket}/` | ||
52 | : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/` | ||
53 | |||
54 | const prefix = webtorrentPrefix || '' | ||
55 | const start = baseUrl + prefix | ||
56 | |||
57 | expectStartWith(file.fileUrl, start) | ||
58 | |||
59 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | ||
60 | const location = res.headers['location'] | ||
61 | expectStartWith(location, start) | ||
62 | |||
63 | await makeRawRequest(location, HttpStatusCode.OK_200) | ||
64 | } | ||
65 | |||
66 | const hls = video.streamingPlaylists[0] | ||
67 | |||
68 | if (hls) { | ||
69 | allFiles = allFiles.concat(hls.files) | ||
70 | |||
71 | const baseUrl = baseMockUrl | ||
72 | ? `${baseMockUrl}/${playlistBucket}/` | ||
73 | : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/` | ||
74 | |||
75 | const prefix = playlistPrefix || '' | ||
76 | const start = baseUrl + prefix | ||
77 | |||
78 | expectStartWith(hls.playlistUrl, start) | ||
79 | expectStartWith(hls.segmentsSha256Url, start) | ||
80 | |||
81 | await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200) | ||
82 | |||
83 | const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200) | ||
84 | expect(JSON.stringify(resSha.body)).to.not.throw | ||
85 | |||
86 | for (const file of hls.files) { | ||
87 | expectStartWith(file.fileUrl, start) | ||
88 | |||
89 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | ||
90 | const location = res.headers['location'] | ||
91 | expectStartWith(location, start) | ||
92 | |||
93 | await makeRawRequest(location, HttpStatusCode.OK_200) | ||
94 | } | ||
95 | } | ||
96 | |||
97 | for (const file of allFiles) { | ||
98 | const torrent = await webtorrentAdd(file.magnetUri, true) | ||
99 | |||
100 | expect(torrent.files).to.be.an('array') | ||
101 | expect(torrent.files.length).to.equal(1) | ||
102 | expect(torrent.files[0].path).to.exist.and.to.not.equal('') | ||
103 | |||
104 | const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
105 | expect(res.body).to.have.length.above(100) | ||
106 | } | ||
107 | |||
108 | return allFiles.map(f => f.fileUrl) | ||
109 | } | ||
110 | |||
111 | function runTestSuite (options: { | ||
112 | playlistBucket: string | ||
113 | playlistPrefix?: string | ||
114 | |||
115 | webtorrentBucket: string | ||
116 | webtorrentPrefix?: string | ||
117 | |||
118 | useMockBaseUrl?: boolean | ||
119 | |||
120 | maxUploadPart?: string | ||
121 | }) { | ||
122 | const mockObjectStorage = new MockObjectStorage() | ||
123 | let baseMockUrl: string | ||
124 | |||
125 | let servers: PeerTubeServer[] | ||
126 | |||
127 | let keptUrls: string[] = [] | ||
128 | |||
129 | const uuidsToDelete: string[] = [] | ||
130 | let deletedUrls: string[] = [] | ||
131 | |||
132 | before(async function () { | ||
133 | this.timeout(120000) | ||
134 | |||
135 | const port = await mockObjectStorage.initialize() | ||
136 | baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined | ||
137 | |||
138 | await ObjectStorageCommand.createBucket(options.playlistBucket) | ||
139 | await ObjectStorageCommand.createBucket(options.webtorrentBucket) | ||
140 | |||
141 | const config = { | ||
142 | object_storage: { | ||
143 | enabled: true, | ||
144 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | ||
145 | region: ObjectStorageCommand.getRegion(), | ||
146 | |||
147 | credentials: ObjectStorageCommand.getCredentialsConfig(), | ||
148 | |||
149 | max_upload_part: options.maxUploadPart || '2MB', | ||
150 | |||
151 | streaming_playlists: { | ||
152 | bucket_name: options.playlistBucket, | ||
153 | prefix: options.playlistPrefix, | ||
154 | base_url: baseMockUrl | ||
155 | ? `${baseMockUrl}/${options.playlistBucket}` | ||
156 | : undefined | ||
157 | }, | ||
158 | |||
159 | videos: { | ||
160 | bucket_name: options.webtorrentBucket, | ||
161 | prefix: options.webtorrentPrefix, | ||
162 | base_url: baseMockUrl | ||
163 | ? `${baseMockUrl}/${options.webtorrentBucket}` | ||
164 | : undefined | ||
165 | } | ||
166 | } | ||
167 | } | ||
168 | |||
169 | servers = await createMultipleServers(2, config) | ||
170 | |||
171 | await setAccessTokensToServers(servers) | ||
172 | await doubleFollow(servers[0], servers[1]) | ||
173 | |||
174 | for (const server of servers) { | ||
175 | const { uuid } = await server.videos.quickUpload({ name: 'video to keep' }) | ||
176 | await waitJobs(servers) | ||
177 | |||
178 | const files = await server.videos.listFiles({ id: uuid }) | ||
179 | keptUrls = keptUrls.concat(files.map(f => f.fileUrl)) | ||
180 | } | ||
181 | }) | ||
182 | |||
183 | it('Should upload a video and move it to the object storage without transcoding', async function () { | ||
184 | this.timeout(20000) | ||
185 | |||
186 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) | ||
187 | uuidsToDelete.push(uuid) | ||
188 | |||
189 | await waitJobs(servers) | ||
190 | |||
191 | for (const server of servers) { | ||
192 | const video = await server.videos.get({ id: uuid }) | ||
193 | const files = await checkFiles({ ...options, video, baseMockUrl }) | ||
194 | |||
195 | deletedUrls = deletedUrls.concat(files) | ||
196 | } | ||
197 | }) | ||
198 | |||
199 | it('Should upload a video and move it to the object storage with transcoding', async function () { | ||
200 | this.timeout(40000) | ||
201 | |||
202 | const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' }) | ||
203 | uuidsToDelete.push(uuid) | ||
204 | |||
205 | await waitJobs(servers) | ||
206 | |||
207 | for (const server of servers) { | ||
208 | const video = await server.videos.get({ id: uuid }) | ||
209 | const files = await checkFiles({ ...options, video, baseMockUrl }) | ||
210 | |||
211 | deletedUrls = deletedUrls.concat(files) | ||
212 | } | ||
213 | }) | ||
214 | |||
215 | it('Should correctly delete the files', async function () { | ||
216 | await servers[0].videos.remove({ id: uuidsToDelete[0] }) | ||
217 | await servers[1].videos.remove({ id: uuidsToDelete[1] }) | ||
218 | |||
219 | await waitJobs(servers) | ||
220 | |||
221 | for (const url of deletedUrls) { | ||
222 | await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404) | ||
223 | } | ||
224 | }) | ||
225 | |||
226 | it('Should have kept other files', async function () { | ||
227 | for (const url of keptUrls) { | ||
228 | await makeRawRequest(url, HttpStatusCode.OK_200) | ||
229 | } | ||
230 | }) | ||
231 | |||
232 | it('Should have an empty tmp directory', async function () { | ||
233 | for (const server of servers) { | ||
234 | await checkTmpIsEmpty(server) | ||
235 | } | ||
236 | }) | ||
237 | |||
238 | after(async function () { | ||
239 | mockObjectStorage.terminate() | ||
240 | |||
241 | await cleanupTests(servers) | ||
242 | }) | ||
243 | } | ||
244 | |||
245 | describe('Object storage for videos', function () { | ||
246 | if (areObjectStorageTestsDisabled()) return | ||
247 | |||
248 | describe('Test config', function () { | ||
249 | let server: PeerTubeServer | ||
250 | |||
251 | const baseConfig = { | ||
252 | object_storage: { | ||
253 | enabled: true, | ||
254 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | ||
255 | region: ObjectStorageCommand.getRegion(), | ||
256 | |||
257 | credentials: ObjectStorageCommand.getCredentialsConfig(), | ||
258 | |||
259 | streaming_playlists: { | ||
260 | bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET | ||
261 | }, | ||
262 | |||
263 | videos: { | ||
264 | bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET | ||
265 | } | ||
266 | } | ||
267 | } | ||
268 | |||
269 | const badCredentials = { | ||
270 | access_key_id: 'AKIAIOSFODNN7EXAMPLE', | ||
271 | secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' | ||
272 | } | ||
273 | |||
274 | it('Should fail with same bucket names without prefix', function (done) { | ||
275 | const config = merge({}, baseConfig, { | ||
276 | object_storage: { | ||
277 | streaming_playlists: { | ||
278 | bucket_name: 'aaa' | ||
279 | }, | ||
280 | |||
281 | videos: { | ||
282 | bucket_name: 'aaa' | ||
283 | } | ||
284 | } | ||
285 | }) | ||
286 | |||
287 | createSingleServer(1, config) | ||
288 | .then(() => done(new Error('Did not throw'))) | ||
289 | .catch(() => done()) | ||
290 | }) | ||
291 | |||
292 | it('Should fail with bad credentials', async function () { | ||
293 | this.timeout(60000) | ||
294 | |||
295 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
296 | |||
297 | const config = merge({}, baseConfig, { | ||
298 | object_storage: { | ||
299 | credentials: badCredentials | ||
300 | } | ||
301 | }) | ||
302 | |||
303 | server = await createSingleServer(1, config) | ||
304 | await setAccessTokensToServers([ server ]) | ||
305 | |||
306 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | ||
307 | |||
308 | await waitJobs([ server ], true) | ||
309 | const video = await server.videos.get({ id: uuid }) | ||
310 | |||
311 | expectStartWith(video.files[0].fileUrl, server.url) | ||
312 | |||
313 | await killallServers([ server ]) | ||
314 | }) | ||
315 | |||
316 | it('Should succeed with credentials from env', async function () { | ||
317 | this.timeout(60000) | ||
318 | |||
319 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
320 | |||
321 | const config = merge({}, baseConfig, { | ||
322 | object_storage: { | ||
323 | credentials: { | ||
324 | access_key_id: '', | ||
325 | secret_access_key: '' | ||
326 | } | ||
327 | } | ||
328 | }) | ||
329 | |||
330 | const goodCredentials = ObjectStorageCommand.getCredentialsConfig() | ||
331 | |||
332 | server = await createSingleServer(1, config, { | ||
333 | env: { | ||
334 | AWS_ACCESS_KEY_ID: goodCredentials.access_key_id, | ||
335 | AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key | ||
336 | } | ||
337 | }) | ||
338 | |||
339 | await setAccessTokensToServers([ server ]) | ||
340 | |||
341 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | ||
342 | |||
343 | await waitJobs([ server ], true) | ||
344 | const video = await server.videos.get({ id: uuid }) | ||
345 | |||
346 | expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | ||
347 | }) | ||
348 | |||
349 | after(async function () { | ||
350 | await killallServers([ server ]) | ||
351 | }) | ||
352 | }) | ||
353 | |||
354 | describe('Test simple object storage', function () { | ||
355 | runTestSuite({ | ||
356 | playlistBucket: 'streaming-playlists', | ||
357 | webtorrentBucket: 'videos' | ||
358 | }) | ||
359 | }) | ||
360 | |||
361 | describe('Test object storage with prefix', function () { | ||
362 | runTestSuite({ | ||
363 | playlistBucket: 'mybucket', | ||
364 | webtorrentBucket: 'mybucket', | ||
365 | |||
366 | playlistPrefix: 'streaming-playlists_', | ||
367 | webtorrentPrefix: 'webtorrent_' | ||
368 | }) | ||
369 | }) | ||
370 | |||
371 | describe('Test object storage with prefix and base URL', function () { | ||
372 | runTestSuite({ | ||
373 | playlistBucket: 'mybucket', | ||
374 | webtorrentBucket: 'mybucket', | ||
375 | |||
376 | playlistPrefix: 'streaming-playlists_', | ||
377 | webtorrentPrefix: 'webtorrent_', | ||
378 | |||
379 | useMockBaseUrl: true | ||
380 | }) | ||
381 | }) | ||
382 | |||
383 | describe('Test object storage with small upload part', function () { | ||
384 | runTestSuite({ | ||
385 | playlistBucket: 'streaming-playlists', | ||
386 | webtorrentBucket: 'videos', | ||
387 | |||
388 | maxUploadPart: '5KB' | ||
389 | }) | ||
390 | }) | ||
391 | }) | ||
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index e1a12f5f8..3400b1d9a 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts | |||
@@ -207,14 +207,14 @@ async function check1PlaylistRedundancies (videoUUID?: string) { | |||
207 | expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) | 207 | expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) |
208 | } | 208 | } |
209 | 209 | ||
210 | const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls' | 210 | const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls/' + videoUUID |
211 | const baseUrlSegment = servers[0].url + '/static/redundancy/hls' | 211 | const baseUrlSegment = servers[0].url + '/static/redundancy/hls/' + videoUUID |
212 | 212 | ||
213 | const video = await servers[0].videos.get({ id: videoUUID }) | 213 | const video = await servers[0].videos.get({ id: videoUUID }) |
214 | const hlsPlaylist = video.streamingPlaylists[0] | 214 | const hlsPlaylist = video.streamingPlaylists[0] |
215 | 215 | ||
216 | for (const resolution of [ 240, 360, 480, 720 ]) { | 216 | for (const resolution of [ 240, 360, 480, 720 ]) { |
217 | await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist }) | 217 | await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist }) |
218 | } | 218 | } |
219 | 219 | ||
220 | const { hlsFilenames } = await ensureSameFilenames(videoUUID) | 220 | const { hlsFilenames } = await ensureSameFilenames(videoUUID) |
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts index 961f0e617..2c829f532 100644 --- a/server/tests/api/videos/video-hls.ts +++ b/server/tests/api/videos/video-hls.ts | |||
@@ -5,6 +5,7 @@ import * as chai from 'chai' | |||
5 | import { basename, join } from 'path' | 5 | import { basename, join } from 'path' |
6 | import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' | 6 | import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' |
7 | import { | 7 | import { |
8 | areObjectStorageTestsDisabled, | ||
8 | checkDirectoryIsEmpty, | 9 | checkDirectoryIsEmpty, |
9 | checkResolutionsInMasterPlaylist, | 10 | checkResolutionsInMasterPlaylist, |
10 | checkSegmentHash, | 11 | checkSegmentHash, |
@@ -12,7 +13,9 @@ import { | |||
12 | cleanupTests, | 13 | cleanupTests, |
13 | createMultipleServers, | 14 | createMultipleServers, |
14 | doubleFollow, | 15 | doubleFollow, |
16 | expectStartWith, | ||
15 | makeRawRequest, | 17 | makeRawRequest, |
18 | ObjectStorageCommand, | ||
16 | PeerTubeServer, | 19 | PeerTubeServer, |
17 | setAccessTokensToServers, | 20 | setAccessTokensToServers, |
18 | waitJobs, | 21 | waitJobs, |
@@ -23,8 +26,19 @@ import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants' | |||
23 | 26 | ||
24 | const expect = chai.expect | 27 | const expect = chai.expect |
25 | 28 | ||
26 | async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, hlsOnly: boolean, resolutions = [ 240, 360, 480, 720 ]) { | 29 | async function checkHlsPlaylist (options: { |
27 | for (const server of servers) { | 30 | servers: PeerTubeServer[] |
31 | videoUUID: string | ||
32 | hlsOnly: boolean | ||
33 | |||
34 | resolutions?: number[] | ||
35 | objectStorageBaseUrl: string | ||
36 | }) { | ||
37 | const { videoUUID, hlsOnly, objectStorageBaseUrl } = options | ||
38 | |||
39 | const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ] | ||
40 | |||
41 | for (const server of options.servers) { | ||
28 | const videoDetails = await server.videos.get({ id: videoUUID }) | 42 | const videoDetails = await server.videos.get({ id: videoUUID }) |
29 | const baseUrl = `http://${videoDetails.account.host}` | 43 | const baseUrl = `http://${videoDetails.account.host}` |
30 | 44 | ||
@@ -48,9 +62,15 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h | |||
48 | expect(file.torrentUrl).to.match( | 62 | expect(file.torrentUrl).to.match( |
49 | new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`) | 63 | new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`) |
50 | ) | 64 | ) |
51 | expect(file.fileUrl).to.match( | 65 | |
52 | new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`) | 66 | if (objectStorageBaseUrl) { |
53 | ) | 67 | expectStartWith(file.fileUrl, objectStorageBaseUrl) |
68 | } else { | ||
69 | expect(file.fileUrl).to.match( | ||
70 | new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`) | ||
71 | ) | ||
72 | } | ||
73 | |||
54 | expect(file.resolution.label).to.equal(resolution + 'p') | 74 | expect(file.resolution.label).to.equal(resolution + 'p') |
55 | 75 | ||
56 | await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200) | 76 | await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200) |
@@ -80,9 +100,11 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h | |||
80 | const file = hlsFiles.find(f => f.resolution.id === resolution) | 100 | const file = hlsFiles.find(f => f.resolution.id === resolution) |
81 | const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' | 101 | const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' |
82 | 102 | ||
83 | const subPlaylist = await server.streamingPlaylists.get({ | 103 | const url = objectStorageBaseUrl |
84 | url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}` | 104 | ? `${objectStorageBaseUrl}hls_${videoUUID}/${playlistName}` |
85 | }) | 105 | : `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}` |
106 | |||
107 | const subPlaylist = await server.streamingPlaylists.get({ url }) | ||
86 | 108 | ||
87 | expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) | 109 | expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) |
88 | expect(subPlaylist).to.contain(basename(file.fileUrl)) | 110 | expect(subPlaylist).to.contain(basename(file.fileUrl)) |
@@ -90,14 +112,15 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h | |||
90 | } | 112 | } |
91 | 113 | ||
92 | { | 114 | { |
93 | const baseUrlAndPath = baseUrl + '/static/streaming-playlists/hls' | 115 | const baseUrlAndPath = objectStorageBaseUrl |
116 | ? objectStorageBaseUrl + 'hls_' + videoUUID | ||
117 | : baseUrl + '/static/streaming-playlists/hls/' + videoUUID | ||
94 | 118 | ||
95 | for (const resolution of resolutions) { | 119 | for (const resolution of resolutions) { |
96 | await checkSegmentHash({ | 120 | await checkSegmentHash({ |
97 | server, | 121 | server, |
98 | baseUrlPlaylist: baseUrlAndPath, | 122 | baseUrlPlaylist: baseUrlAndPath, |
99 | baseUrlSegment: baseUrlAndPath, | 123 | baseUrlSegment: baseUrlAndPath, |
100 | videoUUID, | ||
101 | resolution, | 124 | resolution, |
102 | hlsPlaylist | 125 | hlsPlaylist |
103 | }) | 126 | }) |
@@ -111,7 +134,7 @@ describe('Test HLS videos', function () { | |||
111 | let videoUUID = '' | 134 | let videoUUID = '' |
112 | let videoAudioUUID = '' | 135 | let videoAudioUUID = '' |
113 | 136 | ||
114 | function runTestSuite (hlsOnly: boolean) { | 137 | function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) { |
115 | 138 | ||
116 | it('Should upload a video and transcode it to HLS', async function () { | 139 | it('Should upload a video and transcode it to HLS', async function () { |
117 | this.timeout(120000) | 140 | this.timeout(120000) |
@@ -121,7 +144,7 @@ describe('Test HLS videos', function () { | |||
121 | 144 | ||
122 | await waitJobs(servers) | 145 | await waitJobs(servers) |
123 | 146 | ||
124 | await checkHlsPlaylist(servers, videoUUID, hlsOnly) | 147 | await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl }) |
125 | }) | 148 | }) |
126 | 149 | ||
127 | it('Should upload an audio file and transcode it to HLS', async function () { | 150 | it('Should upload an audio file and transcode it to HLS', async function () { |
@@ -132,7 +155,13 @@ describe('Test HLS videos', function () { | |||
132 | 155 | ||
133 | await waitJobs(servers) | 156 | await waitJobs(servers) |
134 | 157 | ||
135 | await checkHlsPlaylist(servers, videoAudioUUID, hlsOnly, [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ]) | 158 | await checkHlsPlaylist({ |
159 | servers, | ||
160 | videoUUID: videoAudioUUID, | ||
161 | hlsOnly, | ||
162 | resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ], | ||
163 | objectStorageBaseUrl | ||
164 | }) | ||
136 | }) | 165 | }) |
137 | 166 | ||
138 | it('Should update the video', async function () { | 167 | it('Should update the video', async function () { |
@@ -142,7 +171,7 @@ describe('Test HLS videos', function () { | |||
142 | 171 | ||
143 | await waitJobs(servers) | 172 | await waitJobs(servers) |
144 | 173 | ||
145 | await checkHlsPlaylist(servers, videoUUID, hlsOnly) | 174 | await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl }) |
146 | }) | 175 | }) |
147 | 176 | ||
148 | it('Should delete videos', async function () { | 177 | it('Should delete videos', async function () { |
@@ -229,6 +258,22 @@ describe('Test HLS videos', function () { | |||
229 | runTestSuite(true) | 258 | runTestSuite(true) |
230 | }) | 259 | }) |
231 | 260 | ||
261 | describe('With object storage enabled', function () { | ||
262 | if (areObjectStorageTestsDisabled()) return | ||
263 | |||
264 | before(async function () { | ||
265 | this.timeout(120000) | ||
266 | |||
267 | const configOverride = ObjectStorageCommand.getDefaultConfig() | ||
268 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
269 | |||
270 | await servers[0].kill() | ||
271 | await servers[0].run(configOverride) | ||
272 | }) | ||
273 | |||
274 | runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl()) | ||
275 | }) | ||
276 | |||
232 | after(async function () { | 277 | after(async function () { |
233 | await cleanupTests(servers) | 278 | await cleanupTests(servers) |
234 | }) | 279 | }) |
diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts index bddcff5e7..9f1b57a2e 100644 --- a/server/tests/cli/create-import-video-file-job.ts +++ b/server/tests/cli/create-import-video-file-job.ts | |||
@@ -2,8 +2,19 @@ | |||
2 | 2 | ||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils' | 5 | import { |
6 | import { VideoFile } from '@shared/models' | 6 | areObjectStorageTestsDisabled, |
7 | cleanupTests, | ||
8 | createMultipleServers, | ||
9 | doubleFollow, | ||
10 | expectStartWith, | ||
11 | makeRawRequest, | ||
12 | ObjectStorageCommand, | ||
13 | PeerTubeServer, | ||
14 | setAccessTokensToServers, | ||
15 | waitJobs | ||
16 | } from '@shared/extra-utils' | ||
17 | import { HttpStatusCode, VideoDetails, VideoFile } from '@shared/models' | ||
7 | 18 | ||
8 | const expect = chai.expect | 19 | const expect = chai.expect |
9 | 20 | ||
@@ -17,22 +28,35 @@ function assertVideoProperties (video: VideoFile, resolution: number, extname: s | |||
17 | if (size) expect(video.size).to.equal(size) | 28 | if (size) expect(video.size).to.equal(size) |
18 | } | 29 | } |
19 | 30 | ||
20 | describe('Test create import video jobs', function () { | 31 | async function checkFiles (video: VideoDetails, objectStorage: boolean) { |
21 | this.timeout(60000) | 32 | for (const file of video.files) { |
33 | if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | ||
22 | 34 | ||
23 | let servers: PeerTubeServer[] = [] | 35 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) |
36 | } | ||
37 | } | ||
38 | |||
39 | function runTests (objectStorage: boolean) { | ||
24 | let video1UUID: string | 40 | let video1UUID: string |
25 | let video2UUID: string | 41 | let video2UUID: string |
26 | 42 | ||
43 | let servers: PeerTubeServer[] = [] | ||
44 | |||
27 | before(async function () { | 45 | before(async function () { |
28 | this.timeout(90000) | 46 | this.timeout(90000) |
29 | 47 | ||
48 | const config = objectStorage | ||
49 | ? ObjectStorageCommand.getDefaultConfig() | ||
50 | : {} | ||
51 | |||
30 | // Run server 2 to have transcoding enabled | 52 | // Run server 2 to have transcoding enabled |
31 | servers = await createMultipleServers(2) | 53 | servers = await createMultipleServers(2, config) |
32 | await setAccessTokensToServers(servers) | 54 | await setAccessTokensToServers(servers) |
33 | 55 | ||
34 | await doubleFollow(servers[0], servers[1]) | 56 | await doubleFollow(servers[0], servers[1]) |
35 | 57 | ||
58 | if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() | ||
59 | |||
36 | // Upload two videos for our needs | 60 | // Upload two videos for our needs |
37 | { | 61 | { |
38 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video1' } }) | 62 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video1' } }) |
@@ -44,7 +68,6 @@ describe('Test create import video jobs', function () { | |||
44 | video2UUID = uuid | 68 | video2UUID = uuid |
45 | } | 69 | } |
46 | 70 | ||
47 | // Transcoding | ||
48 | await waitJobs(servers) | 71 | await waitJobs(servers) |
49 | }) | 72 | }) |
50 | 73 | ||
@@ -65,6 +88,8 @@ describe('Test create import video jobs', function () { | |||
65 | const [ originalVideo, transcodedVideo ] = videoDetails.files | 88 | const [ originalVideo, transcodedVideo ] = videoDetails.files |
66 | assertVideoProperties(originalVideo, 720, 'webm', 218910) | 89 | assertVideoProperties(originalVideo, 720, 'webm', 218910) |
67 | assertVideoProperties(transcodedVideo, 480, 'webm', 69217) | 90 | assertVideoProperties(transcodedVideo, 480, 'webm', 69217) |
91 | |||
92 | await checkFiles(videoDetails, objectStorage) | ||
68 | } | 93 | } |
69 | }) | 94 | }) |
70 | 95 | ||
@@ -87,6 +112,8 @@ describe('Test create import video jobs', function () { | |||
87 | assertVideoProperties(transcodedVideo420, 480, 'mp4') | 112 | assertVideoProperties(transcodedVideo420, 480, 'mp4') |
88 | assertVideoProperties(transcodedVideo320, 360, 'mp4') | 113 | assertVideoProperties(transcodedVideo320, 360, 'mp4') |
89 | assertVideoProperties(transcodedVideo240, 240, 'mp4') | 114 | assertVideoProperties(transcodedVideo240, 240, 'mp4') |
115 | |||
116 | await checkFiles(videoDetails, objectStorage) | ||
90 | } | 117 | } |
91 | }) | 118 | }) |
92 | 119 | ||
@@ -107,10 +134,25 @@ describe('Test create import video jobs', function () { | |||
107 | const [ video720, video480 ] = videoDetails.files | 134 | const [ video720, video480 ] = videoDetails.files |
108 | assertVideoProperties(video720, 720, 'webm', 942961) | 135 | assertVideoProperties(video720, 720, 'webm', 942961) |
109 | assertVideoProperties(video480, 480, 'webm', 69217) | 136 | assertVideoProperties(video480, 480, 'webm', 69217) |
137 | |||
138 | await checkFiles(videoDetails, objectStorage) | ||
110 | } | 139 | } |
111 | }) | 140 | }) |
112 | 141 | ||
113 | after(async function () { | 142 | after(async function () { |
114 | await cleanupTests(servers) | 143 | await cleanupTests(servers) |
115 | }) | 144 | }) |
145 | } | ||
146 | |||
147 | describe('Test create import video jobs', function () { | ||
148 | |||
149 | describe('On filesystem', function () { | ||
150 | runTests(false) | ||
151 | }) | ||
152 | |||
153 | describe('On object storage', function () { | ||
154 | if (areObjectStorageTestsDisabled()) return | ||
155 | |||
156 | runTests(true) | ||
157 | }) | ||
116 | }) | 158 | }) |
diff --git a/server/tests/cli/create-transcoding-job.ts b/server/tests/cli/create-transcoding-job.ts index df787ccdc..3313a492f 100644 --- a/server/tests/cli/create-transcoding-job.ts +++ b/server/tests/cli/create-transcoding-job.ts | |||
@@ -2,10 +2,15 @@ | |||
2 | 2 | ||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import { HttpStatusCode, VideoFile } from '@shared/models' | ||
5 | import { | 6 | import { |
7 | areObjectStorageTestsDisabled, | ||
6 | cleanupTests, | 8 | cleanupTests, |
7 | createMultipleServers, | 9 | createMultipleServers, |
8 | doubleFollow, | 10 | doubleFollow, |
11 | expectStartWith, | ||
12 | makeRawRequest, | ||
13 | ObjectStorageCommand, | ||
9 | PeerTubeServer, | 14 | PeerTubeServer, |
10 | setAccessTokensToServers, | 15 | setAccessTokensToServers, |
11 | waitJobs | 16 | waitJobs |
@@ -13,39 +18,39 @@ import { | |||
13 | 18 | ||
14 | const expect = chai.expect | 19 | const expect = chai.expect |
15 | 20 | ||
16 | describe('Test create transcoding jobs', function () { | 21 | async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent' | 'playlist') { |
17 | let servers: PeerTubeServer[] = [] | 22 | for (const file of files) { |
18 | const videosUUID: string[] = [] | 23 | const shouldStartWith = type === 'webtorrent' |
24 | ? ObjectStorageCommand.getWebTorrentBaseUrl() | ||
25 | : ObjectStorageCommand.getPlaylistBaseUrl() | ||
19 | 26 | ||
20 | const config = { | 27 | expectStartWith(file.fileUrl, shouldStartWith) |
21 | transcoding: { | 28 | |
22 | enabled: false, | 29 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) |
23 | resolutions: { | ||
24 | '240p': true, | ||
25 | '360p': true, | ||
26 | '480p': true, | ||
27 | '720p': true, | ||
28 | '1080p': true, | ||
29 | '1440p': true, | ||
30 | '2160p': true | ||
31 | }, | ||
32 | hls: { | ||
33 | enabled: false | ||
34 | } | ||
35 | } | ||
36 | } | 30 | } |
31 | } | ||
32 | |||
33 | function runTests (objectStorage: boolean) { | ||
34 | let servers: PeerTubeServer[] = [] | ||
35 | const videosUUID: string[] = [] | ||
37 | 36 | ||
38 | before(async function () { | 37 | before(async function () { |
39 | this.timeout(60000) | 38 | this.timeout(60000) |
40 | 39 | ||
40 | const config = objectStorage | ||
41 | ? ObjectStorageCommand.getDefaultConfig() | ||
42 | : {} | ||
43 | |||
41 | // Run server 2 to have transcoding enabled | 44 | // Run server 2 to have transcoding enabled |
42 | servers = await createMultipleServers(2) | 45 | servers = await createMultipleServers(2, config) |
43 | await setAccessTokensToServers(servers) | 46 | await setAccessTokensToServers(servers) |
44 | 47 | ||
45 | await servers[0].config.updateCustomSubConfig({ newConfig: config }) | 48 | await servers[0].config.disableTranscoding() |
46 | 49 | ||
47 | await doubleFollow(servers[0], servers[1]) | 50 | await doubleFollow(servers[0], servers[1]) |
48 | 51 | ||
52 | if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() | ||
53 | |||
49 | for (let i = 1; i <= 5; i++) { | 54 | for (let i = 1; i <= 5; i++) { |
50 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) | 55 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) |
51 | videosUUID.push(uuid) | 56 | videosUUID.push(uuid) |
@@ -81,27 +86,29 @@ describe('Test create transcoding jobs', function () { | |||
81 | let infoHashes: { [id: number]: string } | 86 | let infoHashes: { [id: number]: string } |
82 | 87 | ||
83 | for (const video of data) { | 88 | for (const video of data) { |
84 | const videoDetail = await server.videos.get({ id: video.uuid }) | 89 | const videoDetails = await server.videos.get({ id: video.uuid }) |
85 | 90 | ||
86 | if (video.uuid === videosUUID[1]) { | 91 | if (video.uuid === videosUUID[1]) { |
87 | expect(videoDetail.files).to.have.lengthOf(4) | 92 | expect(videoDetails.files).to.have.lengthOf(4) |
88 | expect(videoDetail.streamingPlaylists).to.have.lengthOf(0) | 93 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) |
94 | |||
95 | if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent') | ||
89 | 96 | ||
90 | if (!infoHashes) { | 97 | if (!infoHashes) { |
91 | infoHashes = {} | 98 | infoHashes = {} |
92 | 99 | ||
93 | for (const file of videoDetail.files) { | 100 | for (const file of videoDetails.files) { |
94 | infoHashes[file.resolution.id.toString()] = file.magnetUri | 101 | infoHashes[file.resolution.id.toString()] = file.magnetUri |
95 | } | 102 | } |
96 | } else { | 103 | } else { |
97 | for (const resolution of Object.keys(infoHashes)) { | 104 | for (const resolution of Object.keys(infoHashes)) { |
98 | const file = videoDetail.files.find(f => f.resolution.id.toString() === resolution) | 105 | const file = videoDetails.files.find(f => f.resolution.id.toString() === resolution) |
99 | expect(file.magnetUri).to.equal(infoHashes[resolution]) | 106 | expect(file.magnetUri).to.equal(infoHashes[resolution]) |
100 | } | 107 | } |
101 | } | 108 | } |
102 | } else { | 109 | } else { |
103 | expect(videoDetail.files).to.have.lengthOf(1) | 110 | expect(videoDetails.files).to.have.lengthOf(1) |
104 | expect(videoDetail.streamingPlaylists).to.have.lengthOf(0) | 111 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) |
105 | } | 112 | } |
106 | } | 113 | } |
107 | } | 114 | } |
@@ -125,6 +132,8 @@ describe('Test create transcoding jobs', function () { | |||
125 | expect(videoDetails.files[1].resolution.id).to.equal(480) | 132 | expect(videoDetails.files[1].resolution.id).to.equal(480) |
126 | 133 | ||
127 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) | 134 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) |
135 | |||
136 | if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent') | ||
128 | } | 137 | } |
129 | }) | 138 | }) |
130 | 139 | ||
@@ -139,11 +148,15 @@ describe('Test create transcoding jobs', function () { | |||
139 | const videoDetails = await server.videos.get({ id: videosUUID[2] }) | 148 | const videoDetails = await server.videos.get({ id: videosUUID[2] }) |
140 | 149 | ||
141 | expect(videoDetails.files).to.have.lengthOf(1) | 150 | expect(videoDetails.files).to.have.lengthOf(1) |
151 | if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent') | ||
152 | |||
142 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) | 153 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) |
143 | 154 | ||
144 | const files = videoDetails.streamingPlaylists[0].files | 155 | const files = videoDetails.streamingPlaylists[0].files |
145 | expect(files).to.have.lengthOf(1) | 156 | expect(files).to.have.lengthOf(1) |
146 | expect(files[0].resolution.id).to.equal(480) | 157 | expect(files[0].resolution.id).to.equal(480) |
158 | |||
159 | if (objectStorage) await checkFilesInObjectStorage(files, 'playlist') | ||
147 | } | 160 | } |
148 | }) | 161 | }) |
149 | 162 | ||
@@ -160,6 +173,8 @@ describe('Test create transcoding jobs', function () { | |||
160 | const files = videoDetails.streamingPlaylists[0].files | 173 | const files = videoDetails.streamingPlaylists[0].files |
161 | expect(files).to.have.lengthOf(1) | 174 | expect(files).to.have.lengthOf(1) |
162 | expect(files[0].resolution.id).to.equal(480) | 175 | expect(files[0].resolution.id).to.equal(480) |
176 | |||
177 | if (objectStorage) await checkFilesInObjectStorage(files, 'playlist') | ||
163 | } | 178 | } |
164 | }) | 179 | }) |
165 | 180 | ||
@@ -178,15 +193,15 @@ describe('Test create transcoding jobs', function () { | |||
178 | 193 | ||
179 | const files = videoDetails.streamingPlaylists[0].files | 194 | const files = videoDetails.streamingPlaylists[0].files |
180 | expect(files).to.have.lengthOf(4) | 195 | expect(files).to.have.lengthOf(4) |
196 | |||
197 | if (objectStorage) await checkFilesInObjectStorage(files, 'playlist') | ||
181 | } | 198 | } |
182 | }) | 199 | }) |
183 | 200 | ||
184 | it('Should optimize the video file and generate HLS videos if enabled in config', async function () { | 201 | it('Should optimize the video file and generate HLS videos if enabled in config', async function () { |
185 | this.timeout(120000) | 202 | this.timeout(120000) |
186 | 203 | ||
187 | config.transcoding.hls.enabled = true | 204 | await servers[0].config.enableTranscoding() |
188 | await servers[0].config.updateCustomSubConfig({ newConfig: config }) | ||
189 | |||
190 | await servers[0].cli.execWithEnv(`npm run create-transcoding-job -- -v ${videosUUID[4]}`) | 205 | await servers[0].cli.execWithEnv(`npm run create-transcoding-job -- -v ${videosUUID[4]}`) |
191 | 206 | ||
192 | await waitJobs(servers) | 207 | await waitJobs(servers) |
@@ -197,10 +212,28 @@ describe('Test create transcoding jobs', function () { | |||
197 | expect(videoDetails.files).to.have.lengthOf(4) | 212 | expect(videoDetails.files).to.have.lengthOf(4) |
198 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) | 213 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) |
199 | expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(4) | 214 | expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(4) |
215 | |||
216 | if (objectStorage) { | ||
217 | await checkFilesInObjectStorage(videoDetails.files, 'webtorrent') | ||
218 | await checkFilesInObjectStorage(videoDetails.streamingPlaylists[0].files, 'playlist') | ||
219 | } | ||
200 | } | 220 | } |
201 | }) | 221 | }) |
202 | 222 | ||
203 | after(async function () { | 223 | after(async function () { |
204 | await cleanupTests(servers) | 224 | await cleanupTests(servers) |
205 | }) | 225 | }) |
226 | } | ||
227 | |||
228 | describe('Test create transcoding jobs', function () { | ||
229 | |||
230 | describe('On filesystem', function () { | ||
231 | runTests(false) | ||
232 | }) | ||
233 | |||
234 | describe('On object storage', function () { | ||
235 | if (areObjectStorageTestsDisabled()) return | ||
236 | |||
237 | runTests(true) | ||
238 | }) | ||
206 | }) | 239 | }) |
diff --git a/server/tests/helpers/request.ts b/server/tests/helpers/request.ts index 7f7873df3..c9a2eb831 100644 --- a/server/tests/helpers/request.ts +++ b/server/tests/helpers/request.ts | |||
@@ -13,7 +13,7 @@ describe('Request helpers', function () { | |||
13 | 13 | ||
14 | it('Should throw an error when the bytes limit is exceeded for request', async function () { | 14 | it('Should throw an error when the bytes limit is exceeded for request', async function () { |
15 | try { | 15 | try { |
16 | await doRequest(FIXTURE_URLS.video4K, { bodyKBLimit: 3 }) | 16 | await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 3 }) |
17 | } catch { | 17 | } catch { |
18 | return | 18 | return |
19 | } | 19 | } |
@@ -23,7 +23,7 @@ describe('Request helpers', function () { | |||
23 | 23 | ||
24 | it('Should throw an error when the bytes limit is exceeded for request and save file', async function () { | 24 | it('Should throw an error when the bytes limit is exceeded for request and save file', async function () { |
25 | try { | 25 | try { |
26 | await doRequestAndSaveToFile(FIXTURE_URLS.video4K, destPath1, { bodyKBLimit: 3 }) | 26 | await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath1, { bodyKBLimit: 3 }) |
27 | } catch { | 27 | } catch { |
28 | 28 | ||
29 | await wait(500) | 29 | await wait(500) |
@@ -35,8 +35,8 @@ describe('Request helpers', function () { | |||
35 | }) | 35 | }) |
36 | 36 | ||
37 | it('Should succeed if the file is below the limit', async function () { | 37 | it('Should succeed if the file is below the limit', async function () { |
38 | await doRequest(FIXTURE_URLS.video4K, { bodyKBLimit: 5 }) | 38 | await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 5 }) |
39 | await doRequestAndSaveToFile(FIXTURE_URLS.video4K, destPath2, { bodyKBLimit: 5 }) | 39 | await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath2, { bodyKBLimit: 5 }) |
40 | 40 | ||
41 | expect(await pathExists(destPath2)).to.be.true | 41 | expect(await pathExists(destPath2)).to.be.true |
42 | }) | 42 | }) |