aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorJelle Besseling <jelle@pingiun.com>2021-08-17 08:26:20 +0200
committerGitHub <noreply@github.com>2021-08-17 08:26:20 +0200
commit0305db28c98fd6cf43a3c50ba92c76215e99d512 (patch)
tree33b753a19728d9f453c1aa4f19b36ac797e5fe80 /server
parentf88ae8f5bc223579313b28582de9101944a4a814 (diff)
downloadPeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.tar.gz
PeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.tar.zst
PeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.zip
Add support for saving video files to object storage (#4290)
* Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/videos/upload.ts34
-rw-r--r--server/controllers/download.ts25
-rw-r--r--server/helpers/webtorrent.ts31
-rw-r--r--server/initializers/checker-after-init.ts23
-rw-r--r--server/initializers/config.ts20
-rw-r--r--server/initializers/constants.ts14
-rw-r--r--server/initializers/database.ts4
-rw-r--r--server/initializers/migrations/0065-video-file-size.ts28
-rw-r--r--server/initializers/migrations/0660-object-storage.ts58
-rw-r--r--server/lib/activitypub/videos/shared/object-to-model-attributes.ts2
-rw-r--r--server/lib/hls.ts82
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts114
-rw-r--r--server/lib/job-queue/handlers/video-file-import.ts27
-rw-r--r--server/lib/job-queue/handlers/video-import.ts16
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts18
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts128
-rw-r--r--server/lib/job-queue/job-queue.ts13
-rw-r--r--server/lib/live/live-manager.ts2
-rw-r--r--server/lib/live/live-utils.ts4
-rw-r--r--server/lib/live/shared/muxing-session.ts4
-rw-r--r--server/lib/object-storage/index.ts3
-rw-r--r--server/lib/object-storage/keys.ts20
-rw-r--r--server/lib/object-storage/shared/client.ts56
-rw-r--r--server/lib/object-storage/shared/index.ts3
-rw-r--r--server/lib/object-storage/shared/logger.ts7
-rw-r--r--server/lib/object-storage/shared/object-storage-helpers.ts229
-rw-r--r--server/lib/object-storage/urls.ts40
-rw-r--r--server/lib/object-storage/videos.ts72
-rw-r--r--server/lib/paths.ts (renamed from server/lib/video-paths.ts)66
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts2
-rw-r--r--server/lib/thumbnail.ts30
-rw-r--r--server/lib/transcoding/video-transcoding.ts228
-rw-r--r--server/lib/video-path-manager.ts139
-rw-r--r--server/lib/video-state.ts99
-rw-r--r--server/lib/video-urls.ts31
-rw-r--r--server/lib/video.ts47
-rw-r--r--server/models/video/formatter/video-format-utils.ts2
-rw-r--r--server/models/video/sql/shared/video-tables.ts9
-rw-r--r--server/models/video/video-file.ts26
-rw-r--r--server/models/video/video-job-info.ts100
-rw-r--r--server/models/video/video-streaming-playlist.ts30
-rw-r--r--server/models/video/video.ts51
-rw-r--r--server/tests/api/index.ts1
-rw-r--r--server/tests/api/live/live-save-replay.ts30
-rw-r--r--server/tests/api/object-storage/index.ts3
-rw-r--r--server/tests/api/object-storage/live.ts136
-rw-r--r--server/tests/api/object-storage/video-imports.ts112
-rw-r--r--server/tests/api/object-storage/videos.ts391
-rw-r--r--server/tests/api/redundancy/redundancy.ts6
-rw-r--r--server/tests/api/videos/video-hls.ts73
-rw-r--r--server/tests/cli/create-import-video-file-job.ts56
-rw-r--r--server/tests/cli/create-transcoding-job.ts95
-rw-r--r--server/tests/helpers/request.ts8
53 files changed, 2342 insertions, 506 deletions
diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts
index 89f50714d..5c740c041 100644
--- a/server/controllers/api/videos/upload.ts
+++ b/server/controllers/api/videos/upload.ts
@@ -1,12 +1,21 @@
1import * as express from 'express' 1import * as express from 'express'
2import { move } from 'fs-extra' 2import { move } from 'fs-extra'
3import { basename } from 'path'
3import { getLowercaseExtension } from '@server/helpers/core-utils' 4import { getLowercaseExtension } from '@server/helpers/core-utils'
4import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' 5import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload'
5import { uuidToShort } from '@server/helpers/uuid' 6import { uuidToShort } from '@server/helpers/uuid'
6import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 7import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
7import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' 8import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
8import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' 9import { generateWebTorrentVideoFilename } from '@server/lib/paths'
9import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 10import {
11 addMoveToObjectStorageJob,
12 addOptimizeOrMergeAudioJob,
13 buildLocalVideoFromReq,
14 buildVideoThumbnailsFromReq,
15 setVideoTags
16} from '@server/lib/video'
17import { VideoPathManager } from '@server/lib/video-path-manager'
18import { buildNextVideoState } from '@server/lib/video-state'
10import { openapiOperationDoc } from '@server/middlewares/doc' 19import { openapiOperationDoc } from '@server/middlewares/doc'
11import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 20import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
12import { uploadx } from '@uploadx/core' 21import { uploadx } from '@uploadx/core'
@@ -139,23 +148,20 @@ async function addVideo (options: {
139 148
140 const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) 149 const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id)
141 150
142 videoData.state = CONFIG.TRANSCODING.ENABLED 151 videoData.state = buildNextVideoState()
143 ? VideoState.TO_TRANSCODE
144 : VideoState.PUBLISHED
145
146 videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware 152 videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware
147 153
148 const video = new VideoModel(videoData) as MVideoFullLight 154 const video = new VideoModel(videoData) as MVideoFullLight
149 video.VideoChannel = videoChannel 155 video.VideoChannel = videoChannel
150 video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object 156 video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
151 157
152 const videoFile = await buildNewFile(video, videoPhysicalFile) 158 const videoFile = await buildNewFile(videoPhysicalFile)
153 159
154 // Move physical file 160 // Move physical file
155 const destination = getVideoFilePath(video, videoFile) 161 const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile)
156 await move(videoPhysicalFile.path, destination) 162 await move(videoPhysicalFile.path, destination)
157 // This is important in case if there is another attempt in the retry process 163 // This is important in case if there is another attempt in the retry process
158 videoPhysicalFile.filename = getVideoFilePath(video, videoFile) 164 videoPhysicalFile.filename = basename(destination)
159 videoPhysicalFile.path = destination 165 videoPhysicalFile.path = destination
160 166
161 const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ 167 const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
@@ -210,9 +216,13 @@ async function addVideo (options: {
210 216
211 createTorrentFederate(video, videoFile) 217 createTorrentFederate(video, videoFile)
212 .then(() => { 218 .then(() => {
213 if (video.state !== VideoState.TO_TRANSCODE) return 219 if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
220 return addMoveToObjectStorageJob(video)
221 }
214 222
215 return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) 223 if (video.state === VideoState.TO_TRANSCODE) {
224 return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
225 }
216 }) 226 })
217 .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) })) 227 .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
218 228
@@ -227,7 +237,7 @@ async function addVideo (options: {
227 }) 237 })
228} 238}
229 239
230async function buildNewFile (video: MVideo, videoPhysicalFile: express.VideoUploadFile) { 240async function buildNewFile (videoPhysicalFile: express.VideoUploadFile) {
231 const videoFile = new VideoFileModel({ 241 const videoFile = new VideoFileModel({
232 extname: getLowercaseExtension(videoPhysicalFile.filename), 242 extname: getLowercaseExtension(videoPhysicalFile.filename),
233 size: videoPhysicalFile.size, 243 size: videoPhysicalFile.size,
diff --git a/server/controllers/download.ts b/server/controllers/download.ts
index ddacc1b68..ffe40d57e 100644
--- a/server/controllers/download.ts
+++ b/server/controllers/download.ts
@@ -3,9 +3,9 @@ import * as express from 'express'
3import { logger } from '@server/helpers/logger' 3import { logger } from '@server/helpers/logger'
4import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' 4import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache'
5import { Hooks } from '@server/lib/plugins/hooks' 5import { Hooks } from '@server/lib/plugins/hooks'
6import { getVideoFilePath } from '@server/lib/video-paths' 6import { VideoPathManager } from '@server/lib/video-path-manager'
7import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 7import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
8import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models' 8import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models'
9import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' 9import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants'
10import { asyncMiddleware, videosDownloadValidator } from '../middlewares' 10import { asyncMiddleware, videosDownloadValidator } from '../middlewares'
11 11
@@ -81,7 +81,15 @@ async function downloadVideoFile (req: express.Request, res: express.Response) {
81 81
82 if (!checkAllowResult(res, allowParameters, allowedResult)) return 82 if (!checkAllowResult(res, allowParameters, allowedResult)) return
83 83
84 return res.download(getVideoFilePath(video, videoFile), `${video.name}-${videoFile.resolution}p${videoFile.extname}`) 84 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
85 return res.redirect(videoFile.getObjectStorageUrl())
86 }
87
88 await VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, path => {
89 const filename = `${video.name}-${videoFile.resolution}p${videoFile.extname}`
90
91 return res.download(path, filename)
92 })
85} 93}
86 94
87async function downloadHLSVideoFile (req: express.Request, res: express.Response) { 95async function downloadHLSVideoFile (req: express.Request, res: express.Response) {
@@ -107,8 +115,15 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
107 115
108 if (!checkAllowResult(res, allowParameters, allowedResult)) return 116 if (!checkAllowResult(res, allowParameters, allowedResult)) return
109 117
110 const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}` 118 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
111 return res.download(getVideoFilePath(streamingPlaylist, videoFile), filename) 119 return res.redirect(videoFile.getObjectStorageUrl())
120 }
121
122 await VideoPathManager.Instance.makeAvailableVideoFile(streamingPlaylist, videoFile, path => {
123 const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}`
124
125 return res.download(path, filename)
126 })
112} 127}
113 128
114function getVideoFile (req: express.Request, files: MVideoFile[]) { 129function getVideoFile (req: express.Request, files: MVideoFile[]) {
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index ecf63e93e..c84376304 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -6,7 +6,8 @@ import { dirname, join } from 'path'
6import * as WebTorrent from 'webtorrent' 6import * as WebTorrent from 'webtorrent'
7import { isArray } from '@server/helpers/custom-validators/misc' 7import { isArray } from '@server/helpers/custom-validators/misc'
8import { WEBSERVER } from '@server/initializers/constants' 8import { WEBSERVER } from '@server/initializers/constants'
9import { generateTorrentFileName, getVideoFilePath } from '@server/lib/video-paths' 9import { generateTorrentFileName } from '@server/lib/paths'
10import { VideoPathManager } from '@server/lib/video-path-manager'
10import { MVideo } from '@server/types/models/video/video' 11import { MVideo } from '@server/types/models/video/video'
11import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' 12import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file'
12import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' 13import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist'
@@ -78,7 +79,7 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
78 }) 79 })
79} 80}
80 81
81async function createTorrentAndSetInfoHash ( 82function createTorrentAndSetInfoHash (
82 videoOrPlaylist: MVideo | MStreamingPlaylistVideo, 83 videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
83 videoFile: MVideoFile 84 videoFile: MVideoFile
84) { 85) {
@@ -95,22 +96,24 @@ async function createTorrentAndSetInfoHash (
95 urlList: [ videoFile.getFileUrl(video) ] 96 urlList: [ videoFile.getFileUrl(video) ]
96 } 97 }
97 98
98 const torrent = await createTorrentPromise(getVideoFilePath(videoOrPlaylist, videoFile), options) 99 return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => {
100 const torrent = await createTorrentPromise(videoPath, options)
99 101
100 const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) 102 const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
101 const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) 103 const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename)
102 logger.info('Creating torrent %s.', torrentPath) 104 logger.info('Creating torrent %s.', torrentPath)
103 105
104 await writeFile(torrentPath, torrent) 106 await writeFile(torrentPath, torrent)
105 107
106 // Remove old torrent file if it existed 108 // Remove old torrent file if it existed
107 if (videoFile.hasTorrent()) { 109 if (videoFile.hasTorrent()) {
108 await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) 110 await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
109 } 111 }
110 112
111 const parsedTorrent = parseTorrent(torrent) 113 const parsedTorrent = parseTorrent(torrent)
112 videoFile.infoHash = parsedTorrent.infoHash 114 videoFile.infoHash = parsedTorrent.infoHash
113 videoFile.torrentFilename = torrentFilename 115 videoFile.torrentFilename = torrentFilename
116 })
114} 117}
115 118
116function generateMagnetUri ( 119function generateMagnetUri (
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
index 911734fa0..09f587274 100644
--- a/server/initializers/checker-after-init.ts
+++ b/server/initializers/checker-after-init.ts
@@ -153,6 +153,29 @@ function checkConfig () {
153 } 153 }
154 } 154 }
155 155
156 // Object storage
157 if (CONFIG.OBJECT_STORAGE.ENABLED === true) {
158
159 if (!CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME) {
160 return 'videos_bucket should be set when object storage support is enabled.'
161 }
162
163 if (!CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME) {
164 return 'streaming_playlists_bucket should be set when object storage support is enabled.'
165 }
166
167 if (
168 CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME &&
169 CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX
170 ) {
171 if (CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === '') {
172 return 'Object storage bucket prefixes should be set when the same bucket is used for both types of video.'
173 } else {
174 return 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.'
175 }
176 }
177 }
178
156 return null 179 return null
157} 180}
158 181
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 30a9823b9..0e684eef8 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -73,6 +73,26 @@ const CONFIG = {
73 PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')), 73 PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')),
74 CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')) 74 CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides'))
75 }, 75 },
76 OBJECT_STORAGE: {
77 ENABLED: config.get<boolean>('object_storage.enabled'),
78 MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')),
79 ENDPOINT: config.get<string>('object_storage.endpoint'),
80 REGION: config.get<string>('object_storage.region'),
81 CREDENTIALS: {
82 ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'),
83 SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key')
84 },
85 VIDEOS: {
86 BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'),
87 PREFIX: config.get<string>('object_storage.videos.prefix'),
88 BASE_URL: config.get<string>('object_storage.videos.base_url')
89 },
90 STREAMING_PLAYLISTS: {
91 BUCKET_NAME: config.get<string>('object_storage.streaming_playlists.bucket_name'),
92 PREFIX: config.get<string>('object_storage.streaming_playlists.prefix'),
93 BASE_URL: config.get<string>('object_storage.streaming_playlists.base_url')
94 }
95 },
76 WEBSERVER: { 96 WEBSERVER: {
77 SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http', 97 SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http',
78 WS: config.get<boolean>('webserver.https') === true ? 'wss' : 'ws', 98 WS: config.get<boolean>('webserver.https') === true ? 'wss' : 'ws',
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 5f121d9a4..8a1526ae8 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
24 24
25// --------------------------------------------------------------------------- 25// ---------------------------------------------------------------------------
26 26
27const LAST_MIGRATION_VERSION = 655 27const LAST_MIGRATION_VERSION = 660
28 28
29// --------------------------------------------------------------------------- 29// ---------------------------------------------------------------------------
30 30
@@ -147,7 +147,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
147 'videos-views': 1, 147 'videos-views': 1,
148 'activitypub-refresher': 1, 148 'activitypub-refresher': 1,
149 'video-redundancy': 1, 149 'video-redundancy': 1,
150 'video-live-ending': 1 150 'video-live-ending': 1,
151 'move-to-object-storage': 3
151} 152}
152// Excluded keys are jobs that can be configured by admins 153// Excluded keys are jobs that can be configured by admins
153const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = { 154const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = {
@@ -162,7 +163,8 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
162 'videos-views': 1, 163 'videos-views': 1,
163 'activitypub-refresher': 1, 164 'activitypub-refresher': 1,
164 'video-redundancy': 1, 165 'video-redundancy': 1,
165 'video-live-ending': 10 166 'video-live-ending': 10,
167 'move-to-object-storage': 1
166} 168}
167const JOB_TTL: { [id in JobType]: number } = { 169const JOB_TTL: { [id in JobType]: number } = {
168 'activitypub-http-broadcast': 60000 * 10, // 10 minutes 170 'activitypub-http-broadcast': 60000 * 10, // 10 minutes
@@ -178,7 +180,8 @@ const JOB_TTL: { [id in JobType]: number } = {
178 'videos-views': undefined, // Unlimited 180 'videos-views': undefined, // Unlimited
179 'activitypub-refresher': 60000 * 10, // 10 minutes 181 'activitypub-refresher': 60000 * 10, // 10 minutes
180 'video-redundancy': 1000 * 3600 * 3, // 3 hours 182 'video-redundancy': 1000 * 3600 * 3, // 3 hours
181 'video-live-ending': 1000 * 60 * 10 // 10 minutes 183 'video-live-ending': 1000 * 60 * 10, // 10 minutes
184 'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours
182} 185}
183const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { 186const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = {
184 'videos-views': { 187 'videos-views': {
@@ -412,7 +415,8 @@ const VIDEO_STATES: { [ id in VideoState ]: string } = {
412 [VideoState.TO_TRANSCODE]: 'To transcode', 415 [VideoState.TO_TRANSCODE]: 'To transcode',
413 [VideoState.TO_IMPORT]: 'To import', 416 [VideoState.TO_IMPORT]: 'To import',
414 [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream', 417 [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream',
415 [VideoState.LIVE_ENDED]: 'Livestream ended' 418 [VideoState.LIVE_ENDED]: 'Livestream ended',
419 [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage'
416} 420}
417 421
418const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = { 422const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 38e7a76d0..0e690f6ae 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -45,6 +45,7 @@ import { VideoTagModel } from '../models/video/video-tag'
45import { VideoViewModel } from '../models/video/video-view' 45import { VideoViewModel } from '../models/video/video-view'
46import { CONFIG } from './config' 46import { CONFIG } from './config'
47import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' 47import { ActorCustomPageModel } from '@server/models/account/actor-custom-page'
48import { VideoJobInfoModel } from '@server/models/video/video-job-info'
48 49
49require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string 50require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
50 51
@@ -143,7 +144,8 @@ async function initDatabaseModels (silent: boolean) {
143 TrackerModel, 144 TrackerModel,
144 VideoTrackerModel, 145 VideoTrackerModel,
145 PluginModel, 146 PluginModel,
146 ActorCustomPageModel 147 ActorCustomPageModel,
148 VideoJobInfoModel
147 ]) 149 ])
148 150
149 // Check extensions exist in the database 151 // Check extensions exist in the database
diff --git a/server/initializers/migrations/0065-video-file-size.ts b/server/initializers/migrations/0065-video-file-size.ts
index 1aeb27f2d..ac952a98c 100644
--- a/server/initializers/migrations/0065-video-file-size.ts
+++ b/server/initializers/migrations/0065-video-file-size.ts
@@ -1,7 +1,4 @@
1import * as Sequelize from 'sequelize' 1import * as Sequelize from 'sequelize'
2import { stat } from 'fs-extra'
3import { VideoModel } from '../../models/video/video'
4import { getVideoFilePath } from '@server/lib/video-paths'
5 2
6function up (utils: { 3function up (utils: {
7 transaction: Sequelize.Transaction 4 transaction: Sequelize.Transaction
@@ -9,30 +6,7 @@ function up (utils: {
9 sequelize: Sequelize.Sequelize 6 sequelize: Sequelize.Sequelize
10 db: any 7 db: any
11}): Promise<void> { 8}): Promise<void> {
12 return utils.db.Video.listOwnedAndPopulateAuthorAndTags() 9 throw new Error('Removed, please upgrade from a previous version first.')
13 .then((videos: VideoModel[]) => {
14 const tasks: Promise<any>[] = []
15
16 videos.forEach(video => {
17 video.VideoFiles.forEach(videoFile => {
18 const p = new Promise((res, rej) => {
19 stat(getVideoFilePath(video, videoFile), (err, stats) => {
20 if (err) return rej(err)
21
22 videoFile.size = stats.size
23 videoFile.save().then(res).catch(rej)
24 })
25 })
26
27 tasks.push(p)
28 })
29 })
30
31 return tasks
32 })
33 .then((tasks: Promise<any>[]) => {
34 return Promise.all(tasks)
35 })
36} 10}
37 11
38function down (options) { 12function down (options) {
diff --git a/server/initializers/migrations/0660-object-storage.ts b/server/initializers/migrations/0660-object-storage.ts
new file mode 100644
index 000000000..c815c71c6
--- /dev/null
+++ b/server/initializers/migrations/0660-object-storage.ts
@@ -0,0 +1,58 @@
1import * as Sequelize from 'sequelize'
2import { VideoStorage } from '@shared/models'
3
4async function up (utils: {
5 transaction: Sequelize.Transaction
6 queryInterface: Sequelize.QueryInterface
7 sequelize: Sequelize.Sequelize
8 db: any
9}): Promise<void> {
10 {
11 const query = `
12 CREATE TABLE IF NOT EXISTS "videoJobInfo" (
13 "id" serial,
14 "pendingMove" INTEGER NOT NULL,
15 "pendingTranscode" INTEGER NOT NULL,
16 "videoId" serial UNIQUE NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
17 "createdAt" timestamp WITH time zone NOT NULL,
18 "updatedAt" timestamp WITH time zone NOT NULL,
19 PRIMARY KEY ("id")
20 );
21 `
22
23 await utils.sequelize.query(query)
24 }
25
26 {
27 await utils.queryInterface.addColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: true })
28 }
29 {
30 await utils.sequelize.query(
31 `UPDATE "videoFile" SET "storage" = ${VideoStorage.FILE_SYSTEM}`
32 )
33 }
34 {
35 await utils.queryInterface.changeColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: false })
36 }
37
38 {
39 await utils.queryInterface.addColumn('videoStreamingPlaylist', 'storage', { type: Sequelize.INTEGER, allowNull: true })
40 }
41 {
42 await utils.sequelize.query(
43 `UPDATE "videoStreamingPlaylist" SET "storage" = ${VideoStorage.FILE_SYSTEM}`
44 )
45 }
46 {
47 await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'storage', { type: Sequelize.INTEGER, allowNull: false })
48 }
49}
50
51function down (options) {
52 throw new Error('Not implemented.')
53}
54
55export {
56 up,
57 down
58}
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
index 1fa16295d..bd9ed45a9 100644
--- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
+++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -6,7 +6,7 @@ import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/vide
6import { logger } from '@server/helpers/logger' 6import { logger } from '@server/helpers/logger'
7import { getExtFromMimetype } from '@server/helpers/video' 7import { getExtFromMimetype } from '@server/helpers/video'
8import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' 8import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants'
9import { generateTorrentFileName } from '@server/lib/video-paths' 9import { generateTorrentFileName } from '@server/lib/paths'
10import { VideoCaptionModel } from '@server/models/video/video-caption' 10import { VideoCaptionModel } from '@server/models/video/video-caption'
11import { VideoFileModel } from '@server/models/video/video-file' 11import { VideoFileModel } from '@server/models/video/video-file'
12import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' 12import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 0e77ab9fa..0828a2d0f 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -1,4 +1,4 @@
1import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, stat, writeFile } from 'fs-extra' 1import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra'
2import { flatten, uniq } from 'lodash' 2import { flatten, uniq } from 'lodash'
3import { basename, dirname, join } from 'path' 3import { basename, dirname, join } from 'path'
4import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' 4import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models'
@@ -8,11 +8,12 @@ import { logger } from '../helpers/logger'
8import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' 8import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
9import { generateRandomString } from '../helpers/utils' 9import { generateRandomString } from '../helpers/utils'
10import { CONFIG } from '../initializers/config' 10import { CONFIG } from '../initializers/config'
11import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' 11import { P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants'
12import { sequelizeTypescript } from '../initializers/database' 12import { sequelizeTypescript } from '../initializers/database'
13import { VideoFileModel } from '../models/video/video-file' 13import { VideoFileModel } from '../models/video/video-file'
14import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' 14import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
15import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths' 15import { getHlsResolutionPlaylistFilename } from './paths'
16import { VideoPathManager } from './video-path-manager'
16 17
17async function updateStreamingPlaylistsInfohashesIfNeeded () { 18async function updateStreamingPlaylistsInfohashesIfNeeded () {
18 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() 19 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
@@ -31,75 +32,66 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () {
31} 32}
32 33
33async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { 34async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
34 const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
35
36 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] 35 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
37 36
38 const masterPlaylistPath = join(directory, playlist.playlistFilename)
39
40 for (const file of playlist.VideoFiles) { 37 for (const file of playlist.VideoFiles) {
41 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) 38 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
42 39
43 // If we did not generated a playlist for this resolution, skip 40 await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, async videoFilePath => {
44 const filePlaylistPath = join(directory, playlistFilename) 41 const size = await getVideoStreamSize(videoFilePath)
45 if (await pathExists(filePlaylistPath) === false) continue
46
47 const videoFilePath = getVideoFilePath(playlist, file)
48 42
49 const size = await getVideoStreamSize(videoFilePath) 43 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
44 const resolution = `RESOLUTION=${size.width}x${size.height}`
50 45
51 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) 46 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
52 const resolution = `RESOLUTION=${size.width}x${size.height}` 47 if (file.fps) line += ',FRAME-RATE=' + file.fps
53 48
54 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` 49 const codecs = await Promise.all([
55 if (file.fps) line += ',FRAME-RATE=' + file.fps 50 getVideoStreamCodec(videoFilePath),
51 getAudioStreamCodec(videoFilePath)
52 ])
56 53
57 const codecs = await Promise.all([ 54 line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
58 getVideoStreamCodec(videoFilePath),
59 getAudioStreamCodec(videoFilePath)
60 ])
61 55
62 line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` 56 masterPlaylists.push(line)
63 57 masterPlaylists.push(playlistFilename)
64 masterPlaylists.push(line) 58 })
65 masterPlaylists.push(playlistFilename)
66 } 59 }
67 60
68 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') 61 await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => {
62 return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
63 })
69} 64}
70 65
71async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { 66async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
72 const json: { [filename: string]: { [range: string]: string } } = {} 67 const json: { [filename: string]: { [range: string]: string } } = {}
73 68
74 const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
75
76 // For all the resolutions available for this video 69 // For all the resolutions available for this video
77 for (const file of playlist.VideoFiles) { 70 for (const file of playlist.VideoFiles) {
78 const rangeHashes: { [range: string]: string } = {} 71 const rangeHashes: { [range: string]: string } = {}
79 72
80 const videoPath = getVideoFilePath(playlist, file) 73 await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, videoPath => {
81 const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename))
82
83 // Maybe the playlist is not generated for this resolution yet
84 if (!await pathExists(resolutionPlaylistPath)) continue
85 74
86 const playlistContent = await readFile(resolutionPlaylistPath) 75 return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(playlist, file, async resolutionPlaylistPath => {
87 const ranges = getRangesFromPlaylist(playlistContent.toString()) 76 const playlistContent = await readFile(resolutionPlaylistPath)
77 const ranges = getRangesFromPlaylist(playlistContent.toString())
88 78
89 const fd = await open(videoPath, 'r') 79 const fd = await open(videoPath, 'r')
90 for (const range of ranges) { 80 for (const range of ranges) {
91 const buf = Buffer.alloc(range.length) 81 const buf = Buffer.alloc(range.length)
92 await read(fd, buf, 0, range.length, range.offset) 82 await read(fd, buf, 0, range.length, range.offset)
93 83
94 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) 84 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
95 } 85 }
96 await close(fd) 86 await close(fd)
97 87
98 const videoFilename = file.filename 88 const videoFilename = file.filename
99 json[videoFilename] = rangeHashes 89 json[videoFilename] = rangeHashes
90 })
91 })
100 } 92 }
101 93
102 const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename) 94 const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
103 await outputJSON(outputPath, json) 95 await outputJSON(outputPath, json)
104} 96}
105 97
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
new file mode 100644
index 000000000..a0c58d211
--- /dev/null
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -0,0 +1,114 @@
1import * as Bull from 'bull'
2import { remove } from 'fs-extra'
3import { join } from 'path'
4import { logger } from '@server/helpers/logger'
5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
6import { CONFIG } from '@server/initializers/config'
7import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage'
8import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
9import { moveToNextState } from '@server/lib/video-state'
10import { VideoModel } from '@server/models/video/video'
11import { VideoJobInfoModel } from '@server/models/video/video-job-info'
12import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
13import { MoveObjectStoragePayload, VideoStorage } from '../../../../shared'
14
15export async function processMoveToObjectStorage (job: Bull.Job) {
16 const payload = job.data as MoveObjectStoragePayload
17 logger.info('Moving video %s in job %d.', payload.videoUUID, job.id)
18
19 const video = await VideoModel.loadWithFiles(payload.videoUUID)
20 // No video, maybe deleted?
21 if (!video) {
22 logger.info('Can\'t process job %d, video does not exist.', job.id)
23 return undefined
24 }
25
26 if (video.VideoFiles) {
27 await moveWebTorrentFiles(video)
28 }
29
30 if (video.VideoStreamingPlaylists) {
31 await moveHLSFiles(video)
32 }
33
34 const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
35 if (pendingMove === 0) {
36 logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
37 await doAfterLastJob(video, payload.isNewVideo)
38 }
39
40 return payload.videoUUID
41}
42
43// ---------------------------------------------------------------------------
44
45async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
46 for (const file of video.VideoFiles) {
47 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
48
49 const fileUrl = await storeWebTorrentFile(file.filename)
50
51 const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename)
52 await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
53 }
54}
55
56async function moveHLSFiles (video: MVideoWithAllFiles) {
57 for (const playlist of video.VideoStreamingPlaylists) {
58
59 for (const file of playlist.VideoFiles) {
60 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
61
62 // Resolution playlist
63 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
64 await storeHLSFile(playlist, video, playlistFilename)
65
66 // Resolution fragmented file
67 const fileUrl = await storeHLSFile(playlist, video, file.filename)
68
69 const oldPath = join(getHLSDirectory(video), file.filename)
70
71 await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
72 }
73 }
74}
75
76async function doAfterLastJob (video: MVideoWithAllFiles, isNewVideo: boolean) {
77 for (const playlist of video.VideoStreamingPlaylists) {
78 if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
79
80 // Master playlist
81 playlist.playlistUrl = await storeHLSFile(playlist, video, playlist.playlistFilename)
82 // Sha256 segments file
83 playlist.segmentsSha256Url = await storeHLSFile(playlist, video, playlist.segmentsSha256Filename)
84
85 playlist.storage = VideoStorage.OBJECT_STORAGE
86
87 await playlist.save()
88 }
89
90 // Remove empty hls video directory
91 if (video.VideoStreamingPlaylists) {
92 await remove(getHLSDirectory(video))
93 }
94
95 await moveToNextState(video, isNewVideo)
96}
97
98async function onFileMoved (options: {
99 videoOrPlaylist: MVideo | MStreamingPlaylistVideo
100 file: MVideoFile
101 fileUrl: string
102 oldPath: string
103}) {
104 const { videoOrPlaylist, file, fileUrl, oldPath } = options
105
106 file.fileUrl = fileUrl
107 file.storage = VideoStorage.OBJECT_STORAGE
108
109 await createTorrentAndSetInfoHash(videoOrPlaylist, file)
110 await file.save()
111
112 logger.debug('Removing %s because it\'s now on object storage', oldPath)
113 await remove(oldPath)
114}
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts
index 2f4abf730..e8ee1f759 100644
--- a/server/lib/job-queue/handlers/video-file-import.ts
+++ b/server/lib/job-queue/handlers/video-file-import.ts
@@ -2,15 +2,19 @@ import * as Bull from 'bull'
2import { copy, stat } from 'fs-extra' 2import { copy, stat } from 'fs-extra'
3import { getLowercaseExtension } from '@server/helpers/core-utils' 3import { getLowercaseExtension } from '@server/helpers/core-utils'
4import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 4import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
5import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 5import { CONFIG } from '@server/initializers/config'
6import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
7import { generateWebTorrentVideoFilename } from '@server/lib/paths'
8import { addMoveToObjectStorageJob } from '@server/lib/video'
9import { VideoPathManager } from '@server/lib/video-path-manager'
6import { UserModel } from '@server/models/user/user' 10import { UserModel } from '@server/models/user/user'
7import { MVideoFullLight } from '@server/types/models' 11import { MVideoFullLight } from '@server/types/models'
8import { VideoFileImportPayload } from '@shared/models' 12import { VideoFileImportPayload, VideoStorage } from '@shared/models'
9import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' 13import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils'
10import { logger } from '../../../helpers/logger' 14import { logger } from '../../../helpers/logger'
11import { VideoModel } from '../../../models/video/video' 15import { VideoModel } from '../../../models/video/video'
12import { VideoFileModel } from '../../../models/video/video-file' 16import { VideoFileModel } from '../../../models/video/video-file'
13import { onNewWebTorrentFileResolution } from './video-transcoding' 17import { createHlsJobIfEnabled } from './video-transcoding'
14 18
15async function processVideoFileImport (job: Bull.Job) { 19async function processVideoFileImport (job: Bull.Job) {
16 const payload = job.data as VideoFileImportPayload 20 const payload = job.data as VideoFileImportPayload
@@ -29,15 +33,19 @@ async function processVideoFileImport (job: Bull.Job) {
29 33
30 const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId) 34 const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId)
31 35
32 const newResolutionPayload = { 36 await createHlsJobIfEnabled(user, {
33 type: 'new-resolution-to-webtorrent' as 'new-resolution-to-webtorrent',
34 videoUUID: video.uuid, 37 videoUUID: video.uuid,
35 resolution: data.resolution, 38 resolution: data.resolution,
36 isPortraitMode: data.isPortraitMode, 39 isPortraitMode: data.isPortraitMode,
37 copyCodecs: false, 40 copyCodecs: true,
38 isNewVideo: false 41 isMaxQuality: false
42 })
43
44 if (CONFIG.OBJECT_STORAGE.ENABLED) {
45 await addMoveToObjectStorageJob(video)
46 } else {
47 await federateVideoIfNeeded(video, false)
39 } 48 }
40 await onNewWebTorrentFileResolution(video, user, newResolutionPayload)
41 49
42 return video 50 return video
43} 51}
@@ -72,12 +80,13 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) {
72 resolution, 80 resolution,
73 extname: fileExt, 81 extname: fileExt,
74 filename: generateWebTorrentVideoFilename(resolution, fileExt), 82 filename: generateWebTorrentVideoFilename(resolution, fileExt),
83 storage: VideoStorage.FILE_SYSTEM,
75 size, 84 size,
76 fps, 85 fps,
77 videoId: video.id 86 videoId: video.id
78 }) 87 })
79 88
80 const outputPath = getVideoFilePath(video, newVideoFile) 89 const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
81 await copy(inputFilePath, outputPath) 90 await copy(inputFilePath, outputPath)
82 91
83 video.VideoFiles.push(newVideoFile) 92 video.VideoFiles.push(newVideoFile)
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index fec553f2b..a5fa204f5 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -4,11 +4,13 @@ import { getLowercaseExtension } from '@server/helpers/core-utils'
4import { retryTransactionWrapper } from '@server/helpers/database-utils' 4import { retryTransactionWrapper } from '@server/helpers/database-utils'
5import { YoutubeDL } from '@server/helpers/youtube-dl' 5import { YoutubeDL } from '@server/helpers/youtube-dl'
6import { isPostImportVideoAccepted } from '@server/lib/moderation' 6import { isPostImportVideoAccepted } from '@server/lib/moderation'
7import { generateWebTorrentVideoFilename } from '@server/lib/paths'
7import { Hooks } from '@server/lib/plugins/hooks' 8import { Hooks } from '@server/lib/plugins/hooks'
8import { ServerConfigManager } from '@server/lib/server-config-manager' 9import { ServerConfigManager } from '@server/lib/server-config-manager'
9import { isAbleToUploadVideo } from '@server/lib/user' 10import { isAbleToUploadVideo } from '@server/lib/user'
10import { addOptimizeOrMergeAudioJob } from '@server/lib/video' 11import { addMoveToObjectStorageJob, addOptimizeOrMergeAudioJob } from '@server/lib/video'
11import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 12import { VideoPathManager } from '@server/lib/video-path-manager'
13import { buildNextVideoState } from '@server/lib/video-state'
12import { ThumbnailModel } from '@server/models/video/thumbnail' 14import { ThumbnailModel } from '@server/models/video/thumbnail'
13import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' 15import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
14import { 16import {
@@ -25,7 +27,6 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro
25import { logger } from '../../../helpers/logger' 27import { logger } from '../../../helpers/logger'
26import { getSecureTorrentName } from '../../../helpers/utils' 28import { getSecureTorrentName } from '../../../helpers/utils'
27import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' 29import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
28import { CONFIG } from '../../../initializers/config'
29import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' 30import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
30import { sequelizeTypescript } from '../../../initializers/database' 31import { sequelizeTypescript } from '../../../initializers/database'
31import { VideoModel } from '../../../models/video/video' 32import { VideoModel } from '../../../models/video/video'
@@ -100,7 +101,6 @@ type ProcessFileOptions = {
100} 101}
101async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) { 102async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) {
102 let tempVideoPath: string 103 let tempVideoPath: string
103 let videoDestFile: string
104 let videoFile: VideoFileModel 104 let videoFile: VideoFileModel
105 105
106 try { 106 try {
@@ -159,7 +159,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
159 const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) 159 const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
160 160
161 // Move file 161 // Move file
162 videoDestFile = getVideoFilePath(videoImportWithFiles.Video, videoFile) 162 const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile)
163 await move(tempVideoPath, videoDestFile) 163 await move(tempVideoPath, videoDestFile)
164 tempVideoPath = null // This path is not used anymore 164 tempVideoPath = null // This path is not used anymore
165 165
@@ -204,7 +204,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
204 204
205 // Update video DB object 205 // Update video DB object
206 video.duration = duration 206 video.duration = duration
207 video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED 207 video.state = buildNextVideoState(video.state)
208 await video.save({ transaction: t }) 208 await video.save({ transaction: t })
209 209
210 if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) 210 if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t)
@@ -245,6 +245,10 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
245 Notifier.Instance.notifyOnNewVideoIfNeeded(video) 245 Notifier.Instance.notifyOnNewVideoIfNeeded(video)
246 } 246 }
247 247
248 if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
249 return addMoveToObjectStorageJob(videoImportUpdated.Video)
250 }
251
248 // Create transcoding jobs? 252 // Create transcoding jobs?
249 if (video.state === VideoState.TO_TRANSCODE) { 253 if (video.state === VideoState.TO_TRANSCODE) {
250 await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User) 254 await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User)
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index aa5bd573a..9ccf724c2 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -4,10 +4,11 @@ import { join } from 'path'
4import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' 4import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
5import { VIDEO_LIVE } from '@server/initializers/constants' 5import { VIDEO_LIVE } from '@server/initializers/constants'
6import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live' 6import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live'
7import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveDirectory } from '@server/lib/paths'
7import { generateVideoMiniature } from '@server/lib/thumbnail' 8import { generateVideoMiniature } from '@server/lib/thumbnail'
8import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' 9import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding'
9import { publishAndFederateIfNeeded } from '@server/lib/video' 10import { VideoPathManager } from '@server/lib/video-path-manager'
10import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHLSDirectory } from '@server/lib/video-paths' 11import { moveToNextState } from '@server/lib/video-state'
11import { VideoModel } from '@server/models/video/video' 12import { VideoModel } from '@server/models/video/video'
12import { VideoFileModel } from '@server/models/video/video-file' 13import { VideoFileModel } from '@server/models/video/video-file'
13import { VideoLiveModel } from '@server/models/video/video-live' 14import { VideoLiveModel } from '@server/models/video/video-live'
@@ -55,16 +56,15 @@ export {
55// --------------------------------------------------------------------------- 56// ---------------------------------------------------------------------------
56 57
57async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) { 58async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) {
58 const hlsDirectory = getHLSDirectory(video, false) 59 const replayDirectory = VideoPathManager.Instance.getFSHLSOutputPath(video, VIDEO_LIVE.REPLAY_DIRECTORY)
59 const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY)
60 60
61 const rootFiles = await readdir(hlsDirectory) 61 const rootFiles = await readdir(getLiveDirectory(video))
62 62
63 const playlistFiles = rootFiles.filter(file => { 63 const playlistFiles = rootFiles.filter(file => {
64 return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename 64 return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename
65 }) 65 })
66 66
67 await cleanupLiveFiles(hlsDirectory) 67 await cleanupTMPLiveFiles(getLiveDirectory(video))
68 68
69 await live.destroy() 69 await live.destroy()
70 70
@@ -98,7 +98,7 @@ async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MSt
98 98
99 const { resolution, isPortraitMode } = await getVideoFileResolution(concatenatedTsFilePath, probe) 99 const { resolution, isPortraitMode } = await getVideoFileResolution(concatenatedTsFilePath, probe)
100 100
101 const outputPath = await generateHlsPlaylistResolutionFromTS({ 101 const { resolutionPlaylistPath: outputPath } = await generateHlsPlaylistResolutionFromTS({
102 video: videoWithFiles, 102 video: videoWithFiles,
103 concatenatedTsFilePath, 103 concatenatedTsFilePath,
104 resolution, 104 resolution,
@@ -133,10 +133,10 @@ async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MSt
133 }) 133 })
134 } 134 }
135 135
136 await publishAndFederateIfNeeded(videoWithFiles, true) 136 await moveToNextState(videoWithFiles, false)
137} 137}
138 138
139async function cleanupLiveFiles (hlsDirectory: string) { 139async function cleanupTMPLiveFiles (hlsDirectory: string) {
140 if (!await pathExists(hlsDirectory)) return 140 if (!await pathExists(hlsDirectory)) return
141 141
142 const files = await readdir(hlsDirectory) 142 const files = await readdir(hlsDirectory)
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 876d1460c..b3149dde8 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -1,9 +1,11 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils' 2import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils'
3import { getTranscodingJobPriority, publishAndFederateIfNeeded } from '@server/lib/video' 3import { addTranscodingJob, getTranscodingJobPriority } from '@server/lib/video'
4import { getVideoFilePath } from '@server/lib/video-paths' 4import { VideoPathManager } from '@server/lib/video-path-manager'
5import { moveToNextState } from '@server/lib/video-state'
5import { UserModel } from '@server/models/user/user' 6import { UserModel } from '@server/models/user/user'
6import { MUser, MUserId, MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/types/models' 7import { VideoJobInfoModel } from '@server/models/video/video-job-info'
8import { MUser, MUserId, MVideo, MVideoFullLight, MVideoWithFile } from '@server/types/models'
7import { 9import {
8 HLSTranscodingPayload, 10 HLSTranscodingPayload,
9 MergeAudioTranscodingPayload, 11 MergeAudioTranscodingPayload,
@@ -16,17 +18,14 @@ import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils'
16import { logger } from '../../../helpers/logger' 18import { logger } from '../../../helpers/logger'
17import { CONFIG } from '../../../initializers/config' 19import { CONFIG } from '../../../initializers/config'
18import { VideoModel } from '../../../models/video/video' 20import { VideoModel } from '../../../models/video/video'
19import { federateVideoIfNeeded } from '../../activitypub/videos'
20import { Notifier } from '../../notifier'
21import { 21import {
22 generateHlsPlaylistResolution, 22 generateHlsPlaylistResolution,
23 mergeAudioVideofile, 23 mergeAudioVideofile,
24 optimizeOriginalVideofile, 24 optimizeOriginalVideofile,
25 transcodeNewWebTorrentResolution 25 transcodeNewWebTorrentResolution
26} from '../../transcoding/video-transcoding' 26} from '../../transcoding/video-transcoding'
27import { JobQueue } from '../job-queue'
28 27
29type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<any> 28type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void>
30 29
31const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { 30const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = {
32 'new-resolution-to-hls': handleHLSJob, 31 'new-resolution-to-hls': handleHLSJob,
@@ -69,15 +68,16 @@ async function handleHLSJob (job: Bull.Job, payload: HLSTranscodingPayload, vide
69 : video.getMaxQualityFile() 68 : video.getMaxQualityFile()
70 69
71 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() 70 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
72 const videoInputPath = getVideoFilePath(videoOrStreamingPlaylist, videoFileInput)
73 71
74 await generateHlsPlaylistResolution({ 72 await VideoPathManager.Instance.makeAvailableVideoFile(videoOrStreamingPlaylist, videoFileInput, videoInputPath => {
75 video, 73 return generateHlsPlaylistResolution({
76 videoInputPath, 74 video,
77 resolution: payload.resolution, 75 videoInputPath,
78 copyCodecs: payload.copyCodecs, 76 resolution: payload.resolution,
79 isPortraitMode: payload.isPortraitMode || false, 77 copyCodecs: payload.copyCodecs,
80 job 78 isPortraitMode: payload.isPortraitMode || false,
79 job
80 })
81 }) 81 })
82 82
83 await retryTransactionWrapper(onHlsPlaylistGeneration, video, user, payload) 83 await retryTransactionWrapper(onHlsPlaylistGeneration, video, user, payload)
@@ -101,7 +101,7 @@ async function handleWebTorrentMergeAudioJob (job: Bull.Job, payload: MergeAudio
101} 101}
102 102
103async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { 103async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) {
104 const transcodeType = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job) 104 const { transcodeType } = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job)
105 105
106 await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType, user) 106 await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType, user)
107} 107}
@@ -121,10 +121,18 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay
121 video.VideoFiles = [] 121 video.VideoFiles = []
122 122
123 // Create HLS new resolution jobs 123 // Create HLS new resolution jobs
124 await createLowerResolutionsJobs(video, user, payload.resolution, payload.isPortraitMode, 'hls') 124 await createLowerResolutionsJobs({
125 video,
126 user,
127 videoFileResolution: payload.resolution,
128 isPortraitMode: payload.isPortraitMode,
129 isNewVideo: payload.isNewVideo ?? true,
130 type: 'hls'
131 })
125 } 132 }
126 133
127 return publishAndFederateIfNeeded(video) 134 await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
135 await moveToNextState(video, payload.isNewVideo)
128} 136}
129 137
130async function onVideoFileOptimizer ( 138async function onVideoFileOptimizer (
@@ -143,58 +151,54 @@ async function onVideoFileOptimizer (
143 // Video does not exist anymore 151 // Video does not exist anymore
144 if (!videoDatabase) return undefined 152 if (!videoDatabase) return undefined
145 153
146 let videoPublished = false
147
148 // Generate HLS version of the original file 154 // Generate HLS version of the original file
149 const originalFileHLSPayload = Object.assign({}, payload, { 155 const originalFileHLSPayload = {
156 ...payload,
157
150 isPortraitMode, 158 isPortraitMode,
151 resolution: videoDatabase.getMaxQualityFile().resolution, 159 resolution: videoDatabase.getMaxQualityFile().resolution,
152 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues 160 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
153 copyCodecs: transcodeType !== 'quick-transcode', 161 copyCodecs: transcodeType !== 'quick-transcode',
154 isMaxQuality: true 162 isMaxQuality: true
155 }) 163 }
156 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) 164 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
165 const hasNewResolutions = await createLowerResolutionsJobs({
166 video: videoDatabase,
167 user,
168 videoFileResolution: resolution,
169 isPortraitMode,
170 type: 'webtorrent',
171 isNewVideo: payload.isNewVideo ?? true
172 })
157 173
158 const hasNewResolutions = await createLowerResolutionsJobs(videoDatabase, user, resolution, isPortraitMode, 'webtorrent') 174 await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode')
159 175
176 // Move to next state if there are no other resolutions to generate
160 if (!hasHls && !hasNewResolutions) { 177 if (!hasHls && !hasNewResolutions) {
161 // No transcoding to do, it's now published 178 await moveToNextState(videoDatabase, payload.isNewVideo)
162 videoPublished = await videoDatabase.publishIfNeededAndSave(undefined)
163 } 179 }
164
165 await federateVideoIfNeeded(videoDatabase, payload.isNewVideo)
166
167 if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase)
168 if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase)
169} 180}
170 181
171async function onNewWebTorrentFileResolution ( 182async function onNewWebTorrentFileResolution (
172 video: MVideoUUID, 183 video: MVideo,
173 user: MUserId, 184 user: MUserId,
174 payload: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload 185 payload: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload
175) { 186) {
176 await publishAndFederateIfNeeded(video) 187 await createHlsJobIfEnabled(user, { ...payload, copyCodecs: true, isMaxQuality: false })
188 await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
177 189
178 await createHlsJobIfEnabled(user, Object.assign({}, payload, { copyCodecs: true, isMaxQuality: false })) 190 await moveToNextState(video, payload.isNewVideo)
179} 191}
180 192
181// ---------------------------------------------------------------------------
182
183export {
184 processVideoTranscoding,
185 onNewWebTorrentFileResolution
186}
187
188// ---------------------------------------------------------------------------
189
190async function createHlsJobIfEnabled (user: MUserId, payload: { 193async function createHlsJobIfEnabled (user: MUserId, payload: {
191 videoUUID: string 194 videoUUID: string
192 resolution: number 195 resolution: number
193 isPortraitMode?: boolean 196 isPortraitMode?: boolean
194 copyCodecs: boolean 197 copyCodecs: boolean
195 isMaxQuality: boolean 198 isMaxQuality: boolean
199 isNewVideo?: boolean
196}) { 200}) {
197 if (!payload || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false 201 if (!payload || CONFIG.TRANSCODING.ENABLED !== true || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false
198 202
199 const jobOptions = { 203 const jobOptions = {
200 priority: await getTranscodingJobPriority(user) 204 priority: await getTranscodingJobPriority(user)
@@ -206,21 +210,35 @@ async function createHlsJobIfEnabled (user: MUserId, payload: {
206 resolution: payload.resolution, 210 resolution: payload.resolution,
207 isPortraitMode: payload.isPortraitMode, 211 isPortraitMode: payload.isPortraitMode,
208 copyCodecs: payload.copyCodecs, 212 copyCodecs: payload.copyCodecs,
209 isMaxQuality: payload.isMaxQuality 213 isMaxQuality: payload.isMaxQuality,
214 isNewVideo: payload.isNewVideo
210 } 215 }
211 216
212 JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload }, jobOptions) 217 await addTranscodingJob(hlsTranscodingPayload, jobOptions)
213 218
214 return true 219 return true
215} 220}
216 221
217async function createLowerResolutionsJobs ( 222// ---------------------------------------------------------------------------
218 video: MVideoFullLight, 223
219 user: MUserId, 224export {
220 videoFileResolution: number, 225 processVideoTranscoding,
221 isPortraitMode: boolean, 226 createHlsJobIfEnabled,
227 onNewWebTorrentFileResolution
228}
229
230// ---------------------------------------------------------------------------
231
232async function createLowerResolutionsJobs (options: {
233 video: MVideoFullLight
234 user: MUserId
235 videoFileResolution: number
236 isPortraitMode: boolean
237 isNewVideo: boolean
222 type: 'hls' | 'webtorrent' 238 type: 'hls' | 'webtorrent'
223) { 239}) {
240 const { video, user, videoFileResolution, isPortraitMode, isNewVideo, type } = options
241
224 // Create transcoding jobs if there are enabled resolutions 242 // Create transcoding jobs if there are enabled resolutions
225 const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod') 243 const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod')
226 const resolutionCreated: number[] = [] 244 const resolutionCreated: number[] = []
@@ -234,7 +252,8 @@ async function createLowerResolutionsJobs (
234 type: 'new-resolution-to-webtorrent', 252 type: 'new-resolution-to-webtorrent',
235 videoUUID: video.uuid, 253 videoUUID: video.uuid,
236 resolution, 254 resolution,
237 isPortraitMode 255 isPortraitMode,
256 isNewVideo
238 } 257 }
239 } 258 }
240 259
@@ -245,7 +264,8 @@ async function createLowerResolutionsJobs (
245 resolution, 264 resolution,
246 isPortraitMode, 265 isPortraitMode,
247 copyCodecs: false, 266 copyCodecs: false,
248 isMaxQuality: false 267 isMaxQuality: false,
268 isNewVideo
249 } 269 }
250 } 270 }
251 271
@@ -257,7 +277,7 @@ async function createLowerResolutionsJobs (
257 priority: await getTranscodingJobPriority(user) 277 priority: await getTranscodingJobPriority(user)
258 } 278 }
259 279
260 JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput }, jobOptions) 280 await addTranscodingJob(dataInput, jobOptions)
261 } 281 }
262 282
263 if (resolutionCreated.length === 0) { 283 if (resolutionCreated.length === 0) {
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 42e8347b1..7a3a1bf82 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -11,6 +11,7 @@ import {
11 EmailPayload, 11 EmailPayload,
12 JobState, 12 JobState,
13 JobType, 13 JobType,
14 MoveObjectStoragePayload,
14 RefreshPayload, 15 RefreshPayload,
15 VideoFileImportPayload, 16 VideoFileImportPayload,
16 VideoImportPayload, 17 VideoImportPayload,
@@ -34,6 +35,7 @@ import { processVideoImport } from './handlers/video-import'
34import { processVideoLiveEnding } from './handlers/video-live-ending' 35import { processVideoLiveEnding } from './handlers/video-live-ending'
35import { processVideoTranscoding } from './handlers/video-transcoding' 36import { processVideoTranscoding } from './handlers/video-transcoding'
36import { processVideosViews } from './handlers/video-views' 37import { processVideosViews } from './handlers/video-views'
38import { processMoveToObjectStorage } from './handlers/move-to-object-storage'
37 39
38type CreateJobArgument = 40type CreateJobArgument =
39 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 41 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -49,9 +51,10 @@ type CreateJobArgument =
49 { type: 'videos-views', payload: {} } | 51 { type: 'videos-views', payload: {} } |
50 { type: 'video-live-ending', payload: VideoLiveEndingPayload } | 52 { type: 'video-live-ending', payload: VideoLiveEndingPayload } |
51 { type: 'actor-keys', payload: ActorKeysPayload } | 53 { type: 'actor-keys', payload: ActorKeysPayload } |
52 { type: 'video-redundancy', payload: VideoRedundancyPayload } 54 { type: 'video-redundancy', payload: VideoRedundancyPayload } |
55 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload }
53 56
54type CreateJobOptions = { 57export type CreateJobOptions = {
55 delay?: number 58 delay?: number
56 priority?: number 59 priority?: number
57} 60}
@@ -70,7 +73,8 @@ const handlers: { [id in JobType]: (job: Bull.Job) => Promise<any> } = {
70 'activitypub-refresher': refreshAPObject, 73 'activitypub-refresher': refreshAPObject,
71 'video-live-ending': processVideoLiveEnding, 74 'video-live-ending': processVideoLiveEnding,
72 'actor-keys': processActorKeys, 75 'actor-keys': processActorKeys,
73 'video-redundancy': processVideoRedundancy 76 'video-redundancy': processVideoRedundancy,
77 'move-to-object-storage': processMoveToObjectStorage
74} 78}
75 79
76const jobTypes: JobType[] = [ 80const jobTypes: JobType[] = [
@@ -87,7 +91,8 @@ const jobTypes: JobType[] = [
87 'activitypub-refresher', 91 'activitypub-refresher',
88 'video-redundancy', 92 'video-redundancy',
89 'actor-keys', 93 'actor-keys',
90 'video-live-ending' 94 'video-live-ending',
95 'move-to-object-storage'
91] 96]
92 97
93class JobQueue { 98class JobQueue {
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
index 2a429fb33..d7dc841d9 100644
--- a/server/lib/live/live-manager.ts
+++ b/server/lib/live/live-manager.ts
@@ -20,7 +20,7 @@ import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
20import { federateVideoIfNeeded } from '../activitypub/videos' 20import { federateVideoIfNeeded } from '../activitypub/videos'
21import { JobQueue } from '../job-queue' 21import { JobQueue } from '../job-queue'
22import { PeerTubeSocket } from '../peertube-socket' 22import { PeerTubeSocket } from '../peertube-socket'
23import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../video-paths' 23import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../paths'
24import { LiveQuotaStore } from './live-quota-store' 24import { LiveQuotaStore } from './live-quota-store'
25import { LiveSegmentShaStore } from './live-segment-sha-store' 25import { LiveSegmentShaStore } from './live-segment-sha-store'
26import { cleanupLive } from './live-utils' 26import { cleanupLive } from './live-utils'
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts
index e4526c7a5..3bf723b98 100644
--- a/server/lib/live/live-utils.ts
+++ b/server/lib/live/live-utils.ts
@@ -1,7 +1,7 @@
1import { remove } from 'fs-extra' 1import { remove } from 'fs-extra'
2import { basename } from 'path' 2import { basename } from 'path'
3import { MStreamingPlaylist, MVideo } from '@server/types/models' 3import { MStreamingPlaylist, MVideo } from '@server/types/models'
4import { getHLSDirectory } from '../video-paths' 4import { getLiveDirectory } from '../paths'
5 5
6function buildConcatenatedName (segmentOrPlaylistPath: string) { 6function buildConcatenatedName (segmentOrPlaylistPath: string) {
7 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) 7 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/)
@@ -10,7 +10,7 @@ function buildConcatenatedName (segmentOrPlaylistPath: string) {
10} 10}
11 11
12async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { 12async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
13 const hlsDirectory = getHLSDirectory(video) 13 const hlsDirectory = getLiveDirectory(video)
14 14
15 await remove(hlsDirectory) 15 await remove(hlsDirectory)
16 16
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts
index a80abc843..9b5b6c4fc 100644
--- a/server/lib/live/shared/muxing-session.ts
+++ b/server/lib/live/shared/muxing-session.ts
@@ -11,9 +11,9 @@ import { CONFIG } from '@server/initializers/config'
11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' 11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants'
12import { VideoFileModel } from '@server/models/video/video-file' 12import { VideoFileModel } from '@server/models/video/video-file'
13import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' 13import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models'
14import { getLiveDirectory } from '../../paths'
14import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles' 15import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles'
15import { isAbleToUploadVideo } from '../../user' 16import { isAbleToUploadVideo } from '../../user'
16import { getHLSDirectory } from '../../video-paths'
17import { LiveQuotaStore } from '../live-quota-store' 17import { LiveQuotaStore } from '../live-quota-store'
18import { LiveSegmentShaStore } from '../live-segment-sha-store' 18import { LiveSegmentShaStore } from '../live-segment-sha-store'
19import { buildConcatenatedName } from '../live-utils' 19import { buildConcatenatedName } from '../live-utils'
@@ -282,7 +282,7 @@ class MuxingSession extends EventEmitter {
282 } 282 }
283 283
284 private async prepareDirectories () { 284 private async prepareDirectories () {
285 const outPath = getHLSDirectory(this.videoLive.Video) 285 const outPath = getLiveDirectory(this.videoLive.Video)
286 await ensureDir(outPath) 286 await ensureDir(outPath)
287 287
288 const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY) 288 const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY)
diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts
new file mode 100644
index 000000000..8b413a40e
--- /dev/null
+++ b/server/lib/object-storage/index.ts
@@ -0,0 +1,3 @@
1export * from './keys'
2export * from './urls'
3export * from './videos'
diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts
new file mode 100644
index 000000000..519474775
--- /dev/null
+++ b/server/lib/object-storage/keys.ts
@@ -0,0 +1,20 @@
1import { join } from 'path'
2import { MStreamingPlaylist, MVideoUUID } from '@server/types/models'
3
4function generateHLSObjectStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) {
5 return join(generateHLSObjectBaseStorageKey(playlist, video), filename)
6}
7
8function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID) {
9 return playlist.getStringType() + '_' + video.uuid
10}
11
12function generateWebTorrentObjectStorageKey (filename: string) {
13 return filename
14}
15
16export {
17 generateHLSObjectStorageKey,
18 generateHLSObjectBaseStorageKey,
19 generateWebTorrentObjectStorageKey
20}
diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts
new file mode 100644
index 000000000..c9a614593
--- /dev/null
+++ b/server/lib/object-storage/shared/client.ts
@@ -0,0 +1,56 @@
1import { S3Client } from '@aws-sdk/client-s3'
2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config'
4import { lTags } from './logger'
5
6let endpointParsed: URL
7function getEndpointParsed () {
8 if (endpointParsed) return endpointParsed
9
10 endpointParsed = new URL(getEndpoint())
11
12 return endpointParsed
13}
14
15let s3Client: S3Client
16function getClient () {
17 if (s3Client) return s3Client
18
19 const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE
20
21 s3Client = new S3Client({
22 endpoint: getEndpoint(),
23 region: OBJECT_STORAGE.REGION,
24 credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID
25 ? {
26 accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID,
27 secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY
28 }
29 : undefined
30 })
31
32 logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags())
33
34 return s3Client
35}
36
37// ---------------------------------------------------------------------------
38
39export {
40 getEndpointParsed,
41 getClient
42}
43
44// ---------------------------------------------------------------------------
45
46let endpoint: string
47function getEndpoint () {
48 if (endpoint) return endpoint
49
50 const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT
51 endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://')
52 ? CONFIG.OBJECT_STORAGE.ENDPOINT
53 : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT
54
55 return endpoint
56}
diff --git a/server/lib/object-storage/shared/index.ts b/server/lib/object-storage/shared/index.ts
new file mode 100644
index 000000000..11e10aa9f
--- /dev/null
+++ b/server/lib/object-storage/shared/index.ts
@@ -0,0 +1,3 @@
1export * from './client'
2export * from './logger'
3export * from './object-storage-helpers'
diff --git a/server/lib/object-storage/shared/logger.ts b/server/lib/object-storage/shared/logger.ts
new file mode 100644
index 000000000..8ab7cbd71
--- /dev/null
+++ b/server/lib/object-storage/shared/logger.ts
@@ -0,0 +1,7 @@
1import { loggerTagsFactory } from '@server/helpers/logger'
2
3const lTags = loggerTagsFactory('object-storage')
4
5export {
6 lTags
7}
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts
new file mode 100644
index 000000000..e23216907
--- /dev/null
+++ b/server/lib/object-storage/shared/object-storage-helpers.ts
@@ -0,0 +1,229 @@
1import { close, createReadStream, createWriteStream, ensureDir, open, ReadStream, stat } from 'fs-extra'
2import { min } from 'lodash'
3import { dirname } from 'path'
4import { Readable } from 'stream'
5import {
6 CompletedPart,
7 CompleteMultipartUploadCommand,
8 CreateMultipartUploadCommand,
9 DeleteObjectCommand,
10 GetObjectCommand,
11 ListObjectsV2Command,
12 PutObjectCommand,
13 UploadPartCommand
14} from '@aws-sdk/client-s3'
15import { pipelinePromise } from '@server/helpers/core-utils'
16import { isArray } from '@server/helpers/custom-validators/misc'
17import { logger } from '@server/helpers/logger'
18import { CONFIG } from '@server/initializers/config'
19import { getPrivateUrl } from '../urls'
20import { getClient } from './client'
21import { lTags } from './logger'
22
23type BucketInfo = {
24 BUCKET_NAME: string
25 PREFIX?: string
26}
27
28async function storeObject (options: {
29 inputPath: string
30 objectStorageKey: string
31 bucketInfo: BucketInfo
32}): Promise<string> {
33 const { inputPath, objectStorageKey, bucketInfo } = options
34
35 logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
36
37 const stats = await stat(inputPath)
38
39 // If bigger than max allowed size we do a multipart upload
40 if (stats.size > CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART) {
41 return multiPartUpload({ inputPath, objectStorageKey, bucketInfo })
42 }
43
44 const fileStream = createReadStream(inputPath)
45 return objectStoragePut({ objectStorageKey, content: fileStream, bucketInfo })
46}
47
48async function removeObject (filename: string, bucketInfo: BucketInfo) {
49 const command = new DeleteObjectCommand({
50 Bucket: bucketInfo.BUCKET_NAME,
51 Key: buildKey(filename, bucketInfo)
52 })
53
54 return getClient().send(command)
55}
56
57async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
58 const s3Client = getClient()
59
60 const commandPrefix = bucketInfo.PREFIX + prefix
61 const listCommand = new ListObjectsV2Command({
62 Bucket: bucketInfo.BUCKET_NAME,
63 Prefix: commandPrefix
64 })
65
66 const listedObjects = await s3Client.send(listCommand)
67
68 // FIXME: use bulk delete when s3ninja will support this operation
69 // const deleteParams = {
70 // Bucket: bucketInfo.BUCKET_NAME,
71 // Delete: { Objects: [] }
72 // }
73
74 if (isArray(listedObjects.Contents) !== true) {
75 const message = `Cannot remove ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.`
76
77 logger.error(message, { response: listedObjects, ...lTags() })
78 throw new Error(message)
79 }
80
81 for (const object of listedObjects.Contents) {
82 const command = new DeleteObjectCommand({
83 Bucket: bucketInfo.BUCKET_NAME,
84 Key: object.Key
85 })
86
87 await s3Client.send(command)
88
89 // FIXME: use bulk delete when s3ninja will support this operation
90 // deleteParams.Delete.Objects.push({ Key: object.Key })
91 }
92
93 // FIXME: use bulk delete when s3ninja will support this operation
94 // const deleteCommand = new DeleteObjectsCommand(deleteParams)
95 // await s3Client.send(deleteCommand)
96
97 // Repeat if not all objects could be listed at once (limit of 1000?)
98 if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo)
99}
100
101async function makeAvailable (options: {
102 key: string
103 destination: string
104 bucketInfo: BucketInfo
105}) {
106 const { key, destination, bucketInfo } = options
107
108 await ensureDir(dirname(options.destination))
109
110 const command = new GetObjectCommand({
111 Bucket: bucketInfo.BUCKET_NAME,
112 Key: buildKey(key, bucketInfo)
113 })
114 const response = await getClient().send(command)
115
116 const file = createWriteStream(destination)
117 await pipelinePromise(response.Body as Readable, file)
118
119 file.close()
120}
121
122function buildKey (key: string, bucketInfo: BucketInfo) {
123 return bucketInfo.PREFIX + key
124}
125
126// ---------------------------------------------------------------------------
127
128export {
129 BucketInfo,
130 buildKey,
131 storeObject,
132 removeObject,
133 removePrefix,
134 makeAvailable
135}
136
137// ---------------------------------------------------------------------------
138
139async function objectStoragePut (options: {
140 objectStorageKey: string
141 content: ReadStream
142 bucketInfo: BucketInfo
143}) {
144 const { objectStorageKey, content, bucketInfo } = options
145
146 const command = new PutObjectCommand({
147 Bucket: bucketInfo.BUCKET_NAME,
148 Key: buildKey(objectStorageKey, bucketInfo),
149 Body: content
150 })
151
152 await getClient().send(command)
153
154 return getPrivateUrl(bucketInfo, objectStorageKey)
155}
156
157async function multiPartUpload (options: {
158 inputPath: string
159 objectStorageKey: string
160 bucketInfo: BucketInfo
161}) {
162 const { objectStorageKey, inputPath, bucketInfo } = options
163
164 const key = buildKey(objectStorageKey, bucketInfo)
165 const s3Client = getClient()
166
167 const statResult = await stat(inputPath)
168
169 const createMultipartCommand = new CreateMultipartUploadCommand({
170 Bucket: bucketInfo.BUCKET_NAME,
171 Key: key
172 })
173 const createResponse = await s3Client.send(createMultipartCommand)
174
175 const fd = await open(inputPath, 'r')
176 let partNumber = 1
177 const parts: CompletedPart[] = []
178 const partSize = CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART
179
180 for (let start = 0; start < statResult.size; start += partSize) {
181 logger.debug(
182 'Uploading part %d of file to %s%s in bucket %s',
183 partNumber, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()
184 )
185
186 // FIXME: Remove when https://github.com/aws/aws-sdk-js-v3/pull/2637 is released
187 // The s3 sdk needs to know the length of the http body beforehand, but doesn't support
188 // streams with start and end set, so it just tries to stat the file in stream.path.
189 // This fails for us because we only want to send part of the file. The stream type
190 // is modified so we can set the byteLength here, which s3 detects because array buffers
191 // have this field set
192 const stream: ReadStream & { byteLength: number } =
193 createReadStream(
194 inputPath,
195 { fd, autoClose: false, start, end: (start + partSize) - 1 }
196 ) as ReadStream & { byteLength: number }
197
198 // Calculate if the part size is more than what's left over, and in that case use left over bytes for byteLength
199 stream.byteLength = min([ statResult.size - start, partSize ])
200
201 const uploadPartCommand = new UploadPartCommand({
202 Bucket: bucketInfo.BUCKET_NAME,
203 Key: key,
204 UploadId: createResponse.UploadId,
205 PartNumber: partNumber,
206 Body: stream
207 })
208 const uploadResponse = await s3Client.send(uploadPartCommand)
209
210 parts.push({ ETag: uploadResponse.ETag, PartNumber: partNumber })
211 partNumber += 1
212 }
213 await close(fd)
214
215 const completeUploadCommand = new CompleteMultipartUploadCommand({
216 Bucket: bucketInfo.BUCKET_NAME,
217 Key: objectStorageKey,
218 UploadId: createResponse.UploadId,
219 MultipartUpload: { Parts: parts }
220 })
221 await s3Client.send(completeUploadCommand)
222
223 logger.debug(
224 'Completed %s%s in bucket %s in %d parts',
225 bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, partNumber - 1, lTags()
226 )
227
228 return getPrivateUrl(bucketInfo, objectStorageKey)
229}
diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts
new file mode 100644
index 000000000..2a889190b
--- /dev/null
+++ b/server/lib/object-storage/urls.ts
@@ -0,0 +1,40 @@
1import { CONFIG } from '@server/initializers/config'
2import { BucketInfo, buildKey, getEndpointParsed } from './shared'
3
4function getPrivateUrl (config: BucketInfo, keyWithoutPrefix: string) {
5 return getBaseUrl(config) + buildKey(keyWithoutPrefix, config)
6}
7
8function getWebTorrentPublicFileUrl (fileUrl: string) {
9 const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL
10 if (!baseUrl) return fileUrl
11
12 return replaceByBaseUrl(fileUrl, baseUrl)
13}
14
15function getHLSPublicFileUrl (fileUrl: string) {
16 const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL
17 if (!baseUrl) return fileUrl
18
19 return replaceByBaseUrl(fileUrl, baseUrl)
20}
21
22export {
23 getPrivateUrl,
24 getWebTorrentPublicFileUrl,
25 replaceByBaseUrl,
26 getHLSPublicFileUrl
27}
28
29// ---------------------------------------------------------------------------
30
31function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) {
32 if (baseUrl) return baseUrl
33
34 return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/`
35}
36
37const regex = new RegExp('https?://[^/]+')
38function replaceByBaseUrl (fileUrl: string, baseUrl: string) {
39 return fileUrl.replace(regex, baseUrl)
40}
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
new file mode 100644
index 000000000..15b8f58d5
--- /dev/null
+++ b/server/lib/object-storage/videos.ts
@@ -0,0 +1,72 @@
1import { join } from 'path'
2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config'
4import { MStreamingPlaylist, MVideoFile, MVideoUUID } from '@server/types/models'
5import { getHLSDirectory } from '../paths'
6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
7import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
8
9function storeHLSFile (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) {
10 const baseHlsDirectory = getHLSDirectory(video)
11
12 return storeObject({
13 inputPath: join(baseHlsDirectory, filename),
14 objectStorageKey: generateHLSObjectStorageKey(playlist, video, filename),
15 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
16 })
17}
18
19function storeWebTorrentFile (filename: string) {
20 return storeObject({
21 inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename),
22 objectStorageKey: generateWebTorrentObjectStorageKey(filename),
23 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS
24 })
25}
26
27function removeHLSObjectStorage (playlist: MStreamingPlaylist, video: MVideoUUID) {
28 return removePrefix(generateHLSObjectBaseStorageKey(playlist, video), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
29}
30
31function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
32 return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS)
33}
34
35async function makeHLSFileAvailable (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string, destination: string) {
36 const key = generateHLSObjectStorageKey(playlist, video, filename)
37
38 logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags())
39
40 await makeAvailable({
41 key,
42 destination,
43 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
44 })
45
46 return destination
47}
48
49async function makeWebTorrentFileAvailable (filename: string, destination: string) {
50 const key = generateWebTorrentObjectStorageKey(filename)
51
52 logger.info('Fetching WebTorrent file %s from object storage to %s.', key, destination, lTags())
53
54 await makeAvailable({
55 key,
56 destination,
57 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS
58 })
59
60 return destination
61}
62
63export {
64 storeWebTorrentFile,
65 storeHLSFile,
66
67 removeHLSObjectStorage,
68 removeWebTorrentObjectStorage,
69
70 makeWebTorrentFileAvailable,
71 makeHLSFileAvailable
72}
diff --git a/server/lib/video-paths.ts b/server/lib/paths.ts
index 1e4382108..434e637c6 100644
--- a/server/lib/video-paths.ts
+++ b/server/lib/paths.ts
@@ -1,9 +1,8 @@
1import { join } from 'path' 1import { join } from 'path'
2import { extractVideo } from '@server/helpers/video'
3import { CONFIG } from '@server/initializers/config'
4import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants'
5import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
6import { buildUUID } from '@server/helpers/uuid' 2import { buildUUID } from '@server/helpers/uuid'
3import { CONFIG } from '@server/initializers/config'
4import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
5import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
7import { removeFragmentedMP4Ext } from '@shared/core-utils' 6import { removeFragmentedMP4Ext } from '@shared/core-utils'
8 7
9// ################## Video file name ################## 8// ################## Video file name ##################
@@ -16,39 +15,18 @@ function generateHLSVideoFilename (resolution: number) {
16 return `${buildUUID()}-${resolution}-fragmented.mp4` 15 return `${buildUUID()}-${resolution}-fragmented.mp4`
17} 16}
18 17
19function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) { 18// ################## Streaming playlist ##################
20 if (videoFile.isHLS()) {
21 const video = extractVideo(videoOrPlaylist)
22
23 return join(getHLSDirectory(video), videoFile.filename)
24 }
25
26 const baseDir = isRedundancy
27 ? CONFIG.STORAGE.REDUNDANCY_DIR
28 : CONFIG.STORAGE.VIDEOS_DIR
29
30 return join(baseDir, videoFile.filename)
31}
32
33// ################## Redundancy ##################
34 19
35function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { 20function getLiveDirectory (video: MVideoUUID) {
36 // Base URL used by our HLS player 21 return getHLSDirectory(video)
37 return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid
38} 22}
39 23
40function generateWebTorrentRedundancyUrl (file: MVideoFile) { 24function getHLSDirectory (video: MVideoUUID) {
41 return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename 25 return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
42} 26}
43 27
44// ################## Streaming playlist ################## 28function getHLSRedundancyDirectory (video: MVideoUUID) {
45 29 return join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
46function getHLSDirectory (video: MVideoUUID, isRedundancy = false) {
47 const baseDir = isRedundancy
48 ? HLS_REDUNDANCY_DIRECTORY
49 : HLS_STREAMING_PLAYLIST_DIRECTORY
50
51 return join(baseDir, video.uuid)
52} 30}
53 31
54function getHlsResolutionPlaylistFilename (videoFilename: string) { 32function getHlsResolutionPlaylistFilename (videoFilename: string) {
@@ -81,36 +59,24 @@ function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVi
81 return uuid + '-' + resolution + extension 59 return uuid + '-' + resolution + extension
82} 60}
83 61
84function getTorrentFilePath (videoFile: MVideoFile) { 62function getFSTorrentFilePath (videoFile: MVideoFile) {
85 return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) 63 return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)
86} 64}
87 65
88// ################## Meta data ##################
89
90function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) {
91 const path = '/api/v1/videos/'
92
93 return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id
94}
95
96// --------------------------------------------------------------------------- 66// ---------------------------------------------------------------------------
97 67
98export { 68export {
99 generateHLSVideoFilename, 69 generateHLSVideoFilename,
100 generateWebTorrentVideoFilename, 70 generateWebTorrentVideoFilename,
101 71
102 getVideoFilePath,
103
104 generateTorrentFileName, 72 generateTorrentFileName,
105 getTorrentFilePath, 73 getFSTorrentFilePath,
106 74
107 getHLSDirectory, 75 getHLSDirectory,
76 getLiveDirectory,
77 getHLSRedundancyDirectory,
78
108 generateHLSMasterPlaylistFilename, 79 generateHLSMasterPlaylistFilename,
109 generateHlsSha256SegmentsFilename, 80 generateHlsSha256SegmentsFilename,
110 getHlsResolutionPlaylistFilename, 81 getHlsResolutionPlaylistFilename
111
112 getLocalVideoFileMetadataUrl,
113
114 generateWebTorrentRedundancyUrl,
115 generateHLSRedundancyUrl
116} 82}
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 137ae53a0..ebfd015b5 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -24,7 +24,7 @@ import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlayli
24import { getOrCreateAPVideo } from '../activitypub/videos' 24import { getOrCreateAPVideo } from '../activitypub/videos'
25import { downloadPlaylistSegments } from '../hls' 25import { downloadPlaylistSegments } from '../hls'
26import { removeVideoRedundancy } from '../redundancy' 26import { removeVideoRedundancy } from '../redundancy'
27import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' 27import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-urls'
28import { AbstractScheduler } from './abstract-scheduler' 28import { AbstractScheduler } from './abstract-scheduler'
29 29
30type CandidateToDuplicate = { 30type CandidateToDuplicate = {
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts
index c08523988..d2384f53c 100644
--- a/server/lib/thumbnail.ts
+++ b/server/lib/thumbnail.ts
@@ -1,5 +1,4 @@
1import { join } from 'path' 1import { join } from 'path'
2
3import { ThumbnailType } from '../../shared/models/videos/thumbnail.type' 2import { ThumbnailType } from '../../shared/models/videos/thumbnail.type'
4import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils' 3import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils'
5import { generateImageFilename, processImage } from '../helpers/image-utils' 4import { generateImageFilename, processImage } from '../helpers/image-utils'
@@ -10,7 +9,7 @@ import { ThumbnailModel } from '../models/video/thumbnail'
10import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' 9import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models'
11import { MThumbnail } from '../types/models/video/thumbnail' 10import { MThumbnail } from '../types/models/video/thumbnail'
12import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' 11import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist'
13import { getVideoFilePath } from './video-paths' 12import { VideoPathManager } from './video-path-manager'
14 13
15type ImageSize = { height?: number, width?: number } 14type ImageSize = { height?: number, width?: number }
16 15
@@ -116,21 +115,22 @@ function generateVideoMiniature (options: {
116}) { 115}) {
117 const { video, videoFile, type } = options 116 const { video, videoFile, type } = options
118 117
119 const input = getVideoFilePath(video, videoFile) 118 return VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, input => {
119 const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type)
120 120
121 const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) 121 const thumbnailCreator = videoFile.isAudio()
122 const thumbnailCreator = videoFile.isAudio() 122 ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true)
123 ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) 123 : () => generateImageFromVideoFile(input, basePath, filename, { height, width })
124 : () => generateImageFromVideoFile(input, basePath, filename, { height, width })
125 124
126 return updateThumbnailFromFunction({ 125 return updateThumbnailFromFunction({
127 thumbnailCreator, 126 thumbnailCreator,
128 filename, 127 filename,
129 height, 128 height,
130 width, 129 width,
131 type, 130 type,
132 automaticallyGenerated: true, 131 automaticallyGenerated: true,
133 existingThumbnail 132 existingThumbnail
133 })
134 }) 134 })
135} 135}
136 136
diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts
index d2a556360..ee228c011 100644
--- a/server/lib/transcoding/video-transcoding.ts
+++ b/server/lib/transcoding/video-transcoding.ts
@@ -4,13 +4,13 @@ import { basename, extname as extnameUtil, join } from 'path'
4import { toEven } from '@server/helpers/core-utils' 4import { toEven } from '@server/helpers/core-utils'
5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
6import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 6import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
7import { VideoResolution } from '../../../shared/models/videos' 7import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
8import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 8import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
9import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils' 9import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils'
10import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' 10import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils'
11import { logger } from '../../helpers/logger' 11import { logger } from '../../helpers/logger'
12import { CONFIG } from '../../initializers/config' 12import { CONFIG } from '../../initializers/config'
13import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' 13import { P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants'
14import { VideoFileModel } from '../../models/video/video-file' 14import { VideoFileModel } from '../../models/video/video-file'
15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' 15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
16import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' 16import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls'
@@ -19,9 +19,9 @@ import {
19 generateHlsSha256SegmentsFilename, 19 generateHlsSha256SegmentsFilename,
20 generateHLSVideoFilename, 20 generateHLSVideoFilename,
21 generateWebTorrentVideoFilename, 21 generateWebTorrentVideoFilename,
22 getHlsResolutionPlaylistFilename, 22 getHlsResolutionPlaylistFilename
23 getVideoFilePath 23} from '../paths'
24} from '../video-paths' 24import { VideoPathManager } from '../video-path-manager'
25import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' 25import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
26 26
27/** 27/**
@@ -32,159 +32,162 @@ import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
32 */ 32 */
33 33
34// Optimize the original video file and replace it. The resolution is not changed. 34// Optimize the original video file and replace it. The resolution is not changed.
35async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) { 35function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) {
36 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 36 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
37 const newExtname = '.mp4' 37 const newExtname = '.mp4'
38 38
39 const videoInputPath = getVideoFilePath(video, inputVideoFile) 39 return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async videoInputPath => {
40 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 40 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
41 41
42 const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) 42 const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath)
43 ? 'quick-transcode' 43 ? 'quick-transcode'
44 : 'video' 44 : 'video'
45 45
46 const resolution = toEven(inputVideoFile.resolution) 46 const resolution = toEven(inputVideoFile.resolution)
47 47
48 const transcodeOptions: TranscodeOptions = { 48 const transcodeOptions: TranscodeOptions = {
49 type: transcodeType, 49 type: transcodeType,
50 50
51 inputPath: videoInputPath, 51 inputPath: videoInputPath,
52 outputPath: videoTranscodedPath, 52 outputPath: videoTranscodedPath,
53 53
54 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 54 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
55 profile: CONFIG.TRANSCODING.PROFILE, 55 profile: CONFIG.TRANSCODING.PROFILE,
56 56
57 resolution, 57 resolution,
58 58
59 job 59 job
60 } 60 }
61 61
62 // Could be very long! 62 // Could be very long!
63 await transcode(transcodeOptions) 63 await transcode(transcodeOptions)
64 64
65 try { 65 try {
66 await remove(videoInputPath) 66 await remove(videoInputPath)
67 67
68 // Important to do this before getVideoFilename() to take in account the new filename 68 // Important to do this before getVideoFilename() to take in account the new filename
69 inputVideoFile.extname = newExtname 69 inputVideoFile.extname = newExtname
70 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) 70 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
71 inputVideoFile.storage = VideoStorage.FILE_SYSTEM
71 72
72 const videoOutputPath = getVideoFilePath(video, inputVideoFile) 73 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
73 74
74 await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 75 const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
75 76
76 return transcodeType 77 return { transcodeType, videoFile }
77 } catch (err) { 78 } catch (err) {
78 // Auto destruction... 79 // Auto destruction...
79 video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) 80 video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err }))
80 81
81 throw err 82 throw err
82 } 83 }
84 })
83} 85}
84 86
85// Transcode the original video file to a lower resolution. 87// Transcode the original video file to a lower resolution
86async function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) { 88// We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed
89function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) {
87 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 90 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
88 const extname = '.mp4' 91 const extname = '.mp4'
89 92
90 // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed 93 return VideoPathManager.Instance.makeAvailableVideoFile(video, video.getMaxQualityFile(), async videoInputPath => {
91 const videoInputPath = getVideoFilePath(video, video.getMaxQualityFile()) 94 const newVideoFile = new VideoFileModel({
95 resolution,
96 extname,
97 filename: generateWebTorrentVideoFilename(resolution, extname),
98 size: 0,
99 videoId: video.id
100 })
92 101
93 const newVideoFile = new VideoFileModel({ 102 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
94 resolution, 103 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
95 extname,
96 filename: generateWebTorrentVideoFilename(resolution, extname),
97 size: 0,
98 videoId: video.id
99 })
100 104
101 const videoOutputPath = getVideoFilePath(video, newVideoFile) 105 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
102 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) 106 ? {
107 type: 'only-audio' as 'only-audio',
103 108
104 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO 109 inputPath: videoInputPath,
105 ? { 110 outputPath: videoTranscodedPath,
106 type: 'only-audio' as 'only-audio',
107 111
108 inputPath: videoInputPath, 112 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
109 outputPath: videoTranscodedPath, 113 profile: CONFIG.TRANSCODING.PROFILE,
110 114
111 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 115 resolution,
112 profile: CONFIG.TRANSCODING.PROFILE,
113 116
114 resolution, 117 job
118 }
119 : {
120 type: 'video' as 'video',
121 inputPath: videoInputPath,
122 outputPath: videoTranscodedPath,
115 123
116 job 124 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
117 } 125 profile: CONFIG.TRANSCODING.PROFILE,
118 : {
119 type: 'video' as 'video',
120 inputPath: videoInputPath,
121 outputPath: videoTranscodedPath,
122 126
123 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 127 resolution,
124 profile: CONFIG.TRANSCODING.PROFILE, 128 isPortraitMode: isPortrait,
125 129
126 resolution, 130 job
127 isPortraitMode: isPortrait, 131 }
128 132
129 job 133 await transcode(transcodeOptions)
130 }
131
132 await transcode(transcodeOptions)
133 134
134 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) 135 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath)
136 })
135} 137}
136 138
137// Merge an image with an audio file to create a video 139// Merge an image with an audio file to create a video
138async function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) { 140function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) {
139 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 141 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
140 const newExtname = '.mp4' 142 const newExtname = '.mp4'
141 143
142 const inputVideoFile = video.getMinQualityFile() 144 const inputVideoFile = video.getMinQualityFile()
143 145
144 const audioInputPath = getVideoFilePath(video, inputVideoFile) 146 return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async audioInputPath => {
145 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 147 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
146 148
147 // If the user updates the video preview during transcoding 149 // If the user updates the video preview during transcoding
148 const previewPath = video.getPreview().getPath() 150 const previewPath = video.getPreview().getPath()
149 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) 151 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
150 await copyFile(previewPath, tmpPreviewPath) 152 await copyFile(previewPath, tmpPreviewPath)
151 153
152 const transcodeOptions = { 154 const transcodeOptions = {
153 type: 'merge-audio' as 'merge-audio', 155 type: 'merge-audio' as 'merge-audio',
154 156
155 inputPath: tmpPreviewPath, 157 inputPath: tmpPreviewPath,
156 outputPath: videoTranscodedPath, 158 outputPath: videoTranscodedPath,
157 159
158 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 160 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
159 profile: CONFIG.TRANSCODING.PROFILE, 161 profile: CONFIG.TRANSCODING.PROFILE,
160 162
161 audioPath: audioInputPath, 163 audioPath: audioInputPath,
162 resolution, 164 resolution,
163 165
164 job 166 job
165 } 167 }
166 168
167 try { 169 try {
168 await transcode(transcodeOptions) 170 await transcode(transcodeOptions)
169 171
170 await remove(audioInputPath) 172 await remove(audioInputPath)
171 await remove(tmpPreviewPath) 173 await remove(tmpPreviewPath)
172 } catch (err) { 174 } catch (err) {
173 await remove(tmpPreviewPath) 175 await remove(tmpPreviewPath)
174 throw err 176 throw err
175 } 177 }
176 178
177 // Important to do this before getVideoFilename() to take in account the new file extension 179 // Important to do this before getVideoFilename() to take in account the new file extension
178 inputVideoFile.extname = newExtname 180 inputVideoFile.extname = newExtname
179 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) 181 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
180 182
181 const videoOutputPath = getVideoFilePath(video, inputVideoFile) 183 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
182 // ffmpeg generated a new video file, so update the video duration 184 // ffmpeg generated a new video file, so update the video duration
183 // See https://trac.ffmpeg.org/ticket/5456 185 // See https://trac.ffmpeg.org/ticket/5456
184 video.duration = await getDurationFromVideoFile(videoTranscodedPath) 186 video.duration = await getDurationFromVideoFile(videoTranscodedPath)
185 await video.save() 187 await video.save()
186 188
187 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 189 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
190 })
188} 191}
189 192
190// Concat TS segments from a live video to a fragmented mp4 HLS playlist 193// Concat TS segments from a live video to a fragmented mp4 HLS playlist
@@ -258,7 +261,7 @@ async function onWebTorrentVideoFileTranscoding (
258 await VideoFileModel.customUpsert(videoFile, 'video', undefined) 261 await VideoFileModel.customUpsert(videoFile, 'video', undefined)
259 video.VideoFiles = await video.$get('VideoFiles') 262 video.VideoFiles = await video.$get('VideoFiles')
260 263
261 return video 264 return { video, videoFile }
262} 265}
263 266
264async function generateHlsPlaylistCommon (options: { 267async function generateHlsPlaylistCommon (options: {
@@ -335,14 +338,13 @@ async function generateHlsPlaylistCommon (options: {
335 videoStreamingPlaylistId: playlist.id 338 videoStreamingPlaylistId: playlist.id
336 }) 339 })
337 340
338 const videoFilePath = getVideoFilePath(playlist, newVideoFile) 341 const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
339 342
340 // Move files from tmp transcoded directory to the appropriate place 343 // Move files from tmp transcoded directory to the appropriate place
341 const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 344 await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
342 await ensureDir(baseHlsDirectory)
343 345
344 // Move playlist file 346 // Move playlist file
345 const resolutionPlaylistPath = join(baseHlsDirectory, resolutionPlaylistFilename) 347 const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
346 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) 348 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
347 // Move video file 349 // Move video file
348 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) 350 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
@@ -355,7 +357,7 @@ async function generateHlsPlaylistCommon (options: {
355 357
356 await createTorrentAndSetInfoHash(playlist, newVideoFile) 358 await createTorrentAndSetInfoHash(playlist, newVideoFile)
357 359
358 await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) 360 const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
359 361
360 const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo 362 const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo
361 playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') 363 playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles')
@@ -368,5 +370,5 @@ async function generateHlsPlaylistCommon (options: {
368 await updateMasterHLSPlaylist(video, playlistWithFiles) 370 await updateMasterHLSPlaylist(video, playlistWithFiles)
369 await updateSha256VODSegments(video, playlistWithFiles) 371 await updateSha256VODSegments(video, playlistWithFiles)
370 372
371 return resolutionPlaylistPath 373 return { resolutionPlaylistPath, videoFile: savedVideoFile }
372} 374}
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts
new file mode 100644
index 000000000..4c5d0c89d
--- /dev/null
+++ b/server/lib/video-path-manager.ts
@@ -0,0 +1,139 @@
1import { remove } from 'fs-extra'
2import { extname, join } from 'path'
3import { buildUUID } from '@server/helpers/uuid'
4import { extractVideo } from '@server/helpers/video'
5import { CONFIG } from '@server/initializers/config'
6import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
7import { VideoStorage } from '@shared/models'
8import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage'
9import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths'
10
11type MakeAvailableCB <T> = (path: string) => Promise<T> | T
12
13class VideoPathManager {
14
15 private static instance: VideoPathManager
16
17 private constructor () {}
18
19 getFSHLSOutputPath (video: MVideoUUID, filename?: string) {
20 const base = getHLSDirectory(video)
21 if (!filename) return base
22
23 return join(base, filename)
24 }
25
26 getFSRedundancyVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
27 if (videoFile.isHLS()) {
28 const video = extractVideo(videoOrPlaylist)
29
30 return join(getHLSRedundancyDirectory(video), videoFile.filename)
31 }
32
33 return join(CONFIG.STORAGE.REDUNDANCY_DIR, videoFile.filename)
34 }
35
36 getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
37 if (videoFile.isHLS()) {
38 const video = extractVideo(videoOrPlaylist)
39
40 return join(getHLSDirectory(video), videoFile.filename)
41 }
42
43 return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename)
44 }
45
46 async makeAvailableVideoFile <T> (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) {
47 if (videoFile.storage === VideoStorage.FILE_SYSTEM) {
48 return this.makeAvailableFactory(
49 () => this.getFSVideoFileOutputPath(videoOrPlaylist, videoFile),
50 false,
51 cb
52 )
53 }
54
55 const destination = this.buildTMPDestination(videoFile.filename)
56
57 if (videoFile.isHLS()) {
58 const video = extractVideo(videoOrPlaylist)
59
60 return this.makeAvailableFactory(
61 () => makeHLSFileAvailable(videoOrPlaylist as MStreamingPlaylistVideo, video, videoFile.filename, destination),
62 true,
63 cb
64 )
65 }
66
67 return this.makeAvailableFactory(
68 () => makeWebTorrentFileAvailable(videoFile.filename, destination),
69 true,
70 cb
71 )
72 }
73
74 async makeAvailableResolutionPlaylistFile <T> (playlist: MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) {
75 const filename = getHlsResolutionPlaylistFilename(videoFile.filename)
76
77 if (videoFile.storage === VideoStorage.FILE_SYSTEM) {
78 return this.makeAvailableFactory(
79 () => join(getHLSDirectory(playlist.Video), filename),
80 false,
81 cb
82 )
83 }
84
85 return this.makeAvailableFactory(
86 () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)),
87 true,
88 cb
89 )
90 }
91
92 async makeAvailablePlaylistFile <T> (playlist: MStreamingPlaylistVideo, filename: string, cb: MakeAvailableCB<T>) {
93 if (playlist.storage === VideoStorage.FILE_SYSTEM) {
94 return this.makeAvailableFactory(
95 () => join(getHLSDirectory(playlist.Video), filename),
96 false,
97 cb
98 )
99 }
100
101 return this.makeAvailableFactory(
102 () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)),
103 true,
104 cb
105 )
106 }
107
108 private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) {
109 let result: T
110
111 const destination = await method()
112
113 try {
114 result = await cb(destination)
115 } catch (err) {
116 if (destination && clean) await remove(destination)
117 throw err
118 }
119
120 if (clean) await remove(destination)
121
122 return result
123 }
124
125 private buildTMPDestination (filename: string) {
126 return join(CONFIG.STORAGE.TMP_DIR, buildUUID() + extname(filename))
127
128 }
129
130 static get Instance () {
131 return this.instance || (this.instance = new this())
132 }
133}
134
135// ---------------------------------------------------------------------------
136
137export {
138 VideoPathManager
139}
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts
new file mode 100644
index 000000000..0613d94bf
--- /dev/null
+++ b/server/lib/video-state.ts
@@ -0,0 +1,99 @@
1import { Transaction } from 'sequelize'
2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config'
4import { sequelizeTypescript } from '@server/initializers/database'
5import { VideoModel } from '@server/models/video/video'
6import { VideoJobInfoModel } from '@server/models/video/video-job-info'
7import { MVideoFullLight, MVideoUUID } from '@server/types/models'
8import { VideoState } from '@shared/models'
9import { federateVideoIfNeeded } from './activitypub/videos'
10import { Notifier } from './notifier'
11import { addMoveToObjectStorageJob } from './video'
12
13function buildNextVideoState (currentState?: VideoState) {
14 if (currentState === VideoState.PUBLISHED) {
15 throw new Error('Video is already in its final state')
16 }
17
18 if (
19 currentState !== VideoState.TO_TRANSCODE &&
20 currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE &&
21 CONFIG.TRANSCODING.ENABLED
22 ) {
23 return VideoState.TO_TRANSCODE
24 }
25
26 if (
27 currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE &&
28 CONFIG.OBJECT_STORAGE.ENABLED
29 ) {
30 return VideoState.TO_MOVE_TO_EXTERNAL_STORAGE
31 }
32
33 return VideoState.PUBLISHED
34}
35
36function moveToNextState (video: MVideoUUID, isNewVideo = true) {
37 return sequelizeTypescript.transaction(async t => {
38 // Maybe the video changed in database, refresh it
39 const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
40 // Video does not exist anymore
41 if (!videoDatabase) return undefined
42
43 // Already in its final state
44 if (videoDatabase.state === VideoState.PUBLISHED) {
45 return federateVideoIfNeeded(videoDatabase, false, t)
46 }
47
48 const newState = buildNextVideoState(videoDatabase.state)
49
50 if (newState === VideoState.PUBLISHED) {
51 return moveToPublishedState(videoDatabase, isNewVideo, t)
52 }
53
54 if (newState === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
55 return moveToExternalStorageState(videoDatabase, isNewVideo, t)
56 }
57 })
58}
59
60// ---------------------------------------------------------------------------
61
62export {
63 buildNextVideoState,
64 moveToNextState
65}
66
67// ---------------------------------------------------------------------------
68
69async function moveToPublishedState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) {
70 logger.info('Publishing video %s.', video.uuid, { tags: [ video.uuid ] })
71
72 const previousState = video.state
73 await video.setNewState(VideoState.PUBLISHED, transaction)
74
75 // If the video was not published, we consider it is a new one for other instances
76 // Live videos are always federated, so it's not a new video
77 await federateVideoIfNeeded(video, isNewVideo, transaction)
78
79 Notifier.Instance.notifyOnNewVideoIfNeeded(video)
80
81 if (previousState === VideoState.TO_TRANSCODE) {
82 Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video)
83 }
84}
85
86async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) {
87 const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction)
88 const pendingTranscode = videoJobInfo?.pendingTranscode || 0
89
90 // We want to wait all transcoding jobs before moving the video on an external storage
91 if (pendingTranscode !== 0) return
92
93 await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, transaction)
94
95 logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] })
96
97 addMoveToObjectStorageJob(video, isNewVideo)
98 .catch(err => logger.error('Cannot add move to object storage job', { err }))
99}
diff --git a/server/lib/video-urls.ts b/server/lib/video-urls.ts
new file mode 100644
index 000000000..64c2c9bf9
--- /dev/null
+++ b/server/lib/video-urls.ts
@@ -0,0 +1,31 @@
1
2import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants'
3import { MStreamingPlaylist, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
4
5// ################## Redundancy ##################
6
7function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) {
8 // Base URL used by our HLS player
9 return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid
10}
11
12function generateWebTorrentRedundancyUrl (file: MVideoFile) {
13 return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename
14}
15
16// ################## Meta data ##################
17
18function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) {
19 const path = '/api/v1/videos/'
20
21 return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id
22}
23
24// ---------------------------------------------------------------------------
25
26export {
27 getLocalVideoFileMetadataUrl,
28
29 generateWebTorrentRedundancyUrl,
30 generateHLSRedundancyUrl
31}
diff --git a/server/lib/video.ts b/server/lib/video.ts
index 61fee4949..0a2b93cc0 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -1,15 +1,13 @@
1import { UploadFiles } from 'express' 1import { UploadFiles } from 'express'
2import { Transaction } from 'sequelize/types' 2import { Transaction } from 'sequelize/types'
3import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' 3import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants'
4import { sequelizeTypescript } from '@server/initializers/database'
5import { TagModel } from '@server/models/video/tag' 4import { TagModel } from '@server/models/video/tag'
6import { VideoModel } from '@server/models/video/video' 5import { VideoModel } from '@server/models/video/video'
6import { VideoJobInfoModel } from '@server/models/video/video-job-info'
7import { FilteredModelAttributes } from '@server/types' 7import { FilteredModelAttributes } from '@server/types'
8import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' 8import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
9import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models' 9import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models'
10import { federateVideoIfNeeded } from './activitypub/videos' 10import { CreateJobOptions, JobQueue } from './job-queue/job-queue'
11import { JobQueue } from './job-queue/job-queue'
12import { Notifier } from './notifier'
13import { updateVideoMiniatureFromExisting } from './thumbnail' 11import { updateVideoMiniatureFromExisting } from './thumbnail'
14 12
15function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { 13function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
@@ -82,29 +80,6 @@ async function setVideoTags (options: {
82 video.Tags = tagInstances 80 video.Tags = tagInstances
83} 81}
84 82
85async function publishAndFederateIfNeeded (video: MVideoUUID, wasLive = false) {
86 const result = await sequelizeTypescript.transaction(async t => {
87 // Maybe the video changed in database, refresh it
88 const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
89 // Video does not exist anymore
90 if (!videoDatabase) return undefined
91
92 // We transcoded the video file in another format, now we can publish it
93 const videoPublished = await videoDatabase.publishIfNeededAndSave(t)
94
95 // If the video was not published, we consider it is a new one for other instances
96 // Live videos are always federated, so it's not a new video
97 await federateVideoIfNeeded(videoDatabase, !wasLive && videoPublished, t)
98
99 return { videoDatabase, videoPublished }
100 })
101
102 if (result?.videoPublished) {
103 Notifier.Instance.notifyOnNewVideoIfNeeded(result.videoDatabase)
104 Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(result.videoDatabase)
105 }
106}
107
108async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) { 83async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) {
109 let dataInput: VideoTranscodingPayload 84 let dataInput: VideoTranscodingPayload
110 85
@@ -127,7 +102,20 @@ async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoF
127 priority: await getTranscodingJobPriority(user) 102 priority: await getTranscodingJobPriority(user)
128 } 103 }
129 104
130 return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput }, jobOptions) 105 return addTranscodingJob(dataInput, jobOptions)
106}
107
108async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions) {
109 await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
110
111 return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: payload }, options)
112}
113
114async function addMoveToObjectStorageJob (video: MVideoUUID, isNewVideo = true) {
115 await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove')
116
117 const dataInput = { videoUUID: video.uuid, isNewVideo }
118 return JobQueue.Instance.createJobWithPromise({ type: 'move-to-object-storage', payload: dataInput })
131} 119}
132 120
133async function getTranscodingJobPriority (user: MUserId) { 121async function getTranscodingJobPriority (user: MUserId) {
@@ -143,9 +131,10 @@ async function getTranscodingJobPriority (user: MUserId) {
143 131
144export { 132export {
145 buildLocalVideoFromReq, 133 buildLocalVideoFromReq,
146 publishAndFederateIfNeeded,
147 buildVideoThumbnailsFromReq, 134 buildVideoThumbnailsFromReq,
148 setVideoTags, 135 setVideoTags,
149 addOptimizeOrMergeAudioJob, 136 addOptimizeOrMergeAudioJob,
137 addTranscodingJob,
138 addMoveToObjectStorageJob,
150 getTranscodingJobPriority 139 getTranscodingJobPriority
151} 140}
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts
index 8a54de3b0..b3c4f390d 100644
--- a/server/models/video/formatter/video-format-utils.ts
+++ b/server/models/video/formatter/video-format-utils.ts
@@ -1,6 +1,6 @@
1import { uuidToShort } from '@server/helpers/uuid' 1import { uuidToShort } from '@server/helpers/uuid'
2import { generateMagnetUri } from '@server/helpers/webtorrent' 2import { generateMagnetUri } from '@server/helpers/webtorrent'
3import { getLocalVideoFileMetadataUrl } from '@server/lib/video-paths' 3import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
4import { VideoFile } from '@shared/models/videos/video-file.model' 4import { VideoFile } from '@shared/models/videos/video-file.model'
5import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects' 5import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects'
6import { Video, VideoDetails } from '../../../../shared/models/videos' 6import { Video, VideoDetails } from '../../../../shared/models/videos'
diff --git a/server/models/video/sql/shared/video-tables.ts b/server/models/video/sql/shared/video-tables.ts
index 742d19099..75823864d 100644
--- a/server/models/video/sql/shared/video-tables.ts
+++ b/server/models/video/sql/shared/video-tables.ts
@@ -87,7 +87,8 @@ export class VideoTables {
87 'fps', 87 'fps',
88 'metadataUrl', 88 'metadataUrl',
89 'videoStreamingPlaylistId', 89 'videoStreamingPlaylistId',
90 'videoId' 90 'videoId',
91 'storage'
91 ] 92 ]
92 } 93 }
93 94
@@ -102,7 +103,8 @@ export class VideoTables {
102 'segmentsSha256Url', 103 'segmentsSha256Url',
103 'videoId', 104 'videoId',
104 'createdAt', 105 'createdAt',
105 'updatedAt' 106 'updatedAt',
107 'storage'
106 ]) 108 ])
107 } 109 }
108 110
@@ -258,7 +260,8 @@ export class VideoTables {
258 'originallyPublishedAt', 260 'originallyPublishedAt',
259 'channelId', 261 'channelId',
260 'createdAt', 262 'createdAt',
261 'updatedAt' 263 'updatedAt',
264 'moveJobsRunning'
262 ] 265 ]
263 } 266 }
264} 267}
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 09fc5288b..627c95763 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -23,9 +23,11 @@ import validator from 'validator'
23import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' 23import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
24import { logger } from '@server/helpers/logger' 24import { logger } from '@server/helpers/logger'
25import { extractVideo } from '@server/helpers/video' 25import { extractVideo } from '@server/helpers/video'
26import { getTorrentFilePath } from '@server/lib/video-paths' 26import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
27import { getFSTorrentFilePath } from '@server/lib/paths'
27import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' 28import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
28import { AttributesOnly } from '@shared/core-utils' 29import { AttributesOnly } from '@shared/core-utils'
30import { VideoStorage } from '@shared/models'
29import { 31import {
30 isVideoFileExtnameValid, 32 isVideoFileExtnameValid,
31 isVideoFileInfoHashValid, 33 isVideoFileInfoHashValid,
@@ -214,6 +216,11 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
214 @Column 216 @Column
215 videoId: number 217 videoId: number
216 218
219 @AllowNull(false)
220 @Default(VideoStorage.FILE_SYSTEM)
221 @Column
222 storage: VideoStorage
223
217 @BelongsTo(() => VideoModel, { 224 @BelongsTo(() => VideoModel, {
218 foreignKey: { 225 foreignKey: {
219 allowNull: true 226 allowNull: true
@@ -273,7 +280,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
273 280
274 static async doesOwnedWebTorrentVideoFileExist (filename: string) { 281 static async doesOwnedWebTorrentVideoFileExist (filename: string) {
275 const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + 282 const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
276 'WHERE "filename" = $filename LIMIT 1' 283 `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1`
277 284
278 return doesExist(query, { filename }) 285 return doesExist(query, { filename })
279 } 286 }
@@ -450,9 +457,20 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
450 return !!this.videoStreamingPlaylistId 457 return !!this.videoStreamingPlaylistId
451 } 458 }
452 459
460 getObjectStorageUrl () {
461 if (this.isHLS()) {
462 return getHLSPublicFileUrl(this.fileUrl)
463 }
464
465 return getWebTorrentPublicFileUrl(this.fileUrl)
466 }
467
453 getFileUrl (video: MVideo) { 468 getFileUrl (video: MVideo) {
454 if (!this.Video) this.Video = video as VideoModel 469 if (this.storage === VideoStorage.OBJECT_STORAGE) {
470 return this.getObjectStorageUrl()
471 }
455 472
473 if (!this.Video) this.Video = video as VideoModel
456 if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) 474 if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
457 475
458 return this.fileUrl 476 return this.fileUrl
@@ -503,7 +521,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
503 removeTorrent () { 521 removeTorrent () {
504 if (!this.torrentFilename) return null 522 if (!this.torrentFilename) return null
505 523
506 const torrentPath = getTorrentFilePath(this) 524 const torrentPath = getFSTorrentFilePath(this)
507 return remove(torrentPath) 525 return remove(torrentPath)
508 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) 526 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
509 } 527 }
diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts
new file mode 100644
index 000000000..7c1fe6734
--- /dev/null
+++ b/server/models/video/video-job-info.ts
@@ -0,0 +1,100 @@
1import { Op, QueryTypes, Transaction } from 'sequelize'
2import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Model, Table, Unique, UpdatedAt } from 'sequelize-typescript'
3import { AttributesOnly } from '@shared/core-utils'
4import { VideoModel } from './video'
5
6@Table({
7 tableName: 'videoJobInfo',
8 indexes: [
9 {
10 fields: [ 'videoId' ],
11 where: {
12 videoId: {
13 [Op.ne]: null
14 }
15 }
16 }
17 ]
18})
19
20export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfoModel>>> {
21 @CreatedAt
22 createdAt: Date
23
24 @UpdatedAt
25 updatedAt: Date
26
27 @AllowNull(false)
28 @Default(0)
29 @IsInt
30 @Column
31 pendingMove: number
32
33 @AllowNull(false)
34 @Default(0)
35 @IsInt
36 @Column
37 pendingTranscode: number
38
39 @ForeignKey(() => VideoModel)
40 @Unique
41 @Column
42 videoId: number
43
44 @BelongsTo(() => VideoModel, {
45 foreignKey: {
46 allowNull: false
47 },
48 onDelete: 'cascade'
49 })
50 Video: VideoModel
51
52 static load (videoId: number, transaction: Transaction) {
53 const where = {
54 videoId
55 }
56
57 return VideoJobInfoModel.findOne({ where, transaction })
58 }
59
60 static async increaseOrCreate (videoUUID: string, column: 'pendingMove' | 'pendingTranscode'): Promise<number> {
61 const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } }
62
63 const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(`
64 INSERT INTO "videoJobInfo" ("videoId", "${column}", "createdAt", "updatedAt")
65 SELECT
66 "video"."id" AS "videoId", 1, NOW(), NOW()
67 FROM
68 "video"
69 WHERE
70 "video"."uuid" = $videoUUID
71 ON CONFLICT ("videoId") DO UPDATE
72 SET
73 "${column}" = "videoJobInfo"."${column}" + 1,
74 "updatedAt" = NOW()
75 RETURNING
76 "${column}"
77 `, options)
78
79 return pendingMove
80 }
81
82 static async decrease (videoUUID: string, column: 'pendingMove' | 'pendingTranscode'): Promise<number> {
83 const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } }
84
85 const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(`
86 UPDATE
87 "videoJobInfo"
88 SET
89 "${column}" = "videoJobInfo"."${column}" - 1,
90 "updatedAt" = NOW()
91 FROM "video"
92 WHERE
93 "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID
94 RETURNING
95 "${column}";
96 `, options)
97
98 return pendingMove
99 }
100}
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index d591a3134..3e9fd97c7 100644
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -1,10 +1,25 @@
1import * as memoizee from 'memoizee' 1import * as memoizee from 'memoizee'
2import { join } from 'path' 2import { join } from 'path'
3import { Op } from 'sequelize' 3import { Op } from 'sequelize'
4import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' 4import {
5 AllowNull,
6 BelongsTo,
7 Column,
8 CreatedAt,
9 DataType,
10 Default,
11 ForeignKey,
12 HasMany,
13 Is,
14 Model,
15 Table,
16 UpdatedAt
17} from 'sequelize-typescript'
18import { getHLSPublicFileUrl } from '@server/lib/object-storage'
5import { VideoFileModel } from '@server/models/video/video-file' 19import { VideoFileModel } from '@server/models/video/video-file'
6import { MStreamingPlaylist, MVideo } from '@server/types/models' 20import { MStreamingPlaylist, MVideo } from '@server/types/models'
7import { AttributesOnly } from '@shared/core-utils' 21import { AttributesOnly } from '@shared/core-utils'
22import { VideoStorage } from '@shared/models'
8import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 23import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
9import { sha1 } from '../../helpers/core-utils' 24import { sha1 } from '../../helpers/core-utils'
10import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 25import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
@@ -81,6 +96,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
81 @Column 96 @Column
82 videoId: number 97 videoId: number
83 98
99 @AllowNull(false)
100 @Default(VideoStorage.FILE_SYSTEM)
101 @Column
102 storage: VideoStorage
103
84 @BelongsTo(() => VideoModel, { 104 @BelongsTo(() => VideoModel, {
85 foreignKey: { 105 foreignKey: {
86 allowNull: false 106 allowNull: false
@@ -185,12 +205,20 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
185 } 205 }
186 206
187 getMasterPlaylistUrl (video: MVideo) { 207 getMasterPlaylistUrl (video: MVideo) {
208 if (this.storage === VideoStorage.OBJECT_STORAGE) {
209 return getHLSPublicFileUrl(this.playlistUrl)
210 }
211
188 if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) 212 if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid)
189 213
190 return this.playlistUrl 214 return this.playlistUrl
191 } 215 }
192 216
193 getSha256SegmentsUrl (video: MVideo) { 217 getSha256SegmentsUrl (video: MVideo) {
218 if (this.storage === VideoStorage.OBJECT_STORAGE) {
219 return getHLSPublicFileUrl(this.segmentsSha256Url)
220 }
221
194 if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive) 222 if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive)
195 223
196 return this.segmentsSha256Url 224 return this.segmentsSha256Url
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 56a5b0e18..874ad168a 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -28,14 +28,16 @@ import { buildNSFWFilter } from '@server/helpers/express-utils'
28import { uuidToShort } from '@server/helpers/uuid' 28import { uuidToShort } from '@server/helpers/uuid'
29import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' 29import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video'
30import { LiveManager } from '@server/lib/live/live-manager' 30import { LiveManager } from '@server/lib/live/live-manager'
31import { getHLSDirectory, getVideoFilePath } from '@server/lib/video-paths' 31import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
32import { getHLSDirectory, getHLSRedundancyDirectory } from '@server/lib/paths'
33import { VideoPathManager } from '@server/lib/video-path-manager'
32import { getServerActor } from '@server/models/application/application' 34import { getServerActor } from '@server/models/application/application'
33import { ModelCache } from '@server/models/model-cache' 35import { ModelCache } from '@server/models/model-cache'
34import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' 36import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils'
35import { VideoFile } from '@shared/models/videos/video-file.model' 37import { VideoFile } from '@shared/models/videos/video-file.model'
36import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared' 38import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared'
37import { VideoObject } from '../../../shared/models/activitypub/objects' 39import { VideoObject } from '../../../shared/models/activitypub/objects'
38import { Video, VideoDetails, VideoRateType } from '../../../shared/models/videos' 40import { Video, VideoDetails, VideoRateType, VideoStorage } from '../../../shared/models/videos'
39import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' 41import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
40import { VideoFilter } from '../../../shared/models/videos/video-query.type' 42import { VideoFilter } from '../../../shared/models/videos/video-query.type'
41import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 43import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
@@ -114,6 +116,7 @@ import { ScopeNames as VideoChannelScopeNames, SummaryOptions, VideoChannelModel
114import { VideoCommentModel } from './video-comment' 116import { VideoCommentModel } from './video-comment'
115import { VideoFileModel } from './video-file' 117import { VideoFileModel } from './video-file'
116import { VideoImportModel } from './video-import' 118import { VideoImportModel } from './video-import'
119import { VideoJobInfoModel } from './video-job-info'
117import { VideoLiveModel } from './video-live' 120import { VideoLiveModel } from './video-live'
118import { VideoPlaylistElementModel } from './video-playlist-element' 121import { VideoPlaylistElementModel } from './video-playlist-element'
119import { VideoShareModel } from './video-share' 122import { VideoShareModel } from './video-share'
@@ -732,6 +735,15 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
732 }) 735 })
733 VideoCaptions: VideoCaptionModel[] 736 VideoCaptions: VideoCaptionModel[]
734 737
738 @HasOne(() => VideoJobInfoModel, {
739 foreignKey: {
740 name: 'videoId',
741 allowNull: false
742 },
743 onDelete: 'cascade'
744 })
745 VideoJobInfo: VideoJobInfoModel
746
735 @BeforeDestroy 747 @BeforeDestroy
736 static async sendDelete (instance: MVideoAccountLight, options) { 748 static async sendDelete (instance: MVideoAccountLight, options) {
737 if (!instance.isOwned()) return undefined 749 if (!instance.isOwned()) return undefined
@@ -1641,9 +1653,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1641 getMaxQualityResolution () { 1653 getMaxQualityResolution () {
1642 const file = this.getMaxQualityFile() 1654 const file = this.getMaxQualityFile()
1643 const videoOrPlaylist = file.getVideoOrStreamingPlaylist() 1655 const videoOrPlaylist = file.getVideoOrStreamingPlaylist()
1644 const originalFilePath = getVideoFilePath(videoOrPlaylist, file)
1645 1656
1646 return getVideoFileResolution(originalFilePath) 1657 return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, file, originalFilePath => {
1658 return getVideoFileResolution(originalFilePath)
1659 })
1647 } 1660 }
1648 1661
1649 getDescriptionAPIPath () { 1662 getDescriptionAPIPath () {
@@ -1673,16 +1686,24 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1673 } 1686 }
1674 1687
1675 removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) { 1688 removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) {
1676 const filePath = getVideoFilePath(this, videoFile, isRedundancy) 1689 const filePath = isRedundancy
1690 ? VideoPathManager.Instance.getFSRedundancyVideoFilePath(this, videoFile)
1691 : VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile)
1677 1692
1678 const promises: Promise<any>[] = [ remove(filePath) ] 1693 const promises: Promise<any>[] = [ remove(filePath) ]
1679 if (!isRedundancy) promises.push(videoFile.removeTorrent()) 1694 if (!isRedundancy) promises.push(videoFile.removeTorrent())
1680 1695
1696 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
1697 promises.push(removeWebTorrentObjectStorage(videoFile))
1698 }
1699
1681 return Promise.all(promises) 1700 return Promise.all(promises)
1682 } 1701 }
1683 1702
1684 async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) { 1703 async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) {
1685 const directoryPath = getHLSDirectory(this, isRedundancy) 1704 const directoryPath = isRedundancy
1705 ? getHLSRedundancyDirectory(this)
1706 : getHLSDirectory(this)
1686 1707
1687 await remove(directoryPath) 1708 await remove(directoryPath)
1688 1709
@@ -1698,6 +1719,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1698 await Promise.all( 1719 await Promise.all(
1699 streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent()) 1720 streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent())
1700 ) 1721 )
1722
1723 if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
1724 await removeHLSObjectStorage(streamingPlaylist, this)
1725 }
1701 } 1726 }
1702 } 1727 }
1703 1728
@@ -1741,16 +1766,16 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1741 this.privacy === VideoPrivacy.INTERNAL 1766 this.privacy === VideoPrivacy.INTERNAL
1742 } 1767 }
1743 1768
1744 async publishIfNeededAndSave (t: Transaction) { 1769 async setNewState (newState: VideoState, transaction: Transaction) {
1745 if (this.state !== VideoState.PUBLISHED) { 1770 if (this.state === newState) throw new Error('Cannot use same state ' + newState)
1746 this.state = VideoState.PUBLISHED 1771
1747 this.publishedAt = new Date() 1772 this.state = newState
1748 await this.save({ transaction: t })
1749 1773
1750 return true 1774 if (this.state === VideoState.PUBLISHED) {
1775 this.publishedAt = new Date()
1751 } 1776 }
1752 1777
1753 return false 1778 await this.save({ transaction })
1754 } 1779 }
1755 1780
1756 getBandwidthBits (videoFile: MVideoFile) { 1781 getBandwidthBits (videoFile: MVideoFile) {
diff --git a/server/tests/api/index.ts b/server/tests/api/index.ts
index b62e2f5f7..19301c0b9 100644
--- a/server/tests/api/index.ts
+++ b/server/tests/api/index.ts
@@ -2,6 +2,7 @@
2import './activitypub' 2import './activitypub'
3import './check-params' 3import './check-params'
4import './moderation' 4import './moderation'
5import './object-storage'
5import './notifications' 6import './notifications'
6import './redundancy' 7import './redundancy'
7import './search' 8import './search'
diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts
index 8f1fb78a5..6c4ea90ca 100644
--- a/server/tests/api/live/live-save-replay.ts
+++ b/server/tests/api/live/live-save-replay.ts
@@ -15,7 +15,9 @@ import {
15 stopFfmpeg, 15 stopFfmpeg,
16 testFfmpegStreamError, 16 testFfmpegStreamError,
17 wait, 17 wait,
18 waitJobs 18 waitJobs,
19 waitUntilLivePublishedOnAllServers,
20 waitUntilLiveSavedOnAllServers
19} from '@shared/extra-utils' 21} from '@shared/extra-utils'
20import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' 22import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models'
21 23
@@ -66,18 +68,6 @@ describe('Save replay setting', function () {
66 } 68 }
67 } 69 }
68 70
69 async function waitUntilLivePublishedOnAllServers (videoId: string) {
70 for (const server of servers) {
71 await server.live.waitUntilPublished({ videoId })
72 }
73 }
74
75 async function waitUntilLiveSavedOnAllServers (videoId: string) {
76 for (const server of servers) {
77 await server.live.waitUntilSaved({ videoId })
78 }
79 }
80
81 before(async function () { 71 before(async function () {
82 this.timeout(120000) 72 this.timeout(120000)
83 73
@@ -127,7 +117,7 @@ describe('Save replay setting', function () {
127 117
128 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 118 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
129 119
130 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 120 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
131 121
132 await waitJobs(servers) 122 await waitJobs(servers)
133 123
@@ -160,7 +150,7 @@ describe('Save replay setting', function () {
160 150
161 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 151 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
162 152
163 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 153 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
164 154
165 await waitJobs(servers) 155 await waitJobs(servers)
166 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) 156 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -189,7 +179,7 @@ describe('Save replay setting', function () {
189 179
190 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 180 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
191 181
192 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 182 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
193 183
194 await waitJobs(servers) 184 await waitJobs(servers)
195 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) 185 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -224,7 +214,7 @@ describe('Save replay setting', function () {
224 this.timeout(20000) 214 this.timeout(20000)
225 215
226 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 216 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
227 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 217 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
228 218
229 await waitJobs(servers) 219 await waitJobs(servers)
230 220
@@ -237,7 +227,7 @@ describe('Save replay setting', function () {
237 227
238 await stopFfmpeg(ffmpegCommand) 228 await stopFfmpeg(ffmpegCommand)
239 229
240 await waitUntilLiveSavedOnAllServers(liveVideoUUID) 230 await waitUntilLiveSavedOnAllServers(servers, liveVideoUUID)
241 await waitJobs(servers) 231 await waitJobs(servers)
242 232
243 // Live has been transcoded 233 // Live has been transcoded
@@ -268,7 +258,7 @@ describe('Save replay setting', function () {
268 liveVideoUUID = await createLiveWrapper(true) 258 liveVideoUUID = await createLiveWrapper(true)
269 259
270 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 260 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
271 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 261 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
272 262
273 await waitJobs(servers) 263 await waitJobs(servers)
274 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) 264 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -296,7 +286,7 @@ describe('Save replay setting', function () {
296 liveVideoUUID = await createLiveWrapper(true) 286 liveVideoUUID = await createLiveWrapper(true)
297 287
298 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 288 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
299 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 289 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
300 290
301 await waitJobs(servers) 291 await waitJobs(servers)
302 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) 292 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
diff --git a/server/tests/api/object-storage/index.ts b/server/tests/api/object-storage/index.ts
new file mode 100644
index 000000000..f319d6ef5
--- /dev/null
+++ b/server/tests/api/object-storage/index.ts
@@ -0,0 +1,3 @@
1export * from './live'
2export * from './video-imports'
3export * from './videos'
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts
new file mode 100644
index 000000000..d3e6777f2
--- /dev/null
+++ b/server/tests/api/object-storage/live.ts
@@ -0,0 +1,136 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import { FfmpegCommand } from 'fluent-ffmpeg'
6import {
7 areObjectStorageTestsDisabled,
8 createMultipleServers,
9 doubleFollow,
10 expectStartWith,
11 killallServers,
12 makeRawRequest,
13 ObjectStorageCommand,
14 PeerTubeServer,
15 setAccessTokensToServers,
16 setDefaultVideoChannel,
17 stopFfmpeg,
18 waitJobs,
19 waitUntilLivePublishedOnAllServers,
20 waitUntilLiveSavedOnAllServers
21} from '@shared/extra-utils'
22import { HttpStatusCode, LiveVideoCreate, VideoFile, VideoPrivacy } from '@shared/models'
23
24const expect = chai.expect
25
26async function createLive (server: PeerTubeServer) {
27 const attributes: LiveVideoCreate = {
28 channelId: server.store.channel.id,
29 privacy: VideoPrivacy.PUBLIC,
30 name: 'my super live',
31 saveReplay: true
32 }
33
34 const { uuid } = await server.live.create({ fields: attributes })
35
36 return uuid
37}
38
39async function checkFiles (files: VideoFile[]) {
40 for (const file of files) {
41 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
42
43 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
44 }
45}
46
47describe('Object storage for lives', function () {
48 if (areObjectStorageTestsDisabled()) return
49
50 let ffmpegCommand: FfmpegCommand
51 let servers: PeerTubeServer[]
52 let videoUUID: string
53
54 before(async function () {
55 this.timeout(120000)
56
57 await ObjectStorageCommand.prepareDefaultBuckets()
58
59 servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultConfig())
60
61 await setAccessTokensToServers(servers)
62 await setDefaultVideoChannel(servers)
63 await doubleFollow(servers[0], servers[1])
64
65 await servers[0].config.enableTranscoding()
66 })
67
68 describe('Without live transcoding', async function () {
69
70 before(async function () {
71 await servers[0].config.enableLive({ transcoding: false })
72
73 videoUUID = await createLive(servers[0])
74 })
75
76 it('Should create a live and save the replay on object storage', async function () {
77 this.timeout(220000)
78
79 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID })
80 await waitUntilLivePublishedOnAllServers(servers, videoUUID)
81
82 await stopFfmpeg(ffmpegCommand)
83
84 await waitUntilLiveSavedOnAllServers(servers, videoUUID)
85 await waitJobs(servers)
86
87 for (const server of servers) {
88 const video = await server.videos.get({ id: videoUUID })
89
90 expect(video.files).to.have.lengthOf(0)
91 expect(video.streamingPlaylists).to.have.lengthOf(1)
92
93 const files = video.streamingPlaylists[0].files
94
95 await checkFiles(files)
96 }
97 })
98 })
99
100 describe('With live transcoding', async function () {
101
102 before(async function () {
103 await servers[0].config.enableLive({ transcoding: true })
104
105 videoUUID = await createLive(servers[0])
106 })
107
108 it('Should import a video and have sent it to object storage', async function () {
109 this.timeout(240000)
110
111 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID })
112 await waitUntilLivePublishedOnAllServers(servers, videoUUID)
113
114 await stopFfmpeg(ffmpegCommand)
115
116 await waitUntilLiveSavedOnAllServers(servers, videoUUID)
117 await waitJobs(servers)
118
119 for (const server of servers) {
120 const video = await server.videos.get({ id: videoUUID })
121
122 expect(video.files).to.have.lengthOf(0)
123 expect(video.streamingPlaylists).to.have.lengthOf(1)
124
125 const files = video.streamingPlaylists[0].files
126 expect(files).to.have.lengthOf(4)
127
128 await checkFiles(files)
129 }
130 })
131 })
132
133 after(async function () {
134 await killallServers(servers)
135 })
136})
diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts
new file mode 100644
index 000000000..efc01f550
--- /dev/null
+++ b/server/tests/api/object-storage/video-imports.ts
@@ -0,0 +1,112 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import {
6 areObjectStorageTestsDisabled,
7 createSingleServer,
8 expectStartWith,
9 FIXTURE_URLS,
10 killallServers,
11 makeRawRequest,
12 ObjectStorageCommand,
13 PeerTubeServer,
14 setAccessTokensToServers,
15 setDefaultVideoChannel,
16 waitJobs
17} from '@shared/extra-utils'
18import { HttpStatusCode, VideoPrivacy } from '@shared/models'
19
20const expect = chai.expect
21
22async function importVideo (server: PeerTubeServer) {
23 const attributes = {
24 name: 'import 2',
25 privacy: VideoPrivacy.PUBLIC,
26 channelId: server.store.channel.id,
27 targetUrl: FIXTURE_URLS.goodVideo720
28 }
29
30 const { video: { uuid } } = await server.imports.importVideo({ attributes })
31
32 return uuid
33}
34
35describe('Object storage for video import', function () {
36 if (areObjectStorageTestsDisabled()) return
37
38 let server: PeerTubeServer
39
40 before(async function () {
41 this.timeout(120000)
42
43 await ObjectStorageCommand.prepareDefaultBuckets()
44
45 server = await createSingleServer(1, ObjectStorageCommand.getDefaultConfig())
46
47 await setAccessTokensToServers([ server ])
48 await setDefaultVideoChannel([ server ])
49
50 await server.config.enableImports()
51 })
52
53 describe('Without transcoding', async function () {
54
55 before(async function () {
56 await server.config.disableTranscoding()
57 })
58
59 it('Should import a video and have sent it to object storage', async function () {
60 this.timeout(120000)
61
62 const uuid = await importVideo(server)
63 await waitJobs(server)
64
65 const video = await server.videos.get({ id: uuid })
66
67 expect(video.files).to.have.lengthOf(1)
68 expect(video.streamingPlaylists).to.have.lengthOf(0)
69
70 const fileUrl = video.files[0].fileUrl
71 expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
72
73 await makeRawRequest(fileUrl, HttpStatusCode.OK_200)
74 })
75 })
76
77 describe('With transcoding', async function () {
78
79 before(async function () {
80 await server.config.enableTranscoding()
81 })
82
83 it('Should import a video and have sent it to object storage', async function () {
84 this.timeout(120000)
85
86 const uuid = await importVideo(server)
87 await waitJobs(server)
88
89 const video = await server.videos.get({ id: uuid })
90
91 expect(video.files).to.have.lengthOf(4)
92 expect(video.streamingPlaylists).to.have.lengthOf(1)
93 expect(video.streamingPlaylists[0].files).to.have.lengthOf(4)
94
95 for (const file of video.files) {
96 expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
97
98 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
99 }
100
101 for (const file of video.streamingPlaylists[0].files) {
102 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
103
104 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
105 }
106 })
107 })
108
109 after(async function () {
110 await killallServers([ server ])
111 })
112})
diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts
new file mode 100644
index 000000000..3958bd3d7
--- /dev/null
+++ b/server/tests/api/object-storage/videos.ts
@@ -0,0 +1,391 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import { merge } from 'lodash'
6import {
7 areObjectStorageTestsDisabled,
8 checkTmpIsEmpty,
9 cleanupTests,
10 createMultipleServers,
11 createSingleServer,
12 doubleFollow,
13 expectStartWith,
14 killallServers,
15 makeRawRequest,
16 MockObjectStorage,
17 ObjectStorageCommand,
18 PeerTubeServer,
19 setAccessTokensToServers,
20 waitJobs,
21 webtorrentAdd
22} from '@shared/extra-utils'
23import { HttpStatusCode, VideoDetails } from '@shared/models'
24
25const expect = chai.expect
26
27async function checkFiles (options: {
28 video: VideoDetails
29
30 baseMockUrl?: string
31
32 playlistBucket: string
33 playlistPrefix?: string
34
35 webtorrentBucket: string
36 webtorrentPrefix?: string
37}) {
38 const {
39 video,
40 playlistBucket,
41 webtorrentBucket,
42 baseMockUrl,
43 playlistPrefix,
44 webtorrentPrefix
45 } = options
46
47 let allFiles = video.files
48
49 for (const file of video.files) {
50 const baseUrl = baseMockUrl
51 ? `${baseMockUrl}/${webtorrentBucket}/`
52 : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/`
53
54 const prefix = webtorrentPrefix || ''
55 const start = baseUrl + prefix
56
57 expectStartWith(file.fileUrl, start)
58
59 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
60 const location = res.headers['location']
61 expectStartWith(location, start)
62
63 await makeRawRequest(location, HttpStatusCode.OK_200)
64 }
65
66 const hls = video.streamingPlaylists[0]
67
68 if (hls) {
69 allFiles = allFiles.concat(hls.files)
70
71 const baseUrl = baseMockUrl
72 ? `${baseMockUrl}/${playlistBucket}/`
73 : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/`
74
75 const prefix = playlistPrefix || ''
76 const start = baseUrl + prefix
77
78 expectStartWith(hls.playlistUrl, start)
79 expectStartWith(hls.segmentsSha256Url, start)
80
81 await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
82
83 const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
84 expect(JSON.stringify(resSha.body)).to.not.throw
85
86 for (const file of hls.files) {
87 expectStartWith(file.fileUrl, start)
88
89 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
90 const location = res.headers['location']
91 expectStartWith(location, start)
92
93 await makeRawRequest(location, HttpStatusCode.OK_200)
94 }
95 }
96
97 for (const file of allFiles) {
98 const torrent = await webtorrentAdd(file.magnetUri, true)
99
100 expect(torrent.files).to.be.an('array')
101 expect(torrent.files.length).to.equal(1)
102 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
103
104 const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
105 expect(res.body).to.have.length.above(100)
106 }
107
108 return allFiles.map(f => f.fileUrl)
109}
110
111function runTestSuite (options: {
112 playlistBucket: string
113 playlistPrefix?: string
114
115 webtorrentBucket: string
116 webtorrentPrefix?: string
117
118 useMockBaseUrl?: boolean
119
120 maxUploadPart?: string
121}) {
122 const mockObjectStorage = new MockObjectStorage()
123 let baseMockUrl: string
124
125 let servers: PeerTubeServer[]
126
127 let keptUrls: string[] = []
128
129 const uuidsToDelete: string[] = []
130 let deletedUrls: string[] = []
131
132 before(async function () {
133 this.timeout(120000)
134
135 const port = await mockObjectStorage.initialize()
136 baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
137
138 await ObjectStorageCommand.createBucket(options.playlistBucket)
139 await ObjectStorageCommand.createBucket(options.webtorrentBucket)
140
141 const config = {
142 object_storage: {
143 enabled: true,
144 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
145 region: ObjectStorageCommand.getRegion(),
146
147 credentials: ObjectStorageCommand.getCredentialsConfig(),
148
149 max_upload_part: options.maxUploadPart || '2MB',
150
151 streaming_playlists: {
152 bucket_name: options.playlistBucket,
153 prefix: options.playlistPrefix,
154 base_url: baseMockUrl
155 ? `${baseMockUrl}/${options.playlistBucket}`
156 : undefined
157 },
158
159 videos: {
160 bucket_name: options.webtorrentBucket,
161 prefix: options.webtorrentPrefix,
162 base_url: baseMockUrl
163 ? `${baseMockUrl}/${options.webtorrentBucket}`
164 : undefined
165 }
166 }
167 }
168
169 servers = await createMultipleServers(2, config)
170
171 await setAccessTokensToServers(servers)
172 await doubleFollow(servers[0], servers[1])
173
174 for (const server of servers) {
175 const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
176 await waitJobs(servers)
177
178 const files = await server.videos.listFiles({ id: uuid })
179 keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
180 }
181 })
182
183 it('Should upload a video and move it to the object storage without transcoding', async function () {
184 this.timeout(20000)
185
186 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
187 uuidsToDelete.push(uuid)
188
189 await waitJobs(servers)
190
191 for (const server of servers) {
192 const video = await server.videos.get({ id: uuid })
193 const files = await checkFiles({ ...options, video, baseMockUrl })
194
195 deletedUrls = deletedUrls.concat(files)
196 }
197 })
198
199 it('Should upload a video and move it to the object storage with transcoding', async function () {
200 this.timeout(40000)
201
202 const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' })
203 uuidsToDelete.push(uuid)
204
205 await waitJobs(servers)
206
207 for (const server of servers) {
208 const video = await server.videos.get({ id: uuid })
209 const files = await checkFiles({ ...options, video, baseMockUrl })
210
211 deletedUrls = deletedUrls.concat(files)
212 }
213 })
214
215 it('Should correctly delete the files', async function () {
216 await servers[0].videos.remove({ id: uuidsToDelete[0] })
217 await servers[1].videos.remove({ id: uuidsToDelete[1] })
218
219 await waitJobs(servers)
220
221 for (const url of deletedUrls) {
222 await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
223 }
224 })
225
226 it('Should have kept other files', async function () {
227 for (const url of keptUrls) {
228 await makeRawRequest(url, HttpStatusCode.OK_200)
229 }
230 })
231
232 it('Should have an empty tmp directory', async function () {
233 for (const server of servers) {
234 await checkTmpIsEmpty(server)
235 }
236 })
237
238 after(async function () {
239 mockObjectStorage.terminate()
240
241 await cleanupTests(servers)
242 })
243}
244
245describe('Object storage for videos', function () {
246 if (areObjectStorageTestsDisabled()) return
247
248 describe('Test config', function () {
249 let server: PeerTubeServer
250
251 const baseConfig = {
252 object_storage: {
253 enabled: true,
254 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
255 region: ObjectStorageCommand.getRegion(),
256
257 credentials: ObjectStorageCommand.getCredentialsConfig(),
258
259 streaming_playlists: {
260 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET
261 },
262
263 videos: {
264 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET
265 }
266 }
267 }
268
269 const badCredentials = {
270 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
271 secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
272 }
273
274 it('Should fail with same bucket names without prefix', function (done) {
275 const config = merge({}, baseConfig, {
276 object_storage: {
277 streaming_playlists: {
278 bucket_name: 'aaa'
279 },
280
281 videos: {
282 bucket_name: 'aaa'
283 }
284 }
285 })
286
287 createSingleServer(1, config)
288 .then(() => done(new Error('Did not throw')))
289 .catch(() => done())
290 })
291
292 it('Should fail with bad credentials', async function () {
293 this.timeout(60000)
294
295 await ObjectStorageCommand.prepareDefaultBuckets()
296
297 const config = merge({}, baseConfig, {
298 object_storage: {
299 credentials: badCredentials
300 }
301 })
302
303 server = await createSingleServer(1, config)
304 await setAccessTokensToServers([ server ])
305
306 const { uuid } = await server.videos.quickUpload({ name: 'video' })
307
308 await waitJobs([ server ], true)
309 const video = await server.videos.get({ id: uuid })
310
311 expectStartWith(video.files[0].fileUrl, server.url)
312
313 await killallServers([ server ])
314 })
315
316 it('Should succeed with credentials from env', async function () {
317 this.timeout(60000)
318
319 await ObjectStorageCommand.prepareDefaultBuckets()
320
321 const config = merge({}, baseConfig, {
322 object_storage: {
323 credentials: {
324 access_key_id: '',
325 secret_access_key: ''
326 }
327 }
328 })
329
330 const goodCredentials = ObjectStorageCommand.getCredentialsConfig()
331
332 server = await createSingleServer(1, config, {
333 env: {
334 AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
335 AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
336 }
337 })
338
339 await setAccessTokensToServers([ server ])
340
341 const { uuid } = await server.videos.quickUpload({ name: 'video' })
342
343 await waitJobs([ server ], true)
344 const video = await server.videos.get({ id: uuid })
345
346 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
347 })
348
349 after(async function () {
350 await killallServers([ server ])
351 })
352 })
353
354 describe('Test simple object storage', function () {
355 runTestSuite({
356 playlistBucket: 'streaming-playlists',
357 webtorrentBucket: 'videos'
358 })
359 })
360
361 describe('Test object storage with prefix', function () {
362 runTestSuite({
363 playlistBucket: 'mybucket',
364 webtorrentBucket: 'mybucket',
365
366 playlistPrefix: 'streaming-playlists_',
367 webtorrentPrefix: 'webtorrent_'
368 })
369 })
370
371 describe('Test object storage with prefix and base URL', function () {
372 runTestSuite({
373 playlistBucket: 'mybucket',
374 webtorrentBucket: 'mybucket',
375
376 playlistPrefix: 'streaming-playlists_',
377 webtorrentPrefix: 'webtorrent_',
378
379 useMockBaseUrl: true
380 })
381 })
382
383 describe('Test object storage with small upload part', function () {
384 runTestSuite({
385 playlistBucket: 'streaming-playlists',
386 webtorrentBucket: 'videos',
387
388 maxUploadPart: '5KB'
389 })
390 })
391})
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index e1a12f5f8..3400b1d9a 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -207,14 +207,14 @@ async function check1PlaylistRedundancies (videoUUID?: string) {
207 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) 207 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
208 } 208 }
209 209
210 const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls' 210 const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls/' + videoUUID
211 const baseUrlSegment = servers[0].url + '/static/redundancy/hls' 211 const baseUrlSegment = servers[0].url + '/static/redundancy/hls/' + videoUUID
212 212
213 const video = await servers[0].videos.get({ id: videoUUID }) 213 const video = await servers[0].videos.get({ id: videoUUID })
214 const hlsPlaylist = video.streamingPlaylists[0] 214 const hlsPlaylist = video.streamingPlaylists[0]
215 215
216 for (const resolution of [ 240, 360, 480, 720 ]) { 216 for (const resolution of [ 240, 360, 480, 720 ]) {
217 await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist }) 217 await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist })
218 } 218 }
219 219
220 const { hlsFilenames } = await ensureSameFilenames(videoUUID) 220 const { hlsFilenames } = await ensureSameFilenames(videoUUID)
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts
index 961f0e617..2c829f532 100644
--- a/server/tests/api/videos/video-hls.ts
+++ b/server/tests/api/videos/video-hls.ts
@@ -5,6 +5,7 @@ import * as chai from 'chai'
5import { basename, join } from 'path' 5import { basename, join } from 'path'
6import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' 6import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
7import { 7import {
8 areObjectStorageTestsDisabled,
8 checkDirectoryIsEmpty, 9 checkDirectoryIsEmpty,
9 checkResolutionsInMasterPlaylist, 10 checkResolutionsInMasterPlaylist,
10 checkSegmentHash, 11 checkSegmentHash,
@@ -12,7 +13,9 @@ import {
12 cleanupTests, 13 cleanupTests,
13 createMultipleServers, 14 createMultipleServers,
14 doubleFollow, 15 doubleFollow,
16 expectStartWith,
15 makeRawRequest, 17 makeRawRequest,
18 ObjectStorageCommand,
16 PeerTubeServer, 19 PeerTubeServer,
17 setAccessTokensToServers, 20 setAccessTokensToServers,
18 waitJobs, 21 waitJobs,
@@ -23,8 +26,19 @@ import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants'
23 26
24const expect = chai.expect 27const expect = chai.expect
25 28
26async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, hlsOnly: boolean, resolutions = [ 240, 360, 480, 720 ]) { 29async function checkHlsPlaylist (options: {
27 for (const server of servers) { 30 servers: PeerTubeServer[]
31 videoUUID: string
32 hlsOnly: boolean
33
34 resolutions?: number[]
35 objectStorageBaseUrl: string
36}) {
37 const { videoUUID, hlsOnly, objectStorageBaseUrl } = options
38
39 const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ]
40
41 for (const server of options.servers) {
28 const videoDetails = await server.videos.get({ id: videoUUID }) 42 const videoDetails = await server.videos.get({ id: videoUUID })
29 const baseUrl = `http://${videoDetails.account.host}` 43 const baseUrl = `http://${videoDetails.account.host}`
30 44
@@ -48,9 +62,15 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h
48 expect(file.torrentUrl).to.match( 62 expect(file.torrentUrl).to.match(
49 new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`) 63 new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`)
50 ) 64 )
51 expect(file.fileUrl).to.match( 65
52 new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`) 66 if (objectStorageBaseUrl) {
53 ) 67 expectStartWith(file.fileUrl, objectStorageBaseUrl)
68 } else {
69 expect(file.fileUrl).to.match(
70 new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`)
71 )
72 }
73
54 expect(file.resolution.label).to.equal(resolution + 'p') 74 expect(file.resolution.label).to.equal(resolution + 'p')
55 75
56 await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200) 76 await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
@@ -80,9 +100,11 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h
80 const file = hlsFiles.find(f => f.resolution.id === resolution) 100 const file = hlsFiles.find(f => f.resolution.id === resolution)
81 const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' 101 const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
82 102
83 const subPlaylist = await server.streamingPlaylists.get({ 103 const url = objectStorageBaseUrl
84 url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}` 104 ? `${objectStorageBaseUrl}hls_${videoUUID}/${playlistName}`
85 }) 105 : `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
106
107 const subPlaylist = await server.streamingPlaylists.get({ url })
86 108
87 expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) 109 expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
88 expect(subPlaylist).to.contain(basename(file.fileUrl)) 110 expect(subPlaylist).to.contain(basename(file.fileUrl))
@@ -90,14 +112,15 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h
90 } 112 }
91 113
92 { 114 {
93 const baseUrlAndPath = baseUrl + '/static/streaming-playlists/hls' 115 const baseUrlAndPath = objectStorageBaseUrl
116 ? objectStorageBaseUrl + 'hls_' + videoUUID
117 : baseUrl + '/static/streaming-playlists/hls/' + videoUUID
94 118
95 for (const resolution of resolutions) { 119 for (const resolution of resolutions) {
96 await checkSegmentHash({ 120 await checkSegmentHash({
97 server, 121 server,
98 baseUrlPlaylist: baseUrlAndPath, 122 baseUrlPlaylist: baseUrlAndPath,
99 baseUrlSegment: baseUrlAndPath, 123 baseUrlSegment: baseUrlAndPath,
100 videoUUID,
101 resolution, 124 resolution,
102 hlsPlaylist 125 hlsPlaylist
103 }) 126 })
@@ -111,7 +134,7 @@ describe('Test HLS videos', function () {
111 let videoUUID = '' 134 let videoUUID = ''
112 let videoAudioUUID = '' 135 let videoAudioUUID = ''
113 136
114 function runTestSuite (hlsOnly: boolean) { 137 function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) {
115 138
116 it('Should upload a video and transcode it to HLS', async function () { 139 it('Should upload a video and transcode it to HLS', async function () {
117 this.timeout(120000) 140 this.timeout(120000)
@@ -121,7 +144,7 @@ describe('Test HLS videos', function () {
121 144
122 await waitJobs(servers) 145 await waitJobs(servers)
123 146
124 await checkHlsPlaylist(servers, videoUUID, hlsOnly) 147 await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl })
125 }) 148 })
126 149
127 it('Should upload an audio file and transcode it to HLS', async function () { 150 it('Should upload an audio file and transcode it to HLS', async function () {
@@ -132,7 +155,13 @@ describe('Test HLS videos', function () {
132 155
133 await waitJobs(servers) 156 await waitJobs(servers)
134 157
135 await checkHlsPlaylist(servers, videoAudioUUID, hlsOnly, [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ]) 158 await checkHlsPlaylist({
159 servers,
160 videoUUID: videoAudioUUID,
161 hlsOnly,
162 resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ],
163 objectStorageBaseUrl
164 })
136 }) 165 })
137 166
138 it('Should update the video', async function () { 167 it('Should update the video', async function () {
@@ -142,7 +171,7 @@ describe('Test HLS videos', function () {
142 171
143 await waitJobs(servers) 172 await waitJobs(servers)
144 173
145 await checkHlsPlaylist(servers, videoUUID, hlsOnly) 174 await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl })
146 }) 175 })
147 176
148 it('Should delete videos', async function () { 177 it('Should delete videos', async function () {
@@ -229,6 +258,22 @@ describe('Test HLS videos', function () {
229 runTestSuite(true) 258 runTestSuite(true)
230 }) 259 })
231 260
261 describe('With object storage enabled', function () {
262 if (areObjectStorageTestsDisabled()) return
263
264 before(async function () {
265 this.timeout(120000)
266
267 const configOverride = ObjectStorageCommand.getDefaultConfig()
268 await ObjectStorageCommand.prepareDefaultBuckets()
269
270 await servers[0].kill()
271 await servers[0].run(configOverride)
272 })
273
274 runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl())
275 })
276
232 after(async function () { 277 after(async function () {
233 await cleanupTests(servers) 278 await cleanupTests(servers)
234 }) 279 })
diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts
index bddcff5e7..9f1b57a2e 100644
--- a/server/tests/cli/create-import-video-file-job.ts
+++ b/server/tests/cli/create-import-video-file-job.ts
@@ -2,8 +2,19 @@
2 2
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils' 5import {
6import { VideoFile } from '@shared/models' 6 areObjectStorageTestsDisabled,
7 cleanupTests,
8 createMultipleServers,
9 doubleFollow,
10 expectStartWith,
11 makeRawRequest,
12 ObjectStorageCommand,
13 PeerTubeServer,
14 setAccessTokensToServers,
15 waitJobs
16} from '@shared/extra-utils'
17import { HttpStatusCode, VideoDetails, VideoFile } from '@shared/models'
7 18
8const expect = chai.expect 19const expect = chai.expect
9 20
@@ -17,22 +28,35 @@ function assertVideoProperties (video: VideoFile, resolution: number, extname: s
17 if (size) expect(video.size).to.equal(size) 28 if (size) expect(video.size).to.equal(size)
18} 29}
19 30
20describe('Test create import video jobs', function () { 31async function checkFiles (video: VideoDetails, objectStorage: boolean) {
21 this.timeout(60000) 32 for (const file of video.files) {
33 if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
22 34
23 let servers: PeerTubeServer[] = [] 35 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
36 }
37}
38
39function runTests (objectStorage: boolean) {
24 let video1UUID: string 40 let video1UUID: string
25 let video2UUID: string 41 let video2UUID: string
26 42
43 let servers: PeerTubeServer[] = []
44
27 before(async function () { 45 before(async function () {
28 this.timeout(90000) 46 this.timeout(90000)
29 47
48 const config = objectStorage
49 ? ObjectStorageCommand.getDefaultConfig()
50 : {}
51
30 // Run server 2 to have transcoding enabled 52 // Run server 2 to have transcoding enabled
31 servers = await createMultipleServers(2) 53 servers = await createMultipleServers(2, config)
32 await setAccessTokensToServers(servers) 54 await setAccessTokensToServers(servers)
33 55
34 await doubleFollow(servers[0], servers[1]) 56 await doubleFollow(servers[0], servers[1])
35 57
58 if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets()
59
36 // Upload two videos for our needs 60 // Upload two videos for our needs
37 { 61 {
38 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video1' } }) 62 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video1' } })
@@ -44,7 +68,6 @@ describe('Test create import video jobs', function () {
44 video2UUID = uuid 68 video2UUID = uuid
45 } 69 }
46 70
47 // Transcoding
48 await waitJobs(servers) 71 await waitJobs(servers)
49 }) 72 })
50 73
@@ -65,6 +88,8 @@ describe('Test create import video jobs', function () {
65 const [ originalVideo, transcodedVideo ] = videoDetails.files 88 const [ originalVideo, transcodedVideo ] = videoDetails.files
66 assertVideoProperties(originalVideo, 720, 'webm', 218910) 89 assertVideoProperties(originalVideo, 720, 'webm', 218910)
67 assertVideoProperties(transcodedVideo, 480, 'webm', 69217) 90 assertVideoProperties(transcodedVideo, 480, 'webm', 69217)
91
92 await checkFiles(videoDetails, objectStorage)
68 } 93 }
69 }) 94 })
70 95
@@ -87,6 +112,8 @@ describe('Test create import video jobs', function () {
87 assertVideoProperties(transcodedVideo420, 480, 'mp4') 112 assertVideoProperties(transcodedVideo420, 480, 'mp4')
88 assertVideoProperties(transcodedVideo320, 360, 'mp4') 113 assertVideoProperties(transcodedVideo320, 360, 'mp4')
89 assertVideoProperties(transcodedVideo240, 240, 'mp4') 114 assertVideoProperties(transcodedVideo240, 240, 'mp4')
115
116 await checkFiles(videoDetails, objectStorage)
90 } 117 }
91 }) 118 })
92 119
@@ -107,10 +134,25 @@ describe('Test create import video jobs', function () {
107 const [ video720, video480 ] = videoDetails.files 134 const [ video720, video480 ] = videoDetails.files
108 assertVideoProperties(video720, 720, 'webm', 942961) 135 assertVideoProperties(video720, 720, 'webm', 942961)
109 assertVideoProperties(video480, 480, 'webm', 69217) 136 assertVideoProperties(video480, 480, 'webm', 69217)
137
138 await checkFiles(videoDetails, objectStorage)
110 } 139 }
111 }) 140 })
112 141
113 after(async function () { 142 after(async function () {
114 await cleanupTests(servers) 143 await cleanupTests(servers)
115 }) 144 })
145}
146
147describe('Test create import video jobs', function () {
148
149 describe('On filesystem', function () {
150 runTests(false)
151 })
152
153 describe('On object storage', function () {
154 if (areObjectStorageTestsDisabled()) return
155
156 runTests(true)
157 })
116}) 158})
diff --git a/server/tests/cli/create-transcoding-job.ts b/server/tests/cli/create-transcoding-job.ts
index df787ccdc..3313a492f 100644
--- a/server/tests/cli/create-transcoding-job.ts
+++ b/server/tests/cli/create-transcoding-job.ts
@@ -2,10 +2,15 @@
2 2
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { HttpStatusCode, VideoFile } from '@shared/models'
5import { 6import {
7 areObjectStorageTestsDisabled,
6 cleanupTests, 8 cleanupTests,
7 createMultipleServers, 9 createMultipleServers,
8 doubleFollow, 10 doubleFollow,
11 expectStartWith,
12 makeRawRequest,
13 ObjectStorageCommand,
9 PeerTubeServer, 14 PeerTubeServer,
10 setAccessTokensToServers, 15 setAccessTokensToServers,
11 waitJobs 16 waitJobs
@@ -13,39 +18,39 @@ import {
13 18
14const expect = chai.expect 19const expect = chai.expect
15 20
16describe('Test create transcoding jobs', function () { 21async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent' | 'playlist') {
17 let servers: PeerTubeServer[] = [] 22 for (const file of files) {
18 const videosUUID: string[] = [] 23 const shouldStartWith = type === 'webtorrent'
24 ? ObjectStorageCommand.getWebTorrentBaseUrl()
25 : ObjectStorageCommand.getPlaylistBaseUrl()
19 26
20 const config = { 27 expectStartWith(file.fileUrl, shouldStartWith)
21 transcoding: { 28
22 enabled: false, 29 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
23 resolutions: {
24 '240p': true,
25 '360p': true,
26 '480p': true,
27 '720p': true,
28 '1080p': true,
29 '1440p': true,
30 '2160p': true
31 },
32 hls: {
33 enabled: false
34 }
35 }
36 } 30 }
31}
32
33function runTests (objectStorage: boolean) {
34 let servers: PeerTubeServer[] = []
35 const videosUUID: string[] = []
37 36
38 before(async function () { 37 before(async function () {
39 this.timeout(60000) 38 this.timeout(60000)
40 39
40 const config = objectStorage
41 ? ObjectStorageCommand.getDefaultConfig()
42 : {}
43
41 // Run server 2 to have transcoding enabled 44 // Run server 2 to have transcoding enabled
42 servers = await createMultipleServers(2) 45 servers = await createMultipleServers(2, config)
43 await setAccessTokensToServers(servers) 46 await setAccessTokensToServers(servers)
44 47
45 await servers[0].config.updateCustomSubConfig({ newConfig: config }) 48 await servers[0].config.disableTranscoding()
46 49
47 await doubleFollow(servers[0], servers[1]) 50 await doubleFollow(servers[0], servers[1])
48 51
52 if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets()
53
49 for (let i = 1; i <= 5; i++) { 54 for (let i = 1; i <= 5; i++) {
50 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) 55 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } })
51 videosUUID.push(uuid) 56 videosUUID.push(uuid)
@@ -81,27 +86,29 @@ describe('Test create transcoding jobs', function () {
81 let infoHashes: { [id: number]: string } 86 let infoHashes: { [id: number]: string }
82 87
83 for (const video of data) { 88 for (const video of data) {
84 const videoDetail = await server.videos.get({ id: video.uuid }) 89 const videoDetails = await server.videos.get({ id: video.uuid })
85 90
86 if (video.uuid === videosUUID[1]) { 91 if (video.uuid === videosUUID[1]) {
87 expect(videoDetail.files).to.have.lengthOf(4) 92 expect(videoDetails.files).to.have.lengthOf(4)
88 expect(videoDetail.streamingPlaylists).to.have.lengthOf(0) 93 expect(videoDetails.streamingPlaylists).to.have.lengthOf(0)
94
95 if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent')
89 96
90 if (!infoHashes) { 97 if (!infoHashes) {
91 infoHashes = {} 98 infoHashes = {}
92 99
93 for (const file of videoDetail.files) { 100 for (const file of videoDetails.files) {
94 infoHashes[file.resolution.id.toString()] = file.magnetUri 101 infoHashes[file.resolution.id.toString()] = file.magnetUri
95 } 102 }
96 } else { 103 } else {
97 for (const resolution of Object.keys(infoHashes)) { 104 for (const resolution of Object.keys(infoHashes)) {
98 const file = videoDetail.files.find(f => f.resolution.id.toString() === resolution) 105 const file = videoDetails.files.find(f => f.resolution.id.toString() === resolution)
99 expect(file.magnetUri).to.equal(infoHashes[resolution]) 106 expect(file.magnetUri).to.equal(infoHashes[resolution])
100 } 107 }
101 } 108 }
102 } else { 109 } else {
103 expect(videoDetail.files).to.have.lengthOf(1) 110 expect(videoDetails.files).to.have.lengthOf(1)
104 expect(videoDetail.streamingPlaylists).to.have.lengthOf(0) 111 expect(videoDetails.streamingPlaylists).to.have.lengthOf(0)
105 } 112 }
106 } 113 }
107 } 114 }
@@ -125,6 +132,8 @@ describe('Test create transcoding jobs', function () {
125 expect(videoDetails.files[1].resolution.id).to.equal(480) 132 expect(videoDetails.files[1].resolution.id).to.equal(480)
126 133
127 expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) 134 expect(videoDetails.streamingPlaylists).to.have.lengthOf(0)
135
136 if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent')
128 } 137 }
129 }) 138 })
130 139
@@ -139,11 +148,15 @@ describe('Test create transcoding jobs', function () {
139 const videoDetails = await server.videos.get({ id: videosUUID[2] }) 148 const videoDetails = await server.videos.get({ id: videosUUID[2] })
140 149
141 expect(videoDetails.files).to.have.lengthOf(1) 150 expect(videoDetails.files).to.have.lengthOf(1)
151 if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent')
152
142 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) 153 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
143 154
144 const files = videoDetails.streamingPlaylists[0].files 155 const files = videoDetails.streamingPlaylists[0].files
145 expect(files).to.have.lengthOf(1) 156 expect(files).to.have.lengthOf(1)
146 expect(files[0].resolution.id).to.equal(480) 157 expect(files[0].resolution.id).to.equal(480)
158
159 if (objectStorage) await checkFilesInObjectStorage(files, 'playlist')
147 } 160 }
148 }) 161 })
149 162
@@ -160,6 +173,8 @@ describe('Test create transcoding jobs', function () {
160 const files = videoDetails.streamingPlaylists[0].files 173 const files = videoDetails.streamingPlaylists[0].files
161 expect(files).to.have.lengthOf(1) 174 expect(files).to.have.lengthOf(1)
162 expect(files[0].resolution.id).to.equal(480) 175 expect(files[0].resolution.id).to.equal(480)
176
177 if (objectStorage) await checkFilesInObjectStorage(files, 'playlist')
163 } 178 }
164 }) 179 })
165 180
@@ -178,15 +193,15 @@ describe('Test create transcoding jobs', function () {
178 193
179 const files = videoDetails.streamingPlaylists[0].files 194 const files = videoDetails.streamingPlaylists[0].files
180 expect(files).to.have.lengthOf(4) 195 expect(files).to.have.lengthOf(4)
196
197 if (objectStorage) await checkFilesInObjectStorage(files, 'playlist')
181 } 198 }
182 }) 199 })
183 200
184 it('Should optimize the video file and generate HLS videos if enabled in config', async function () { 201 it('Should optimize the video file and generate HLS videos if enabled in config', async function () {
185 this.timeout(120000) 202 this.timeout(120000)
186 203
187 config.transcoding.hls.enabled = true 204 await servers[0].config.enableTranscoding()
188 await servers[0].config.updateCustomSubConfig({ newConfig: config })
189
190 await servers[0].cli.execWithEnv(`npm run create-transcoding-job -- -v ${videosUUID[4]}`) 205 await servers[0].cli.execWithEnv(`npm run create-transcoding-job -- -v ${videosUUID[4]}`)
191 206
192 await waitJobs(servers) 207 await waitJobs(servers)
@@ -197,10 +212,28 @@ describe('Test create transcoding jobs', function () {
197 expect(videoDetails.files).to.have.lengthOf(4) 212 expect(videoDetails.files).to.have.lengthOf(4)
198 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) 213 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
199 expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(4) 214 expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(4)
215
216 if (objectStorage) {
217 await checkFilesInObjectStorage(videoDetails.files, 'webtorrent')
218 await checkFilesInObjectStorage(videoDetails.streamingPlaylists[0].files, 'playlist')
219 }
200 } 220 }
201 }) 221 })
202 222
203 after(async function () { 223 after(async function () {
204 await cleanupTests(servers) 224 await cleanupTests(servers)
205 }) 225 })
226}
227
228describe('Test create transcoding jobs', function () {
229
230 describe('On filesystem', function () {
231 runTests(false)
232 })
233
234 describe('On object storage', function () {
235 if (areObjectStorageTestsDisabled()) return
236
237 runTests(true)
238 })
206}) 239})
diff --git a/server/tests/helpers/request.ts b/server/tests/helpers/request.ts
index 7f7873df3..c9a2eb831 100644
--- a/server/tests/helpers/request.ts
+++ b/server/tests/helpers/request.ts
@@ -13,7 +13,7 @@ describe('Request helpers', function () {
13 13
14 it('Should throw an error when the bytes limit is exceeded for request', async function () { 14 it('Should throw an error when the bytes limit is exceeded for request', async function () {
15 try { 15 try {
16 await doRequest(FIXTURE_URLS.video4K, { bodyKBLimit: 3 }) 16 await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 3 })
17 } catch { 17 } catch {
18 return 18 return
19 } 19 }
@@ -23,7 +23,7 @@ describe('Request helpers', function () {
23 23
24 it('Should throw an error when the bytes limit is exceeded for request and save file', async function () { 24 it('Should throw an error when the bytes limit is exceeded for request and save file', async function () {
25 try { 25 try {
26 await doRequestAndSaveToFile(FIXTURE_URLS.video4K, destPath1, { bodyKBLimit: 3 }) 26 await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath1, { bodyKBLimit: 3 })
27 } catch { 27 } catch {
28 28
29 await wait(500) 29 await wait(500)
@@ -35,8 +35,8 @@ describe('Request helpers', function () {
35 }) 35 })
36 36
37 it('Should succeed if the file is below the limit', async function () { 37 it('Should succeed if the file is below the limit', async function () {
38 await doRequest(FIXTURE_URLS.video4K, { bodyKBLimit: 5 }) 38 await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 5 })
39 await doRequestAndSaveToFile(FIXTURE_URLS.video4K, destPath2, { bodyKBLimit: 5 }) 39 await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath2, { bodyKBLimit: 5 })
40 40
41 expect(await pathExists(destPath2)).to.be.true 41 expect(await pathExists(destPath2)).to.be.true
42 }) 42 })