aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/server/debug.ts2
-rw-r--r--server/controllers/api/videos/index.ts2
-rw-r--r--server/controllers/api/videos/token.ts33
-rw-r--r--server/controllers/api/videos/update.ts76
-rw-r--r--server/controllers/download.ts4
-rw-r--r--server/controllers/static.ts31
-rw-r--r--server/helpers/ffmpeg/ffmpeg-vod.ts13
-rw-r--r--server/helpers/upload.ts6
-rw-r--r--server/helpers/webtorrent.ts7
-rw-r--r--server/initializers/constants.ts37
-rw-r--r--server/initializers/installer.ts10
-rw-r--r--server/lib/auth/oauth.ts9
-rw-r--r--server/lib/job-queue/handlers/manage-video-torrent.ts2
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts6
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts22
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts95
-rw-r--r--server/lib/object-storage/videos.ts9
-rw-r--r--server/lib/paths.ts17
-rw-r--r--server/lib/schedulers/update-videos-scheduler.ts66
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts4
-rw-r--r--server/lib/transcoding/transcoding.ts367
-rw-r--r--server/lib/video-path-manager.ts51
-rw-r--r--server/lib/video-privacy.ts96
-rw-r--r--server/lib/video-tokens-manager.ts49
-rw-r--r--server/lib/video.ts61
-rw-r--r--server/middlewares/auth.ts8
-rw-r--r--server/middlewares/validators/index.ts7
-rw-r--r--server/middlewares/validators/shared/videos.ts54
-rw-r--r--server/middlewares/validators/static.ts131
-rw-r--r--server/middlewares/validators/videos/videos.ts33
-rw-r--r--server/models/video/formatter/video-format-utils.ts22
-rw-r--r--server/models/video/video-file.ts29
-rw-r--r--server/models/video/video-streaming-playlist.ts21
-rw-r--r--server/models/video/video.ts24
-rw-r--r--server/tests/api/check-params/index.ts1
-rw-r--r--server/tests/api/check-params/live.ts17
-rw-r--r--server/tests/api/check-params/video-files.ts217
-rw-r--r--server/tests/api/check-params/video-token.ts44
-rw-r--r--server/tests/api/live/live-fast-restream.ts4
-rw-r--r--server/tests/api/live/live.ts13
-rw-r--r--server/tests/api/object-storage/live.ts2
-rw-r--r--server/tests/api/object-storage/video-imports.ts6
-rw-r--r--server/tests/api/object-storage/videos.ts20
-rw-r--r--server/tests/api/redundancy/redundancy.ts2
-rw-r--r--server/tests/api/server/open-telemetry.ts6
-rw-r--r--server/tests/api/transcoding/create-transcoding.ts10
-rw-r--r--server/tests/api/transcoding/hls.ts163
-rw-r--r--server/tests/api/transcoding/index.ts1
-rw-r--r--server/tests/api/transcoding/update-while-transcoding.ts151
-rw-r--r--server/tests/api/videos/index.ts1
-rw-r--r--server/tests/api/videos/video-files.ts2
-rw-r--r--server/tests/api/videos/video-static-file-privacy.ts389
-rw-r--r--server/tests/cli/create-import-video-file-job.ts2
-rw-r--r--server/tests/cli/create-move-video-storage-job.ts4
-rw-r--r--server/tests/cli/create-transcoding-job.ts2
-rw-r--r--server/tests/cli/prune-storage.ts41
-rw-r--r--server/tests/cli/regenerate-thumbnails.ts20
-rw-r--r--server/tests/feeds/feeds.ts2
-rw-r--r--server/tests/plugins/filter-hooks.ts36
-rw-r--r--server/tests/plugins/plugin-helpers.ts6
-rw-r--r--server/tests/shared/streaming-playlists.ts122
-rw-r--r--server/tests/shared/videos.ts6
62 files changed, 1990 insertions, 704 deletions
diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts
index 4e5333782..f3792bfc8 100644
--- a/server/controllers/api/server/debug.ts
+++ b/server/controllers/api/server/debug.ts
@@ -8,6 +8,7 @@ import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
8import { UserRight } from '../../../../shared/models/users' 8import { UserRight } from '../../../../shared/models/users'
9import { authenticate, ensureUserHasRight } from '../../../middlewares' 9import { authenticate, ensureUserHasRight } from '../../../middlewares'
10import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler' 10import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
11import { UpdateVideosScheduler } from '@server/lib/schedulers/update-videos-scheduler'
11 12
12const debugRouter = express.Router() 13const debugRouter = express.Router()
13 14
@@ -45,6 +46,7 @@ async function runCommand (req: express.Request, res: express.Response) {
45 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), 46 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
46 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), 47 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
47 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(), 48 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
49 'process-update-videos-scheduler': () => UpdateVideosScheduler.Instance.execute(),
48 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute() 50 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
49 } 51 }
50 52
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index b301515df..ea081e5ab 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -41,6 +41,7 @@ import { ownershipVideoRouter } from './ownership'
41import { rateVideoRouter } from './rate' 41import { rateVideoRouter } from './rate'
42import { statsRouter } from './stats' 42import { statsRouter } from './stats'
43import { studioRouter } from './studio' 43import { studioRouter } from './studio'
44import { tokenRouter } from './token'
44import { transcodingRouter } from './transcoding' 45import { transcodingRouter } from './transcoding'
45import { updateRouter } from './update' 46import { updateRouter } from './update'
46import { uploadRouter } from './upload' 47import { uploadRouter } from './upload'
@@ -63,6 +64,7 @@ videosRouter.use('/', uploadRouter)
63videosRouter.use('/', updateRouter) 64videosRouter.use('/', updateRouter)
64videosRouter.use('/', filesRouter) 65videosRouter.use('/', filesRouter)
65videosRouter.use('/', transcodingRouter) 66videosRouter.use('/', transcodingRouter)
67videosRouter.use('/', tokenRouter)
66 68
67videosRouter.get('/categories', 69videosRouter.get('/categories',
68 openapiOperationDoc({ operationId: 'getCategories' }), 70 openapiOperationDoc({ operationId: 'getCategories' }),
diff --git a/server/controllers/api/videos/token.ts b/server/controllers/api/videos/token.ts
new file mode 100644
index 000000000..009b6dfb6
--- /dev/null
+++ b/server/controllers/api/videos/token.ts
@@ -0,0 +1,33 @@
1import express from 'express'
2import { VideoTokensManager } from '@server/lib/video-tokens-manager'
3import { VideoToken } from '@shared/models'
4import { asyncMiddleware, authenticate, videosCustomGetValidator } from '../../../middlewares'
5
6const tokenRouter = express.Router()
7
8tokenRouter.post('/:id/token',
9 authenticate,
10 asyncMiddleware(videosCustomGetValidator('only-video')),
11 generateToken
12)
13
14// ---------------------------------------------------------------------------
15
16export {
17 tokenRouter
18}
19
20// ---------------------------------------------------------------------------
21
22function generateToken (req: express.Request, res: express.Response) {
23 const video = res.locals.onlyVideo
24
25 const { token, expires } = VideoTokensManager.Instance.create(video.uuid)
26
27 return res.json({
28 files: {
29 token,
30 expires
31 }
32 } as VideoToken)
33}
diff --git a/server/controllers/api/videos/update.ts b/server/controllers/api/videos/update.ts
index ab1a23d9a..0a910379a 100644
--- a/server/controllers/api/videos/update.ts
+++ b/server/controllers/api/videos/update.ts
@@ -1,12 +1,12 @@
1import express from 'express' 1import express from 'express'
2import { Transaction } from 'sequelize/types' 2import { Transaction } from 'sequelize/types'
3import { changeVideoChannelShare } from '@server/lib/activitypub/share' 3import { changeVideoChannelShare } from '@server/lib/activitypub/share'
4import { CreateJobArgument, JobQueue } from '@server/lib/job-queue' 4import { addVideoJobsAfterUpdate, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
5import { buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' 5import { setVideoPrivacy } from '@server/lib/video-privacy'
6import { openapiOperationDoc } from '@server/middlewares/doc' 6import { openapiOperationDoc } from '@server/middlewares/doc'
7import { FilteredModelAttributes } from '@server/types' 7import { FilteredModelAttributes } from '@server/types'
8import { MVideoFullLight } from '@server/types/models' 8import { MVideoFullLight } from '@server/types/models'
9import { HttpStatusCode, ManageVideoTorrentPayload, VideoUpdate } from '@shared/models' 9import { HttpStatusCode, VideoUpdate } from '@shared/models'
10import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' 10import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
11import { resetSequelizeInstance } from '../../../helpers/database-utils' 11import { resetSequelizeInstance } from '../../../helpers/database-utils'
12import { createReqFiles } from '../../../helpers/express-utils' 12import { createReqFiles } from '../../../helpers/express-utils'
@@ -18,6 +18,7 @@ import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
18import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosUpdateValidator } from '../../../middlewares' 18import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosUpdateValidator } from '../../../middlewares'
19import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' 19import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
20import { VideoModel } from '../../../models/video/video' 20import { VideoModel } from '../../../models/video/video'
21import { VideoPathManager } from '@server/lib/video-path-manager'
21 22
22const lTags = loggerTagsFactory('api', 'video') 23const lTags = loggerTagsFactory('api', 'video')
23const auditLogger = auditLoggerFactory('videos') 24const auditLogger = auditLoggerFactory('videos')
@@ -47,8 +48,8 @@ async function updateVideo (req: express.Request, res: express.Response) {
47 const oldVideoAuditView = new VideoAuditView(videoFromReq.toFormattedDetailsJSON()) 48 const oldVideoAuditView = new VideoAuditView(videoFromReq.toFormattedDetailsJSON())
48 const videoInfoToUpdate: VideoUpdate = req.body 49 const videoInfoToUpdate: VideoUpdate = req.body
49 50
50 const wasConfidentialVideo = videoFromReq.isConfidential()
51 const hadPrivacyForFederation = videoFromReq.hasPrivacyForFederation() 51 const hadPrivacyForFederation = videoFromReq.hasPrivacyForFederation()
52 const oldPrivacy = videoFromReq.privacy
52 53
53 const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ 54 const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
54 video: videoFromReq, 55 video: videoFromReq,
@@ -57,12 +58,13 @@ async function updateVideo (req: express.Request, res: express.Response) {
57 automaticallyGenerated: false 58 automaticallyGenerated: false
58 }) 59 })
59 60
61 const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoFromReq.uuid)
62
60 try { 63 try {
61 const { videoInstanceUpdated, isNewVideo } = await sequelizeTypescript.transaction(async t => { 64 const { videoInstanceUpdated, isNewVideo } = await sequelizeTypescript.transaction(async t => {
62 // Refresh video since thumbnails to prevent concurrent updates 65 // Refresh video since thumbnails to prevent concurrent updates
63 const video = await VideoModel.loadFull(videoFromReq.id, t) 66 const video = await VideoModel.loadFull(videoFromReq.id, t)
64 67
65 const sequelizeOptions = { transaction: t }
66 const oldVideoChannel = video.VideoChannel 68 const oldVideoChannel = video.VideoChannel
67 69
68 const keysToUpdate: (keyof VideoUpdate & FilteredModelAttributes<VideoModel>)[] = [ 70 const keysToUpdate: (keyof VideoUpdate & FilteredModelAttributes<VideoModel>)[] = [
@@ -97,7 +99,7 @@ async function updateVideo (req: express.Request, res: express.Response) {
97 await video.setAsRefreshed(t) 99 await video.setAsRefreshed(t)
98 } 100 }
99 101
100 const videoInstanceUpdated = await video.save(sequelizeOptions) as MVideoFullLight 102 const videoInstanceUpdated = await video.save({ transaction: t }) as MVideoFullLight
101 103
102 // Thumbnail & preview updates? 104 // Thumbnail & preview updates?
103 if (thumbnailModel) await videoInstanceUpdated.addAndSaveThumbnail(thumbnailModel, t) 105 if (thumbnailModel) await videoInstanceUpdated.addAndSaveThumbnail(thumbnailModel, t)
@@ -113,7 +115,9 @@ async function updateVideo (req: express.Request, res: express.Response) {
113 await videoInstanceUpdated.$set('VideoChannel', res.locals.videoChannel, { transaction: t }) 115 await videoInstanceUpdated.$set('VideoChannel', res.locals.videoChannel, { transaction: t })
114 videoInstanceUpdated.VideoChannel = res.locals.videoChannel 116 videoInstanceUpdated.VideoChannel = res.locals.videoChannel
115 117
116 if (hadPrivacyForFederation === true) await changeVideoChannelShare(videoInstanceUpdated, oldVideoChannel, t) 118 if (hadPrivacyForFederation === true) {
119 await changeVideoChannelShare(videoInstanceUpdated, oldVideoChannel, t)
120 }
117 } 121 }
118 122
119 // Schedule an update in the future? 123 // Schedule an update in the future?
@@ -139,7 +143,12 @@ async function updateVideo (req: express.Request, res: express.Response) {
139 143
140 Hooks.runAction('action:api.video.updated', { video: videoInstanceUpdated, body: req.body, req, res }) 144 Hooks.runAction('action:api.video.updated', { video: videoInstanceUpdated, body: req.body, req, res })
141 145
142 await addVideoJobsAfterUpdate({ video: videoInstanceUpdated, videoInfoToUpdate, wasConfidentialVideo, isNewVideo }) 146 await addVideoJobsAfterUpdate({
147 video: videoInstanceUpdated,
148 nameChanged: !!videoInfoToUpdate.name,
149 oldPrivacy,
150 isNewVideo
151 })
143 } catch (err) { 152 } catch (err) {
144 // Force fields we want to update 153 // Force fields we want to update
145 // If the transaction is retried, sequelize will think the object has not changed 154 // If the transaction is retried, sequelize will think the object has not changed
@@ -147,6 +156,8 @@ async function updateVideo (req: express.Request, res: express.Response) {
147 resetSequelizeInstance(videoFromReq, videoFieldsSave) 156 resetSequelizeInstance(videoFromReq, videoFieldsSave)
148 157
149 throw err 158 throw err
159 } finally {
160 videoFileLockReleaser()
150 } 161 }
151 162
152 return res.type('json') 163 return res.type('json')
@@ -164,7 +175,7 @@ async function updateVideoPrivacy (options: {
164 const isNewVideo = videoInstance.isNewVideo(videoInfoToUpdate.privacy) 175 const isNewVideo = videoInstance.isNewVideo(videoInfoToUpdate.privacy)
165 176
166 const newPrivacy = parseInt(videoInfoToUpdate.privacy.toString(), 10) 177 const newPrivacy = parseInt(videoInfoToUpdate.privacy.toString(), 10)
167 videoInstance.setPrivacy(newPrivacy) 178 setVideoPrivacy(videoInstance, newPrivacy)
168 179
169 // Unfederate the video if the new privacy is not compatible with federation 180 // Unfederate the video if the new privacy is not compatible with federation
170 if (hadPrivacyForFederation && !videoInstance.hasPrivacyForFederation()) { 181 if (hadPrivacyForFederation && !videoInstance.hasPrivacyForFederation()) {
@@ -185,50 +196,3 @@ function updateSchedule (videoInstance: MVideoFullLight, videoInfoToUpdate: Vide
185 return ScheduleVideoUpdateModel.deleteByVideoId(videoInstance.id, transaction) 196 return ScheduleVideoUpdateModel.deleteByVideoId(videoInstance.id, transaction)
186 } 197 }
187} 198}
188
189async function addVideoJobsAfterUpdate (options: {
190 video: MVideoFullLight
191 videoInfoToUpdate: VideoUpdate
192 wasConfidentialVideo: boolean
193 isNewVideo: boolean
194}) {
195 const { video, videoInfoToUpdate, wasConfidentialVideo, isNewVideo } = options
196 const jobs: CreateJobArgument[] = []
197
198 if (!video.isLive && videoInfoToUpdate.name) {
199
200 for (const file of (video.VideoFiles || [])) {
201 const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }
202
203 jobs.push({ type: 'manage-video-torrent', payload })
204 }
205
206 const hls = video.getHLSPlaylist()
207
208 for (const file of (hls?.VideoFiles || [])) {
209 const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }
210
211 jobs.push({ type: 'manage-video-torrent', payload })
212 }
213 }
214
215 jobs.push({
216 type: 'federate-video',
217 payload: {
218 videoUUID: video.uuid,
219 isNewVideo
220 }
221 })
222
223 if (wasConfidentialVideo) {
224 jobs.push({
225 type: 'notify',
226 payload: {
227 action: 'new-video',
228 videoUUID: video.uuid
229 }
230 })
231 }
232
233 return JobQueue.Instance.createSequentialJobFlow(...jobs)
234}
diff --git a/server/controllers/download.ts b/server/controllers/download.ts
index a270180c0..abd1df26f 100644
--- a/server/controllers/download.ts
+++ b/server/controllers/download.ts
@@ -7,7 +7,7 @@ import { VideoPathManager } from '@server/lib/video-path-manager'
7import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 7import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
8import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models' 8import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models'
9import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' 9import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants'
10import { asyncMiddleware, videosDownloadValidator } from '../middlewares' 10import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares'
11 11
12const downloadRouter = express.Router() 12const downloadRouter = express.Router()
13 13
@@ -20,12 +20,14 @@ downloadRouter.use(
20 20
21downloadRouter.use( 21downloadRouter.use(
22 STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension', 22 STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension',
23 optionalAuthenticate,
23 asyncMiddleware(videosDownloadValidator), 24 asyncMiddleware(videosDownloadValidator),
24 asyncMiddleware(downloadVideoFile) 25 asyncMiddleware(downloadVideoFile)
25) 26)
26 27
27downloadRouter.use( 28downloadRouter.use(
28 STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + ':id-:resolution([0-9]+)-fragmented.:extension', 29 STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + ':id-:resolution([0-9]+)-fragmented.:extension',
30 optionalAuthenticate,
29 asyncMiddleware(videosDownloadValidator), 31 asyncMiddleware(videosDownloadValidator),
30 asyncMiddleware(downloadHLSVideoFile) 32 asyncMiddleware(downloadHLSVideoFile)
31) 33)
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index 33c429eb1..dc091455a 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -1,20 +1,34 @@
1import cors from 'cors' 1import cors from 'cors'
2import express from 'express' 2import express from 'express'
3import { handleStaticError } from '@server/middlewares' 3import {
4 asyncMiddleware,
5 ensureCanAccessPrivateVideoHLSFiles,
6 ensureCanAccessVideoPrivateWebTorrentFiles,
7 handleStaticError,
8 optionalAuthenticate
9} from '@server/middlewares'
4import { CONFIG } from '../initializers/config' 10import { CONFIG } from '../initializers/config'
5import { HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants' 11import { DIRECTORIES, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants'
6 12
7const staticRouter = express.Router() 13const staticRouter = express.Router()
8 14
9// Cors is very important to let other servers access torrent and video files 15// Cors is very important to let other servers access torrent and video files
10staticRouter.use(cors()) 16staticRouter.use(cors())
11 17
12// Videos path for webseed 18// WebTorrent/Classic videos
19staticRouter.use(
20 STATIC_PATHS.PRIVATE_WEBSEED,
21 optionalAuthenticate,
22 asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles),
23 express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }),
24 handleStaticError
25)
13staticRouter.use( 26staticRouter.use(
14 STATIC_PATHS.WEBSEED, 27 STATIC_PATHS.WEBSEED,
15 express.static(CONFIG.STORAGE.VIDEOS_DIR, { fallthrough: false }), 28 express.static(DIRECTORIES.VIDEOS.PUBLIC, { fallthrough: false }),
16 handleStaticError 29 handleStaticError
17) 30)
31
18staticRouter.use( 32staticRouter.use(
19 STATIC_PATHS.REDUNDANCY, 33 STATIC_PATHS.REDUNDANCY,
20 express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }), 34 express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }),
@@ -23,8 +37,15 @@ staticRouter.use(
23 37
24// HLS 38// HLS
25staticRouter.use( 39staticRouter.use(
40 STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS,
41 optionalAuthenticate,
42 asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles),
43 express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }),
44 handleStaticError
45)
46staticRouter.use(
26 STATIC_PATHS.STREAMING_PLAYLISTS.HLS, 47 STATIC_PATHS.STREAMING_PLAYLISTS.HLS,
27 express.static(HLS_STREAMING_PLAYLIST_DIRECTORY, { fallthrough: false }), 48 express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, { fallthrough: false }),
28 handleStaticError 49 handleStaticError
29) 50)
30 51
diff --git a/server/helpers/ffmpeg/ffmpeg-vod.ts b/server/helpers/ffmpeg/ffmpeg-vod.ts
index 7a81a1313..d84703eb9 100644
--- a/server/helpers/ffmpeg/ffmpeg-vod.ts
+++ b/server/helpers/ffmpeg/ffmpeg-vod.ts
@@ -1,14 +1,15 @@
1import { MutexInterface } from 'async-mutex'
1import { Job } from 'bullmq' 2import { Job } from 'bullmq'
2import { FfmpegCommand } from 'fluent-ffmpeg' 3import { FfmpegCommand } from 'fluent-ffmpeg'
3import { readFile, writeFile } from 'fs-extra' 4import { readFile, writeFile } from 'fs-extra'
4import { dirname } from 'path' 5import { dirname } from 'path'
6import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
5import { pick } from '@shared/core-utils' 7import { pick } from '@shared/core-utils'
6import { AvailableEncoders, VideoResolution } from '@shared/models' 8import { AvailableEncoders, VideoResolution } from '@shared/models'
7import { logger, loggerTagsFactory } from '../logger' 9import { logger, loggerTagsFactory } from '../logger'
8import { getFFmpeg, runCommand } from './ffmpeg-commons' 10import { getFFmpeg, runCommand } from './ffmpeg-commons'
9import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets' 11import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets'
10import { computeFPS, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from './ffprobe-utils' 12import { computeFPS, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from './ffprobe-utils'
11import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
12 13
13const lTags = loggerTagsFactory('ffmpeg') 14const lTags = loggerTagsFactory('ffmpeg')
14 15
@@ -22,6 +23,10 @@ interface BaseTranscodeVODOptions {
22 inputPath: string 23 inputPath: string
23 outputPath: string 24 outputPath: string
24 25
26 // Will be released after the ffmpeg started
27 // To prevent a bug where the input file does not exist anymore when running ffmpeg
28 inputFileMutexReleaser: MutexInterface.Releaser
29
25 availableEncoders: AvailableEncoders 30 availableEncoders: AvailableEncoders
26 profile: string 31 profile: string
27 32
@@ -94,6 +99,12 @@ async function transcodeVOD (options: TranscodeVODOptions) {
94 99
95 command = await builders[options.type](command, options) 100 command = await builders[options.type](command, options)
96 101
102 command.on('start', () => {
103 setTimeout(() => {
104 options.inputFileMutexReleaser()
105 }, 1000)
106 })
107
97 await runCommand({ command, job: options.job }) 108 await runCommand({ command, job: options.job })
98 109
99 await fixHLSPlaylistIfNeeded(options) 110 await fixHLSPlaylistIfNeeded(options)
diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts
index 3cb17edd0..f5f476913 100644
--- a/server/helpers/upload.ts
+++ b/server/helpers/upload.ts
@@ -1,10 +1,10 @@
1import { join } from 'path' 1import { join } from 'path'
2import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants' 2import { DIRECTORIES } from '@server/initializers/constants'
3 3
4function getResumableUploadPath (filename?: string) { 4function getResumableUploadPath (filename?: string) {
5 if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename) 5 if (filename) return join(DIRECTORIES.RESUMABLE_UPLOAD, filename)
6 6
7 return RESUMABLE_UPLOAD_DIRECTORY 7 return DIRECTORIES.RESUMABLE_UPLOAD
8} 8}
9 9
10// --------------------------------------------------------------------------- 10// ---------------------------------------------------------------------------
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 88bdb16b6..6d87c74f7 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -164,7 +164,10 @@ function generateMagnetUri (
164) { 164) {
165 const xs = videoFile.getTorrentUrl() 165 const xs = videoFile.getTorrentUrl()
166 const announce = trackerUrls 166 const announce = trackerUrls
167 let urlList = [ videoFile.getFileUrl(video) ] 167
168 let urlList = video.requiresAuth(video.uuid)
169 ? []
170 : [ videoFile.getFileUrl(video) ]
168 171
169 const redundancies = videoFile.RedundancyVideos 172 const redundancies = videoFile.RedundancyVideos
170 if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) 173 if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl))
@@ -240,6 +243,8 @@ function buildAnnounceList () {
240} 243}
241 244
242function buildUrlList (video: MVideo, videoFile: MVideoFile) { 245function buildUrlList (video: MVideo, videoFile: MVideoFile) {
246 if (video.requiresAuth(video.uuid)) return []
247
243 return [ videoFile.getFileUrl(video) ] 248 return [ videoFile.getFileUrl(video) ]
244} 249}
245 250
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index cab61948a..88bdd07fe 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -662,10 +662,15 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
662// Express static paths (router) 662// Express static paths (router)
663const STATIC_PATHS = { 663const STATIC_PATHS = {
664 THUMBNAILS: '/static/thumbnails/', 664 THUMBNAILS: '/static/thumbnails/',
665
665 WEBSEED: '/static/webseed/', 666 WEBSEED: '/static/webseed/',
667 PRIVATE_WEBSEED: '/static/webseed/private/',
668
666 REDUNDANCY: '/static/redundancy/', 669 REDUNDANCY: '/static/redundancy/',
670
667 STREAMING_PLAYLISTS: { 671 STREAMING_PLAYLISTS: {
668 HLS: '/static/streaming-playlists/hls' 672 HLS: '/static/streaming-playlists/hls',
673 PRIVATE_HLS: '/static/streaming-playlists/hls/private/'
669 } 674 }
670} 675}
671const STATIC_DOWNLOAD_PATHS = { 676const STATIC_DOWNLOAD_PATHS = {
@@ -745,12 +750,32 @@ const LRU_CACHE = {
745 }, 750 },
746 ACTOR_IMAGE_STATIC: { 751 ACTOR_IMAGE_STATIC: {
747 MAX_SIZE: 500 752 MAX_SIZE: 500
753 },
754 STATIC_VIDEO_FILES_RIGHTS_CHECK: {
755 MAX_SIZE: 5000,
756 TTL: parseDurationToMs('10 seconds')
757 },
758 VIDEO_TOKENS: {
759 MAX_SIZE: 100_000,
760 TTL: parseDurationToMs('8 hours')
748 } 761 }
749} 762}
750 763
751const RESUMABLE_UPLOAD_DIRECTORY = join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads') 764const DIRECTORIES = {
752const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls') 765 RESUMABLE_UPLOAD: join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads'),
753const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') 766
767 HLS_STREAMING_PLAYLIST: {
768 PUBLIC: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls'),
769 PRIVATE: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls', 'private')
770 },
771
772 VIDEOS: {
773 PUBLIC: CONFIG.STORAGE.VIDEOS_DIR,
774 PRIVATE: join(CONFIG.STORAGE.VIDEOS_DIR, 'private')
775 },
776
777 HLS_REDUNDANCY: join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
778}
754 779
755const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS 780const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS
756 781
@@ -971,9 +996,8 @@ export {
971 PEERTUBE_VERSION, 996 PEERTUBE_VERSION,
972 LAZY_STATIC_PATHS, 997 LAZY_STATIC_PATHS,
973 SEARCH_INDEX, 998 SEARCH_INDEX,
974 RESUMABLE_UPLOAD_DIRECTORY, 999 DIRECTORIES,
975 RESUMABLE_UPLOAD_SESSION_LIFETIME, 1000 RESUMABLE_UPLOAD_SESSION_LIFETIME,
976 HLS_REDUNDANCY_DIRECTORY,
977 P2P_MEDIA_LOADER_PEER_VERSION, 1001 P2P_MEDIA_LOADER_PEER_VERSION,
978 ACTOR_IMAGES_SIZE, 1002 ACTOR_IMAGES_SIZE,
979 ACCEPT_HEADERS, 1003 ACCEPT_HEADERS,
@@ -1007,7 +1031,6 @@ export {
1007 VIDEO_FILTERS, 1031 VIDEO_FILTERS,
1008 ROUTE_CACHE_LIFETIME, 1032 ROUTE_CACHE_LIFETIME,
1009 SORTABLE_COLUMNS, 1033 SORTABLE_COLUMNS,
1010 HLS_STREAMING_PLAYLIST_DIRECTORY,
1011 JOB_TTL, 1034 JOB_TTL,
1012 DEFAULT_THEME_NAME, 1035 DEFAULT_THEME_NAME,
1013 NSFW_POLICY_TYPES, 1036 NSFW_POLICY_TYPES,
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index b02be9567..f5d8eedf1 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -10,7 +10,7 @@ import { ApplicationModel } from '../models/application/application'
10import { OAuthClientModel } from '../models/oauth/oauth-client' 10import { OAuthClientModel } from '../models/oauth/oauth-client'
11import { applicationExist, clientsExist, usersExist } from './checker-after-init' 11import { applicationExist, clientsExist, usersExist } from './checker-after-init'
12import { CONFIG } from './config' 12import { CONFIG } from './config'
13import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants' 13import { DIRECTORIES, FILES_CACHE, LAST_MIGRATION_VERSION } from './constants'
14import { sequelizeTypescript } from './database' 14import { sequelizeTypescript } from './database'
15 15
16async function installApplication () { 16async function installApplication () {
@@ -92,11 +92,13 @@ function createDirectoriesIfNotExist () {
92 tasks.push(ensureDir(dir)) 92 tasks.push(ensureDir(dir))
93 } 93 }
94 94
95 // Playlist directories 95 tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE))
96 tasks.push(ensureDir(HLS_STREAMING_PLAYLIST_DIRECTORY)) 96 tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC))
97 tasks.push(ensureDir(DIRECTORIES.VIDEOS.PUBLIC))
98 tasks.push(ensureDir(DIRECTORIES.VIDEOS.PRIVATE))
97 99
98 // Resumable upload directory 100 // Resumable upload directory
99 tasks.push(ensureDir(RESUMABLE_UPLOAD_DIRECTORY)) 101 tasks.push(ensureDir(DIRECTORIES.RESUMABLE_UPLOAD))
100 102
101 return Promise.all(tasks) 103 return Promise.all(tasks)
102} 104}
diff --git a/server/lib/auth/oauth.ts b/server/lib/auth/oauth.ts
index 35b05ec5a..bc0d4301f 100644
--- a/server/lib/auth/oauth.ts
+++ b/server/lib/auth/oauth.ts
@@ -95,14 +95,9 @@ async function handleOAuthToken (req: express.Request, options: { refreshTokenAu
95 95
96function handleOAuthAuthenticate ( 96function handleOAuthAuthenticate (
97 req: express.Request, 97 req: express.Request,
98 res: express.Response, 98 res: express.Response
99 authenticateInQuery = false
100) { 99) {
101 const options = authenticateInQuery 100 return oAuthServer.authenticate(new Request(req), new Response(res))
102 ? { allowBearerTokensInQueryString: true }
103 : {}
104
105 return oAuthServer.authenticate(new Request(req), new Response(res), options)
106} 101}
107 102
108export { 103export {
diff --git a/server/lib/job-queue/handlers/manage-video-torrent.ts b/server/lib/job-queue/handlers/manage-video-torrent.ts
index 03aa414c9..425915c96 100644
--- a/server/lib/job-queue/handlers/manage-video-torrent.ts
+++ b/server/lib/job-queue/handlers/manage-video-torrent.ts
@@ -82,7 +82,7 @@ async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) {
82async function loadFileOrLog (videoFileId: number) { 82async function loadFileOrLog (videoFileId: number) {
83 if (!videoFileId) return undefined 83 if (!videoFileId) return undefined
84 84
85 const file = await VideoFileModel.loadWithVideo(videoFileId) 85 const file = await VideoFileModel.load(videoFileId)
86 86
87 if (!file) { 87 if (!file) {
88 logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId) 88 logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId)
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index 28c3d325d..0b68555d1 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -3,10 +3,10 @@ import { remove } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import { logger, loggerTagsFactory } from '@server/helpers/logger' 4import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { updateTorrentMetadata } from '@server/helpers/webtorrent' 5import { updateTorrentMetadata } from '@server/helpers/webtorrent'
6import { CONFIG } from '@server/initializers/config'
7import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' 6import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
8import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage' 7import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage'
9import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' 8import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
9import { VideoPathManager } from '@server/lib/video-path-manager'
10import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' 10import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
11import { VideoModel } from '@server/models/video/video' 11import { VideoModel } from '@server/models/video/video'
12import { VideoJobInfoModel } from '@server/models/video/video-job-info' 12import { VideoJobInfoModel } from '@server/models/video/video-job-info'
@@ -72,9 +72,9 @@ async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
72 for (const file of video.VideoFiles) { 72 for (const file of video.VideoFiles) {
73 if (file.storage !== VideoStorage.FILE_SYSTEM) continue 73 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
74 74
75 const fileUrl = await storeWebTorrentFile(file.filename) 75 const fileUrl = await storeWebTorrentFile(video, file)
76 76
77 const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename) 77 const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
78 await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) 78 await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
79 } 79 }
80} 80}
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 7dbffc955..c6263f55a 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -18,6 +18,7 @@ import { VideoStreamingPlaylistModel } from '@server/models/video/video-streamin
18import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models' 18import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models'
19import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' 19import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
20import { logger, loggerTagsFactory } from '../../../helpers/logger' 20import { logger, loggerTagsFactory } from '../../../helpers/logger'
21import { VideoPathManager } from '@server/lib/video-path-manager'
21 22
22const lTags = loggerTagsFactory('live', 'job') 23const lTags = loggerTagsFactory('live', 'job')
23 24
@@ -205,18 +206,27 @@ async function assignReplayFilesToVideo (options: {
205 const concatenatedTsFiles = await readdir(replayDirectory) 206 const concatenatedTsFiles = await readdir(replayDirectory)
206 207
207 for (const concatenatedTsFile of concatenatedTsFiles) { 208 for (const concatenatedTsFile of concatenatedTsFiles) {
209 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
210
208 const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) 211 const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile)
209 212
210 const probe = await ffprobePromise(concatenatedTsFilePath) 213 const probe = await ffprobePromise(concatenatedTsFilePath)
211 const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe) 214 const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe)
212 const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe) 215 const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe)
213 216
214 await generateHlsPlaylistResolutionFromTS({ 217 try {
215 video, 218 await generateHlsPlaylistResolutionFromTS({
216 concatenatedTsFilePath, 219 video,
217 resolution, 220 inputFileMutexReleaser,
218 isAAC: audioStream?.codec_name === 'aac' 221 concatenatedTsFilePath,
219 }) 222 resolution,
223 isAAC: audioStream?.codec_name === 'aac'
224 })
225 } catch (err) {
226 logger.error('Cannot generate HLS playlist resolution from TS files.', { err })
227 }
228
229 inputFileMutexReleaser()
220 } 230 }
221 231
222 return video 232 return video
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index b0e92acf7..48c675678 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -94,15 +94,24 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV
94 94
95 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() 95 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
96 96
97 await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { 97 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
98 return generateHlsPlaylistResolution({ 98
99 video, 99 try {
100 videoInputPath, 100 await videoFileInput.getVideo().reload()
101 resolution: payload.resolution, 101
102 copyCodecs: payload.copyCodecs, 102 await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
103 job 103 return generateHlsPlaylistResolution({
104 video,
105 videoInputPath,
106 inputFileMutexReleaser,
107 resolution: payload.resolution,
108 copyCodecs: payload.copyCodecs,
109 job
110 })
104 }) 111 })
105 }) 112 } finally {
113 inputFileMutexReleaser()
114 }
106 115
107 logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid)) 116 logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid))
108 117
@@ -177,38 +186,44 @@ async function onVideoFirstWebTorrentTranscoding (
177 transcodeType: TranscodeVODOptionsType, 186 transcodeType: TranscodeVODOptionsType,
178 user: MUserId 187 user: MUserId
179) { 188) {
180 const { resolution, audioStream } = await videoArg.probeMaxQualityFile() 189 const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)
181 190
182 // Maybe the video changed in database, refresh it 191 try {
183 const videoDatabase = await VideoModel.loadFull(videoArg.uuid) 192 // Maybe the video changed in database, refresh it
184 // Video does not exist anymore 193 const videoDatabase = await VideoModel.loadFull(videoArg.uuid)
185 if (!videoDatabase) return undefined 194 // Video does not exist anymore
186 195 if (!videoDatabase) return undefined
187 // Generate HLS version of the original file 196
188 const originalFileHLSPayload = { 197 const { resolution, audioStream } = await videoDatabase.probeMaxQualityFile()
189 ...payload, 198
190 199 // Generate HLS version of the original file
191 hasAudio: !!audioStream, 200 const originalFileHLSPayload = {
192 resolution: videoDatabase.getMaxQualityFile().resolution, 201 ...payload,
193 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues 202
194 copyCodecs: transcodeType !== 'quick-transcode', 203 hasAudio: !!audioStream,
195 isMaxQuality: true 204 resolution: videoDatabase.getMaxQualityFile().resolution,
196 } 205 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
197 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) 206 copyCodecs: transcodeType !== 'quick-transcode',
198 const hasNewResolutions = await createLowerResolutionsJobs({ 207 isMaxQuality: true
199 video: videoDatabase, 208 }
200 user, 209 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
201 videoFileResolution: resolution, 210 const hasNewResolutions = await createLowerResolutionsJobs({
202 hasAudio: !!audioStream, 211 video: videoDatabase,
203 type: 'webtorrent', 212 user,
204 isNewVideo: payload.isNewVideo ?? true 213 videoFileResolution: resolution,
205 }) 214 hasAudio: !!audioStream,
206 215 type: 'webtorrent',
207 await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode') 216 isNewVideo: payload.isNewVideo ?? true
208 217 })
209 // Move to next state if there are no other resolutions to generate 218
210 if (!hasHls && !hasNewResolutions) { 219 await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode')
211 await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo }) 220
221 // Move to next state if there are no other resolutions to generate
222 if (!hasHls && !hasNewResolutions) {
223 await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo })
224 }
225 } finally {
226 mutexReleaser()
212 } 227 }
213} 228}
214 229
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
index 62aae248b..e323baaa2 100644
--- a/server/lib/object-storage/videos.ts
+++ b/server/lib/object-storage/videos.ts
@@ -1,8 +1,9 @@
1import { basename, join } from 'path' 1import { basename, join } from 'path'
2import { logger } from '@server/helpers/logger' 2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config' 3import { CONFIG } from '@server/initializers/config'
4import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' 4import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models'
5import { getHLSDirectory } from '../paths' 5import { getHLSDirectory } from '../paths'
6import { VideoPathManager } from '../video-path-manager'
6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' 7import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
7import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' 8import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
8 9
@@ -30,10 +31,10 @@ function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string)
30 31
31// --------------------------------------------------------------------------- 32// ---------------------------------------------------------------------------
32 33
33function storeWebTorrentFile (filename: string) { 34function storeWebTorrentFile (video: MVideo, file: MVideoFile) {
34 return storeObject({ 35 return storeObject({
35 inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename), 36 inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file),
36 objectStorageKey: generateWebTorrentObjectStorageKey(filename), 37 objectStorageKey: generateWebTorrentObjectStorageKey(file.filename),
37 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS 38 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS
38 }) 39 })
39} 40}
diff --git a/server/lib/paths.ts b/server/lib/paths.ts
index b29854700..470970f55 100644
--- a/server/lib/paths.ts
+++ b/server/lib/paths.ts
@@ -1,9 +1,10 @@
1import { join } from 'path' 1import { join } from 'path'
2import { CONFIG } from '@server/initializers/config' 2import { CONFIG } from '@server/initializers/config'
3import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, VIDEO_LIVE } from '@server/initializers/constants' 3import { DIRECTORIES, VIDEO_LIVE } from '@server/initializers/constants'
4import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' 4import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
5import { removeFragmentedMP4Ext } from '@shared/core-utils' 5import { removeFragmentedMP4Ext } from '@shared/core-utils'
6import { buildUUID } from '@shared/extra-utils' 6import { buildUUID } from '@shared/extra-utils'
7import { isVideoInPrivateDirectory } from './video-privacy'
7 8
8// ################## Video file name ################## 9// ################## Video file name ##################
9 10
@@ -17,20 +18,24 @@ function generateHLSVideoFilename (resolution: number) {
17 18
18// ################## Streaming playlist ################## 19// ################## Streaming playlist ##################
19 20
20function getLiveDirectory (video: MVideoUUID) { 21function getLiveDirectory (video: MVideo) {
21 return getHLSDirectory(video) 22 return getHLSDirectory(video)
22} 23}
23 24
24function getLiveReplayBaseDirectory (video: MVideoUUID) { 25function getLiveReplayBaseDirectory (video: MVideo) {
25 return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY) 26 return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY)
26} 27}
27 28
28function getHLSDirectory (video: MVideoUUID) { 29function getHLSDirectory (video: MVideo) {
29 return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 30 if (isVideoInPrivateDirectory(video.privacy)) {
31 return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, video.uuid)
32 }
33
34 return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, video.uuid)
30} 35}
31 36
32function getHLSRedundancyDirectory (video: MVideoUUID) { 37function getHLSRedundancyDirectory (video: MVideoUUID) {
33 return join(HLS_REDUNDANCY_DIRECTORY, video.uuid) 38 return join(DIRECTORIES.HLS_REDUNDANCY, video.uuid)
34} 39}
35 40
36function getHlsResolutionPlaylistFilename (videoFilename: string) { 41function getHlsResolutionPlaylistFilename (videoFilename: string) {
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts
index 5bfbc3cd2..30bf189db 100644
--- a/server/lib/schedulers/update-videos-scheduler.ts
+++ b/server/lib/schedulers/update-videos-scheduler.ts
@@ -1,11 +1,14 @@
1import { VideoModel } from '@server/models/video/video' 1import { VideoModel } from '@server/models/video/video'
2import { MVideoFullLight } from '@server/types/models' 2import { MScheduleVideoUpdate } from '@server/types/models'
3import { VideoPrivacy, VideoState } from '@shared/models'
3import { logger } from '../../helpers/logger' 4import { logger } from '../../helpers/logger'
4import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 5import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
5import { sequelizeTypescript } from '../../initializers/database' 6import { sequelizeTypescript } from '../../initializers/database'
6import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' 7import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update'
7import { federateVideoIfNeeded } from '../activitypub/videos'
8import { Notifier } from '../notifier' 8import { Notifier } from '../notifier'
9import { addVideoJobsAfterUpdate } from '../video'
10import { VideoPathManager } from '../video-path-manager'
11import { setVideoPrivacy } from '../video-privacy'
9import { AbstractScheduler } from './abstract-scheduler' 12import { AbstractScheduler } from './abstract-scheduler'
10 13
11export class UpdateVideosScheduler extends AbstractScheduler { 14export class UpdateVideosScheduler extends AbstractScheduler {
@@ -26,35 +29,54 @@ export class UpdateVideosScheduler extends AbstractScheduler {
26 if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined 29 if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined
27 30
28 const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate() 31 const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate()
29 const publishedVideos: MVideoFullLight[] = []
30 32
31 for (const schedule of schedules) { 33 for (const schedule of schedules) {
32 await sequelizeTypescript.transaction(async t => { 34 const videoOnly = await VideoModel.load(schedule.videoId)
33 const video = await VideoModel.loadFull(schedule.videoId, t) 35 const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid)
34 36
35 logger.info('Executing scheduled video update on %s.', video.uuid) 37 try {
38 const { video, published } = await this.updateAVideo(schedule)
36 39
37 if (schedule.privacy) { 40 if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video)
38 const wasConfidentialVideo = video.isConfidential() 41 } catch (err) {
39 const isNewVideo = video.isNewVideo(schedule.privacy) 42 logger.error('Cannot update video', { err })
43 }
40 44
41 video.setPrivacy(schedule.privacy) 45 mutexReleaser()
42 await video.save({ transaction: t }) 46 }
43 await federateVideoIfNeeded(video, isNewVideo, t) 47 }
48
49 private async updateAVideo (schedule: MScheduleVideoUpdate) {
50 let oldPrivacy: VideoPrivacy
51 let isNewVideo: boolean
52 let published = false
53
54 const video = await sequelizeTypescript.transaction(async t => {
55 const video = await VideoModel.loadFull(schedule.videoId, t)
56 if (video.state === VideoState.TO_TRANSCODE) return
57
58 logger.info('Executing scheduled video update on %s.', video.uuid)
59
60 if (schedule.privacy) {
61 isNewVideo = video.isNewVideo(schedule.privacy)
62 oldPrivacy = video.privacy
44 63
45 if (wasConfidentialVideo) { 64 setVideoPrivacy(video, schedule.privacy)
46 publishedVideos.push(video) 65 await video.save({ transaction: t })
47 } 66
67 if (oldPrivacy === VideoPrivacy.PRIVATE) {
68 published = true
48 } 69 }
70 }
49 71
50 await schedule.destroy({ transaction: t }) 72 await schedule.destroy({ transaction: t })
51 })
52 }
53 73
54 for (const v of publishedVideos) { 74 return video
55 Notifier.Instance.notifyOnNewVideoIfNeeded(v) 75 })
56 Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(v) 76
57 } 77 await addVideoJobsAfterUpdate({ video, oldPrivacy, isNewVideo, nameChanged: false })
78
79 return { video, published }
58 } 80 }
59 81
60 static get Instance () { 82 static get Instance () {
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 91c217615..78245fa6a 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -16,7 +16,7 @@ import { VideosRedundancyStrategy } from '../../../shared/models/redundancy'
16import { logger, loggerTagsFactory } from '../../helpers/logger' 16import { logger, loggerTagsFactory } from '../../helpers/logger'
17import { downloadWebTorrentVideo } from '../../helpers/webtorrent' 17import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
18import { CONFIG } from '../../initializers/config' 18import { CONFIG } from '../../initializers/config'
19import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' 19import { DIRECTORIES, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants'
20import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 20import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
21import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 21import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
22import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' 22import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
@@ -262,7 +262,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
262 262
263 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid)) 263 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid))
264 264
265 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) 265 const destDirectory = join(DIRECTORIES.HLS_REDUNDANCY, video.uuid)
266 const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) 266 const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
267 267
268 const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000 268 const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000
diff --git a/server/lib/transcoding/transcoding.ts b/server/lib/transcoding/transcoding.ts
index 44e26754d..736e96e65 100644
--- a/server/lib/transcoding/transcoding.ts
+++ b/server/lib/transcoding/transcoding.ts
@@ -1,3 +1,4 @@
1import { MutexInterface } from 'async-mutex'
1import { Job } from 'bullmq' 2import { Job } from 'bullmq'
2import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' 3import { copyFile, ensureDir, move, remove, stat } from 'fs-extra'
3import { basename, extname as extnameUtil, join } from 'path' 4import { basename, extname as extnameUtil, join } from 'path'
@@ -6,11 +7,13 @@ import { retryTransactionWrapper } from '@server/helpers/database-utils'
6import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 7import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
7import { sequelizeTypescript } from '@server/initializers/database' 8import { sequelizeTypescript } from '@server/initializers/database'
8import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 9import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
10import { pick } from '@shared/core-utils'
9import { VideoResolution, VideoStorage } from '../../../shared/models/videos' 11import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
10import { 12import {
11 buildFileMetadata, 13 buildFileMetadata,
12 canDoQuickTranscode, 14 canDoQuickTranscode,
13 computeResolutionsToTranscode, 15 computeResolutionsToTranscode,
16 ffprobePromise,
14 getVideoStreamDuration, 17 getVideoStreamDuration,
15 getVideoStreamFPS, 18 getVideoStreamFPS,
16 transcodeVOD, 19 transcodeVOD,
@@ -33,7 +36,7 @@ import { VideoTranscodingProfilesManager } from './default-transcoding-profiles'
33 */ 36 */
34 37
35// Optimize the original video file and replace it. The resolution is not changed. 38// Optimize the original video file and replace it. The resolution is not changed.
36function optimizeOriginalVideofile (options: { 39async function optimizeOriginalVideofile (options: {
37 video: MVideoFullLight 40 video: MVideoFullLight
38 inputVideoFile: MVideoFile 41 inputVideoFile: MVideoFile
39 job: Job 42 job: Job
@@ -43,49 +46,61 @@ function optimizeOriginalVideofile (options: {
43 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 46 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
44 const newExtname = '.mp4' 47 const newExtname = '.mp4'
45 48
46 return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async videoInputPath => { 49 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
47 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
48 50
49 const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath) 51 try {
50 ? 'quick-transcode' 52 await video.reload()
51 : 'video'
52 53
53 const resolution = buildOriginalFileResolution(inputVideoFile.resolution) 54 const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
54 55
55 const transcodeOptions: TranscodeVODOptions = { 56 const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => {
56 type: transcodeType, 57 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
57 58
58 inputPath: videoInputPath, 59 const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath)
59 outputPath: videoTranscodedPath, 60 ? 'quick-transcode'
61 : 'video'
60 62
61 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 63 const resolution = buildOriginalFileResolution(inputVideoFile.resolution)
62 profile: CONFIG.TRANSCODING.PROFILE,
63 64
64 resolution, 65 const transcodeOptions: TranscodeVODOptions = {
66 type: transcodeType,
65 67
66 job 68 inputPath: videoInputPath,
67 } 69 outputPath: videoTranscodedPath,
68 70
69 // Could be very long! 71 inputFileMutexReleaser,
70 await transcodeVOD(transcodeOptions)
71 72
72 // Important to do this before getVideoFilename() to take in account the new filename 73 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
73 inputVideoFile.resolution = resolution 74 profile: CONFIG.TRANSCODING.PROFILE,
74 inputVideoFile.extname = newExtname
75 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
76 inputVideoFile.storage = VideoStorage.FILE_SYSTEM
77 75
78 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) 76 resolution,
79 77
80 const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 78 job
81 await remove(videoInputPath) 79 }
82 80
83 return { transcodeType, videoFile } 81 // Could be very long!
84 }) 82 await transcodeVOD(transcodeOptions)
83
84 // Important to do this before getVideoFilename() to take in account the new filename
85 inputVideoFile.resolution = resolution
86 inputVideoFile.extname = newExtname
87 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
88 inputVideoFile.storage = VideoStorage.FILE_SYSTEM
89
90 const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
91 await remove(videoInputPath)
92
93 return { transcodeType, videoFile }
94 })
95
96 return result
97 } finally {
98 inputFileMutexReleaser()
99 }
85} 100}
86 101
87// Transcode the original video file to a lower resolution compatible with WebTorrent 102// Transcode the original video file to a lower resolution compatible with WebTorrent
88function transcodeNewWebTorrentResolution (options: { 103async function transcodeNewWebTorrentResolution (options: {
89 video: MVideoFullLight 104 video: MVideoFullLight
90 resolution: VideoResolution 105 resolution: VideoResolution
91 job: Job 106 job: Job
@@ -95,53 +110,68 @@ function transcodeNewWebTorrentResolution (options: {
95 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 110 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
96 const newExtname = '.mp4' 111 const newExtname = '.mp4'
97 112
98 return VideoPathManager.Instance.makeAvailableVideoFile(video.getMaxQualityFile().withVideoOrPlaylist(video), async videoInputPath => { 113 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
99 const newVideoFile = new VideoFileModel({
100 resolution,
101 extname: newExtname,
102 filename: generateWebTorrentVideoFilename(resolution, newExtname),
103 size: 0,
104 videoId: video.id
105 })
106 114
107 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) 115 try {
108 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) 116 await video.reload()
109 117
110 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO 118 const file = video.getMaxQualityFile().withVideoOrPlaylist(video)
111 ? {
112 type: 'only-audio' as 'only-audio',
113 119
114 inputPath: videoInputPath, 120 const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => {
115 outputPath: videoTranscodedPath, 121 const newVideoFile = new VideoFileModel({
122 resolution,
123 extname: newExtname,
124 filename: generateWebTorrentVideoFilename(resolution, newExtname),
125 size: 0,
126 videoId: video.id
127 })
116 128
117 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 129 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
118 profile: CONFIG.TRANSCODING.PROFILE,
119 130
120 resolution, 131 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
132 ? {
133 type: 'only-audio' as 'only-audio',
121 134
122 job 135 inputPath: videoInputPath,
123 } 136 outputPath: videoTranscodedPath,
124 : {
125 type: 'video' as 'video',
126 inputPath: videoInputPath,
127 outputPath: videoTranscodedPath,
128 137
129 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 138 inputFileMutexReleaser,
130 profile: CONFIG.TRANSCODING.PROFILE,
131 139
132 resolution, 140 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
141 profile: CONFIG.TRANSCODING.PROFILE,
133 142
134 job 143 resolution,
135 }
136 144
137 await transcodeVOD(transcodeOptions) 145 job
146 }
147 : {
148 type: 'video' as 'video',
149 inputPath: videoInputPath,
150 outputPath: videoTranscodedPath,
138 151
139 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) 152 inputFileMutexReleaser,
140 }) 153
154 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
155 profile: CONFIG.TRANSCODING.PROFILE,
156
157 resolution,
158
159 job
160 }
161
162 await transcodeVOD(transcodeOptions)
163
164 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile)
165 })
166
167 return result
168 } finally {
169 inputFileMutexReleaser()
170 }
141} 171}
142 172
143// Merge an image with an audio file to create a video 173// Merge an image with an audio file to create a video
144function mergeAudioVideofile (options: { 174async function mergeAudioVideofile (options: {
145 video: MVideoFullLight 175 video: MVideoFullLight
146 resolution: VideoResolution 176 resolution: VideoResolution
147 job: Job 177 job: Job
@@ -151,54 +181,67 @@ function mergeAudioVideofile (options: {
151 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 181 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
152 const newExtname = '.mp4' 182 const newExtname = '.mp4'
153 183
154 const inputVideoFile = video.getMinQualityFile() 184 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
155 185
156 return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async audioInputPath => { 186 try {
157 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 187 await video.reload()
158 188
159 // If the user updates the video preview during transcoding 189 const inputVideoFile = video.getMinQualityFile()
160 const previewPath = video.getPreview().getPath()
161 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
162 await copyFile(previewPath, tmpPreviewPath)
163 190
164 const transcodeOptions = { 191 const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
165 type: 'merge-audio' as 'merge-audio',
166 192
167 inputPath: tmpPreviewPath, 193 const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => {
168 outputPath: videoTranscodedPath, 194 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
169 195
170 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 196 // If the user updates the video preview during transcoding
171 profile: CONFIG.TRANSCODING.PROFILE, 197 const previewPath = video.getPreview().getPath()
198 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
199 await copyFile(previewPath, tmpPreviewPath)
172 200
173 audioPath: audioInputPath, 201 const transcodeOptions = {
174 resolution, 202 type: 'merge-audio' as 'merge-audio',
175 203
176 job 204 inputPath: tmpPreviewPath,
177 } 205 outputPath: videoTranscodedPath,
178 206
179 try { 207 inputFileMutexReleaser,
180 await transcodeVOD(transcodeOptions)
181 208
182 await remove(audioInputPath) 209 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
183 await remove(tmpPreviewPath) 210 profile: CONFIG.TRANSCODING.PROFILE,
184 } catch (err) {
185 await remove(tmpPreviewPath)
186 throw err
187 }
188 211
189 // Important to do this before getVideoFilename() to take in account the new file extension 212 audioPath: audioInputPath,
190 inputVideoFile.extname = newExtname 213 resolution,
191 inputVideoFile.resolution = resolution
192 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
193 214
194 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) 215 job
195 // ffmpeg generated a new video file, so update the video duration 216 }
196 // See https://trac.ffmpeg.org/ticket/5456
197 video.duration = await getVideoStreamDuration(videoTranscodedPath)
198 await video.save()
199 217
200 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 218 try {
201 }) 219 await transcodeVOD(transcodeOptions)
220
221 await remove(audioInputPath)
222 await remove(tmpPreviewPath)
223 } catch (err) {
224 await remove(tmpPreviewPath)
225 throw err
226 }
227
228 // Important to do this before getVideoFilename() to take in account the new file extension
229 inputVideoFile.extname = newExtname
230 inputVideoFile.resolution = resolution
231 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
232
233 // ffmpeg generated a new video file, so update the video duration
234 // See https://trac.ffmpeg.org/ticket/5456
235 video.duration = await getVideoStreamDuration(videoTranscodedPath)
236 await video.save()
237
238 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
239 })
240
241 return result
242 } finally {
243 inputFileMutexReleaser()
244 }
202} 245}
203 246
204// Concat TS segments from a live video to a fragmented mp4 HLS playlist 247// Concat TS segments from a live video to a fragmented mp4 HLS playlist
@@ -207,13 +250,13 @@ async function generateHlsPlaylistResolutionFromTS (options: {
207 concatenatedTsFilePath: string 250 concatenatedTsFilePath: string
208 resolution: VideoResolution 251 resolution: VideoResolution
209 isAAC: boolean 252 isAAC: boolean
253 inputFileMutexReleaser: MutexInterface.Releaser
210}) { 254}) {
211 return generateHlsPlaylistCommon({ 255 return generateHlsPlaylistCommon({
212 video: options.video,
213 resolution: options.resolution,
214 inputPath: options.concatenatedTsFilePath,
215 type: 'hls-from-ts' as 'hls-from-ts', 256 type: 'hls-from-ts' as 'hls-from-ts',
216 isAAC: options.isAAC 257 inputPath: options.concatenatedTsFilePath,
258
259 ...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ])
217 }) 260 })
218} 261}
219 262
@@ -223,15 +266,14 @@ function generateHlsPlaylistResolution (options: {
223 videoInputPath: string 266 videoInputPath: string
224 resolution: VideoResolution 267 resolution: VideoResolution
225 copyCodecs: boolean 268 copyCodecs: boolean
269 inputFileMutexReleaser: MutexInterface.Releaser
226 job?: Job 270 job?: Job
227}) { 271}) {
228 return generateHlsPlaylistCommon({ 272 return generateHlsPlaylistCommon({
229 video: options.video,
230 resolution: options.resolution,
231 copyCodecs: options.copyCodecs,
232 inputPath: options.videoInputPath,
233 type: 'hls' as 'hls', 273 type: 'hls' as 'hls',
234 job: options.job 274 inputPath: options.videoInputPath,
275
276 ...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ])
235 }) 277 })
236} 278}
237 279
@@ -251,27 +293,39 @@ async function onWebTorrentVideoFileTranscoding (
251 video: MVideoFullLight, 293 video: MVideoFullLight,
252 videoFile: MVideoFile, 294 videoFile: MVideoFile,
253 transcodingPath: string, 295 transcodingPath: string,
254 outputPath: string 296 newVideoFile: MVideoFile
255) { 297) {
256 const stats = await stat(transcodingPath) 298 const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
257 const fps = await getVideoStreamFPS(transcodingPath) 299
258 const metadata = await buildFileMetadata(transcodingPath) 300 try {
301 await video.reload()
302
303 const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
259 304
260 await move(transcodingPath, outputPath, { overwrite: true }) 305 const stats = await stat(transcodingPath)
261 306
262 videoFile.size = stats.size 307 const probe = await ffprobePromise(transcodingPath)
263 videoFile.fps = fps 308 const fps = await getVideoStreamFPS(transcodingPath, probe)
264 videoFile.metadata = metadata 309 const metadata = await buildFileMetadata(transcodingPath, probe)
265 310
266 await createTorrentAndSetInfoHash(video, videoFile) 311 await move(transcodingPath, outputPath, { overwrite: true })
267 312
268 const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) 313 videoFile.size = stats.size
269 if (oldFile) await video.removeWebTorrentFile(oldFile) 314 videoFile.fps = fps
315 videoFile.metadata = metadata
270 316
271 await VideoFileModel.customUpsert(videoFile, 'video', undefined) 317 await createTorrentAndSetInfoHash(video, videoFile)
272 video.VideoFiles = await video.$get('VideoFiles')
273 318
274 return { video, videoFile } 319 const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
320 if (oldFile) await video.removeWebTorrentFile(oldFile)
321
322 await VideoFileModel.customUpsert(videoFile, 'video', undefined)
323 video.VideoFiles = await video.$get('VideoFiles')
324
325 return { video, videoFile }
326 } finally {
327 mutexReleaser()
328 }
275} 329}
276 330
277async function generateHlsPlaylistCommon (options: { 331async function generateHlsPlaylistCommon (options: {
@@ -279,12 +333,15 @@ async function generateHlsPlaylistCommon (options: {
279 video: MVideo 333 video: MVideo
280 inputPath: string 334 inputPath: string
281 resolution: VideoResolution 335 resolution: VideoResolution
336
337 inputFileMutexReleaser: MutexInterface.Releaser
338
282 copyCodecs?: boolean 339 copyCodecs?: boolean
283 isAAC?: boolean 340 isAAC?: boolean
284 341
285 job?: Job 342 job?: Job
286}) { 343}) {
287 const { type, video, inputPath, resolution, copyCodecs, isAAC, job } = options 344 const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options
288 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 345 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
289 346
290 const videoTranscodedBasePath = join(transcodeDirectory, type) 347 const videoTranscodedBasePath = join(transcodeDirectory, type)
@@ -308,6 +365,8 @@ async function generateHlsPlaylistCommon (options: {
308 365
309 isAAC, 366 isAAC,
310 367
368 inputFileMutexReleaser,
369
311 hlsPlaylist: { 370 hlsPlaylist: {
312 videoFilename 371 videoFilename
313 }, 372 },
@@ -333,40 +392,54 @@ async function generateHlsPlaylistCommon (options: {
333 videoStreamingPlaylistId: playlist.id 392 videoStreamingPlaylistId: playlist.id
334 }) 393 })
335 394
336 const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) 395 const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
337 await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
338 396
339 // Move playlist file 397 try {
340 const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename) 398 // VOD transcoding is a long task, refresh video attributes
341 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) 399 await video.reload()
342 // Move video file
343 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
344 400
345 // Update video duration if it was not set (in case of a live for example) 401 const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
346 if (!video.duration) { 402 await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
347 video.duration = await getVideoStreamDuration(videoFilePath)
348 await video.save()
349 }
350 403
351 const stats = await stat(videoFilePath) 404 // Move playlist file
405 const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
406 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
407 // Move video file
408 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
352 409
353 newVideoFile.size = stats.size 410 // Update video duration if it was not set (in case of a live for example)
354 newVideoFile.fps = await getVideoStreamFPS(videoFilePath) 411 if (!video.duration) {
355 newVideoFile.metadata = await buildFileMetadata(videoFilePath) 412 video.duration = await getVideoStreamDuration(videoFilePath)
413 await video.save()
414 }
356 415
357 await createTorrentAndSetInfoHash(playlist, newVideoFile) 416 const stats = await stat(videoFilePath)
358 417
359 const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution }) 418 newVideoFile.size = stats.size
360 if (oldFile) { 419 newVideoFile.fps = await getVideoStreamFPS(videoFilePath)
361 await video.removeStreamingPlaylistVideoFile(playlist, oldFile) 420 newVideoFile.metadata = await buildFileMetadata(videoFilePath)
362 await oldFile.destroy() 421
363 } 422 await createTorrentAndSetInfoHash(playlist, newVideoFile)
364 423
365 const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) 424 const oldFile = await VideoFileModel.loadHLSFile({
425 playlistId: playlist.id,
426 fps: newVideoFile.fps,
427 resolution: newVideoFile.resolution
428 })
429
430 if (oldFile) {
431 await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
432 await oldFile.destroy()
433 }
366 434
367 await updatePlaylistAfterFileChange(video, playlist) 435 const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
368 436
369 return { resolutionPlaylistPath, videoFile: savedVideoFile } 437 await updatePlaylistAfterFileChange(video, playlist)
438
439 return { resolutionPlaylistPath, videoFile: savedVideoFile }
440 } finally {
441 mutexReleaser()
442 }
370} 443}
371 444
372function buildOriginalFileResolution (inputResolution: number) { 445function buildOriginalFileResolution (inputResolution: number) {
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts
index c3f55fd95..9953cae5d 100644
--- a/server/lib/video-path-manager.ts
+++ b/server/lib/video-path-manager.ts
@@ -1,29 +1,31 @@
1import { Mutex } from 'async-mutex'
1import { remove } from 'fs-extra' 2import { remove } from 'fs-extra'
2import { extname, join } from 'path' 3import { extname, join } from 'path'
4import { logger, loggerTagsFactory } from '@server/helpers/logger'
3import { extractVideo } from '@server/helpers/video' 5import { extractVideo } from '@server/helpers/video'
4import { CONFIG } from '@server/initializers/config' 6import { CONFIG } from '@server/initializers/config'
5import { 7import { DIRECTORIES } from '@server/initializers/constants'
6 MStreamingPlaylistVideo, 8import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '@server/types/models'
7 MVideo,
8 MVideoFile,
9 MVideoFileStreamingPlaylistVideo,
10 MVideoFileVideo,
11 MVideoUUID
12} from '@server/types/models'
13import { buildUUID } from '@shared/extra-utils' 9import { buildUUID } from '@shared/extra-utils'
14import { VideoStorage } from '@shared/models' 10import { VideoStorage } from '@shared/models'
15import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage' 11import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage'
16import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' 12import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths'
13import { isVideoInPrivateDirectory } from './video-privacy'
17 14
18type MakeAvailableCB <T> = (path: string) => Promise<T> | T 15type MakeAvailableCB <T> = (path: string) => Promise<T> | T
19 16
17const lTags = loggerTagsFactory('video-path-manager')
18
20class VideoPathManager { 19class VideoPathManager {
21 20
22 private static instance: VideoPathManager 21 private static instance: VideoPathManager
23 22
23 // Key is a video UUID
24 private readonly videoFileMutexStore = new Map<string, Mutex>()
25
24 private constructor () {} 26 private constructor () {}
25 27
26 getFSHLSOutputPath (video: MVideoUUID, filename?: string) { 28 getFSHLSOutputPath (video: MVideo, filename?: string) {
27 const base = getHLSDirectory(video) 29 const base = getHLSDirectory(video)
28 if (!filename) return base 30 if (!filename) return base
29 31
@@ -41,13 +43,17 @@ class VideoPathManager {
41 } 43 }
42 44
43 getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { 45 getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
44 if (videoFile.isHLS()) { 46 const video = extractVideo(videoOrPlaylist)
45 const video = extractVideo(videoOrPlaylist)
46 47
48 if (videoFile.isHLS()) {
47 return join(getHLSDirectory(video), videoFile.filename) 49 return join(getHLSDirectory(video), videoFile.filename)
48 } 50 }
49 51
50 return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename) 52 if (isVideoInPrivateDirectory(video.privacy)) {
53 return join(DIRECTORIES.VIDEOS.PRIVATE, videoFile.filename)
54 }
55
56 return join(DIRECTORIES.VIDEOS.PUBLIC, videoFile.filename)
51 } 57 }
52 58
53 async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) { 59 async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) {
@@ -113,6 +119,27 @@ class VideoPathManager {
113 ) 119 )
114 } 120 }
115 121
122 async lockFiles (videoUUID: string) {
123 if (!this.videoFileMutexStore.has(videoUUID)) {
124 this.videoFileMutexStore.set(videoUUID, new Mutex())
125 }
126
127 const mutex = this.videoFileMutexStore.get(videoUUID)
128 const releaser = await mutex.acquire()
129
130 logger.debug('Locked files of %s.', videoUUID, lTags(videoUUID))
131
132 return releaser
133 }
134
135 unlockFiles (videoUUID: string) {
136 const mutex = this.videoFileMutexStore.get(videoUUID)
137
138 mutex.release()
139
140 logger.debug('Released lockfiles of %s.', videoUUID, lTags(videoUUID))
141 }
142
116 private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) { 143 private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) {
117 let result: T 144 let result: T
118 145
diff --git a/server/lib/video-privacy.ts b/server/lib/video-privacy.ts
new file mode 100644
index 000000000..1a4a5a22d
--- /dev/null
+++ b/server/lib/video-privacy.ts
@@ -0,0 +1,96 @@
1import { move } from 'fs-extra'
2import { join } from 'path'
3import { logger } from '@server/helpers/logger'
4import { DIRECTORIES } from '@server/initializers/constants'
5import { MVideo, MVideoFullLight } from '@server/types/models'
6import { VideoPrivacy } from '@shared/models'
7
8function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) {
9 if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {
10 video.publishedAt = new Date()
11 }
12
13 video.privacy = newPrivacy
14}
15
16function isVideoInPrivateDirectory (privacy: VideoPrivacy) {
17 return privacy === VideoPrivacy.PRIVATE || privacy === VideoPrivacy.INTERNAL
18}
19
20function isVideoInPublicDirectory (privacy: VideoPrivacy) {
21 return !isVideoInPrivateDirectory(privacy)
22}
23
24async function moveFilesIfPrivacyChanged (video: MVideoFullLight, oldPrivacy: VideoPrivacy) {
25 // Now public, previously private
26 if (isVideoInPublicDirectory(video.privacy) && isVideoInPrivateDirectory(oldPrivacy)) {
27 await moveFiles({ type: 'private-to-public', video })
28
29 return true
30 }
31
32 // Now private, previously public
33 if (isVideoInPrivateDirectory(video.privacy) && isVideoInPublicDirectory(oldPrivacy)) {
34 await moveFiles({ type: 'public-to-private', video })
35
36 return true
37 }
38
39 return false
40}
41
42export {
43 setVideoPrivacy,
44
45 isVideoInPrivateDirectory,
46 isVideoInPublicDirectory,
47
48 moveFilesIfPrivacyChanged
49}
50
51// ---------------------------------------------------------------------------
52
53async function moveFiles (options: {
54 type: 'private-to-public' | 'public-to-private'
55 video: MVideoFullLight
56}) {
57 const { type, video } = options
58
59 const directories = type === 'private-to-public'
60 ? {
61 webtorrent: { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC },
62 hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC }
63 }
64 : {
65 webtorrent: { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE },
66 hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE }
67 }
68
69 for (const file of video.VideoFiles) {
70 const source = join(directories.webtorrent.old, file.filename)
71 const destination = join(directories.webtorrent.new, file.filename)
72
73 try {
74 logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
75
76 await move(source, destination)
77 } catch (err) {
78 logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err })
79 }
80 }
81
82 const hls = video.getHLSPlaylist()
83
84 if (hls) {
85 const source = join(directories.hls.old, video.uuid)
86 const destination = join(directories.hls.new, video.uuid)
87
88 try {
89 logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
90
91 await move(source, destination)
92 } catch (err) {
93 logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err })
94 }
95 }
96}
diff --git a/server/lib/video-tokens-manager.ts b/server/lib/video-tokens-manager.ts
new file mode 100644
index 000000000..c43085d16
--- /dev/null
+++ b/server/lib/video-tokens-manager.ts
@@ -0,0 +1,49 @@
1import LRUCache from 'lru-cache'
2import { LRU_CACHE } from '@server/initializers/constants'
3import { buildUUID } from '@shared/extra-utils'
4
5// ---------------------------------------------------------------------------
6// Create temporary tokens that can be used as URL query parameters to access video static files
7// ---------------------------------------------------------------------------
8
9class VideoTokensManager {
10
11 private static instance: VideoTokensManager
12
13 private readonly lruCache = new LRUCache<string, string>({
14 max: LRU_CACHE.VIDEO_TOKENS.MAX_SIZE,
15 ttl: LRU_CACHE.VIDEO_TOKENS.TTL
16 })
17
18 private constructor () {}
19
20 create (videoUUID: string) {
21 const token = buildUUID()
22
23 const expires = new Date(new Date().getTime() + LRU_CACHE.VIDEO_TOKENS.TTL)
24
25 this.lruCache.set(token, videoUUID)
26
27 return { token, expires }
28 }
29
30 hasToken (options: {
31 token: string
32 videoUUID: string
33 }) {
34 const value = this.lruCache.get(options.token)
35 if (!value) return false
36
37 return value === options.videoUUID
38 }
39
40 static get Instance () {
41 return this.instance || (this.instance = new this())
42 }
43}
44
45// ---------------------------------------------------------------------------
46
47export {
48 VideoTokensManager
49}
diff --git a/server/lib/video.ts b/server/lib/video.ts
index 6c4f3ce7b..aacc41a7a 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -7,10 +7,11 @@ import { TagModel } from '@server/models/video/tag'
7import { VideoModel } from '@server/models/video/video' 7import { VideoModel } from '@server/models/video/video'
8import { VideoJobInfoModel } from '@server/models/video/video-job-info' 8import { VideoJobInfoModel } from '@server/models/video/video-job-info'
9import { FilteredModelAttributes } from '@server/types' 9import { FilteredModelAttributes } from '@server/types'
10import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' 10import { MThumbnail, MUserId, MVideoFile, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
11import { ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models' 11import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models'
12import { CreateJobOptions } from './job-queue/job-queue' 12import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue'
13import { updateVideoMiniatureFromExisting } from './thumbnail' 13import { updateVideoMiniatureFromExisting } from './thumbnail'
14import { moveFilesIfPrivacyChanged } from './video-privacy'
14 15
15function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { 16function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
16 return { 17 return {
@@ -177,6 +178,59 @@ const getCachedVideoDuration = memoizee(getVideoDuration, {
177 178
178// --------------------------------------------------------------------------- 179// ---------------------------------------------------------------------------
179 180
181async function addVideoJobsAfterUpdate (options: {
182 video: MVideoFullLight
183 isNewVideo: boolean
184
185 nameChanged: boolean
186 oldPrivacy: VideoPrivacy
187}) {
188 const { video, nameChanged, oldPrivacy, isNewVideo } = options
189 const jobs: CreateJobArgument[] = []
190
191 const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy)
192
193 if (!video.isLive && (nameChanged || filePathChanged)) {
194 for (const file of (video.VideoFiles || [])) {
195 const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }
196
197 jobs.push({ type: 'manage-video-torrent', payload })
198 }
199
200 const hls = video.getHLSPlaylist()
201
202 for (const file of (hls?.VideoFiles || [])) {
203 const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }
204
205 jobs.push({ type: 'manage-video-torrent', payload })
206 }
207 }
208
209 jobs.push({
210 type: 'federate-video',
211 payload: {
212 videoUUID: video.uuid,
213 isNewVideo
214 }
215 })
216
217 const wasConfidentialVideo = new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED, VideoPrivacy.INTERNAL ]).has(oldPrivacy)
218
219 if (wasConfidentialVideo) {
220 jobs.push({
221 type: 'notify',
222 payload: {
223 action: 'new-video',
224 videoUUID: video.uuid
225 }
226 })
227 }
228
229 return JobQueue.Instance.createSequentialJobFlow(...jobs)
230}
231
232// ---------------------------------------------------------------------------
233
180export { 234export {
181 buildLocalVideoFromReq, 235 buildLocalVideoFromReq,
182 buildVideoThumbnailsFromReq, 236 buildVideoThumbnailsFromReq,
@@ -185,5 +239,6 @@ export {
185 buildTranscodingJob, 239 buildTranscodingJob,
186 buildMoveToObjectStorageJob, 240 buildMoveToObjectStorageJob,
187 getTranscodingJobPriority, 241 getTranscodingJobPriority,
242 addVideoJobsAfterUpdate,
188 getCachedVideoDuration 243 getCachedVideoDuration
189} 244}
diff --git a/server/middlewares/auth.ts b/server/middlewares/auth.ts
index 904d47efd..e6025c8ce 100644
--- a/server/middlewares/auth.ts
+++ b/server/middlewares/auth.ts
@@ -5,8 +5,8 @@ import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
5import { logger } from '../helpers/logger' 5import { logger } from '../helpers/logger'
6import { handleOAuthAuthenticate } from '../lib/auth/oauth' 6import { handleOAuthAuthenticate } from '../lib/auth/oauth'
7 7
8function authenticate (req: express.Request, res: express.Response, next: express.NextFunction, authenticateInQuery = false) { 8function authenticate (req: express.Request, res: express.Response, next: express.NextFunction) {
9 handleOAuthAuthenticate(req, res, authenticateInQuery) 9 handleOAuthAuthenticate(req, res)
10 .then((token: any) => { 10 .then((token: any) => {
11 res.locals.oauth = { token } 11 res.locals.oauth = { token }
12 res.locals.authenticated = true 12 res.locals.authenticated = true
@@ -47,7 +47,7 @@ function authenticateSocket (socket: Socket, next: (err?: any) => void) {
47 .catch(err => logger.error('Cannot get access token.', { err })) 47 .catch(err => logger.error('Cannot get access token.', { err }))
48} 48}
49 49
50function authenticatePromise (req: express.Request, res: express.Response, authenticateInQuery = false) { 50function authenticatePromise (req: express.Request, res: express.Response) {
51 return new Promise<void>(resolve => { 51 return new Promise<void>(resolve => {
52 // Already authenticated? (or tried to) 52 // Already authenticated? (or tried to)
53 if (res.locals.oauth?.token.User) return resolve() 53 if (res.locals.oauth?.token.User) return resolve()
@@ -59,7 +59,7 @@ function authenticatePromise (req: express.Request, res: express.Response, authe
59 }) 59 })
60 } 60 }
61 61
62 authenticate(req, res, () => resolve(), authenticateInQuery) 62 authenticate(req, res, () => resolve())
63 }) 63 })
64} 64}
65 65
diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts
index ffadb3b49..899da229a 100644
--- a/server/middlewares/validators/index.ts
+++ b/server/middlewares/validators/index.ts
@@ -1,7 +1,6 @@
1export * from './activitypub'
2export * from './videos'
3export * from './abuse' 1export * from './abuse'
4export * from './account' 2export * from './account'
3export * from './activitypub'
5export * from './actor-image' 4export * from './actor-image'
6export * from './blocklist' 5export * from './blocklist'
7export * from './bulk' 6export * from './bulk'
@@ -10,8 +9,8 @@ export * from './express'
10export * from './feeds' 9export * from './feeds'
11export * from './follows' 10export * from './follows'
12export * from './jobs' 11export * from './jobs'
13export * from './metrics'
14export * from './logs' 12export * from './logs'
13export * from './metrics'
15export * from './oembed' 14export * from './oembed'
16export * from './pagination' 15export * from './pagination'
17export * from './plugins' 16export * from './plugins'
@@ -19,9 +18,11 @@ export * from './redundancy'
19export * from './search' 18export * from './search'
20export * from './server' 19export * from './server'
21export * from './sort' 20export * from './sort'
21export * from './static'
22export * from './themes' 22export * from './themes'
23export * from './user-history' 23export * from './user-history'
24export * from './user-notifications' 24export * from './user-notifications'
25export * from './user-subscriptions' 25export * from './user-subscriptions'
26export * from './users' 26export * from './users'
27export * from './videos'
27export * from './webfinger' 28export * from './webfinger'
diff --git a/server/middlewares/validators/shared/videos.ts b/server/middlewares/validators/shared/videos.ts
index e3a98c58f..c29751eca 100644
--- a/server/middlewares/validators/shared/videos.ts
+++ b/server/middlewares/validators/shared/videos.ts
@@ -1,7 +1,7 @@
1import { Request, Response } from 'express' 1import { Request, Response } from 'express'
2import { isUUIDValid } from '@server/helpers/custom-validators/misc'
3import { loadVideo, VideoLoadType } from '@server/lib/model-loaders' 2import { loadVideo, VideoLoadType } from '@server/lib/model-loaders'
4import { isAbleToUploadVideo } from '@server/lib/user' 3import { isAbleToUploadVideo } from '@server/lib/user'
4import { VideoTokensManager } from '@server/lib/video-tokens-manager'
5import { authenticatePromise } from '@server/middlewares/auth' 5import { authenticatePromise } from '@server/middlewares/auth'
6import { VideoModel } from '@server/models/video/video' 6import { VideoModel } from '@server/models/video/video'
7import { VideoChannelModel } from '@server/models/video/video-channel' 7import { VideoChannelModel } from '@server/models/video/video-channel'
@@ -108,26 +108,21 @@ async function checkCanSeeVideo (options: {
108 res: Response 108 res: Response
109 paramId: string 109 paramId: string
110 video: MVideo 110 video: MVideo
111 authenticateInQuery?: boolean // default false
112}) { 111}) {
113 const { req, res, video, paramId, authenticateInQuery = false } = options 112 const { req, res, video, paramId } = options
114 113
115 if (video.requiresAuth()) { 114 if (video.requiresAuth(paramId)) {
116 return checkCanSeeAuthVideo(req, res, video, authenticateInQuery) 115 return checkCanSeeAuthVideo(req, res, video)
117 } 116 }
118 117
119 if (video.privacy === VideoPrivacy.UNLISTED) { 118 if (video.privacy === VideoPrivacy.UNLISTED || video.privacy === VideoPrivacy.PUBLIC) {
120 if (isUUIDValid(paramId)) return true 119 return true
121
122 return checkCanSeeAuthVideo(req, res, video, authenticateInQuery)
123 } 120 }
124 121
125 if (video.privacy === VideoPrivacy.PUBLIC) return true 122 throw new Error('Unknown video privacy when checking video right ' + video.url)
126
127 throw new Error('Fatal error when checking video right ' + video.url)
128} 123}
129 124
130async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoId | MVideoWithRights, authenticateInQuery = false) { 125async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoId | MVideoWithRights) {
131 const fail = () => { 126 const fail = () => {
132 res.fail({ 127 res.fail({
133 status: HttpStatusCode.FORBIDDEN_403, 128 status: HttpStatusCode.FORBIDDEN_403,
@@ -137,7 +132,7 @@ async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoI
137 return false 132 return false
138 } 133 }
139 134
140 await authenticatePromise(req, res, authenticateInQuery) 135 await authenticatePromise(req, res)
141 136
142 const user = res.locals.oauth?.token.User 137 const user = res.locals.oauth?.token.User
143 if (!user) return fail() 138 if (!user) return fail()
@@ -173,6 +168,36 @@ async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoI
173 168
174// --------------------------------------------------------------------------- 169// ---------------------------------------------------------------------------
175 170
171async function checkCanAccessVideoStaticFiles (options: {
172 video: MVideo
173 req: Request
174 res: Response
175 paramId: string
176}) {
177 const { video, req, res, paramId } = options
178
179 if (res.locals.oauth?.token.User) {
180 return checkCanSeeVideo(options)
181 }
182
183 if (!video.requiresAuth(paramId)) return true
184
185 const videoFileToken = req.query.videoFileToken
186 if (!videoFileToken) {
187 res.sendStatus(HttpStatusCode.FORBIDDEN_403)
188 return false
189 }
190
191 if (VideoTokensManager.Instance.hasToken({ token: videoFileToken, videoUUID: video.uuid })) {
192 return true
193 }
194
195 res.sendStatus(HttpStatusCode.FORBIDDEN_403)
196 return false
197}
198
199// ---------------------------------------------------------------------------
200
176function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRight, res: Response, onlyOwned = true) { 201function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRight, res: Response, onlyOwned = true) {
177 // Retrieve the user who did the request 202 // Retrieve the user who did the request
178 if (onlyOwned && video.isOwned() === false) { 203 if (onlyOwned && video.isOwned() === false) {
@@ -220,6 +245,7 @@ export {
220 doesVideoExist, 245 doesVideoExist,
221 doesVideoFileOfVideoExist, 246 doesVideoFileOfVideoExist,
222 247
248 checkCanAccessVideoStaticFiles,
223 checkUserCanManageVideo, 249 checkUserCanManageVideo,
224 checkCanSeeVideo, 250 checkCanSeeVideo,
225 checkUserQuota 251 checkUserQuota
diff --git a/server/middlewares/validators/static.ts b/server/middlewares/validators/static.ts
new file mode 100644
index 000000000..ff9e6ae6e
--- /dev/null
+++ b/server/middlewares/validators/static.ts
@@ -0,0 +1,131 @@
1import express from 'express'
2import { query } from 'express-validator'
3import LRUCache from 'lru-cache'
4import { basename, dirname } from 'path'
5import { exists, isUUIDValid } from '@server/helpers/custom-validators/misc'
6import { logger } from '@server/helpers/logger'
7import { LRU_CACHE } from '@server/initializers/constants'
8import { VideoModel } from '@server/models/video/video'
9import { VideoFileModel } from '@server/models/video/video-file'
10import { HttpStatusCode } from '@shared/models'
11import { areValidationErrors, checkCanAccessVideoStaticFiles } from './shared'
12
13const staticFileTokenBypass = new LRUCache<string, boolean>({
14 max: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.MAX_SIZE,
15 ttl: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.TTL
16})
17
18const ensureCanAccessVideoPrivateWebTorrentFiles = [
19 query('videoFileToken').optional().custom(exists),
20
21 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
22 if (areValidationErrors(req, res)) return
23
24 const token = extractTokenOrDie(req, res)
25 if (!token) return
26
27 const cacheKey = token + '-' + req.originalUrl
28
29 if (staticFileTokenBypass.has(cacheKey)) {
30 const allowedFromCache = staticFileTokenBypass.get(cacheKey)
31
32 if (allowedFromCache === true) return next()
33
34 return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
35 }
36
37 const allowed = await isWebTorrentAllowed(req, res)
38
39 staticFileTokenBypass.set(cacheKey, allowed)
40
41 if (allowed !== true) return
42
43 return next()
44 }
45]
46
47const ensureCanAccessPrivateVideoHLSFiles = [
48 query('videoFileToken').optional().custom(exists),
49
50 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
51 if (areValidationErrors(req, res)) return
52
53 const videoUUID = basename(dirname(req.originalUrl))
54
55 if (!isUUIDValid(videoUUID)) {
56 logger.debug('Path does not contain valid video UUID to serve static file %s', req.originalUrl)
57
58 return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
59 }
60
61 const token = extractTokenOrDie(req, res)
62 if (!token) return
63
64 const cacheKey = token + '-' + videoUUID
65
66 if (staticFileTokenBypass.has(cacheKey)) {
67 const allowedFromCache = staticFileTokenBypass.get(cacheKey)
68
69 if (allowedFromCache === true) return next()
70
71 return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
72 }
73
74 const allowed = await isHLSAllowed(req, res, videoUUID)
75
76 staticFileTokenBypass.set(cacheKey, allowed)
77
78 if (allowed !== true) return
79
80 return next()
81 }
82]
83
84export {
85 ensureCanAccessVideoPrivateWebTorrentFiles,
86 ensureCanAccessPrivateVideoHLSFiles
87}
88
89// ---------------------------------------------------------------------------
90
91async function isWebTorrentAllowed (req: express.Request, res: express.Response) {
92 const filename = basename(req.path)
93
94 const file = await VideoFileModel.loadWithVideoByFilename(filename)
95 if (!file) {
96 logger.debug('Unknown static file %s to serve', req.originalUrl, { filename })
97
98 res.sendStatus(HttpStatusCode.FORBIDDEN_403)
99 return false
100 }
101
102 const video = file.getVideo()
103
104 return checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
105}
106
107async function isHLSAllowed (req: express.Request, res: express.Response, videoUUID: string) {
108 const video = await VideoModel.load(videoUUID)
109
110 if (!video) {
111 logger.debug('Unknown static file %s to serve', req.originalUrl, { videoUUID })
112
113 res.sendStatus(HttpStatusCode.FORBIDDEN_403)
114 return false
115 }
116
117 return checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
118}
119
120function extractTokenOrDie (req: express.Request, res: express.Response) {
121 const token = res.locals.oauth?.token.accessToken || req.query.videoFileToken
122
123 if (!token) {
124 return res.fail({
125 message: 'Bearer token is missing in headers or video file token is missing in URL query parameters',
126 status: HttpStatusCode.FORBIDDEN_403
127 })
128 }
129
130 return token
131}
diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts
index 7fd2b03d1..e29eb4a32 100644
--- a/server/middlewares/validators/videos/videos.ts
+++ b/server/middlewares/validators/videos/videos.ts
@@ -7,7 +7,7 @@ import { getServerActor } from '@server/models/application/application'
7import { ExpressPromiseHandler } from '@server/types/express-handler' 7import { ExpressPromiseHandler } from '@server/types/express-handler'
8import { MUserAccountId, MVideoFullLight } from '@server/types/models' 8import { MUserAccountId, MVideoFullLight } from '@server/types/models'
9import { arrayify, getAllPrivacies } from '@shared/core-utils' 9import { arrayify, getAllPrivacies } from '@shared/core-utils'
10import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude } from '@shared/models' 10import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude, VideoState } from '@shared/models'
11import { 11import {
12 exists, 12 exists,
13 isBooleanValid, 13 isBooleanValid,
@@ -48,6 +48,7 @@ import { Hooks } from '../../../lib/plugins/hooks'
48import { VideoModel } from '../../../models/video/video' 48import { VideoModel } from '../../../models/video/video'
49import { 49import {
50 areValidationErrors, 50 areValidationErrors,
51 checkCanAccessVideoStaticFiles,
51 checkCanSeeVideo, 52 checkCanSeeVideo,
52 checkUserCanManageVideo, 53 checkUserCanManageVideo,
53 checkUserQuota, 54 checkUserQuota,
@@ -232,6 +233,11 @@ const videosUpdateValidator = getCommonVideoEditAttributes().concat([
232 if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req) 233 if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req)
233 if (!await doesVideoExist(req.params.id, res)) return cleanUpReqFiles(req) 234 if (!await doesVideoExist(req.params.id, res)) return cleanUpReqFiles(req)
234 235
236 const video = getVideoWithAttributes(res)
237 if (req.body.privacy && video.isLive && video.state !== VideoState.WAITING_FOR_LIVE) {
238 return res.fail({ message: 'Cannot update privacy of a live that has already started' })
239 }
240
235 // Check if the user who did the request is able to update the video 241 // Check if the user who did the request is able to update the video
236 const user = res.locals.oauth.token.User 242 const user = res.locals.oauth.token.User
237 if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) 243 if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req)
@@ -271,10 +277,7 @@ async function checkVideoFollowConstraints (req: express.Request, res: express.R
271 }) 277 })
272} 278}
273 279
274const videosCustomGetValidator = ( 280const videosCustomGetValidator = (fetchType: 'for-api' | 'all' | 'only-video' | 'only-immutable-attributes') => {
275 fetchType: 'for-api' | 'all' | 'only-video' | 'only-immutable-attributes',
276 authenticateInQuery = false
277) => {
278 return [ 281 return [
279 isValidVideoIdParam('id'), 282 isValidVideoIdParam('id'),
280 283
@@ -287,7 +290,7 @@ const videosCustomGetValidator = (
287 290
288 const video = getVideoWithAttributes(res) as MVideoFullLight 291 const video = getVideoWithAttributes(res) as MVideoFullLight
289 292
290 if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.id, authenticateInQuery })) return 293 if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.id })) return
291 294
292 return next() 295 return next()
293 } 296 }
@@ -295,7 +298,6 @@ const videosCustomGetValidator = (
295} 298}
296 299
297const videosGetValidator = videosCustomGetValidator('all') 300const videosGetValidator = videosCustomGetValidator('all')
298const videosDownloadValidator = videosCustomGetValidator('all', true)
299 301
300const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([ 302const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([
301 isValidVideoIdParam('id'), 303 isValidVideoIdParam('id'),
@@ -311,6 +313,21 @@ const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([
311 } 313 }
312]) 314])
313 315
316const videosDownloadValidator = [
317 isValidVideoIdParam('id'),
318
319 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
320 if (areValidationErrors(req, res)) return
321 if (!await doesVideoExist(req.params.id, res, 'all')) return
322
323 const video = getVideoWithAttributes(res)
324
325 if (!await checkCanAccessVideoStaticFiles({ req, res, video, paramId: req.params.id })) return
326
327 return next()
328 }
329]
330
314const videosRemoveValidator = [ 331const videosRemoveValidator = [
315 isValidVideoIdParam('id'), 332 isValidVideoIdParam('id'),
316 333
@@ -372,7 +389,7 @@ function getCommonVideoEditAttributes () {
372 .custom(isBooleanValid).withMessage('Should have a valid waitTranscoding boolean'), 389 .custom(isBooleanValid).withMessage('Should have a valid waitTranscoding boolean'),
373 body('privacy') 390 body('privacy')
374 .optional() 391 .optional()
375 .customSanitizer(toValueOrNull) 392 .customSanitizer(toIntOrNull)
376 .custom(isVideoPrivacyValid), 393 .custom(isVideoPrivacyValid),
377 body('description') 394 body('description')
378 .optional() 395 .optional()
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts
index e1b0eb610..76745f4b5 100644
--- a/server/models/video/formatter/video-format-utils.ts
+++ b/server/models/video/formatter/video-format-utils.ts
@@ -34,6 +34,7 @@ import {
34import { 34import {
35 MServer, 35 MServer,
36 MStreamingPlaylistRedundanciesOpt, 36 MStreamingPlaylistRedundanciesOpt,
37 MUserId,
37 MVideo, 38 MVideo,
38 MVideoAP, 39 MVideoAP,
39 MVideoFile, 40 MVideoFile,
@@ -245,8 +246,12 @@ function sortByResolutionDesc (fileA: MVideoFile, fileB: MVideoFile) {
245function videoFilesModelToFormattedJSON ( 246function videoFilesModelToFormattedJSON (
246 video: MVideoFormattable, 247 video: MVideoFormattable,
247 videoFiles: MVideoFileRedundanciesOpt[], 248 videoFiles: MVideoFileRedundanciesOpt[],
248 includeMagnet = true 249 options: {
250 includeMagnet?: boolean // default true
251 } = {}
249): VideoFile[] { 252): VideoFile[] {
253 const { includeMagnet = true } = options
254
250 const trackerUrls = includeMagnet 255 const trackerUrls = includeMagnet
251 ? video.getTrackerUrls() 256 ? video.getTrackerUrls()
252 : [] 257 : []
@@ -281,11 +286,14 @@ function videoFilesModelToFormattedJSON (
281 }) 286 })
282} 287}
283 288
284function addVideoFilesInAPAcc ( 289function addVideoFilesInAPAcc (options: {
285 acc: ActivityUrlObject[] | ActivityTagObject[], 290 acc: ActivityUrlObject[] | ActivityTagObject[]
286 video: MVideo, 291 video: MVideo
287 files: MVideoFile[] 292 files: MVideoFile[]
288) { 293 user?: MUserId
294}) {
295 const { acc, video, files } = options
296
289 const trackerUrls = video.getTrackerUrls() 297 const trackerUrls = video.getTrackerUrls()
290 298
291 const sortedFiles = (files || []) 299 const sortedFiles = (files || [])
@@ -370,7 +378,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
370 } 378 }
371 ] 379 ]
372 380
373 addVideoFilesInAPAcc(url, video, video.VideoFiles || []) 381 addVideoFilesInAPAcc({ acc: url, video, files: video.VideoFiles || [] })
374 382
375 for (const playlist of (video.VideoStreamingPlaylists || [])) { 383 for (const playlist of (video.VideoStreamingPlaylists || [])) {
376 const tag = playlist.p2pMediaLoaderInfohashes 384 const tag = playlist.p2pMediaLoaderInfohashes
@@ -382,7 +390,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
382 href: playlist.getSha256SegmentsUrl(video) 390 href: playlist.getSha256SegmentsUrl(video)
383 }) 391 })
384 392
385 addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || []) 393 addVideoFilesInAPAcc({ acc: tag, video, files: playlist.VideoFiles || [] })
386 394
387 url.push({ 395 url.push({
388 type: 'Link', 396 type: 'Link',
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index d4f07f85f..1a608932f 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -24,6 +24,7 @@ import { extractVideo } from '@server/helpers/video'
24import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url' 24import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url'
25import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage' 25import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
26import { getFSTorrentFilePath } from '@server/lib/paths' 26import { getFSTorrentFilePath } from '@server/lib/paths'
27import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
27import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' 28import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
28import { VideoResolution, VideoStorage } from '@shared/models' 29import { VideoResolution, VideoStorage } from '@shared/models'
29import { AttributesOnly } from '@shared/typescript-utils' 30import { AttributesOnly } from '@shared/typescript-utils'
@@ -295,6 +296,16 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
295 return VideoFileModel.findOne(query) 296 return VideoFileModel.findOne(query)
296 } 297 }
297 298
299 static loadWithVideoByFilename (filename: string): Promise<MVideoFileVideo | MVideoFileStreamingPlaylistVideo> {
300 const query = {
301 where: {
302 filename
303 }
304 }
305
306 return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
307 }
308
298 static loadWithVideoOrPlaylistByTorrentFilename (filename: string) { 309 static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
299 const query = { 310 const query = {
300 where: { 311 where: {
@@ -305,6 +316,10 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
305 return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query) 316 return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
306 } 317 }
307 318
319 static load (id: number): Promise<MVideoFile> {
320 return VideoFileModel.findByPk(id)
321 }
322
308 static loadWithMetadata (id: number) { 323 static loadWithMetadata (id: number) {
309 return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id) 324 return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
310 } 325 }
@@ -467,7 +482,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
467 } 482 }
468 483
469 getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo { 484 getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
470 if (this.videoId) return (this as MVideoFileVideo).Video 485 if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video
471 486
472 return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist 487 return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
473 } 488 }
@@ -508,7 +523,17 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
508 } 523 }
509 524
510 getFileStaticPath (video: MVideo) { 525 getFileStaticPath (video: MVideo) {
511 if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename) 526 if (this.isHLS()) {
527 if (isVideoInPrivateDirectory(video.privacy)) {
528 return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
529 }
530
531 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
532 }
533
534 if (isVideoInPrivateDirectory(video.privacy)) {
535 return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename)
536 }
512 537
513 return join(STATIC_PATHS.WEBSEED, this.filename) 538 return join(STATIC_PATHS.WEBSEED, this.filename)
514 } 539 }
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index 2b6771f27..b919046ed 100644
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -17,6 +17,7 @@ import {
17} from 'sequelize-typescript' 17} from 'sequelize-typescript'
18import { getHLSPublicFileUrl } from '@server/lib/object-storage' 18import { getHLSPublicFileUrl } from '@server/lib/object-storage'
19import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths' 19import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths'
20import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
20import { VideoFileModel } from '@server/models/video/video-file' 21import { VideoFileModel } from '@server/models/video/video-file'
21import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' 22import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models'
22import { sha1 } from '@shared/extra-utils' 23import { sha1 } from '@shared/extra-utils'
@@ -250,7 +251,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
250 return getHLSPublicFileUrl(this.playlistUrl) 251 return getHLSPublicFileUrl(this.playlistUrl)
251 } 252 }
252 253
253 return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) 254 return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video)
254 } 255 }
255 256
256 return this.playlistUrl 257 return this.playlistUrl
@@ -262,7 +263,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
262 return getHLSPublicFileUrl(this.segmentsSha256Url) 263 return getHLSPublicFileUrl(this.segmentsSha256Url)
263 } 264 }
264 265
265 return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid) 266 return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video)
266 } 267 }
267 268
268 return this.segmentsSha256Url 269 return this.segmentsSha256Url
@@ -287,11 +288,19 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
287 return Object.assign(this, { Video: video }) 288 return Object.assign(this, { Video: video })
288 } 289 }
289 290
290 private getMasterPlaylistStaticPath (videoUUID: string) { 291 private getMasterPlaylistStaticPath (video: MVideo) {
291 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename) 292 if (isVideoInPrivateDirectory(video.privacy)) {
293 return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.playlistFilename)
294 }
295
296 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.playlistFilename)
292 } 297 }
293 298
294 private getSha256SegmentsStaticPath (videoUUID: string) { 299 private getSha256SegmentsStaticPath (video: MVideo) {
295 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename) 300 if (isVideoInPrivateDirectory(video.privacy)) {
301 return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.segmentsSha256Filename)
302 }
303
304 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.segmentsSha256Filename)
296 } 305 }
297} 306}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 468117504..82362917e 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -52,7 +52,7 @@ import {
52import { AttributesOnly } from '@shared/typescript-utils' 52import { AttributesOnly } from '@shared/typescript-utils'
53import { peertubeTruncate } from '../../helpers/core-utils' 53import { peertubeTruncate } from '../../helpers/core-utils'
54import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 54import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
55import { exists, isBooleanValid } from '../../helpers/custom-validators/misc' 55import { exists, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc'
56import { 56import {
57 isVideoDescriptionValid, 57 isVideoDescriptionValid,
58 isVideoDurationValid, 58 isVideoDurationValid,
@@ -1696,12 +1696,12 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1696 let files: VideoFile[] = [] 1696 let files: VideoFile[] = []
1697 1697
1698 if (Array.isArray(this.VideoFiles)) { 1698 if (Array.isArray(this.VideoFiles)) {
1699 const result = videoFilesModelToFormattedJSON(this, this.VideoFiles, includeMagnet) 1699 const result = videoFilesModelToFormattedJSON(this, this.VideoFiles, { includeMagnet })
1700 files = files.concat(result) 1700 files = files.concat(result)
1701 } 1701 }
1702 1702
1703 for (const p of (this.VideoStreamingPlaylists || [])) { 1703 for (const p of (this.VideoStreamingPlaylists || [])) {
1704 const result = videoFilesModelToFormattedJSON(this, p.VideoFiles, includeMagnet) 1704 const result = videoFilesModelToFormattedJSON(this, p.VideoFiles, { includeMagnet })
1705 files = files.concat(result) 1705 files = files.concat(result)
1706 } 1706 }
1707 1707
@@ -1868,22 +1868,14 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1868 return setAsUpdated('video', this.id, transaction) 1868 return setAsUpdated('video', this.id, transaction)
1869 } 1869 }
1870 1870
1871 requiresAuth () { 1871 requiresAuth (paramId: string) {
1872 return this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL || !!this.VideoBlacklist 1872 if (this.privacy === VideoPrivacy.UNLISTED) {
1873 } 1873 if (!isUUIDValid(paramId)) return true
1874 1874
1875 setPrivacy (newPrivacy: VideoPrivacy) { 1875 return false
1876 if (this.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {
1877 this.publishedAt = new Date()
1878 } 1876 }
1879 1877
1880 this.privacy = newPrivacy 1878 return this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL || !!this.VideoBlacklist
1881 }
1882
1883 isConfidential () {
1884 return this.privacy === VideoPrivacy.PRIVATE ||
1885 this.privacy === VideoPrivacy.UNLISTED ||
1886 this.privacy === VideoPrivacy.INTERNAL
1887 } 1879 }
1888 1880
1889 async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) { 1881 async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) {
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index 33dc8fb76..961093bb5 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -34,6 +34,7 @@ import './video-imports'
34import './video-playlists' 34import './video-playlists'
35import './video-source' 35import './video-source'
36import './video-studio' 36import './video-studio'
37import './video-token'
37import './videos-common-filters' 38import './videos-common-filters'
38import './videos-history' 39import './videos-history'
39import './videos-overviews' 40import './videos-overviews'
diff --git a/server/tests/api/check-params/live.ts b/server/tests/api/check-params/live.ts
index 3f553c42b..2eff9414b 100644
--- a/server/tests/api/check-params/live.ts
+++ b/server/tests/api/check-params/live.ts
@@ -502,6 +502,23 @@ describe('Test video lives API validator', function () {
502 await stopFfmpeg(ffmpegCommand) 502 await stopFfmpeg(ffmpegCommand)
503 }) 503 })
504 504
505 it('Should fail to change live privacy if it has already started', async function () {
506 this.timeout(40000)
507
508 const live = await command.get({ videoId: video.id })
509
510 const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
511
512 await command.waitUntilPublished({ videoId: video.id })
513 await server.videos.update({
514 id: video.id,
515 attributes: { privacy: VideoPrivacy.PUBLIC },
516 expectedStatus: HttpStatusCode.BAD_REQUEST_400
517 })
518
519 await stopFfmpeg(ffmpegCommand)
520 })
521
505 it('Should fail to stream twice in the save live', async function () { 522 it('Should fail to stream twice in the save live', async function () {
506 this.timeout(40000) 523 this.timeout(40000)
507 524
diff --git a/server/tests/api/check-params/video-files.ts b/server/tests/api/check-params/video-files.ts
index aa4de2c83..9dc59a1b5 100644
--- a/server/tests/api/check-params/video-files.ts
+++ b/server/tests/api/check-params/video-files.ts
@@ -1,10 +1,12 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { HttpStatusCode, UserRole } from '@shared/models' 3import { getAllFiles } from '@shared/core-utils'
4import { HttpStatusCode, UserRole, VideoDetails, VideoPrivacy } from '@shared/models'
4import { 5import {
5 cleanupTests, 6 cleanupTests,
6 createMultipleServers, 7 createMultipleServers,
7 doubleFollow, 8 doubleFollow,
9 makeRawRequest,
8 PeerTubeServer, 10 PeerTubeServer,
9 setAccessTokensToServers, 11 setAccessTokensToServers,
10 waitJobs 12 waitJobs
@@ -13,22 +15,9 @@ import {
13describe('Test videos files', function () { 15describe('Test videos files', function () {
14 let servers: PeerTubeServer[] 16 let servers: PeerTubeServer[]
15 17
16 let webtorrentId: string
17 let hlsId: string
18 let remoteId: string
19
20 let userToken: string 18 let userToken: string
21 let moderatorToken: string 19 let moderatorToken: string
22 20
23 let validId1: string
24 let validId2: string
25
26 let hlsFileId: number
27 let webtorrentFileId: number
28
29 let remoteHLSFileId: number
30 let remoteWebtorrentFileId: number
31
32 // --------------------------------------------------------------- 21 // ---------------------------------------------------------------
33 22
34 before(async function () { 23 before(async function () {
@@ -41,117 +30,163 @@ describe('Test videos files', function () {
41 30
42 userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER) 31 userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER)
43 moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR) 32 moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR)
33 })
44 34
45 { 35 describe('Getting metadata', function () {
46 const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) 36 let video: VideoDetails
47 await waitJobs(servers) 37
38 before(async function () {
39 const { uuid } = await servers[0].videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
40 video = await servers[0].videos.getWithToken({ id: uuid })
41 })
42
43 it('Should not get metadata of private video without token', async function () {
44 for (const file of getAllFiles(video)) {
45 await makeRawRequest({ url: file.metadataUrl, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
46 }
47 })
48
49 it('Should not get metadata of private video without the appropriate token', async function () {
50 for (const file of getAllFiles(video)) {
51 await makeRawRequest({ url: file.metadataUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
52 }
53 })
54
55 it('Should get metadata of private video with the appropriate token', async function () {
56 for (const file of getAllFiles(video)) {
57 await makeRawRequest({ url: file.metadataUrl, token: servers[0].accessToken, expectedStatus: HttpStatusCode.OK_200 })
58 }
59 })
60 })
61
62 describe('Deleting files', function () {
63 let webtorrentId: string
64 let hlsId: string
65 let remoteId: string
66
67 let validId1: string
68 let validId2: string
48 69
49 const video = await servers[1].videos.get({ id: uuid }) 70 let hlsFileId: number
50 remoteId = video.uuid 71 let webtorrentFileId: number
51 remoteHLSFileId = video.streamingPlaylists[0].files[0].id
52 remoteWebtorrentFileId = video.files[0].id
53 }
54 72
55 { 73 let remoteHLSFileId: number
56 await servers[0].config.enableTranscoding(true, true) 74 let remoteWebtorrentFileId: number
75
76 before(async function () {
77 this.timeout(300_000)
57 78
58 { 79 {
59 const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' }) 80 const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' })
60 await waitJobs(servers) 81 await waitJobs(servers)
61 82
62 const video = await servers[0].videos.get({ id: uuid }) 83 const video = await servers[1].videos.get({ id: uuid })
63 validId1 = video.uuid 84 remoteId = video.uuid
64 hlsFileId = video.streamingPlaylists[0].files[0].id 85 remoteHLSFileId = video.streamingPlaylists[0].files[0].id
65 webtorrentFileId = video.files[0].id 86 remoteWebtorrentFileId = video.files[0].id
66 } 87 }
67 88
68 { 89 {
69 const { uuid } = await servers[0].videos.quickUpload({ name: 'both 2' }) 90 await servers[0].config.enableTranscoding(true, true)
70 validId2 = uuid 91
92 {
93 const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' })
94 await waitJobs(servers)
95
96 const video = await servers[0].videos.get({ id: uuid })
97 validId1 = video.uuid
98 hlsFileId = video.streamingPlaylists[0].files[0].id
99 webtorrentFileId = video.files[0].id
100 }
101
102 {
103 const { uuid } = await servers[0].videos.quickUpload({ name: 'both 2' })
104 validId2 = uuid
105 }
71 } 106 }
72 }
73 107
74 await waitJobs(servers) 108 await waitJobs(servers)
75 109
76 { 110 {
77 await servers[0].config.enableTranscoding(false, true) 111 await servers[0].config.enableTranscoding(false, true)
78 const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' }) 112 const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' })
79 hlsId = uuid 113 hlsId = uuid
80 } 114 }
81 115
82 await waitJobs(servers) 116 await waitJobs(servers)
83 117
84 { 118 {
85 await servers[0].config.enableTranscoding(false, true) 119 await servers[0].config.enableTranscoding(false, true)
86 const { uuid } = await servers[0].videos.quickUpload({ name: 'webtorrent' }) 120 const { uuid } = await servers[0].videos.quickUpload({ name: 'webtorrent' })
87 webtorrentId = uuid 121 webtorrentId = uuid
88 } 122 }
89 123
90 await waitJobs(servers) 124 await waitJobs(servers)
91 }) 125 })
92 126
93 it('Should not delete files of a unknown video', async function () { 127 it('Should not delete files of a unknown video', async function () {
94 const expectedStatus = HttpStatusCode.NOT_FOUND_404 128 const expectedStatus = HttpStatusCode.NOT_FOUND_404
95 129
96 await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus }) 130 await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus })
97 await servers[0].videos.removeAllWebTorrentFiles({ videoId: 404, expectedStatus }) 131 await servers[0].videos.removeAllWebTorrentFiles({ videoId: 404, expectedStatus })
98 132
99 await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus }) 133 await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus })
100 await servers[0].videos.removeWebTorrentFile({ videoId: 404, fileId: webtorrentFileId, expectedStatus }) 134 await servers[0].videos.removeWebTorrentFile({ videoId: 404, fileId: webtorrentFileId, expectedStatus })
101 }) 135 })
102 136
103 it('Should not delete unknown files', async function () { 137 it('Should not delete unknown files', async function () {
104 const expectedStatus = HttpStatusCode.NOT_FOUND_404 138 const expectedStatus = HttpStatusCode.NOT_FOUND_404
105 139
106 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webtorrentFileId, expectedStatus }) 140 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webtorrentFileId, expectedStatus })
107 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: hlsFileId, expectedStatus }) 141 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: hlsFileId, expectedStatus })
108 }) 142 })
109 143
110 it('Should not delete files of a remote video', async function () { 144 it('Should not delete files of a remote video', async function () {
111 const expectedStatus = HttpStatusCode.BAD_REQUEST_400 145 const expectedStatus = HttpStatusCode.BAD_REQUEST_400
112 146
113 await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus }) 147 await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus })
114 await servers[0].videos.removeAllWebTorrentFiles({ videoId: remoteId, expectedStatus }) 148 await servers[0].videos.removeAllWebTorrentFiles({ videoId: remoteId, expectedStatus })
115 149
116 await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus }) 150 await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus })
117 await servers[0].videos.removeWebTorrentFile({ videoId: remoteId, fileId: remoteWebtorrentFileId, expectedStatus }) 151 await servers[0].videos.removeWebTorrentFile({ videoId: remoteId, fileId: remoteWebtorrentFileId, expectedStatus })
118 }) 152 })
119 153
120 it('Should not delete files by a non admin user', async function () { 154 it('Should not delete files by a non admin user', async function () {
121 const expectedStatus = HttpStatusCode.FORBIDDEN_403 155 const expectedStatus = HttpStatusCode.FORBIDDEN_403
122 156
123 await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus }) 157 await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus })
124 await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus }) 158 await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus })
125 159
126 await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus }) 160 await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus })
127 await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus }) 161 await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus })
128 162
129 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus }) 163 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus })
130 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus }) 164 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus })
131 165
132 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: userToken, expectedStatus }) 166 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: userToken, expectedStatus })
133 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: moderatorToken, expectedStatus }) 167 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: moderatorToken, expectedStatus })
134 }) 168 })
135 169
136 it('Should not delete files if the files are not available', async function () { 170 it('Should not delete files if the files are not available', async function () {
137 await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) 171 await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
138 await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) 172 await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
139 173
140 await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) 174 await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
141 await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) 175 await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
142 }) 176 })
143 177
144 it('Should not delete files if no both versions are available', async function () { 178 it('Should not delete files if no both versions are available', async function () {
145 await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) 179 await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
146 await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) 180 await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
147 }) 181 })
148 182
149 it('Should delete files if both versions are available', async function () { 183 it('Should delete files if both versions are available', async function () {
150 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId }) 184 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId })
151 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId }) 185 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId })
152 186
153 await servers[0].videos.removeHLSPlaylist({ videoId: validId1 }) 187 await servers[0].videos.removeHLSPlaylist({ videoId: validId1 })
154 await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId2 }) 188 await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId2 })
189 })
155 }) 190 })
156 191
157 after(async function () { 192 after(async function () {
diff --git a/server/tests/api/check-params/video-token.ts b/server/tests/api/check-params/video-token.ts
new file mode 100644
index 000000000..7acb9d580
--- /dev/null
+++ b/server/tests/api/check-params/video-token.ts
@@ -0,0 +1,44 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { HttpStatusCode, VideoPrivacy } from '@shared/models'
4import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
5
6describe('Test video tokens', function () {
7 let server: PeerTubeServer
8 let videoId: string
9 let userToken: string
10
11 // ---------------------------------------------------------------
12
13 before(async function () {
14 this.timeout(300_000)
15
16 server = await createSingleServer(1)
17 await setAccessTokensToServers([ server ])
18
19 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
20 videoId = uuid
21
22 userToken = await server.users.generateUserAndToken('user1')
23 })
24
25 it('Should not generate tokens for unauthenticated user', async function () {
26 await server.videoToken.create({ videoId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
27 })
28
29 it('Should not generate tokens of unknown video', async function () {
30 await server.videoToken.create({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
31 })
32
33 it('Should not generate tokens of a non owned video', async function () {
34 await server.videoToken.create({ videoId, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
35 })
36
37 it('Should generate token', async function () {
38 await server.videoToken.create({ videoId })
39 })
40
41 after(async function () {
42 await cleanupTests([ server ])
43 })
44})
diff --git a/server/tests/api/live/live-fast-restream.ts b/server/tests/api/live/live-fast-restream.ts
index 772ea792d..971df1a61 100644
--- a/server/tests/api/live/live-fast-restream.ts
+++ b/server/tests/api/live/live-fast-restream.ts
@@ -79,8 +79,8 @@ describe('Fast restream in live', function () {
79 expect(video.streamingPlaylists).to.have.lengthOf(1) 79 expect(video.streamingPlaylists).to.have.lengthOf(1)
80 80
81 await server.live.getSegmentFile({ videoUUID: liveId, segment: 0, playlistNumber: 0 }) 81 await server.live.getSegmentFile({ videoUUID: liveId, segment: 0, playlistNumber: 0 })
82 await makeRawRequest(video.streamingPlaylists[0].playlistUrl, HttpStatusCode.OK_200) 82 await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
83 await makeRawRequest(video.streamingPlaylists[0].segmentsSha256Url, HttpStatusCode.OK_200) 83 await makeRawRequest({ url: video.streamingPlaylists[0].segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
84 84
85 await wait(100) 85 await wait(100)
86 } 86 }
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts
index 3f2a304be..003cc934f 100644
--- a/server/tests/api/live/live.ts
+++ b/server/tests/api/live/live.ts
@@ -21,6 +21,7 @@ import {
21 doubleFollow, 21 doubleFollow,
22 killallServers, 22 killallServers,
23 LiveCommand, 23 LiveCommand,
24 makeGetRequest,
24 makeRawRequest, 25 makeRawRequest,
25 PeerTubeServer, 26 PeerTubeServer,
26 sendRTMPStream, 27 sendRTMPStream,
@@ -157,8 +158,8 @@ describe('Test live', function () {
157 expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED) 158 expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED)
158 expect(video.nsfw).to.be.true 159 expect(video.nsfw).to.be.true
159 160
160 await makeRawRequest(server.url + video.thumbnailPath, HttpStatusCode.OK_200) 161 await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
161 await makeRawRequest(server.url + video.previewPath, HttpStatusCode.OK_200) 162 await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 })
162 } 163 }
163 }) 164 })
164 165
@@ -532,8 +533,8 @@ describe('Test live', function () {
532 expect(video.files).to.have.lengthOf(0) 533 expect(video.files).to.have.lengthOf(0)
533 534
534 const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS) 535 const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
535 await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200) 536 await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
536 await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200) 537 await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
537 538
538 // We should have generated random filenames 539 // We should have generated random filenames
539 expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8') 540 expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8')
@@ -564,8 +565,8 @@ describe('Test live', function () {
564 expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height]) 565 expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height])
565 expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height]) 566 expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height])
566 567
567 await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200) 568 await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 })
568 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 569 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
569 } 570 }
570 } 571 }
571 }) 572 })
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts
index 7e16b4c89..77f3a8066 100644
--- a/server/tests/api/object-storage/live.ts
+++ b/server/tests/api/object-storage/live.ts
@@ -48,7 +48,7 @@ async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, nu
48 for (const file of files) { 48 for (const file of files) {
49 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 49 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
50 50
51 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 51 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
52 } 52 }
53 } 53 }
54} 54}
diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts
index f688c7018..90988ea9a 100644
--- a/server/tests/api/object-storage/video-imports.ts
+++ b/server/tests/api/object-storage/video-imports.ts
@@ -66,7 +66,7 @@ describe('Object storage for video import', function () {
66 const fileUrl = video.files[0].fileUrl 66 const fileUrl = video.files[0].fileUrl
67 expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 67 expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
68 68
69 await makeRawRequest(fileUrl, HttpStatusCode.OK_200) 69 await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 })
70 }) 70 })
71 }) 71 })
72 72
@@ -91,13 +91,13 @@ describe('Object storage for video import', function () {
91 for (const file of video.files) { 91 for (const file of video.files) {
92 expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 92 expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
93 93
94 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 94 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
95 } 95 }
96 96
97 for (const file of video.streamingPlaylists[0].files) { 97 for (const file of video.streamingPlaylists[0].files) {
98 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 98 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
99 99
100 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 100 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
101 } 101 }
102 }) 102 })
103 }) 103 })
diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts
index 3e65e1093..63f5179c7 100644
--- a/server/tests/api/object-storage/videos.ts
+++ b/server/tests/api/object-storage/videos.ts
@@ -59,11 +59,11 @@ async function checkFiles (options: {
59 59
60 expectStartWith(file.fileUrl, start) 60 expectStartWith(file.fileUrl, start)
61 61
62 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) 62 const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
63 const location = res.headers['location'] 63 const location = res.headers['location']
64 expectStartWith(location, start) 64 expectStartWith(location, start)
65 65
66 await makeRawRequest(location, HttpStatusCode.OK_200) 66 await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
67 } 67 }
68 68
69 const hls = video.streamingPlaylists[0] 69 const hls = video.streamingPlaylists[0]
@@ -81,19 +81,19 @@ async function checkFiles (options: {
81 expectStartWith(hls.playlistUrl, start) 81 expectStartWith(hls.playlistUrl, start)
82 expectStartWith(hls.segmentsSha256Url, start) 82 expectStartWith(hls.segmentsSha256Url, start)
83 83
84 await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200) 84 await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
85 85
86 const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200) 86 const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
87 expect(JSON.stringify(resSha.body)).to.not.throw 87 expect(JSON.stringify(resSha.body)).to.not.throw
88 88
89 for (const file of hls.files) { 89 for (const file of hls.files) {
90 expectStartWith(file.fileUrl, start) 90 expectStartWith(file.fileUrl, start)
91 91
92 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) 92 const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
93 const location = res.headers['location'] 93 const location = res.headers['location']
94 expectStartWith(location, start) 94 expectStartWith(location, start)
95 95
96 await makeRawRequest(location, HttpStatusCode.OK_200) 96 await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
97 } 97 }
98 } 98 }
99 99
@@ -104,7 +104,7 @@ async function checkFiles (options: {
104 expect(torrent.files.length).to.equal(1) 104 expect(torrent.files.length).to.equal(1)
105 expect(torrent.files[0].path).to.exist.and.to.not.equal('') 105 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
106 106
107 const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 107 const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
108 expect(res.body).to.have.length.above(100) 108 expect(res.body).to.have.length.above(100)
109 } 109 }
110 110
@@ -220,7 +220,7 @@ function runTestSuite (options: {
220 220
221 it('Should fetch correctly all the files', async function () { 221 it('Should fetch correctly all the files', async function () {
222 for (const url of deletedUrls.concat(keptUrls)) { 222 for (const url of deletedUrls.concat(keptUrls)) {
223 await makeRawRequest(url, HttpStatusCode.OK_200) 223 await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
224 } 224 }
225 }) 225 })
226 226
@@ -231,13 +231,13 @@ function runTestSuite (options: {
231 await waitJobs(servers) 231 await waitJobs(servers)
232 232
233 for (const url of deletedUrls) { 233 for (const url of deletedUrls) {
234 await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404) 234 await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
235 } 235 }
236 }) 236 })
237 237
238 it('Should have kept other files', async function () { 238 it('Should have kept other files', async function () {
239 for (const url of keptUrls) { 239 for (const url of keptUrls) {
240 await makeRawRequest(url, HttpStatusCode.OK_200) 240 await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
241 } 241 }
242 }) 242 })
243 243
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index f349a7a76..ba6b00e0b 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -39,7 +39,7 @@ async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], ser
39 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) 39 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
40 40
41 for (const url of parsed.urlList) { 41 for (const url of parsed.urlList) {
42 await makeRawRequest(url, HttpStatusCode.OK_200) 42 await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
43 } 43 }
44} 44}
45 45
diff --git a/server/tests/api/server/open-telemetry.ts b/server/tests/api/server/open-telemetry.ts
index 43a27cc32..7a294be82 100644
--- a/server/tests/api/server/open-telemetry.ts
+++ b/server/tests/api/server/open-telemetry.ts
@@ -18,7 +18,7 @@ describe('Open Telemetry', function () {
18 18
19 let hasError = false 19 let hasError = false
20 try { 20 try {
21 await makeRawRequest(metricsUrl, HttpStatusCode.NOT_FOUND_404) 21 await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
22 } catch (err) { 22 } catch (err) {
23 hasError = err.message.includes('ECONNREFUSED') 23 hasError = err.message.includes('ECONNREFUSED')
24 } 24 }
@@ -37,7 +37,7 @@ describe('Open Telemetry', function () {
37 } 37 }
38 }) 38 })
39 39
40 const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200) 40 const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 })
41 expect(res.text).to.contain('peertube_job_queue_total{') 41 expect(res.text).to.contain('peertube_job_queue_total{')
42 }) 42 })
43 43
@@ -60,7 +60,7 @@ describe('Open Telemetry', function () {
60 } 60 }
61 }) 61 })
62 62
63 const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200) 63 const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 })
64 expect(res.text).to.contain('peertube_playback_http_downloaded_bytes_total{') 64 expect(res.text).to.contain('peertube_playback_http_downloaded_bytes_total{')
65 }) 65 })
66 66
diff --git a/server/tests/api/transcoding/create-transcoding.ts b/server/tests/api/transcoding/create-transcoding.ts
index a50bf7654..372f5332a 100644
--- a/server/tests/api/transcoding/create-transcoding.ts
+++ b/server/tests/api/transcoding/create-transcoding.ts
@@ -20,7 +20,7 @@ import {
20async function checkFilesInObjectStorage (video: VideoDetails) { 20async function checkFilesInObjectStorage (video: VideoDetails) {
21 for (const file of video.files) { 21 for (const file of video.files) {
22 expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 22 expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
23 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 23 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
24 } 24 }
25 25
26 if (video.streamingPlaylists.length === 0) return 26 if (video.streamingPlaylists.length === 0) return
@@ -28,14 +28,14 @@ async function checkFilesInObjectStorage (video: VideoDetails) {
28 const hlsPlaylist = video.streamingPlaylists[0] 28 const hlsPlaylist = video.streamingPlaylists[0]
29 for (const file of hlsPlaylist.files) { 29 for (const file of hlsPlaylist.files) {
30 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 30 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
31 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 31 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
32 } 32 }
33 33
34 expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 34 expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getPlaylistBaseUrl())
35 await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200) 35 await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
36 36
37 expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getPlaylistBaseUrl()) 37 expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getPlaylistBaseUrl())
38 await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200) 38 await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
39} 39}
40 40
41function runTests (objectStorage: boolean) { 41function runTests (objectStorage: boolean) {
@@ -234,7 +234,7 @@ function runTests (objectStorage: boolean) {
234 234
235 it('Should have correctly deleted previous files', async function () { 235 it('Should have correctly deleted previous files', async function () {
236 for (const fileUrl of shouldBeDeleted) { 236 for (const fileUrl of shouldBeDeleted) {
237 await makeRawRequest(fileUrl, HttpStatusCode.NOT_FOUND_404) 237 await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
238 } 238 }
239 }) 239 })
240 240
diff --git a/server/tests/api/transcoding/hls.ts b/server/tests/api/transcoding/hls.ts
index 252422e5d..7b5492cd4 100644
--- a/server/tests/api/transcoding/hls.ts
+++ b/server/tests/api/transcoding/hls.ts
@@ -1,168 +1,48 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { join } from 'path'
4import { basename, join } from 'path' 4import { checkDirectoryIsEmpty, checkTmpIsEmpty, completeCheckHlsPlaylist } from '@server/tests/shared'
5import { 5import { areObjectStorageTestsDisabled } from '@shared/core-utils'
6 checkDirectoryIsEmpty, 6import { HttpStatusCode } from '@shared/models'
7 checkResolutionsInMasterPlaylist,
8 checkSegmentHash,
9 checkTmpIsEmpty,
10 expectStartWith,
11 hlsInfohashExist
12} from '@server/tests/shared'
13import { areObjectStorageTestsDisabled, removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
14import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models'
15import { 7import {
16 cleanupTests, 8 cleanupTests,
17 createMultipleServers, 9 createMultipleServers,
18 doubleFollow, 10 doubleFollow,
19 makeRawRequest,
20 ObjectStorageCommand, 11 ObjectStorageCommand,
21 PeerTubeServer, 12 PeerTubeServer,
22 setAccessTokensToServers, 13 setAccessTokensToServers,
23 waitJobs, 14 waitJobs
24 webtorrentAdd
25} from '@shared/server-commands' 15} from '@shared/server-commands'
26import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants' 16import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants'
27 17
28async function checkHlsPlaylist (options: {
29 servers: PeerTubeServer[]
30 videoUUID: string
31 hlsOnly: boolean
32
33 resolutions?: number[]
34 objectStorageBaseUrl: string
35}) {
36 const { videoUUID, hlsOnly, objectStorageBaseUrl } = options
37
38 const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ]
39
40 for (const server of options.servers) {
41 const videoDetails = await server.videos.get({ id: videoUUID })
42 const baseUrl = `http://${videoDetails.account.host}`
43
44 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
45
46 const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
47 expect(hlsPlaylist).to.not.be.undefined
48
49 const hlsFiles = hlsPlaylist.files
50 expect(hlsFiles).to.have.lengthOf(resolutions.length)
51
52 if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0)
53 else expect(videoDetails.files).to.have.lengthOf(resolutions.length)
54
55 // Check JSON files
56 for (const resolution of resolutions) {
57 const file = hlsFiles.find(f => f.resolution.id === resolution)
58 expect(file).to.not.be.undefined
59
60 expect(file.magnetUri).to.have.lengthOf.above(2)
61 expect(file.torrentUrl).to.match(
62 new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`)
63 )
64
65 if (objectStorageBaseUrl) {
66 expectStartWith(file.fileUrl, objectStorageBaseUrl)
67 } else {
68 expect(file.fileUrl).to.match(
69 new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`)
70 )
71 }
72
73 expect(file.resolution.label).to.equal(resolution + 'p')
74
75 await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
76 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
77
78 const torrent = await webtorrentAdd(file.magnetUri, true)
79 expect(torrent.files).to.be.an('array')
80 expect(torrent.files.length).to.equal(1)
81 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
82 }
83
84 // Check master playlist
85 {
86 await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
87
88 const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl })
89
90 let i = 0
91 for (const resolution of resolutions) {
92 expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
93 expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
94
95 const url = 'http://' + videoDetails.account.host
96 await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i)
97
98 i++
99 }
100 }
101
102 // Check resolution playlists
103 {
104 for (const resolution of resolutions) {
105 const file = hlsFiles.find(f => f.resolution.id === resolution)
106 const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
107
108 const url = objectStorageBaseUrl
109 ? `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}`
110 : `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
111
112 const subPlaylist = await server.streamingPlaylists.get({ url })
113
114 expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
115 expect(subPlaylist).to.contain(basename(file.fileUrl))
116 }
117 }
118
119 {
120 const baseUrlAndPath = objectStorageBaseUrl
121 ? objectStorageBaseUrl + 'hls/' + videoUUID
122 : baseUrl + '/static/streaming-playlists/hls/' + videoUUID
123
124 for (const resolution of resolutions) {
125 await checkSegmentHash({
126 server,
127 baseUrlPlaylist: baseUrlAndPath,
128 baseUrlSegment: baseUrlAndPath,
129 resolution,
130 hlsPlaylist
131 })
132 }
133 }
134 }
135}
136
137describe('Test HLS videos', function () { 18describe('Test HLS videos', function () {
138 let servers: PeerTubeServer[] = [] 19 let servers: PeerTubeServer[] = []
139 let videoUUID = ''
140 let videoAudioUUID = ''
141 20
142 function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) { 21 function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) {
22 const videoUUIDs: string[] = []
143 23
144 it('Should upload a video and transcode it to HLS', async function () { 24 it('Should upload a video and transcode it to HLS', async function () {
145 this.timeout(120000) 25 this.timeout(120000)
146 26
147 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 1', fixture: 'video_short.webm' } }) 27 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 1', fixture: 'video_short.webm' } })
148 videoUUID = uuid 28 videoUUIDs.push(uuid)
149 29
150 await waitJobs(servers) 30 await waitJobs(servers)
151 31
152 await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl }) 32 await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
153 }) 33 })
154 34
155 it('Should upload an audio file and transcode it to HLS', async function () { 35 it('Should upload an audio file and transcode it to HLS', async function () {
156 this.timeout(120000) 36 this.timeout(120000)
157 37
158 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video audio', fixture: 'sample.ogg' } }) 38 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video audio', fixture: 'sample.ogg' } })
159 videoAudioUUID = uuid 39 videoUUIDs.push(uuid)
160 40
161 await waitJobs(servers) 41 await waitJobs(servers)
162 42
163 await checkHlsPlaylist({ 43 await completeCheckHlsPlaylist({
164 servers, 44 servers,
165 videoUUID: videoAudioUUID, 45 videoUUID: uuid,
166 hlsOnly, 46 hlsOnly,
167 resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ], 47 resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ],
168 objectStorageBaseUrl 48 objectStorageBaseUrl
@@ -172,31 +52,36 @@ describe('Test HLS videos', function () {
172 it('Should update the video', async function () { 52 it('Should update the video', async function () {
173 this.timeout(30000) 53 this.timeout(30000)
174 54
175 await servers[0].videos.update({ id: videoUUID, attributes: { name: 'video 1 updated' } }) 55 await servers[0].videos.update({ id: videoUUIDs[0], attributes: { name: 'video 1 updated' } })
176 56
177 await waitJobs(servers) 57 await waitJobs(servers)
178 58
179 await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl }) 59 await completeCheckHlsPlaylist({ servers, videoUUID: videoUUIDs[0], hlsOnly, objectStorageBaseUrl })
180 }) 60 })
181 61
182 it('Should delete videos', async function () { 62 it('Should delete videos', async function () {
183 this.timeout(10000) 63 this.timeout(10000)
184 64
185 await servers[0].videos.remove({ id: videoUUID }) 65 for (const uuid of videoUUIDs) {
186 await servers[0].videos.remove({ id: videoAudioUUID }) 66 await servers[0].videos.remove({ id: uuid })
67 }
187 68
188 await waitJobs(servers) 69 await waitJobs(servers)
189 70
190 for (const server of servers) { 71 for (const server of servers) {
191 await server.videos.get({ id: videoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) 72 for (const uuid of videoUUIDs) {
192 await server.videos.get({ id: videoAudioUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) 73 await server.videos.get({ id: uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
74 }
193 } 75 }
194 }) 76 })
195 77
196 it('Should have the playlists/segment deleted from the disk', async function () { 78 it('Should have the playlists/segment deleted from the disk', async function () {
197 for (const server of servers) { 79 for (const server of servers) {
198 await checkDirectoryIsEmpty(server, 'videos') 80 await checkDirectoryIsEmpty(server, 'videos', [ 'private' ])
199 await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls')) 81 await checkDirectoryIsEmpty(server, join('videos', 'private'))
82
83 await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls'), [ 'private' ])
84 await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls', 'private'))
200 } 85 }
201 }) 86 })
202 87
diff --git a/server/tests/api/transcoding/index.ts b/server/tests/api/transcoding/index.ts
index 0cc28b4a4..9866418d6 100644
--- a/server/tests/api/transcoding/index.ts
+++ b/server/tests/api/transcoding/index.ts
@@ -2,4 +2,5 @@ export * from './audio-only'
2export * from './create-transcoding' 2export * from './create-transcoding'
3export * from './hls' 3export * from './hls'
4export * from './transcoder' 4export * from './transcoder'
5export * from './update-while-transcoding'
5export * from './video-studio' 6export * from './video-studio'
diff --git a/server/tests/api/transcoding/update-while-transcoding.ts b/server/tests/api/transcoding/update-while-transcoding.ts
new file mode 100644
index 000000000..5ca923392
--- /dev/null
+++ b/server/tests/api/transcoding/update-while-transcoding.ts
@@ -0,0 +1,151 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { completeCheckHlsPlaylist } from '@server/tests/shared'
4import { areObjectStorageTestsDisabled, wait } from '@shared/core-utils'
5import { VideoPrivacy } from '@shared/models'
6import {
7 cleanupTests,
8 createMultipleServers,
9 doubleFollow,
10 ObjectStorageCommand,
11 PeerTubeServer,
12 setAccessTokensToServers,
13 waitJobs
14} from '@shared/server-commands'
15
16describe('Test update video privacy while transcoding', function () {
17 let servers: PeerTubeServer[] = []
18
19 const videoUUIDs: string[] = []
20
21 function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) {
22
23 it('Should not have an error while quickly updating a private video to public after upload #1', async function () {
24 this.timeout(360_000)
25
26 const attributes = {
27 name: 'quick update',
28 privacy: VideoPrivacy.PRIVATE
29 }
30
31 const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: false })
32 await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
33 videoUUIDs.push(uuid)
34
35 await waitJobs(servers)
36
37 await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
38 })
39
40 it('Should not have an error while quickly updating a private video to public after upload #2', async function () {
41
42 {
43 const attributes = {
44 name: 'quick update 2',
45 privacy: VideoPrivacy.PRIVATE
46 }
47
48 const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true })
49 await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
50 videoUUIDs.push(uuid)
51
52 await waitJobs(servers)
53
54 await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
55 }
56 })
57
58 it('Should not have an error while quickly updating a private video to public after upload #3', async function () {
59 const attributes = {
60 name: 'quick update 3',
61 privacy: VideoPrivacy.PRIVATE
62 }
63
64 const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true })
65 await wait(1000)
66 await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
67 videoUUIDs.push(uuid)
68
69 await waitJobs(servers)
70
71 await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
72 })
73 }
74
75 before(async function () {
76 this.timeout(120000)
77
78 const configOverride = {
79 transcoding: {
80 enabled: true,
81 allow_audio_files: true,
82 hls: {
83 enabled: true
84 }
85 }
86 }
87 servers = await createMultipleServers(2, configOverride)
88
89 // Get the access tokens
90 await setAccessTokensToServers(servers)
91
92 // Server 1 and server 2 follow each other
93 await doubleFollow(servers[0], servers[1])
94 })
95
96 describe('With WebTorrent & HLS enabled', function () {
97 runTestSuite(false)
98 })
99
100 describe('With only HLS enabled', function () {
101
102 before(async function () {
103 await servers[0].config.updateCustomSubConfig({
104 newConfig: {
105 transcoding: {
106 enabled: true,
107 allowAudioFiles: true,
108 resolutions: {
109 '144p': false,
110 '240p': true,
111 '360p': true,
112 '480p': true,
113 '720p': true,
114 '1080p': true,
115 '1440p': true,
116 '2160p': true
117 },
118 hls: {
119 enabled: true
120 },
121 webtorrent: {
122 enabled: false
123 }
124 }
125 }
126 })
127 })
128
129 runTestSuite(true)
130 })
131
132 describe('With object storage enabled', function () {
133 if (areObjectStorageTestsDisabled()) return
134
135 before(async function () {
136 this.timeout(120000)
137
138 const configOverride = ObjectStorageCommand.getDefaultConfig()
139 await ObjectStorageCommand.prepareDefaultBuckets()
140
141 await servers[0].kill()
142 await servers[0].run(configOverride)
143 })
144
145 runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl())
146 })
147
148 after(async function () {
149 await cleanupTests(servers)
150 })
151})
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index 266155297..357c08199 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -19,3 +19,4 @@ import './videos-common-filters'
19import './videos-history' 19import './videos-history'
20import './videos-overview' 20import './videos-overview'
21import './video-source' 21import './video-source'
22import './video-static-file-privacy'
diff --git a/server/tests/api/videos/video-files.ts b/server/tests/api/videos/video-files.ts
index c0b886aad..8c913bf31 100644
--- a/server/tests/api/videos/video-files.ts
+++ b/server/tests/api/videos/video-files.ts
@@ -153,7 +153,7 @@ describe('Test videos files', function () {
153 expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1) 153 expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1)
154 expect(video.streamingPlaylists[0].files.find(f => f.id === toDelete.id)).to.not.exist 154 expect(video.streamingPlaylists[0].files.find(f => f.id === toDelete.id)).to.not.exist
155 155
156 const { text } = await makeRawRequest(video.streamingPlaylists[0].playlistUrl) 156 const { text } = await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
157 157
158 expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false 158 expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false
159 expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true 159 expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true
diff --git a/server/tests/api/videos/video-static-file-privacy.ts b/server/tests/api/videos/video-static-file-privacy.ts
new file mode 100644
index 000000000..e38fdec6e
--- /dev/null
+++ b/server/tests/api/videos/video-static-file-privacy.ts
@@ -0,0 +1,389 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { decode } from 'magnet-uri'
5import { expectStartWith } from '@server/tests/shared'
6import { getAllFiles, wait } from '@shared/core-utils'
7import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models'
8import {
9 cleanupTests,
10 createSingleServer,
11 findExternalSavedVideo,
12 makeRawRequest,
13 parseTorrentVideo,
14 PeerTubeServer,
15 sendRTMPStream,
16 setAccessTokensToServers,
17 setDefaultVideoChannel,
18 stopFfmpeg,
19 waitJobs
20} from '@shared/server-commands'
21
22describe('Test video static file privacy', function () {
23 let server: PeerTubeServer
24 let userToken: string
25
26 before(async function () {
27 this.timeout(50000)
28
29 server = await createSingleServer(1)
30 await setAccessTokensToServers([ server ])
31 await setDefaultVideoChannel([ server ])
32
33 userToken = await server.users.generateUserAndToken('user1')
34 })
35
36 describe('VOD static file path', function () {
37
38 function runSuite () {
39
40 async function checkPrivateWebTorrentFiles (uuid: string) {
41 const video = await server.videos.getWithToken({ id: uuid })
42
43 for (const file of video.files) {
44 expect(file.fileDownloadUrl).to.not.include('/private/')
45 expectStartWith(file.fileUrl, server.url + '/static/webseed/private/')
46
47 const torrent = await parseTorrentVideo(server, file)
48 expect(torrent.urlList).to.have.lengthOf(0)
49
50 const magnet = decode(file.magnetUri)
51 expect(magnet.urlList).to.have.lengthOf(0)
52
53 await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
54 }
55
56 const hls = video.streamingPlaylists[0]
57 if (hls) {
58 expectStartWith(hls.playlistUrl, server.url + '/static/streaming-playlists/hls/private/')
59 expectStartWith(hls.segmentsSha256Url, server.url + '/static/streaming-playlists/hls/private/')
60
61 await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
62 await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
63 }
64 }
65
66 async function checkPublicWebTorrentFiles (uuid: string) {
67 const video = await server.videos.get({ id: uuid })
68
69 for (const file of getAllFiles(video)) {
70 expect(file.fileDownloadUrl).to.not.include('/private/')
71 expect(file.fileUrl).to.not.include('/private/')
72
73 const torrent = await parseTorrentVideo(server, file)
74 expect(torrent.urlList[0]).to.not.include('private')
75
76 const magnet = decode(file.magnetUri)
77 expect(magnet.urlList[0]).to.not.include('private')
78
79 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
80 await makeRawRequest({ url: torrent.urlList[0], expectedStatus: HttpStatusCode.OK_200 })
81 await makeRawRequest({ url: magnet.urlList[0], expectedStatus: HttpStatusCode.OK_200 })
82 }
83
84 const hls = video.streamingPlaylists[0]
85 if (hls) {
86 expect(hls.playlistUrl).to.not.include('private')
87 expect(hls.segmentsSha256Url).to.not.include('private')
88
89 await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
90 await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
91 }
92 }
93
94 it('Should upload a private/internal video and have a private static path', async function () {
95 this.timeout(120000)
96
97 for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) {
98 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy })
99 await waitJobs([ server ])
100
101 await checkPrivateWebTorrentFiles(uuid)
102 }
103 })
104
105 it('Should upload a public video and update it as private/internal to have a private static path', async function () {
106 this.timeout(120000)
107
108 for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) {
109 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PUBLIC })
110 await waitJobs([ server ])
111
112 await server.videos.update({ id: uuid, attributes: { privacy } })
113 await waitJobs([ server ])
114
115 await checkPrivateWebTorrentFiles(uuid)
116 }
117 })
118
119 it('Should upload a private video and update it to unlisted to have a public static path', async function () {
120 this.timeout(120000)
121
122 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
123 await waitJobs([ server ])
124
125 await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } })
126 await waitJobs([ server ])
127
128 await checkPublicWebTorrentFiles(uuid)
129 })
130
131 it('Should upload an internal video and update it to public to have a public static path', async function () {
132 this.timeout(120000)
133
134 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL })
135 await waitJobs([ server ])
136
137 await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
138 await waitJobs([ server ])
139
140 await checkPublicWebTorrentFiles(uuid)
141 })
142
143 it('Should upload an internal video and schedule a public publish', async function () {
144 this.timeout(120000)
145
146 const attributes = {
147 name: 'video',
148 privacy: VideoPrivacy.PRIVATE,
149 scheduleUpdate: {
150 updateAt: new Date(Date.now() + 1000).toISOString(),
151 privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC
152 }
153 }
154
155 const { uuid } = await server.videos.upload({ attributes })
156
157 await waitJobs([ server ])
158 await wait(1000)
159 await server.debug.sendCommand({ body: { command: 'process-update-videos-scheduler' } })
160
161 await waitJobs([ server ])
162
163 await checkPublicWebTorrentFiles(uuid)
164 })
165 }
166
167 describe('Without transcoding', function () {
168 runSuite()
169 })
170
171 describe('With transcoding', function () {
172
173 before(async function () {
174 await server.config.enableMinimumTranscoding()
175 })
176
177 runSuite()
178 })
179 })
180
181 describe('VOD static file right check', function () {
182 let unrelatedFileToken: string
183
184 async function checkVideoFiles (options: {
185 id: string
186 expectedStatus: HttpStatusCode
187 token: string
188 videoFileToken: string
189 }) {
190 const { id, expectedStatus, token, videoFileToken } = options
191
192 const video = await server.videos.getWithToken({ id })
193
194 for (const file of getAllFiles(video)) {
195 await makeRawRequest({ url: file.fileUrl, token, expectedStatus })
196 await makeRawRequest({ url: file.fileDownloadUrl, token, expectedStatus })
197
198 await makeRawRequest({ url: file.fileUrl, query: { videoFileToken }, expectedStatus })
199 await makeRawRequest({ url: file.fileDownloadUrl, query: { videoFileToken }, expectedStatus })
200 }
201
202 const hls = video.streamingPlaylists[0]
203 await makeRawRequest({ url: hls.playlistUrl, token, expectedStatus })
204 await makeRawRequest({ url: hls.segmentsSha256Url, token, expectedStatus })
205
206 await makeRawRequest({ url: hls.playlistUrl, query: { videoFileToken }, expectedStatus })
207 await makeRawRequest({ url: hls.segmentsSha256Url, query: { videoFileToken }, expectedStatus })
208 }
209
210 before(async function () {
211 await server.config.enableMinimumTranscoding()
212
213 const { uuid } = await server.videos.quickUpload({ name: 'another video' })
214 unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
215 })
216
217 it('Should not be able to access a private video files without OAuth token and file token', async function () {
218 this.timeout(120000)
219
220 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL })
221 await waitJobs([ server ])
222
223 await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403, token: null, videoFileToken: null })
224 })
225
226 it('Should not be able to access an internal video files without appropriate OAuth token and file token', async function () {
227 this.timeout(120000)
228
229 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
230 await waitJobs([ server ])
231
232 await checkVideoFiles({
233 id: uuid,
234 expectedStatus: HttpStatusCode.FORBIDDEN_403,
235 token: userToken,
236 videoFileToken: unrelatedFileToken
237 })
238 })
239
240 it('Should be able to access a private video files with appropriate OAuth token or file token', async function () {
241 this.timeout(120000)
242
243 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
244 const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
245
246 await waitJobs([ server ])
247
248 await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken })
249 })
250
251 it('Should be able to access a private video of another user with an admin OAuth token or file token', async function () {
252 this.timeout(120000)
253
254 const { uuid } = await server.videos.quickUpload({ name: 'video', token: userToken, privacy: VideoPrivacy.PRIVATE })
255 const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
256
257 await waitJobs([ server ])
258
259 await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken })
260 })
261 })
262
263 describe('Live static file path and check', function () {
264 let normalLiveId: string
265 let normalLive: LiveVideo
266
267 let permanentLiveId: string
268 let permanentLive: LiveVideo
269
270 let unrelatedFileToken: string
271
272 async function checkLiveFiles (live: LiveVideo, liveId: string) {
273 const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
274 await server.live.waitUntilPublished({ videoId: liveId })
275
276 const video = await server.videos.getWithToken({ id: liveId })
277 const fileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid })
278
279 const hls = video.streamingPlaylists[0]
280
281 for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
282 expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/')
283
284 await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
285 await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
286
287 await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
288 await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
289 await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
290 }
291
292 await stopFfmpeg(ffmpegCommand)
293 }
294
295 async function checkReplay (replay: VideoDetails) {
296 const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid })
297
298 const hls = replay.streamingPlaylists[0]
299 expect(hls.files).to.not.have.lengthOf(0)
300
301 for (const file of hls.files) {
302 await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
303 await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
304
305 await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
306 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
307 await makeRawRequest({
308 url: file.fileUrl,
309 query: { videoFileToken: unrelatedFileToken },
310 expectedStatus: HttpStatusCode.FORBIDDEN_403
311 })
312 }
313
314 for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
315 expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/')
316
317 await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
318 await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
319
320 await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
321 await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
322 await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
323 }
324 }
325
326 before(async function () {
327 await server.config.enableMinimumTranscoding()
328
329 const { uuid } = await server.videos.quickUpload({ name: 'another video' })
330 unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
331
332 await server.config.enableLive({
333 allowReplay: true,
334 transcoding: true,
335 resolutions: 'min'
336 })
337
338 {
339 const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: false, privacy: VideoPrivacy.PRIVATE })
340 normalLiveId = video.uuid
341 normalLive = live
342 }
343
344 {
345 const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: true, privacy: VideoPrivacy.PRIVATE })
346 permanentLiveId = video.uuid
347 permanentLive = live
348 }
349 })
350
351 it('Should create a private normal live and have a private static path', async function () {
352 this.timeout(240000)
353
354 await checkLiveFiles(normalLive, normalLiveId)
355 })
356
357 it('Should create a private permanent live and have a private static path', async function () {
358 this.timeout(240000)
359
360 await checkLiveFiles(permanentLive, permanentLiveId)
361 })
362
363 it('Should have created a replay of the normal live with a private static path', async function () {
364 this.timeout(240000)
365
366 await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId })
367
368 const replay = await server.videos.getWithToken({ id: normalLiveId })
369 await checkReplay(replay)
370 })
371
372 it('Should have created a replay of the permanent live with a private static path', async function () {
373 this.timeout(240000)
374
375 await server.live.waitUntilWaiting({ videoId: permanentLiveId })
376 await waitJobs([ server ])
377
378 const live = await server.videos.getWithToken({ id: permanentLiveId })
379 const replayFromList = await findExternalSavedVideo(server, live)
380 const replay = await server.videos.getWithToken({ id: replayFromList.id })
381
382 await checkReplay(replay)
383 })
384 })
385
386 after(async function () {
387 await cleanupTests([ server ])
388 })
389})
diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts
index 2cf2dd8f8..a4aa5f699 100644
--- a/server/tests/cli/create-import-video-file-job.ts
+++ b/server/tests/cli/create-import-video-file-job.ts
@@ -29,7 +29,7 @@ async function checkFiles (video: VideoDetails, objectStorage: boolean) {
29 for (const file of video.files) { 29 for (const file of video.files) {
30 if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 30 if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
31 31
32 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 32 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
33 } 33 }
34} 34}
35 35
diff --git a/server/tests/cli/create-move-video-storage-job.ts b/server/tests/cli/create-move-video-storage-job.ts
index 6a12a2c6c..ecdd75b76 100644
--- a/server/tests/cli/create-move-video-storage-job.ts
+++ b/server/tests/cli/create-move-video-storage-job.ts
@@ -22,7 +22,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject
22 22
23 expectStartWith(file.fileUrl, start) 23 expectStartWith(file.fileUrl, start)
24 24
25 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 25 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
26 } 26 }
27 27
28 const start = inObjectStorage 28 const start = inObjectStorage
@@ -36,7 +36,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject
36 for (const file of hls.files) { 36 for (const file of hls.files) {
37 expectStartWith(file.fileUrl, start) 37 expectStartWith(file.fileUrl, start)
38 38
39 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 39 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
40 } 40 }
41} 41}
42 42
diff --git a/server/tests/cli/create-transcoding-job.ts b/server/tests/cli/create-transcoding-job.ts
index 8897d8c23..51bf04a80 100644
--- a/server/tests/cli/create-transcoding-job.ts
+++ b/server/tests/cli/create-transcoding-job.ts
@@ -23,7 +23,7 @@ async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent'
23 23
24 expectStartWith(file.fileUrl, shouldStartWith) 24 expectStartWith(file.fileUrl, shouldStartWith)
25 25
26 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 26 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
27 } 27 }
28} 28}
29 29
diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts
index a89e17e3c..ba0fa1f86 100644
--- a/server/tests/cli/prune-storage.ts
+++ b/server/tests/cli/prune-storage.ts
@@ -5,7 +5,7 @@ import { createFile, readdir } from 'fs-extra'
5import { join } from 'path' 5import { join } from 'path'
6import { wait } from '@shared/core-utils' 6import { wait } from '@shared/core-utils'
7import { buildUUID } from '@shared/extra-utils' 7import { buildUUID } from '@shared/extra-utils'
8import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models' 8import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models'
9import { 9import {
10 cleanupTests, 10 cleanupTests,
11 CLICommand, 11 CLICommand,
@@ -36,22 +36,28 @@ async function assertNotExists (server: PeerTubeServer, directory: string, subst
36async function assertCountAreOkay (servers: PeerTubeServer[]) { 36async function assertCountAreOkay (servers: PeerTubeServer[]) {
37 for (const server of servers) { 37 for (const server of servers) {
38 const videosCount = await countFiles(server, 'videos') 38 const videosCount = await countFiles(server, 'videos')
39 expect(videosCount).to.equal(8) 39 expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory
40
41 const privateVideosCount = await countFiles(server, 'videos/private')
42 expect(privateVideosCount).to.equal(4)
40 43
41 const torrentsCount = await countFiles(server, 'torrents') 44 const torrentsCount = await countFiles(server, 'torrents')
42 expect(torrentsCount).to.equal(16) 45 expect(torrentsCount).to.equal(24)
43 46
44 const previewsCount = await countFiles(server, 'previews') 47 const previewsCount = await countFiles(server, 'previews')
45 expect(previewsCount).to.equal(2) 48 expect(previewsCount).to.equal(3)
46 49
47 const thumbnailsCount = await countFiles(server, 'thumbnails') 50 const thumbnailsCount = await countFiles(server, 'thumbnails')
48 expect(thumbnailsCount).to.equal(6) 51 expect(thumbnailsCount).to.equal(7) // 3 local videos, 1 local playlist, 2 remotes videos and 1 remote playlist
49 52
50 const avatarsCount = await countFiles(server, 'avatars') 53 const avatarsCount = await countFiles(server, 'avatars')
51 expect(avatarsCount).to.equal(4) 54 expect(avatarsCount).to.equal(4)
52 55
53 const hlsRootCount = await countFiles(server, 'streaming-playlists/hls') 56 const hlsRootCount = await countFiles(server, join('streaming-playlists', 'hls'))
54 expect(hlsRootCount).to.equal(2) 57 expect(hlsRootCount).to.equal(3) // 2 videos + private directory
58
59 const hlsPrivateRootCount = await countFiles(server, join('streaming-playlists', 'hls', 'private'))
60 expect(hlsPrivateRootCount).to.equal(1)
55 } 61 }
56} 62}
57 63
@@ -67,8 +73,10 @@ describe('Test prune storage scripts', function () {
67 await setDefaultVideoChannel(servers) 73 await setDefaultVideoChannel(servers)
68 74
69 for (const server of servers) { 75 for (const server of servers) {
70 await server.videos.upload({ attributes: { name: 'video 1' } }) 76 await server.videos.upload({ attributes: { name: 'video 1', privacy: VideoPrivacy.PUBLIC } })
71 await server.videos.upload({ attributes: { name: 'video 2' } }) 77 await server.videos.upload({ attributes: { name: 'video 2', privacy: VideoPrivacy.PUBLIC } })
78
79 await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } })
72 80
73 await server.users.updateMyAvatar({ fixture: 'avatar.png' }) 81 await server.users.updateMyAvatar({ fixture: 'avatar.png' })
74 82
@@ -123,13 +131,16 @@ describe('Test prune storage scripts', function () {
123 it('Should create some dirty files', async function () { 131 it('Should create some dirty files', async function () {
124 for (let i = 0; i < 2; i++) { 132 for (let i = 0; i < 2; i++) {
125 { 133 {
126 const base = servers[0].servers.buildDirectory('videos') 134 const basePublic = servers[0].servers.buildDirectory('videos')
135 const basePrivate = servers[0].servers.buildDirectory(join('videos', 'private'))
127 136
128 const n1 = buildUUID() + '.mp4' 137 const n1 = buildUUID() + '.mp4'
129 const n2 = buildUUID() + '.webm' 138 const n2 = buildUUID() + '.webm'
130 139
131 await createFile(join(base, n1)) 140 await createFile(join(basePublic, n1))
132 await createFile(join(base, n2)) 141 await createFile(join(basePublic, n2))
142 await createFile(join(basePrivate, n1))
143 await createFile(join(basePrivate, n2))
133 144
134 badNames['videos'] = [ n1, n2 ] 145 badNames['videos'] = [ n1, n2 ]
135 } 146 }
@@ -184,10 +195,12 @@ describe('Test prune storage scripts', function () {
184 195
185 { 196 {
186 const directory = join('streaming-playlists', 'hls') 197 const directory = join('streaming-playlists', 'hls')
187 const base = servers[0].servers.buildDirectory(directory) 198 const basePublic = servers[0].servers.buildDirectory(directory)
199 const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private'))
188 200
189 const n1 = buildUUID() 201 const n1 = buildUUID()
190 await createFile(join(base, n1)) 202 await createFile(join(basePublic, n1))
203 await createFile(join(basePrivate, n1))
191 badNames[directory] = [ n1 ] 204 badNames[directory] = [ n1 ]
192 } 205 }
193 } 206 }
diff --git a/server/tests/cli/regenerate-thumbnails.ts b/server/tests/cli/regenerate-thumbnails.ts
index f459b11b8..16a8adcda 100644
--- a/server/tests/cli/regenerate-thumbnails.ts
+++ b/server/tests/cli/regenerate-thumbnails.ts
@@ -6,7 +6,7 @@ import {
6 cleanupTests, 6 cleanupTests,
7 createMultipleServers, 7 createMultipleServers,
8 doubleFollow, 8 doubleFollow,
9 makeRawRequest, 9 makeGetRequest,
10 PeerTubeServer, 10 PeerTubeServer,
11 setAccessTokensToServers, 11 setAccessTokensToServers,
12 waitJobs 12 waitJobs
@@ -16,8 +16,8 @@ async function testThumbnail (server: PeerTubeServer, videoId: number | string)
16 const video = await server.videos.get({ id: videoId }) 16 const video = await server.videos.get({ id: videoId })
17 17
18 const requests = [ 18 const requests = [
19 makeRawRequest(join(server.url, video.thumbnailPath), HttpStatusCode.OK_200), 19 makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }),
20 makeRawRequest(join(server.url, video.thumbnailPath), HttpStatusCode.OK_200) 20 makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
21 ] 21 ]
22 22
23 for (const req of requests) { 23 for (const req of requests) {
@@ -69,17 +69,17 @@ describe('Test regenerate thumbnails script', function () {
69 69
70 it('Should have empty thumbnails', async function () { 70 it('Should have empty thumbnails', async function () {
71 { 71 {
72 const res = await makeRawRequest(join(servers[0].url, video1.thumbnailPath), HttpStatusCode.OK_200) 72 const res = await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
73 expect(res.body).to.have.lengthOf(0) 73 expect(res.body).to.have.lengthOf(0)
74 } 74 }
75 75
76 { 76 {
77 const res = await makeRawRequest(join(servers[0].url, video2.thumbnailPath), HttpStatusCode.OK_200) 77 const res = await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
78 expect(res.body).to.not.have.lengthOf(0) 78 expect(res.body).to.not.have.lengthOf(0)
79 } 79 }
80 80
81 { 81 {
82 const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200) 82 const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
83 expect(res.body).to.have.lengthOf(0) 83 expect(res.body).to.have.lengthOf(0)
84 } 84 }
85 }) 85 })
@@ -94,21 +94,21 @@ describe('Test regenerate thumbnails script', function () {
94 await testThumbnail(servers[0], video1.uuid) 94 await testThumbnail(servers[0], video1.uuid)
95 await testThumbnail(servers[0], video2.uuid) 95 await testThumbnail(servers[0], video2.uuid)
96 96
97 const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200) 97 const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
98 expect(res.body).to.have.lengthOf(0) 98 expect(res.body).to.have.lengthOf(0)
99 }) 99 })
100 100
101 it('Should have deleted old thumbnail files', async function () { 101 it('Should have deleted old thumbnail files', async function () {
102 { 102 {
103 await makeRawRequest(join(servers[0].url, video1.thumbnailPath), HttpStatusCode.NOT_FOUND_404) 103 await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
104 } 104 }
105 105
106 { 106 {
107 await makeRawRequest(join(servers[0].url, video2.thumbnailPath), HttpStatusCode.NOT_FOUND_404) 107 await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
108 } 108 }
109 109
110 { 110 {
111 const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200) 111 const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
112 expect(res.body).to.have.lengthOf(0) 112 expect(res.body).to.have.lengthOf(0)
113 } 113 }
114 }) 114 })
diff --git a/server/tests/feeds/feeds.ts b/server/tests/feeds/feeds.ts
index 0ddb641e6..c49175d5e 100644
--- a/server/tests/feeds/feeds.ts
+++ b/server/tests/feeds/feeds.ts
@@ -314,7 +314,7 @@ describe('Test syndication feeds', () => {
314 const jsonObj = JSON.parse(json) 314 const jsonObj = JSON.parse(json)
315 const imageUrl = jsonObj.icon 315 const imageUrl = jsonObj.icon
316 expect(imageUrl).to.include('/lazy-static/avatars/') 316 expect(imageUrl).to.include('/lazy-static/avatars/')
317 await makeRawRequest(imageUrl) 317 await makeRawRequest({ url: imageUrl })
318 }) 318 })
319 }) 319 })
320 320
diff --git a/server/tests/plugins/filter-hooks.ts b/server/tests/plugins/filter-hooks.ts
index ae4b3cf5f..083fd43ca 100644
--- a/server/tests/plugins/filter-hooks.ts
+++ b/server/tests/plugins/filter-hooks.ts
@@ -6,6 +6,7 @@ import {
6 cleanupTests, 6 cleanupTests,
7 createMultipleServers, 7 createMultipleServers,
8 doubleFollow, 8 doubleFollow,
9 makeGetRequest,
9 makeRawRequest, 10 makeRawRequest,
10 PeerTubeServer, 11 PeerTubeServer,
11 PluginsCommand, 12 PluginsCommand,
@@ -461,30 +462,41 @@ describe('Test plugin filter hooks', function () {
461 }) 462 })
462 463
463 it('Should run filter:api.download.torrent.allowed.result', async function () { 464 it('Should run filter:api.download.torrent.allowed.result', async function () {
464 const res = await makeRawRequest(downloadVideos[0].files[0].torrentDownloadUrl, 403) 465 const res = await makeRawRequest({ url: downloadVideos[0].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
465 expect(res.body.error).to.equal('Liu Bei') 466 expect(res.body.error).to.equal('Liu Bei')
466 467
467 await makeRawRequest(downloadVideos[1].files[0].torrentDownloadUrl, 200) 468 await makeRawRequest({ url: downloadVideos[1].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
468 await makeRawRequest(downloadVideos[2].files[0].torrentDownloadUrl, 200) 469 await makeRawRequest({ url: downloadVideos[2].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
469 }) 470 })
470 471
471 it('Should run filter:api.download.video.allowed.result', async function () { 472 it('Should run filter:api.download.video.allowed.result', async function () {
472 { 473 {
473 const res = await makeRawRequest(downloadVideos[1].files[0].fileDownloadUrl, 403) 474 const res = await makeRawRequest({ url: downloadVideos[1].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
474 expect(res.body.error).to.equal('Cao Cao') 475 expect(res.body.error).to.equal('Cao Cao')
475 476
476 await makeRawRequest(downloadVideos[0].files[0].fileDownloadUrl, 200) 477 await makeRawRequest({ url: downloadVideos[0].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
477 await makeRawRequest(downloadVideos[2].files[0].fileDownloadUrl, 200) 478 await makeRawRequest({ url: downloadVideos[2].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
478 } 479 }
479 480
480 { 481 {
481 const res = await makeRawRequest(downloadVideos[2].streamingPlaylists[0].files[0].fileDownloadUrl, 403) 482 const res = await makeRawRequest({
483 url: downloadVideos[2].streamingPlaylists[0].files[0].fileDownloadUrl,
484 expectedStatus: HttpStatusCode.FORBIDDEN_403
485 })
486
482 expect(res.body.error).to.equal('Sun Jian') 487 expect(res.body.error).to.equal('Sun Jian')
483 488
484 await makeRawRequest(downloadVideos[2].files[0].fileDownloadUrl, 200) 489 await makeRawRequest({ url: downloadVideos[2].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
490
491 await makeRawRequest({
492 url: downloadVideos[0].streamingPlaylists[0].files[0].fileDownloadUrl,
493 expectedStatus: HttpStatusCode.OK_200
494 })
485 495
486 await makeRawRequest(downloadVideos[0].streamingPlaylists[0].files[0].fileDownloadUrl, 200) 496 await makeRawRequest({
487 await makeRawRequest(downloadVideos[1].streamingPlaylists[0].files[0].fileDownloadUrl, 200) 497 url: downloadVideos[1].streamingPlaylists[0].files[0].fileDownloadUrl,
498 expectedStatus: HttpStatusCode.OK_200
499 })
488 } 500 }
489 }) 501 })
490 }) 502 })
@@ -515,12 +527,12 @@ describe('Test plugin filter hooks', function () {
515 }) 527 })
516 528
517 it('Should run filter:html.embed.video.allowed.result', async function () { 529 it('Should run filter:html.embed.video.allowed.result', async function () {
518 const res = await makeRawRequest(servers[0].url + embedVideos[0].embedPath, 200) 530 const res = await makeGetRequest({ url: servers[0].url, path: embedVideos[0].embedPath, expectedStatus: HttpStatusCode.OK_200 })
519 expect(res.text).to.equal('Lu Bu') 531 expect(res.text).to.equal('Lu Bu')
520 }) 532 })
521 533
522 it('Should run filter:html.embed.video-playlist.allowed.result', async function () { 534 it('Should run filter:html.embed.video-playlist.allowed.result', async function () {
523 const res = await makeRawRequest(servers[0].url + embedPlaylists[0].embedPath, 200) 535 const res = await makeGetRequest({ url: servers[0].url, path: embedPlaylists[0].embedPath, expectedStatus: HttpStatusCode.OK_200 })
524 expect(res.text).to.equal('Diao Chan') 536 expect(res.text).to.equal('Diao Chan')
525 }) 537 })
526 }) 538 })
diff --git a/server/tests/plugins/plugin-helpers.ts b/server/tests/plugins/plugin-helpers.ts
index 31c18350a..f2bada4ee 100644
--- a/server/tests/plugins/plugin-helpers.ts
+++ b/server/tests/plugins/plugin-helpers.ts
@@ -307,7 +307,7 @@ describe('Test plugin helpers', function () {
307 expect(file.fps).to.equal(25) 307 expect(file.fps).to.equal(25)
308 308
309 expect(await pathExists(file.path)).to.be.true 309 expect(await pathExists(file.path)).to.be.true
310 await makeRawRequest(file.url, HttpStatusCode.OK_200) 310 await makeRawRequest({ url: file.url, expectedStatus: HttpStatusCode.OK_200 })
311 } 311 }
312 } 312 }
313 313
@@ -321,12 +321,12 @@ describe('Test plugin helpers', function () {
321 const miniature = body.thumbnails.find(t => t.type === ThumbnailType.MINIATURE) 321 const miniature = body.thumbnails.find(t => t.type === ThumbnailType.MINIATURE)
322 expect(miniature).to.exist 322 expect(miniature).to.exist
323 expect(await pathExists(miniature.path)).to.be.true 323 expect(await pathExists(miniature.path)).to.be.true
324 await makeRawRequest(miniature.url, HttpStatusCode.OK_200) 324 await makeRawRequest({ url: miniature.url, expectedStatus: HttpStatusCode.OK_200 })
325 325
326 const preview = body.thumbnails.find(t => t.type === ThumbnailType.PREVIEW) 326 const preview = body.thumbnails.find(t => t.type === ThumbnailType.PREVIEW)
327 expect(preview).to.exist 327 expect(preview).to.exist
328 expect(await pathExists(preview.path)).to.be.true 328 expect(await pathExists(preview.path)).to.be.true
329 await makeRawRequest(preview.url, HttpStatusCode.OK_200) 329 await makeRawRequest({ url: preview.url, expectedStatus: HttpStatusCode.OK_200 })
330 } 330 }
331 }) 331 })
332 332
diff --git a/server/tests/shared/streaming-playlists.ts b/server/tests/shared/streaming-playlists.ts
index 74c25e99c..8ee04d921 100644
--- a/server/tests/shared/streaming-playlists.ts
+++ b/server/tests/shared/streaming-playlists.ts
@@ -1,9 +1,13 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
1import { expect } from 'chai' 3import { expect } from 'chai'
2import { basename } from 'path' 4import { basename } from 'path'
3import { removeFragmentedMP4Ext } from '@shared/core-utils' 5import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
4import { sha256 } from '@shared/extra-utils' 6import { sha256 } from '@shared/extra-utils'
5import { HttpStatusCode, VideoStreamingPlaylist } from '@shared/models' 7import { HttpStatusCode, VideoStreamingPlaylist, VideoStreamingPlaylistType } from '@shared/models'
6import { PeerTubeServer } from '@shared/server-commands' 8import { makeRawRequest, PeerTubeServer, webtorrentAdd } from '@shared/server-commands'
9import { expectStartWith } from './checks'
10import { hlsInfohashExist } from './tracker'
7 11
8async function checkSegmentHash (options: { 12async function checkSegmentHash (options: {
9 server: PeerTubeServer 13 server: PeerTubeServer
@@ -75,8 +79,118 @@ async function checkResolutionsInMasterPlaylist (options: {
75 expect(playlistsLength).to.have.lengthOf(resolutions.length) 79 expect(playlistsLength).to.have.lengthOf(resolutions.length)
76} 80}
77 81
82async function completeCheckHlsPlaylist (options: {
83 servers: PeerTubeServer[]
84 videoUUID: string
85 hlsOnly: boolean
86
87 resolutions?: number[]
88 objectStorageBaseUrl: string
89}) {
90 const { videoUUID, hlsOnly, objectStorageBaseUrl } = options
91
92 const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ]
93
94 for (const server of options.servers) {
95 const videoDetails = await server.videos.get({ id: videoUUID })
96 const baseUrl = `http://${videoDetails.account.host}`
97
98 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
99
100 const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
101 expect(hlsPlaylist).to.not.be.undefined
102
103 const hlsFiles = hlsPlaylist.files
104 expect(hlsFiles).to.have.lengthOf(resolutions.length)
105
106 if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0)
107 else expect(videoDetails.files).to.have.lengthOf(resolutions.length)
108
109 // Check JSON files
110 for (const resolution of resolutions) {
111 const file = hlsFiles.find(f => f.resolution.id === resolution)
112 expect(file).to.not.be.undefined
113
114 expect(file.magnetUri).to.have.lengthOf.above(2)
115 expect(file.torrentUrl).to.match(
116 new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`)
117 )
118
119 if (objectStorageBaseUrl) {
120 expectStartWith(file.fileUrl, objectStorageBaseUrl)
121 } else {
122 expect(file.fileUrl).to.match(
123 new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`)
124 )
125 }
126
127 expect(file.resolution.label).to.equal(resolution + 'p')
128
129 await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 })
130 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
131
132 const torrent = await webtorrentAdd(file.magnetUri, true)
133 expect(torrent.files).to.be.an('array')
134 expect(torrent.files.length).to.equal(1)
135 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
136 }
137
138 // Check master playlist
139 {
140 await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
141
142 const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl })
143
144 let i = 0
145 for (const resolution of resolutions) {
146 expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
147 expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
148
149 const url = 'http://' + videoDetails.account.host
150 await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i)
151
152 i++
153 }
154 }
155
156 // Check resolution playlists
157 {
158 for (const resolution of resolutions) {
159 const file = hlsFiles.find(f => f.resolution.id === resolution)
160 const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
161
162 const url = objectStorageBaseUrl
163 ? `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}`
164 : `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
165
166 const subPlaylist = await server.streamingPlaylists.get({ url })
167
168 expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
169 expect(subPlaylist).to.contain(basename(file.fileUrl))
170 }
171 }
172
173 {
174 const baseUrlAndPath = objectStorageBaseUrl
175 ? objectStorageBaseUrl + 'hls/' + videoUUID
176 : baseUrl + '/static/streaming-playlists/hls/' + videoUUID
177
178 for (const resolution of resolutions) {
179 await checkSegmentHash({
180 server,
181 baseUrlPlaylist: baseUrlAndPath,
182 baseUrlSegment: baseUrlAndPath,
183 resolution,
184 hlsPlaylist
185 })
186 }
187 }
188 }
189}
190
78export { 191export {
79 checkSegmentHash, 192 checkSegmentHash,
80 checkLiveSegmentHash, 193 checkLiveSegmentHash,
81 checkResolutionsInMasterPlaylist 194 checkResolutionsInMasterPlaylist,
195 completeCheckHlsPlaylist
82} 196}
diff --git a/server/tests/shared/videos.ts b/server/tests/shared/videos.ts
index e18329e07..c8339584b 100644
--- a/server/tests/shared/videos.ts
+++ b/server/tests/shared/videos.ts
@@ -125,9 +125,9 @@ async function completeVideoCheck (
125 expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`)) 125 expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`))
126 126
127 await Promise.all([ 127 await Promise.all([
128 makeRawRequest(file.torrentUrl, 200), 128 makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }),
129 makeRawRequest(file.torrentDownloadUrl, 200), 129 makeRawRequest({ url: file.torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }),
130 makeRawRequest(file.metadataUrl, 200) 130 makeRawRequest({ url: file.metadataUrl, expectedStatus: HttpStatusCode.OK_200 })
131 ]) 131 ])
132 132
133 expect(file.resolution.id).to.equal(attributeFile.resolution) 133 expect(file.resolution.id).to.equal(attributeFile.resolution)