aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-11-18 14:35:08 +0100
committerChocobozzz <me@florianbigard.com>2021-11-18 15:20:57 +0100
commitad5db1044c8599eaaaa2a578b350777ae996b068 (patch)
tree3e003cccf021152405d49b21c6c91b703c8ae96c /server
parentb46cf4b920984492df598c1b61179acfc7f6f22e (diff)
downloadPeerTube-ad5db1044c8599eaaaa2a578b350777ae996b068.tar.gz
PeerTube-ad5db1044c8599eaaaa2a578b350777ae996b068.tar.zst
PeerTube-ad5db1044c8599eaaaa2a578b350777ae996b068.zip
Add ability to run transcoding jobs
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/videos/files.ts5
-rw-r--r--server/controllers/api/videos/index.ts2
-rw-r--r--server/controllers/api/videos/transcoding.ts62
-rw-r--r--server/controllers/download.ts4
-rw-r--r--server/helpers/custom-validators/video-transcoding.ts12
-rw-r--r--server/helpers/ffprobe-utils.ts6
-rw-r--r--server/helpers/webtorrent.ts2
-rw-r--r--server/lib/hls.ts7
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts11
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts10
-rw-r--r--server/lib/live/live-manager.ts4
-rw-r--r--server/lib/object-storage/keys.ts10
-rw-r--r--server/lib/object-storage/videos.ts16
-rw-r--r--server/lib/thumbnail.ts2
-rw-r--r--server/lib/transcoding/video-transcoding.ts6
-rw-r--r--server/lib/video-path-manager.ts26
-rw-r--r--server/lib/video-state.ts2
-rw-r--r--server/lib/video.ts2
-rw-r--r--server/middlewares/validators/videos/index.ts1
-rw-r--r--server/middlewares/validators/videos/video-files.ts21
-rw-r--r--server/middlewares/validators/videos/video-transcoding.ts55
-rw-r--r--server/models/video/formatter/video-format-utils.ts3
-rw-r--r--server/models/video/video-file.ts8
-rw-r--r--server/models/video/video-job-info.ts2
-rw-r--r--server/models/video/video-streaming-playlist.ts4
-rw-r--r--server/models/video/video.ts9
-rw-r--r--server/tests/api/check-params/index.ts1
-rw-r--r--server/tests/api/check-params/transcoding.ts104
-rw-r--r--server/tests/api/check-params/video-files.ts17
-rw-r--r--server/tests/api/videos/index.ts1
-rw-r--r--server/tests/api/videos/video-create-transcoding.ts156
31 files changed, 495 insertions, 76 deletions
diff --git a/server/controllers/api/videos/files.ts b/server/controllers/api/videos/files.ts
index 2fe4b5a3f..a8b32411d 100644
--- a/server/controllers/api/videos/files.ts
+++ b/server/controllers/api/videos/files.ts
@@ -3,10 +3,11 @@ import toInt from 'validator/lib/toInt'
3import { logger, loggerTagsFactory } from '@server/helpers/logger' 3import { logger, loggerTagsFactory } from '@server/helpers/logger'
4import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' 4import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
5import { VideoFileModel } from '@server/models/video/video-file' 5import { VideoFileModel } from '@server/models/video/video-file'
6import { HttpStatusCode } from '@shared/models' 6import { HttpStatusCode, UserRight } from '@shared/models'
7import { 7import {
8 asyncMiddleware, 8 asyncMiddleware,
9 authenticate, 9 authenticate,
10 ensureUserHasRight,
10 videoFileMetadataGetValidator, 11 videoFileMetadataGetValidator,
11 videoFilesDeleteHLSValidator, 12 videoFilesDeleteHLSValidator,
12 videoFilesDeleteWebTorrentValidator 13 videoFilesDeleteWebTorrentValidator
@@ -22,12 +23,14 @@ filesRouter.get('/:id/metadata/:videoFileId',
22 23
23filesRouter.delete('/:id/hls', 24filesRouter.delete('/:id/hls',
24 authenticate, 25 authenticate,
26 ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES),
25 asyncMiddleware(videoFilesDeleteHLSValidator), 27 asyncMiddleware(videoFilesDeleteHLSValidator),
26 asyncMiddleware(removeHLSPlaylist) 28 asyncMiddleware(removeHLSPlaylist)
27) 29)
28 30
29filesRouter.delete('/:id/webtorrent', 31filesRouter.delete('/:id/webtorrent',
30 authenticate, 32 authenticate,
33 ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES),
31 asyncMiddleware(videoFilesDeleteWebTorrentValidator), 34 asyncMiddleware(videoFilesDeleteWebTorrentValidator),
32 asyncMiddleware(removeWebTorrentFiles) 35 asyncMiddleware(removeWebTorrentFiles)
33) 36)
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 2d088a73e..fc1bcc73d 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -40,6 +40,7 @@ import { videoImportsRouter } from './import'
40import { liveRouter } from './live' 40import { liveRouter } from './live'
41import { ownershipVideoRouter } from './ownership' 41import { ownershipVideoRouter } from './ownership'
42import { rateVideoRouter } from './rate' 42import { rateVideoRouter } from './rate'
43import { transcodingRouter } from './transcoding'
43import { updateRouter } from './update' 44import { updateRouter } from './update'
44import { uploadRouter } from './upload' 45import { uploadRouter } from './upload'
45import { watchingRouter } from './watching' 46import { watchingRouter } from './watching'
@@ -58,6 +59,7 @@ videosRouter.use('/', liveRouter)
58videosRouter.use('/', uploadRouter) 59videosRouter.use('/', uploadRouter)
59videosRouter.use('/', updateRouter) 60videosRouter.use('/', updateRouter)
60videosRouter.use('/', filesRouter) 61videosRouter.use('/', filesRouter)
62videosRouter.use('/', transcodingRouter)
61 63
62videosRouter.get('/categories', 64videosRouter.get('/categories',
63 openapiOperationDoc({ operationId: 'getCategories' }), 65 openapiOperationDoc({ operationId: 'getCategories' }),
diff --git a/server/controllers/api/videos/transcoding.ts b/server/controllers/api/videos/transcoding.ts
new file mode 100644
index 000000000..dd6fbd3de
--- /dev/null
+++ b/server/controllers/api/videos/transcoding.ts
@@ -0,0 +1,62 @@
1import express from 'express'
2import { computeLowerResolutionsToTranscode } from '@server/helpers/ffprobe-utils'
3import { logger, loggerTagsFactory } from '@server/helpers/logger'
4import { addTranscodingJob } from '@server/lib/video'
5import { HttpStatusCode, UserRight, VideoState, VideoTranscodingCreate } from '@shared/models'
6import { asyncMiddleware, authenticate, createTranscodingValidator, ensureUserHasRight } from '../../../middlewares'
7
8const lTags = loggerTagsFactory('api', 'video')
9const transcodingRouter = express.Router()
10
11transcodingRouter.post('/:videoId/transcoding',
12 authenticate,
13 ensureUserHasRight(UserRight.RUN_VIDEO_TRANSCODING),
14 asyncMiddleware(createTranscodingValidator),
15 asyncMiddleware(createTranscoding)
16)
17
18// ---------------------------------------------------------------------------
19
20export {
21 transcodingRouter
22}
23
24// ---------------------------------------------------------------------------
25
26async function createTranscoding (req: express.Request, res: express.Response) {
27 const video = res.locals.videoAll
28 logger.info('Creating %s transcoding job for %s.', req.body.type, video.url, lTags())
29
30 const body: VideoTranscodingCreate = req.body
31
32 const { resolution: maxResolution, isPortraitMode } = await video.getMaxQualityResolution()
33 const resolutions = computeLowerResolutionsToTranscode(maxResolution, 'vod').concat([ maxResolution ])
34
35 video.state = VideoState.TO_TRANSCODE
36 await video.save()
37
38 for (const resolution of resolutions) {
39 if (body.transcodingType === 'hls') {
40 await addTranscodingJob({
41 type: 'new-resolution-to-hls',
42 videoUUID: video.uuid,
43 resolution,
44 isPortraitMode,
45 copyCodecs: false,
46 isNewVideo: false,
47 autoDeleteWebTorrentIfNeeded: false,
48 isMaxQuality: maxResolution === resolution
49 })
50 } else if (body.transcodingType === 'webtorrent') {
51 await addTranscodingJob({
52 type: 'new-resolution-to-webtorrent',
53 videoUUID: video.uuid,
54 isNewVideo: false,
55 resolution: resolution,
56 isPortraitMode
57 })
58 }
59 }
60
61 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
62}
diff --git a/server/controllers/download.ts b/server/controllers/download.ts
index 8da710669..43d525f83 100644
--- a/server/controllers/download.ts
+++ b/server/controllers/download.ts
@@ -85,7 +85,7 @@ async function downloadVideoFile (req: express.Request, res: express.Response) {
85 return res.redirect(videoFile.getObjectStorageUrl()) 85 return res.redirect(videoFile.getObjectStorageUrl())
86 } 86 }
87 87
88 await VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, path => { 88 await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => {
89 const filename = `${video.name}-${videoFile.resolution}p${videoFile.extname}` 89 const filename = `${video.name}-${videoFile.resolution}p${videoFile.extname}`
90 90
91 return res.download(path, filename) 91 return res.download(path, filename)
@@ -119,7 +119,7 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
119 return res.redirect(videoFile.getObjectStorageUrl()) 119 return res.redirect(videoFile.getObjectStorageUrl())
120 } 120 }
121 121
122 await VideoPathManager.Instance.makeAvailableVideoFile(streamingPlaylist, videoFile, path => { 122 await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => {
123 const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}` 123 const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}`
124 124
125 return res.download(path, filename) 125 return res.download(path, filename)
diff --git a/server/helpers/custom-validators/video-transcoding.ts b/server/helpers/custom-validators/video-transcoding.ts
new file mode 100644
index 000000000..cf792f996
--- /dev/null
+++ b/server/helpers/custom-validators/video-transcoding.ts
@@ -0,0 +1,12 @@
1import { exists } from './misc'
2
3function isValidCreateTranscodingType (value: any) {
4 return exists(value) &&
5 (value === 'hls' || value === 'webtorrent')
6}
7
8// ---------------------------------------------------------------------------
9
10export {
11 isValidCreateTranscodingType
12}
diff --git a/server/helpers/ffprobe-utils.ts b/server/helpers/ffprobe-utils.ts
index 907f13651..e15628e2a 100644
--- a/server/helpers/ffprobe-utils.ts
+++ b/server/helpers/ffprobe-utils.ts
@@ -206,7 +206,7 @@ async function getVideoStreamFromFile (path: string, existingProbe?: FfprobeData
206 return metadata.streams.find(s => s.codec_type === 'video') || null 206 return metadata.streams.find(s => s.codec_type === 'video') || null
207} 207}
208 208
209function computeResolutionsToTranscode (videoFileResolution: number, type: 'vod' | 'live') { 209function computeLowerResolutionsToTranscode (videoFileResolution: number, type: 'vod' | 'live') {
210 const configResolutions = type === 'vod' 210 const configResolutions = type === 'vod'
211 ? CONFIG.TRANSCODING.RESOLUTIONS 211 ? CONFIG.TRANSCODING.RESOLUTIONS
212 : CONFIG.LIVE.TRANSCODING.RESOLUTIONS 212 : CONFIG.LIVE.TRANSCODING.RESOLUTIONS
@@ -214,7 +214,7 @@ function computeResolutionsToTranscode (videoFileResolution: number, type: 'vod'
214 const resolutionsEnabled: number[] = [] 214 const resolutionsEnabled: number[] = []
215 215
216 // Put in the order we want to proceed jobs 216 // Put in the order we want to proceed jobs
217 const resolutions = [ 217 const resolutions: VideoResolution[] = [
218 VideoResolution.H_NOVIDEO, 218 VideoResolution.H_NOVIDEO,
219 VideoResolution.H_480P, 219 VideoResolution.H_480P,
220 VideoResolution.H_360P, 220 VideoResolution.H_360P,
@@ -327,7 +327,7 @@ export {
327 getVideoFileFPS, 327 getVideoFileFPS,
328 ffprobePromise, 328 ffprobePromise,
329 getClosestFramerateStandard, 329 getClosestFramerateStandard,
330 computeResolutionsToTranscode, 330 computeLowerResolutionsToTranscode,
331 getVideoFileBitrate, 331 getVideoFileBitrate,
332 canDoQuickTranscode, 332 canDoQuickTranscode,
333 canDoQuickVideoTranscode, 333 canDoQuickVideoTranscode,
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 5e1ea6198..c75c058e4 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -100,7 +100,7 @@ function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlayli
100 urlList: buildUrlList(video, videoFile) 100 urlList: buildUrlList(video, videoFile)
101 } 101 }
102 102
103 return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => { 103 return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(videoOrPlaylist), async videoPath => {
104 const torrentContent = await createTorrentPromise(videoPath, options) 104 const torrentContent = await createTorrentPromise(videoPath, options)
105 105
106 const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) 106 const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 8160e7949..d969549b8 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -37,7 +37,7 @@ async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlayl
37 for (const file of playlist.VideoFiles) { 37 for (const file of playlist.VideoFiles) {
38 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) 38 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
39 39
40 await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, async videoFilePath => { 40 await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => {
41 const size = await getVideoStreamSize(videoFilePath) 41 const size = await getVideoStreamSize(videoFilePath)
42 42
43 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) 43 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
@@ -69,10 +69,11 @@ async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingP
69 // For all the resolutions available for this video 69 // For all the resolutions available for this video
70 for (const file of playlist.VideoFiles) { 70 for (const file of playlist.VideoFiles) {
71 const rangeHashes: { [range: string]: string } = {} 71 const rangeHashes: { [range: string]: string } = {}
72 const fileWithPlaylist = file.withVideoOrPlaylist(playlist)
72 73
73 await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, videoPath => { 74 await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => {
74 75
75 return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(playlist, file, async resolutionPlaylistPath => { 76 return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => {
76 const playlistContent = await readFile(resolutionPlaylistPath) 77 const playlistContent = await readFile(resolutionPlaylistPath)
77 const ranges = getRangesFromPlaylist(playlistContent.toString()) 78 const ranges = getRangesFromPlaylist(playlistContent.toString())
78 79
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index 4beca3d75..54a7c566b 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -56,16 +56,17 @@ async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
56 56
57async function moveHLSFiles (video: MVideoWithAllFiles) { 57async function moveHLSFiles (video: MVideoWithAllFiles) {
58 for (const playlist of video.VideoStreamingPlaylists) { 58 for (const playlist of video.VideoStreamingPlaylists) {
59 const playlistWithVideo = playlist.withVideo(video)
59 60
60 for (const file of playlist.VideoFiles) { 61 for (const file of playlist.VideoFiles) {
61 if (file.storage !== VideoStorage.FILE_SYSTEM) continue 62 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
62 63
63 // Resolution playlist 64 // Resolution playlist
64 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) 65 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
65 await storeHLSFile(playlist, video, playlistFilename) 66 await storeHLSFile(playlistWithVideo, playlistFilename)
66 67
67 // Resolution fragmented file 68 // Resolution fragmented file
68 const fileUrl = await storeHLSFile(playlist, video, file.filename) 69 const fileUrl = await storeHLSFile(playlistWithVideo, file.filename)
69 70
70 const oldPath = join(getHLSDirectory(video), file.filename) 71 const oldPath = join(getHLSDirectory(video), file.filename)
71 72
@@ -78,10 +79,12 @@ async function doAfterLastJob (video: MVideoWithAllFiles, isNewVideo: boolean) {
78 for (const playlist of video.VideoStreamingPlaylists) { 79 for (const playlist of video.VideoStreamingPlaylists) {
79 if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue 80 if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
80 81
82 const playlistWithVideo = playlist.withVideo(video)
83
81 // Master playlist 84 // Master playlist
82 playlist.playlistUrl = await storeHLSFile(playlist, video, playlist.playlistFilename) 85 playlist.playlistUrl = await storeHLSFile(playlistWithVideo, playlist.playlistFilename)
83 // Sha256 segments file 86 // Sha256 segments file
84 playlist.segmentsSha256Url = await storeHLSFile(playlist, video, playlist.segmentsSha256Filename) 87 playlist.segmentsSha256Url = await storeHLSFile(playlistWithVideo, playlist.segmentsSha256Filename)
85 88
86 playlist.storage = VideoStorage.OBJECT_STORAGE 89 playlist.storage = VideoStorage.OBJECT_STORAGE
87 90
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 904ef2e3c..2d0798e12 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -14,7 +14,7 @@ import {
14 VideoTranscodingPayload 14 VideoTranscodingPayload
15} from '../../../../shared' 15} from '../../../../shared'
16import { retryTransactionWrapper } from '../../../helpers/database-utils' 16import { retryTransactionWrapper } from '../../../helpers/database-utils'
17import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils' 17import { computeLowerResolutionsToTranscode } from '../../../helpers/ffprobe-utils'
18import { logger, loggerTagsFactory } from '../../../helpers/logger' 18import { logger, loggerTagsFactory } from '../../../helpers/logger'
19import { CONFIG } from '../../../initializers/config' 19import { CONFIG } from '../../../initializers/config'
20import { VideoModel } from '../../../models/video/video' 20import { VideoModel } from '../../../models/video/video'
@@ -81,7 +81,7 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV
81 81
82 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() 82 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
83 83
84 await VideoPathManager.Instance.makeAvailableVideoFile(videoOrStreamingPlaylist, videoFileInput, videoInputPath => { 84 await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
85 return generateHlsPlaylistResolution({ 85 return generateHlsPlaylistResolution({
86 video, 86 video,
87 videoInputPath, 87 videoInputPath,
@@ -135,7 +135,7 @@ async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodi
135// --------------------------------------------------------------------------- 135// ---------------------------------------------------------------------------
136 136
137async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, payload: HLSTranscodingPayload) { 137async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, payload: HLSTranscodingPayload) {
138 if (payload.isMaxQuality && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { 138 if (payload.isMaxQuality && payload.autoDeleteWebTorrentIfNeeded && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
139 // Remove webtorrent files if not enabled 139 // Remove webtorrent files if not enabled
140 for (const file of video.VideoFiles) { 140 for (const file of video.VideoFiles) {
141 await video.removeWebTorrentFileAndTorrent(file) 141 await video.removeWebTorrentFileAndTorrent(file)
@@ -232,6 +232,7 @@ async function createHlsJobIfEnabled (user: MUserId, payload: {
232 isPortraitMode: payload.isPortraitMode, 232 isPortraitMode: payload.isPortraitMode,
233 copyCodecs: payload.copyCodecs, 233 copyCodecs: payload.copyCodecs,
234 isMaxQuality: payload.isMaxQuality, 234 isMaxQuality: payload.isMaxQuality,
235 autoDeleteWebTorrentIfNeeded: true,
235 isNewVideo: payload.isNewVideo 236 isNewVideo: payload.isNewVideo
236 } 237 }
237 238
@@ -261,7 +262,7 @@ async function createLowerResolutionsJobs (options: {
261 const { video, user, videoFileResolution, isPortraitMode, isNewVideo, type } = options 262 const { video, user, videoFileResolution, isPortraitMode, isNewVideo, type } = options
262 263
263 // Create transcoding jobs if there are enabled resolutions 264 // Create transcoding jobs if there are enabled resolutions
264 const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod') 265 const resolutionsEnabled = computeLowerResolutionsToTranscode(videoFileResolution, 'vod')
265 const resolutionCreated: string[] = [] 266 const resolutionCreated: string[] = []
266 267
267 for (const resolution of resolutionsEnabled) { 268 for (const resolution of resolutionsEnabled) {
@@ -288,6 +289,7 @@ async function createLowerResolutionsJobs (options: {
288 isPortraitMode, 289 isPortraitMode,
289 copyCodecs: false, 290 copyCodecs: false,
290 isMaxQuality: false, 291 isMaxQuality: false,
292 autoDeleteWebTorrentIfNeeded: true,
291 isNewVideo 293 isNewVideo
292 } 294 }
293 295
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
index 2562edb75..b3bf5a999 100644
--- a/server/lib/live/live-manager.ts
+++ b/server/lib/live/live-manager.ts
@@ -3,7 +3,7 @@ import { readFile } from 'fs-extra'
3import { createServer, Server } from 'net' 3import { createServer, Server } from 'net'
4import { createServer as createServerTLS, Server as ServerTLS } from 'tls' 4import { createServer as createServerTLS, Server as ServerTLS } from 'tls'
5import { 5import {
6 computeResolutionsToTranscode, 6 computeLowerResolutionsToTranscode,
7 ffprobePromise, 7 ffprobePromise,
8 getVideoFileBitrate, 8 getVideoFileBitrate,
9 getVideoFileFPS, 9 getVideoFileFPS,
@@ -402,7 +402,7 @@ class LiveManager {
402 402
403 private buildAllResolutionsToTranscode (originResolution: number) { 403 private buildAllResolutionsToTranscode (originResolution: number) {
404 const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED 404 const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
405 ? computeResolutionsToTranscode(originResolution, 'live') 405 ? computeLowerResolutionsToTranscode(originResolution, 'live')
406 : [] 406 : []
407 407
408 return resolutionsEnabled.concat([ originResolution ]) 408 return resolutionsEnabled.concat([ originResolution ])
diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts
index 12acb3aec..4f17073f4 100644
--- a/server/lib/object-storage/keys.ts
+++ b/server/lib/object-storage/keys.ts
@@ -1,12 +1,12 @@
1import { join } from 'path' 1import { join } from 'path'
2import { MStreamingPlaylist, MVideoUUID } from '@server/types/models' 2import { MStreamingPlaylistVideo } from '@server/types/models'
3 3
4function generateHLSObjectStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) { 4function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) {
5 return join(generateHLSObjectBaseStorageKey(playlist, video), filename) 5 return join(generateHLSObjectBaseStorageKey(playlist), filename)
6} 6}
7 7
8function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID) { 8function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) {
9 return join(playlist.getStringType(), video.uuid) 9 return join(playlist.getStringType(), playlist.Video.uuid)
10} 10}
11 11
12function generateWebTorrentObjectStorageKey (filename: string) { 12function generateWebTorrentObjectStorageKey (filename: string) {
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
index 15b8f58d5..8988f3e2a 100644
--- a/server/lib/object-storage/videos.ts
+++ b/server/lib/object-storage/videos.ts
@@ -1,17 +1,17 @@
1import { join } from 'path' 1import { join } from 'path'
2import { logger } from '@server/helpers/logger' 2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config' 3import { CONFIG } from '@server/initializers/config'
4import { MStreamingPlaylist, MVideoFile, MVideoUUID } from '@server/types/models' 4import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models'
5import { getHLSDirectory } from '../paths' 5import { getHLSDirectory } from '../paths'
6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' 6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
7import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' 7import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
8 8
9function storeHLSFile (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) { 9function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string) {
10 const baseHlsDirectory = getHLSDirectory(video) 10 const baseHlsDirectory = getHLSDirectory(playlist.Video)
11 11
12 return storeObject({ 12 return storeObject({
13 inputPath: join(baseHlsDirectory, filename), 13 inputPath: join(baseHlsDirectory, filename),
14 objectStorageKey: generateHLSObjectStorageKey(playlist, video, filename), 14 objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
15 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS 15 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
16 }) 16 })
17} 17}
@@ -24,16 +24,16 @@ function storeWebTorrentFile (filename: string) {
24 }) 24 })
25} 25}
26 26
27function removeHLSObjectStorage (playlist: MStreamingPlaylist, video: MVideoUUID) { 27function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
28 return removePrefix(generateHLSObjectBaseStorageKey(playlist, video), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) 28 return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
29} 29}
30 30
31function removeWebTorrentObjectStorage (videoFile: MVideoFile) { 31function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
32 return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) 32 return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS)
33} 33}
34 34
35async function makeHLSFileAvailable (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string, destination: string) { 35async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
36 const key = generateHLSObjectStorageKey(playlist, video, filename) 36 const key = generateHLSObjectStorageKey(playlist, filename)
37 37
38 logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags()) 38 logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags())
39 39
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts
index d2384f53c..36270e5c1 100644
--- a/server/lib/thumbnail.ts
+++ b/server/lib/thumbnail.ts
@@ -115,7 +115,7 @@ function generateVideoMiniature (options: {
115}) { 115}) {
116 const { video, videoFile, type } = options 116 const { video, videoFile, type } = options
117 117
118 return VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, input => { 118 return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), input => {
119 const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) 119 const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type)
120 120
121 const thumbnailCreator = videoFile.isAudio() 121 const thumbnailCreator = videoFile.isAudio()
diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts
index 250a678eb..d0db05216 100644
--- a/server/lib/transcoding/video-transcoding.ts
+++ b/server/lib/transcoding/video-transcoding.ts
@@ -35,7 +35,7 @@ function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVid
35 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 35 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
36 const newExtname = '.mp4' 36 const newExtname = '.mp4'
37 37
38 return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async videoInputPath => { 38 return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async videoInputPath => {
39 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 39 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
40 40
41 const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) 41 const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath)
@@ -81,7 +81,7 @@ function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: V
81 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 81 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
82 const extname = '.mp4' 82 const extname = '.mp4'
83 83
84 return VideoPathManager.Instance.makeAvailableVideoFile(video, video.getMaxQualityFile(), async videoInputPath => { 84 return VideoPathManager.Instance.makeAvailableVideoFile(video.getMaxQualityFile().withVideoOrPlaylist(video), async videoInputPath => {
85 const newVideoFile = new VideoFileModel({ 85 const newVideoFile = new VideoFileModel({
86 resolution, 86 resolution,
87 extname, 87 extname,
@@ -134,7 +134,7 @@ function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolutio
134 134
135 const inputVideoFile = video.getMinQualityFile() 135 const inputVideoFile = video.getMinQualityFile()
136 136
137 return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async audioInputPath => { 137 return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async audioInputPath => {
138 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 138 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
139 139
140 // If the user updates the video preview during transcoding 140 // If the user updates the video preview during transcoding
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts
index 4c5d0c89d..27058005c 100644
--- a/server/lib/video-path-manager.ts
+++ b/server/lib/video-path-manager.ts
@@ -3,7 +3,14 @@ import { extname, join } from 'path'
3import { buildUUID } from '@server/helpers/uuid' 3import { buildUUID } from '@server/helpers/uuid'
4import { extractVideo } from '@server/helpers/video' 4import { extractVideo } from '@server/helpers/video'
5import { CONFIG } from '@server/initializers/config' 5import { CONFIG } from '@server/initializers/config'
6import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' 6import {
7 MStreamingPlaylistVideo,
8 MVideo,
9 MVideoFile,
10 MVideoFileStreamingPlaylistVideo,
11 MVideoFileVideo,
12 MVideoUUID
13} from '@server/types/models'
7import { VideoStorage } from '@shared/models' 14import { VideoStorage } from '@shared/models'
8import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage' 15import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage'
9import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' 16import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths'
@@ -43,10 +50,10 @@ class VideoPathManager {
43 return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename) 50 return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename)
44 } 51 }
45 52
46 async makeAvailableVideoFile <T> (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) { 53 async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) {
47 if (videoFile.storage === VideoStorage.FILE_SYSTEM) { 54 if (videoFile.storage === VideoStorage.FILE_SYSTEM) {
48 return this.makeAvailableFactory( 55 return this.makeAvailableFactory(
49 () => this.getFSVideoFileOutputPath(videoOrPlaylist, videoFile), 56 () => this.getFSVideoFileOutputPath(videoFile.getVideoOrStreamingPlaylist(), videoFile),
50 false, 57 false,
51 cb 58 cb
52 ) 59 )
@@ -55,10 +62,10 @@ class VideoPathManager {
55 const destination = this.buildTMPDestination(videoFile.filename) 62 const destination = this.buildTMPDestination(videoFile.filename)
56 63
57 if (videoFile.isHLS()) { 64 if (videoFile.isHLS()) {
58 const video = extractVideo(videoOrPlaylist) 65 const playlist = (videoFile as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
59 66
60 return this.makeAvailableFactory( 67 return this.makeAvailableFactory(
61 () => makeHLSFileAvailable(videoOrPlaylist as MStreamingPlaylistVideo, video, videoFile.filename, destination), 68 () => makeHLSFileAvailable(playlist, videoFile.filename, destination),
62 true, 69 true,
63 cb 70 cb
64 ) 71 )
@@ -71,19 +78,20 @@ class VideoPathManager {
71 ) 78 )
72 } 79 }
73 80
74 async makeAvailableResolutionPlaylistFile <T> (playlist: MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) { 81 async makeAvailableResolutionPlaylistFile <T> (videoFile: MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) {
75 const filename = getHlsResolutionPlaylistFilename(videoFile.filename) 82 const filename = getHlsResolutionPlaylistFilename(videoFile.filename)
76 83
77 if (videoFile.storage === VideoStorage.FILE_SYSTEM) { 84 if (videoFile.storage === VideoStorage.FILE_SYSTEM) {
78 return this.makeAvailableFactory( 85 return this.makeAvailableFactory(
79 () => join(getHLSDirectory(playlist.Video), filename), 86 () => join(getHLSDirectory(videoFile.getVideo()), filename),
80 false, 87 false,
81 cb 88 cb
82 ) 89 )
83 } 90 }
84 91
92 const playlist = videoFile.VideoStreamingPlaylist
85 return this.makeAvailableFactory( 93 return this.makeAvailableFactory(
86 () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)), 94 () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)),
87 true, 95 true,
88 cb 96 cb
89 ) 97 )
@@ -99,7 +107,7 @@ class VideoPathManager {
99 } 107 }
100 108
101 return this.makeAvailableFactory( 109 return this.makeAvailableFactory(
102 () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)), 110 () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)),
103 true, 111 true,
104 cb 112 cb
105 ) 113 )
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts
index 0b51f5c6b..bf6dd4bc8 100644
--- a/server/lib/video-state.ts
+++ b/server/lib/video-state.ts
@@ -80,6 +80,8 @@ async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: b
80} 80}
81 81
82function moveToFailedTranscodingState (video: MVideoFullLight) { 82function moveToFailedTranscodingState (video: MVideoFullLight) {
83 if (video.state === VideoState.TRANSCODING_FAILED) return
84
83 return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined) 85 return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined)
84} 86}
85 87
diff --git a/server/lib/video.ts b/server/lib/video.ts
index 0a2b93cc0..1cfe4f27c 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -105,7 +105,7 @@ async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoF
105 return addTranscodingJob(dataInput, jobOptions) 105 return addTranscodingJob(dataInput, jobOptions)
106} 106}
107 107
108async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions) { 108async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions = {}) {
109 await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') 109 await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
110 110
111 return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: payload }, options) 111 return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: payload }, options)
diff --git a/server/middlewares/validators/videos/index.ts b/server/middlewares/validators/videos/index.ts
index fd1d58093..f365d8ee1 100644
--- a/server/middlewares/validators/videos/index.ts
+++ b/server/middlewares/validators/videos/index.ts
@@ -9,4 +9,5 @@ export * from './video-ownership-changes'
9export * from './video-watch' 9export * from './video-watch'
10export * from './video-rates' 10export * from './video-rates'
11export * from './video-shares' 11export * from './video-shares'
12export * from './video-transcoding'
12export * from './videos' 13export * from './videos'
diff --git a/server/middlewares/validators/videos/video-files.ts b/server/middlewares/validators/videos/video-files.ts
index 282594ab6..c1fa77502 100644
--- a/server/middlewares/validators/videos/video-files.ts
+++ b/server/middlewares/validators/videos/video-files.ts
@@ -1,6 +1,6 @@
1import express from 'express' 1import express from 'express'
2import { MUser, MVideo } from '@server/types/models' 2import { MVideo } from '@server/types/models'
3import { HttpStatusCode, UserRight } from '../../../../shared' 3import { HttpStatusCode } from '../../../../shared'
4import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
5import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' 5import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared'
6 6
@@ -14,9 +14,7 @@ const videoFilesDeleteWebTorrentValidator = [
14 if (!await doesVideoExist(req.params.id, res)) return 14 if (!await doesVideoExist(req.params.id, res)) return
15 15
16 const video = res.locals.videoAll 16 const video = res.locals.videoAll
17 const user = res.locals.oauth.token.User
18 17
19 if (!checkUserCanDeleteFiles(user, res)) return
20 if (!checkLocalVideo(video, res)) return 18 if (!checkLocalVideo(video, res)) return
21 19
22 if (!video.hasWebTorrentFiles()) { 20 if (!video.hasWebTorrentFiles()) {
@@ -47,9 +45,7 @@ const videoFilesDeleteHLSValidator = [
47 if (!await doesVideoExist(req.params.id, res)) return 45 if (!await doesVideoExist(req.params.id, res)) return
48 46
49 const video = res.locals.videoAll 47 const video = res.locals.videoAll
50 const user = res.locals.oauth.token.User
51 48
52 if (!checkUserCanDeleteFiles(user, res)) return
53 if (!checkLocalVideo(video, res)) return 49 if (!checkLocalVideo(video, res)) return
54 50
55 if (!video.getHLSPlaylist()) { 51 if (!video.getHLSPlaylist()) {
@@ -89,16 +85,3 @@ function checkLocalVideo (video: MVideo, res: express.Response) {
89 85
90 return true 86 return true
91} 87}
92
93function checkUserCanDeleteFiles (user: MUser, res: express.Response) {
94 if (user.hasRight(UserRight.MANAGE_VIDEO_FILES) !== true) {
95 res.fail({
96 status: HttpStatusCode.FORBIDDEN_403,
97 message: 'User cannot update video files'
98 })
99
100 return false
101 }
102
103 return true
104}
diff --git a/server/middlewares/validators/videos/video-transcoding.ts b/server/middlewares/validators/videos/video-transcoding.ts
new file mode 100644
index 000000000..34f231d45
--- /dev/null
+++ b/server/middlewares/validators/videos/video-transcoding.ts
@@ -0,0 +1,55 @@
1import express from 'express'
2import { body } from 'express-validator'
3import { isValidCreateTranscodingType } from '@server/helpers/custom-validators/video-transcoding'
4import { CONFIG } from '@server/initializers/config'
5import { VideoJobInfoModel } from '@server/models/video/video-job-info'
6import { HttpStatusCode } from '@shared/models'
7import { logger } from '../../../helpers/logger'
8import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared'
9
10const createTranscodingValidator = [
11 isValidVideoIdParam('videoId'),
12
13 body('transcodingType')
14 .custom(isValidCreateTranscodingType).withMessage('Should have a valid transcoding type'),
15
16 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
17 logger.debug('Checking createTranscodingValidator parameters', { parameters: req.body })
18
19 if (areValidationErrors(req, res)) return
20 if (!await doesVideoExist(req.params.videoId, res, 'all')) return
21
22 const video = res.locals.videoAll
23
24 if (video.remote) {
25 return res.fail({
26 status: HttpStatusCode.BAD_REQUEST_400,
27 message: 'Cannot run transcoding job on a remote video'
28 })
29 }
30
31 if (CONFIG.TRANSCODING.ENABLED !== true) {
32 return res.fail({
33 status: HttpStatusCode.BAD_REQUEST_400,
34 message: 'Cannot run transcoding job because transcoding is disabled on this instance'
35 })
36 }
37
38 // Prefer using job info table instead of video state because before 4.0 failed transcoded video were stuck in "TO_TRANSCODE" state
39 const info = await VideoJobInfoModel.load(video.id)
40 if (info && info.pendingTranscode !== 0) {
41 return res.fail({
42 status: HttpStatusCode.CONFLICT_409,
43 message: 'This video is already being transcoded'
44 })
45 }
46
47 return next()
48 }
49]
50
51// ---------------------------------------------------------------------------
52
53export {
54 createTranscodingValidator
55}
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts
index 461e296df..fd4da68ed 100644
--- a/server/models/video/formatter/video-format-utils.ts
+++ b/server/models/video/formatter/video-format-utils.ts
@@ -2,8 +2,7 @@ import { uuidToShort } from '@server/helpers/uuid'
2import { generateMagnetUri } from '@server/helpers/webtorrent' 2import { generateMagnetUri } from '@server/helpers/webtorrent'
3import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls' 3import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
4import { VideoViews } from '@server/lib/video-views' 4import { VideoViews } from '@server/lib/video-views'
5import { VideosCommonQueryAfterSanitize } from '@shared/models' 5import { VideoFile, VideosCommonQueryAfterSanitize } from '@shared/models'
6import { VideoFile } from '@shared/models/videos/video-file.model'
7import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects' 6import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects'
8import { Video, VideoDetails, VideoInclude } from '../../../../shared/models/videos' 7import { Video, VideoDetails, VideoInclude } from '../../../../shared/models/videos'
9import { VideoStreamingPlaylist } from '../../../../shared/models/videos/video-streaming-playlist.model' 8import { VideoStreamingPlaylist } from '../../../../shared/models/videos/video-streaming-playlist.model'
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 106f9602b..87311c0ed 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -25,7 +25,7 @@ import { logger } from '@server/helpers/logger'
25import { extractVideo } from '@server/helpers/video' 25import { extractVideo } from '@server/helpers/video'
26import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage' 26import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
27import { getFSTorrentFilePath } from '@server/lib/paths' 27import { getFSTorrentFilePath } from '@server/lib/paths'
28import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' 28import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
29import { AttributesOnly } from '@shared/core-utils' 29import { AttributesOnly } from '@shared/core-utils'
30import { VideoStorage } from '@shared/models' 30import { VideoStorage } from '@shared/models'
31import { 31import {
@@ -536,4 +536,10 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
536 (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId) 536 (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
537 ) 537 )
538 } 538 }
539
540 withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
541 if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist })
542
543 return Object.assign(this, { Video: videoOrPlaylist })
544 }
539} 545}
diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts
index 7c1fe6734..cb1f3f2f0 100644
--- a/server/models/video/video-job-info.ts
+++ b/server/models/video/video-job-info.ts
@@ -49,7 +49,7 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo
49 }) 49 })
50 Video: VideoModel 50 Video: VideoModel
51 51
52 static load (videoId: number, transaction: Transaction) { 52 static load (videoId: number, transaction?: Transaction) {
53 const where = { 53 const where = {
54 videoId 54 videoId
55 } 55 }
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index 18d96c750..4643c5452 100644
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -239,6 +239,10 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
239 this.videoId === other.videoId 239 this.videoId === other.videoId
240 } 240 }
241 241
242 withVideo (video: MVideo) {
243 return Object.assign(this, { Video: video })
244 }
245
242 private getMasterPlaylistStaticPath (videoUUID: string) { 246 private getMasterPlaylistStaticPath (videoUUID: string) {
243 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename) 247 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename)
244 } 248 }
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 6eeb6b312..c49df1d5e 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -33,9 +33,8 @@ import { getHLSDirectory, getHLSRedundancyDirectory } from '@server/lib/paths'
33import { VideoPathManager } from '@server/lib/video-path-manager' 33import { VideoPathManager } from '@server/lib/video-path-manager'
34import { getServerActor } from '@server/models/application/application' 34import { getServerActor } from '@server/models/application/application'
35import { ModelCache } from '@server/models/model-cache' 35import { ModelCache } from '@server/models/model-cache'
36import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' 36import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, isThisWeek, pick } from '@shared/core-utils'
37import { VideoInclude } from '@shared/models' 37import { VideoFile, VideoInclude } from '@shared/models'
38import { VideoFile } from '@shared/models/videos/video-file.model'
39import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared' 38import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared'
40import { VideoObject } from '../../../shared/models/activitypub/objects' 39import { VideoObject } from '../../../shared/models/activitypub/objects'
41import { Video, VideoDetails, VideoRateType, VideoStorage } from '../../../shared/models/videos' 40import { Video, VideoDetails, VideoRateType, VideoStorage } from '../../../shared/models/videos'
@@ -1673,7 +1672,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1673 const file = this.getMaxQualityFile() 1672 const file = this.getMaxQualityFile()
1674 const videoOrPlaylist = file.getVideoOrStreamingPlaylist() 1673 const videoOrPlaylist = file.getVideoOrStreamingPlaylist()
1675 1674
1676 return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, file, originalFilePath => { 1675 return VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(videoOrPlaylist), originalFilePath => {
1677 return getVideoFileResolution(originalFilePath) 1676 return getVideoFileResolution(originalFilePath)
1678 }) 1677 })
1679 } 1678 }
@@ -1742,7 +1741,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1742 ) 1741 )
1743 1742
1744 if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { 1743 if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
1745 await removeHLSObjectStorage(streamingPlaylist, this) 1744 await removeHLSObjectStorage(streamingPlaylist.withVideo(this))
1746 } 1745 }
1747 } 1746 }
1748 } 1747 }
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index ff7dc4abb..e052296db 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -15,6 +15,7 @@ import './plugins'
15import './redundancy' 15import './redundancy'
16import './search' 16import './search'
17import './services' 17import './services'
18import './transcoding'
18import './upload-quota' 19import './upload-quota'
19import './user-notifications' 20import './user-notifications'
20import './user-subscriptions' 21import './user-subscriptions'
diff --git a/server/tests/api/check-params/transcoding.ts b/server/tests/api/check-params/transcoding.ts
new file mode 100644
index 000000000..a8daafe3e
--- /dev/null
+++ b/server/tests/api/check-params/transcoding.ts
@@ -0,0 +1,104 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils'
5import { HttpStatusCode, UserRole } from '@shared/models'
6
7describe('Test transcoding API validators', function () {
8 let servers: PeerTubeServer[]
9
10 let userToken: string
11 let moderatorToken: string
12
13 let remoteId: string
14 let validId: string
15
16 // ---------------------------------------------------------------
17
18 before(async function () {
19 this.timeout(60000)
20
21 servers = await createMultipleServers(2)
22 await setAccessTokensToServers(servers)
23
24 await doubleFollow(servers[0], servers[1])
25
26 userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER)
27 moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR)
28
29 {
30 const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' })
31 remoteId = uuid
32 }
33
34 {
35 const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' })
36 validId = uuid
37 }
38
39 await waitJobs(servers)
40
41 await servers[0].config.enableTranscoding()
42 })
43
44 it('Should not run transcoding of a unknown video', async function () {
45 await servers[0].videos.runTranscoding({ videoId: 404, transcodingType: 'hls', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
46 await servers[0].videos.runTranscoding({ videoId: 404, transcodingType: 'webtorrent', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
47 })
48
49 it('Should not run transcoding of a remote video', async function () {
50 const expectedStatus = HttpStatusCode.BAD_REQUEST_400
51
52 await servers[0].videos.runTranscoding({ videoId: remoteId, transcodingType: 'hls', expectedStatus })
53 await servers[0].videos.runTranscoding({ videoId: remoteId, transcodingType: 'webtorrent', expectedStatus })
54 })
55
56 it('Should not run transcoding by a non admin user', async function () {
57 const expectedStatus = HttpStatusCode.FORBIDDEN_403
58
59 await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls', token: userToken, expectedStatus })
60 await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent', token: moderatorToken, expectedStatus })
61 })
62
63 it('Should not run transcoding without transcoding type', async function () {
64 await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: undefined, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
65 })
66
67 it('Should not run transcoding with an incorrect transcoding type', async function () {
68 const expectedStatus = HttpStatusCode.BAD_REQUEST_400
69
70 await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'toto' as any, expectedStatus })
71 })
72
73 it('Should not run transcoding if the instance disabled it', async function () {
74 const expectedStatus = HttpStatusCode.BAD_REQUEST_400
75
76 await servers[0].config.disableTranscoding()
77
78 await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls', expectedStatus })
79 await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent', expectedStatus })
80 })
81
82 it('Should run transcoding', async function () {
83 this.timeout(120_000)
84
85 await servers[0].config.enableTranscoding()
86
87 await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls' })
88 await waitJobs(servers)
89
90 await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent' })
91 await waitJobs(servers)
92 })
93
94 it('Should not run transcoding on a video that is already being transcoded', async function () {
95 await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent' })
96
97 const expectedStatus = HttpStatusCode.CONFLICT_409
98 await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent', expectedStatus })
99 })
100
101 after(async function () {
102 await cleanupTests(servers)
103 })
104})
diff --git a/server/tests/api/check-params/video-files.ts b/server/tests/api/check-params/video-files.ts
index 48b10d2b5..61936d562 100644
--- a/server/tests/api/check-params/video-files.ts
+++ b/server/tests/api/check-params/video-files.ts
@@ -1,16 +1,19 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import 'mocha' 3import 'mocha'
4import { cleanupTests, createMultipleServers, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils' 4import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils'
5import { HttpStatusCode, UserRole } from '@shared/models' 5import { HttpStatusCode, UserRole } from '@shared/models'
6 6
7describe('Test videos files', function () { 7describe('Test videos files', function () {
8 let servers: PeerTubeServer[] 8 let servers: PeerTubeServer[]
9
9 let webtorrentId: string 10 let webtorrentId: string
10 let hlsId: string 11 let hlsId: string
11 let remoteId: string 12 let remoteId: string
13
12 let userToken: string 14 let userToken: string
13 let moderatorToken: string 15 let moderatorToken: string
16
14 let validId1: string 17 let validId1: string
15 let validId2: string 18 let validId2: string
16 19
@@ -22,10 +25,17 @@ describe('Test videos files', function () {
22 servers = await createMultipleServers(2) 25 servers = await createMultipleServers(2)
23 await setAccessTokensToServers(servers) 26 await setAccessTokensToServers(servers)
24 27
28 await doubleFollow(servers[0], servers[1])
29
25 userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER) 30 userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER)
26 moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR) 31 moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR)
27 32
28 { 33 {
34 const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' })
35 remoteId = uuid
36 }
37
38 {
29 await servers[0].config.enableTranscoding(true, true) 39 await servers[0].config.enableTranscoding(true, true)
30 40
31 { 41 {
@@ -58,6 +68,11 @@ describe('Test videos files', function () {
58 await waitJobs(servers) 68 await waitJobs(servers)
59 }) 69 })
60 70
71 it('Should not delete files of a unknown video', async function () {
72 await servers[0].videos.removeHLSFiles({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
73 await servers[0].videos.removeWebTorrentFiles({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
74 })
75
61 it('Should not delete files of a remote video', async function () { 76 it('Should not delete files of a remote video', async function () {
62 await servers[0].videos.removeHLSFiles({ videoId: remoteId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) 77 await servers[0].videos.removeHLSFiles({ videoId: remoteId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
63 await servers[0].videos.removeWebTorrentFiles({ videoId: remoteId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) 78 await servers[0].videos.removeWebTorrentFiles({ videoId: remoteId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index f92e339e7..bedb9b8b6 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -6,6 +6,7 @@ import './video-captions'
6import './video-change-ownership' 6import './video-change-ownership'
7import './video-channels' 7import './video-channels'
8import './video-comments' 8import './video-comments'
9import './video-create-transcoding'
9import './video-description' 10import './video-description'
10import './video-files' 11import './video-files'
11import './video-hls' 12import './video-hls'
diff --git a/server/tests/api/videos/video-create-transcoding.ts b/server/tests/api/videos/video-create-transcoding.ts
new file mode 100644
index 000000000..bae06ac6c
--- /dev/null
+++ b/server/tests/api/videos/video-create-transcoding.ts
@@ -0,0 +1,156 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import {
6 areObjectStorageTestsDisabled,
7 cleanupTests,
8 createMultipleServers,
9 doubleFollow,
10 expectStartWith,
11 makeRawRequest,
12 ObjectStorageCommand,
13 PeerTubeServer,
14 setAccessTokensToServers,
15 waitJobs
16} from '@shared/extra-utils'
17import { HttpStatusCode, VideoDetails } from '@shared/models'
18
19const expect = chai.expect
20
21async function checkFilesInObjectStorage (video: VideoDetails) {
22 for (const file of video.files) {
23 expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
24 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
25 }
26
27 for (const file of video.streamingPlaylists[0].files) {
28 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
29 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
30 }
31}
32
33async function expectNoFailedTranscodingJob (server: PeerTubeServer) {
34 const { data } = await server.jobs.listFailed({ jobType: 'video-transcoding' })
35 expect(data).to.have.lengthOf(0)
36}
37
38function runTests (objectStorage: boolean) {
39 let servers: PeerTubeServer[] = []
40 let videoUUID: string
41 let publishedAt: string
42
43 before(async function () {
44 this.timeout(120000)
45
46 const config = objectStorage
47 ? ObjectStorageCommand.getDefaultConfig()
48 : {}
49
50 // Run server 2 to have transcoding enabled
51 servers = await createMultipleServers(2, config)
52 await setAccessTokensToServers(servers)
53
54 await servers[0].config.disableTranscoding()
55
56 await doubleFollow(servers[0], servers[1])
57
58 if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets()
59
60 const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' })
61 videoUUID = shortUUID
62
63 const video = await servers[0].videos.get({ id: videoUUID })
64 publishedAt = video.publishedAt as string
65
66 await servers[0].config.enableTranscoding()
67
68 await waitJobs(servers)
69 })
70
71 it('Should generate HLS', async function () {
72 this.timeout(60000)
73
74 await servers[0].videos.runTranscoding({
75 videoId: videoUUID,
76 transcodingType: 'hls'
77 })
78
79 await waitJobs(servers)
80 await expectNoFailedTranscodingJob(servers[0])
81
82 for (const server of servers) {
83 const videoDetails = await server.videos.get({ id: videoUUID })
84
85 expect(videoDetails.files).to.have.lengthOf(1)
86 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
87 expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
88
89 if (objectStorage) await checkFilesInObjectStorage(videoDetails)
90 }
91 })
92
93 it('Should generate WebTorrent', async function () {
94 this.timeout(60000)
95
96 await servers[0].videos.runTranscoding({
97 videoId: videoUUID,
98 transcodingType: 'webtorrent'
99 })
100
101 await waitJobs(servers)
102
103 for (const server of servers) {
104 const videoDetails = await server.videos.get({ id: videoUUID })
105
106 expect(videoDetails.files).to.have.lengthOf(5)
107 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
108 expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
109
110 if (objectStorage) await checkFilesInObjectStorage(videoDetails)
111 }
112 })
113
114 it('Should generate WebTorrent from HLS only video', async function () {
115 this.timeout(60000)
116
117 await servers[0].videos.removeWebTorrentFiles({ videoId: videoUUID })
118 await waitJobs(servers)
119
120 await servers[0].videos.runTranscoding({ videoId: videoUUID, transcodingType: 'webtorrent' })
121 await waitJobs(servers)
122
123 for (const server of servers) {
124 const videoDetails = await server.videos.get({ id: videoUUID })
125
126 expect(videoDetails.files).to.have.lengthOf(5)
127 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
128 expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
129
130 if (objectStorage) await checkFilesInObjectStorage(videoDetails)
131 }
132 })
133
134 it('Should not have updated published at attributes', async function () {
135 const video = await servers[0].videos.get({ id: videoUUID })
136
137 expect(video.publishedAt).to.equal(publishedAt)
138 })
139
140 after(async function () {
141 await cleanupTests(servers)
142 })
143}
144
145describe('Test create transcoding jobs from API', function () {
146
147 describe('On filesystem', function () {
148 runTests(false)
149 })
150
151 describe('On object storage', function () {
152 if (areObjectStorageTestsDisabled()) return
153
154 runTests(true)
155 })
156})