aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/activitypub/client.ts15
-rw-r--r--server/controllers/api/config.ts8
-rw-r--r--server/controllers/api/videos/index.ts19
-rw-r--r--server/controllers/static.ts9
-rw-r--r--server/controllers/tracker.ts25
-rw-r--r--server/helpers/activitypub.ts5
-rw-r--r--server/helpers/core-utils.ts8
-rw-r--r--server/helpers/custom-validators/activitypub/cache-file.ts12
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts8
-rw-r--r--server/helpers/custom-validators/misc.ts5
-rw-r--r--server/helpers/ffmpeg-utils.ts32
-rw-r--r--server/helpers/video.ts4
-rw-r--r--server/initializers/checker-before-init.ts2
-rw-r--r--server/initializers/constants.ts14
-rw-r--r--server/initializers/database.ts4
-rw-r--r--server/initializers/installer.ts5
-rw-r--r--server/initializers/migrations/0330-video-streaming-playlist.ts51
-rw-r--r--server/lib/activitypub/cache-file.ts23
-rw-r--r--server/lib/activitypub/send/send-create.ts9
-rw-r--r--server/lib/activitypub/send/send-undo.ts3
-rw-r--r--server/lib/activitypub/send/send-update.ts2
-rw-r--r--server/lib/activitypub/url.ts7
-rw-r--r--server/lib/activitypub/videos.ts97
-rw-r--r--server/lib/hls.ts110
-rw-r--r--server/lib/job-queue/handlers/video-file.ts59
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts189
-rw-r--r--server/lib/video-transcoding.ts49
-rw-r--r--server/middlewares/validators/redundancy.ts33
-rw-r--r--server/models/redundancy/video-redundancy.ts139
-rw-r--r--server/models/video/video-file.ts6
-rw-r--r--server/models/video/video-format-utils.ts61
-rw-r--r--server/models/video/video-streaming-playlist.ts154
-rw-r--r--server/models/video/video.ts179
-rw-r--r--server/tests/api/check-params/config.ts3
-rw-r--r--server/tests/api/redundancy/redundancy.ts212
-rw-r--r--server/tests/api/server/config.ts6
-rw-r--r--server/tests/api/videos/index.ts1
-rw-r--r--server/tests/api/videos/video-hls.ts145
-rw-r--r--server/tests/cli/update-host.ts11
39 files changed, 1452 insertions, 272 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts
index 1a4e28dc8..32a83aa5f 100644
--- a/server/controllers/activitypub/client.ts
+++ b/server/controllers/activitypub/client.ts
@@ -37,7 +37,7 @@ import {
37 getVideoSharesActivityPubUrl 37 getVideoSharesActivityPubUrl
38} from '../../lib/activitypub' 38} from '../../lib/activitypub'
39import { VideoCaptionModel } from '../../models/video/video-caption' 39import { VideoCaptionModel } from '../../models/video/video-caption'
40import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy' 40import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy'
41import { getServerActor } from '../../helpers/utils' 41import { getServerActor } from '../../helpers/utils'
42import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 42import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
43 43
@@ -66,11 +66,11 @@ activityPubClientRouter.get('/accounts?/:name/dislikes/:videoId',
66 66
67activityPubClientRouter.get('/videos/watch/:id', 67activityPubClientRouter.get('/videos/watch/:id',
68 executeIfActivityPub(asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS))), 68 executeIfActivityPub(asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS))),
69 executeIfActivityPub(asyncMiddleware(videosGetValidator)), 69 executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))),
70 executeIfActivityPub(asyncMiddleware(videoController)) 70 executeIfActivityPub(asyncMiddleware(videoController))
71) 71)
72activityPubClientRouter.get('/videos/watch/:id/activity', 72activityPubClientRouter.get('/videos/watch/:id/activity',
73 executeIfActivityPub(asyncMiddleware(videosGetValidator)), 73 executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))),
74 executeIfActivityPub(asyncMiddleware(videoController)) 74 executeIfActivityPub(asyncMiddleware(videoController))
75) 75)
76activityPubClientRouter.get('/videos/watch/:id/announces', 76activityPubClientRouter.get('/videos/watch/:id/announces',
@@ -116,7 +116,11 @@ activityPubClientRouter.get('/video-channels/:name/following',
116) 116)
117 117
118activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?', 118activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?',
119 executeIfActivityPub(asyncMiddleware(videoRedundancyGetValidator)), 119 executeIfActivityPub(asyncMiddleware(videoFileRedundancyGetValidator)),
120 executeIfActivityPub(asyncMiddleware(videoRedundancyController))
121)
122activityPubClientRouter.get('/redundancy/video-playlists/:streamingPlaylistType/:videoId',
123 executeIfActivityPub(asyncMiddleware(videoPlaylistRedundancyGetValidator)),
120 executeIfActivityPub(asyncMiddleware(videoRedundancyController)) 124 executeIfActivityPub(asyncMiddleware(videoRedundancyController))
121) 125)
122 126
@@ -163,7 +167,8 @@ function getAccountVideoRate (rateType: VideoRateType) {
163} 167}
164 168
165async function videoController (req: express.Request, res: express.Response) { 169async function videoController (req: express.Request, res: express.Response) {
166 const video: VideoModel = res.locals.video 170 // We need more attributes
171 const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id)
167 172
168 if (video.url.startsWith(CONFIG.WEBSERVER.URL) === false) return res.redirect(video.url) 173 if (video.url.startsWith(CONFIG.WEBSERVER.URL) === false) return res.redirect(video.url)
169 174
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 255026f46..1f3341bc0 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -1,5 +1,5 @@
1import * as express from 'express' 1import * as express from 'express'
2import { omit, snakeCase } from 'lodash' 2import { snakeCase } from 'lodash'
3import { ServerConfig, UserRight } from '../../../shared' 3import { ServerConfig, UserRight } from '../../../shared'
4import { About } from '../../../shared/models/server/about.model' 4import { About } from '../../../shared/models/server/about.model'
5import { CustomConfig } from '../../../shared/models/server/custom-config.model' 5import { CustomConfig } from '../../../shared/models/server/custom-config.model'
@@ -78,6 +78,9 @@ async function getConfig (req: express.Request, res: express.Response) {
78 requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION 78 requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION
79 }, 79 },
80 transcoding: { 80 transcoding: {
81 hls: {
82 enabled: CONFIG.TRANSCODING.HLS.ENABLED
83 },
81 enabledResolutions 84 enabledResolutions
82 }, 85 },
83 import: { 86 import: {
@@ -246,6 +249,9 @@ function customConfig (): CustomConfig {
246 '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ], 249 '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ],
247 '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ], 250 '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ],
248 '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ] 251 '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ]
252 },
253 hls: {
254 enabled: CONFIG.TRANSCODING.HLS.ENABLED
249 } 255 }
250 }, 256 },
251 import: { 257 import: {
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 2b2dfa7ca..e04fc8186 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -37,6 +37,7 @@ import {
37 setDefaultPagination, 37 setDefaultPagination,
38 setDefaultSort, 38 setDefaultSort,
39 videosAddValidator, 39 videosAddValidator,
40 videosCustomGetValidator,
40 videosGetValidator, 41 videosGetValidator,
41 videosRemoveValidator, 42 videosRemoveValidator,
42 videosSortValidator, 43 videosSortValidator,
@@ -123,9 +124,9 @@ videosRouter.get('/:id/description',
123) 124)
124videosRouter.get('/:id', 125videosRouter.get('/:id',
125 optionalAuthenticate, 126 optionalAuthenticate,
126 asyncMiddleware(videosGetValidator), 127 asyncMiddleware(videosCustomGetValidator('only-video-with-rights')),
127 asyncMiddleware(checkVideoFollowConstraints), 128 asyncMiddleware(checkVideoFollowConstraints),
128 getVideo 129 asyncMiddleware(getVideo)
129) 130)
130videosRouter.post('/:id/views', 131videosRouter.post('/:id/views',
131 asyncMiddleware(videosGetValidator), 132 asyncMiddleware(videosGetValidator),
@@ -395,15 +396,17 @@ async function updateVideo (req: express.Request, res: express.Response) {
395 return res.type('json').status(204).end() 396 return res.type('json').status(204).end()
396} 397}
397 398
398function getVideo (req: express.Request, res: express.Response) { 399async function getVideo (req: express.Request, res: express.Response) {
399 const videoInstance = res.locals.video 400 // We need more attributes
401 const userId: number = res.locals.oauth ? res.locals.oauth.token.User.id : null
402 const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id, undefined, userId)
400 403
401 if (videoInstance.isOutdated()) { 404 if (video.isOutdated()) {
402 JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoInstance.url } }) 405 JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })
403 .catch(err => logger.error('Cannot create AP refresher job for video %s.', videoInstance.url, { err })) 406 .catch(err => logger.error('Cannot create AP refresher job for video %s.', video.url, { err }))
404 } 407 }
405 408
406 return res.json(videoInstance.toFormattedDetailsJSON()) 409 return res.json(video.toFormattedDetailsJSON())
407} 410}
408 411
409async function viewVideo (req: express.Request, res: express.Response) { 412async function viewVideo (req: express.Request, res: express.Response) {
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index 4fd58f70c..b21f9da00 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -1,6 +1,6 @@
1import * as cors from 'cors' 1import * as cors from 'cors'
2import * as express from 'express' 2import * as express from 'express'
3import { CONFIG, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers' 3import { CONFIG, HLS_PLAYLIST_DIRECTORY, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers'
4import { VideosPreviewCache } from '../lib/cache' 4import { VideosPreviewCache } from '../lib/cache'
5import { cacheRoute } from '../middlewares/cache' 5import { cacheRoute } from '../middlewares/cache'
6import { asyncMiddleware, videosGetValidator } from '../middlewares' 6import { asyncMiddleware, videosGetValidator } from '../middlewares'
@@ -51,6 +51,13 @@ staticRouter.use(
51 asyncMiddleware(downloadVideoFile) 51 asyncMiddleware(downloadVideoFile)
52) 52)
53 53
54// HLS
55staticRouter.use(
56 STATIC_PATHS.PLAYLISTS.HLS,
57 cors(),
58 express.static(HLS_PLAYLIST_DIRECTORY, { fallthrough: false }) // 404 if the file does not exist
59)
60
54// Thumbnails path for express 61// Thumbnails path for express
55const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR 62const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR
56staticRouter.use( 63staticRouter.use(
diff --git a/server/controllers/tracker.ts b/server/controllers/tracker.ts
index 1deb8c402..8b77d9de7 100644
--- a/server/controllers/tracker.ts
+++ b/server/controllers/tracker.ts
@@ -7,6 +7,7 @@ import { Server as WebSocketServer } from 'ws'
7import { CONFIG, TRACKER_RATE_LIMITS } from '../initializers/constants' 7import { CONFIG, TRACKER_RATE_LIMITS } from '../initializers/constants'
8import { VideoFileModel } from '../models/video/video-file' 8import { VideoFileModel } from '../models/video/video-file'
9import { parse } from 'url' 9import { parse } from 'url'
10import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
10 11
11const TrackerServer = bitTorrentTracker.Server 12const TrackerServer = bitTorrentTracker.Server
12 13
@@ -21,7 +22,7 @@ const trackerServer = new TrackerServer({
21 udp: false, 22 udp: false,
22 ws: false, 23 ws: false,
23 dht: false, 24 dht: false,
24 filter: function (infoHash, params, cb) { 25 filter: async function (infoHash, params, cb) {
25 let ip: string 26 let ip: string
26 27
27 if (params.type === 'ws') { 28 if (params.type === 'ws') {
@@ -32,19 +33,25 @@ const trackerServer = new TrackerServer({
32 33
33 const key = ip + '-' + infoHash 34 const key = ip + '-' + infoHash
34 35
35 peersIps[ip] = peersIps[ip] ? peersIps[ip] + 1 : 1 36 peersIps[ ip ] = peersIps[ ip ] ? peersIps[ ip ] + 1 : 1
36 peersIpInfoHash[key] = peersIpInfoHash[key] ? peersIpInfoHash[key] + 1 : 1 37 peersIpInfoHash[ key ] = peersIpInfoHash[ key ] ? peersIpInfoHash[ key ] + 1 : 1
37 38
38 if (peersIpInfoHash[key] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) { 39 if (peersIpInfoHash[ key ] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) {
39 return cb(new Error(`Too many requests (${peersIpInfoHash[ key ]} of ip ${ip} for torrent ${infoHash}`)) 40 return cb(new Error(`Too many requests (${peersIpInfoHash[ key ]} of ip ${ip} for torrent ${infoHash}`))
40 } 41 }
41 42
42 VideoFileModel.isInfohashExists(infoHash) 43 try {
43 .then(exists => { 44 const videoFileExists = await VideoFileModel.doesInfohashExist(infoHash)
44 if (exists === false) return cb(new Error(`Unknown infoHash ${infoHash}`)) 45 if (videoFileExists === true) return cb()
45 46
46 return cb() 47 const playlistExists = await VideoStreamingPlaylistModel.doesInfohashExist(infoHash)
47 }) 48 if (playlistExists === true) return cb()
49
50 return cb(new Error(`Unknown infoHash ${infoHash}`))
51 } catch (err) {
52 logger.error('Error in tracker filter.', { err })
53 return cb(err)
54 }
48 } 55 }
49}) 56})
50 57
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index f1430055f..eba552524 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -15,7 +15,7 @@ function activityPubContextify <T> (data: T) {
15 'https://w3id.org/security/v1', 15 'https://w3id.org/security/v1',
16 { 16 {
17 RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', 17 RsaSignature2017: 'https://w3id.org/security#RsaSignature2017',
18 pt: 'https://joinpeertube.org/ns', 18 pt: 'https://joinpeertube.org/ns#',
19 sc: 'http://schema.org#', 19 sc: 'http://schema.org#',
20 Hashtag: 'as:Hashtag', 20 Hashtag: 'as:Hashtag',
21 uuid: 'sc:identifier', 21 uuid: 'sc:identifier',
@@ -32,7 +32,8 @@ function activityPubContextify <T> (data: T) {
32 waitTranscoding: 'sc:Boolean', 32 waitTranscoding: 'sc:Boolean',
33 expires: 'sc:expires', 33 expires: 'sc:expires',
34 support: 'sc:Text', 34 support: 'sc:Text',
35 CacheFile: 'pt:CacheFile' 35 CacheFile: 'pt:CacheFile',
36 Infohash: 'pt:Infohash'
36 }, 37 },
37 { 38 {
38 likes: { 39 likes: {
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 3fb824e36..f38b82d97 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -193,10 +193,14 @@ function peertubeTruncate (str: string, maxLength: number) {
193 return truncate(str, options) 193 return truncate(str, options)
194} 194}
195 195
196function sha256 (str: string, encoding: HexBase64Latin1Encoding = 'hex') { 196function sha256 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') {
197 return createHash('sha256').update(str).digest(encoding) 197 return createHash('sha256').update(str).digest(encoding)
198} 198}
199 199
200function sha1 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') {
201 return createHash('sha1').update(str).digest(encoding)
202}
203
200function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> { 204function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> {
201 return function promisified (): Promise<A> { 205 return function promisified (): Promise<A> {
202 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { 206 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
@@ -262,7 +266,9 @@ export {
262 sanitizeHost, 266 sanitizeHost,
263 buildPath, 267 buildPath,
264 peertubeTruncate, 268 peertubeTruncate,
269
265 sha256, 270 sha256,
271 sha1,
266 272
267 promisify0, 273 promisify0,
268 promisify1, 274 promisify1,
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts
index e2bd0c55e..21d5c53ca 100644
--- a/server/helpers/custom-validators/activitypub/cache-file.ts
+++ b/server/helpers/custom-validators/activitypub/cache-file.ts
@@ -8,9 +8,19 @@ function isCacheFileObjectValid (object: CacheFileObject) {
8 object.type === 'CacheFile' && 8 object.type === 'CacheFile' &&
9 isDateValid(object.expires) && 9 isDateValid(object.expires) &&
10 isActivityPubUrlValid(object.object) && 10 isActivityPubUrlValid(object.object) &&
11 isRemoteVideoUrlValid(object.url) 11 (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url))
12} 12}
13 13
14// ---------------------------------------------------------------------------
15
14export { 16export {
15 isCacheFileObjectValid 17 isCacheFileObjectValid
16} 18}
19
20// ---------------------------------------------------------------------------
21
22function isPlaylistRedundancyUrlValid (url: any) {
23 return url.type === 'Link' &&
24 (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
25 isActivityPubUrlValid(url.href)
26}
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index 0f34aab21..ad99c2724 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -1,7 +1,7 @@
1import * as validator from 'validator' 1import * as validator from 'validator'
2import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers' 2import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers'
3import { peertubeTruncate } from '../../core-utils' 3import { peertubeTruncate } from '../../core-utils'
4import { exists, isBooleanValid, isDateValid, isUUIDValid } from '../misc' 4import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc'
5import { 5import {
6 isVideoDurationValid, 6 isVideoDurationValid,
7 isVideoNameValid, 7 isVideoNameValid,
@@ -12,7 +12,6 @@ import {
12} from '../videos' 12} from '../videos'
13import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' 13import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc'
14import { VideoState } from '../../../../shared/models/videos' 14import { VideoState } from '../../../../shared/models/videos'
15import { isVideoAbuseReasonValid } from '../video-abuses'
16 15
17function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { 16function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) {
18 return isBaseActivityValid(activity, 'Update') && 17 return isBaseActivityValid(activity, 'Update') &&
@@ -81,6 +80,11 @@ function isRemoteVideoUrlValid (url: any) {
81 ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType || url.mimeType) !== -1 && 80 ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType || url.mimeType) !== -1 &&
82 validator.isLength(url.href, { min: 5 }) && 81 validator.isLength(url.href, { min: 5 }) &&
83 validator.isInt(url.height + '', { min: 0 }) 82 validator.isInt(url.height + '', { min: 0 })
83 ) ||
84 (
85 (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
86 isActivityPubUrlValid(url.href) &&
87 isArray(url.tag)
84 ) 88 )
85} 89}
86 90
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts
index b6f0ebe6f..76647fea2 100644
--- a/server/helpers/custom-validators/misc.ts
+++ b/server/helpers/custom-validators/misc.ts
@@ -13,6 +13,10 @@ function isNotEmptyIntArray (value: any) {
13 return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0 13 return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0
14} 14}
15 15
16function isArrayOf (value: any, validator: (value: any) => boolean) {
17 return isArray(value) && value.every(v => validator(v))
18}
19
16function isDateValid (value: string) { 20function isDateValid (value: string) {
17 return exists(value) && validator.isISO8601(value) 21 return exists(value) && validator.isISO8601(value)
18} 22}
@@ -82,6 +86,7 @@ function isFileValid (
82 86
83export { 87export {
84 exists, 88 exists,
89 isArrayOf,
85 isNotEmptyIntArray, 90 isNotEmptyIntArray,
86 isArray, 91 isArray,
87 isIdValid, 92 isIdValid,
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 132f4690e..5ad8ed48e 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -1,5 +1,5 @@
1import * as ffmpeg from 'fluent-ffmpeg' 1import * as ffmpeg from 'fluent-ffmpeg'
2import { join } from 'path' 2import { dirname, join } from 'path'
3import { getTargetBitrate, VideoResolution } from '../../shared/models/videos' 3import { getTargetBitrate, VideoResolution } from '../../shared/models/videos'
4import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' 4import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
5import { processImage } from './image-utils' 5import { processImage } from './image-utils'
@@ -29,12 +29,21 @@ function computeResolutionsToTranscode (videoFileHeight: number) {
29 return resolutionsEnabled 29 return resolutionsEnabled
30} 30}
31 31
32async function getVideoFileResolution (path: string) { 32async function getVideoFileSize (path: string) {
33 const videoStream = await getVideoFileStream(path) 33 const videoStream = await getVideoFileStream(path)
34 34
35 return { 35 return {
36 videoFileResolution: Math.min(videoStream.height, videoStream.width), 36 width: videoStream.width,
37 isPortraitMode: videoStream.height > videoStream.width 37 height: videoStream.height
38 }
39}
40
41async function getVideoFileResolution (path: string) {
42 const size = await getVideoFileSize(path)
43
44 return {
45 videoFileResolution: Math.min(size.height, size.width),
46 isPortraitMode: size.height > size.width
38 } 47 }
39} 48}
40 49
@@ -110,8 +119,10 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima
110type TranscodeOptions = { 119type TranscodeOptions = {
111 inputPath: string 120 inputPath: string
112 outputPath: string 121 outputPath: string
113 resolution?: VideoResolution 122 resolution: VideoResolution
114 isPortraitMode?: boolean 123 isPortraitMode?: boolean
124
125 generateHlsPlaylist?: boolean
115} 126}
116 127
117function transcode (options: TranscodeOptions) { 128function transcode (options: TranscodeOptions) {
@@ -150,6 +161,16 @@ function transcode (options: TranscodeOptions) {
150 command = command.withFPS(fps) 161 command = command.withFPS(fps)
151 } 162 }
152 163
164 if (options.generateHlsPlaylist) {
165 const segmentFilename = `${dirname(options.outputPath)}/${options.resolution}_%03d.ts`
166
167 command = command.outputOption('-hls_time 4')
168 .outputOption('-hls_list_size 0')
169 .outputOption('-hls_playlist_type vod')
170 .outputOption('-hls_segment_filename ' + segmentFilename)
171 .outputOption('-f hls')
172 }
173
153 command 174 command
154 .on('error', (err, stdout, stderr) => { 175 .on('error', (err, stdout, stderr) => {
155 logger.error('Error in transcoding job.', { stdout, stderr }) 176 logger.error('Error in transcoding job.', { stdout, stderr })
@@ -166,6 +187,7 @@ function transcode (options: TranscodeOptions) {
166// --------------------------------------------------------------------------- 187// ---------------------------------------------------------------------------
167 188
168export { 189export {
190 getVideoFileSize,
169 getVideoFileResolution, 191 getVideoFileResolution,
170 getDurationFromVideoFile, 192 getDurationFromVideoFile,
171 generateImageFromVideoFile, 193 generateImageFromVideoFile,
diff --git a/server/helpers/video.ts b/server/helpers/video.ts
index 1bd21467d..c90fe06c7 100644
--- a/server/helpers/video.ts
+++ b/server/helpers/video.ts
@@ -1,10 +1,12 @@
1import { VideoModel } from '../models/video/video' 1import { VideoModel } from '../models/video/video'
2 2
3type VideoFetchType = 'all' | 'only-video' | 'id' | 'none' 3type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none'
4 4
5function fetchVideo (id: number | string, fetchType: VideoFetchType, userId?: number) { 5function fetchVideo (id: number | string, fetchType: VideoFetchType, userId?: number) {
6 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId) 6 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId)
7 7
8 if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id)
9
8 if (fetchType === 'only-video') return VideoModel.load(id) 10 if (fetchType === 'only-video') return VideoModel.load(id)
9 11
10 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) 12 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 7905d9ffa..29fdb263e 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -12,7 +12,7 @@ function checkMissedConfig () {
12 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max', 12 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max',
13 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', 13 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
14 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', 14 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
15 'storage.redundancy', 'storage.tmp', 15 'storage.redundancy', 'storage.tmp', 'storage.playlists',
16 'log.level', 16 'log.level',
17 'user.video_quota', 'user.video_quota_daily', 17 'user.video_quota', 'user.video_quota_daily',
18 'cache.previews.size', 'admin.email', 'contact_form.enabled', 18 'cache.previews.size', 'admin.email', 'contact_form.enabled',
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 6f3ebb9aa..98f8f8694 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -16,7 +16,7 @@ let config: IConfig = require('config')
16 16
17// --------------------------------------------------------------------------- 17// ---------------------------------------------------------------------------
18 18
19const LAST_MIGRATION_VERSION = 325 19const LAST_MIGRATION_VERSION = 330
20 20
21// --------------------------------------------------------------------------- 21// ---------------------------------------------------------------------------
22 22
@@ -192,6 +192,7 @@ const CONFIG = {
192 AVATARS_DIR: buildPath(config.get<string>('storage.avatars')), 192 AVATARS_DIR: buildPath(config.get<string>('storage.avatars')),
193 LOG_DIR: buildPath(config.get<string>('storage.logs')), 193 LOG_DIR: buildPath(config.get<string>('storage.logs')),
194 VIDEOS_DIR: buildPath(config.get<string>('storage.videos')), 194 VIDEOS_DIR: buildPath(config.get<string>('storage.videos')),
195 PLAYLISTS_DIR: buildPath(config.get<string>('storage.playlists')),
195 REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')), 196 REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
196 THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')), 197 THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
197 PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')), 198 PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
@@ -259,6 +260,9 @@ const CONFIG = {
259 get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') }, 260 get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') },
260 get '720p' () { return config.get<boolean>('transcoding.resolutions.720p') }, 261 get '720p' () { return config.get<boolean>('transcoding.resolutions.720p') },
261 get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') } 262 get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') }
263 },
264 HLS: {
265 get ENABLED () { return config.get<boolean>('transcoding.hls.enabled') }
262 } 266 }
263 }, 267 },
264 IMPORT: { 268 IMPORT: {
@@ -590,6 +594,9 @@ const STATIC_PATHS = {
590 TORRENTS: '/static/torrents/', 594 TORRENTS: '/static/torrents/',
591 WEBSEED: '/static/webseed/', 595 WEBSEED: '/static/webseed/',
592 REDUNDANCY: '/static/redundancy/', 596 REDUNDANCY: '/static/redundancy/',
597 PLAYLISTS: {
598 HLS: '/static/playlists/hls'
599 },
593 AVATARS: '/static/avatars/', 600 AVATARS: '/static/avatars/',
594 VIDEO_CAPTIONS: '/static/video-captions/' 601 VIDEO_CAPTIONS: '/static/video-captions/'
595} 602}
@@ -632,6 +639,9 @@ const CACHE = {
632 } 639 }
633} 640}
634 641
642const HLS_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.PLAYLISTS_DIR, 'hls')
643const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
644
635const MEMOIZE_TTL = { 645const MEMOIZE_TTL = {
636 OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours 646 OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
637} 647}
@@ -709,6 +719,7 @@ updateWebserverUrls()
709 719
710export { 720export {
711 API_VERSION, 721 API_VERSION,
722 HLS_REDUNDANCY_DIRECTORY,
712 AVATARS_SIZE, 723 AVATARS_SIZE,
713 ACCEPT_HEADERS, 724 ACCEPT_HEADERS,
714 BCRYPT_SALT_SIZE, 725 BCRYPT_SALT_SIZE,
@@ -733,6 +744,7 @@ export {
733 PRIVATE_RSA_KEY_SIZE, 744 PRIVATE_RSA_KEY_SIZE,
734 ROUTE_CACHE_LIFETIME, 745 ROUTE_CACHE_LIFETIME,
735 SORTABLE_COLUMNS, 746 SORTABLE_COLUMNS,
747 HLS_PLAYLIST_DIRECTORY,
736 FEEDS, 748 FEEDS,
737 JOB_TTL, 749 JOB_TTL,
738 NSFW_POLICY_TYPES, 750 NSFW_POLICY_TYPES,
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 84ad2079b..fe296142d 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -33,6 +33,7 @@ import { AccountBlocklistModel } from '../models/account/account-blocklist'
33import { ServerBlocklistModel } from '../models/server/server-blocklist' 33import { ServerBlocklistModel } from '../models/server/server-blocklist'
34import { UserNotificationModel } from '../models/account/user-notification' 34import { UserNotificationModel } from '../models/account/user-notification'
35import { UserNotificationSettingModel } from '../models/account/user-notification-setting' 35import { UserNotificationSettingModel } from '../models/account/user-notification-setting'
36import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
36 37
37require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string 38require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
38 39
@@ -99,7 +100,8 @@ async function initDatabaseModels (silent: boolean) {
99 AccountBlocklistModel, 100 AccountBlocklistModel,
100 ServerBlocklistModel, 101 ServerBlocklistModel,
101 UserNotificationModel, 102 UserNotificationModel,
102 UserNotificationSettingModel 103 UserNotificationSettingModel,
104 VideoStreamingPlaylistModel
103 ]) 105 ])
104 106
105 // Check extensions exist in the database 107 // Check extensions exist in the database
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index b9a9da183..2b22e16fe 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user'
6import { ApplicationModel } from '../models/application/application' 6import { ApplicationModel } from '../models/application/application'
7import { OAuthClientModel } from '../models/oauth/oauth-client' 7import { OAuthClientModel } from '../models/oauth/oauth-client'
8import { applicationExist, clientsExist, usersExist } from './checker-after-init' 8import { applicationExist, clientsExist, usersExist } from './checker-after-init'
9import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants' 9import { CACHE, CONFIG, HLS_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
10import { sequelizeTypescript } from './database' 10import { sequelizeTypescript } from './database'
11import { remove, ensureDir } from 'fs-extra' 11import { remove, ensureDir } from 'fs-extra'
12 12
@@ -73,6 +73,9 @@ function createDirectoriesIfNotExist () {
73 tasks.push(ensureDir(dir)) 73 tasks.push(ensureDir(dir))
74 } 74 }
75 75
76 // Playlist directories
77 tasks.push(ensureDir(HLS_PLAYLIST_DIRECTORY))
78
76 return Promise.all(tasks) 79 return Promise.all(tasks)
77} 80}
78 81
diff --git a/server/initializers/migrations/0330-video-streaming-playlist.ts b/server/initializers/migrations/0330-video-streaming-playlist.ts
new file mode 100644
index 000000000..c85a762ab
--- /dev/null
+++ b/server/initializers/migrations/0330-video-streaming-playlist.ts
@@ -0,0 +1,51 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction,
5 queryInterface: Sequelize.QueryInterface,
6 sequelize: Sequelize.Sequelize
7}): Promise<void> {
8
9 {
10 const query = `
11 CREATE TABLE IF NOT EXISTS "videoStreamingPlaylist"
12(
13 "id" SERIAL,
14 "type" INTEGER NOT NULL,
15 "playlistUrl" VARCHAR(2000) NOT NULL,
16 "p2pMediaLoaderInfohashes" VARCHAR(255)[] NOT NULL,
17 "segmentsSha256Url" VARCHAR(255) NOT NULL,
18 "videoId" INTEGER NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
19 "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
20 "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
21 PRIMARY KEY ("id")
22);`
23 await utils.sequelize.query(query)
24 }
25
26 {
27 const data = {
28 type: Sequelize.INTEGER,
29 allowNull: true,
30 defaultValue: null
31 }
32
33 await utils.queryInterface.changeColumn('videoRedundancy', 'videoFileId', data)
34 }
35
36 {
37 const query = 'ALTER TABLE "videoRedundancy" ADD COLUMN "videoStreamingPlaylistId" INTEGER NULL ' +
38 'REFERENCES "videoStreamingPlaylist" ("id") ON DELETE CASCADE ON UPDATE CASCADE'
39
40 await utils.sequelize.query(query)
41 }
42}
43
44function down (options) {
45 throw new Error('Not implemented.')
46}
47
48export {
49 up,
50 down
51}
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index f6f068b45..9a40414bb 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -1,11 +1,28 @@
1import { CacheFileObject } from '../../../shared/index' 1import { ActivityPlaylistUrlObject, ActivityVideoUrlObject, CacheFileObject } from '../../../shared/index'
2import { VideoModel } from '../../models/video/video' 2import { VideoModel } from '../../models/video/video'
3import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 3import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
4import { Transaction } from 'sequelize' 4import { Transaction } from 'sequelize'
5import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
5 6
6function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { 7function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
7 const url = cacheFileObject.url
8 8
9 if (cacheFileObject.url.mediaType === 'application/x-mpegURL') {
10 const url = cacheFileObject.url
11
12 const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS)
13 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
14
15 return {
16 expiresOn: new Date(cacheFileObject.expires),
17 url: cacheFileObject.id,
18 fileUrl: url.href,
19 strategy: null,
20 videoStreamingPlaylistId: playlist.id,
21 actorId: byActor.id
22 }
23 }
24
25 const url = cacheFileObject.url
9 const videoFile = video.VideoFiles.find(f => { 26 const videoFile = video.VideoFiles.find(f => {
10 return f.resolution === url.height && f.fps === url.fps 27 return f.resolution === url.height && f.fps === url.fps
11 }) 28 })
@@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
15 return { 32 return {
16 expiresOn: new Date(cacheFileObject.expires), 33 expiresOn: new Date(cacheFileObject.expires),
17 url: cacheFileObject.id, 34 url: cacheFileObject.id,
18 fileUrl: cacheFileObject.url.href, 35 fileUrl: url.href,
19 strategy: null, 36 strategy: null,
20 videoFileId: videoFile.id, 37 videoFileId: videoFile.id,
21 actorId: byActor.id 38 actorId: byActor.id
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index e3fca0a17..605aaba06 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -1,6 +1,6 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub' 2import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub'
3import { VideoPrivacy } from '../../../../shared/models/videos' 3import { Video, VideoPrivacy } from '../../../../shared/models/videos'
4import { ActorModel } from '../../../models/activitypub/actor' 4import { ActorModel } from '../../../models/activitypub/actor'
5import { VideoModel } from '../../../models/video/video' 5import { VideoModel } from '../../../models/video/video'
6import { VideoAbuseModel } from '../../../models/video/video-abuse' 6import { VideoAbuseModel } from '../../../models/video/video-abuse'
@@ -39,17 +39,14 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
39 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 39 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
40} 40}
41 41
42async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { 42async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) {
43 logger.info('Creating job to send file cache of %s.', fileRedundancy.url) 43 logger.info('Creating job to send file cache of %s.', fileRedundancy.url)
44 44
45 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
46 const redundancyObject = fileRedundancy.toActivityPubObject()
47
48 return sendVideoRelatedCreateActivity({ 45 return sendVideoRelatedCreateActivity({
49 byActor, 46 byActor,
50 video, 47 video,
51 url: fileRedundancy.url, 48 url: fileRedundancy.url,
52 object: redundancyObject 49 object: fileRedundancy.toActivityPubObject()
53 }) 50 })
54} 51}
55 52
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts
index bf1b6e117..8976fcbc8 100644
--- a/server/lib/activitypub/send/send-undo.ts
+++ b/server/lib/activitypub/send/send-undo.ts
@@ -73,7 +73,8 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans
73async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { 73async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
74 logger.info('Creating job to undo cache file %s.', redundancyModel.url) 74 logger.info('Creating job to undo cache file %s.', redundancyModel.url)
75 75
76 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 76 const videoId = redundancyModel.getVideo().id
77 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
77 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) 78 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())
78 79
79 return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) 80 return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t })
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index a68f03edf..839f66470 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -61,7 +61,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { 61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
62 logger.info('Creating job to update cache file %s.', redundancyModel.url) 62 logger.info('Creating job to update cache file %s.', redundancyModel.url)
63 63
64 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 64 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id)
65 65
66 const activityBuilder = (audience: ActivityAudience) => { 66 const activityBuilder = (audience: ActivityAudience) => {
67 const redundancyObject = redundancyModel.toActivityPubObject() 67 const redundancyObject = redundancyModel.toActivityPubObject()
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts
index 38f15448c..4229fe094 100644
--- a/server/lib/activitypub/url.ts
+++ b/server/lib/activitypub/url.ts
@@ -5,6 +5,8 @@ import { VideoModel } from '../../models/video/video'
5import { VideoAbuseModel } from '../../models/video/video-abuse' 5import { VideoAbuseModel } from '../../models/video/video-abuse'
6import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
7import { VideoFileModel } from '../../models/video/video-file' 7import { VideoFileModel } from '../../models/video/video-file'
8import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
9import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
8 10
9function getVideoActivityPubUrl (video: VideoModel) { 11function getVideoActivityPubUrl (video: VideoModel) {
10 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid 12 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid
@@ -16,6 +18,10 @@ function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) {
16 return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` 18 return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}`
17} 19}
18 20
21function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) {
22 return `${CONFIG.WEBSERVER.URL}/redundancy/video-playlists/${playlist.getStringType()}/${video.uuid}`
23}
24
19function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { 25function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) {
20 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id 26 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id
21} 27}
@@ -92,6 +98,7 @@ function getUndoActivityPubUrl (originalUrl: string) {
92 98
93export { 99export {
94 getVideoActivityPubUrl, 100 getVideoActivityPubUrl,
101 getVideoCacheStreamingPlaylistActivityPubUrl,
95 getVideoChannelActivityPubUrl, 102 getVideoChannelActivityPubUrl,
96 getAccountActivityPubUrl, 103 getAccountActivityPubUrl,
97 getVideoAbuseActivityPubUrl, 104 getVideoAbuseActivityPubUrl,
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index e1e523499..edd01234f 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -2,7 +2,14 @@ import * as Bluebird from 'bluebird'
2import * as sequelize from 'sequelize' 2import * as sequelize from 'sequelize'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import * as request from 'request' 4import * as request from 'request'
5import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' 5import {
6 ActivityIconObject,
7 ActivityPlaylistSegmentHashesObject,
8 ActivityPlaylistUrlObject,
9 ActivityUrlObject,
10 ActivityVideoUrlObject,
11 VideoState
12} from '../../../shared/index'
6import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 13import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
7import { VideoPrivacy } from '../../../shared/models/videos' 14import { VideoPrivacy } from '../../../shared/models/videos'
8import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' 15import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
@@ -30,6 +37,9 @@ import { AccountModel } from '../../models/account/account'
30import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' 37import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
31import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' 38import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
32import { Notifier } from '../notifier' 39import { Notifier } from '../notifier'
40import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
41import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
42import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
33 43
34async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 44async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
35 // If the video is not private and published, we federate it 45 // If the video is not private and published, we federate it
@@ -264,6 +274,25 @@ async function updateVideoFromAP (options: {
264 } 274 }
265 275
266 { 276 {
277 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject)
278 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
279
280 // Remove video files that do not exist anymore
281 const destroyTasks = options.video.VideoStreamingPlaylists
282 .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f)))
283 .map(f => f.destroy(sequelizeOptions))
284 await Promise.all(destroyTasks)
285
286 // Update or add other one
287 const upsertTasks = streamingPlaylistAttributes.map(a => {
288 return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t })
289 .then(([ streamingPlaylist ]) => streamingPlaylist)
290 })
291
292 options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks)
293 }
294
295 {
267 // Update Tags 296 // Update Tags
268 const tags = options.videoObject.tag.map(tag => tag.name) 297 const tags = options.videoObject.tag.map(tag => tag.name)
269 const tagInstances = await TagModel.findOrCreateTags(tags, t) 298 const tagInstances = await TagModel.findOrCreateTags(tags, t)
@@ -367,13 +396,25 @@ export {
367 396
368// --------------------------------------------------------------------------- 397// ---------------------------------------------------------------------------
369 398
370function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { 399function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
371 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) 400 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
372 401
373 const urlMediaType = url.mediaType || url.mimeType 402 const urlMediaType = url.mediaType || url.mimeType
374 return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') 403 return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
375} 404}
376 405
406function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
407 const urlMediaType = url.mediaType || url.mimeType
408
409 return urlMediaType === 'application/x-mpegURL'
410}
411
412function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
413 const urlMediaType = tag.mediaType || tag.mimeType
414
415 return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json'
416}
417
377async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { 418async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
378 logger.debug('Adding remote video %s.', videoObject.id) 419 logger.debug('Adding remote video %s.', videoObject.id)
379 420
@@ -394,8 +435,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
394 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) 435 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
395 await Promise.all(videoFilePromises) 436 await Promise.all(videoFilePromises)
396 437
438 const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject)
439 const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
440 await Promise.all(playlistPromises)
441
397 // Process tags 442 // Process tags
398 const tags = videoObject.tag.map(t => t.name) 443 const tags = videoObject.tag
444 .filter(t => t.type === 'Hashtag')
445 .map(t => t.name)
399 const tagInstances = await TagModel.findOrCreateTags(tags, t) 446 const tagInstances = await TagModel.findOrCreateTags(tags, t)
400 await videoCreated.$set('Tags', tagInstances, sequelizeOptions) 447 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
401 448
@@ -473,13 +520,13 @@ async function videoActivityObjectToDBAttributes (
473} 520}
474 521
475function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { 522function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
476 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] 523 const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
477 524
478 if (fileUrls.length === 0) { 525 if (fileUrls.length === 0) {
479 throw new Error('Cannot find video files for ' + video.url) 526 throw new Error('Cannot find video files for ' + video.url)
480 } 527 }
481 528
482 const attributes: VideoFileModel[] = [] 529 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
483 for (const fileUrl of fileUrls) { 530 for (const fileUrl of fileUrls) {
484 // Fetch associated magnet uri 531 // Fetch associated magnet uri
485 const magnet = videoObject.url.find(u => { 532 const magnet = videoObject.url.find(u => {
@@ -502,7 +549,45 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid
502 size: fileUrl.size, 549 size: fileUrl.size,
503 videoId: video.id, 550 videoId: video.id,
504 fps: fileUrl.fps || -1 551 fps: fileUrl.fps || -1
505 } as VideoFileModel 552 }
553
554 attributes.push(attribute)
555 }
556
557 return attributes
558}
559
560function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
561 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
562 if (playlistUrls.length === 0) return []
563
564 const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
565 for (const playlistUrlObject of playlistUrls) {
566 const p2pMediaLoaderInfohashes = playlistUrlObject.tag
567 .filter(t => t.type === 'Infohash')
568 .map(t => t.name)
569 if (p2pMediaLoaderInfohashes.length === 0) {
570 logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject })
571 continue
572 }
573
574 const segmentsSha256UrlObject = playlistUrlObject.tag
575 .find(t => {
576 return isAPPlaylistSegmentHashesUrlObject(t)
577 }) as ActivityPlaylistSegmentHashesObject
578 if (!segmentsSha256UrlObject) {
579 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
580 continue
581 }
582
583 const attribute = {
584 type: VideoStreamingPlaylistType.HLS,
585 playlistUrl: playlistUrlObject.href,
586 segmentsSha256Url: segmentsSha256UrlObject.href,
587 p2pMediaLoaderInfohashes,
588 videoId: video.id
589 }
590
506 attributes.push(attribute) 591 attributes.push(attribute)
507 } 592 }
508 593
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
new file mode 100644
index 000000000..10db6c3c3
--- /dev/null
+++ b/server/lib/hls.ts
@@ -0,0 +1,110 @@
1import { VideoModel } from '../models/video/video'
2import { basename, dirname, join } from 'path'
3import { HLS_PLAYLIST_DIRECTORY, CONFIG } from '../initializers'
4import { outputJSON, pathExists, readdir, readFile, remove, writeFile, move } from 'fs-extra'
5import { getVideoFileSize } from '../helpers/ffmpeg-utils'
6import { sha256 } from '../helpers/core-utils'
7import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
8import HLSDownloader from 'hlsdownloader'
9import { logger } from '../helpers/logger'
10import { parse } from 'url'
11
12async function updateMasterHLSPlaylist (video: VideoModel) {
13 const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
14 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
15 const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
16
17 for (const file of video.VideoFiles) {
18 // If we did not generated a playlist for this resolution, skip
19 const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
20 if (await pathExists(filePlaylistPath) === false) continue
21
22 const videoFilePath = video.getVideoFilePath(file)
23
24 const size = await getVideoFileSize(videoFilePath)
25
26 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
27 const resolution = `RESOLUTION=${size.width}x${size.height}`
28
29 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
30 if (file.fps) line += ',FRAME-RATE=' + file.fps
31
32 masterPlaylists.push(line)
33 masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
34 }
35
36 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
37}
38
39async function updateSha256Segments (video: VideoModel) {
40 const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
41 const files = await readdir(directory)
42 const json: { [filename: string]: string} = {}
43
44 for (const file of files) {
45 if (file.endsWith('.ts') === false) continue
46
47 const buffer = await readFile(join(directory, file))
48 const filename = basename(file)
49
50 json[filename] = sha256(buffer)
51 }
52
53 const outputPath = join(directory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
54 await outputJSON(outputPath, json)
55}
56
57function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
58 let timer
59
60 logger.info('Importing HLS playlist %s', playlistUrl)
61
62 const params = {
63 playlistURL: playlistUrl,
64 destination: CONFIG.STORAGE.TMP_DIR
65 }
66 const downloader = new HLSDownloader(params)
67
68 const hlsDestinationDir = join(CONFIG.STORAGE.TMP_DIR, dirname(parse(playlistUrl).pathname))
69
70 return new Promise<string>(async (res, rej) => {
71 downloader.startDownload(err => {
72 clearTimeout(timer)
73
74 if (err) {
75 deleteTmpDirectory(hlsDestinationDir)
76
77 return rej(err)
78 }
79
80 move(hlsDestinationDir, destinationDir, { overwrite: true })
81 .then(() => res())
82 .catch(err => {
83 deleteTmpDirectory(hlsDestinationDir)
84
85 return rej(err)
86 })
87 })
88
89 timer = setTimeout(() => {
90 deleteTmpDirectory(hlsDestinationDir)
91
92 return rej(new Error('HLS download timeout.'))
93 }, timeout)
94
95 function deleteTmpDirectory (directory: string) {
96 remove(directory)
97 .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
98 }
99 })
100}
101
102// ---------------------------------------------------------------------------
103
104export {
105 updateMasterHLSPlaylist,
106 updateSha256Segments,
107 downloadPlaylistSegments
108}
109
110// ---------------------------------------------------------------------------
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts
index 217d666b6..7119ce0ca 100644
--- a/server/lib/job-queue/handlers/video-file.ts
+++ b/server/lib/job-queue/handlers/video-file.ts
@@ -5,17 +5,18 @@ import { VideoModel } from '../../../models/video/video'
5import { JobQueue } from '../job-queue' 5import { JobQueue } from '../job-queue'
6import { federateVideoIfNeeded } from '../../activitypub' 6import { federateVideoIfNeeded } from '../../activitypub'
7import { retryTransactionWrapper } from '../../../helpers/database-utils' 7import { retryTransactionWrapper } from '../../../helpers/database-utils'
8import { sequelizeTypescript } from '../../../initializers' 8import { sequelizeTypescript, CONFIG } from '../../../initializers'
9import * as Bluebird from 'bluebird' 9import * as Bluebird from 'bluebird'
10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' 10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
11import { importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' 11import { generateHlsPlaylist, importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding'
12import { Notifier } from '../../notifier' 12import { Notifier } from '../../notifier'
13 13
14export type VideoFilePayload = { 14export type VideoFilePayload = {
15 videoUUID: string 15 videoUUID: string
16 isNewVideo?: boolean
17 resolution?: VideoResolution 16 resolution?: VideoResolution
17 isNewVideo?: boolean
18 isPortraitMode?: boolean 18 isPortraitMode?: boolean
19 generateHlsPlaylist?: boolean
19} 20}
20 21
21export type VideoFileImportPayload = { 22export type VideoFileImportPayload = {
@@ -51,21 +52,38 @@ async function processVideoFile (job: Bull.Job) {
51 return undefined 52 return undefined
52 } 53 }
53 54
54 // Transcoding in other resolution 55 if (payload.generateHlsPlaylist) {
55 if (payload.resolution) { 56 await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false)
57
58 await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video)
59 } else if (payload.resolution) { // Transcoding in other resolution
56 await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) 60 await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false)
57 61
58 await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) 62 await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video, payload)
59 } else { 63 } else {
60 await optimizeVideofile(video) 64 await optimizeVideofile(video)
61 65
62 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) 66 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload)
63 } 67 }
64 68
65 return video 69 return video
66} 70}
67 71
68async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { 72async function onHlsPlaylistGenerationSuccess (video: VideoModel) {
73 if (video === undefined) return undefined
74
75 await sequelizeTypescript.transaction(async t => {
76 // Maybe the video changed in database, refresh it
77 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
78 // Video does not exist anymore
79 if (!videoDatabase) return undefined
80
81 // If the video was not published, we consider it is a new one for other instances
82 await federateVideoIfNeeded(videoDatabase, false, t)
83 })
84}
85
86async function onVideoFileTranscoderOrImportSuccess (video: VideoModel, payload?: VideoFilePayload) {
69 if (video === undefined) return undefined 87 if (video === undefined) return undefined
70 88
71 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { 89 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
@@ -96,9 +114,11 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) {
96 Notifier.Instance.notifyOnNewVideo(videoDatabase) 114 Notifier.Instance.notifyOnNewVideo(videoDatabase)
97 Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) 115 Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase)
98 } 116 }
117
118 await createHlsJobIfEnabled(payload)
99} 119}
100 120
101async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) { 121async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoFilePayload) {
102 if (videoArg === undefined) return undefined 122 if (videoArg === undefined) return undefined
103 123
104 // Outside the transaction (IO on disk) 124 // Outside the transaction (IO on disk)
@@ -145,7 +165,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo
145 logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) 165 logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy })
146 } 166 }
147 167
148 await federateVideoIfNeeded(videoDatabase, isNewVideo, t) 168 await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t)
149 169
150 return { videoDatabase, videoPublished } 170 return { videoDatabase, videoPublished }
151 }) 171 })
@@ -155,6 +175,8 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo
155 if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) 175 if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase)
156 if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) 176 if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase)
157 } 177 }
178
179 await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution }))
158} 180}
159 181
160// --------------------------------------------------------------------------- 182// ---------------------------------------------------------------------------
@@ -163,3 +185,20 @@ export {
163 processVideoFile, 185 processVideoFile,
164 processVideoFileImport 186 processVideoFileImport
165} 187}
188
189// ---------------------------------------------------------------------------
190
191function createHlsJobIfEnabled (payload?: VideoFilePayload) {
192 // Generate HLS playlist?
193 if (payload && CONFIG.TRANSCODING.HLS.ENABLED) {
194 const hlsTranscodingPayload = {
195 videoUUID: payload.videoUUID,
196 resolution: payload.resolution,
197 isPortraitMode: payload.isPortraitMode,
198
199 generateHlsPlaylist: true
200 }
201
202 return JobQueue.Instance.createJob({ type: 'video-file', payload: hlsTranscodingPayload })
203 }
204}
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index f643ee226..1a48f2bd0 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,5 +1,5 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' 2import { CONFIG, HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideosRedundancy } from '../../../shared/models/redundancy' 4import { VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
@@ -9,9 +9,19 @@ import { join } from 'path'
9import { move } from 'fs-extra' 9import { move } from 'fs-extra'
10import { getServerActor } from '../../helpers/utils' 10import { getServerActor } from '../../helpers/utils'
11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
12import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' 12import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
13import { removeVideoRedundancy } from '../redundancy' 13import { removeVideoRedundancy } from '../redundancy'
14import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' 14import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
16import { VideoModel } from '../../models/video/video'
17import { downloadPlaylistSegments } from '../hls'
18
19type CandidateToDuplicate = {
20 redundancy: VideosRedundancy,
21 video: VideoModel,
22 files: VideoFileModel[],
23 streamingPlaylists: VideoStreamingPlaylistModel[]
24}
15 25
16export class VideosRedundancyScheduler extends AbstractScheduler { 26export class VideosRedundancyScheduler extends AbstractScheduler {
17 27
@@ -24,28 +34,32 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
24 } 34 }
25 35
26 protected async internalExecute () { 36 protected async internalExecute () {
27 for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { 37 for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
28 logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) 38 logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy)
29 39
30 try { 40 try {
31 const videoToDuplicate = await this.findVideoToDuplicate(obj) 41 const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig)
32 if (!videoToDuplicate) continue 42 if (!videoToDuplicate) continue
33 43
34 const videoFiles = videoToDuplicate.VideoFiles 44 const candidateToDuplicate = {
35 videoFiles.forEach(f => f.Video = videoToDuplicate) 45 video: videoToDuplicate,
46 redundancy: redundancyConfig,
47 files: videoToDuplicate.VideoFiles,
48 streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
49 }
36 50
37 await this.purgeCacheIfNeeded(obj, videoFiles) 51 await this.purgeCacheIfNeeded(candidateToDuplicate)
38 52
39 if (await this.isTooHeavy(obj, videoFiles)) { 53 if (await this.isTooHeavy(candidateToDuplicate)) {
40 logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) 54 logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
41 continue 55 continue
42 } 56 }
43 57
44 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) 58 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy)
45 59
46 await this.createVideoRedundancy(obj, videoFiles) 60 await this.createVideoRedundancies(candidateToDuplicate)
47 } catch (err) { 61 } catch (err) {
48 logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) 62 logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err })
49 } 63 }
50 } 64 }
51 65
@@ -63,25 +77,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
63 77
64 for (const redundancyModel of expired) { 78 for (const redundancyModel of expired) {
65 try { 79 try {
66 await this.extendsOrDeleteRedundancy(redundancyModel) 80 const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
81 const candidate = {
82 redundancy: redundancyConfig,
83 video: null,
84 files: [],
85 streamingPlaylists: []
86 }
87
88 // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it
89 if (!redundancyConfig || await this.isTooHeavy(candidate)) {
90 logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy)
91 await removeVideoRedundancy(redundancyModel)
92 } else {
93 await this.extendsRedundancy(redundancyModel)
94 }
67 } catch (err) { 95 } catch (err) {
68 logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel)) 96 logger.error(
97 'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel),
98 { err }
99 )
69 } 100 }
70 } 101 }
71 } 102 }
72 103
73 private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) { 104 private async extendsRedundancy (redundancyModel: VideoRedundancyModel) {
74 // Refresh the video, maybe it was deleted
75 const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url)
76
77 if (!video) {
78 logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url)
79
80 await redundancyModel.destroy()
81 return
82 }
83
84 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) 105 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
106 // Redundancy strategy disabled, remove our redundancy instead of extending expiration
107 if (!redundancy) await removeVideoRedundancy(redundancyModel)
108
85 await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) 109 await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
86 } 110 }
87 111
@@ -112,49 +136,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
112 } 136 }
113 } 137 }
114 138
115 private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 139 private async createVideoRedundancies (data: CandidateToDuplicate) {
116 const serverActor = await getServerActor() 140 const video = await this.loadAndRefreshVideo(data.video.url)
141
142 if (!video) {
143 logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url)
117 144
118 for (const file of filesToDuplicate) { 145 return
119 const video = await this.loadAndRefreshVideo(file.Video.url) 146 }
120 147
148 for (const file of data.files) {
121 const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) 149 const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
122 if (existingRedundancy) { 150 if (existingRedundancy) {
123 await this.extendsOrDeleteRedundancy(existingRedundancy) 151 await this.extendsRedundancy(existingRedundancy)
124 152
125 continue 153 continue
126 } 154 }
127 155
128 if (!video) { 156 await this.createVideoFileRedundancy(data.redundancy, video, file)
129 logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url) 157 }
158
159 for (const streamingPlaylist of data.streamingPlaylists) {
160 const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id)
161 if (existingRedundancy) {
162 await this.extendsRedundancy(existingRedundancy)
130 163
131 continue 164 continue
132 } 165 }
133 166
134 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) 167 await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist)
168 }
169 }
135 170
136 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 171 private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) {
137 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) 172 file.Video = video
138 173
139 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) 174 const serverActor = await getServerActor()
140 175
141 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) 176 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
142 await move(tmpPath, destPath)
143 177
144 const createdModel = await VideoRedundancyModel.create({ 178 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
145 expiresOn: this.buildNewExpiration(redundancy.minLifetime), 179 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
146 url: getVideoCacheFileActivityPubUrl(file),
147 fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
148 strategy: redundancy.strategy,
149 videoFileId: file.id,
150 actorId: serverActor.id
151 })
152 createdModel.VideoFile = file
153 180
154 await sendCreateCacheFile(serverActor, createdModel) 181 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
155 182
156 logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) 183 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file))
157 } 184 await move(tmpPath, destPath)
185
186 const createdModel = await VideoRedundancyModel.create({
187 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
188 url: getVideoCacheFileActivityPubUrl(file),
189 fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
190 strategy: redundancy.strategy,
191 videoFileId: file.id,
192 actorId: serverActor.id
193 })
194
195 createdModel.VideoFile = file
196
197 await sendCreateCacheFile(serverActor, video, createdModel)
198
199 logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url)
200 }
201
202 private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) {
203 playlist.Video = video
204
205 const serverActor = await getServerActor()
206
207 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy)
208
209 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
210 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
211
212 const createdModel = await VideoRedundancyModel.create({
213 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
214 url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
215 fileUrl: playlist.getVideoRedundancyUrl(CONFIG.WEBSERVER.URL),
216 strategy: redundancy.strategy,
217 videoStreamingPlaylistId: playlist.id,
218 actorId: serverActor.id
219 })
220
221 createdModel.VideoStreamingPlaylist = playlist
222
223 await sendCreateCacheFile(serverActor, video, createdModel)
224
225 logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url)
158 } 226 }
159 227
160 private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { 228 private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) {
@@ -168,8 +236,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
168 await sendUpdateCacheFile(serverActor, redundancy) 236 await sendUpdateCacheFile(serverActor, redundancy)
169 } 237 }
170 238
171 private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 239 private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) {
172 while (this.isTooHeavy(redundancy, filesToDuplicate)) { 240 while (this.isTooHeavy(candidateToDuplicate)) {
241 const redundancy = candidateToDuplicate.redundancy
173 const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) 242 const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime)
174 if (!toDelete) return 243 if (!toDelete) return
175 244
@@ -177,11 +246,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
177 } 246 }
178 } 247 }
179 248
180 private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 249 private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) {
181 const maxSize = redundancy.size 250 const maxSize = candidateToDuplicate.redundancy.size
182 251
183 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) 252 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy)
184 const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate) 253 const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists)
185 254
186 return totalWillDuplicate > maxSize 255 return totalWillDuplicate > maxSize
187 } 256 }
@@ -191,13 +260,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
191 } 260 }
192 261
193 private buildEntryLogId (object: VideoRedundancyModel) { 262 private buildEntryLogId (object: VideoRedundancyModel) {
194 return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` 263 if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
264
265 return `${object.VideoStreamingPlaylist.playlistUrl}`
195 } 266 }
196 267
197 private getTotalFileSizes (files: VideoFileModel[]) { 268 private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) {
198 const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size 269 const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size
199 270
200 return files.reduce(fileReducer, 0) 271 return files.reduce(fileReducer, 0) * playlists.length
201 } 272 }
202 273
203 private async loadAndRefreshVideo (videoUrl: string) { 274 private async loadAndRefreshVideo (videoUrl: string) {
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
index 4460f46e4..608badfef 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -1,11 +1,14 @@
1import { CONFIG } from '../initializers' 1import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers'
2import { extname, join } from 'path' 2import { extname, join } from 'path'
3import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' 3import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils'
4import { copy, remove, move, stat } from 'fs-extra' 4import { copy, ensureDir, move, remove, stat } from 'fs-extra'
5import { logger } from '../helpers/logger' 5import { logger } from '../helpers/logger'
6import { VideoResolution } from '../../shared/models/videos' 6import { VideoResolution } from '../../shared/models/videos'
7import { VideoFileModel } from '../models/video/video-file' 7import { VideoFileModel } from '../models/video/video-file'
8import { VideoModel } from '../models/video/video' 8import { VideoModel } from '../models/video/video'
9import { updateMasterHLSPlaylist, updateSha256Segments } from './hls'
10import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
11import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type'
9 12
10async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { 13async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) {
11 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR 14 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
@@ -17,7 +20,8 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi
17 20
18 const transcodeOptions = { 21 const transcodeOptions = {
19 inputPath: videoInputPath, 22 inputPath: videoInputPath,
20 outputPath: videoTranscodedPath 23 outputPath: videoTranscodedPath,
24 resolution: inputVideoFile.resolution
21 } 25 }
22 26
23 // Could be very long! 27 // Could be very long!
@@ -47,7 +51,7 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi
47 } 51 }
48} 52}
49 53
50async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { 54async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) {
51 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR 55 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
52 const extname = '.mp4' 56 const extname = '.mp4'
53 57
@@ -60,13 +64,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
60 size: 0, 64 size: 0,
61 videoId: video.id 65 videoId: video.id
62 }) 66 })
63 const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) 67 const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile))
64 68
65 const transcodeOptions = { 69 const transcodeOptions = {
66 inputPath: videoInputPath, 70 inputPath: videoInputPath,
67 outputPath: videoOutputPath, 71 outputPath: videoOutputPath,
68 resolution, 72 resolution,
69 isPortraitMode 73 isPortraitMode: isPortrait
70 } 74 }
71 75
72 await transcode(transcodeOptions) 76 await transcode(transcodeOptions)
@@ -84,6 +88,38 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
84 video.VideoFiles.push(newVideoFile) 88 video.VideoFiles.push(newVideoFile)
85} 89}
86 90
91async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) {
92 const baseHlsDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
93 await ensureDir(join(HLS_PLAYLIST_DIRECTORY, video.uuid))
94
95 const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile()))
96 const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
97
98 const transcodeOptions = {
99 inputPath: videoInputPath,
100 outputPath,
101 resolution,
102 isPortraitMode,
103 generateHlsPlaylist: true
104 }
105
106 await transcode(transcodeOptions)
107
108 await updateMasterHLSPlaylist(video)
109 await updateSha256Segments(video)
110
111 const playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
112
113 await VideoStreamingPlaylistModel.upsert({
114 videoId: video.id,
115 playlistUrl,
116 segmentsSha256Url: CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid),
117 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles),
118
119 type: VideoStreamingPlaylistType.HLS
120 })
121}
122
87async function importVideoFile (video: VideoModel, inputFilePath: string) { 123async function importVideoFile (video: VideoModel, inputFilePath: string) {
88 const { videoFileResolution } = await getVideoFileResolution(inputFilePath) 124 const { videoFileResolution } = await getVideoFileResolution(inputFilePath)
89 const { size } = await stat(inputFilePath) 125 const { size } = await stat(inputFilePath)
@@ -125,6 +161,7 @@ async function importVideoFile (video: VideoModel, inputFilePath: string) {
125} 161}
126 162
127export { 163export {
164 generateHlsPlaylist,
128 optimizeVideofile, 165 optimizeVideofile,
129 transcodeOriginalVideofile, 166 transcodeOriginalVideofile,
130 importVideoFile 167 importVideoFile
diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts
index c72ab78b2..329322509 100644
--- a/server/middlewares/validators/redundancy.ts
+++ b/server/middlewares/validators/redundancy.ts
@@ -13,7 +13,7 @@ import { ActorFollowModel } from '../../models/activitypub/actor-follow'
13import { SERVER_ACTOR_NAME } from '../../initializers' 13import { SERVER_ACTOR_NAME } from '../../initializers'
14import { ServerModel } from '../../models/server/server' 14import { ServerModel } from '../../models/server/server'
15 15
16const videoRedundancyGetValidator = [ 16const videoFileRedundancyGetValidator = [
17 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), 17 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
18 param('resolution') 18 param('resolution')
19 .customSanitizer(toIntOrNull) 19 .customSanitizer(toIntOrNull)
@@ -24,7 +24,7 @@ const videoRedundancyGetValidator = [
24 .custom(exists).withMessage('Should have a valid fps'), 24 .custom(exists).withMessage('Should have a valid fps'),
25 25
26 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 26 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
27 logger.debug('Checking videoRedundancyGetValidator parameters', { parameters: req.params }) 27 logger.debug('Checking videoFileRedundancyGetValidator parameters', { parameters: req.params })
28 28
29 if (areValidationErrors(req, res)) return 29 if (areValidationErrors(req, res)) return
30 if (!await isVideoExist(req.params.videoId, res)) return 30 if (!await isVideoExist(req.params.videoId, res)) return
@@ -38,7 +38,31 @@ const videoRedundancyGetValidator = [
38 res.locals.videoFile = videoFile 38 res.locals.videoFile = videoFile
39 39
40 const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) 40 const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
41 if (!videoRedundancy)return res.status(404).json({ error: 'Video redundancy not found.' }) 41 if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' })
42 res.locals.videoRedundancy = videoRedundancy
43
44 return next()
45 }
46]
47
48const videoPlaylistRedundancyGetValidator = [
49 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
50 param('streamingPlaylistType').custom(exists).withMessage('Should have a valid streaming playlist type'),
51
52 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
53 logger.debug('Checking videoPlaylistRedundancyGetValidator parameters', { parameters: req.params })
54
55 if (areValidationErrors(req, res)) return
56 if (!await isVideoExist(req.params.videoId, res)) return
57
58 const video: VideoModel = res.locals.video
59 const videoStreamingPlaylist = video.VideoStreamingPlaylists.find(p => p === req.params.streamingPlaylistType)
60
61 if (!videoStreamingPlaylist) return res.status(404).json({ error: 'Video playlist not found.' })
62 res.locals.videoStreamingPlaylist = videoStreamingPlaylist
63
64 const videoRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(videoStreamingPlaylist.id)
65 if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' })
42 res.locals.videoRedundancy = videoRedundancy 66 res.locals.videoRedundancy = videoRedundancy
43 67
44 return next() 68 return next()
@@ -75,6 +99,7 @@ const updateServerRedundancyValidator = [
75// --------------------------------------------------------------------------- 99// ---------------------------------------------------------------------------
76 100
77export { 101export {
78 videoRedundancyGetValidator, 102 videoFileRedundancyGetValidator,
103 videoPlaylistRedundancyGetValidator,
79 updateServerRedundancyValidator 104 updateServerRedundancyValidator
80} 105}
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index 8f2ef2d9a..b722bed14 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -28,6 +28,7 @@ import { sample } from 'lodash'
28import { isTestInstance } from '../../helpers/core-utils' 28import { isTestInstance } from '../../helpers/core-utils'
29import * as Bluebird from 'bluebird' 29import * as Bluebird from 'bluebird'
30import * as Sequelize from 'sequelize' 30import * as Sequelize from 'sequelize'
31import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
31 32
32export enum ScopeNames { 33export enum ScopeNames {
33 WITH_VIDEO = 'WITH_VIDEO' 34 WITH_VIDEO = 'WITH_VIDEO'
@@ -38,7 +39,17 @@ export enum ScopeNames {
38 include: [ 39 include: [
39 { 40 {
40 model: () => VideoFileModel, 41 model: () => VideoFileModel,
41 required: true, 42 required: false,
43 include: [
44 {
45 model: () => VideoModel,
46 required: true
47 }
48 ]
49 },
50 {
51 model: () => VideoStreamingPlaylistModel,
52 required: false,
42 include: [ 53 include: [
43 { 54 {
44 model: () => VideoModel, 55 model: () => VideoModel,
@@ -97,12 +108,24 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
97 108
98 @BelongsTo(() => VideoFileModel, { 109 @BelongsTo(() => VideoFileModel, {
99 foreignKey: { 110 foreignKey: {
100 allowNull: false 111 allowNull: true
101 }, 112 },
102 onDelete: 'cascade' 113 onDelete: 'cascade'
103 }) 114 })
104 VideoFile: VideoFileModel 115 VideoFile: VideoFileModel
105 116
117 @ForeignKey(() => VideoStreamingPlaylistModel)
118 @Column
119 videoStreamingPlaylistId: number
120
121 @BelongsTo(() => VideoStreamingPlaylistModel, {
122 foreignKey: {
123 allowNull: true
124 },
125 onDelete: 'cascade'
126 })
127 VideoStreamingPlaylist: VideoStreamingPlaylistModel
128
106 @ForeignKey(() => ActorModel) 129 @ForeignKey(() => ActorModel)
107 @Column 130 @Column
108 actorId: number 131 actorId: number
@@ -119,13 +142,25 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
119 static async removeFile (instance: VideoRedundancyModel) { 142 static async removeFile (instance: VideoRedundancyModel) {
120 if (!instance.isOwned()) return 143 if (!instance.isOwned()) return
121 144
122 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId) 145 if (instance.videoFileId) {
146 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
123 147
124 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` 148 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
125 logger.info('Removing duplicated video file %s.', logIdentifier) 149 logger.info('Removing duplicated video file %s.', logIdentifier)
126 150
127 videoFile.Video.removeFile(videoFile, true) 151 videoFile.Video.removeFile(videoFile, true)
128 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) 152 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
153 }
154
155 if (instance.videoStreamingPlaylistId) {
156 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
157
158 const videoUUID = videoStreamingPlaylist.Video.uuid
159 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
160
161 videoStreamingPlaylist.Video.removeStreamingPlaylist(true)
162 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
163 }
129 164
130 return undefined 165 return undefined
131 } 166 }
@@ -143,6 +178,19 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
143 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) 178 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
144 } 179 }
145 180
181 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number) {
182 const actor = await getServerActor()
183
184 const query = {
185 where: {
186 actorId: actor.id,
187 videoStreamingPlaylistId
188 }
189 }
190
191 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
192 }
193
146 static loadByUrl (url: string, transaction?: Sequelize.Transaction) { 194 static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
147 const query = { 195 const query = {
148 where: { 196 where: {
@@ -191,7 +239,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
191 const ids = rows.map(r => r.id) 239 const ids = rows.map(r => r.id)
192 const id = sample(ids) 240 const id = sample(ids)
193 241
194 return VideoModel.loadWithFile(id, undefined, !isTestInstance()) 242 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
195 } 243 }
196 244
197 static async findMostViewToDuplicate (randomizedFactor: number) { 245 static async findMostViewToDuplicate (randomizedFactor: number) {
@@ -333,40 +381,44 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
333 381
334 static async listLocalOfServer (serverId: number) { 382 static async listLocalOfServer (serverId: number) {
335 const actor = await getServerActor() 383 const actor = await getServerActor()
336 384 const buildVideoInclude = () => ({
337 const query = { 385 model: VideoModel,
338 where: { 386 required: true,
339 actorId: actor.id
340 },
341 include: [ 387 include: [
342 { 388 {
343 model: VideoFileModel, 389 attributes: [],
390 model: VideoChannelModel.unscoped(),
344 required: true, 391 required: true,
345 include: [ 392 include: [
346 { 393 {
347 model: VideoModel, 394 attributes: [],
395 model: ActorModel.unscoped(),
348 required: true, 396 required: true,
349 include: [ 397 where: {
350 { 398 serverId
351 attributes: [], 399 }
352 model: VideoChannelModel.unscoped(),
353 required: true,
354 include: [
355 {
356 attributes: [],
357 model: ActorModel.unscoped(),
358 required: true,
359 where: {
360 serverId
361 }
362 }
363 ]
364 }
365 ]
366 } 400 }
367 ] 401 ]
368 } 402 }
369 ] 403 ]
404 })
405
406 const query = {
407 where: {
408 actorId: actor.id
409 },
410 include: [
411 {
412 model: VideoFileModel,
413 required: false,
414 include: [ buildVideoInclude() ]
415 },
416 {
417 model: VideoStreamingPlaylistModel,
418 required: false,
419 include: [ buildVideoInclude() ]
420 }
421 ]
370 } 422 }
371 423
372 return VideoRedundancyModel.findAll(query) 424 return VideoRedundancyModel.findAll(query)
@@ -403,11 +455,32 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
403 })) 455 }))
404 } 456 }
405 457
458 getVideo () {
459 if (this.VideoFile) return this.VideoFile.Video
460
461 return this.VideoStreamingPlaylist.Video
462 }
463
406 isOwned () { 464 isOwned () {
407 return !!this.strategy 465 return !!this.strategy
408 } 466 }
409 467
410 toActivityPubObject (): CacheFileObject { 468 toActivityPubObject (): CacheFileObject {
469 if (this.VideoStreamingPlaylist) {
470 return {
471 id: this.url,
472 type: 'CacheFile' as 'CacheFile',
473 object: this.VideoStreamingPlaylist.Video.url,
474 expires: this.expiresOn.toISOString(),
475 url: {
476 type: 'Link',
477 mimeType: 'application/x-mpegURL',
478 mediaType: 'application/x-mpegURL',
479 href: this.fileUrl
480 }
481 }
482 }
483
411 return { 484 return {
412 id: this.url, 485 id: this.url,
413 type: 'CacheFile' as 'CacheFile', 486 type: 'CacheFile' as 'CacheFile',
@@ -431,7 +504,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
431 504
432 const notIn = Sequelize.literal( 505 const notIn = Sequelize.literal(
433 '(' + 506 '(' +
434 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` + 507 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
435 ')' 508 ')'
436 ) 509 )
437 510
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 1f1b76c1e..7d1e371b9 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -62,7 +62,7 @@ export class VideoFileModel extends Model<VideoFileModel> {
62 extname: string 62 extname: string
63 63
64 @AllowNull(false) 64 @AllowNull(false)
65 @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash')) 65 @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash'))
66 @Column 66 @Column
67 infoHash: string 67 infoHash: string
68 68
@@ -86,14 +86,14 @@ export class VideoFileModel extends Model<VideoFileModel> {
86 86
87 @HasMany(() => VideoRedundancyModel, { 87 @HasMany(() => VideoRedundancyModel, {
88 foreignKey: { 88 foreignKey: {
89 allowNull: false 89 allowNull: true
90 }, 90 },
91 onDelete: 'CASCADE', 91 onDelete: 'CASCADE',
92 hooks: true 92 hooks: true
93 }) 93 })
94 RedundancyVideos: VideoRedundancyModel[] 94 RedundancyVideos: VideoRedundancyModel[]
95 95
96 static isInfohashExists (infoHash: string) { 96 static doesInfohashExist (infoHash: string) {
97 const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' 97 const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
98 const options = { 98 const options = {
99 type: Sequelize.QueryTypes.SELECT, 99 type: Sequelize.QueryTypes.SELECT,
diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts
index de0747f22..e49dbee30 100644
--- a/server/models/video/video-format-utils.ts
+++ b/server/models/video/video-format-utils.ts
@@ -1,7 +1,12 @@
1import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' 1import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
2import { VideoModel } from './video' 2import { VideoModel } from './video'
3import { VideoFileModel } from './video-file' 3import { VideoFileModel } from './video-file'
4import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects' 4import {
5 ActivityPlaylistInfohashesObject,
6 ActivityPlaylistSegmentHashesObject,
7 ActivityUrlObject,
8 VideoTorrentObject
9} from '../../../shared/models/activitypub/objects'
5import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers' 10import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers'
6import { VideoCaptionModel } from './video-caption' 11import { VideoCaptionModel } from './video-caption'
7import { 12import {
@@ -11,6 +16,8 @@ import {
11 getVideoSharesActivityPubUrl 16 getVideoSharesActivityPubUrl
12} from '../../lib/activitypub' 17} from '../../lib/activitypub'
13import { isArray } from '../../helpers/custom-validators/misc' 18import { isArray } from '../../helpers/custom-validators/misc'
19import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
20import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
14 21
15export type VideoFormattingJSONOptions = { 22export type VideoFormattingJSONOptions = {
16 completeDescription?: boolean 23 completeDescription?: boolean
@@ -120,7 +127,12 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
120 } 127 }
121 }) 128 })
122 129
130 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
131
123 const tags = video.Tags ? video.Tags.map(t => t.name) : [] 132 const tags = video.Tags ? video.Tags.map(t => t.name) : []
133
134 const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists)
135
124 const detailsJson = { 136 const detailsJson = {
125 support: video.support, 137 support: video.support,
126 descriptionPath: video.getDescriptionAPIPath(), 138 descriptionPath: video.getDescriptionAPIPath(),
@@ -133,7 +145,11 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
133 id: video.state, 145 id: video.state,
134 label: VideoModel.getStateLabel(video.state) 146 label: VideoModel.getStateLabel(video.state)
135 }, 147 },
136 files: [] 148
149 trackerUrls: video.getTrackerUrls(baseUrlHttp, baseUrlWs),
150
151 files: [],
152 streamingPlaylists
137 } 153 }
138 154
139 // Format and sort video files 155 // Format and sort video files
@@ -142,6 +158,25 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
142 return Object.assign(formattedJson, detailsJson) 158 return Object.assign(formattedJson, detailsJson)
143} 159}
144 160
161function streamingPlaylistsModelToFormattedJSON (video: VideoModel, playlists: VideoStreamingPlaylistModel[]): VideoStreamingPlaylist[] {
162 if (isArray(playlists) === false) return []
163
164 return playlists
165 .map(playlist => {
166 const redundancies = isArray(playlist.RedundancyVideos)
167 ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl }))
168 : []
169
170 return {
171 id: playlist.id,
172 type: playlist.type,
173 playlistUrl: playlist.playlistUrl,
174 segmentsSha256Url: playlist.segmentsSha256Url,
175 redundancies
176 } as VideoStreamingPlaylist
177 })
178}
179
145function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] { 180function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] {
146 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 181 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
147 182
@@ -232,6 +267,28 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject {
232 }) 267 })
233 } 268 }
234 269
270 for (const playlist of (video.VideoStreamingPlaylists || [])) {
271 let tag: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[]
272
273 tag = playlist.p2pMediaLoaderInfohashes
274 .map(i => ({ type: 'Infohash' as 'Infohash', name: i }))
275 tag.push({
276 type: 'Link',
277 name: 'sha256',
278 mimeType: 'application/json' as 'application/json',
279 mediaType: 'application/json' as 'application/json',
280 href: playlist.segmentsSha256Url
281 })
282
283 url.push({
284 type: 'Link',
285 mimeType: 'application/x-mpegURL' as 'application/x-mpegURL',
286 mediaType: 'application/x-mpegURL' as 'application/x-mpegURL',
287 href: playlist.playlistUrl,
288 tag
289 })
290 }
291
235 // Add video url too 292 // Add video url too
236 url.push({ 293 url.push({
237 type: 'Link', 294 type: 'Link',
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
new file mode 100644
index 000000000..bce537781
--- /dev/null
+++ b/server/models/video/video-streaming-playlist.ts
@@ -0,0 +1,154 @@
1import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
2import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
3import { throwIfNotValid } from '../utils'
4import { VideoModel } from './video'
5import * as Sequelize from 'sequelize'
6import { VideoRedundancyModel } from '../redundancy/video-redundancy'
7import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
8import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
9import { CONSTRAINTS_FIELDS, STATIC_PATHS } from '../../initializers'
10import { VideoFileModel } from './video-file'
11import { join } from 'path'
12import { sha1 } from '../../helpers/core-utils'
13import { isArrayOf } from '../../helpers/custom-validators/misc'
14
15@Table({
16 tableName: 'videoStreamingPlaylist',
17 indexes: [
18 {
19 fields: [ 'videoId' ]
20 },
21 {
22 fields: [ 'videoId', 'type' ],
23 unique: true
24 },
25 {
26 fields: [ 'p2pMediaLoaderInfohashes' ],
27 using: 'gin'
28 }
29 ]
30})
31export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistModel> {
32 @CreatedAt
33 createdAt: Date
34
35 @UpdatedAt
36 updatedAt: Date
37
38 @AllowNull(false)
39 @Column
40 type: VideoStreamingPlaylistType
41
42 @AllowNull(false)
43 @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url'))
44 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max))
45 playlistUrl: string
46
47 @AllowNull(false)
48 @Is('VideoStreamingPlaylistInfoHashes', value => throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes'))
49 @Column(DataType.ARRAY(DataType.STRING))
50 p2pMediaLoaderInfohashes: string[]
51
52 @AllowNull(false)
53 @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url'))
54 @Column
55 segmentsSha256Url: string
56
57 @ForeignKey(() => VideoModel)
58 @Column
59 videoId: number
60
61 @BelongsTo(() => VideoModel, {
62 foreignKey: {
63 allowNull: false
64 },
65 onDelete: 'CASCADE'
66 })
67 Video: VideoModel
68
69 @HasMany(() => VideoRedundancyModel, {
70 foreignKey: {
71 allowNull: false
72 },
73 onDelete: 'CASCADE',
74 hooks: true
75 })
76 RedundancyVideos: VideoRedundancyModel[]
77
78 static doesInfohashExist (infoHash: string) {
79 const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1'
80 const options = {
81 type: Sequelize.QueryTypes.SELECT,
82 bind: { infoHash },
83 raw: true
84 }
85
86 return VideoModel.sequelize.query(query, options)
87 .then(results => {
88 return results.length === 1
89 })
90 }
91
92 static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: VideoFileModel[]) {
93 const hashes: string[] = []
94
95 // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L97
96 for (let i = 0; i < videoFiles.length; i++) {
97 hashes.push(sha1(`1${playlistUrl}+V${i}`))
98 }
99
100 return hashes
101 }
102
103 static loadWithVideo (id: number) {
104 const options = {
105 include: [
106 {
107 model: VideoModel.unscoped(),
108 required: true
109 }
110 ]
111 }
112
113 return VideoStreamingPlaylistModel.findById(id, options)
114 }
115
116 static getHlsPlaylistFilename (resolution: number) {
117 return resolution + '.m3u8'
118 }
119
120 static getMasterHlsPlaylistFilename () {
121 return 'master.m3u8'
122 }
123
124 static getHlsSha256SegmentsFilename () {
125 return 'segments-sha256.json'
126 }
127
128 static getHlsMasterPlaylistStaticPath (videoUUID: string) {
129 return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
130 }
131
132 static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) {
133 return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
134 }
135
136 static getHlsSha256SegmentsStaticPath (videoUUID: string) {
137 return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
138 }
139
140 getStringType () {
141 if (this.type === VideoStreamingPlaylistType.HLS) return 'hls'
142
143 return 'unknown'
144 }
145
146 getVideoRedundancyUrl (baseUrlHttp: string) {
147 return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getStringType() + '/' + this.Video.uuid
148 }
149
150 hasSameUniqueKeysThan (other: VideoStreamingPlaylistModel) {
151 return this.type === other.type &&
152 this.videoId === other.videoId
153 }
154}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 80a6c7832..702260772 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -52,7 +52,7 @@ import {
52 ACTIVITY_PUB, 52 ACTIVITY_PUB,
53 API_VERSION, 53 API_VERSION,
54 CONFIG, 54 CONFIG,
55 CONSTRAINTS_FIELDS, 55 CONSTRAINTS_FIELDS, HLS_PLAYLIST_DIRECTORY, HLS_REDUNDANCY_DIRECTORY,
56 PREVIEWS_SIZE, 56 PREVIEWS_SIZE,
57 REMOTE_SCHEME, 57 REMOTE_SCHEME,
58 STATIC_DOWNLOAD_PATHS, 58 STATIC_DOWNLOAD_PATHS,
@@ -95,6 +95,7 @@ import * as validator from 'validator'
95import { UserVideoHistoryModel } from '../account/user-video-history' 95import { UserVideoHistoryModel } from '../account/user-video-history'
96import { UserModel } from '../account/user' 96import { UserModel } from '../account/user'
97import { VideoImportModel } from './video-import' 97import { VideoImportModel } from './video-import'
98import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
98 99
99// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation 100// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
100const indexes: Sequelize.DefineIndexesOptions[] = [ 101const indexes: Sequelize.DefineIndexesOptions[] = [
@@ -159,7 +160,9 @@ export enum ScopeNames {
159 WITH_FILES = 'WITH_FILES', 160 WITH_FILES = 'WITH_FILES',
160 WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', 161 WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE',
161 WITH_BLACKLISTED = 'WITH_BLACKLISTED', 162 WITH_BLACKLISTED = 'WITH_BLACKLISTED',
162 WITH_USER_HISTORY = 'WITH_USER_HISTORY' 163 WITH_USER_HISTORY = 'WITH_USER_HISTORY',
164 WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS',
165 WITH_USER_ID = 'WITH_USER_ID'
163} 166}
164 167
165type ForAPIOptions = { 168type ForAPIOptions = {
@@ -463,6 +466,22 @@ type AvailableForListIDsOptions = {
463 466
464 return query 467 return query
465 }, 468 },
469 [ ScopeNames.WITH_USER_ID ]: {
470 include: [
471 {
472 attributes: [ 'accountId' ],
473 model: () => VideoChannelModel.unscoped(),
474 required: true,
475 include: [
476 {
477 attributes: [ 'userId' ],
478 model: () => AccountModel.unscoped(),
479 required: true
480 }
481 ]
482 }
483 ]
484 },
466 [ ScopeNames.WITH_ACCOUNT_DETAILS ]: { 485 [ ScopeNames.WITH_ACCOUNT_DETAILS ]: {
467 include: [ 486 include: [
468 { 487 {
@@ -527,22 +546,55 @@ type AvailableForListIDsOptions = {
527 } 546 }
528 ] 547 ]
529 }, 548 },
530 [ ScopeNames.WITH_FILES ]: { 549 [ ScopeNames.WITH_FILES ]: (withRedundancies = false) => {
531 include: [ 550 let subInclude: any[] = []
532 { 551
533 model: () => VideoFileModel.unscoped(), 552 if (withRedundancies === true) {
534 // FIXME: typings 553 subInclude = [
535 [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join 554 {
536 required: false, 555 attributes: [ 'fileUrl' ],
537 include: [ 556 model: VideoRedundancyModel.unscoped(),
538 { 557 required: false
539 attributes: [ 'fileUrl' ], 558 }
540 model: () => VideoRedundancyModel.unscoped(), 559 ]
541 required: false 560 }
542 } 561
543 ] 562 return {
544 } 563 include: [
545 ] 564 {
565 model: VideoFileModel.unscoped(),
566 // FIXME: typings
567 [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join
568 required: false,
569 include: subInclude
570 }
571 ]
572 }
573 },
574 [ ScopeNames.WITH_STREAMING_PLAYLISTS ]: (withRedundancies = false) => {
575 let subInclude: any[] = []
576
577 if (withRedundancies === true) {
578 subInclude = [
579 {
580 attributes: [ 'fileUrl' ],
581 model: VideoRedundancyModel.unscoped(),
582 required: false
583 }
584 ]
585 }
586
587 return {
588 include: [
589 {
590 model: VideoStreamingPlaylistModel.unscoped(),
591 // FIXME: typings
592 [ 'separate' as any ]: true, // We may have multiple streaming playlists, having multiple redundancies so let's separate this join
593 required: false,
594 include: subInclude
595 }
596 ]
597 }
546 }, 598 },
547 [ ScopeNames.WITH_SCHEDULED_UPDATE ]: { 599 [ ScopeNames.WITH_SCHEDULED_UPDATE ]: {
548 include: [ 600 include: [
@@ -722,6 +774,16 @@ export class VideoModel extends Model<VideoModel> {
722 }) 774 })
723 VideoFiles: VideoFileModel[] 775 VideoFiles: VideoFileModel[]
724 776
777 @HasMany(() => VideoStreamingPlaylistModel, {
778 foreignKey: {
779 name: 'videoId',
780 allowNull: false
781 },
782 hooks: true,
783 onDelete: 'cascade'
784 })
785 VideoStreamingPlaylists: VideoStreamingPlaylistModel[]
786
725 @HasMany(() => VideoShareModel, { 787 @HasMany(() => VideoShareModel, {
726 foreignKey: { 788 foreignKey: {
727 name: 'videoId', 789 name: 'videoId',
@@ -847,6 +909,9 @@ export class VideoModel extends Model<VideoModel> {
847 tasks.push(instance.removeFile(file)) 909 tasks.push(instance.removeFile(file))
848 tasks.push(instance.removeTorrent(file)) 910 tasks.push(instance.removeTorrent(file))
849 }) 911 })
912
913 // Remove playlists file
914 tasks.push(instance.removeStreamingPlaylist())
850 } 915 }
851 916
852 // Do not wait video deletion because we could be in a transaction 917 // Do not wait video deletion because we could be in a transaction
@@ -858,10 +923,6 @@ export class VideoModel extends Model<VideoModel> {
858 return undefined 923 return undefined
859 } 924 }
860 925
861 static list () {
862 return VideoModel.scope(ScopeNames.WITH_FILES).findAll()
863 }
864
865 static listLocal () { 926 static listLocal () {
866 const query = { 927 const query = {
867 where: { 928 where: {
@@ -869,7 +930,7 @@ export class VideoModel extends Model<VideoModel> {
869 } 930 }
870 } 931 }
871 932
872 return VideoModel.scope(ScopeNames.WITH_FILES).findAll(query) 933 return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]).findAll(query)
873 } 934 }
874 935
875 static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { 936 static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) {
@@ -1200,6 +1261,16 @@ export class VideoModel extends Model<VideoModel> {
1200 return VideoModel.findOne(options) 1261 return VideoModel.findOne(options)
1201 } 1262 }
1202 1263
1264 static loadWithRights (id: number | string, t?: Sequelize.Transaction) {
1265 const where = VideoModel.buildWhereIdOrUUID(id)
1266 const options = {
1267 where,
1268 transaction: t
1269 }
1270
1271 return VideoModel.scope([ ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_USER_ID ]).findOne(options)
1272 }
1273
1203 static loadOnlyId (id: number | string, t?: Sequelize.Transaction) { 1274 static loadOnlyId (id: number | string, t?: Sequelize.Transaction) {
1204 const where = VideoModel.buildWhereIdOrUUID(id) 1275 const where = VideoModel.buildWhereIdOrUUID(id)
1205 1276
@@ -1212,8 +1283,8 @@ export class VideoModel extends Model<VideoModel> {
1212 return VideoModel.findOne(options) 1283 return VideoModel.findOne(options)
1213 } 1284 }
1214 1285
1215 static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) { 1286 static loadWithFiles (id: number, t?: Sequelize.Transaction, logging?: boolean) {
1216 return VideoModel.scope(ScopeNames.WITH_FILES) 1287 return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ])
1217 .findById(id, { transaction: t, logging }) 1288 .findById(id, { transaction: t, logging })
1218 } 1289 }
1219 1290
@@ -1224,9 +1295,7 @@ export class VideoModel extends Model<VideoModel> {
1224 } 1295 }
1225 } 1296 }
1226 1297
1227 return VideoModel 1298 return VideoModel.findOne(options)
1228 .scope([ ScopeNames.WITH_FILES ])
1229 .findOne(options)
1230 } 1299 }
1231 1300
1232 static loadByUrl (url: string, transaction?: Sequelize.Transaction) { 1301 static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
@@ -1248,7 +1317,11 @@ export class VideoModel extends Model<VideoModel> {
1248 transaction 1317 transaction
1249 } 1318 }
1250 1319
1251 return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) 1320 return VideoModel.scope([
1321 ScopeNames.WITH_ACCOUNT_DETAILS,
1322 ScopeNames.WITH_FILES,
1323 ScopeNames.WITH_STREAMING_PLAYLISTS
1324 ]).findOne(query)
1252 } 1325 }
1253 1326
1254 static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) { 1327 static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) {
@@ -1263,9 +1336,37 @@ export class VideoModel extends Model<VideoModel> {
1263 const scopes = [ 1336 const scopes = [
1264 ScopeNames.WITH_TAGS, 1337 ScopeNames.WITH_TAGS,
1265 ScopeNames.WITH_BLACKLISTED, 1338 ScopeNames.WITH_BLACKLISTED,
1339 ScopeNames.WITH_ACCOUNT_DETAILS,
1340 ScopeNames.WITH_SCHEDULED_UPDATE,
1266 ScopeNames.WITH_FILES, 1341 ScopeNames.WITH_FILES,
1342 ScopeNames.WITH_STREAMING_PLAYLISTS
1343 ]
1344
1345 if (userId) {
1346 scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings
1347 }
1348
1349 return VideoModel
1350 .scope(scopes)
1351 .findOne(options)
1352 }
1353
1354 static loadForGetAPI (id: number | string, t?: Sequelize.Transaction, userId?: number) {
1355 const where = VideoModel.buildWhereIdOrUUID(id)
1356
1357 const options = {
1358 order: [ [ 'Tags', 'name', 'ASC' ] ],
1359 where,
1360 transaction: t
1361 }
1362
1363 const scopes = [
1364 ScopeNames.WITH_TAGS,
1365 ScopeNames.WITH_BLACKLISTED,
1267 ScopeNames.WITH_ACCOUNT_DETAILS, 1366 ScopeNames.WITH_ACCOUNT_DETAILS,
1268 ScopeNames.WITH_SCHEDULED_UPDATE 1367 ScopeNames.WITH_SCHEDULED_UPDATE,
1368 { method: [ ScopeNames.WITH_FILES, true ] } as any, // FIXME: typings
1369 { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } as any // FIXME: typings
1269 ] 1370 ]
1270 1371
1271 if (userId) { 1372 if (userId) {
@@ -1612,6 +1713,14 @@ export class VideoModel extends Model<VideoModel> {
1612 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) 1713 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
1613 } 1714 }
1614 1715
1716 removeStreamingPlaylist (isRedundancy = false) {
1717 const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_PLAYLIST_DIRECTORY
1718
1719 const filePath = join(baseDir, this.uuid)
1720 return remove(filePath)
1721 .catch(err => logger.warn('Cannot delete playlist directory %s.', filePath, { err }))
1722 }
1723
1615 isOutdated () { 1724 isOutdated () {
1616 if (this.isOwned()) return false 1725 if (this.isOwned()) return false
1617 1726
@@ -1646,7 +1755,7 @@ export class VideoModel extends Model<VideoModel> {
1646 1755
1647 generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { 1756 generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
1648 const xs = this.getTorrentUrl(videoFile, baseUrlHttp) 1757 const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
1649 const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] 1758 const announce = this.getTrackerUrls(baseUrlHttp, baseUrlWs)
1650 let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] 1759 let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
1651 1760
1652 const redundancies = videoFile.RedundancyVideos 1761 const redundancies = videoFile.RedundancyVideos
@@ -1663,6 +1772,10 @@ export class VideoModel extends Model<VideoModel> {
1663 return magnetUtil.encode(magnetHash) 1772 return magnetUtil.encode(magnetHash)
1664 } 1773 }
1665 1774
1775 getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) {
1776 return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
1777 }
1778
1666 getThumbnailUrl (baseUrlHttp: string) { 1779 getThumbnailUrl (baseUrlHttp: string) {
1667 return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() 1780 return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName()
1668 } 1781 }
@@ -1686,4 +1799,8 @@ export class VideoModel extends Model<VideoModel> {
1686 getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { 1799 getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
1687 return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) 1800 return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
1688 } 1801 }
1802
1803 getBandwidthBits (videoFile: VideoFileModel) {
1804 return Math.ceil((videoFile.size * 8) / this.duration)
1805 }
1689} 1806}
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 4038ecbf0..07de2b5a5 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -65,6 +65,9 @@ describe('Test config API validators', function () {
65 '480p': true, 65 '480p': true,
66 '720p': false, 66 '720p': false,
67 '1080p': false 67 '1080p': false
68 },
69 hls: {
70 enabled: false
68 } 71 }
69 }, 72 },
70 import: { 73 import: {
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 9d3ce8153..5b99309fb 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -17,7 +17,7 @@ import {
17 viewVideo, 17 viewVideo,
18 wait, 18 wait,
19 waitUntilLog, 19 waitUntilLog,
20 checkVideoFilesWereRemoved, removeVideo, getVideoWithToken 20 checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer
21} from '../../../../shared/utils' 21} from '../../../../shared/utils'
22import { waitJobs } from '../../../../shared/utils/server/jobs' 22import { waitJobs } from '../../../../shared/utils/server/jobs'
23 23
@@ -48,6 +48,11 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
48 48
49async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { 49async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
50 const config = { 50 const config = {
51 transcoding: {
52 hls: {
53 enabled: true
54 }
55 },
51 redundancy: { 56 redundancy: {
52 videos: { 57 videos: {
53 check_interval: '5 seconds', 58 check_interval: '5 seconds',
@@ -85,7 +90,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
85 await waitJobs(servers) 90 await waitJobs(servers)
86} 91}
87 92
88async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { 93async function check1WebSeed (videoUUID?: string) {
89 if (!videoUUID) videoUUID = video1Server2UUID 94 if (!videoUUID) videoUUID = video1Server2UUID
90 95
91 const webseeds = [ 96 const webseeds = [
@@ -93,47 +98,17 @@ async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: str
93 ] 98 ]
94 99
95 for (const server of servers) { 100 for (const server of servers) {
96 { 101 // With token to avoid issues with video follow constraints
97 // With token to avoid issues with video follow constraints 102 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
98 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
99 103
100 const video: VideoDetails = res.body 104 const video: VideoDetails = res.body
101 for (const f of video.files) { 105 for (const f of video.files) {
102 checkMagnetWebseeds(f, webseeds, server) 106 checkMagnetWebseeds(f, webseeds, server)
103 }
104 } 107 }
105 } 108 }
106} 109}
107 110
108async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { 111async function check2Webseeds (videoUUID?: string) {
109 const res = await getStats(servers[0].url)
110 const data: ServerStats = res.body
111
112 expect(data.videosRedundancy).to.have.lengthOf(1)
113 const stat = data.videosRedundancy[0]
114
115 expect(stat.strategy).to.equal(strategy)
116 expect(stat.totalSize).to.equal(204800)
117 expect(stat.totalUsed).to.be.at.least(1).and.below(204801)
118 expect(stat.totalVideoFiles).to.equal(4)
119 expect(stat.totalVideos).to.equal(1)
120}
121
122async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
123 const res = await getStats(servers[0].url)
124 const data: ServerStats = res.body
125
126 expect(data.videosRedundancy).to.have.lengthOf(1)
127
128 const stat = data.videosRedundancy[0]
129 expect(stat.strategy).to.equal(strategy)
130 expect(stat.totalSize).to.equal(204800)
131 expect(stat.totalUsed).to.equal(0)
132 expect(stat.totalVideoFiles).to.equal(0)
133 expect(stat.totalVideos).to.equal(0)
134}
135
136async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
137 if (!videoUUID) videoUUID = video1Server2UUID 112 if (!videoUUID) videoUUID = video1Server2UUID
138 113
139 const webseeds = [ 114 const webseeds = [
@@ -158,7 +133,7 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st
158 await makeGetRequest({ 133 await makeGetRequest({
159 url: servers[1].url, 134 url: servers[1].url,
160 statusCodeExpected: 200, 135 statusCodeExpected: 200,
161 path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, 136 path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
162 contentType: null 137 contentType: null
163 }) 138 })
164 } 139 }
@@ -174,6 +149,81 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st
174 } 149 }
175} 150}
176 151
152async function check0PlaylistRedundancies (videoUUID?: string) {
153 if (!videoUUID) videoUUID = video1Server2UUID
154
155 for (const server of servers) {
156 // With token to avoid issues with video follow constraints
157 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
158 const video: VideoDetails = res.body
159
160 expect(video.streamingPlaylists).to.be.an('array')
161 expect(video.streamingPlaylists).to.have.lengthOf(1)
162 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
163 }
164}
165
166async function check1PlaylistRedundancies (videoUUID?: string) {
167 if (!videoUUID) videoUUID = video1Server2UUID
168
169 for (const server of servers) {
170 const res = await getVideo(server.url, videoUUID)
171 const video: VideoDetails = res.body
172
173 expect(video.streamingPlaylists).to.have.lengthOf(1)
174 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
175
176 const redundancy = video.streamingPlaylists[0].redundancies[0]
177
178 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
179 }
180
181 await makeGetRequest({
182 url: servers[0].url,
183 statusCodeExpected: 200,
184 path: `/static/redundancy/hls/${videoUUID}/360_000.ts`,
185 contentType: null
186 })
187
188 for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) {
189 const files = await readdir(join(root(), directory, videoUUID))
190 expect(files).to.have.length.at.least(4)
191
192 for (const resolution of [ 240, 360, 480, 720 ]) {
193 expect(files.find(f => f === `${resolution}_000.ts`)).to.not.be.undefined
194 expect(files.find(f => f === `${resolution}_001.ts`)).to.not.be.undefined
195 }
196 }
197}
198
199async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
200 const res = await getStats(servers[0].url)
201 const data: ServerStats = res.body
202
203 expect(data.videosRedundancy).to.have.lengthOf(1)
204 const stat = data.videosRedundancy[0]
205
206 expect(stat.strategy).to.equal(strategy)
207 expect(stat.totalSize).to.equal(204800)
208 expect(stat.totalUsed).to.be.at.least(1).and.below(204801)
209 expect(stat.totalVideoFiles).to.equal(4)
210 expect(stat.totalVideos).to.equal(1)
211}
212
213async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
214 const res = await getStats(servers[0].url)
215 const data: ServerStats = res.body
216
217 expect(data.videosRedundancy).to.have.lengthOf(1)
218
219 const stat = data.videosRedundancy[0]
220 expect(stat.strategy).to.equal(strategy)
221 expect(stat.totalSize).to.equal(204800)
222 expect(stat.totalUsed).to.equal(0)
223 expect(stat.totalVideoFiles).to.equal(0)
224 expect(stat.totalVideos).to.equal(0)
225}
226
177async function enableRedundancyOnServer1 () { 227async function enableRedundancyOnServer1 () {
178 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) 228 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
179 229
@@ -220,7 +270,8 @@ describe('Test videos redundancy', function () {
220 }) 270 })
221 271
222 it('Should have 1 webseed on the first video', async function () { 272 it('Should have 1 webseed on the first video', async function () {
223 await check1WebSeed(strategy) 273 await check1WebSeed()
274 await check0PlaylistRedundancies()
224 await checkStatsWith1Webseed(strategy) 275 await checkStatsWith1Webseed(strategy)
225 }) 276 })
226 277
@@ -229,27 +280,29 @@ describe('Test videos redundancy', function () {
229 }) 280 })
230 281
231 it('Should have 2 webseeds on the first video', async function () { 282 it('Should have 2 webseeds on the first video', async function () {
232 this.timeout(40000) 283 this.timeout(80000)
233 284
234 await waitJobs(servers) 285 await waitJobs(servers)
235 await waitUntilLog(servers[0], 'Duplicated ', 4) 286 await waitUntilLog(servers[0], 'Duplicated ', 5)
236 await waitJobs(servers) 287 await waitJobs(servers)
237 288
238 await check2Webseeds(strategy) 289 await check2Webseeds()
290 await check1PlaylistRedundancies()
239 await checkStatsWith2Webseed(strategy) 291 await checkStatsWith2Webseed(strategy)
240 }) 292 })
241 293
242 it('Should undo redundancy on server 1 and remove duplicated videos', async function () { 294 it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
243 this.timeout(40000) 295 this.timeout(80000)
244 296
245 await disableRedundancyOnServer1() 297 await disableRedundancyOnServer1()
246 298
247 await waitJobs(servers) 299 await waitJobs(servers)
248 await wait(5000) 300 await wait(5000)
249 301
250 await check1WebSeed(strategy) 302 await check1WebSeed()
303 await check0PlaylistRedundancies()
251 304
252 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) 305 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ])
253 }) 306 })
254 307
255 after(function () { 308 after(function () {
@@ -267,7 +320,8 @@ describe('Test videos redundancy', function () {
267 }) 320 })
268 321
269 it('Should have 1 webseed on the first video', async function () { 322 it('Should have 1 webseed on the first video', async function () {
270 await check1WebSeed(strategy) 323 await check1WebSeed()
324 await check0PlaylistRedundancies()
271 await checkStatsWith1Webseed(strategy) 325 await checkStatsWith1Webseed(strategy)
272 }) 326 })
273 327
@@ -276,25 +330,27 @@ describe('Test videos redundancy', function () {
276 }) 330 })
277 331
278 it('Should have 2 webseeds on the first video', async function () { 332 it('Should have 2 webseeds on the first video', async function () {
279 this.timeout(40000) 333 this.timeout(80000)
280 334
281 await waitJobs(servers) 335 await waitJobs(servers)
282 await waitUntilLog(servers[0], 'Duplicated ', 4) 336 await waitUntilLog(servers[0], 'Duplicated ', 5)
283 await waitJobs(servers) 337 await waitJobs(servers)
284 338
285 await check2Webseeds(strategy) 339 await check2Webseeds()
340 await check1PlaylistRedundancies()
286 await checkStatsWith2Webseed(strategy) 341 await checkStatsWith2Webseed(strategy)
287 }) 342 })
288 343
289 it('Should unfollow on server 1 and remove duplicated videos', async function () { 344 it('Should unfollow on server 1 and remove duplicated videos', async function () {
290 this.timeout(40000) 345 this.timeout(80000)
291 346
292 await unfollow(servers[0].url, servers[0].accessToken, servers[1]) 347 await unfollow(servers[0].url, servers[0].accessToken, servers[1])
293 348
294 await waitJobs(servers) 349 await waitJobs(servers)
295 await wait(5000) 350 await wait(5000)
296 351
297 await check1WebSeed(strategy) 352 await check1WebSeed()
353 await check0PlaylistRedundancies()
298 354
299 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) 355 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
300 }) 356 })
@@ -314,7 +370,8 @@ describe('Test videos redundancy', function () {
314 }) 370 })
315 371
316 it('Should have 1 webseed on the first video', async function () { 372 it('Should have 1 webseed on the first video', async function () {
317 await check1WebSeed(strategy) 373 await check1WebSeed()
374 await check0PlaylistRedundancies()
318 await checkStatsWith1Webseed(strategy) 375 await checkStatsWith1Webseed(strategy)
319 }) 376 })
320 377
@@ -323,18 +380,19 @@ describe('Test videos redundancy', function () {
323 }) 380 })
324 381
325 it('Should still have 1 webseed on the first video', async function () { 382 it('Should still have 1 webseed on the first video', async function () {
326 this.timeout(40000) 383 this.timeout(80000)
327 384
328 await waitJobs(servers) 385 await waitJobs(servers)
329 await wait(15000) 386 await wait(15000)
330 await waitJobs(servers) 387 await waitJobs(servers)
331 388
332 await check1WebSeed(strategy) 389 await check1WebSeed()
390 await check0PlaylistRedundancies()
333 await checkStatsWith1Webseed(strategy) 391 await checkStatsWith1Webseed(strategy)
334 }) 392 })
335 393
336 it('Should view 2 times the first video to have > min_views config', async function () { 394 it('Should view 2 times the first video to have > min_views config', async function () {
337 this.timeout(40000) 395 this.timeout(80000)
338 396
339 await viewVideo(servers[ 0 ].url, video1Server2UUID) 397 await viewVideo(servers[ 0 ].url, video1Server2UUID)
340 await viewVideo(servers[ 2 ].url, video1Server2UUID) 398 await viewVideo(servers[ 2 ].url, video1Server2UUID)
@@ -344,13 +402,14 @@ describe('Test videos redundancy', function () {
344 }) 402 })
345 403
346 it('Should have 2 webseeds on the first video', async function () { 404 it('Should have 2 webseeds on the first video', async function () {
347 this.timeout(40000) 405 this.timeout(80000)
348 406
349 await waitJobs(servers) 407 await waitJobs(servers)
350 await waitUntilLog(servers[0], 'Duplicated ', 4) 408 await waitUntilLog(servers[0], 'Duplicated ', 5)
351 await waitJobs(servers) 409 await waitJobs(servers)
352 410
353 await check2Webseeds(strategy) 411 await check2Webseeds()
412 await check1PlaylistRedundancies()
354 await checkStatsWith2Webseed(strategy) 413 await checkStatsWith2Webseed(strategy)
355 }) 414 })
356 415
@@ -405,7 +464,7 @@ describe('Test videos redundancy', function () {
405 }) 464 })
406 465
407 it('Should still have 2 webseeds after 10 seconds', async function () { 466 it('Should still have 2 webseeds after 10 seconds', async function () {
408 this.timeout(40000) 467 this.timeout(80000)
409 468
410 await wait(10000) 469 await wait(10000)
411 470
@@ -420,7 +479,7 @@ describe('Test videos redundancy', function () {
420 }) 479 })
421 480
422 it('Should stop server 1 and expire video redundancy', async function () { 481 it('Should stop server 1 and expire video redundancy', async function () {
423 this.timeout(40000) 482 this.timeout(80000)
424 483
425 killallServers([ servers[0] ]) 484 killallServers([ servers[0] ])
426 485
@@ -446,10 +505,11 @@ describe('Test videos redundancy', function () {
446 await enableRedundancyOnServer1() 505 await enableRedundancyOnServer1()
447 506
448 await waitJobs(servers) 507 await waitJobs(servers)
449 await waitUntilLog(servers[0], 'Duplicated ', 4) 508 await waitUntilLog(servers[0], 'Duplicated ', 5)
450 await waitJobs(servers) 509 await waitJobs(servers)
451 510
452 await check2Webseeds(strategy) 511 await check2Webseeds()
512 await check1PlaylistRedundancies()
453 await checkStatsWith2Webseed(strategy) 513 await checkStatsWith2Webseed(strategy)
454 514
455 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) 515 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
@@ -467,8 +527,10 @@ describe('Test videos redundancy', function () {
467 await wait(1000) 527 await wait(1000)
468 528
469 try { 529 try {
470 await check1WebSeed(strategy, video1Server2UUID) 530 await check1WebSeed(video1Server2UUID)
471 await check2Webseeds(strategy, video2Server2UUID) 531 await check0PlaylistRedundancies(video1Server2UUID)
532 await check2Webseeds(video2Server2UUID)
533 await check1PlaylistRedundancies(video2Server2UUID)
472 534
473 checked = true 535 checked = true
474 } catch { 536 } catch {
@@ -477,6 +539,26 @@ describe('Test videos redundancy', function () {
477 } 539 }
478 }) 540 })
479 541
542 it('Should disable strategy and remove redundancies', async function () {
543 this.timeout(80000)
544
545 await waitJobs(servers)
546
547 killallServers([ servers[ 0 ] ])
548 await reRunServer(servers[ 0 ], {
549 redundancy: {
550 videos: {
551 check_interval: '1 second',
552 strategies: []
553 }
554 }
555 })
556
557 await waitJobs(servers)
558
559 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ])
560 })
561
480 after(function () { 562 after(function () {
481 return cleanServers() 563 return cleanServers()
482 }) 564 })
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index bebfc7398..0dfe6e4fe 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -57,6 +57,8 @@ function checkInitialConfig (data: CustomConfig) {
57 expect(data.transcoding.resolutions['480p']).to.be.true 57 expect(data.transcoding.resolutions['480p']).to.be.true
58 expect(data.transcoding.resolutions['720p']).to.be.true 58 expect(data.transcoding.resolutions['720p']).to.be.true
59 expect(data.transcoding.resolutions['1080p']).to.be.true 59 expect(data.transcoding.resolutions['1080p']).to.be.true
60 expect(data.transcoding.hls.enabled).to.be.true
61
60 expect(data.import.videos.http.enabled).to.be.true 62 expect(data.import.videos.http.enabled).to.be.true
61 expect(data.import.videos.torrent.enabled).to.be.true 63 expect(data.import.videos.torrent.enabled).to.be.true
62} 64}
@@ -95,6 +97,7 @@ function checkUpdatedConfig (data: CustomConfig) {
95 expect(data.transcoding.resolutions['480p']).to.be.true 97 expect(data.transcoding.resolutions['480p']).to.be.true
96 expect(data.transcoding.resolutions['720p']).to.be.false 98 expect(data.transcoding.resolutions['720p']).to.be.false
97 expect(data.transcoding.resolutions['1080p']).to.be.false 99 expect(data.transcoding.resolutions['1080p']).to.be.false
100 expect(data.transcoding.hls.enabled).to.be.false
98 101
99 expect(data.import.videos.http.enabled).to.be.false 102 expect(data.import.videos.http.enabled).to.be.false
100 expect(data.import.videos.torrent.enabled).to.be.false 103 expect(data.import.videos.torrent.enabled).to.be.false
@@ -205,6 +208,9 @@ describe('Test config', function () {
205 '480p': true, 208 '480p': true,
206 '720p': false, 209 '720p': false,
207 '1080p': false 210 '1080p': false
211 },
212 hls: {
213 enabled: false
208 } 214 }
209 }, 215 },
210 import: { 216 import: {
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index 97f467aae..a501a80b2 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -8,6 +8,7 @@ import './video-change-ownership'
8import './video-channels' 8import './video-channels'
9import './video-comments' 9import './video-comments'
10import './video-description' 10import './video-description'
11import './video-hls'
11import './video-imports' 12import './video-imports'
12import './video-nsfw' 13import './video-nsfw'
13import './video-privacy' 14import './video-privacy'
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts
new file mode 100644
index 000000000..71d863b12
--- /dev/null
+++ b/server/tests/api/videos/video-hls.ts
@@ -0,0 +1,145 @@
1/* tslint:disable:no-unused-expression */
2
3import * as chai from 'chai'
4import 'mocha'
5import {
6 checkDirectoryIsEmpty,
7 checkTmpIsEmpty,
8 doubleFollow,
9 flushAndRunMultipleServers,
10 flushTests,
11 getPlaylist,
12 getSegment,
13 getSegmentSha256,
14 getVideo,
15 killallServers,
16 removeVideo,
17 ServerInfo,
18 setAccessTokensToServers,
19 updateVideo,
20 uploadVideo,
21 waitJobs
22} from '../../../../shared/utils'
23import { VideoDetails } from '../../../../shared/models/videos'
24import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type'
25import { sha256 } from '../../../helpers/core-utils'
26import { join } from 'path'
27
28const expect = chai.expect
29
30async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string) {
31 const resolutions = [ 240, 360, 480, 720 ]
32
33 for (const server of servers) {
34 const res = await getVideo(server.url, videoUUID)
35 const videoDetails: VideoDetails = res.body
36
37 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
38
39 const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
40 expect(hlsPlaylist).to.not.be.undefined
41
42 {
43 const res2 = await getPlaylist(hlsPlaylist.playlistUrl)
44
45 const masterPlaylist = res2.text
46
47 expect(masterPlaylist).to.contain('#EXT-X-STREAM-INF:BANDWIDTH=55472,RESOLUTION=640x360,FRAME-RATE=25')
48
49 for (const resolution of resolutions) {
50 expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
51 }
52 }
53
54 {
55 for (const resolution of resolutions) {
56 const res2 = await getPlaylist(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}.m3u8`)
57
58 const subPlaylist = res2.text
59 expect(subPlaylist).to.contain(resolution + '_000.ts')
60 }
61 }
62
63 {
64 for (const resolution of resolutions) {
65
66 const res2 = await getSegment(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}_000.ts`)
67
68 const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url)
69
70 const sha256Server = resSha.body[ resolution + '_000.ts' ]
71 expect(sha256(res2.body)).to.equal(sha256Server)
72 }
73 }
74 }
75}
76
77describe('Test HLS videos', function () {
78 let servers: ServerInfo[] = []
79 let videoUUID = ''
80
81 before(async function () {
82 this.timeout(120000)
83
84 servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true, hls: { enabled: true } } })
85
86 // Get the access tokens
87 await setAccessTokensToServers(servers)
88
89 // Server 1 and server 2 follow each other
90 await doubleFollow(servers[0], servers[1])
91 })
92
93 it('Should upload a video and transcode it to HLS', async function () {
94 this.timeout(120000)
95
96 {
97 const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1', fixture: 'video_short.webm' })
98 videoUUID = res.body.video.uuid
99 }
100
101 await waitJobs(servers)
102
103 await checkHlsPlaylist(servers, videoUUID)
104 })
105
106 it('Should update the video', async function () {
107 await updateVideo(servers[0].url, servers[0].accessToken, videoUUID, { name: 'video 1 updated' })
108
109 await waitJobs(servers)
110
111 await checkHlsPlaylist(servers, videoUUID)
112 })
113
114 it('Should delete the video', async function () {
115 await removeVideo(servers[0].url, servers[0].accessToken, videoUUID)
116
117 await waitJobs(servers)
118
119 for (const server of servers) {
120 await getVideo(server.url, videoUUID, 404)
121 }
122 })
123
124 it('Should have the playlists/segment deleted from the disk', async function () {
125 for (const server of servers) {
126 await checkDirectoryIsEmpty(server, 'videos')
127 await checkDirectoryIsEmpty(server, join('playlists', 'hls'))
128 }
129 })
130
131 it('Should have an empty tmp directory', async function () {
132 for (const server of servers) {
133 await checkTmpIsEmpty(server)
134 }
135 })
136
137 after(async function () {
138 killallServers(servers)
139
140 // Keep the logs if the test failed
141 if (this['ok']) {
142 await flushTests()
143 }
144 })
145})
diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts
index 811ea6a9f..d38bb4331 100644
--- a/server/tests/cli/update-host.ts
+++ b/server/tests/cli/update-host.ts
@@ -86,6 +86,13 @@ describe('Test update host scripts', function () {
86 const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid) 86 const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid)
87 87
88 expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid) 88 expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid)
89
90 const res = await getVideo(server.url, video.uuid)
91 const videoDetails: VideoDetails = res.body
92
93 expect(videoDetails.trackerUrls[0]).to.include(server.host)
94 expect(videoDetails.streamingPlaylists[0].playlistUrl).to.include(server.host)
95 expect(videoDetails.streamingPlaylists[0].segmentsSha256Url).to.include(server.host)
89 } 96 }
90 }) 97 })
91 98
@@ -100,7 +107,7 @@ describe('Test update host scripts', function () {
100 } 107 }
101 }) 108 })
102 109
103 it('Should have update accounts url', async function () { 110 it('Should have updated accounts url', async function () {
104 const res = await getAccountsList(server.url) 111 const res = await getAccountsList(server.url)
105 expect(res.body.total).to.equal(3) 112 expect(res.body.total).to.equal(3)
106 113
@@ -112,7 +119,7 @@ describe('Test update host scripts', function () {
112 } 119 }
113 }) 120 })
114 121
115 it('Should update torrent hosts', async function () { 122 it('Should have updated torrent hosts', async function () {
116 this.timeout(30000) 123 this.timeout(30000)
117 124
118 const res = await getVideosList(server.url) 125 const res = await getVideosList(server.url)