aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/activitypub/client.ts15
-rw-r--r--server/controllers/api/config.ts8
-rw-r--r--server/controllers/api/users/index.ts19
-rw-r--r--server/controllers/api/users/me.ts2
-rw-r--r--server/controllers/api/videos/index.ts19
-rw-r--r--server/controllers/static.ts9
-rw-r--r--server/controllers/tracker.ts25
-rw-r--r--server/helpers/activitypub.ts5
-rw-r--r--server/helpers/core-utils.ts8
-rw-r--r--server/helpers/custom-validators/activitypub/cache-file.ts12
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts8
-rw-r--r--server/helpers/custom-validators/misc.ts5
-rw-r--r--server/helpers/custom-validators/videos.ts4
-rw-r--r--server/helpers/ffmpeg-utils.ts38
-rw-r--r--server/helpers/requests.ts2
-rw-r--r--server/helpers/utils.ts1
-rw-r--r--server/helpers/video.ts4
-rw-r--r--server/initializers/checker-before-init.ts2
-rw-r--r--server/initializers/constants.ts20
-rw-r--r--server/initializers/database.ts4
-rw-r--r--server/initializers/installer.ts5
-rw-r--r--server/initializers/migrations/0330-video-streaming-playlist.ts51
-rw-r--r--server/lib/activitypub/actor.ts4
-rw-r--r--server/lib/activitypub/cache-file.ts23
-rw-r--r--server/lib/activitypub/send/send-create.ts7
-rw-r--r--server/lib/activitypub/send/send-undo.ts3
-rw-r--r--server/lib/activitypub/send/send-update.ts2
-rw-r--r--server/lib/activitypub/url.ts7
-rw-r--r--server/lib/activitypub/videos.ts97
-rw-r--r--server/lib/emailer.ts4
-rw-r--r--server/lib/hls.ts164
-rw-r--r--server/lib/job-queue/handlers/video-file.ts69
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts189
-rw-r--r--server/lib/video-transcoding.ts52
-rw-r--r--server/middlewares/csp.ts2
-rw-r--r--server/middlewares/validators/redundancy.ts33
-rw-r--r--server/middlewares/validators/users.ts2
-rw-r--r--server/models/account/user-notification.ts171
-rw-r--r--server/models/redundancy/video-redundancy.ts139
-rw-r--r--server/models/video/video-file.ts6
-rw-r--r--server/models/video/video-format-utils.ts61
-rw-r--r--server/models/video/video-streaming-playlist.ts158
-rw-r--r--server/models/video/video.ts179
-rw-r--r--server/tests/api/check-params/accounts.ts2
-rw-r--r--server/tests/api/check-params/config.ts3
-rw-r--r--server/tests/api/check-params/users.ts18
-rw-r--r--server/tests/api/redundancy/redundancy.ts216
-rw-r--r--server/tests/api/server/config.ts6
-rw-r--r--server/tests/api/users/user-notifications.ts36
-rw-r--r--server/tests/api/users/users.ts16
-rw-r--r--server/tests/api/videos/index.ts1
-rw-r--r--server/tests/api/videos/video-hls.ts139
-rw-r--r--server/tests/cli/update-host.ts11
53 files changed, 1752 insertions, 334 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts
index 7e87f6f3b..31c0a5fbd 100644
--- a/server/controllers/activitypub/client.ts
+++ b/server/controllers/activitypub/client.ts
@@ -33,7 +33,7 @@ import {
33 getVideoSharesActivityPubUrl 33 getVideoSharesActivityPubUrl
34} from '../../lib/activitypub' 34} from '../../lib/activitypub'
35import { VideoCaptionModel } from '../../models/video/video-caption' 35import { VideoCaptionModel } from '../../models/video/video-caption'
36import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy' 36import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy'
37import { getServerActor } from '../../helpers/utils' 37import { getServerActor } from '../../helpers/utils'
38import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 38import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
39import { buildDislikeActivity } from '../../lib/activitypub/send/send-dislike' 39import { buildDislikeActivity } from '../../lib/activitypub/send/send-dislike'
@@ -63,11 +63,11 @@ activityPubClientRouter.get('/accounts?/:name/dislikes/:videoId',
63 63
64activityPubClientRouter.get('/videos/watch/:id', 64activityPubClientRouter.get('/videos/watch/:id',
65 executeIfActivityPub(asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS))), 65 executeIfActivityPub(asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS))),
66 executeIfActivityPub(asyncMiddleware(videosGetValidator)), 66 executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))),
67 executeIfActivityPub(asyncMiddleware(videoController)) 67 executeIfActivityPub(asyncMiddleware(videoController))
68) 68)
69activityPubClientRouter.get('/videos/watch/:id/activity', 69activityPubClientRouter.get('/videos/watch/:id/activity',
70 executeIfActivityPub(asyncMiddleware(videosGetValidator)), 70 executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))),
71 executeIfActivityPub(asyncMiddleware(videoController)) 71 executeIfActivityPub(asyncMiddleware(videoController))
72) 72)
73activityPubClientRouter.get('/videos/watch/:id/announces', 73activityPubClientRouter.get('/videos/watch/:id/announces',
@@ -113,7 +113,11 @@ activityPubClientRouter.get('/video-channels/:name/following',
113) 113)
114 114
115activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?', 115activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?',
116 executeIfActivityPub(asyncMiddleware(videoRedundancyGetValidator)), 116 executeIfActivityPub(asyncMiddleware(videoFileRedundancyGetValidator)),
117 executeIfActivityPub(asyncMiddleware(videoRedundancyController))
118)
119activityPubClientRouter.get('/redundancy/video-playlists/:streamingPlaylistType/:videoId',
120 executeIfActivityPub(asyncMiddleware(videoPlaylistRedundancyGetValidator)),
117 executeIfActivityPub(asyncMiddleware(videoRedundancyController)) 121 executeIfActivityPub(asyncMiddleware(videoRedundancyController))
118) 122)
119 123
@@ -160,7 +164,8 @@ function getAccountVideoRate (rateType: VideoRateType) {
160} 164}
161 165
162async function videoController (req: express.Request, res: express.Response) { 166async function videoController (req: express.Request, res: express.Response) {
163 const video: VideoModel = res.locals.video 167 // We need more attributes
168 const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id)
164 169
165 if (video.url.startsWith(CONFIG.WEBSERVER.URL) === false) return res.redirect(video.url) 170 if (video.url.startsWith(CONFIG.WEBSERVER.URL) === false) return res.redirect(video.url)
166 171
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 255026f46..1f3341bc0 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -1,5 +1,5 @@
1import * as express from 'express' 1import * as express from 'express'
2import { omit, snakeCase } from 'lodash' 2import { snakeCase } from 'lodash'
3import { ServerConfig, UserRight } from '../../../shared' 3import { ServerConfig, UserRight } from '../../../shared'
4import { About } from '../../../shared/models/server/about.model' 4import { About } from '../../../shared/models/server/about.model'
5import { CustomConfig } from '../../../shared/models/server/custom-config.model' 5import { CustomConfig } from '../../../shared/models/server/custom-config.model'
@@ -78,6 +78,9 @@ async function getConfig (req: express.Request, res: express.Response) {
78 requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION 78 requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION
79 }, 79 },
80 transcoding: { 80 transcoding: {
81 hls: {
82 enabled: CONFIG.TRANSCODING.HLS.ENABLED
83 },
81 enabledResolutions 84 enabledResolutions
82 }, 85 },
83 import: { 86 import: {
@@ -246,6 +249,9 @@ function customConfig (): CustomConfig {
246 '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ], 249 '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ],
247 '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ], 250 '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ],
248 '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ] 251 '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ]
252 },
253 hls: {
254 enabled: CONFIG.TRANSCODING.HLS.ENABLED
249 } 255 }
250 }, 256 },
251 import: { 257 import: {
diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts
index dbe0718d4..e3533a7f6 100644
--- a/server/controllers/api/users/index.ts
+++ b/server/controllers/api/users/index.ts
@@ -229,7 +229,7 @@ async function unblockUser (req: express.Request, res: express.Response, next: e
229 return res.status(204).end() 229 return res.status(204).end()
230} 230}
231 231
232async function blockUser (req: express.Request, res: express.Response, next: express.NextFunction) { 232async function blockUser (req: express.Request, res: express.Response) {
233 const user: UserModel = res.locals.user 233 const user: UserModel = res.locals.user
234 const reason = req.body.reason 234 const reason = req.body.reason
235 235
@@ -238,23 +238,23 @@ async function blockUser (req: express.Request, res: express.Response, next: exp
238 return res.status(204).end() 238 return res.status(204).end()
239} 239}
240 240
241function getUser (req: express.Request, res: express.Response, next: express.NextFunction) { 241function getUser (req: express.Request, res: express.Response) {
242 return res.json((res.locals.user as UserModel).toFormattedJSON()) 242 return res.json((res.locals.user as UserModel).toFormattedJSON())
243} 243}
244 244
245async function autocompleteUsers (req: express.Request, res: express.Response, next: express.NextFunction) { 245async function autocompleteUsers (req: express.Request, res: express.Response) {
246 const resultList = await UserModel.autoComplete(req.query.search as string) 246 const resultList = await UserModel.autoComplete(req.query.search as string)
247 247
248 return res.json(resultList) 248 return res.json(resultList)
249} 249}
250 250
251async function listUsers (req: express.Request, res: express.Response, next: express.NextFunction) { 251async function listUsers (req: express.Request, res: express.Response) {
252 const resultList = await UserModel.listForApi(req.query.start, req.query.count, req.query.sort, req.query.search) 252 const resultList = await UserModel.listForApi(req.query.start, req.query.count, req.query.sort, req.query.search)
253 253
254 return res.json(getFormattedObjects(resultList.data, resultList.total)) 254 return res.json(getFormattedObjects(resultList.data, resultList.total))
255} 255}
256 256
257async function removeUser (req: express.Request, res: express.Response, next: express.NextFunction) { 257async function removeUser (req: express.Request, res: express.Response) {
258 const user: UserModel = res.locals.user 258 const user: UserModel = res.locals.user
259 259
260 await user.destroy() 260 await user.destroy()
@@ -264,12 +264,13 @@ async function removeUser (req: express.Request, res: express.Response, next: ex
264 return res.sendStatus(204) 264 return res.sendStatus(204)
265} 265}
266 266
267async function updateUser (req: express.Request, res: express.Response, next: express.NextFunction) { 267async function updateUser (req: express.Request, res: express.Response) {
268 const body: UserUpdate = req.body 268 const body: UserUpdate = req.body
269 const userToUpdate = res.locals.user as UserModel 269 const userToUpdate = res.locals.user as UserModel
270 const oldUserAuditView = new UserAuditView(userToUpdate.toFormattedJSON()) 270 const oldUserAuditView = new UserAuditView(userToUpdate.toFormattedJSON())
271 const roleChanged = body.role !== undefined && body.role !== userToUpdate.role 271 const roleChanged = body.role !== undefined && body.role !== userToUpdate.role
272 272
273 if (body.password !== undefined) userToUpdate.password = body.password
273 if (body.email !== undefined) userToUpdate.email = body.email 274 if (body.email !== undefined) userToUpdate.email = body.email
274 if (body.emailVerified !== undefined) userToUpdate.emailVerified = body.emailVerified 275 if (body.emailVerified !== undefined) userToUpdate.emailVerified = body.emailVerified
275 if (body.videoQuota !== undefined) userToUpdate.videoQuota = body.videoQuota 276 if (body.videoQuota !== undefined) userToUpdate.videoQuota = body.videoQuota
@@ -279,11 +280,11 @@ async function updateUser (req: express.Request, res: express.Response, next: ex
279 const user = await userToUpdate.save() 280 const user = await userToUpdate.save()
280 281
281 // Destroy user token to refresh rights 282 // Destroy user token to refresh rights
282 if (roleChanged) await deleteUserToken(userToUpdate.id) 283 if (roleChanged || body.password !== undefined) await deleteUserToken(userToUpdate.id)
283 284
284 auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) 285 auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView)
285 286
286 // Don't need to send this update to followers, these attributes are not propagated 287 // Don't need to send this update to followers, these attributes are not federated
287 288
288 return res.sendStatus(204) 289 return res.sendStatus(204)
289} 290}
@@ -293,7 +294,7 @@ async function askResetUserPassword (req: express.Request, res: express.Response
293 294
294 const verificationString = await Redis.Instance.setResetPasswordVerificationString(user.id) 295 const verificationString = await Redis.Instance.setResetPasswordVerificationString(user.id)
295 const url = CONFIG.WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString 296 const url = CONFIG.WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString
296 await Emailer.Instance.addForgetPasswordEmailJob(user.email, url) 297 await Emailer.Instance.addPasswordResetEmailJob(user.email, url)
297 298
298 return res.status(204).end() 299 return res.status(204).end()
299} 300}
diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts
index 94a2b8732..d5e154869 100644
--- a/server/controllers/api/users/me.ts
+++ b/server/controllers/api/users/me.ts
@@ -167,7 +167,7 @@ async function deleteMe (req: express.Request, res: express.Response) {
167 return res.sendStatus(204) 167 return res.sendStatus(204)
168} 168}
169 169
170async function updateMe (req: express.Request, res: express.Response, next: express.NextFunction) { 170async function updateMe (req: express.Request, res: express.Response) {
171 const body: UserUpdateMe = req.body 171 const body: UserUpdateMe = req.body
172 172
173 const user: UserModel = res.locals.oauth.token.user 173 const user: UserModel = res.locals.oauth.token.user
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 8414ca42c..3b1c2d255 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -36,6 +36,7 @@ import {
36 setDefaultPagination, 36 setDefaultPagination,
37 setDefaultSort, 37 setDefaultSort,
38 videosAddValidator, 38 videosAddValidator,
39 videosCustomGetValidator,
39 videosGetValidator, 40 videosGetValidator,
40 videosRemoveValidator, 41 videosRemoveValidator,
41 videosSortValidator, 42 videosSortValidator,
@@ -123,9 +124,9 @@ videosRouter.get('/:id/description',
123) 124)
124videosRouter.get('/:id', 125videosRouter.get('/:id',
125 optionalAuthenticate, 126 optionalAuthenticate,
126 asyncMiddleware(videosGetValidator), 127 asyncMiddleware(videosCustomGetValidator('only-video-with-rights')),
127 asyncMiddleware(checkVideoFollowConstraints), 128 asyncMiddleware(checkVideoFollowConstraints),
128 getVideo 129 asyncMiddleware(getVideo)
129) 130)
130videosRouter.post('/:id/views', 131videosRouter.post('/:id/views',
131 asyncMiddleware(videosGetValidator), 132 asyncMiddleware(videosGetValidator),
@@ -395,15 +396,17 @@ async function updateVideo (req: express.Request, res: express.Response) {
395 return res.type('json').status(204).end() 396 return res.type('json').status(204).end()
396} 397}
397 398
398function getVideo (req: express.Request, res: express.Response) { 399async function getVideo (req: express.Request, res: express.Response) {
399 const videoInstance = res.locals.video 400 // We need more attributes
401 const userId: number = res.locals.oauth ? res.locals.oauth.token.User.id : null
402 const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id, undefined, userId)
400 403
401 if (videoInstance.isOutdated()) { 404 if (video.isOutdated()) {
402 JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoInstance.url } }) 405 JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })
403 .catch(err => logger.error('Cannot create AP refresher job for video %s.', videoInstance.url, { err })) 406 .catch(err => logger.error('Cannot create AP refresher job for video %s.', video.url, { err }))
404 } 407 }
405 408
406 return res.json(videoInstance.toFormattedDetailsJSON()) 409 return res.json(video.toFormattedDetailsJSON())
407} 410}
408 411
409async function viewVideo (req: express.Request, res: express.Response) { 412async function viewVideo (req: express.Request, res: express.Response) {
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index 4fd58f70c..b21f9da00 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -1,6 +1,6 @@
1import * as cors from 'cors' 1import * as cors from 'cors'
2import * as express from 'express' 2import * as express from 'express'
3import { CONFIG, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers' 3import { CONFIG, HLS_PLAYLIST_DIRECTORY, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers'
4import { VideosPreviewCache } from '../lib/cache' 4import { VideosPreviewCache } from '../lib/cache'
5import { cacheRoute } from '../middlewares/cache' 5import { cacheRoute } from '../middlewares/cache'
6import { asyncMiddleware, videosGetValidator } from '../middlewares' 6import { asyncMiddleware, videosGetValidator } from '../middlewares'
@@ -51,6 +51,13 @@ staticRouter.use(
51 asyncMiddleware(downloadVideoFile) 51 asyncMiddleware(downloadVideoFile)
52) 52)
53 53
54// HLS
55staticRouter.use(
56 STATIC_PATHS.PLAYLISTS.HLS,
57 cors(),
58 express.static(HLS_PLAYLIST_DIRECTORY, { fallthrough: false }) // 404 if the file does not exist
59)
60
54// Thumbnails path for express 61// Thumbnails path for express
55const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR 62const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR
56staticRouter.use( 63staticRouter.use(
diff --git a/server/controllers/tracker.ts b/server/controllers/tracker.ts
index 1deb8c402..8b77d9de7 100644
--- a/server/controllers/tracker.ts
+++ b/server/controllers/tracker.ts
@@ -7,6 +7,7 @@ import { Server as WebSocketServer } from 'ws'
7import { CONFIG, TRACKER_RATE_LIMITS } from '../initializers/constants' 7import { CONFIG, TRACKER_RATE_LIMITS } from '../initializers/constants'
8import { VideoFileModel } from '../models/video/video-file' 8import { VideoFileModel } from '../models/video/video-file'
9import { parse } from 'url' 9import { parse } from 'url'
10import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
10 11
11const TrackerServer = bitTorrentTracker.Server 12const TrackerServer = bitTorrentTracker.Server
12 13
@@ -21,7 +22,7 @@ const trackerServer = new TrackerServer({
21 udp: false, 22 udp: false,
22 ws: false, 23 ws: false,
23 dht: false, 24 dht: false,
24 filter: function (infoHash, params, cb) { 25 filter: async function (infoHash, params, cb) {
25 let ip: string 26 let ip: string
26 27
27 if (params.type === 'ws') { 28 if (params.type === 'ws') {
@@ -32,19 +33,25 @@ const trackerServer = new TrackerServer({
32 33
33 const key = ip + '-' + infoHash 34 const key = ip + '-' + infoHash
34 35
35 peersIps[ip] = peersIps[ip] ? peersIps[ip] + 1 : 1 36 peersIps[ ip ] = peersIps[ ip ] ? peersIps[ ip ] + 1 : 1
36 peersIpInfoHash[key] = peersIpInfoHash[key] ? peersIpInfoHash[key] + 1 : 1 37 peersIpInfoHash[ key ] = peersIpInfoHash[ key ] ? peersIpInfoHash[ key ] + 1 : 1
37 38
38 if (peersIpInfoHash[key] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) { 39 if (peersIpInfoHash[ key ] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) {
39 return cb(new Error(`Too many requests (${peersIpInfoHash[ key ]} of ip ${ip} for torrent ${infoHash}`)) 40 return cb(new Error(`Too many requests (${peersIpInfoHash[ key ]} of ip ${ip} for torrent ${infoHash}`))
40 } 41 }
41 42
42 VideoFileModel.isInfohashExists(infoHash) 43 try {
43 .then(exists => { 44 const videoFileExists = await VideoFileModel.doesInfohashExist(infoHash)
44 if (exists === false) return cb(new Error(`Unknown infoHash ${infoHash}`)) 45 if (videoFileExists === true) return cb()
45 46
46 return cb() 47 const playlistExists = await VideoStreamingPlaylistModel.doesInfohashExist(infoHash)
47 }) 48 if (playlistExists === true) return cb()
49
50 return cb(new Error(`Unknown infoHash ${infoHash}`))
51 } catch (err) {
52 logger.error('Error in tracker filter.', { err })
53 return cb(err)
54 }
48 } 55 }
49}) 56})
50 57
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index f1430055f..eba552524 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -15,7 +15,7 @@ function activityPubContextify <T> (data: T) {
15 'https://w3id.org/security/v1', 15 'https://w3id.org/security/v1',
16 { 16 {
17 RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', 17 RsaSignature2017: 'https://w3id.org/security#RsaSignature2017',
18 pt: 'https://joinpeertube.org/ns', 18 pt: 'https://joinpeertube.org/ns#',
19 sc: 'http://schema.org#', 19 sc: 'http://schema.org#',
20 Hashtag: 'as:Hashtag', 20 Hashtag: 'as:Hashtag',
21 uuid: 'sc:identifier', 21 uuid: 'sc:identifier',
@@ -32,7 +32,8 @@ function activityPubContextify <T> (data: T) {
32 waitTranscoding: 'sc:Boolean', 32 waitTranscoding: 'sc:Boolean',
33 expires: 'sc:expires', 33 expires: 'sc:expires',
34 support: 'sc:Text', 34 support: 'sc:Text',
35 CacheFile: 'pt:CacheFile' 35 CacheFile: 'pt:CacheFile',
36 Infohash: 'pt:Infohash'
36 }, 37 },
37 { 38 {
38 likes: { 39 likes: {
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 3fb824e36..f38b82d97 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -193,10 +193,14 @@ function peertubeTruncate (str: string, maxLength: number) {
193 return truncate(str, options) 193 return truncate(str, options)
194} 194}
195 195
196function sha256 (str: string, encoding: HexBase64Latin1Encoding = 'hex') { 196function sha256 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') {
197 return createHash('sha256').update(str).digest(encoding) 197 return createHash('sha256').update(str).digest(encoding)
198} 198}
199 199
200function sha1 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') {
201 return createHash('sha1').update(str).digest(encoding)
202}
203
200function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> { 204function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> {
201 return function promisified (): Promise<A> { 205 return function promisified (): Promise<A> {
202 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { 206 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
@@ -262,7 +266,9 @@ export {
262 sanitizeHost, 266 sanitizeHost,
263 buildPath, 267 buildPath,
264 peertubeTruncate, 268 peertubeTruncate,
269
265 sha256, 270 sha256,
271 sha1,
266 272
267 promisify0, 273 promisify0,
268 promisify1, 274 promisify1,
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts
index e2bd0c55e..21d5c53ca 100644
--- a/server/helpers/custom-validators/activitypub/cache-file.ts
+++ b/server/helpers/custom-validators/activitypub/cache-file.ts
@@ -8,9 +8,19 @@ function isCacheFileObjectValid (object: CacheFileObject) {
8 object.type === 'CacheFile' && 8 object.type === 'CacheFile' &&
9 isDateValid(object.expires) && 9 isDateValid(object.expires) &&
10 isActivityPubUrlValid(object.object) && 10 isActivityPubUrlValid(object.object) &&
11 isRemoteVideoUrlValid(object.url) 11 (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url))
12} 12}
13 13
14// ---------------------------------------------------------------------------
15
14export { 16export {
15 isCacheFileObjectValid 17 isCacheFileObjectValid
16} 18}
19
20// ---------------------------------------------------------------------------
21
22function isPlaylistRedundancyUrlValid (url: any) {
23 return url.type === 'Link' &&
24 (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
25 isActivityPubUrlValid(url.href)
26}
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index 0f34aab21..ad99c2724 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -1,7 +1,7 @@
1import * as validator from 'validator' 1import * as validator from 'validator'
2import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers' 2import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers'
3import { peertubeTruncate } from '../../core-utils' 3import { peertubeTruncate } from '../../core-utils'
4import { exists, isBooleanValid, isDateValid, isUUIDValid } from '../misc' 4import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc'
5import { 5import {
6 isVideoDurationValid, 6 isVideoDurationValid,
7 isVideoNameValid, 7 isVideoNameValid,
@@ -12,7 +12,6 @@ import {
12} from '../videos' 12} from '../videos'
13import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' 13import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc'
14import { VideoState } from '../../../../shared/models/videos' 14import { VideoState } from '../../../../shared/models/videos'
15import { isVideoAbuseReasonValid } from '../video-abuses'
16 15
17function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { 16function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) {
18 return isBaseActivityValid(activity, 'Update') && 17 return isBaseActivityValid(activity, 'Update') &&
@@ -81,6 +80,11 @@ function isRemoteVideoUrlValid (url: any) {
81 ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType || url.mimeType) !== -1 && 80 ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType || url.mimeType) !== -1 &&
82 validator.isLength(url.href, { min: 5 }) && 81 validator.isLength(url.href, { min: 5 }) &&
83 validator.isInt(url.height + '', { min: 0 }) 82 validator.isInt(url.height + '', { min: 0 })
83 ) ||
84 (
85 (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
86 isActivityPubUrlValid(url.href) &&
87 isArray(url.tag)
84 ) 88 )
85} 89}
86 90
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts
index b6f0ebe6f..76647fea2 100644
--- a/server/helpers/custom-validators/misc.ts
+++ b/server/helpers/custom-validators/misc.ts
@@ -13,6 +13,10 @@ function isNotEmptyIntArray (value: any) {
13 return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0 13 return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0
14} 14}
15 15
16function isArrayOf (value: any, validator: (value: any) => boolean) {
17 return isArray(value) && value.every(v => validator(v))
18}
19
16function isDateValid (value: string) { 20function isDateValid (value: string) {
17 return exists(value) && validator.isISO8601(value) 21 return exists(value) && validator.isISO8601(value)
18} 22}
@@ -82,6 +86,7 @@ function isFileValid (
82 86
83export { 87export {
84 exists, 88 exists,
89 isArrayOf,
85 isNotEmptyIntArray, 90 isNotEmptyIntArray,
86 isArray, 91 isArray,
87 isIdValid, 92 isIdValid,
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index e6f22e6c5..95e256b8f 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -88,8 +88,8 @@ function isVideoFileExtnameValid (value: string) {
88 88
89function isVideoFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { 89function isVideoFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) {
90 const videoFileTypesRegex = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) 90 const videoFileTypesRegex = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
91 .map(m => `(${m})`) 91 .map(m => `(${m})`)
92 .join('|') 92 .join('|')
93 93
94 return isFileValid(files, videoFileTypesRegex, 'videofile', null) 94 return isFileValid(files, videoFileTypesRegex, 'videofile', null)
95} 95}
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index c7296054d..133b1b03b 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -1,5 +1,5 @@
1import * as ffmpeg from 'fluent-ffmpeg' 1import * as ffmpeg from 'fluent-ffmpeg'
2import { join } from 'path' 2import { dirname, join } from 'path'
3import { getTargetBitrate, VideoResolution } from '../../shared/models/videos' 3import { getTargetBitrate, VideoResolution } from '../../shared/models/videos'
4import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' 4import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
5import { processImage } from './image-utils' 5import { processImage } from './image-utils'
@@ -29,19 +29,28 @@ function computeResolutionsToTranscode (videoFileHeight: number) {
29 return resolutionsEnabled 29 return resolutionsEnabled
30} 30}
31 31
32async function getVideoFileResolution (path: string) { 32async function getVideoFileSize (path: string) {
33 const videoStream = await getVideoFileStream(path) 33 const videoStream = await getVideoFileStream(path)
34 34
35 return { 35 return {
36 videoFileResolution: Math.min(videoStream.height, videoStream.width), 36 width: videoStream.width,
37 isPortraitMode: videoStream.height > videoStream.width 37 height: videoStream.height
38 }
39}
40
41async function getVideoFileResolution (path: string) {
42 const size = await getVideoFileSize(path)
43
44 return {
45 videoFileResolution: Math.min(size.height, size.width),
46 isPortraitMode: size.height > size.width
38 } 47 }
39} 48}
40 49
41async function getVideoFileFPS (path: string) { 50async function getVideoFileFPS (path: string) {
42 const videoStream = await getVideoFileStream(path) 51 const videoStream = await getVideoFileStream(path)
43 52
44 for (const key of [ 'r_frame_rate' , 'avg_frame_rate' ]) { 53 for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) {
45 const valuesText: string = videoStream[key] 54 const valuesText: string = videoStream[key]
46 if (!valuesText) continue 55 if (!valuesText) continue
47 56
@@ -110,8 +119,12 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima
110type TranscodeOptions = { 119type TranscodeOptions = {
111 inputPath: string 120 inputPath: string
112 outputPath: string 121 outputPath: string
113 resolution?: VideoResolution 122 resolution: VideoResolution
114 isPortraitMode?: boolean 123 isPortraitMode?: boolean
124
125 hlsPlaylist?: {
126 videoFilename: string
127 }
115} 128}
116 129
117function transcode (options: TranscodeOptions) { 130function transcode (options: TranscodeOptions) {
@@ -150,6 +163,18 @@ function transcode (options: TranscodeOptions) {
150 command = command.withFPS(fps) 163 command = command.withFPS(fps)
151 } 164 }
152 165
166 if (options.hlsPlaylist) {
167 const videoPath = `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}`
168
169 command = command.outputOption('-hls_time 4')
170 .outputOption('-hls_list_size 0')
171 .outputOption('-hls_playlist_type vod')
172 .outputOption('-hls_segment_filename ' + videoPath)
173 .outputOption('-hls_segment_type fmp4')
174 .outputOption('-f hls')
175 .outputOption('-hls_flags single_file')
176 }
177
153 command 178 command
154 .on('error', (err, stdout, stderr) => { 179 .on('error', (err, stdout, stderr) => {
155 logger.error('Error in transcoding job.', { stdout, stderr }) 180 logger.error('Error in transcoding job.', { stdout, stderr })
@@ -166,6 +191,7 @@ function transcode (options: TranscodeOptions) {
166// --------------------------------------------------------------------------- 191// ---------------------------------------------------------------------------
167 192
168export { 193export {
194 getVideoFileSize,
169 getVideoFileResolution, 195 getVideoFileResolution,
170 getDurationFromVideoFile, 196 getDurationFromVideoFile,
171 generateImageFromVideoFile, 197 generateImageFromVideoFile,
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts
index 3fc776f1a..5c6dc5e19 100644
--- a/server/helpers/requests.ts
+++ b/server/helpers/requests.ts
@@ -7,7 +7,7 @@ import { join } from 'path'
7 7
8function doRequest <T> ( 8function doRequest <T> (
9 requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean } 9 requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean }
10): Bluebird<{ response: request.RequestResponse, body: any }> { 10): Bluebird<{ response: request.RequestResponse, body: T }> {
11 if (requestOptions.activityPub === true) { 11 if (requestOptions.activityPub === true) {
12 if (!Array.isArray(requestOptions.headers)) requestOptions.headers = {} 12 if (!Array.isArray(requestOptions.headers)) requestOptions.headers = {}
13 requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER 13 requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index 3c3406e38..cb0e823c5 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -7,7 +7,6 @@ import { join } from 'path'
7import { Instance as ParseTorrent } from 'parse-torrent' 7import { Instance as ParseTorrent } from 'parse-torrent'
8import { remove } from 'fs-extra' 8import { remove } from 'fs-extra'
9import * as memoizee from 'memoizee' 9import * as memoizee from 'memoizee'
10import { isArray } from './custom-validators/misc'
11 10
12function deleteFileAsync (path: string) { 11function deleteFileAsync (path: string) {
13 remove(path) 12 remove(path)
diff --git a/server/helpers/video.ts b/server/helpers/video.ts
index 1bd21467d..c90fe06c7 100644
--- a/server/helpers/video.ts
+++ b/server/helpers/video.ts
@@ -1,10 +1,12 @@
1import { VideoModel } from '../models/video/video' 1import { VideoModel } from '../models/video/video'
2 2
3type VideoFetchType = 'all' | 'only-video' | 'id' | 'none' 3type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none'
4 4
5function fetchVideo (id: number | string, fetchType: VideoFetchType, userId?: number) { 5function fetchVideo (id: number | string, fetchType: VideoFetchType, userId?: number) {
6 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId) 6 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId)
7 7
8 if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id)
9
8 if (fetchType === 'only-video') return VideoModel.load(id) 10 if (fetchType === 'only-video') return VideoModel.load(id)
9 11
10 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) 12 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 7905d9ffa..29fdb263e 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -12,7 +12,7 @@ function checkMissedConfig () {
12 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max', 12 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max',
13 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', 13 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
14 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', 14 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
15 'storage.redundancy', 'storage.tmp', 15 'storage.redundancy', 'storage.tmp', 'storage.playlists',
16 'log.level', 16 'log.level',
17 'user.video_quota', 'user.video_quota_daily', 17 'user.video_quota', 'user.video_quota_daily',
18 'cache.previews.size', 'admin.email', 'contact_form.enabled', 18 'cache.previews.size', 'admin.email', 'contact_form.enabled',
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 93fdd3f03..e5c4c4e63 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -16,7 +16,7 @@ let config: IConfig = require('config')
16 16
17// --------------------------------------------------------------------------- 17// ---------------------------------------------------------------------------
18 18
19const LAST_MIGRATION_VERSION = 325 19const LAST_MIGRATION_VERSION = 330
20 20
21// --------------------------------------------------------------------------- 21// ---------------------------------------------------------------------------
22 22
@@ -192,6 +192,7 @@ const CONFIG = {
192 AVATARS_DIR: buildPath(config.get<string>('storage.avatars')), 192 AVATARS_DIR: buildPath(config.get<string>('storage.avatars')),
193 LOG_DIR: buildPath(config.get<string>('storage.logs')), 193 LOG_DIR: buildPath(config.get<string>('storage.logs')),
194 VIDEOS_DIR: buildPath(config.get<string>('storage.videos')), 194 VIDEOS_DIR: buildPath(config.get<string>('storage.videos')),
195 PLAYLISTS_DIR: buildPath(config.get<string>('storage.playlists')),
195 REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')), 196 REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
196 THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')), 197 THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
197 PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')), 198 PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
@@ -259,6 +260,9 @@ const CONFIG = {
259 get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') }, 260 get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') },
260 get '720p' () { return config.get<boolean>('transcoding.resolutions.720p') }, 261 get '720p' () { return config.get<boolean>('transcoding.resolutions.720p') },
261 get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') } 262 get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') }
263 },
264 HLS: {
265 get ENABLED () { return config.get<boolean>('transcoding.hls.enabled') }
262 } 266 }
263 }, 267 },
264 IMPORT: { 268 IMPORT: {
@@ -590,6 +594,9 @@ const STATIC_PATHS = {
590 TORRENTS: '/static/torrents/', 594 TORRENTS: '/static/torrents/',
591 WEBSEED: '/static/webseed/', 595 WEBSEED: '/static/webseed/',
592 REDUNDANCY: '/static/redundancy/', 596 REDUNDANCY: '/static/redundancy/',
597 PLAYLISTS: {
598 HLS: '/static/playlists/hls'
599 },
593 AVATARS: '/static/avatars/', 600 AVATARS: '/static/avatars/',
594 VIDEO_CAPTIONS: '/static/video-captions/' 601 VIDEO_CAPTIONS: '/static/video-captions/'
595} 602}
@@ -632,6 +639,9 @@ const CACHE = {
632 } 639 }
633} 640}
634 641
642const HLS_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.PLAYLISTS_DIR, 'hls')
643const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
644
635const MEMOIZE_TTL = { 645const MEMOIZE_TTL = {
636 OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours 646 OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
637} 647}
@@ -701,6 +711,8 @@ if (isTestInstance() === true) {
701 CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 711 CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
702 MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1 712 MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1
703 ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms' 713 ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms'
714
715 RATES_LIMIT.LOGIN.MAX = 20
704} 716}
705 717
706updateWebserverUrls() 718updateWebserverUrls()
@@ -709,6 +721,7 @@ updateWebserverUrls()
709 721
710export { 722export {
711 API_VERSION, 723 API_VERSION,
724 HLS_REDUNDANCY_DIRECTORY,
712 AVATARS_SIZE, 725 AVATARS_SIZE,
713 ACCEPT_HEADERS, 726 ACCEPT_HEADERS,
714 BCRYPT_SALT_SIZE, 727 BCRYPT_SALT_SIZE,
@@ -733,6 +746,7 @@ export {
733 PRIVATE_RSA_KEY_SIZE, 746 PRIVATE_RSA_KEY_SIZE,
734 ROUTE_CACHE_LIFETIME, 747 ROUTE_CACHE_LIFETIME,
735 SORTABLE_COLUMNS, 748 SORTABLE_COLUMNS,
749 HLS_PLAYLIST_DIRECTORY,
736 FEEDS, 750 FEEDS,
737 JOB_TTL, 751 JOB_TTL,
738 NSFW_POLICY_TYPES, 752 NSFW_POLICY_TYPES,
@@ -795,7 +809,9 @@ function buildVideoMimetypeExt () {
795 'video/quicktime': '.mov', 809 'video/quicktime': '.mov',
796 'video/x-msvideo': '.avi', 810 'video/x-msvideo': '.avi',
797 'video/x-flv': '.flv', 811 'video/x-flv': '.flv',
798 'video/x-matroska': '.mkv' 812 'video/x-matroska': '.mkv',
813 'application/octet-stream': '.mkv',
814 'video/avi': '.avi'
799 }) 815 })
800 } 816 }
801 817
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 84ad2079b..fe296142d 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -33,6 +33,7 @@ import { AccountBlocklistModel } from '../models/account/account-blocklist'
33import { ServerBlocklistModel } from '../models/server/server-blocklist' 33import { ServerBlocklistModel } from '../models/server/server-blocklist'
34import { UserNotificationModel } from '../models/account/user-notification' 34import { UserNotificationModel } from '../models/account/user-notification'
35import { UserNotificationSettingModel } from '../models/account/user-notification-setting' 35import { UserNotificationSettingModel } from '../models/account/user-notification-setting'
36import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
36 37
37require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string 38require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
38 39
@@ -99,7 +100,8 @@ async function initDatabaseModels (silent: boolean) {
99 AccountBlocklistModel, 100 AccountBlocklistModel,
100 ServerBlocklistModel, 101 ServerBlocklistModel,
101 UserNotificationModel, 102 UserNotificationModel,
102 UserNotificationSettingModel 103 UserNotificationSettingModel,
104 VideoStreamingPlaylistModel
103 ]) 105 ])
104 106
105 // Check extensions exist in the database 107 // Check extensions exist in the database
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index b9a9da183..2b22e16fe 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user'
6import { ApplicationModel } from '../models/application/application' 6import { ApplicationModel } from '../models/application/application'
7import { OAuthClientModel } from '../models/oauth/oauth-client' 7import { OAuthClientModel } from '../models/oauth/oauth-client'
8import { applicationExist, clientsExist, usersExist } from './checker-after-init' 8import { applicationExist, clientsExist, usersExist } from './checker-after-init'
9import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants' 9import { CACHE, CONFIG, HLS_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
10import { sequelizeTypescript } from './database' 10import { sequelizeTypescript } from './database'
11import { remove, ensureDir } from 'fs-extra' 11import { remove, ensureDir } from 'fs-extra'
12 12
@@ -73,6 +73,9 @@ function createDirectoriesIfNotExist () {
73 tasks.push(ensureDir(dir)) 73 tasks.push(ensureDir(dir))
74 } 74 }
75 75
76 // Playlist directories
77 tasks.push(ensureDir(HLS_PLAYLIST_DIRECTORY))
78
76 return Promise.all(tasks) 79 return Promise.all(tasks)
77} 80}
78 81
diff --git a/server/initializers/migrations/0330-video-streaming-playlist.ts b/server/initializers/migrations/0330-video-streaming-playlist.ts
new file mode 100644
index 000000000..c85a762ab
--- /dev/null
+++ b/server/initializers/migrations/0330-video-streaming-playlist.ts
@@ -0,0 +1,51 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction,
5 queryInterface: Sequelize.QueryInterface,
6 sequelize: Sequelize.Sequelize
7}): Promise<void> {
8
9 {
10 const query = `
11 CREATE TABLE IF NOT EXISTS "videoStreamingPlaylist"
12(
13 "id" SERIAL,
14 "type" INTEGER NOT NULL,
15 "playlistUrl" VARCHAR(2000) NOT NULL,
16 "p2pMediaLoaderInfohashes" VARCHAR(255)[] NOT NULL,
17 "segmentsSha256Url" VARCHAR(255) NOT NULL,
18 "videoId" INTEGER NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
19 "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
20 "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
21 PRIMARY KEY ("id")
22);`
23 await utils.sequelize.query(query)
24 }
25
26 {
27 const data = {
28 type: Sequelize.INTEGER,
29 allowNull: true,
30 defaultValue: null
31 }
32
33 await utils.queryInterface.changeColumn('videoRedundancy', 'videoFileId', data)
34 }
35
36 {
37 const query = 'ALTER TABLE "videoRedundancy" ADD COLUMN "videoStreamingPlaylistId" INTEGER NULL ' +
38 'REFERENCES "videoStreamingPlaylist" ("id") ON DELETE CASCADE ON UPDATE CASCADE'
39
40 await utils.sequelize.query(query)
41 }
42}
43
44function down (options) {
45 throw new Error('Not implemented.')
46}
47
48export {
49 up,
50 down
51}
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 8215840da..a3f379b76 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -355,10 +355,10 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
355 355
356 logger.info('Fetching remote actor %s.', actorUrl) 356 logger.info('Fetching remote actor %s.', actorUrl)
357 357
358 const requestResult = await doRequest(options) 358 const requestResult = await doRequest<ActivityPubActor>(options)
359 normalizeActor(requestResult.body) 359 normalizeActor(requestResult.body)
360 360
361 const actorJSON: ActivityPubActor = requestResult.body 361 const actorJSON = requestResult.body
362 if (isActorObjectValid(actorJSON) === false) { 362 if (isActorObjectValid(actorJSON) === false) {
363 logger.debug('Remote actor JSON is not valid.', { actorJSON }) 363 logger.debug('Remote actor JSON is not valid.', { actorJSON })
364 return { result: undefined, statusCode: requestResult.response.statusCode } 364 return { result: undefined, statusCode: requestResult.response.statusCode }
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index f6f068b45..9a40414bb 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -1,11 +1,28 @@
1import { CacheFileObject } from '../../../shared/index' 1import { ActivityPlaylistUrlObject, ActivityVideoUrlObject, CacheFileObject } from '../../../shared/index'
2import { VideoModel } from '../../models/video/video' 2import { VideoModel } from '../../models/video/video'
3import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 3import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
4import { Transaction } from 'sequelize' 4import { Transaction } from 'sequelize'
5import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
5 6
6function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { 7function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
7 const url = cacheFileObject.url
8 8
9 if (cacheFileObject.url.mediaType === 'application/x-mpegURL') {
10 const url = cacheFileObject.url
11
12 const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS)
13 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
14
15 return {
16 expiresOn: new Date(cacheFileObject.expires),
17 url: cacheFileObject.id,
18 fileUrl: url.href,
19 strategy: null,
20 videoStreamingPlaylistId: playlist.id,
21 actorId: byActor.id
22 }
23 }
24
25 const url = cacheFileObject.url
9 const videoFile = video.VideoFiles.find(f => { 26 const videoFile = video.VideoFiles.find(f => {
10 return f.resolution === url.height && f.fps === url.fps 27 return f.resolution === url.height && f.fps === url.fps
11 }) 28 })
@@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
15 return { 32 return {
16 expiresOn: new Date(cacheFileObject.expires), 33 expiresOn: new Date(cacheFileObject.expires),
17 url: cacheFileObject.id, 34 url: cacheFileObject.id,
18 fileUrl: cacheFileObject.url.href, 35 fileUrl: url.href,
19 strategy: null, 36 strategy: null,
20 videoFileId: videoFile.id, 37 videoFileId: videoFile.id,
21 actorId: byActor.id 38 actorId: byActor.id
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index 73e667ad4..ef20e404c 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -23,17 +23,14 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) {
23 return broadcastToFollowers(createActivity, byActor, [ byActor ], t) 23 return broadcastToFollowers(createActivity, byActor, [ byActor ], t)
24} 24}
25 25
26async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { 26async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) {
27 logger.info('Creating job to send file cache of %s.', fileRedundancy.url) 27 logger.info('Creating job to send file cache of %s.', fileRedundancy.url)
28 28
29 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
30 const redundancyObject = fileRedundancy.toActivityPubObject()
31
32 return sendVideoRelatedCreateActivity({ 29 return sendVideoRelatedCreateActivity({
33 byActor, 30 byActor,
34 video, 31 video,
35 url: fileRedundancy.url, 32 url: fileRedundancy.url,
36 object: redundancyObject 33 object: fileRedundancy.toActivityPubObject()
37 }) 34 })
38} 35}
39 36
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts
index eb18a6cb6..ecbf605d6 100644
--- a/server/lib/activitypub/send/send-undo.ts
+++ b/server/lib/activitypub/send/send-undo.ts
@@ -73,7 +73,8 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans
73async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { 73async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
74 logger.info('Creating job to undo cache file %s.', redundancyModel.url) 74 logger.info('Creating job to undo cache file %s.', redundancyModel.url)
75 75
76 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 76 const videoId = redundancyModel.getVideo().id
77 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
77 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) 78 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())
78 79
79 return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) 80 return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t })
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index a68f03edf..839f66470 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -61,7 +61,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { 61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
62 logger.info('Creating job to update cache file %s.', redundancyModel.url) 62 logger.info('Creating job to update cache file %s.', redundancyModel.url)
63 63
64 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 64 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id)
65 65
66 const activityBuilder = (audience: ActivityAudience) => { 66 const activityBuilder = (audience: ActivityAudience) => {
67 const redundancyObject = redundancyModel.toActivityPubObject() 67 const redundancyObject = redundancyModel.toActivityPubObject()
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts
index 38f15448c..4229fe094 100644
--- a/server/lib/activitypub/url.ts
+++ b/server/lib/activitypub/url.ts
@@ -5,6 +5,8 @@ import { VideoModel } from '../../models/video/video'
5import { VideoAbuseModel } from '../../models/video/video-abuse' 5import { VideoAbuseModel } from '../../models/video/video-abuse'
6import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
7import { VideoFileModel } from '../../models/video/video-file' 7import { VideoFileModel } from '../../models/video/video-file'
8import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
9import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
8 10
9function getVideoActivityPubUrl (video: VideoModel) { 11function getVideoActivityPubUrl (video: VideoModel) {
10 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid 12 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid
@@ -16,6 +18,10 @@ function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) {
16 return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` 18 return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}`
17} 19}
18 20
21function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) {
22 return `${CONFIG.WEBSERVER.URL}/redundancy/video-playlists/${playlist.getStringType()}/${video.uuid}`
23}
24
19function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { 25function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) {
20 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id 26 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id
21} 27}
@@ -92,6 +98,7 @@ function getUndoActivityPubUrl (originalUrl: string) {
92 98
93export { 99export {
94 getVideoActivityPubUrl, 100 getVideoActivityPubUrl,
101 getVideoCacheStreamingPlaylistActivityPubUrl,
95 getVideoChannelActivityPubUrl, 102 getVideoChannelActivityPubUrl,
96 getAccountActivityPubUrl, 103 getAccountActivityPubUrl,
97 getVideoAbuseActivityPubUrl, 104 getVideoAbuseActivityPubUrl,
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index e1e523499..edd01234f 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -2,7 +2,14 @@ import * as Bluebird from 'bluebird'
2import * as sequelize from 'sequelize' 2import * as sequelize from 'sequelize'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import * as request from 'request' 4import * as request from 'request'
5import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' 5import {
6 ActivityIconObject,
7 ActivityPlaylistSegmentHashesObject,
8 ActivityPlaylistUrlObject,
9 ActivityUrlObject,
10 ActivityVideoUrlObject,
11 VideoState
12} from '../../../shared/index'
6import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 13import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
7import { VideoPrivacy } from '../../../shared/models/videos' 14import { VideoPrivacy } from '../../../shared/models/videos'
8import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' 15import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
@@ -30,6 +37,9 @@ import { AccountModel } from '../../models/account/account'
30import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' 37import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
31import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' 38import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
32import { Notifier } from '../notifier' 39import { Notifier } from '../notifier'
40import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
41import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
42import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
33 43
34async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 44async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
35 // If the video is not private and published, we federate it 45 // If the video is not private and published, we federate it
@@ -264,6 +274,25 @@ async function updateVideoFromAP (options: {
264 } 274 }
265 275
266 { 276 {
277 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject)
278 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
279
280 // Remove video files that do not exist anymore
281 const destroyTasks = options.video.VideoStreamingPlaylists
282 .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f)))
283 .map(f => f.destroy(sequelizeOptions))
284 await Promise.all(destroyTasks)
285
286 // Update or add other one
287 const upsertTasks = streamingPlaylistAttributes.map(a => {
288 return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t })
289 .then(([ streamingPlaylist ]) => streamingPlaylist)
290 })
291
292 options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks)
293 }
294
295 {
267 // Update Tags 296 // Update Tags
268 const tags = options.videoObject.tag.map(tag => tag.name) 297 const tags = options.videoObject.tag.map(tag => tag.name)
269 const tagInstances = await TagModel.findOrCreateTags(tags, t) 298 const tagInstances = await TagModel.findOrCreateTags(tags, t)
@@ -367,13 +396,25 @@ export {
367 396
368// --------------------------------------------------------------------------- 397// ---------------------------------------------------------------------------
369 398
370function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { 399function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
371 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) 400 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
372 401
373 const urlMediaType = url.mediaType || url.mimeType 402 const urlMediaType = url.mediaType || url.mimeType
374 return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') 403 return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
375} 404}
376 405
406function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
407 const urlMediaType = url.mediaType || url.mimeType
408
409 return urlMediaType === 'application/x-mpegURL'
410}
411
412function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
413 const urlMediaType = tag.mediaType || tag.mimeType
414
415 return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json'
416}
417
377async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { 418async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
378 logger.debug('Adding remote video %s.', videoObject.id) 419 logger.debug('Adding remote video %s.', videoObject.id)
379 420
@@ -394,8 +435,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
394 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) 435 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
395 await Promise.all(videoFilePromises) 436 await Promise.all(videoFilePromises)
396 437
438 const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject)
439 const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
440 await Promise.all(playlistPromises)
441
397 // Process tags 442 // Process tags
398 const tags = videoObject.tag.map(t => t.name) 443 const tags = videoObject.tag
444 .filter(t => t.type === 'Hashtag')
445 .map(t => t.name)
399 const tagInstances = await TagModel.findOrCreateTags(tags, t) 446 const tagInstances = await TagModel.findOrCreateTags(tags, t)
400 await videoCreated.$set('Tags', tagInstances, sequelizeOptions) 447 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
401 448
@@ -473,13 +520,13 @@ async function videoActivityObjectToDBAttributes (
473} 520}
474 521
475function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { 522function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
476 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] 523 const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
477 524
478 if (fileUrls.length === 0) { 525 if (fileUrls.length === 0) {
479 throw new Error('Cannot find video files for ' + video.url) 526 throw new Error('Cannot find video files for ' + video.url)
480 } 527 }
481 528
482 const attributes: VideoFileModel[] = [] 529 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
483 for (const fileUrl of fileUrls) { 530 for (const fileUrl of fileUrls) {
484 // Fetch associated magnet uri 531 // Fetch associated magnet uri
485 const magnet = videoObject.url.find(u => { 532 const magnet = videoObject.url.find(u => {
@@ -502,7 +549,45 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid
502 size: fileUrl.size, 549 size: fileUrl.size,
503 videoId: video.id, 550 videoId: video.id,
504 fps: fileUrl.fps || -1 551 fps: fileUrl.fps || -1
505 } as VideoFileModel 552 }
553
554 attributes.push(attribute)
555 }
556
557 return attributes
558}
559
560function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
561 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
562 if (playlistUrls.length === 0) return []
563
564 const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
565 for (const playlistUrlObject of playlistUrls) {
566 const p2pMediaLoaderInfohashes = playlistUrlObject.tag
567 .filter(t => t.type === 'Infohash')
568 .map(t => t.name)
569 if (p2pMediaLoaderInfohashes.length === 0) {
570 logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject })
571 continue
572 }
573
574 const segmentsSha256UrlObject = playlistUrlObject.tag
575 .find(t => {
576 return isAPPlaylistSegmentHashesUrlObject(t)
577 }) as ActivityPlaylistSegmentHashesObject
578 if (!segmentsSha256UrlObject) {
579 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
580 continue
581 }
582
583 const attribute = {
584 type: VideoStreamingPlaylistType.HLS,
585 playlistUrl: playlistUrlObject.href,
586 segmentsSha256Url: segmentsSha256UrlObject.href,
587 p2pMediaLoaderInfohashes,
588 videoId: video.id
589 }
590
506 attributes.push(attribute) 591 attributes.push(attribute)
507 } 592 }
508 593
diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts
index f384a254e..672414cc0 100644
--- a/server/lib/emailer.ts
+++ b/server/lib/emailer.ts
@@ -296,9 +296,9 @@ class Emailer {
296 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) 296 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
297 } 297 }
298 298
299 addForgetPasswordEmailJob (to: string, resetPasswordUrl: string) { 299 addPasswordResetEmailJob (to: string, resetPasswordUrl: string) {
300 const text = `Hi dear user,\n\n` + 300 const text = `Hi dear user,\n\n` +
301 `It seems you forgot your password on ${CONFIG.WEBSERVER.HOST}! ` + 301 `A reset password procedure for your account ${to} has been requested on ${CONFIG.WEBSERVER.HOST} ` +
302 `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + 302 `Please follow this link to reset it: ${resetPasswordUrl}\n\n` +
303 `If you are not the person who initiated this request, please ignore this email.\n\n` + 303 `If you are not the person who initiated this request, please ignore this email.\n\n` +
304 `Cheers,\n` + 304 `Cheers,\n` +
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
new file mode 100644
index 000000000..3575981f4
--- /dev/null
+++ b/server/lib/hls.ts
@@ -0,0 +1,164 @@
1import { VideoModel } from '../models/video/video'
2import { basename, join, dirname } from 'path'
3import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers'
4import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
5import { getVideoFileSize } from '../helpers/ffmpeg-utils'
6import { sha256 } from '../helpers/core-utils'
7import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
8import { logger } from '../helpers/logger'
9import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
10import { generateRandomString } from '../helpers/utils'
11import { flatten, uniq } from 'lodash'
12
13async function updateMasterHLSPlaylist (video: VideoModel) {
14 const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
15 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
16 const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
17
18 for (const file of video.VideoFiles) {
19 // If we did not generated a playlist for this resolution, skip
20 const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
21 if (await pathExists(filePlaylistPath) === false) continue
22
23 const videoFilePath = video.getVideoFilePath(file)
24
25 const size = await getVideoFileSize(videoFilePath)
26
27 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
28 const resolution = `RESOLUTION=${size.width}x${size.height}`
29
30 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
31 if (file.fps) line += ',FRAME-RATE=' + file.fps
32
33 masterPlaylists.push(line)
34 masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
35 }
36
37 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
38}
39
40async function updateSha256Segments (video: VideoModel) {
41 const json: { [filename: string]: { [range: string]: string } } = {}
42
43 const playlistDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
44
45 // For all the resolutions available for this video
46 for (const file of video.VideoFiles) {
47 const rangeHashes: { [range: string]: string } = {}
48
49 const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution))
50 const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
51
52 // Maybe the playlist is not generated for this resolution yet
53 if (!await pathExists(playlistPath)) continue
54
55 const playlistContent = await readFile(playlistPath)
56 const ranges = getRangesFromPlaylist(playlistContent.toString())
57
58 const fd = await open(videoPath, 'r')
59 for (const range of ranges) {
60 const buf = Buffer.alloc(range.length)
61 await read(fd, buf, 0, range.length, range.offset)
62
63 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
64 }
65 await close(fd)
66
67 const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)
68 json[videoFilename] = rangeHashes
69 }
70
71 const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
72 await outputJSON(outputPath, json)
73}
74
75function getRangesFromPlaylist (playlistContent: string) {
76 const ranges: { offset: number, length: number }[] = []
77 const lines = playlistContent.split('\n')
78 const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
79
80 for (const line of lines) {
81 const captured = regex.exec(line)
82
83 if (captured) {
84 ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })
85 }
86 }
87
88 return ranges
89}
90
91function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
92 let timer
93
94 logger.info('Importing HLS playlist %s', playlistUrl)
95
96 return new Promise<string>(async (res, rej) => {
97 const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
98
99 await ensureDir(tmpDirectory)
100
101 timer = setTimeout(() => {
102 deleteTmpDirectory(tmpDirectory)
103
104 return rej(new Error('HLS download timeout.'))
105 }, timeout)
106
107 try {
108 // Fetch master playlist
109 const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
110
111 const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
112 const fileUrls = uniq(flatten(await Promise.all(subRequests)))
113
114 logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
115
116 for (const fileUrl of fileUrls) {
117 const destPath = join(tmpDirectory, basename(fileUrl))
118
119 await doRequestAndSaveToFile({ uri: fileUrl }, destPath)
120 }
121
122 clearTimeout(timer)
123
124 await move(tmpDirectory, destinationDir, { overwrite: true })
125
126 return res()
127 } catch (err) {
128 deleteTmpDirectory(tmpDirectory)
129
130 return rej(err)
131 }
132 })
133
134 function deleteTmpDirectory (directory: string) {
135 remove(directory)
136 .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
137 }
138
139 async function fetchUniqUrls (playlistUrl: string) {
140 const { body } = await doRequest<string>({ uri: playlistUrl })
141
142 if (!body) return []
143
144 const urls = body.split('\n')
145 .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
146 .map(url => {
147 if (url.startsWith('http://') || url.startsWith('https://')) return url
148
149 return `${dirname(playlistUrl)}/${url}`
150 })
151
152 return uniq(urls)
153 }
154}
155
156// ---------------------------------------------------------------------------
157
158export {
159 updateMasterHLSPlaylist,
160 updateSha256Segments,
161 downloadPlaylistSegments
162}
163
164// ---------------------------------------------------------------------------
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts
index 593e43cc5..04983155c 100644
--- a/server/lib/job-queue/handlers/video-file.ts
+++ b/server/lib/job-queue/handlers/video-file.ts
@@ -5,17 +5,18 @@ import { VideoModel } from '../../../models/video/video'
5import { JobQueue } from '../job-queue' 5import { JobQueue } from '../job-queue'
6import { federateVideoIfNeeded } from '../../activitypub' 6import { federateVideoIfNeeded } from '../../activitypub'
7import { retryTransactionWrapper } from '../../../helpers/database-utils' 7import { retryTransactionWrapper } from '../../../helpers/database-utils'
8import { sequelizeTypescript } from '../../../initializers' 8import { sequelizeTypescript, CONFIG } from '../../../initializers'
9import * as Bluebird from 'bluebird' 9import * as Bluebird from 'bluebird'
10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' 10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
11import { importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' 11import { generateHlsPlaylist, importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding'
12import { Notifier } from '../../notifier' 12import { Notifier } from '../../notifier'
13 13
14export type VideoFilePayload = { 14export type VideoFilePayload = {
15 videoUUID: string 15 videoUUID: string
16 isNewVideo?: boolean
17 resolution?: VideoResolution 16 resolution?: VideoResolution
17 isNewVideo?: boolean
18 isPortraitMode?: boolean 18 isPortraitMode?: boolean
19 generateHlsPlaylist?: boolean
19} 20}
20 21
21export type VideoFileImportPayload = { 22export type VideoFileImportPayload = {
@@ -51,21 +52,38 @@ async function processVideoFile (job: Bull.Job) {
51 return undefined 52 return undefined
52 } 53 }
53 54
54 // Transcoding in other resolution 55 if (payload.generateHlsPlaylist) {
55 if (payload.resolution) { 56 await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false)
57
58 await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video)
59 } else if (payload.resolution) { // Transcoding in other resolution
56 await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) 60 await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false)
57 61
58 await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) 62 await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video, payload)
59 } else { 63 } else {
60 await optimizeVideofile(video) 64 await optimizeVideofile(video)
61 65
62 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) 66 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload)
63 } 67 }
64 68
65 return video 69 return video
66} 70}
67 71
68async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { 72async function onHlsPlaylistGenerationSuccess (video: VideoModel) {
73 if (video === undefined) return undefined
74
75 await sequelizeTypescript.transaction(async t => {
76 // Maybe the video changed in database, refresh it
77 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
78 // Video does not exist anymore
79 if (!videoDatabase) return undefined
80
81 // If the video was not published, we consider it is a new one for other instances
82 await federateVideoIfNeeded(videoDatabase, false, t)
83 })
84}
85
86async function onVideoFileTranscoderOrImportSuccess (video: VideoModel, payload?: VideoFilePayload) {
69 if (video === undefined) return undefined 87 if (video === undefined) return undefined
70 88
71 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { 89 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
@@ -91,13 +109,16 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) {
91 return { videoDatabase, videoPublished } 109 return { videoDatabase, videoPublished }
92 }) 110 })
93 111
94 if (videoPublished) { 112 // don't notify prior to scheduled video update
113 if (videoPublished && !videoDatabase.ScheduleVideoUpdate) {
95 Notifier.Instance.notifyOnNewVideo(videoDatabase) 114 Notifier.Instance.notifyOnNewVideo(videoDatabase)
96 Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) 115 Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase)
97 } 116 }
117
118 await createHlsJobIfEnabled(payload)
98} 119}
99 120
100async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) { 121async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoFilePayload) {
101 if (videoArg === undefined) return undefined 122 if (videoArg === undefined) return undefined
102 123
103 // Outside the transaction (IO on disk) 124 // Outside the transaction (IO on disk)
@@ -144,13 +165,18 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo
144 logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) 165 logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy })
145 } 166 }
146 167
147 await federateVideoIfNeeded(videoDatabase, isNewVideo, t) 168 await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t)
148 169
149 return { videoDatabase, videoPublished } 170 return { videoDatabase, videoPublished }
150 }) 171 })
151 172
152 if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) 173 // don't notify prior to scheduled video update
153 if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) 174 if (!videoDatabase.ScheduleVideoUpdate) {
175 if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase)
176 if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase)
177 }
178
179 await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution }))
154} 180}
155 181
156// --------------------------------------------------------------------------- 182// ---------------------------------------------------------------------------
@@ -159,3 +185,20 @@ export {
159 processVideoFile, 185 processVideoFile,
160 processVideoFileImport 186 processVideoFileImport
161} 187}
188
189// ---------------------------------------------------------------------------
190
191function createHlsJobIfEnabled (payload?: VideoFilePayload) {
192 // Generate HLS playlist?
193 if (payload && CONFIG.TRANSCODING.HLS.ENABLED) {
194 const hlsTranscodingPayload = {
195 videoUUID: payload.videoUUID,
196 resolution: payload.resolution,
197 isPortraitMode: payload.isPortraitMode,
198
199 generateHlsPlaylist: true
200 }
201
202 return JobQueue.Instance.createJob({ type: 'video-file', payload: hlsTranscodingPayload })
203 }
204}
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index f643ee226..1a48f2bd0 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,5 +1,5 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' 2import { CONFIG, HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideosRedundancy } from '../../../shared/models/redundancy' 4import { VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
@@ -9,9 +9,19 @@ import { join } from 'path'
9import { move } from 'fs-extra' 9import { move } from 'fs-extra'
10import { getServerActor } from '../../helpers/utils' 10import { getServerActor } from '../../helpers/utils'
11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
12import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' 12import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
13import { removeVideoRedundancy } from '../redundancy' 13import { removeVideoRedundancy } from '../redundancy'
14import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' 14import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
16import { VideoModel } from '../../models/video/video'
17import { downloadPlaylistSegments } from '../hls'
18
19type CandidateToDuplicate = {
20 redundancy: VideosRedundancy,
21 video: VideoModel,
22 files: VideoFileModel[],
23 streamingPlaylists: VideoStreamingPlaylistModel[]
24}
15 25
16export class VideosRedundancyScheduler extends AbstractScheduler { 26export class VideosRedundancyScheduler extends AbstractScheduler {
17 27
@@ -24,28 +34,32 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
24 } 34 }
25 35
26 protected async internalExecute () { 36 protected async internalExecute () {
27 for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { 37 for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
28 logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) 38 logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy)
29 39
30 try { 40 try {
31 const videoToDuplicate = await this.findVideoToDuplicate(obj) 41 const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig)
32 if (!videoToDuplicate) continue 42 if (!videoToDuplicate) continue
33 43
34 const videoFiles = videoToDuplicate.VideoFiles 44 const candidateToDuplicate = {
35 videoFiles.forEach(f => f.Video = videoToDuplicate) 45 video: videoToDuplicate,
46 redundancy: redundancyConfig,
47 files: videoToDuplicate.VideoFiles,
48 streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
49 }
36 50
37 await this.purgeCacheIfNeeded(obj, videoFiles) 51 await this.purgeCacheIfNeeded(candidateToDuplicate)
38 52
39 if (await this.isTooHeavy(obj, videoFiles)) { 53 if (await this.isTooHeavy(candidateToDuplicate)) {
40 logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) 54 logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
41 continue 55 continue
42 } 56 }
43 57
44 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) 58 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy)
45 59
46 await this.createVideoRedundancy(obj, videoFiles) 60 await this.createVideoRedundancies(candidateToDuplicate)
47 } catch (err) { 61 } catch (err) {
48 logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) 62 logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err })
49 } 63 }
50 } 64 }
51 65
@@ -63,25 +77,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
63 77
64 for (const redundancyModel of expired) { 78 for (const redundancyModel of expired) {
65 try { 79 try {
66 await this.extendsOrDeleteRedundancy(redundancyModel) 80 const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
81 const candidate = {
82 redundancy: redundancyConfig,
83 video: null,
84 files: [],
85 streamingPlaylists: []
86 }
87
88 // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it
89 if (!redundancyConfig || await this.isTooHeavy(candidate)) {
90 logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy)
91 await removeVideoRedundancy(redundancyModel)
92 } else {
93 await this.extendsRedundancy(redundancyModel)
94 }
67 } catch (err) { 95 } catch (err) {
68 logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel)) 96 logger.error(
97 'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel),
98 { err }
99 )
69 } 100 }
70 } 101 }
71 } 102 }
72 103
73 private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) { 104 private async extendsRedundancy (redundancyModel: VideoRedundancyModel) {
74 // Refresh the video, maybe it was deleted
75 const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url)
76
77 if (!video) {
78 logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url)
79
80 await redundancyModel.destroy()
81 return
82 }
83
84 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) 105 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
106 // Redundancy strategy disabled, remove our redundancy instead of extending expiration
107 if (!redundancy) await removeVideoRedundancy(redundancyModel)
108
85 await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) 109 await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
86 } 110 }
87 111
@@ -112,49 +136,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
112 } 136 }
113 } 137 }
114 138
115 private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 139 private async createVideoRedundancies (data: CandidateToDuplicate) {
116 const serverActor = await getServerActor() 140 const video = await this.loadAndRefreshVideo(data.video.url)
141
142 if (!video) {
143 logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url)
117 144
118 for (const file of filesToDuplicate) { 145 return
119 const video = await this.loadAndRefreshVideo(file.Video.url) 146 }
120 147
148 for (const file of data.files) {
121 const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) 149 const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
122 if (existingRedundancy) { 150 if (existingRedundancy) {
123 await this.extendsOrDeleteRedundancy(existingRedundancy) 151 await this.extendsRedundancy(existingRedundancy)
124 152
125 continue 153 continue
126 } 154 }
127 155
128 if (!video) { 156 await this.createVideoFileRedundancy(data.redundancy, video, file)
129 logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url) 157 }
158
159 for (const streamingPlaylist of data.streamingPlaylists) {
160 const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id)
161 if (existingRedundancy) {
162 await this.extendsRedundancy(existingRedundancy)
130 163
131 continue 164 continue
132 } 165 }
133 166
134 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) 167 await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist)
168 }
169 }
135 170
136 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 171 private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) {
137 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) 172 file.Video = video
138 173
139 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) 174 const serverActor = await getServerActor()
140 175
141 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) 176 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
142 await move(tmpPath, destPath)
143 177
144 const createdModel = await VideoRedundancyModel.create({ 178 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
145 expiresOn: this.buildNewExpiration(redundancy.minLifetime), 179 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
146 url: getVideoCacheFileActivityPubUrl(file),
147 fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
148 strategy: redundancy.strategy,
149 videoFileId: file.id,
150 actorId: serverActor.id
151 })
152 createdModel.VideoFile = file
153 180
154 await sendCreateCacheFile(serverActor, createdModel) 181 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
155 182
156 logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) 183 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file))
157 } 184 await move(tmpPath, destPath)
185
186 const createdModel = await VideoRedundancyModel.create({
187 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
188 url: getVideoCacheFileActivityPubUrl(file),
189 fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
190 strategy: redundancy.strategy,
191 videoFileId: file.id,
192 actorId: serverActor.id
193 })
194
195 createdModel.VideoFile = file
196
197 await sendCreateCacheFile(serverActor, video, createdModel)
198
199 logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url)
200 }
201
202 private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) {
203 playlist.Video = video
204
205 const serverActor = await getServerActor()
206
207 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy)
208
209 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
210 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
211
212 const createdModel = await VideoRedundancyModel.create({
213 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
214 url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
215 fileUrl: playlist.getVideoRedundancyUrl(CONFIG.WEBSERVER.URL),
216 strategy: redundancy.strategy,
217 videoStreamingPlaylistId: playlist.id,
218 actorId: serverActor.id
219 })
220
221 createdModel.VideoStreamingPlaylist = playlist
222
223 await sendCreateCacheFile(serverActor, video, createdModel)
224
225 logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url)
158 } 226 }
159 227
160 private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { 228 private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) {
@@ -168,8 +236,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
168 await sendUpdateCacheFile(serverActor, redundancy) 236 await sendUpdateCacheFile(serverActor, redundancy)
169 } 237 }
170 238
171 private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 239 private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) {
172 while (this.isTooHeavy(redundancy, filesToDuplicate)) { 240 while (this.isTooHeavy(candidateToDuplicate)) {
241 const redundancy = candidateToDuplicate.redundancy
173 const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) 242 const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime)
174 if (!toDelete) return 243 if (!toDelete) return
175 244
@@ -177,11 +246,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
177 } 246 }
178 } 247 }
179 248
180 private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 249 private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) {
181 const maxSize = redundancy.size 250 const maxSize = candidateToDuplicate.redundancy.size
182 251
183 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) 252 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy)
184 const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate) 253 const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists)
185 254
186 return totalWillDuplicate > maxSize 255 return totalWillDuplicate > maxSize
187 } 256 }
@@ -191,13 +260,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
191 } 260 }
192 261
193 private buildEntryLogId (object: VideoRedundancyModel) { 262 private buildEntryLogId (object: VideoRedundancyModel) {
194 return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` 263 if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
264
265 return `${object.VideoStreamingPlaylist.playlistUrl}`
195 } 266 }
196 267
197 private getTotalFileSizes (files: VideoFileModel[]) { 268 private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) {
198 const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size 269 const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size
199 270
200 return files.reduce(fileReducer, 0) 271 return files.reduce(fileReducer, 0) * playlists.length
201 } 272 }
202 273
203 private async loadAndRefreshVideo (videoUrl: string) { 274 private async loadAndRefreshVideo (videoUrl: string) {
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
index 4460f46e4..086b860a2 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -1,11 +1,14 @@
1import { CONFIG } from '../initializers' 1import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers'
2import { extname, join } from 'path' 2import { extname, join } from 'path'
3import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' 3import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils'
4import { copy, remove, move, stat } from 'fs-extra' 4import { copy, ensureDir, move, remove, stat } from 'fs-extra'
5import { logger } from '../helpers/logger' 5import { logger } from '../helpers/logger'
6import { VideoResolution } from '../../shared/models/videos' 6import { VideoResolution } from '../../shared/models/videos'
7import { VideoFileModel } from '../models/video/video-file' 7import { VideoFileModel } from '../models/video/video-file'
8import { VideoModel } from '../models/video/video' 8import { VideoModel } from '../models/video/video'
9import { updateMasterHLSPlaylist, updateSha256Segments } from './hls'
10import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
11import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type'
9 12
10async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { 13async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) {
11 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR 14 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
@@ -17,7 +20,8 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi
17 20
18 const transcodeOptions = { 21 const transcodeOptions = {
19 inputPath: videoInputPath, 22 inputPath: videoInputPath,
20 outputPath: videoTranscodedPath 23 outputPath: videoTranscodedPath,
24 resolution: inputVideoFile.resolution
21 } 25 }
22 26
23 // Could be very long! 27 // Could be very long!
@@ -47,7 +51,7 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi
47 } 51 }
48} 52}
49 53
50async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { 54async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) {
51 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR 55 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
52 const extname = '.mp4' 56 const extname = '.mp4'
53 57
@@ -60,13 +64,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
60 size: 0, 64 size: 0,
61 videoId: video.id 65 videoId: video.id
62 }) 66 })
63 const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) 67 const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile))
64 68
65 const transcodeOptions = { 69 const transcodeOptions = {
66 inputPath: videoInputPath, 70 inputPath: videoInputPath,
67 outputPath: videoOutputPath, 71 outputPath: videoOutputPath,
68 resolution, 72 resolution,
69 isPortraitMode 73 isPortraitMode: isPortrait
70 } 74 }
71 75
72 await transcode(transcodeOptions) 76 await transcode(transcodeOptions)
@@ -84,6 +88,41 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
84 video.VideoFiles.push(newVideoFile) 88 video.VideoFiles.push(newVideoFile)
85} 89}
86 90
91async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) {
92 const baseHlsDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
93 await ensureDir(join(HLS_PLAYLIST_DIRECTORY, video.uuid))
94
95 const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile()))
96 const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
97
98 const transcodeOptions = {
99 inputPath: videoInputPath,
100 outputPath,
101 resolution,
102 isPortraitMode,
103
104 hlsPlaylist: {
105 videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution)
106 }
107 }
108
109 await transcode(transcodeOptions)
110
111 await updateMasterHLSPlaylist(video)
112 await updateSha256Segments(video)
113
114 const playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
115
116 await VideoStreamingPlaylistModel.upsert({
117 videoId: video.id,
118 playlistUrl,
119 segmentsSha256Url: CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid),
120 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles),
121
122 type: VideoStreamingPlaylistType.HLS
123 })
124}
125
87async function importVideoFile (video: VideoModel, inputFilePath: string) { 126async function importVideoFile (video: VideoModel, inputFilePath: string) {
88 const { videoFileResolution } = await getVideoFileResolution(inputFilePath) 127 const { videoFileResolution } = await getVideoFileResolution(inputFilePath)
89 const { size } = await stat(inputFilePath) 128 const { size } = await stat(inputFilePath)
@@ -125,6 +164,7 @@ async function importVideoFile (video: VideoModel, inputFilePath: string) {
125} 164}
126 165
127export { 166export {
167 generateHlsPlaylist,
128 optimizeVideofile, 168 optimizeVideofile,
129 transcodeOriginalVideofile, 169 transcodeOriginalVideofile,
130 importVideoFile 170 importVideoFile
diff --git a/server/middlewares/csp.ts b/server/middlewares/csp.ts
index 8b919af0d..5fa9d1ab5 100644
--- a/server/middlewares/csp.ts
+++ b/server/middlewares/csp.ts
@@ -16,7 +16,7 @@ const baseDirectives = Object.assign({},
16 baseUri: ["'self'"], 16 baseUri: ["'self'"],
17 manifestSrc: ["'self'"], 17 manifestSrc: ["'self'"],
18 frameSrc: ["'self'"], // instead of deprecated child-src / self because of test-embed 18 frameSrc: ["'self'"], // instead of deprecated child-src / self because of test-embed
19 workerSrc: ["'self'"] // instead of deprecated child-src 19 workerSrc: ["'self'", 'blob:'] // instead of deprecated child-src
20 }, 20 },
21 CONFIG.SERVICES['CSP-LOGGER'] ? { reportUri: CONFIG.SERVICES['CSP-LOGGER'] } : {}, 21 CONFIG.SERVICES['CSP-LOGGER'] ? { reportUri: CONFIG.SERVICES['CSP-LOGGER'] } : {},
22 CONFIG.WEBSERVER.SCHEME === 'https' ? { upgradeInsecureRequests: true } : {} 22 CONFIG.WEBSERVER.SCHEME === 'https' ? { upgradeInsecureRequests: true } : {}
diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts
index c72ab78b2..329322509 100644
--- a/server/middlewares/validators/redundancy.ts
+++ b/server/middlewares/validators/redundancy.ts
@@ -13,7 +13,7 @@ import { ActorFollowModel } from '../../models/activitypub/actor-follow'
13import { SERVER_ACTOR_NAME } from '../../initializers' 13import { SERVER_ACTOR_NAME } from '../../initializers'
14import { ServerModel } from '../../models/server/server' 14import { ServerModel } from '../../models/server/server'
15 15
16const videoRedundancyGetValidator = [ 16const videoFileRedundancyGetValidator = [
17 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), 17 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
18 param('resolution') 18 param('resolution')
19 .customSanitizer(toIntOrNull) 19 .customSanitizer(toIntOrNull)
@@ -24,7 +24,7 @@ const videoRedundancyGetValidator = [
24 .custom(exists).withMessage('Should have a valid fps'), 24 .custom(exists).withMessage('Should have a valid fps'),
25 25
26 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 26 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
27 logger.debug('Checking videoRedundancyGetValidator parameters', { parameters: req.params }) 27 logger.debug('Checking videoFileRedundancyGetValidator parameters', { parameters: req.params })
28 28
29 if (areValidationErrors(req, res)) return 29 if (areValidationErrors(req, res)) return
30 if (!await isVideoExist(req.params.videoId, res)) return 30 if (!await isVideoExist(req.params.videoId, res)) return
@@ -38,7 +38,31 @@ const videoRedundancyGetValidator = [
38 res.locals.videoFile = videoFile 38 res.locals.videoFile = videoFile
39 39
40 const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) 40 const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
41 if (!videoRedundancy)return res.status(404).json({ error: 'Video redundancy not found.' }) 41 if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' })
42 res.locals.videoRedundancy = videoRedundancy
43
44 return next()
45 }
46]
47
48const videoPlaylistRedundancyGetValidator = [
49 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
50 param('streamingPlaylistType').custom(exists).withMessage('Should have a valid streaming playlist type'),
51
52 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
53 logger.debug('Checking videoPlaylistRedundancyGetValidator parameters', { parameters: req.params })
54
55 if (areValidationErrors(req, res)) return
56 if (!await isVideoExist(req.params.videoId, res)) return
57
58 const video: VideoModel = res.locals.video
59 const videoStreamingPlaylist = video.VideoStreamingPlaylists.find(p => p === req.params.streamingPlaylistType)
60
61 if (!videoStreamingPlaylist) return res.status(404).json({ error: 'Video playlist not found.' })
62 res.locals.videoStreamingPlaylist = videoStreamingPlaylist
63
64 const videoRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(videoStreamingPlaylist.id)
65 if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' })
42 res.locals.videoRedundancy = videoRedundancy 66 res.locals.videoRedundancy = videoRedundancy
43 67
44 return next() 68 return next()
@@ -75,6 +99,7 @@ const updateServerRedundancyValidator = [
75// --------------------------------------------------------------------------- 99// ---------------------------------------------------------------------------
76 100
77export { 101export {
78 videoRedundancyGetValidator, 102 videoFileRedundancyGetValidator,
103 videoPlaylistRedundancyGetValidator,
79 updateServerRedundancyValidator 104 updateServerRedundancyValidator
80} 105}
diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts
index 1bb0bfb1b..a52e3060a 100644
--- a/server/middlewares/validators/users.ts
+++ b/server/middlewares/validators/users.ts
@@ -113,6 +113,7 @@ const deleteMeValidator = [
113 113
114const usersUpdateValidator = [ 114const usersUpdateValidator = [
115 param('id').isInt().not().isEmpty().withMessage('Should have a valid id'), 115 param('id').isInt().not().isEmpty().withMessage('Should have a valid id'),
116 body('password').optional().custom(isUserPasswordValid).withMessage('Should have a valid password'),
116 body('email').optional().isEmail().withMessage('Should have a valid email attribute'), 117 body('email').optional().isEmail().withMessage('Should have a valid email attribute'),
117 body('emailVerified').optional().isBoolean().withMessage('Should have a valid email verified attribute'), 118 body('emailVerified').optional().isBoolean().withMessage('Should have a valid email verified attribute'),
118 body('videoQuota').optional().custom(isUserVideoQuotaValid).withMessage('Should have a valid user quota'), 119 body('videoQuota').optional().custom(isUserVideoQuotaValid).withMessage('Should have a valid user quota'),
@@ -233,6 +234,7 @@ const usersAskResetPasswordValidator = [
233 logger.debug('Checking usersAskResetPassword parameters', { parameters: req.body }) 234 logger.debug('Checking usersAskResetPassword parameters', { parameters: req.body })
234 235
235 if (areValidationErrors(req, res)) return 236 if (areValidationErrors(req, res)) return
237
236 const exists = await checkUserEmailExist(req.body.email, res, false) 238 const exists = await checkUserEmailExist(req.body.email, res, false)
237 if (!exists) { 239 if (!exists) {
238 logger.debug('User with email %s does not exist (asking reset password).', req.body.email) 240 logger.debug('User with email %s does not exist (asking reset password).', req.body.email)
diff --git a/server/models/account/user-notification.ts b/server/models/account/user-notification.ts
index 9e4f982a3..6cdbb827b 100644
--- a/server/models/account/user-notification.ts
+++ b/server/models/account/user-notification.ts
@@ -27,11 +27,33 @@ import { VideoBlacklistModel } from '../video/video-blacklist'
27import { VideoImportModel } from '../video/video-import' 27import { VideoImportModel } from '../video/video-import'
28import { ActorModel } from '../activitypub/actor' 28import { ActorModel } from '../activitypub/actor'
29import { ActorFollowModel } from '../activitypub/actor-follow' 29import { ActorFollowModel } from '../activitypub/actor-follow'
30import { AvatarModel } from '../avatar/avatar'
31import { ServerModel } from '../server/server'
30 32
31enum ScopeNames { 33enum ScopeNames {
32 WITH_ALL = 'WITH_ALL' 34 WITH_ALL = 'WITH_ALL'
33} 35}
34 36
37function buildActorWithAvatarInclude () {
38 return {
39 attributes: [ 'preferredUsername' ],
40 model: () => ActorModel.unscoped(),
41 required: true,
42 include: [
43 {
44 attributes: [ 'filename' ],
45 model: () => AvatarModel.unscoped(),
46 required: false
47 },
48 {
49 attributes: [ 'host' ],
50 model: () => ServerModel.unscoped(),
51 required: false
52 }
53 ]
54 }
55}
56
35function buildVideoInclude (required: boolean) { 57function buildVideoInclude (required: boolean) {
36 return { 58 return {
37 attributes: [ 'id', 'uuid', 'name' ], 59 attributes: [ 'id', 'uuid', 'name' ],
@@ -40,19 +62,21 @@ function buildVideoInclude (required: boolean) {
40 } 62 }
41} 63}
42 64
43function buildChannelInclude (required: boolean) { 65function buildChannelInclude (required: boolean, withActor = false) {
44 return { 66 return {
45 required, 67 required,
46 attributes: [ 'id', 'name' ], 68 attributes: [ 'id', 'name' ],
47 model: () => VideoChannelModel.unscoped() 69 model: () => VideoChannelModel.unscoped(),
70 include: withActor === true ? [ buildActorWithAvatarInclude() ] : []
48 } 71 }
49} 72}
50 73
51function buildAccountInclude (required: boolean) { 74function buildAccountInclude (required: boolean, withActor = false) {
52 return { 75 return {
53 required, 76 required,
54 attributes: [ 'id', 'name' ], 77 attributes: [ 'id', 'name' ],
55 model: () => AccountModel.unscoped() 78 model: () => AccountModel.unscoped(),
79 include: withActor === true ? [ buildActorWithAvatarInclude() ] : []
56 } 80 }
57} 81}
58 82
@@ -60,47 +84,40 @@ function buildAccountInclude (required: boolean) {
60 [ScopeNames.WITH_ALL]: { 84 [ScopeNames.WITH_ALL]: {
61 include: [ 85 include: [
62 Object.assign(buildVideoInclude(false), { 86 Object.assign(buildVideoInclude(false), {
63 include: [ buildChannelInclude(true) ] 87 include: [ buildChannelInclude(true, true) ]
64 }), 88 }),
89
65 { 90 {
66 attributes: [ 'id', 'originCommentId' ], 91 attributes: [ 'id', 'originCommentId' ],
67 model: () => VideoCommentModel.unscoped(), 92 model: () => VideoCommentModel.unscoped(),
68 required: false, 93 required: false,
69 include: [ 94 include: [
70 buildAccountInclude(true), 95 buildAccountInclude(true, true),
71 buildVideoInclude(true) 96 buildVideoInclude(true)
72 ] 97 ]
73 }, 98 },
99
74 { 100 {
75 attributes: [ 'id' ], 101 attributes: [ 'id' ],
76 model: () => VideoAbuseModel.unscoped(), 102 model: () => VideoAbuseModel.unscoped(),
77 required: false, 103 required: false,
78 include: [ buildVideoInclude(true) ] 104 include: [ buildVideoInclude(true) ]
79 }, 105 },
106
80 { 107 {
81 attributes: [ 'id' ], 108 attributes: [ 'id' ],
82 model: () => VideoBlacklistModel.unscoped(), 109 model: () => VideoBlacklistModel.unscoped(),
83 required: false, 110 required: false,
84 include: [ buildVideoInclude(true) ] 111 include: [ buildVideoInclude(true) ]
85 }, 112 },
113
86 { 114 {
87 attributes: [ 'id', 'magnetUri', 'targetUrl', 'torrentName' ], 115 attributes: [ 'id', 'magnetUri', 'targetUrl', 'torrentName' ],
88 model: () => VideoImportModel.unscoped(), 116 model: () => VideoImportModel.unscoped(),
89 required: false, 117 required: false,
90 include: [ buildVideoInclude(false) ] 118 include: [ buildVideoInclude(false) ]
91 }, 119 },
92 { 120
93 attributes: [ 'id', 'name' ],
94 model: () => AccountModel.unscoped(),
95 required: false,
96 include: [
97 {
98 attributes: [ 'id', 'preferredUsername' ],
99 model: () => ActorModel.unscoped(),
100 required: true
101 }
102 ]
103 },
104 { 121 {
105 attributes: [ 'id' ], 122 attributes: [ 'id' ],
106 model: () => ActorFollowModel.unscoped(), 123 model: () => ActorFollowModel.unscoped(),
@@ -111,7 +128,23 @@ function buildAccountInclude (required: boolean) {
111 model: () => ActorModel.unscoped(), 128 model: () => ActorModel.unscoped(),
112 required: true, 129 required: true,
113 as: 'ActorFollower', 130 as: 'ActorFollower',
114 include: [ buildAccountInclude(true) ] 131 include: [
132 {
133 attributes: [ 'id', 'name' ],
134 model: () => AccountModel.unscoped(),
135 required: true
136 },
137 {
138 attributes: [ 'filename' ],
139 model: () => AvatarModel.unscoped(),
140 required: false
141 },
142 {
143 attributes: [ 'host' ],
144 model: () => ServerModel.unscoped(),
145 required: false
146 }
147 ]
115 }, 148 },
116 { 149 {
117 attributes: [ 'preferredUsername' ], 150 attributes: [ 'preferredUsername' ],
@@ -124,7 +157,9 @@ function buildAccountInclude (required: boolean) {
124 ] 157 ]
125 } 158 }
126 ] 159 ]
127 } 160 },
161
162 buildAccountInclude(false, true)
128 ] 163 ]
129 } 164 }
130}) 165})
@@ -132,10 +167,63 @@ function buildAccountInclude (required: boolean) {
132 tableName: 'userNotification', 167 tableName: 'userNotification',
133 indexes: [ 168 indexes: [
134 { 169 {
135 fields: [ 'videoId' ] 170 fields: [ 'userId' ]
171 },
172 {
173 fields: [ 'videoId' ],
174 where: {
175 videoId: {
176 [Op.ne]: null
177 }
178 }
136 }, 179 },
137 { 180 {
138 fields: [ 'commentId' ] 181 fields: [ 'commentId' ],
182 where: {
183 commentId: {
184 [Op.ne]: null
185 }
186 }
187 },
188 {
189 fields: [ 'videoAbuseId' ],
190 where: {
191 videoAbuseId: {
192 [Op.ne]: null
193 }
194 }
195 },
196 {
197 fields: [ 'videoBlacklistId' ],
198 where: {
199 videoBlacklistId: {
200 [Op.ne]: null
201 }
202 }
203 },
204 {
205 fields: [ 'videoImportId' ],
206 where: {
207 videoImportId: {
208 [Op.ne]: null
209 }
210 }
211 },
212 {
213 fields: [ 'accountId' ],
214 where: {
215 accountId: {
216 [Op.ne]: null
217 }
218 }
219 },
220 {
221 fields: [ 'actorFollowId' ],
222 where: {
223 actorFollowId: {
224 [Op.ne]: null
225 }
226 }
139 } 227 }
140 ] 228 ]
141}) 229})
@@ -297,12 +385,9 @@ export class UserNotificationModel extends Model<UserNotificationModel> {
297 } 385 }
298 386
299 toFormattedJSON (): UserNotification { 387 toFormattedJSON (): UserNotification {
300 const video = this.Video ? Object.assign(this.formatVideo(this.Video), { 388 const video = this.Video
301 channel: { 389 ? Object.assign(this.formatVideo(this.Video),{ channel: this.formatActor(this.Video.VideoChannel) })
302 id: this.Video.VideoChannel.id, 390 : undefined
303 displayName: this.Video.VideoChannel.getDisplayName()
304 }
305 }) : undefined
306 391
307 const videoImport = this.VideoImport ? { 392 const videoImport = this.VideoImport ? {
308 id: this.VideoImport.id, 393 id: this.VideoImport.id,
@@ -315,10 +400,7 @@ export class UserNotificationModel extends Model<UserNotificationModel> {
315 const comment = this.Comment ? { 400 const comment = this.Comment ? {
316 id: this.Comment.id, 401 id: this.Comment.id,
317 threadId: this.Comment.getThreadId(), 402 threadId: this.Comment.getThreadId(),
318 account: { 403 account: this.formatActor(this.Comment.Account),
319 id: this.Comment.Account.id,
320 displayName: this.Comment.Account.getDisplayName()
321 },
322 video: this.formatVideo(this.Comment.Video) 404 video: this.formatVideo(this.Comment.Video)
323 } : undefined 405 } : undefined
324 406
@@ -332,17 +414,16 @@ export class UserNotificationModel extends Model<UserNotificationModel> {
332 video: this.formatVideo(this.VideoBlacklist.Video) 414 video: this.formatVideo(this.VideoBlacklist.Video)
333 } : undefined 415 } : undefined
334 416
335 const account = this.Account ? { 417 const account = this.Account ? this.formatActor(this.Account) : undefined
336 id: this.Account.id,
337 displayName: this.Account.getDisplayName(),
338 name: this.Account.Actor.preferredUsername
339 } : undefined
340 418
341 const actorFollow = this.ActorFollow ? { 419 const actorFollow = this.ActorFollow ? {
342 id: this.ActorFollow.id, 420 id: this.ActorFollow.id,
343 follower: { 421 follower: {
422 id: this.ActorFollow.ActorFollower.Account.id,
344 displayName: this.ActorFollow.ActorFollower.Account.getDisplayName(), 423 displayName: this.ActorFollow.ActorFollower.Account.getDisplayName(),
345 name: this.ActorFollow.ActorFollower.preferredUsername 424 name: this.ActorFollow.ActorFollower.preferredUsername,
425 avatar: this.ActorFollow.ActorFollower.Avatar ? { path: this.ActorFollow.ActorFollower.Avatar.getWebserverPath() } : undefined,
426 host: this.ActorFollow.ActorFollower.getHost()
346 }, 427 },
347 following: { 428 following: {
348 type: this.ActorFollow.ActorFollowing.VideoChannel ? 'channel' as 'channel' : 'account' as 'account', 429 type: this.ActorFollow.ActorFollowing.VideoChannel ? 'channel' as 'channel' : 'account' as 'account',
@@ -374,4 +455,18 @@ export class UserNotificationModel extends Model<UserNotificationModel> {
374 name: video.name 455 name: video.name
375 } 456 }
376 } 457 }
458
459 private formatActor (accountOrChannel: AccountModel | VideoChannelModel) {
460 const avatar = accountOrChannel.Actor.Avatar
461 ? { path: accountOrChannel.Actor.Avatar.getWebserverPath() }
462 : undefined
463
464 return {
465 id: accountOrChannel.id,
466 displayName: accountOrChannel.getDisplayName(),
467 name: accountOrChannel.Actor.preferredUsername,
468 host: accountOrChannel.Actor.getHost(),
469 avatar
470 }
471 }
377} 472}
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index 8f2ef2d9a..b722bed14 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -28,6 +28,7 @@ import { sample } from 'lodash'
28import { isTestInstance } from '../../helpers/core-utils' 28import { isTestInstance } from '../../helpers/core-utils'
29import * as Bluebird from 'bluebird' 29import * as Bluebird from 'bluebird'
30import * as Sequelize from 'sequelize' 30import * as Sequelize from 'sequelize'
31import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
31 32
32export enum ScopeNames { 33export enum ScopeNames {
33 WITH_VIDEO = 'WITH_VIDEO' 34 WITH_VIDEO = 'WITH_VIDEO'
@@ -38,7 +39,17 @@ export enum ScopeNames {
38 include: [ 39 include: [
39 { 40 {
40 model: () => VideoFileModel, 41 model: () => VideoFileModel,
41 required: true, 42 required: false,
43 include: [
44 {
45 model: () => VideoModel,
46 required: true
47 }
48 ]
49 },
50 {
51 model: () => VideoStreamingPlaylistModel,
52 required: false,
42 include: [ 53 include: [
43 { 54 {
44 model: () => VideoModel, 55 model: () => VideoModel,
@@ -97,12 +108,24 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
97 108
98 @BelongsTo(() => VideoFileModel, { 109 @BelongsTo(() => VideoFileModel, {
99 foreignKey: { 110 foreignKey: {
100 allowNull: false 111 allowNull: true
101 }, 112 },
102 onDelete: 'cascade' 113 onDelete: 'cascade'
103 }) 114 })
104 VideoFile: VideoFileModel 115 VideoFile: VideoFileModel
105 116
117 @ForeignKey(() => VideoStreamingPlaylistModel)
118 @Column
119 videoStreamingPlaylistId: number
120
121 @BelongsTo(() => VideoStreamingPlaylistModel, {
122 foreignKey: {
123 allowNull: true
124 },
125 onDelete: 'cascade'
126 })
127 VideoStreamingPlaylist: VideoStreamingPlaylistModel
128
106 @ForeignKey(() => ActorModel) 129 @ForeignKey(() => ActorModel)
107 @Column 130 @Column
108 actorId: number 131 actorId: number
@@ -119,13 +142,25 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
119 static async removeFile (instance: VideoRedundancyModel) { 142 static async removeFile (instance: VideoRedundancyModel) {
120 if (!instance.isOwned()) return 143 if (!instance.isOwned()) return
121 144
122 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId) 145 if (instance.videoFileId) {
146 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
123 147
124 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` 148 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
125 logger.info('Removing duplicated video file %s.', logIdentifier) 149 logger.info('Removing duplicated video file %s.', logIdentifier)
126 150
127 videoFile.Video.removeFile(videoFile, true) 151 videoFile.Video.removeFile(videoFile, true)
128 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) 152 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
153 }
154
155 if (instance.videoStreamingPlaylistId) {
156 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
157
158 const videoUUID = videoStreamingPlaylist.Video.uuid
159 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
160
161 videoStreamingPlaylist.Video.removeStreamingPlaylist(true)
162 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
163 }
129 164
130 return undefined 165 return undefined
131 } 166 }
@@ -143,6 +178,19 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
143 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) 178 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
144 } 179 }
145 180
181 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number) {
182 const actor = await getServerActor()
183
184 const query = {
185 where: {
186 actorId: actor.id,
187 videoStreamingPlaylistId
188 }
189 }
190
191 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
192 }
193
146 static loadByUrl (url: string, transaction?: Sequelize.Transaction) { 194 static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
147 const query = { 195 const query = {
148 where: { 196 where: {
@@ -191,7 +239,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
191 const ids = rows.map(r => r.id) 239 const ids = rows.map(r => r.id)
192 const id = sample(ids) 240 const id = sample(ids)
193 241
194 return VideoModel.loadWithFile(id, undefined, !isTestInstance()) 242 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
195 } 243 }
196 244
197 static async findMostViewToDuplicate (randomizedFactor: number) { 245 static async findMostViewToDuplicate (randomizedFactor: number) {
@@ -333,40 +381,44 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
333 381
334 static async listLocalOfServer (serverId: number) { 382 static async listLocalOfServer (serverId: number) {
335 const actor = await getServerActor() 383 const actor = await getServerActor()
336 384 const buildVideoInclude = () => ({
337 const query = { 385 model: VideoModel,
338 where: { 386 required: true,
339 actorId: actor.id
340 },
341 include: [ 387 include: [
342 { 388 {
343 model: VideoFileModel, 389 attributes: [],
390 model: VideoChannelModel.unscoped(),
344 required: true, 391 required: true,
345 include: [ 392 include: [
346 { 393 {
347 model: VideoModel, 394 attributes: [],
395 model: ActorModel.unscoped(),
348 required: true, 396 required: true,
349 include: [ 397 where: {
350 { 398 serverId
351 attributes: [], 399 }
352 model: VideoChannelModel.unscoped(),
353 required: true,
354 include: [
355 {
356 attributes: [],
357 model: ActorModel.unscoped(),
358 required: true,
359 where: {
360 serverId
361 }
362 }
363 ]
364 }
365 ]
366 } 400 }
367 ] 401 ]
368 } 402 }
369 ] 403 ]
404 })
405
406 const query = {
407 where: {
408 actorId: actor.id
409 },
410 include: [
411 {
412 model: VideoFileModel,
413 required: false,
414 include: [ buildVideoInclude() ]
415 },
416 {
417 model: VideoStreamingPlaylistModel,
418 required: false,
419 include: [ buildVideoInclude() ]
420 }
421 ]
370 } 422 }
371 423
372 return VideoRedundancyModel.findAll(query) 424 return VideoRedundancyModel.findAll(query)
@@ -403,11 +455,32 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
403 })) 455 }))
404 } 456 }
405 457
458 getVideo () {
459 if (this.VideoFile) return this.VideoFile.Video
460
461 return this.VideoStreamingPlaylist.Video
462 }
463
406 isOwned () { 464 isOwned () {
407 return !!this.strategy 465 return !!this.strategy
408 } 466 }
409 467
410 toActivityPubObject (): CacheFileObject { 468 toActivityPubObject (): CacheFileObject {
469 if (this.VideoStreamingPlaylist) {
470 return {
471 id: this.url,
472 type: 'CacheFile' as 'CacheFile',
473 object: this.VideoStreamingPlaylist.Video.url,
474 expires: this.expiresOn.toISOString(),
475 url: {
476 type: 'Link',
477 mimeType: 'application/x-mpegURL',
478 mediaType: 'application/x-mpegURL',
479 href: this.fileUrl
480 }
481 }
482 }
483
411 return { 484 return {
412 id: this.url, 485 id: this.url,
413 type: 'CacheFile' as 'CacheFile', 486 type: 'CacheFile' as 'CacheFile',
@@ -431,7 +504,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
431 504
432 const notIn = Sequelize.literal( 505 const notIn = Sequelize.literal(
433 '(' + 506 '(' +
434 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` + 507 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
435 ')' 508 ')'
436 ) 509 )
437 510
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 1f1b76c1e..7d1e371b9 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -62,7 +62,7 @@ export class VideoFileModel extends Model<VideoFileModel> {
62 extname: string 62 extname: string
63 63
64 @AllowNull(false) 64 @AllowNull(false)
65 @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash')) 65 @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash'))
66 @Column 66 @Column
67 infoHash: string 67 infoHash: string
68 68
@@ -86,14 +86,14 @@ export class VideoFileModel extends Model<VideoFileModel> {
86 86
87 @HasMany(() => VideoRedundancyModel, { 87 @HasMany(() => VideoRedundancyModel, {
88 foreignKey: { 88 foreignKey: {
89 allowNull: false 89 allowNull: true
90 }, 90 },
91 onDelete: 'CASCADE', 91 onDelete: 'CASCADE',
92 hooks: true 92 hooks: true
93 }) 93 })
94 RedundancyVideos: VideoRedundancyModel[] 94 RedundancyVideos: VideoRedundancyModel[]
95 95
96 static isInfohashExists (infoHash: string) { 96 static doesInfohashExist (infoHash: string) {
97 const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' 97 const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
98 const options = { 98 const options = {
99 type: Sequelize.QueryTypes.SELECT, 99 type: Sequelize.QueryTypes.SELECT,
diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts
index de0747f22..e49dbee30 100644
--- a/server/models/video/video-format-utils.ts
+++ b/server/models/video/video-format-utils.ts
@@ -1,7 +1,12 @@
1import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' 1import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
2import { VideoModel } from './video' 2import { VideoModel } from './video'
3import { VideoFileModel } from './video-file' 3import { VideoFileModel } from './video-file'
4import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects' 4import {
5 ActivityPlaylistInfohashesObject,
6 ActivityPlaylistSegmentHashesObject,
7 ActivityUrlObject,
8 VideoTorrentObject
9} from '../../../shared/models/activitypub/objects'
5import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers' 10import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers'
6import { VideoCaptionModel } from './video-caption' 11import { VideoCaptionModel } from './video-caption'
7import { 12import {
@@ -11,6 +16,8 @@ import {
11 getVideoSharesActivityPubUrl 16 getVideoSharesActivityPubUrl
12} from '../../lib/activitypub' 17} from '../../lib/activitypub'
13import { isArray } from '../../helpers/custom-validators/misc' 18import { isArray } from '../../helpers/custom-validators/misc'
19import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
20import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
14 21
15export type VideoFormattingJSONOptions = { 22export type VideoFormattingJSONOptions = {
16 completeDescription?: boolean 23 completeDescription?: boolean
@@ -120,7 +127,12 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
120 } 127 }
121 }) 128 })
122 129
130 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
131
123 const tags = video.Tags ? video.Tags.map(t => t.name) : [] 132 const tags = video.Tags ? video.Tags.map(t => t.name) : []
133
134 const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists)
135
124 const detailsJson = { 136 const detailsJson = {
125 support: video.support, 137 support: video.support,
126 descriptionPath: video.getDescriptionAPIPath(), 138 descriptionPath: video.getDescriptionAPIPath(),
@@ -133,7 +145,11 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
133 id: video.state, 145 id: video.state,
134 label: VideoModel.getStateLabel(video.state) 146 label: VideoModel.getStateLabel(video.state)
135 }, 147 },
136 files: [] 148
149 trackerUrls: video.getTrackerUrls(baseUrlHttp, baseUrlWs),
150
151 files: [],
152 streamingPlaylists
137 } 153 }
138 154
139 // Format and sort video files 155 // Format and sort video files
@@ -142,6 +158,25 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
142 return Object.assign(formattedJson, detailsJson) 158 return Object.assign(formattedJson, detailsJson)
143} 159}
144 160
161function streamingPlaylistsModelToFormattedJSON (video: VideoModel, playlists: VideoStreamingPlaylistModel[]): VideoStreamingPlaylist[] {
162 if (isArray(playlists) === false) return []
163
164 return playlists
165 .map(playlist => {
166 const redundancies = isArray(playlist.RedundancyVideos)
167 ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl }))
168 : []
169
170 return {
171 id: playlist.id,
172 type: playlist.type,
173 playlistUrl: playlist.playlistUrl,
174 segmentsSha256Url: playlist.segmentsSha256Url,
175 redundancies
176 } as VideoStreamingPlaylist
177 })
178}
179
145function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] { 180function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] {
146 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 181 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
147 182
@@ -232,6 +267,28 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject {
232 }) 267 })
233 } 268 }
234 269
270 for (const playlist of (video.VideoStreamingPlaylists || [])) {
271 let tag: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[]
272
273 tag = playlist.p2pMediaLoaderInfohashes
274 .map(i => ({ type: 'Infohash' as 'Infohash', name: i }))
275 tag.push({
276 type: 'Link',
277 name: 'sha256',
278 mimeType: 'application/json' as 'application/json',
279 mediaType: 'application/json' as 'application/json',
280 href: playlist.segmentsSha256Url
281 })
282
283 url.push({
284 type: 'Link',
285 mimeType: 'application/x-mpegURL' as 'application/x-mpegURL',
286 mediaType: 'application/x-mpegURL' as 'application/x-mpegURL',
287 href: playlist.playlistUrl,
288 tag
289 })
290 }
291
235 // Add video url too 292 // Add video url too
236 url.push({ 293 url.push({
237 type: 'Link', 294 type: 'Link',
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
new file mode 100644
index 000000000..bf6f7b0c4
--- /dev/null
+++ b/server/models/video/video-streaming-playlist.ts
@@ -0,0 +1,158 @@
1import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
2import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
3import { throwIfNotValid } from '../utils'
4import { VideoModel } from './video'
5import * as Sequelize from 'sequelize'
6import { VideoRedundancyModel } from '../redundancy/video-redundancy'
7import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
8import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
9import { CONSTRAINTS_FIELDS, STATIC_PATHS } from '../../initializers'
10import { VideoFileModel } from './video-file'
11import { join } from 'path'
12import { sha1 } from '../../helpers/core-utils'
13import { isArrayOf } from '../../helpers/custom-validators/misc'
14
15@Table({
16 tableName: 'videoStreamingPlaylist',
17 indexes: [
18 {
19 fields: [ 'videoId' ]
20 },
21 {
22 fields: [ 'videoId', 'type' ],
23 unique: true
24 },
25 {
26 fields: [ 'p2pMediaLoaderInfohashes' ],
27 using: 'gin'
28 }
29 ]
30})
31export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistModel> {
32 @CreatedAt
33 createdAt: Date
34
35 @UpdatedAt
36 updatedAt: Date
37
38 @AllowNull(false)
39 @Column
40 type: VideoStreamingPlaylistType
41
42 @AllowNull(false)
43 @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url'))
44 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max))
45 playlistUrl: string
46
47 @AllowNull(false)
48 @Is('VideoStreamingPlaylistInfoHashes', value => throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes'))
49 @Column(DataType.ARRAY(DataType.STRING))
50 p2pMediaLoaderInfohashes: string[]
51
52 @AllowNull(false)
53 @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url'))
54 @Column
55 segmentsSha256Url: string
56
57 @ForeignKey(() => VideoModel)
58 @Column
59 videoId: number
60
61 @BelongsTo(() => VideoModel, {
62 foreignKey: {
63 allowNull: false
64 },
65 onDelete: 'CASCADE'
66 })
67 Video: VideoModel
68
69 @HasMany(() => VideoRedundancyModel, {
70 foreignKey: {
71 allowNull: false
72 },
73 onDelete: 'CASCADE',
74 hooks: true
75 })
76 RedundancyVideos: VideoRedundancyModel[]
77
78 static doesInfohashExist (infoHash: string) {
79 const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1'
80 const options = {
81 type: Sequelize.QueryTypes.SELECT,
82 bind: { infoHash },
83 raw: true
84 }
85
86 return VideoModel.sequelize.query(query, options)
87 .then(results => {
88 return results.length === 1
89 })
90 }
91
92 static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: VideoFileModel[]) {
93 const hashes: string[] = []
94
95 // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L97
96 for (let i = 0; i < videoFiles.length; i++) {
97 hashes.push(sha1(`1${playlistUrl}+V${i}`))
98 }
99
100 return hashes
101 }
102
103 static loadWithVideo (id: number) {
104 const options = {
105 include: [
106 {
107 model: VideoModel.unscoped(),
108 required: true
109 }
110 ]
111 }
112
113 return VideoStreamingPlaylistModel.findById(id, options)
114 }
115
116 static getHlsPlaylistFilename (resolution: number) {
117 return resolution + '.m3u8'
118 }
119
120 static getMasterHlsPlaylistFilename () {
121 return 'master.m3u8'
122 }
123
124 static getHlsSha256SegmentsFilename () {
125 return 'segments-sha256.json'
126 }
127
128 static getHlsVideoName (uuid: string, resolution: number) {
129 return `${uuid}-${resolution}-fragmented.mp4`
130 }
131
132 static getHlsMasterPlaylistStaticPath (videoUUID: string) {
133 return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
134 }
135
136 static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) {
137 return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
138 }
139
140 static getHlsSha256SegmentsStaticPath (videoUUID: string) {
141 return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
142 }
143
144 getStringType () {
145 if (this.type === VideoStreamingPlaylistType.HLS) return 'hls'
146
147 return 'unknown'
148 }
149
150 getVideoRedundancyUrl (baseUrlHttp: string) {
151 return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getStringType() + '/' + this.Video.uuid
152 }
153
154 hasSameUniqueKeysThan (other: VideoStreamingPlaylistModel) {
155 return this.type === other.type &&
156 this.videoId === other.videoId
157 }
158}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 80a6c7832..702260772 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -52,7 +52,7 @@ import {
52 ACTIVITY_PUB, 52 ACTIVITY_PUB,
53 API_VERSION, 53 API_VERSION,
54 CONFIG, 54 CONFIG,
55 CONSTRAINTS_FIELDS, 55 CONSTRAINTS_FIELDS, HLS_PLAYLIST_DIRECTORY, HLS_REDUNDANCY_DIRECTORY,
56 PREVIEWS_SIZE, 56 PREVIEWS_SIZE,
57 REMOTE_SCHEME, 57 REMOTE_SCHEME,
58 STATIC_DOWNLOAD_PATHS, 58 STATIC_DOWNLOAD_PATHS,
@@ -95,6 +95,7 @@ import * as validator from 'validator'
95import { UserVideoHistoryModel } from '../account/user-video-history' 95import { UserVideoHistoryModel } from '../account/user-video-history'
96import { UserModel } from '../account/user' 96import { UserModel } from '../account/user'
97import { VideoImportModel } from './video-import' 97import { VideoImportModel } from './video-import'
98import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
98 99
99// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation 100// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
100const indexes: Sequelize.DefineIndexesOptions[] = [ 101const indexes: Sequelize.DefineIndexesOptions[] = [
@@ -159,7 +160,9 @@ export enum ScopeNames {
159 WITH_FILES = 'WITH_FILES', 160 WITH_FILES = 'WITH_FILES',
160 WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', 161 WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE',
161 WITH_BLACKLISTED = 'WITH_BLACKLISTED', 162 WITH_BLACKLISTED = 'WITH_BLACKLISTED',
162 WITH_USER_HISTORY = 'WITH_USER_HISTORY' 163 WITH_USER_HISTORY = 'WITH_USER_HISTORY',
164 WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS',
165 WITH_USER_ID = 'WITH_USER_ID'
163} 166}
164 167
165type ForAPIOptions = { 168type ForAPIOptions = {
@@ -463,6 +466,22 @@ type AvailableForListIDsOptions = {
463 466
464 return query 467 return query
465 }, 468 },
469 [ ScopeNames.WITH_USER_ID ]: {
470 include: [
471 {
472 attributes: [ 'accountId' ],
473 model: () => VideoChannelModel.unscoped(),
474 required: true,
475 include: [
476 {
477 attributes: [ 'userId' ],
478 model: () => AccountModel.unscoped(),
479 required: true
480 }
481 ]
482 }
483 ]
484 },
466 [ ScopeNames.WITH_ACCOUNT_DETAILS ]: { 485 [ ScopeNames.WITH_ACCOUNT_DETAILS ]: {
467 include: [ 486 include: [
468 { 487 {
@@ -527,22 +546,55 @@ type AvailableForListIDsOptions = {
527 } 546 }
528 ] 547 ]
529 }, 548 },
530 [ ScopeNames.WITH_FILES ]: { 549 [ ScopeNames.WITH_FILES ]: (withRedundancies = false) => {
531 include: [ 550 let subInclude: any[] = []
532 { 551
533 model: () => VideoFileModel.unscoped(), 552 if (withRedundancies === true) {
534 // FIXME: typings 553 subInclude = [
535 [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join 554 {
536 required: false, 555 attributes: [ 'fileUrl' ],
537 include: [ 556 model: VideoRedundancyModel.unscoped(),
538 { 557 required: false
539 attributes: [ 'fileUrl' ], 558 }
540 model: () => VideoRedundancyModel.unscoped(), 559 ]
541 required: false 560 }
542 } 561
543 ] 562 return {
544 } 563 include: [
545 ] 564 {
565 model: VideoFileModel.unscoped(),
566 // FIXME: typings
567 [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join
568 required: false,
569 include: subInclude
570 }
571 ]
572 }
573 },
574 [ ScopeNames.WITH_STREAMING_PLAYLISTS ]: (withRedundancies = false) => {
575 let subInclude: any[] = []
576
577 if (withRedundancies === true) {
578 subInclude = [
579 {
580 attributes: [ 'fileUrl' ],
581 model: VideoRedundancyModel.unscoped(),
582 required: false
583 }
584 ]
585 }
586
587 return {
588 include: [
589 {
590 model: VideoStreamingPlaylistModel.unscoped(),
591 // FIXME: typings
592 [ 'separate' as any ]: true, // We may have multiple streaming playlists, having multiple redundancies so let's separate this join
593 required: false,
594 include: subInclude
595 }
596 ]
597 }
546 }, 598 },
547 [ ScopeNames.WITH_SCHEDULED_UPDATE ]: { 599 [ ScopeNames.WITH_SCHEDULED_UPDATE ]: {
548 include: [ 600 include: [
@@ -722,6 +774,16 @@ export class VideoModel extends Model<VideoModel> {
722 }) 774 })
723 VideoFiles: VideoFileModel[] 775 VideoFiles: VideoFileModel[]
724 776
777 @HasMany(() => VideoStreamingPlaylistModel, {
778 foreignKey: {
779 name: 'videoId',
780 allowNull: false
781 },
782 hooks: true,
783 onDelete: 'cascade'
784 })
785 VideoStreamingPlaylists: VideoStreamingPlaylistModel[]
786
725 @HasMany(() => VideoShareModel, { 787 @HasMany(() => VideoShareModel, {
726 foreignKey: { 788 foreignKey: {
727 name: 'videoId', 789 name: 'videoId',
@@ -847,6 +909,9 @@ export class VideoModel extends Model<VideoModel> {
847 tasks.push(instance.removeFile(file)) 909 tasks.push(instance.removeFile(file))
848 tasks.push(instance.removeTorrent(file)) 910 tasks.push(instance.removeTorrent(file))
849 }) 911 })
912
913 // Remove playlists file
914 tasks.push(instance.removeStreamingPlaylist())
850 } 915 }
851 916
852 // Do not wait video deletion because we could be in a transaction 917 // Do not wait video deletion because we could be in a transaction
@@ -858,10 +923,6 @@ export class VideoModel extends Model<VideoModel> {
858 return undefined 923 return undefined
859 } 924 }
860 925
861 static list () {
862 return VideoModel.scope(ScopeNames.WITH_FILES).findAll()
863 }
864
865 static listLocal () { 926 static listLocal () {
866 const query = { 927 const query = {
867 where: { 928 where: {
@@ -869,7 +930,7 @@ export class VideoModel extends Model<VideoModel> {
869 } 930 }
870 } 931 }
871 932
872 return VideoModel.scope(ScopeNames.WITH_FILES).findAll(query) 933 return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]).findAll(query)
873 } 934 }
874 935
875 static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { 936 static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) {
@@ -1200,6 +1261,16 @@ export class VideoModel extends Model<VideoModel> {
1200 return VideoModel.findOne(options) 1261 return VideoModel.findOne(options)
1201 } 1262 }
1202 1263
1264 static loadWithRights (id: number | string, t?: Sequelize.Transaction) {
1265 const where = VideoModel.buildWhereIdOrUUID(id)
1266 const options = {
1267 where,
1268 transaction: t
1269 }
1270
1271 return VideoModel.scope([ ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_USER_ID ]).findOne(options)
1272 }
1273
1203 static loadOnlyId (id: number | string, t?: Sequelize.Transaction) { 1274 static loadOnlyId (id: number | string, t?: Sequelize.Transaction) {
1204 const where = VideoModel.buildWhereIdOrUUID(id) 1275 const where = VideoModel.buildWhereIdOrUUID(id)
1205 1276
@@ -1212,8 +1283,8 @@ export class VideoModel extends Model<VideoModel> {
1212 return VideoModel.findOne(options) 1283 return VideoModel.findOne(options)
1213 } 1284 }
1214 1285
1215 static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) { 1286 static loadWithFiles (id: number, t?: Sequelize.Transaction, logging?: boolean) {
1216 return VideoModel.scope(ScopeNames.WITH_FILES) 1287 return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ])
1217 .findById(id, { transaction: t, logging }) 1288 .findById(id, { transaction: t, logging })
1218 } 1289 }
1219 1290
@@ -1224,9 +1295,7 @@ export class VideoModel extends Model<VideoModel> {
1224 } 1295 }
1225 } 1296 }
1226 1297
1227 return VideoModel 1298 return VideoModel.findOne(options)
1228 .scope([ ScopeNames.WITH_FILES ])
1229 .findOne(options)
1230 } 1299 }
1231 1300
1232 static loadByUrl (url: string, transaction?: Sequelize.Transaction) { 1301 static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
@@ -1248,7 +1317,11 @@ export class VideoModel extends Model<VideoModel> {
1248 transaction 1317 transaction
1249 } 1318 }
1250 1319
1251 return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) 1320 return VideoModel.scope([
1321 ScopeNames.WITH_ACCOUNT_DETAILS,
1322 ScopeNames.WITH_FILES,
1323 ScopeNames.WITH_STREAMING_PLAYLISTS
1324 ]).findOne(query)
1252 } 1325 }
1253 1326
1254 static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) { 1327 static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) {
@@ -1263,9 +1336,37 @@ export class VideoModel extends Model<VideoModel> {
1263 const scopes = [ 1336 const scopes = [
1264 ScopeNames.WITH_TAGS, 1337 ScopeNames.WITH_TAGS,
1265 ScopeNames.WITH_BLACKLISTED, 1338 ScopeNames.WITH_BLACKLISTED,
1339 ScopeNames.WITH_ACCOUNT_DETAILS,
1340 ScopeNames.WITH_SCHEDULED_UPDATE,
1266 ScopeNames.WITH_FILES, 1341 ScopeNames.WITH_FILES,
1342 ScopeNames.WITH_STREAMING_PLAYLISTS
1343 ]
1344
1345 if (userId) {
1346 scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings
1347 }
1348
1349 return VideoModel
1350 .scope(scopes)
1351 .findOne(options)
1352 }
1353
1354 static loadForGetAPI (id: number | string, t?: Sequelize.Transaction, userId?: number) {
1355 const where = VideoModel.buildWhereIdOrUUID(id)
1356
1357 const options = {
1358 order: [ [ 'Tags', 'name', 'ASC' ] ],
1359 where,
1360 transaction: t
1361 }
1362
1363 const scopes = [
1364 ScopeNames.WITH_TAGS,
1365 ScopeNames.WITH_BLACKLISTED,
1267 ScopeNames.WITH_ACCOUNT_DETAILS, 1366 ScopeNames.WITH_ACCOUNT_DETAILS,
1268 ScopeNames.WITH_SCHEDULED_UPDATE 1367 ScopeNames.WITH_SCHEDULED_UPDATE,
1368 { method: [ ScopeNames.WITH_FILES, true ] } as any, // FIXME: typings
1369 { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } as any // FIXME: typings
1269 ] 1370 ]
1270 1371
1271 if (userId) { 1372 if (userId) {
@@ -1612,6 +1713,14 @@ export class VideoModel extends Model<VideoModel> {
1612 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) 1713 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
1613 } 1714 }
1614 1715
1716 removeStreamingPlaylist (isRedundancy = false) {
1717 const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_PLAYLIST_DIRECTORY
1718
1719 const filePath = join(baseDir, this.uuid)
1720 return remove(filePath)
1721 .catch(err => logger.warn('Cannot delete playlist directory %s.', filePath, { err }))
1722 }
1723
1615 isOutdated () { 1724 isOutdated () {
1616 if (this.isOwned()) return false 1725 if (this.isOwned()) return false
1617 1726
@@ -1646,7 +1755,7 @@ export class VideoModel extends Model<VideoModel> {
1646 1755
1647 generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { 1756 generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
1648 const xs = this.getTorrentUrl(videoFile, baseUrlHttp) 1757 const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
1649 const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] 1758 const announce = this.getTrackerUrls(baseUrlHttp, baseUrlWs)
1650 let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] 1759 let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
1651 1760
1652 const redundancies = videoFile.RedundancyVideos 1761 const redundancies = videoFile.RedundancyVideos
@@ -1663,6 +1772,10 @@ export class VideoModel extends Model<VideoModel> {
1663 return magnetUtil.encode(magnetHash) 1772 return magnetUtil.encode(magnetHash)
1664 } 1773 }
1665 1774
1775 getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) {
1776 return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
1777 }
1778
1666 getThumbnailUrl (baseUrlHttp: string) { 1779 getThumbnailUrl (baseUrlHttp: string) {
1667 return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() 1780 return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName()
1668 } 1781 }
@@ -1686,4 +1799,8 @@ export class VideoModel extends Model<VideoModel> {
1686 getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { 1799 getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
1687 return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) 1800 return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
1688 } 1801 }
1802
1803 getBandwidthBits (videoFile: VideoFileModel) {
1804 return Math.ceil((videoFile.size * 8) / this.duration)
1805 }
1689} 1806}
diff --git a/server/tests/api/check-params/accounts.ts b/server/tests/api/check-params/accounts.ts
index 567fd072c..68f9519c6 100644
--- a/server/tests/api/check-params/accounts.ts
+++ b/server/tests/api/check-params/accounts.ts
@@ -10,7 +10,7 @@ import {
10} from '../../../../shared/utils/requests/check-api-params' 10} from '../../../../shared/utils/requests/check-api-params'
11import { getAccount } from '../../../../shared/utils/users/accounts' 11import { getAccount } from '../../../../shared/utils/users/accounts'
12 12
13describe('Test users API validators', function () { 13describe('Test accounts API validators', function () {
14 const path = '/api/v1/accounts/' 14 const path = '/api/v1/accounts/'
15 let server: ServerInfo 15 let server: ServerInfo
16 16
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 4038ecbf0..07de2b5a5 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -65,6 +65,9 @@ describe('Test config API validators', function () {
65 '480p': true, 65 '480p': true,
66 '720p': false, 66 '720p': false,
67 '1080p': false 67 '1080p': false
68 },
69 hls: {
70 enabled: false
68 } 71 }
69 }, 72 },
70 import: { 73 import: {
diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts
index a3e8e2e9c..13be8b460 100644
--- a/server/tests/api/check-params/users.ts
+++ b/server/tests/api/check-params/users.ts
@@ -464,6 +464,24 @@ describe('Test users API validators', function () {
464 await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) 464 await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields })
465 }) 465 })
466 466
467 it('Should fail with a too small password', async function () {
468 const fields = {
469 currentPassword: 'my super password',
470 password: 'bla'
471 }
472
473 await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields })
474 })
475
476 it('Should fail with a too long password', async function () {
477 const fields = {
478 currentPassword: 'my super password',
479 password: 'super'.repeat(61)
480 }
481
482 await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields })
483 })
484
467 it('Should fail with an non authenticated user', async function () { 485 it('Should fail with an non authenticated user', async function () {
468 const fields = { 486 const fields = {
469 videoQuota: 42 487 videoQuota: 42
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 9d3ce8153..778611fff 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -17,7 +17,7 @@ import {
17 viewVideo, 17 viewVideo,
18 wait, 18 wait,
19 waitUntilLog, 19 waitUntilLog,
20 checkVideoFilesWereRemoved, removeVideo, getVideoWithToken 20 checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer, checkSegmentHash
21} from '../../../../shared/utils' 21} from '../../../../shared/utils'
22import { waitJobs } from '../../../../shared/utils/server/jobs' 22import { waitJobs } from '../../../../shared/utils/server/jobs'
23 23
@@ -48,6 +48,11 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
48 48
49async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { 49async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
50 const config = { 50 const config = {
51 transcoding: {
52 hls: {
53 enabled: true
54 }
55 },
51 redundancy: { 56 redundancy: {
52 videos: { 57 videos: {
53 check_interval: '5 seconds', 58 check_interval: '5 seconds',
@@ -85,7 +90,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
85 await waitJobs(servers) 90 await waitJobs(servers)
86} 91}
87 92
88async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { 93async function check1WebSeed (videoUUID?: string) {
89 if (!videoUUID) videoUUID = video1Server2UUID 94 if (!videoUUID) videoUUID = video1Server2UUID
90 95
91 const webseeds = [ 96 const webseeds = [
@@ -93,47 +98,17 @@ async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: str
93 ] 98 ]
94 99
95 for (const server of servers) { 100 for (const server of servers) {
96 { 101 // With token to avoid issues with video follow constraints
97 // With token to avoid issues with video follow constraints 102 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
98 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
99 103
100 const video: VideoDetails = res.body 104 const video: VideoDetails = res.body
101 for (const f of video.files) { 105 for (const f of video.files) {
102 checkMagnetWebseeds(f, webseeds, server) 106 checkMagnetWebseeds(f, webseeds, server)
103 }
104 } 107 }
105 } 108 }
106} 109}
107 110
108async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { 111async function check2Webseeds (videoUUID?: string) {
109 const res = await getStats(servers[0].url)
110 const data: ServerStats = res.body
111
112 expect(data.videosRedundancy).to.have.lengthOf(1)
113 const stat = data.videosRedundancy[0]
114
115 expect(stat.strategy).to.equal(strategy)
116 expect(stat.totalSize).to.equal(204800)
117 expect(stat.totalUsed).to.be.at.least(1).and.below(204801)
118 expect(stat.totalVideoFiles).to.equal(4)
119 expect(stat.totalVideos).to.equal(1)
120}
121
122async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
123 const res = await getStats(servers[0].url)
124 const data: ServerStats = res.body
125
126 expect(data.videosRedundancy).to.have.lengthOf(1)
127
128 const stat = data.videosRedundancy[0]
129 expect(stat.strategy).to.equal(strategy)
130 expect(stat.totalSize).to.equal(204800)
131 expect(stat.totalUsed).to.equal(0)
132 expect(stat.totalVideoFiles).to.equal(0)
133 expect(stat.totalVideos).to.equal(0)
134}
135
136async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
137 if (!videoUUID) videoUUID = video1Server2UUID 112 if (!videoUUID) videoUUID = video1Server2UUID
138 113
139 const webseeds = [ 114 const webseeds = [
@@ -158,7 +133,7 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st
158 await makeGetRequest({ 133 await makeGetRequest({
159 url: servers[1].url, 134 url: servers[1].url,
160 statusCodeExpected: 200, 135 statusCodeExpected: 200,
161 path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, 136 path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
162 contentType: null 137 contentType: null
163 }) 138 })
164 } 139 }
@@ -174,6 +149,85 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st
174 } 149 }
175} 150}
176 151
152async function check0PlaylistRedundancies (videoUUID?: string) {
153 if (!videoUUID) videoUUID = video1Server2UUID
154
155 for (const server of servers) {
156 // With token to avoid issues with video follow constraints
157 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
158 const video: VideoDetails = res.body
159
160 expect(video.streamingPlaylists).to.be.an('array')
161 expect(video.streamingPlaylists).to.have.lengthOf(1)
162 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
163 }
164}
165
166async function check1PlaylistRedundancies (videoUUID?: string) {
167 if (!videoUUID) videoUUID = video1Server2UUID
168
169 for (const server of servers) {
170 const res = await getVideo(server.url, videoUUID)
171 const video: VideoDetails = res.body
172
173 expect(video.streamingPlaylists).to.have.lengthOf(1)
174 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
175
176 const redundancy = video.streamingPlaylists[0].redundancies[0]
177
178 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
179 }
180
181 const baseUrlPlaylist = servers[1].url + '/static/playlists/hls'
182 const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
183
184 const res = await getVideo(servers[0].url, videoUUID)
185 const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
186
187 for (const resolution of [ 240, 360, 480, 720 ]) {
188 await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist)
189 }
190
191 for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) {
192 const files = await readdir(join(root(), directory, videoUUID))
193 expect(files).to.have.length.at.least(4)
194
195 for (const resolution of [ 240, 360, 480, 720 ]) {
196 const filename = `${videoUUID}-${resolution}-fragmented.mp4`
197
198 expect(files.find(f => f === filename)).to.not.be.undefined
199 }
200 }
201}
202
203async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
204 const res = await getStats(servers[0].url)
205 const data: ServerStats = res.body
206
207 expect(data.videosRedundancy).to.have.lengthOf(1)
208 const stat = data.videosRedundancy[0]
209
210 expect(stat.strategy).to.equal(strategy)
211 expect(stat.totalSize).to.equal(204800)
212 expect(stat.totalUsed).to.be.at.least(1).and.below(204801)
213 expect(stat.totalVideoFiles).to.equal(4)
214 expect(stat.totalVideos).to.equal(1)
215}
216
217async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
218 const res = await getStats(servers[0].url)
219 const data: ServerStats = res.body
220
221 expect(data.videosRedundancy).to.have.lengthOf(1)
222
223 const stat = data.videosRedundancy[0]
224 expect(stat.strategy).to.equal(strategy)
225 expect(stat.totalSize).to.equal(204800)
226 expect(stat.totalUsed).to.equal(0)
227 expect(stat.totalVideoFiles).to.equal(0)
228 expect(stat.totalVideos).to.equal(0)
229}
230
177async function enableRedundancyOnServer1 () { 231async function enableRedundancyOnServer1 () {
178 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) 232 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
179 233
@@ -220,7 +274,8 @@ describe('Test videos redundancy', function () {
220 }) 274 })
221 275
222 it('Should have 1 webseed on the first video', async function () { 276 it('Should have 1 webseed on the first video', async function () {
223 await check1WebSeed(strategy) 277 await check1WebSeed()
278 await check0PlaylistRedundancies()
224 await checkStatsWith1Webseed(strategy) 279 await checkStatsWith1Webseed(strategy)
225 }) 280 })
226 281
@@ -229,27 +284,29 @@ describe('Test videos redundancy', function () {
229 }) 284 })
230 285
231 it('Should have 2 webseeds on the first video', async function () { 286 it('Should have 2 webseeds on the first video', async function () {
232 this.timeout(40000) 287 this.timeout(80000)
233 288
234 await waitJobs(servers) 289 await waitJobs(servers)
235 await waitUntilLog(servers[0], 'Duplicated ', 4) 290 await waitUntilLog(servers[0], 'Duplicated ', 5)
236 await waitJobs(servers) 291 await waitJobs(servers)
237 292
238 await check2Webseeds(strategy) 293 await check2Webseeds()
294 await check1PlaylistRedundancies()
239 await checkStatsWith2Webseed(strategy) 295 await checkStatsWith2Webseed(strategy)
240 }) 296 })
241 297
242 it('Should undo redundancy on server 1 and remove duplicated videos', async function () { 298 it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
243 this.timeout(40000) 299 this.timeout(80000)
244 300
245 await disableRedundancyOnServer1() 301 await disableRedundancyOnServer1()
246 302
247 await waitJobs(servers) 303 await waitJobs(servers)
248 await wait(5000) 304 await wait(5000)
249 305
250 await check1WebSeed(strategy) 306 await check1WebSeed()
307 await check0PlaylistRedundancies()
251 308
252 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) 309 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ])
253 }) 310 })
254 311
255 after(function () { 312 after(function () {
@@ -267,7 +324,8 @@ describe('Test videos redundancy', function () {
267 }) 324 })
268 325
269 it('Should have 1 webseed on the first video', async function () { 326 it('Should have 1 webseed on the first video', async function () {
270 await check1WebSeed(strategy) 327 await check1WebSeed()
328 await check0PlaylistRedundancies()
271 await checkStatsWith1Webseed(strategy) 329 await checkStatsWith1Webseed(strategy)
272 }) 330 })
273 331
@@ -276,25 +334,27 @@ describe('Test videos redundancy', function () {
276 }) 334 })
277 335
278 it('Should have 2 webseeds on the first video', async function () { 336 it('Should have 2 webseeds on the first video', async function () {
279 this.timeout(40000) 337 this.timeout(80000)
280 338
281 await waitJobs(servers) 339 await waitJobs(servers)
282 await waitUntilLog(servers[0], 'Duplicated ', 4) 340 await waitUntilLog(servers[0], 'Duplicated ', 5)
283 await waitJobs(servers) 341 await waitJobs(servers)
284 342
285 await check2Webseeds(strategy) 343 await check2Webseeds()
344 await check1PlaylistRedundancies()
286 await checkStatsWith2Webseed(strategy) 345 await checkStatsWith2Webseed(strategy)
287 }) 346 })
288 347
289 it('Should unfollow on server 1 and remove duplicated videos', async function () { 348 it('Should unfollow on server 1 and remove duplicated videos', async function () {
290 this.timeout(40000) 349 this.timeout(80000)
291 350
292 await unfollow(servers[0].url, servers[0].accessToken, servers[1]) 351 await unfollow(servers[0].url, servers[0].accessToken, servers[1])
293 352
294 await waitJobs(servers) 353 await waitJobs(servers)
295 await wait(5000) 354 await wait(5000)
296 355
297 await check1WebSeed(strategy) 356 await check1WebSeed()
357 await check0PlaylistRedundancies()
298 358
299 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) 359 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
300 }) 360 })
@@ -314,7 +374,8 @@ describe('Test videos redundancy', function () {
314 }) 374 })
315 375
316 it('Should have 1 webseed on the first video', async function () { 376 it('Should have 1 webseed on the first video', async function () {
317 await check1WebSeed(strategy) 377 await check1WebSeed()
378 await check0PlaylistRedundancies()
318 await checkStatsWith1Webseed(strategy) 379 await checkStatsWith1Webseed(strategy)
319 }) 380 })
320 381
@@ -323,18 +384,19 @@ describe('Test videos redundancy', function () {
323 }) 384 })
324 385
325 it('Should still have 1 webseed on the first video', async function () { 386 it('Should still have 1 webseed on the first video', async function () {
326 this.timeout(40000) 387 this.timeout(80000)
327 388
328 await waitJobs(servers) 389 await waitJobs(servers)
329 await wait(15000) 390 await wait(15000)
330 await waitJobs(servers) 391 await waitJobs(servers)
331 392
332 await check1WebSeed(strategy) 393 await check1WebSeed()
394 await check0PlaylistRedundancies()
333 await checkStatsWith1Webseed(strategy) 395 await checkStatsWith1Webseed(strategy)
334 }) 396 })
335 397
336 it('Should view 2 times the first video to have > min_views config', async function () { 398 it('Should view 2 times the first video to have > min_views config', async function () {
337 this.timeout(40000) 399 this.timeout(80000)
338 400
339 await viewVideo(servers[ 0 ].url, video1Server2UUID) 401 await viewVideo(servers[ 0 ].url, video1Server2UUID)
340 await viewVideo(servers[ 2 ].url, video1Server2UUID) 402 await viewVideo(servers[ 2 ].url, video1Server2UUID)
@@ -344,13 +406,14 @@ describe('Test videos redundancy', function () {
344 }) 406 })
345 407
346 it('Should have 2 webseeds on the first video', async function () { 408 it('Should have 2 webseeds on the first video', async function () {
347 this.timeout(40000) 409 this.timeout(80000)
348 410
349 await waitJobs(servers) 411 await waitJobs(servers)
350 await waitUntilLog(servers[0], 'Duplicated ', 4) 412 await waitUntilLog(servers[0], 'Duplicated ', 5)
351 await waitJobs(servers) 413 await waitJobs(servers)
352 414
353 await check2Webseeds(strategy) 415 await check2Webseeds()
416 await check1PlaylistRedundancies()
354 await checkStatsWith2Webseed(strategy) 417 await checkStatsWith2Webseed(strategy)
355 }) 418 })
356 419
@@ -405,7 +468,7 @@ describe('Test videos redundancy', function () {
405 }) 468 })
406 469
407 it('Should still have 2 webseeds after 10 seconds', async function () { 470 it('Should still have 2 webseeds after 10 seconds', async function () {
408 this.timeout(40000) 471 this.timeout(80000)
409 472
410 await wait(10000) 473 await wait(10000)
411 474
@@ -420,7 +483,7 @@ describe('Test videos redundancy', function () {
420 }) 483 })
421 484
422 it('Should stop server 1 and expire video redundancy', async function () { 485 it('Should stop server 1 and expire video redundancy', async function () {
423 this.timeout(40000) 486 this.timeout(80000)
424 487
425 killallServers([ servers[0] ]) 488 killallServers([ servers[0] ])
426 489
@@ -446,10 +509,11 @@ describe('Test videos redundancy', function () {
446 await enableRedundancyOnServer1() 509 await enableRedundancyOnServer1()
447 510
448 await waitJobs(servers) 511 await waitJobs(servers)
449 await waitUntilLog(servers[0], 'Duplicated ', 4) 512 await waitUntilLog(servers[0], 'Duplicated ', 5)
450 await waitJobs(servers) 513 await waitJobs(servers)
451 514
452 await check2Webseeds(strategy) 515 await check2Webseeds()
516 await check1PlaylistRedundancies()
453 await checkStatsWith2Webseed(strategy) 517 await checkStatsWith2Webseed(strategy)
454 518
455 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) 519 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
@@ -467,8 +531,10 @@ describe('Test videos redundancy', function () {
467 await wait(1000) 531 await wait(1000)
468 532
469 try { 533 try {
470 await check1WebSeed(strategy, video1Server2UUID) 534 await check1WebSeed(video1Server2UUID)
471 await check2Webseeds(strategy, video2Server2UUID) 535 await check0PlaylistRedundancies(video1Server2UUID)
536 await check2Webseeds(video2Server2UUID)
537 await check1PlaylistRedundancies(video2Server2UUID)
472 538
473 checked = true 539 checked = true
474 } catch { 540 } catch {
@@ -477,6 +543,26 @@ describe('Test videos redundancy', function () {
477 } 543 }
478 }) 544 })
479 545
546 it('Should disable strategy and remove redundancies', async function () {
547 this.timeout(80000)
548
549 await waitJobs(servers)
550
551 killallServers([ servers[ 0 ] ])
552 await reRunServer(servers[ 0 ], {
553 redundancy: {
554 videos: {
555 check_interval: '1 second',
556 strategies: []
557 }
558 }
559 })
560
561 await waitJobs(servers)
562
563 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ])
564 })
565
480 after(function () { 566 after(function () {
481 return cleanServers() 567 return cleanServers()
482 }) 568 })
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index bebfc7398..0dfe6e4fe 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -57,6 +57,8 @@ function checkInitialConfig (data: CustomConfig) {
57 expect(data.transcoding.resolutions['480p']).to.be.true 57 expect(data.transcoding.resolutions['480p']).to.be.true
58 expect(data.transcoding.resolutions['720p']).to.be.true 58 expect(data.transcoding.resolutions['720p']).to.be.true
59 expect(data.transcoding.resolutions['1080p']).to.be.true 59 expect(data.transcoding.resolutions['1080p']).to.be.true
60 expect(data.transcoding.hls.enabled).to.be.true
61
60 expect(data.import.videos.http.enabled).to.be.true 62 expect(data.import.videos.http.enabled).to.be.true
61 expect(data.import.videos.torrent.enabled).to.be.true 63 expect(data.import.videos.torrent.enabled).to.be.true
62} 64}
@@ -95,6 +97,7 @@ function checkUpdatedConfig (data: CustomConfig) {
95 expect(data.transcoding.resolutions['480p']).to.be.true 97 expect(data.transcoding.resolutions['480p']).to.be.true
96 expect(data.transcoding.resolutions['720p']).to.be.false 98 expect(data.transcoding.resolutions['720p']).to.be.false
97 expect(data.transcoding.resolutions['1080p']).to.be.false 99 expect(data.transcoding.resolutions['1080p']).to.be.false
100 expect(data.transcoding.hls.enabled).to.be.false
98 101
99 expect(data.import.videos.http.enabled).to.be.false 102 expect(data.import.videos.http.enabled).to.be.false
100 expect(data.import.videos.torrent.enabled).to.be.false 103 expect(data.import.videos.torrent.enabled).to.be.false
@@ -205,6 +208,9 @@ describe('Test config', function () {
205 '480p': true, 208 '480p': true,
206 '720p': false, 209 '720p': false,
207 '1080p': false 210 '1080p': false
211 },
212 hls: {
213 enabled: false
208 } 214 }
209 }, 215 },
210 import: { 216 import: {
diff --git a/server/tests/api/users/user-notifications.ts b/server/tests/api/users/user-notifications.ts
index 5260d64cc..69e51677e 100644
--- a/server/tests/api/users/user-notifications.ts
+++ b/server/tests/api/users/user-notifications.ts
@@ -165,6 +165,8 @@ describe('Test users notifications', function () {
165 }) 165 })
166 166
167 it('Should not send notifications if the user does not follow the video publisher', async function () { 167 it('Should not send notifications if the user does not follow the video publisher', async function () {
168 this.timeout(10000)
169
168 await uploadVideoByLocalAccount(servers) 170 await uploadVideoByLocalAccount(servers)
169 171
170 const notification = await getLastNotification(servers[ 0 ].url, userAccessToken) 172 const notification = await getLastNotification(servers[ 0 ].url, userAccessToken)
@@ -644,6 +646,8 @@ describe('Test users notifications', function () {
644 }) 646 })
645 647
646 it('Should not send a notification if transcoding is not enabled', async function () { 648 it('Should not send a notification if transcoding is not enabled', async function () {
649 this.timeout(10000)
650
647 const { name, uuid } = await uploadVideoByLocalAccount(servers) 651 const { name, uuid } = await uploadVideoByLocalAccount(servers)
648 await waitJobs(servers) 652 await waitJobs(servers)
649 653
@@ -717,6 +721,24 @@ describe('Test users notifications', function () {
717 await wait(6000) 721 await wait(6000)
718 await checkVideoIsPublished(baseParams, name, uuid, 'presence') 722 await checkVideoIsPublished(baseParams, name, uuid, 'presence')
719 }) 723 })
724
725 it('Should not send a notification before the video is published', async function () {
726 this.timeout(20000)
727
728 let updateAt = new Date(new Date().getTime() + 100000)
729
730 const data = {
731 privacy: VideoPrivacy.PRIVATE,
732 scheduleUpdate: {
733 updateAt: updateAt.toISOString(),
734 privacy: VideoPrivacy.PUBLIC
735 }
736 }
737 const { name, uuid } = await uploadVideoByRemoteAccount(servers, data)
738
739 await wait(6000)
740 await checkVideoIsPublished(baseParams, name, uuid, 'absence')
741 })
720 }) 742 })
721 743
722 describe('My video is imported', function () { 744 describe('My video is imported', function () {
@@ -781,6 +803,8 @@ describe('Test users notifications', function () {
781 }) 803 })
782 804
783 it('Should send a notification only to moderators when a user registers on the instance', async function () { 805 it('Should send a notification only to moderators when a user registers on the instance', async function () {
806 this.timeout(10000)
807
784 await registerUser(servers[0].url, 'user_45', 'password') 808 await registerUser(servers[0].url, 'user_45', 'password')
785 809
786 await waitJobs(servers) 810 await waitJobs(servers)
@@ -849,6 +873,8 @@ describe('Test users notifications', function () {
849 }) 873 })
850 874
851 it('Should notify when a local account is following one of our channel', async function () { 875 it('Should notify when a local account is following one of our channel', async function () {
876 this.timeout(10000)
877
852 await addUserSubscription(servers[0].url, servers[0].accessToken, 'user_1@localhost:9001') 878 await addUserSubscription(servers[0].url, servers[0].accessToken, 'user_1@localhost:9001')
853 879
854 await waitJobs(servers) 880 await waitJobs(servers)
@@ -857,6 +883,8 @@ describe('Test users notifications', function () {
857 }) 883 })
858 884
859 it('Should notify when a remote account is following one of our channel', async function () { 885 it('Should notify when a remote account is following one of our channel', async function () {
886 this.timeout(10000)
887
860 await addUserSubscription(servers[1].url, servers[1].accessToken, 'user_1@localhost:9001') 888 await addUserSubscription(servers[1].url, servers[1].accessToken, 'user_1@localhost:9001')
861 889
862 await waitJobs(servers) 890 await waitJobs(servers)
@@ -926,6 +954,8 @@ describe('Test users notifications', function () {
926 }) 954 })
927 955
928 it('Should not have notifications', async function () { 956 it('Should not have notifications', async function () {
957 this.timeout(10000)
958
929 await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { 959 await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, {
930 newVideoFromSubscription: UserNotificationSettingValue.NONE 960 newVideoFromSubscription: UserNotificationSettingValue.NONE
931 })) 961 }))
@@ -943,6 +973,8 @@ describe('Test users notifications', function () {
943 }) 973 })
944 974
945 it('Should only have web notifications', async function () { 975 it('Should only have web notifications', async function () {
976 this.timeout(10000)
977
946 await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { 978 await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, {
947 newVideoFromSubscription: UserNotificationSettingValue.WEB 979 newVideoFromSubscription: UserNotificationSettingValue.WEB
948 })) 980 }))
@@ -967,6 +999,8 @@ describe('Test users notifications', function () {
967 }) 999 })
968 1000
969 it('Should only have mail notifications', async function () { 1001 it('Should only have mail notifications', async function () {
1002 this.timeout(10000)
1003
970 await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { 1004 await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, {
971 newVideoFromSubscription: UserNotificationSettingValue.EMAIL 1005 newVideoFromSubscription: UserNotificationSettingValue.EMAIL
972 })) 1006 }))
@@ -991,6 +1025,8 @@ describe('Test users notifications', function () {
991 }) 1025 })
992 1026
993 it('Should have email and web notifications', async function () { 1027 it('Should have email and web notifications', async function () {
1028 this.timeout(10000)
1029
994 await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { 1030 await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, {
995 newVideoFromSubscription: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL 1031 newVideoFromSubscription: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL
996 })) 1032 }))
diff --git a/server/tests/api/users/users.ts b/server/tests/api/users/users.ts
index ad98ab1c7..c4465d541 100644
--- a/server/tests/api/users/users.ts
+++ b/server/tests/api/users/users.ts
@@ -501,6 +501,22 @@ describe('Test users', function () {
501 accessTokenUser = await userLogin(server, user) 501 accessTokenUser = await userLogin(server, user)
502 }) 502 })
503 503
504 it('Should be able to update another user password', async function () {
505 await updateUser({
506 url: server.url,
507 userId,
508 accessToken,
509 password: 'password updated'
510 })
511
512 await getMyUserVideoQuotaUsed(server.url, accessTokenUser, 401)
513
514 await userLogin(server, user, 400)
515
516 user.password = 'password updated'
517 accessTokenUser = await userLogin(server, user)
518 })
519
504 it('Should be able to list video blacklist by a moderator', async function () { 520 it('Should be able to list video blacklist by a moderator', async function () {
505 await getBlacklistedVideosList(server.url, accessTokenUser) 521 await getBlacklistedVideosList(server.url, accessTokenUser)
506 }) 522 })
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index 97f467aae..a501a80b2 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -8,6 +8,7 @@ import './video-change-ownership'
8import './video-channels' 8import './video-channels'
9import './video-comments' 9import './video-comments'
10import './video-description' 10import './video-description'
11import './video-hls'
11import './video-imports' 12import './video-imports'
12import './video-nsfw' 13import './video-nsfw'
13import './video-privacy' 14import './video-privacy'
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts
new file mode 100644
index 000000000..a1214bad1
--- /dev/null
+++ b/server/tests/api/videos/video-hls.ts
@@ -0,0 +1,139 @@
1/* tslint:disable:no-unused-expression */
2
3import * as chai from 'chai'
4import 'mocha'
5import {
6 checkDirectoryIsEmpty,
7 checkSegmentHash,
8 checkTmpIsEmpty,
9 doubleFollow,
10 flushAndRunMultipleServers,
11 flushTests,
12 getPlaylist,
13 getVideo,
14 killallServers,
15 removeVideo,
16 ServerInfo,
17 setAccessTokensToServers,
18 updateVideo,
19 uploadVideo,
20 waitJobs
21} from '../../../../shared/utils'
22import { VideoDetails } from '../../../../shared/models/videos'
23import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type'
24import { join } from 'path'
25
26const expect = chai.expect
27
28async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string) {
29 const resolutions = [ 240, 360, 480, 720 ]
30
31 for (const server of servers) {
32 const res = await getVideo(server.url, videoUUID)
33 const videoDetails: VideoDetails = res.body
34
35 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
36
37 const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
38 expect(hlsPlaylist).to.not.be.undefined
39
40 {
41 const res2 = await getPlaylist(hlsPlaylist.playlistUrl)
42
43 const masterPlaylist = res2.text
44
45 expect(masterPlaylist).to.contain('#EXT-X-STREAM-INF:BANDWIDTH=55472,RESOLUTION=640x360,FRAME-RATE=25')
46
47 for (const resolution of resolutions) {
48 expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
49 }
50 }
51
52 {
53 for (const resolution of resolutions) {
54 const res2 = await getPlaylist(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}.m3u8`)
55
56 const subPlaylist = res2.text
57 expect(subPlaylist).to.contain(`${videoUUID}-${resolution}-fragmented.mp4`)
58 }
59 }
60
61 {
62 const baseUrl = 'http://localhost:9001/static/playlists/hls'
63
64 for (const resolution of resolutions) {
65 await checkSegmentHash(baseUrl, baseUrl, videoUUID, resolution, hlsPlaylist)
66 }
67 }
68 }
69}
70
71describe('Test HLS videos', function () {
72 let servers: ServerInfo[] = []
73 let videoUUID = ''
74
75 before(async function () {
76 this.timeout(120000)
77
78 servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true, hls: { enabled: true } } })
79
80 // Get the access tokens
81 await setAccessTokensToServers(servers)
82
83 // Server 1 and server 2 follow each other
84 await doubleFollow(servers[0], servers[1])
85 })
86
87 it('Should upload a video and transcode it to HLS', async function () {
88 this.timeout(120000)
89
90 {
91 const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1', fixture: 'video_short.webm' })
92 videoUUID = res.body.video.uuid
93 }
94
95 await waitJobs(servers)
96
97 await checkHlsPlaylist(servers, videoUUID)
98 })
99
100 it('Should update the video', async function () {
101 await updateVideo(servers[0].url, servers[0].accessToken, videoUUID, { name: 'video 1 updated' })
102
103 await waitJobs(servers)
104
105 await checkHlsPlaylist(servers, videoUUID)
106 })
107
108 it('Should delete the video', async function () {
109 await removeVideo(servers[0].url, servers[0].accessToken, videoUUID)
110
111 await waitJobs(servers)
112
113 for (const server of servers) {
114 await getVideo(server.url, videoUUID, 404)
115 }
116 })
117
118 it('Should have the playlists/segment deleted from the disk', async function () {
119 for (const server of servers) {
120 await checkDirectoryIsEmpty(server, 'videos')
121 await checkDirectoryIsEmpty(server, join('playlists', 'hls'))
122 }
123 })
124
125 it('Should have an empty tmp directory', async function () {
126 for (const server of servers) {
127 await checkTmpIsEmpty(server)
128 }
129 })
130
131 after(async function () {
132 killallServers(servers)
133
134 // Keep the logs if the test failed
135 if (this['ok']) {
136 await flushTests()
137 }
138 })
139})
diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts
index 811ea6a9f..d38bb4331 100644
--- a/server/tests/cli/update-host.ts
+++ b/server/tests/cli/update-host.ts
@@ -86,6 +86,13 @@ describe('Test update host scripts', function () {
86 const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid) 86 const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid)
87 87
88 expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid) 88 expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid)
89
90 const res = await getVideo(server.url, video.uuid)
91 const videoDetails: VideoDetails = res.body
92
93 expect(videoDetails.trackerUrls[0]).to.include(server.host)
94 expect(videoDetails.streamingPlaylists[0].playlistUrl).to.include(server.host)
95 expect(videoDetails.streamingPlaylists[0].segmentsSha256Url).to.include(server.host)
89 } 96 }
90 }) 97 })
91 98
@@ -100,7 +107,7 @@ describe('Test update host scripts', function () {
100 } 107 }
101 }) 108 })
102 109
103 it('Should have update accounts url', async function () { 110 it('Should have updated accounts url', async function () {
104 const res = await getAccountsList(server.url) 111 const res = await getAccountsList(server.url)
105 expect(res.body.total).to.equal(3) 112 expect(res.body.total).to.equal(3)
106 113
@@ -112,7 +119,7 @@ describe('Test update host scripts', function () {
112 } 119 }
113 }) 120 })
114 121
115 it('Should update torrent hosts', async function () { 122 it('Should have updated torrent hosts', async function () {
116 this.timeout(30000) 123 this.timeout(30000)
117 124
118 const res = await getVideosList(server.url) 125 const res = await getVideosList(server.url)