diff options
Diffstat (limited to 'server')
53 files changed, 1752 insertions, 334 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index 7e87f6f3b..31c0a5fbd 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts | |||
@@ -33,7 +33,7 @@ import { | |||
33 | getVideoSharesActivityPubUrl | 33 | getVideoSharesActivityPubUrl |
34 | } from '../../lib/activitypub' | 34 | } from '../../lib/activitypub' |
35 | import { VideoCaptionModel } from '../../models/video/video-caption' | 35 | import { VideoCaptionModel } from '../../models/video/video-caption' |
36 | import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy' | 36 | import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy' |
37 | import { getServerActor } from '../../helpers/utils' | 37 | import { getServerActor } from '../../helpers/utils' |
38 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 38 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
39 | import { buildDislikeActivity } from '../../lib/activitypub/send/send-dislike' | 39 | import { buildDislikeActivity } from '../../lib/activitypub/send/send-dislike' |
@@ -63,11 +63,11 @@ activityPubClientRouter.get('/accounts?/:name/dislikes/:videoId', | |||
63 | 63 | ||
64 | activityPubClientRouter.get('/videos/watch/:id', | 64 | activityPubClientRouter.get('/videos/watch/:id', |
65 | executeIfActivityPub(asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS))), | 65 | executeIfActivityPub(asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS))), |
66 | executeIfActivityPub(asyncMiddleware(videosGetValidator)), | 66 | executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))), |
67 | executeIfActivityPub(asyncMiddleware(videoController)) | 67 | executeIfActivityPub(asyncMiddleware(videoController)) |
68 | ) | 68 | ) |
69 | activityPubClientRouter.get('/videos/watch/:id/activity', | 69 | activityPubClientRouter.get('/videos/watch/:id/activity', |
70 | executeIfActivityPub(asyncMiddleware(videosGetValidator)), | 70 | executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))), |
71 | executeIfActivityPub(asyncMiddleware(videoController)) | 71 | executeIfActivityPub(asyncMiddleware(videoController)) |
72 | ) | 72 | ) |
73 | activityPubClientRouter.get('/videos/watch/:id/announces', | 73 | activityPubClientRouter.get('/videos/watch/:id/announces', |
@@ -113,7 +113,11 @@ activityPubClientRouter.get('/video-channels/:name/following', | |||
113 | ) | 113 | ) |
114 | 114 | ||
115 | activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?', | 115 | activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?', |
116 | executeIfActivityPub(asyncMiddleware(videoRedundancyGetValidator)), | 116 | executeIfActivityPub(asyncMiddleware(videoFileRedundancyGetValidator)), |
117 | executeIfActivityPub(asyncMiddleware(videoRedundancyController)) | ||
118 | ) | ||
119 | activityPubClientRouter.get('/redundancy/video-playlists/:streamingPlaylistType/:videoId', | ||
120 | executeIfActivityPub(asyncMiddleware(videoPlaylistRedundancyGetValidator)), | ||
117 | executeIfActivityPub(asyncMiddleware(videoRedundancyController)) | 121 | executeIfActivityPub(asyncMiddleware(videoRedundancyController)) |
118 | ) | 122 | ) |
119 | 123 | ||
@@ -160,7 +164,8 @@ function getAccountVideoRate (rateType: VideoRateType) { | |||
160 | } | 164 | } |
161 | 165 | ||
162 | async function videoController (req: express.Request, res: express.Response) { | 166 | async function videoController (req: express.Request, res: express.Response) { |
163 | const video: VideoModel = res.locals.video | 167 | // We need more attributes |
168 | const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id) | ||
164 | 169 | ||
165 | if (video.url.startsWith(CONFIG.WEBSERVER.URL) === false) return res.redirect(video.url) | 170 | if (video.url.startsWith(CONFIG.WEBSERVER.URL) === false) return res.redirect(video.url) |
166 | 171 | ||
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 255026f46..1f3341bc0 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { omit, snakeCase } from 'lodash' | 2 | import { snakeCase } from 'lodash' |
3 | import { ServerConfig, UserRight } from '../../../shared' | 3 | import { ServerConfig, UserRight } from '../../../shared' |
4 | import { About } from '../../../shared/models/server/about.model' | 4 | import { About } from '../../../shared/models/server/about.model' |
5 | import { CustomConfig } from '../../../shared/models/server/custom-config.model' | 5 | import { CustomConfig } from '../../../shared/models/server/custom-config.model' |
@@ -78,6 +78,9 @@ async function getConfig (req: express.Request, res: express.Response) { | |||
78 | requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION | 78 | requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION |
79 | }, | 79 | }, |
80 | transcoding: { | 80 | transcoding: { |
81 | hls: { | ||
82 | enabled: CONFIG.TRANSCODING.HLS.ENABLED | ||
83 | }, | ||
81 | enabledResolutions | 84 | enabledResolutions |
82 | }, | 85 | }, |
83 | import: { | 86 | import: { |
@@ -246,6 +249,9 @@ function customConfig (): CustomConfig { | |||
246 | '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ], | 249 | '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ], |
247 | '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ], | 250 | '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ], |
248 | '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ] | 251 | '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ] |
252 | }, | ||
253 | hls: { | ||
254 | enabled: CONFIG.TRANSCODING.HLS.ENABLED | ||
249 | } | 255 | } |
250 | }, | 256 | }, |
251 | import: { | 257 | import: { |
diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts index dbe0718d4..e3533a7f6 100644 --- a/server/controllers/api/users/index.ts +++ b/server/controllers/api/users/index.ts | |||
@@ -229,7 +229,7 @@ async function unblockUser (req: express.Request, res: express.Response, next: e | |||
229 | return res.status(204).end() | 229 | return res.status(204).end() |
230 | } | 230 | } |
231 | 231 | ||
232 | async function blockUser (req: express.Request, res: express.Response, next: express.NextFunction) { | 232 | async function blockUser (req: express.Request, res: express.Response) { |
233 | const user: UserModel = res.locals.user | 233 | const user: UserModel = res.locals.user |
234 | const reason = req.body.reason | 234 | const reason = req.body.reason |
235 | 235 | ||
@@ -238,23 +238,23 @@ async function blockUser (req: express.Request, res: express.Response, next: exp | |||
238 | return res.status(204).end() | 238 | return res.status(204).end() |
239 | } | 239 | } |
240 | 240 | ||
241 | function getUser (req: express.Request, res: express.Response, next: express.NextFunction) { | 241 | function getUser (req: express.Request, res: express.Response) { |
242 | return res.json((res.locals.user as UserModel).toFormattedJSON()) | 242 | return res.json((res.locals.user as UserModel).toFormattedJSON()) |
243 | } | 243 | } |
244 | 244 | ||
245 | async function autocompleteUsers (req: express.Request, res: express.Response, next: express.NextFunction) { | 245 | async function autocompleteUsers (req: express.Request, res: express.Response) { |
246 | const resultList = await UserModel.autoComplete(req.query.search as string) | 246 | const resultList = await UserModel.autoComplete(req.query.search as string) |
247 | 247 | ||
248 | return res.json(resultList) | 248 | return res.json(resultList) |
249 | } | 249 | } |
250 | 250 | ||
251 | async function listUsers (req: express.Request, res: express.Response, next: express.NextFunction) { | 251 | async function listUsers (req: express.Request, res: express.Response) { |
252 | const resultList = await UserModel.listForApi(req.query.start, req.query.count, req.query.sort, req.query.search) | 252 | const resultList = await UserModel.listForApi(req.query.start, req.query.count, req.query.sort, req.query.search) |
253 | 253 | ||
254 | return res.json(getFormattedObjects(resultList.data, resultList.total)) | 254 | return res.json(getFormattedObjects(resultList.data, resultList.total)) |
255 | } | 255 | } |
256 | 256 | ||
257 | async function removeUser (req: express.Request, res: express.Response, next: express.NextFunction) { | 257 | async function removeUser (req: express.Request, res: express.Response) { |
258 | const user: UserModel = res.locals.user | 258 | const user: UserModel = res.locals.user |
259 | 259 | ||
260 | await user.destroy() | 260 | await user.destroy() |
@@ -264,12 +264,13 @@ async function removeUser (req: express.Request, res: express.Response, next: ex | |||
264 | return res.sendStatus(204) | 264 | return res.sendStatus(204) |
265 | } | 265 | } |
266 | 266 | ||
267 | async function updateUser (req: express.Request, res: express.Response, next: express.NextFunction) { | 267 | async function updateUser (req: express.Request, res: express.Response) { |
268 | const body: UserUpdate = req.body | 268 | const body: UserUpdate = req.body |
269 | const userToUpdate = res.locals.user as UserModel | 269 | const userToUpdate = res.locals.user as UserModel |
270 | const oldUserAuditView = new UserAuditView(userToUpdate.toFormattedJSON()) | 270 | const oldUserAuditView = new UserAuditView(userToUpdate.toFormattedJSON()) |
271 | const roleChanged = body.role !== undefined && body.role !== userToUpdate.role | 271 | const roleChanged = body.role !== undefined && body.role !== userToUpdate.role |
272 | 272 | ||
273 | if (body.password !== undefined) userToUpdate.password = body.password | ||
273 | if (body.email !== undefined) userToUpdate.email = body.email | 274 | if (body.email !== undefined) userToUpdate.email = body.email |
274 | if (body.emailVerified !== undefined) userToUpdate.emailVerified = body.emailVerified | 275 | if (body.emailVerified !== undefined) userToUpdate.emailVerified = body.emailVerified |
275 | if (body.videoQuota !== undefined) userToUpdate.videoQuota = body.videoQuota | 276 | if (body.videoQuota !== undefined) userToUpdate.videoQuota = body.videoQuota |
@@ -279,11 +280,11 @@ async function updateUser (req: express.Request, res: express.Response, next: ex | |||
279 | const user = await userToUpdate.save() | 280 | const user = await userToUpdate.save() |
280 | 281 | ||
281 | // Destroy user token to refresh rights | 282 | // Destroy user token to refresh rights |
282 | if (roleChanged) await deleteUserToken(userToUpdate.id) | 283 | if (roleChanged || body.password !== undefined) await deleteUserToken(userToUpdate.id) |
283 | 284 | ||
284 | auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) | 285 | auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) |
285 | 286 | ||
286 | // Don't need to send this update to followers, these attributes are not propagated | 287 | // Don't need to send this update to followers, these attributes are not federated |
287 | 288 | ||
288 | return res.sendStatus(204) | 289 | return res.sendStatus(204) |
289 | } | 290 | } |
@@ -293,7 +294,7 @@ async function askResetUserPassword (req: express.Request, res: express.Response | |||
293 | 294 | ||
294 | const verificationString = await Redis.Instance.setResetPasswordVerificationString(user.id) | 295 | const verificationString = await Redis.Instance.setResetPasswordVerificationString(user.id) |
295 | const url = CONFIG.WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString | 296 | const url = CONFIG.WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString |
296 | await Emailer.Instance.addForgetPasswordEmailJob(user.email, url) | 297 | await Emailer.Instance.addPasswordResetEmailJob(user.email, url) |
297 | 298 | ||
298 | return res.status(204).end() | 299 | return res.status(204).end() |
299 | } | 300 | } |
diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index 94a2b8732..d5e154869 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts | |||
@@ -167,7 +167,7 @@ async function deleteMe (req: express.Request, res: express.Response) { | |||
167 | return res.sendStatus(204) | 167 | return res.sendStatus(204) |
168 | } | 168 | } |
169 | 169 | ||
170 | async function updateMe (req: express.Request, res: express.Response, next: express.NextFunction) { | 170 | async function updateMe (req: express.Request, res: express.Response) { |
171 | const body: UserUpdateMe = req.body | 171 | const body: UserUpdateMe = req.body |
172 | 172 | ||
173 | const user: UserModel = res.locals.oauth.token.user | 173 | const user: UserModel = res.locals.oauth.token.user |
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 8414ca42c..3b1c2d255 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts | |||
@@ -36,6 +36,7 @@ import { | |||
36 | setDefaultPagination, | 36 | setDefaultPagination, |
37 | setDefaultSort, | 37 | setDefaultSort, |
38 | videosAddValidator, | 38 | videosAddValidator, |
39 | videosCustomGetValidator, | ||
39 | videosGetValidator, | 40 | videosGetValidator, |
40 | videosRemoveValidator, | 41 | videosRemoveValidator, |
41 | videosSortValidator, | 42 | videosSortValidator, |
@@ -123,9 +124,9 @@ videosRouter.get('/:id/description', | |||
123 | ) | 124 | ) |
124 | videosRouter.get('/:id', | 125 | videosRouter.get('/:id', |
125 | optionalAuthenticate, | 126 | optionalAuthenticate, |
126 | asyncMiddleware(videosGetValidator), | 127 | asyncMiddleware(videosCustomGetValidator('only-video-with-rights')), |
127 | asyncMiddleware(checkVideoFollowConstraints), | 128 | asyncMiddleware(checkVideoFollowConstraints), |
128 | getVideo | 129 | asyncMiddleware(getVideo) |
129 | ) | 130 | ) |
130 | videosRouter.post('/:id/views', | 131 | videosRouter.post('/:id/views', |
131 | asyncMiddleware(videosGetValidator), | 132 | asyncMiddleware(videosGetValidator), |
@@ -395,15 +396,17 @@ async function updateVideo (req: express.Request, res: express.Response) { | |||
395 | return res.type('json').status(204).end() | 396 | return res.type('json').status(204).end() |
396 | } | 397 | } |
397 | 398 | ||
398 | function getVideo (req: express.Request, res: express.Response) { | 399 | async function getVideo (req: express.Request, res: express.Response) { |
399 | const videoInstance = res.locals.video | 400 | // We need more attributes |
401 | const userId: number = res.locals.oauth ? res.locals.oauth.token.User.id : null | ||
402 | const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id, undefined, userId) | ||
400 | 403 | ||
401 | if (videoInstance.isOutdated()) { | 404 | if (video.isOutdated()) { |
402 | JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoInstance.url } }) | 405 | JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } }) |
403 | .catch(err => logger.error('Cannot create AP refresher job for video %s.', videoInstance.url, { err })) | 406 | .catch(err => logger.error('Cannot create AP refresher job for video %s.', video.url, { err })) |
404 | } | 407 | } |
405 | 408 | ||
406 | return res.json(videoInstance.toFormattedDetailsJSON()) | 409 | return res.json(video.toFormattedDetailsJSON()) |
407 | } | 410 | } |
408 | 411 | ||
409 | async function viewVideo (req: express.Request, res: express.Response) { | 412 | async function viewVideo (req: express.Request, res: express.Response) { |
diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 4fd58f70c..b21f9da00 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import * as cors from 'cors' | 1 | import * as cors from 'cors' |
2 | import * as express from 'express' | 2 | import * as express from 'express' |
3 | import { CONFIG, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers' | 3 | import { CONFIG, HLS_PLAYLIST_DIRECTORY, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers' |
4 | import { VideosPreviewCache } from '../lib/cache' | 4 | import { VideosPreviewCache } from '../lib/cache' |
5 | import { cacheRoute } from '../middlewares/cache' | 5 | import { cacheRoute } from '../middlewares/cache' |
6 | import { asyncMiddleware, videosGetValidator } from '../middlewares' | 6 | import { asyncMiddleware, videosGetValidator } from '../middlewares' |
@@ -51,6 +51,13 @@ staticRouter.use( | |||
51 | asyncMiddleware(downloadVideoFile) | 51 | asyncMiddleware(downloadVideoFile) |
52 | ) | 52 | ) |
53 | 53 | ||
54 | // HLS | ||
55 | staticRouter.use( | ||
56 | STATIC_PATHS.PLAYLISTS.HLS, | ||
57 | cors(), | ||
58 | express.static(HLS_PLAYLIST_DIRECTORY, { fallthrough: false }) // 404 if the file does not exist | ||
59 | ) | ||
60 | |||
54 | // Thumbnails path for express | 61 | // Thumbnails path for express |
55 | const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR | 62 | const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR |
56 | staticRouter.use( | 63 | staticRouter.use( |
diff --git a/server/controllers/tracker.ts b/server/controllers/tracker.ts index 1deb8c402..8b77d9de7 100644 --- a/server/controllers/tracker.ts +++ b/server/controllers/tracker.ts | |||
@@ -7,6 +7,7 @@ import { Server as WebSocketServer } from 'ws' | |||
7 | import { CONFIG, TRACKER_RATE_LIMITS } from '../initializers/constants' | 7 | import { CONFIG, TRACKER_RATE_LIMITS } from '../initializers/constants' |
8 | import { VideoFileModel } from '../models/video/video-file' | 8 | import { VideoFileModel } from '../models/video/video-file' |
9 | import { parse } from 'url' | 9 | import { parse } from 'url' |
10 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
10 | 11 | ||
11 | const TrackerServer = bitTorrentTracker.Server | 12 | const TrackerServer = bitTorrentTracker.Server |
12 | 13 | ||
@@ -21,7 +22,7 @@ const trackerServer = new TrackerServer({ | |||
21 | udp: false, | 22 | udp: false, |
22 | ws: false, | 23 | ws: false, |
23 | dht: false, | 24 | dht: false, |
24 | filter: function (infoHash, params, cb) { | 25 | filter: async function (infoHash, params, cb) { |
25 | let ip: string | 26 | let ip: string |
26 | 27 | ||
27 | if (params.type === 'ws') { | 28 | if (params.type === 'ws') { |
@@ -32,19 +33,25 @@ const trackerServer = new TrackerServer({ | |||
32 | 33 | ||
33 | const key = ip + '-' + infoHash | 34 | const key = ip + '-' + infoHash |
34 | 35 | ||
35 | peersIps[ip] = peersIps[ip] ? peersIps[ip] + 1 : 1 | 36 | peersIps[ ip ] = peersIps[ ip ] ? peersIps[ ip ] + 1 : 1 |
36 | peersIpInfoHash[key] = peersIpInfoHash[key] ? peersIpInfoHash[key] + 1 : 1 | 37 | peersIpInfoHash[ key ] = peersIpInfoHash[ key ] ? peersIpInfoHash[ key ] + 1 : 1 |
37 | 38 | ||
38 | if (peersIpInfoHash[key] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) { | 39 | if (peersIpInfoHash[ key ] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) { |
39 | return cb(new Error(`Too many requests (${peersIpInfoHash[ key ]} of ip ${ip} for torrent ${infoHash}`)) | 40 | return cb(new Error(`Too many requests (${peersIpInfoHash[ key ]} of ip ${ip} for torrent ${infoHash}`)) |
40 | } | 41 | } |
41 | 42 | ||
42 | VideoFileModel.isInfohashExists(infoHash) | 43 | try { |
43 | .then(exists => { | 44 | const videoFileExists = await VideoFileModel.doesInfohashExist(infoHash) |
44 | if (exists === false) return cb(new Error(`Unknown infoHash ${infoHash}`)) | 45 | if (videoFileExists === true) return cb() |
45 | 46 | ||
46 | return cb() | 47 | const playlistExists = await VideoStreamingPlaylistModel.doesInfohashExist(infoHash) |
47 | }) | 48 | if (playlistExists === true) return cb() |
49 | |||
50 | return cb(new Error(`Unknown infoHash ${infoHash}`)) | ||
51 | } catch (err) { | ||
52 | logger.error('Error in tracker filter.', { err }) | ||
53 | return cb(err) | ||
54 | } | ||
48 | } | 55 | } |
49 | }) | 56 | }) |
50 | 57 | ||
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index f1430055f..eba552524 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts | |||
@@ -15,7 +15,7 @@ function activityPubContextify <T> (data: T) { | |||
15 | 'https://w3id.org/security/v1', | 15 | 'https://w3id.org/security/v1', |
16 | { | 16 | { |
17 | RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', | 17 | RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', |
18 | pt: 'https://joinpeertube.org/ns', | 18 | pt: 'https://joinpeertube.org/ns#', |
19 | sc: 'http://schema.org#', | 19 | sc: 'http://schema.org#', |
20 | Hashtag: 'as:Hashtag', | 20 | Hashtag: 'as:Hashtag', |
21 | uuid: 'sc:identifier', | 21 | uuid: 'sc:identifier', |
@@ -32,7 +32,8 @@ function activityPubContextify <T> (data: T) { | |||
32 | waitTranscoding: 'sc:Boolean', | 32 | waitTranscoding: 'sc:Boolean', |
33 | expires: 'sc:expires', | 33 | expires: 'sc:expires', |
34 | support: 'sc:Text', | 34 | support: 'sc:Text', |
35 | CacheFile: 'pt:CacheFile' | 35 | CacheFile: 'pt:CacheFile', |
36 | Infohash: 'pt:Infohash' | ||
36 | }, | 37 | }, |
37 | { | 38 | { |
38 | likes: { | 39 | likes: { |
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 3fb824e36..f38b82d97 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts | |||
@@ -193,10 +193,14 @@ function peertubeTruncate (str: string, maxLength: number) { | |||
193 | return truncate(str, options) | 193 | return truncate(str, options) |
194 | } | 194 | } |
195 | 195 | ||
196 | function sha256 (str: string, encoding: HexBase64Latin1Encoding = 'hex') { | 196 | function sha256 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') { |
197 | return createHash('sha256').update(str).digest(encoding) | 197 | return createHash('sha256').update(str).digest(encoding) |
198 | } | 198 | } |
199 | 199 | ||
200 | function sha1 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') { | ||
201 | return createHash('sha1').update(str).digest(encoding) | ||
202 | } | ||
203 | |||
200 | function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> { | 204 | function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> { |
201 | return function promisified (): Promise<A> { | 205 | return function promisified (): Promise<A> { |
202 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { | 206 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { |
@@ -262,7 +266,9 @@ export { | |||
262 | sanitizeHost, | 266 | sanitizeHost, |
263 | buildPath, | 267 | buildPath, |
264 | peertubeTruncate, | 268 | peertubeTruncate, |
269 | |||
265 | sha256, | 270 | sha256, |
271 | sha1, | ||
266 | 272 | ||
267 | promisify0, | 273 | promisify0, |
268 | promisify1, | 274 | promisify1, |
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts index e2bd0c55e..21d5c53ca 100644 --- a/server/helpers/custom-validators/activitypub/cache-file.ts +++ b/server/helpers/custom-validators/activitypub/cache-file.ts | |||
@@ -8,9 +8,19 @@ function isCacheFileObjectValid (object: CacheFileObject) { | |||
8 | object.type === 'CacheFile' && | 8 | object.type === 'CacheFile' && |
9 | isDateValid(object.expires) && | 9 | isDateValid(object.expires) && |
10 | isActivityPubUrlValid(object.object) && | 10 | isActivityPubUrlValid(object.object) && |
11 | isRemoteVideoUrlValid(object.url) | 11 | (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) |
12 | } | 12 | } |
13 | 13 | ||
14 | // --------------------------------------------------------------------------- | ||
15 | |||
14 | export { | 16 | export { |
15 | isCacheFileObjectValid | 17 | isCacheFileObjectValid |
16 | } | 18 | } |
19 | |||
20 | // --------------------------------------------------------------------------- | ||
21 | |||
22 | function isPlaylistRedundancyUrlValid (url: any) { | ||
23 | return url.type === 'Link' && | ||
24 | (url.mediaType || url.mimeType) === 'application/x-mpegURL' && | ||
25 | isActivityPubUrlValid(url.href) | ||
26 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 0f34aab21..ad99c2724 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import * as validator from 'validator' | 1 | import * as validator from 'validator' |
2 | import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers' | 2 | import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers' |
3 | import { peertubeTruncate } from '../../core-utils' | 3 | import { peertubeTruncate } from '../../core-utils' |
4 | import { exists, isBooleanValid, isDateValid, isUUIDValid } from '../misc' | 4 | import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' |
5 | import { | 5 | import { |
6 | isVideoDurationValid, | 6 | isVideoDurationValid, |
7 | isVideoNameValid, | 7 | isVideoNameValid, |
@@ -12,7 +12,6 @@ import { | |||
12 | } from '../videos' | 12 | } from '../videos' |
13 | import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' | 13 | import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' |
14 | import { VideoState } from '../../../../shared/models/videos' | 14 | import { VideoState } from '../../../../shared/models/videos' |
15 | import { isVideoAbuseReasonValid } from '../video-abuses' | ||
16 | 15 | ||
17 | function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { | 16 | function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { |
18 | return isBaseActivityValid(activity, 'Update') && | 17 | return isBaseActivityValid(activity, 'Update') && |
@@ -81,6 +80,11 @@ function isRemoteVideoUrlValid (url: any) { | |||
81 | ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType || url.mimeType) !== -1 && | 80 | ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType || url.mimeType) !== -1 && |
82 | validator.isLength(url.href, { min: 5 }) && | 81 | validator.isLength(url.href, { min: 5 }) && |
83 | validator.isInt(url.height + '', { min: 0 }) | 82 | validator.isInt(url.height + '', { min: 0 }) |
83 | ) || | ||
84 | ( | ||
85 | (url.mediaType || url.mimeType) === 'application/x-mpegURL' && | ||
86 | isActivityPubUrlValid(url.href) && | ||
87 | isArray(url.tag) | ||
84 | ) | 88 | ) |
85 | } | 89 | } |
86 | 90 | ||
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index b6f0ebe6f..76647fea2 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts | |||
@@ -13,6 +13,10 @@ function isNotEmptyIntArray (value: any) { | |||
13 | return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0 | 13 | return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0 |
14 | } | 14 | } |
15 | 15 | ||
16 | function isArrayOf (value: any, validator: (value: any) => boolean) { | ||
17 | return isArray(value) && value.every(v => validator(v)) | ||
18 | } | ||
19 | |||
16 | function isDateValid (value: string) { | 20 | function isDateValid (value: string) { |
17 | return exists(value) && validator.isISO8601(value) | 21 | return exists(value) && validator.isISO8601(value) |
18 | } | 22 | } |
@@ -82,6 +86,7 @@ function isFileValid ( | |||
82 | 86 | ||
83 | export { | 87 | export { |
84 | exists, | 88 | exists, |
89 | isArrayOf, | ||
85 | isNotEmptyIntArray, | 90 | isNotEmptyIntArray, |
86 | isArray, | 91 | isArray, |
87 | isIdValid, | 92 | isIdValid, |
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index e6f22e6c5..95e256b8f 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts | |||
@@ -88,8 +88,8 @@ function isVideoFileExtnameValid (value: string) { | |||
88 | 88 | ||
89 | function isVideoFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { | 89 | function isVideoFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { |
90 | const videoFileTypesRegex = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) | 90 | const videoFileTypesRegex = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) |
91 | .map(m => `(${m})`) | 91 | .map(m => `(${m})`) |
92 | .join('|') | 92 | .join('|') |
93 | 93 | ||
94 | return isFileValid(files, videoFileTypesRegex, 'videofile', null) | 94 | return isFileValid(files, videoFileTypesRegex, 'videofile', null) |
95 | } | 95 | } |
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index c7296054d..133b1b03b 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import * as ffmpeg from 'fluent-ffmpeg' | 1 | import * as ffmpeg from 'fluent-ffmpeg' |
2 | import { join } from 'path' | 2 | import { dirname, join } from 'path' |
3 | import { getTargetBitrate, VideoResolution } from '../../shared/models/videos' | 3 | import { getTargetBitrate, VideoResolution } from '../../shared/models/videos' |
4 | import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' | 4 | import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' |
5 | import { processImage } from './image-utils' | 5 | import { processImage } from './image-utils' |
@@ -29,19 +29,28 @@ function computeResolutionsToTranscode (videoFileHeight: number) { | |||
29 | return resolutionsEnabled | 29 | return resolutionsEnabled |
30 | } | 30 | } |
31 | 31 | ||
32 | async function getVideoFileResolution (path: string) { | 32 | async function getVideoFileSize (path: string) { |
33 | const videoStream = await getVideoFileStream(path) | 33 | const videoStream = await getVideoFileStream(path) |
34 | 34 | ||
35 | return { | 35 | return { |
36 | videoFileResolution: Math.min(videoStream.height, videoStream.width), | 36 | width: videoStream.width, |
37 | isPortraitMode: videoStream.height > videoStream.width | 37 | height: videoStream.height |
38 | } | ||
39 | } | ||
40 | |||
41 | async function getVideoFileResolution (path: string) { | ||
42 | const size = await getVideoFileSize(path) | ||
43 | |||
44 | return { | ||
45 | videoFileResolution: Math.min(size.height, size.width), | ||
46 | isPortraitMode: size.height > size.width | ||
38 | } | 47 | } |
39 | } | 48 | } |
40 | 49 | ||
41 | async function getVideoFileFPS (path: string) { | 50 | async function getVideoFileFPS (path: string) { |
42 | const videoStream = await getVideoFileStream(path) | 51 | const videoStream = await getVideoFileStream(path) |
43 | 52 | ||
44 | for (const key of [ 'r_frame_rate' , 'avg_frame_rate' ]) { | 53 | for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) { |
45 | const valuesText: string = videoStream[key] | 54 | const valuesText: string = videoStream[key] |
46 | if (!valuesText) continue | 55 | if (!valuesText) continue |
47 | 56 | ||
@@ -110,8 +119,12 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima | |||
110 | type TranscodeOptions = { | 119 | type TranscodeOptions = { |
111 | inputPath: string | 120 | inputPath: string |
112 | outputPath: string | 121 | outputPath: string |
113 | resolution?: VideoResolution | 122 | resolution: VideoResolution |
114 | isPortraitMode?: boolean | 123 | isPortraitMode?: boolean |
124 | |||
125 | hlsPlaylist?: { | ||
126 | videoFilename: string | ||
127 | } | ||
115 | } | 128 | } |
116 | 129 | ||
117 | function transcode (options: TranscodeOptions) { | 130 | function transcode (options: TranscodeOptions) { |
@@ -150,6 +163,18 @@ function transcode (options: TranscodeOptions) { | |||
150 | command = command.withFPS(fps) | 163 | command = command.withFPS(fps) |
151 | } | 164 | } |
152 | 165 | ||
166 | if (options.hlsPlaylist) { | ||
167 | const videoPath = `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}` | ||
168 | |||
169 | command = command.outputOption('-hls_time 4') | ||
170 | .outputOption('-hls_list_size 0') | ||
171 | .outputOption('-hls_playlist_type vod') | ||
172 | .outputOption('-hls_segment_filename ' + videoPath) | ||
173 | .outputOption('-hls_segment_type fmp4') | ||
174 | .outputOption('-f hls') | ||
175 | .outputOption('-hls_flags single_file') | ||
176 | } | ||
177 | |||
153 | command | 178 | command |
154 | .on('error', (err, stdout, stderr) => { | 179 | .on('error', (err, stdout, stderr) => { |
155 | logger.error('Error in transcoding job.', { stdout, stderr }) | 180 | logger.error('Error in transcoding job.', { stdout, stderr }) |
@@ -166,6 +191,7 @@ function transcode (options: TranscodeOptions) { | |||
166 | // --------------------------------------------------------------------------- | 191 | // --------------------------------------------------------------------------- |
167 | 192 | ||
168 | export { | 193 | export { |
194 | getVideoFileSize, | ||
169 | getVideoFileResolution, | 195 | getVideoFileResolution, |
170 | getDurationFromVideoFile, | 196 | getDurationFromVideoFile, |
171 | generateImageFromVideoFile, | 197 | generateImageFromVideoFile, |
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index 3fc776f1a..5c6dc5e19 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts | |||
@@ -7,7 +7,7 @@ import { join } from 'path' | |||
7 | 7 | ||
8 | function doRequest <T> ( | 8 | function doRequest <T> ( |
9 | requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean } | 9 | requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean } |
10 | ): Bluebird<{ response: request.RequestResponse, body: any }> { | 10 | ): Bluebird<{ response: request.RequestResponse, body: T }> { |
11 | if (requestOptions.activityPub === true) { | 11 | if (requestOptions.activityPub === true) { |
12 | if (!Array.isArray(requestOptions.headers)) requestOptions.headers = {} | 12 | if (!Array.isArray(requestOptions.headers)) requestOptions.headers = {} |
13 | requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER | 13 | requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER |
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 3c3406e38..cb0e823c5 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts | |||
@@ -7,7 +7,6 @@ import { join } from 'path' | |||
7 | import { Instance as ParseTorrent } from 'parse-torrent' | 7 | import { Instance as ParseTorrent } from 'parse-torrent' |
8 | import { remove } from 'fs-extra' | 8 | import { remove } from 'fs-extra' |
9 | import * as memoizee from 'memoizee' | 9 | import * as memoizee from 'memoizee' |
10 | import { isArray } from './custom-validators/misc' | ||
11 | 10 | ||
12 | function deleteFileAsync (path: string) { | 11 | function deleteFileAsync (path: string) { |
13 | remove(path) | 12 | remove(path) |
diff --git a/server/helpers/video.ts b/server/helpers/video.ts index 1bd21467d..c90fe06c7 100644 --- a/server/helpers/video.ts +++ b/server/helpers/video.ts | |||
@@ -1,10 +1,12 @@ | |||
1 | import { VideoModel } from '../models/video/video' | 1 | import { VideoModel } from '../models/video/video' |
2 | 2 | ||
3 | type VideoFetchType = 'all' | 'only-video' | 'id' | 'none' | 3 | type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' |
4 | 4 | ||
5 | function fetchVideo (id: number | string, fetchType: VideoFetchType, userId?: number) { | 5 | function fetchVideo (id: number | string, fetchType: VideoFetchType, userId?: number) { |
6 | if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId) | 6 | if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId) |
7 | 7 | ||
8 | if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id) | ||
9 | |||
8 | if (fetchType === 'only-video') return VideoModel.load(id) | 10 | if (fetchType === 'only-video') return VideoModel.load(id) |
9 | 11 | ||
10 | if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) | 12 | if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) |
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 7905d9ffa..29fdb263e 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts | |||
@@ -12,7 +12,7 @@ function checkMissedConfig () { | |||
12 | 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max', | 12 | 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max', |
13 | 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', | 13 | 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', |
14 | 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', | 14 | 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', |
15 | 'storage.redundancy', 'storage.tmp', | 15 | 'storage.redundancy', 'storage.tmp', 'storage.playlists', |
16 | 'log.level', | 16 | 'log.level', |
17 | 'user.video_quota', 'user.video_quota_daily', | 17 | 'user.video_quota', 'user.video_quota_daily', |
18 | 'cache.previews.size', 'admin.email', 'contact_form.enabled', | 18 | 'cache.previews.size', 'admin.email', 'contact_form.enabled', |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 93fdd3f03..e5c4c4e63 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -16,7 +16,7 @@ let config: IConfig = require('config') | |||
16 | 16 | ||
17 | // --------------------------------------------------------------------------- | 17 | // --------------------------------------------------------------------------- |
18 | 18 | ||
19 | const LAST_MIGRATION_VERSION = 325 | 19 | const LAST_MIGRATION_VERSION = 330 |
20 | 20 | ||
21 | // --------------------------------------------------------------------------- | 21 | // --------------------------------------------------------------------------- |
22 | 22 | ||
@@ -192,6 +192,7 @@ const CONFIG = { | |||
192 | AVATARS_DIR: buildPath(config.get<string>('storage.avatars')), | 192 | AVATARS_DIR: buildPath(config.get<string>('storage.avatars')), |
193 | LOG_DIR: buildPath(config.get<string>('storage.logs')), | 193 | LOG_DIR: buildPath(config.get<string>('storage.logs')), |
194 | VIDEOS_DIR: buildPath(config.get<string>('storage.videos')), | 194 | VIDEOS_DIR: buildPath(config.get<string>('storage.videos')), |
195 | PLAYLISTS_DIR: buildPath(config.get<string>('storage.playlists')), | ||
195 | REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')), | 196 | REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')), |
196 | THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')), | 197 | THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')), |
197 | PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')), | 198 | PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')), |
@@ -259,6 +260,9 @@ const CONFIG = { | |||
259 | get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') }, | 260 | get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') }, |
260 | get '720p' () { return config.get<boolean>('transcoding.resolutions.720p') }, | 261 | get '720p' () { return config.get<boolean>('transcoding.resolutions.720p') }, |
261 | get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') } | 262 | get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') } |
263 | }, | ||
264 | HLS: { | ||
265 | get ENABLED () { return config.get<boolean>('transcoding.hls.enabled') } | ||
262 | } | 266 | } |
263 | }, | 267 | }, |
264 | IMPORT: { | 268 | IMPORT: { |
@@ -590,6 +594,9 @@ const STATIC_PATHS = { | |||
590 | TORRENTS: '/static/torrents/', | 594 | TORRENTS: '/static/torrents/', |
591 | WEBSEED: '/static/webseed/', | 595 | WEBSEED: '/static/webseed/', |
592 | REDUNDANCY: '/static/redundancy/', | 596 | REDUNDANCY: '/static/redundancy/', |
597 | PLAYLISTS: { | ||
598 | HLS: '/static/playlists/hls' | ||
599 | }, | ||
593 | AVATARS: '/static/avatars/', | 600 | AVATARS: '/static/avatars/', |
594 | VIDEO_CAPTIONS: '/static/video-captions/' | 601 | VIDEO_CAPTIONS: '/static/video-captions/' |
595 | } | 602 | } |
@@ -632,6 +639,9 @@ const CACHE = { | |||
632 | } | 639 | } |
633 | } | 640 | } |
634 | 641 | ||
642 | const HLS_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.PLAYLISTS_DIR, 'hls') | ||
643 | const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') | ||
644 | |||
635 | const MEMOIZE_TTL = { | 645 | const MEMOIZE_TTL = { |
636 | OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours | 646 | OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours |
637 | } | 647 | } |
@@ -701,6 +711,8 @@ if (isTestInstance() === true) { | |||
701 | CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 | 711 | CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 |
702 | MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1 | 712 | MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1 |
703 | ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms' | 713 | ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms' |
714 | |||
715 | RATES_LIMIT.LOGIN.MAX = 20 | ||
704 | } | 716 | } |
705 | 717 | ||
706 | updateWebserverUrls() | 718 | updateWebserverUrls() |
@@ -709,6 +721,7 @@ updateWebserverUrls() | |||
709 | 721 | ||
710 | export { | 722 | export { |
711 | API_VERSION, | 723 | API_VERSION, |
724 | HLS_REDUNDANCY_DIRECTORY, | ||
712 | AVATARS_SIZE, | 725 | AVATARS_SIZE, |
713 | ACCEPT_HEADERS, | 726 | ACCEPT_HEADERS, |
714 | BCRYPT_SALT_SIZE, | 727 | BCRYPT_SALT_SIZE, |
@@ -733,6 +746,7 @@ export { | |||
733 | PRIVATE_RSA_KEY_SIZE, | 746 | PRIVATE_RSA_KEY_SIZE, |
734 | ROUTE_CACHE_LIFETIME, | 747 | ROUTE_CACHE_LIFETIME, |
735 | SORTABLE_COLUMNS, | 748 | SORTABLE_COLUMNS, |
749 | HLS_PLAYLIST_DIRECTORY, | ||
736 | FEEDS, | 750 | FEEDS, |
737 | JOB_TTL, | 751 | JOB_TTL, |
738 | NSFW_POLICY_TYPES, | 752 | NSFW_POLICY_TYPES, |
@@ -795,7 +809,9 @@ function buildVideoMimetypeExt () { | |||
795 | 'video/quicktime': '.mov', | 809 | 'video/quicktime': '.mov', |
796 | 'video/x-msvideo': '.avi', | 810 | 'video/x-msvideo': '.avi', |
797 | 'video/x-flv': '.flv', | 811 | 'video/x-flv': '.flv', |
798 | 'video/x-matroska': '.mkv' | 812 | 'video/x-matroska': '.mkv', |
813 | 'application/octet-stream': '.mkv', | ||
814 | 'video/avi': '.avi' | ||
799 | }) | 815 | }) |
800 | } | 816 | } |
801 | 817 | ||
diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 84ad2079b..fe296142d 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts | |||
@@ -33,6 +33,7 @@ import { AccountBlocklistModel } from '../models/account/account-blocklist' | |||
33 | import { ServerBlocklistModel } from '../models/server/server-blocklist' | 33 | import { ServerBlocklistModel } from '../models/server/server-blocklist' |
34 | import { UserNotificationModel } from '../models/account/user-notification' | 34 | import { UserNotificationModel } from '../models/account/user-notification' |
35 | import { UserNotificationSettingModel } from '../models/account/user-notification-setting' | 35 | import { UserNotificationSettingModel } from '../models/account/user-notification-setting' |
36 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
36 | 37 | ||
37 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string | 38 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string |
38 | 39 | ||
@@ -99,7 +100,8 @@ async function initDatabaseModels (silent: boolean) { | |||
99 | AccountBlocklistModel, | 100 | AccountBlocklistModel, |
100 | ServerBlocklistModel, | 101 | ServerBlocklistModel, |
101 | UserNotificationModel, | 102 | UserNotificationModel, |
102 | UserNotificationSettingModel | 103 | UserNotificationSettingModel, |
104 | VideoStreamingPlaylistModel | ||
103 | ]) | 105 | ]) |
104 | 106 | ||
105 | // Check extensions exist in the database | 107 | // Check extensions exist in the database |
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index b9a9da183..2b22e16fe 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts | |||
@@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user' | |||
6 | import { ApplicationModel } from '../models/application/application' | 6 | import { ApplicationModel } from '../models/application/application' |
7 | import { OAuthClientModel } from '../models/oauth/oauth-client' | 7 | import { OAuthClientModel } from '../models/oauth/oauth-client' |
8 | import { applicationExist, clientsExist, usersExist } from './checker-after-init' | 8 | import { applicationExist, clientsExist, usersExist } from './checker-after-init' |
9 | import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants' | 9 | import { CACHE, CONFIG, HLS_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants' |
10 | import { sequelizeTypescript } from './database' | 10 | import { sequelizeTypescript } from './database' |
11 | import { remove, ensureDir } from 'fs-extra' | 11 | import { remove, ensureDir } from 'fs-extra' |
12 | 12 | ||
@@ -73,6 +73,9 @@ function createDirectoriesIfNotExist () { | |||
73 | tasks.push(ensureDir(dir)) | 73 | tasks.push(ensureDir(dir)) |
74 | } | 74 | } |
75 | 75 | ||
76 | // Playlist directories | ||
77 | tasks.push(ensureDir(HLS_PLAYLIST_DIRECTORY)) | ||
78 | |||
76 | return Promise.all(tasks) | 79 | return Promise.all(tasks) |
77 | } | 80 | } |
78 | 81 | ||
diff --git a/server/initializers/migrations/0330-video-streaming-playlist.ts b/server/initializers/migrations/0330-video-streaming-playlist.ts new file mode 100644 index 000000000..c85a762ab --- /dev/null +++ b/server/initializers/migrations/0330-video-streaming-playlist.ts | |||
@@ -0,0 +1,51 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | |||
3 | async function up (utils: { | ||
4 | transaction: Sequelize.Transaction, | ||
5 | queryInterface: Sequelize.QueryInterface, | ||
6 | sequelize: Sequelize.Sequelize | ||
7 | }): Promise<void> { | ||
8 | |||
9 | { | ||
10 | const query = ` | ||
11 | CREATE TABLE IF NOT EXISTS "videoStreamingPlaylist" | ||
12 | ( | ||
13 | "id" SERIAL, | ||
14 | "type" INTEGER NOT NULL, | ||
15 | "playlistUrl" VARCHAR(2000) NOT NULL, | ||
16 | "p2pMediaLoaderInfohashes" VARCHAR(255)[] NOT NULL, | ||
17 | "segmentsSha256Url" VARCHAR(255) NOT NULL, | ||
18 | "videoId" INTEGER NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, | ||
19 | "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, | ||
20 | "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL, | ||
21 | PRIMARY KEY ("id") | ||
22 | );` | ||
23 | await utils.sequelize.query(query) | ||
24 | } | ||
25 | |||
26 | { | ||
27 | const data = { | ||
28 | type: Sequelize.INTEGER, | ||
29 | allowNull: true, | ||
30 | defaultValue: null | ||
31 | } | ||
32 | |||
33 | await utils.queryInterface.changeColumn('videoRedundancy', 'videoFileId', data) | ||
34 | } | ||
35 | |||
36 | { | ||
37 | const query = 'ALTER TABLE "videoRedundancy" ADD COLUMN "videoStreamingPlaylistId" INTEGER NULL ' + | ||
38 | 'REFERENCES "videoStreamingPlaylist" ("id") ON DELETE CASCADE ON UPDATE CASCADE' | ||
39 | |||
40 | await utils.sequelize.query(query) | ||
41 | } | ||
42 | } | ||
43 | |||
44 | function down (options) { | ||
45 | throw new Error('Not implemented.') | ||
46 | } | ||
47 | |||
48 | export { | ||
49 | up, | ||
50 | down | ||
51 | } | ||
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 8215840da..a3f379b76 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts | |||
@@ -355,10 +355,10 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe | |||
355 | 355 | ||
356 | logger.info('Fetching remote actor %s.', actorUrl) | 356 | logger.info('Fetching remote actor %s.', actorUrl) |
357 | 357 | ||
358 | const requestResult = await doRequest(options) | 358 | const requestResult = await doRequest<ActivityPubActor>(options) |
359 | normalizeActor(requestResult.body) | 359 | normalizeActor(requestResult.body) |
360 | 360 | ||
361 | const actorJSON: ActivityPubActor = requestResult.body | 361 | const actorJSON = requestResult.body |
362 | if (isActorObjectValid(actorJSON) === false) { | 362 | if (isActorObjectValid(actorJSON) === false) { |
363 | logger.debug('Remote actor JSON is not valid.', { actorJSON }) | 363 | logger.debug('Remote actor JSON is not valid.', { actorJSON }) |
364 | return { result: undefined, statusCode: requestResult.response.statusCode } | 364 | return { result: undefined, statusCode: requestResult.response.statusCode } |
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index f6f068b45..9a40414bb 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts | |||
@@ -1,11 +1,28 @@ | |||
1 | import { CacheFileObject } from '../../../shared/index' | 1 | import { ActivityPlaylistUrlObject, ActivityVideoUrlObject, CacheFileObject } from '../../../shared/index' |
2 | import { VideoModel } from '../../models/video/video' | 2 | import { VideoModel } from '../../models/video/video' |
3 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 3 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
4 | import { Transaction } from 'sequelize' | 4 | import { Transaction } from 'sequelize' |
5 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
5 | 6 | ||
6 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { | 7 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { |
7 | const url = cacheFileObject.url | ||
8 | 8 | ||
9 | if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { | ||
10 | const url = cacheFileObject.url | ||
11 | |||
12 | const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) | ||
13 | if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) | ||
14 | |||
15 | return { | ||
16 | expiresOn: new Date(cacheFileObject.expires), | ||
17 | url: cacheFileObject.id, | ||
18 | fileUrl: url.href, | ||
19 | strategy: null, | ||
20 | videoStreamingPlaylistId: playlist.id, | ||
21 | actorId: byActor.id | ||
22 | } | ||
23 | } | ||
24 | |||
25 | const url = cacheFileObject.url | ||
9 | const videoFile = video.VideoFiles.find(f => { | 26 | const videoFile = video.VideoFiles.find(f => { |
10 | return f.resolution === url.height && f.fps === url.fps | 27 | return f.resolution === url.height && f.fps === url.fps |
11 | }) | 28 | }) |
@@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject | |||
15 | return { | 32 | return { |
16 | expiresOn: new Date(cacheFileObject.expires), | 33 | expiresOn: new Date(cacheFileObject.expires), |
17 | url: cacheFileObject.id, | 34 | url: cacheFileObject.id, |
18 | fileUrl: cacheFileObject.url.href, | 35 | fileUrl: url.href, |
19 | strategy: null, | 36 | strategy: null, |
20 | videoFileId: videoFile.id, | 37 | videoFileId: videoFile.id, |
21 | actorId: byActor.id | 38 | actorId: byActor.id |
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index 73e667ad4..ef20e404c 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts | |||
@@ -23,17 +23,14 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) { | |||
23 | return broadcastToFollowers(createActivity, byActor, [ byActor ], t) | 23 | return broadcastToFollowers(createActivity, byActor, [ byActor ], t) |
24 | } | 24 | } |
25 | 25 | ||
26 | async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { | 26 | async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) { |
27 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url) | 27 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url) |
28 | 28 | ||
29 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) | ||
30 | const redundancyObject = fileRedundancy.toActivityPubObject() | ||
31 | |||
32 | return sendVideoRelatedCreateActivity({ | 29 | return sendVideoRelatedCreateActivity({ |
33 | byActor, | 30 | byActor, |
34 | video, | 31 | video, |
35 | url: fileRedundancy.url, | 32 | url: fileRedundancy.url, |
36 | object: redundancyObject | 33 | object: fileRedundancy.toActivityPubObject() |
37 | }) | 34 | }) |
38 | } | 35 | } |
39 | 36 | ||
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index eb18a6cb6..ecbf605d6 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts | |||
@@ -73,7 +73,8 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans | |||
73 | async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { | 73 | async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { |
74 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) | 74 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) |
75 | 75 | ||
76 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | 76 | const videoId = redundancyModel.getVideo().id |
77 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
77 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) | 78 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) |
78 | 79 | ||
79 | return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) | 80 | return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) |
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index a68f03edf..839f66470 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts | |||
@@ -61,7 +61,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod | |||
61 | async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { | 61 | async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { |
62 | logger.info('Creating job to update cache file %s.', redundancyModel.url) | 62 | logger.info('Creating job to update cache file %s.', redundancyModel.url) |
63 | 63 | ||
64 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | 64 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id) |
65 | 65 | ||
66 | const activityBuilder = (audience: ActivityAudience) => { | 66 | const activityBuilder = (audience: ActivityAudience) => { |
67 | const redundancyObject = redundancyModel.toActivityPubObject() | 67 | const redundancyObject = redundancyModel.toActivityPubObject() |
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts index 38f15448c..4229fe094 100644 --- a/server/lib/activitypub/url.ts +++ b/server/lib/activitypub/url.ts | |||
@@ -5,6 +5,8 @@ import { VideoModel } from '../../models/video/video' | |||
5 | import { VideoAbuseModel } from '../../models/video/video-abuse' | 5 | import { VideoAbuseModel } from '../../models/video/video-abuse' |
6 | import { VideoCommentModel } from '../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../models/video/video-comment' |
7 | import { VideoFileModel } from '../../models/video/video-file' | 7 | import { VideoFileModel } from '../../models/video/video-file' |
8 | import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' | ||
9 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
8 | 10 | ||
9 | function getVideoActivityPubUrl (video: VideoModel) { | 11 | function getVideoActivityPubUrl (video: VideoModel) { |
10 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid | 12 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid |
@@ -16,6 +18,10 @@ function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) { | |||
16 | return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` | 18 | return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` |
17 | } | 19 | } |
18 | 20 | ||
21 | function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) { | ||
22 | return `${CONFIG.WEBSERVER.URL}/redundancy/video-playlists/${playlist.getStringType()}/${video.uuid}` | ||
23 | } | ||
24 | |||
19 | function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { | 25 | function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { |
20 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id | 26 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id |
21 | } | 27 | } |
@@ -92,6 +98,7 @@ function getUndoActivityPubUrl (originalUrl: string) { | |||
92 | 98 | ||
93 | export { | 99 | export { |
94 | getVideoActivityPubUrl, | 100 | getVideoActivityPubUrl, |
101 | getVideoCacheStreamingPlaylistActivityPubUrl, | ||
95 | getVideoChannelActivityPubUrl, | 102 | getVideoChannelActivityPubUrl, |
96 | getAccountActivityPubUrl, | 103 | getAccountActivityPubUrl, |
97 | getVideoAbuseActivityPubUrl, | 104 | getVideoAbuseActivityPubUrl, |
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index e1e523499..edd01234f 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -2,7 +2,14 @@ import * as Bluebird from 'bluebird' | |||
2 | import * as sequelize from 'sequelize' | 2 | import * as sequelize from 'sequelize' |
3 | import * as magnetUtil from 'magnet-uri' | 3 | import * as magnetUtil from 'magnet-uri' |
4 | import * as request from 'request' | 4 | import * as request from 'request' |
5 | import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' | 5 | import { |
6 | ActivityIconObject, | ||
7 | ActivityPlaylistSegmentHashesObject, | ||
8 | ActivityPlaylistUrlObject, | ||
9 | ActivityUrlObject, | ||
10 | ActivityVideoUrlObject, | ||
11 | VideoState | ||
12 | } from '../../../shared/index' | ||
6 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 13 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
7 | import { VideoPrivacy } from '../../../shared/models/videos' | 14 | import { VideoPrivacy } from '../../../shared/models/videos' |
8 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' | 15 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' |
@@ -30,6 +37,9 @@ import { AccountModel } from '../../models/account/account' | |||
30 | import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' | 37 | import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' |
31 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | 38 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
32 | import { Notifier } from '../notifier' | 39 | import { Notifier } from '../notifier' |
40 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
41 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
42 | import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' | ||
33 | 43 | ||
34 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { | 44 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { |
35 | // If the video is not private and published, we federate it | 45 | // If the video is not private and published, we federate it |
@@ -264,6 +274,25 @@ async function updateVideoFromAP (options: { | |||
264 | } | 274 | } |
265 | 275 | ||
266 | { | 276 | { |
277 | const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject) | ||
278 | const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) | ||
279 | |||
280 | // Remove video files that do not exist anymore | ||
281 | const destroyTasks = options.video.VideoStreamingPlaylists | ||
282 | .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f))) | ||
283 | .map(f => f.destroy(sequelizeOptions)) | ||
284 | await Promise.all(destroyTasks) | ||
285 | |||
286 | // Update or add other one | ||
287 | const upsertTasks = streamingPlaylistAttributes.map(a => { | ||
288 | return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t }) | ||
289 | .then(([ streamingPlaylist ]) => streamingPlaylist) | ||
290 | }) | ||
291 | |||
292 | options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks) | ||
293 | } | ||
294 | |||
295 | { | ||
267 | // Update Tags | 296 | // Update Tags |
268 | const tags = options.videoObject.tag.map(tag => tag.name) | 297 | const tags = options.videoObject.tag.map(tag => tag.name) |
269 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 298 | const tagInstances = await TagModel.findOrCreateTags(tags, t) |
@@ -367,13 +396,25 @@ export { | |||
367 | 396 | ||
368 | // --------------------------------------------------------------------------- | 397 | // --------------------------------------------------------------------------- |
369 | 398 | ||
370 | function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { | 399 | function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { |
371 | const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) | 400 | const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) |
372 | 401 | ||
373 | const urlMediaType = url.mediaType || url.mimeType | 402 | const urlMediaType = url.mediaType || url.mimeType |
374 | return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') | 403 | return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') |
375 | } | 404 | } |
376 | 405 | ||
406 | function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject { | ||
407 | const urlMediaType = url.mediaType || url.mimeType | ||
408 | |||
409 | return urlMediaType === 'application/x-mpegURL' | ||
410 | } | ||
411 | |||
412 | function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { | ||
413 | const urlMediaType = tag.mediaType || tag.mimeType | ||
414 | |||
415 | return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json' | ||
416 | } | ||
417 | |||
377 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { | 418 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { |
378 | logger.debug('Adding remote video %s.', videoObject.id) | 419 | logger.debug('Adding remote video %s.', videoObject.id) |
379 | 420 | ||
@@ -394,8 +435,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor | |||
394 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) | 435 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) |
395 | await Promise.all(videoFilePromises) | 436 | await Promise.all(videoFilePromises) |
396 | 437 | ||
438 | const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject) | ||
439 | const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t })) | ||
440 | await Promise.all(playlistPromises) | ||
441 | |||
397 | // Process tags | 442 | // Process tags |
398 | const tags = videoObject.tag.map(t => t.name) | 443 | const tags = videoObject.tag |
444 | .filter(t => t.type === 'Hashtag') | ||
445 | .map(t => t.name) | ||
399 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 446 | const tagInstances = await TagModel.findOrCreateTags(tags, t) |
400 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) | 447 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) |
401 | 448 | ||
@@ -473,13 +520,13 @@ async function videoActivityObjectToDBAttributes ( | |||
473 | } | 520 | } |
474 | 521 | ||
475 | function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { | 522 | function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { |
476 | const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] | 523 | const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] |
477 | 524 | ||
478 | if (fileUrls.length === 0) { | 525 | if (fileUrls.length === 0) { |
479 | throw new Error('Cannot find video files for ' + video.url) | 526 | throw new Error('Cannot find video files for ' + video.url) |
480 | } | 527 | } |
481 | 528 | ||
482 | const attributes: VideoFileModel[] = [] | 529 | const attributes: FilteredModelAttributes<VideoFileModel>[] = [] |
483 | for (const fileUrl of fileUrls) { | 530 | for (const fileUrl of fileUrls) { |
484 | // Fetch associated magnet uri | 531 | // Fetch associated magnet uri |
485 | const magnet = videoObject.url.find(u => { | 532 | const magnet = videoObject.url.find(u => { |
@@ -502,7 +549,45 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid | |||
502 | size: fileUrl.size, | 549 | size: fileUrl.size, |
503 | videoId: video.id, | 550 | videoId: video.id, |
504 | fps: fileUrl.fps || -1 | 551 | fps: fileUrl.fps || -1 |
505 | } as VideoFileModel | 552 | } |
553 | |||
554 | attributes.push(attribute) | ||
555 | } | ||
556 | |||
557 | return attributes | ||
558 | } | ||
559 | |||
560 | function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { | ||
561 | const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] | ||
562 | if (playlistUrls.length === 0) return [] | ||
563 | |||
564 | const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = [] | ||
565 | for (const playlistUrlObject of playlistUrls) { | ||
566 | const p2pMediaLoaderInfohashes = playlistUrlObject.tag | ||
567 | .filter(t => t.type === 'Infohash') | ||
568 | .map(t => t.name) | ||
569 | if (p2pMediaLoaderInfohashes.length === 0) { | ||
570 | logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject }) | ||
571 | continue | ||
572 | } | ||
573 | |||
574 | const segmentsSha256UrlObject = playlistUrlObject.tag | ||
575 | .find(t => { | ||
576 | return isAPPlaylistSegmentHashesUrlObject(t) | ||
577 | }) as ActivityPlaylistSegmentHashesObject | ||
578 | if (!segmentsSha256UrlObject) { | ||
579 | logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) | ||
580 | continue | ||
581 | } | ||
582 | |||
583 | const attribute = { | ||
584 | type: VideoStreamingPlaylistType.HLS, | ||
585 | playlistUrl: playlistUrlObject.href, | ||
586 | segmentsSha256Url: segmentsSha256UrlObject.href, | ||
587 | p2pMediaLoaderInfohashes, | ||
588 | videoId: video.id | ||
589 | } | ||
590 | |||
506 | attributes.push(attribute) | 591 | attributes.push(attribute) |
507 | } | 592 | } |
508 | 593 | ||
diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index f384a254e..672414cc0 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts | |||
@@ -296,9 +296,9 @@ class Emailer { | |||
296 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | 296 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) |
297 | } | 297 | } |
298 | 298 | ||
299 | addForgetPasswordEmailJob (to: string, resetPasswordUrl: string) { | 299 | addPasswordResetEmailJob (to: string, resetPasswordUrl: string) { |
300 | const text = `Hi dear user,\n\n` + | 300 | const text = `Hi dear user,\n\n` + |
301 | `It seems you forgot your password on ${CONFIG.WEBSERVER.HOST}! ` + | 301 | `A reset password procedure for your account ${to} has been requested on ${CONFIG.WEBSERVER.HOST} ` + |
302 | `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + | 302 | `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + |
303 | `If you are not the person who initiated this request, please ignore this email.\n\n` + | 303 | `If you are not the person who initiated this request, please ignore this email.\n\n` + |
304 | `Cheers,\n` + | 304 | `Cheers,\n` + |
diff --git a/server/lib/hls.ts b/server/lib/hls.ts new file mode 100644 index 000000000..3575981f4 --- /dev/null +++ b/server/lib/hls.ts | |||
@@ -0,0 +1,164 @@ | |||
1 | import { VideoModel } from '../models/video/video' | ||
2 | import { basename, join, dirname } from 'path' | ||
3 | import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers' | ||
4 | import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' | ||
5 | import { getVideoFileSize } from '../helpers/ffmpeg-utils' | ||
6 | import { sha256 } from '../helpers/core-utils' | ||
7 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
8 | import { logger } from '../helpers/logger' | ||
9 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' | ||
10 | import { generateRandomString } from '../helpers/utils' | ||
11 | import { flatten, uniq } from 'lodash' | ||
12 | |||
13 | async function updateMasterHLSPlaylist (video: VideoModel) { | ||
14 | const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) | ||
15 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] | ||
16 | const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) | ||
17 | |||
18 | for (const file of video.VideoFiles) { | ||
19 | // If we did not generated a playlist for this resolution, skip | ||
20 | const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
21 | if (await pathExists(filePlaylistPath) === false) continue | ||
22 | |||
23 | const videoFilePath = video.getVideoFilePath(file) | ||
24 | |||
25 | const size = await getVideoFileSize(videoFilePath) | ||
26 | |||
27 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | ||
28 | const resolution = `RESOLUTION=${size.width}x${size.height}` | ||
29 | |||
30 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | ||
31 | if (file.fps) line += ',FRAME-RATE=' + file.fps | ||
32 | |||
33 | masterPlaylists.push(line) | ||
34 | masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
35 | } | ||
36 | |||
37 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | ||
38 | } | ||
39 | |||
40 | async function updateSha256Segments (video: VideoModel) { | ||
41 | const json: { [filename: string]: { [range: string]: string } } = {} | ||
42 | |||
43 | const playlistDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) | ||
44 | |||
45 | // For all the resolutions available for this video | ||
46 | for (const file of video.VideoFiles) { | ||
47 | const rangeHashes: { [range: string]: string } = {} | ||
48 | |||
49 | const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)) | ||
50 | const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
51 | |||
52 | // Maybe the playlist is not generated for this resolution yet | ||
53 | if (!await pathExists(playlistPath)) continue | ||
54 | |||
55 | const playlistContent = await readFile(playlistPath) | ||
56 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | ||
57 | |||
58 | const fd = await open(videoPath, 'r') | ||
59 | for (const range of ranges) { | ||
60 | const buf = Buffer.alloc(range.length) | ||
61 | await read(fd, buf, 0, range.length, range.offset) | ||
62 | |||
63 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) | ||
64 | } | ||
65 | await close(fd) | ||
66 | |||
67 | const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution) | ||
68 | json[videoFilename] = rangeHashes | ||
69 | } | ||
70 | |||
71 | const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) | ||
72 | await outputJSON(outputPath, json) | ||
73 | } | ||
74 | |||
75 | function getRangesFromPlaylist (playlistContent: string) { | ||
76 | const ranges: { offset: number, length: number }[] = [] | ||
77 | const lines = playlistContent.split('\n') | ||
78 | const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ | ||
79 | |||
80 | for (const line of lines) { | ||
81 | const captured = regex.exec(line) | ||
82 | |||
83 | if (captured) { | ||
84 | ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) | ||
85 | } | ||
86 | } | ||
87 | |||
88 | return ranges | ||
89 | } | ||
90 | |||
91 | function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { | ||
92 | let timer | ||
93 | |||
94 | logger.info('Importing HLS playlist %s', playlistUrl) | ||
95 | |||
96 | return new Promise<string>(async (res, rej) => { | ||
97 | const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) | ||
98 | |||
99 | await ensureDir(tmpDirectory) | ||
100 | |||
101 | timer = setTimeout(() => { | ||
102 | deleteTmpDirectory(tmpDirectory) | ||
103 | |||
104 | return rej(new Error('HLS download timeout.')) | ||
105 | }, timeout) | ||
106 | |||
107 | try { | ||
108 | // Fetch master playlist | ||
109 | const subPlaylistUrls = await fetchUniqUrls(playlistUrl) | ||
110 | |||
111 | const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) | ||
112 | const fileUrls = uniq(flatten(await Promise.all(subRequests))) | ||
113 | |||
114 | logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) | ||
115 | |||
116 | for (const fileUrl of fileUrls) { | ||
117 | const destPath = join(tmpDirectory, basename(fileUrl)) | ||
118 | |||
119 | await doRequestAndSaveToFile({ uri: fileUrl }, destPath) | ||
120 | } | ||
121 | |||
122 | clearTimeout(timer) | ||
123 | |||
124 | await move(tmpDirectory, destinationDir, { overwrite: true }) | ||
125 | |||
126 | return res() | ||
127 | } catch (err) { | ||
128 | deleteTmpDirectory(tmpDirectory) | ||
129 | |||
130 | return rej(err) | ||
131 | } | ||
132 | }) | ||
133 | |||
134 | function deleteTmpDirectory (directory: string) { | ||
135 | remove(directory) | ||
136 | .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) | ||
137 | } | ||
138 | |||
139 | async function fetchUniqUrls (playlistUrl: string) { | ||
140 | const { body } = await doRequest<string>({ uri: playlistUrl }) | ||
141 | |||
142 | if (!body) return [] | ||
143 | |||
144 | const urls = body.split('\n') | ||
145 | .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) | ||
146 | .map(url => { | ||
147 | if (url.startsWith('http://') || url.startsWith('https://')) return url | ||
148 | |||
149 | return `${dirname(playlistUrl)}/${url}` | ||
150 | }) | ||
151 | |||
152 | return uniq(urls) | ||
153 | } | ||
154 | } | ||
155 | |||
156 | // --------------------------------------------------------------------------- | ||
157 | |||
158 | export { | ||
159 | updateMasterHLSPlaylist, | ||
160 | updateSha256Segments, | ||
161 | downloadPlaylistSegments | ||
162 | } | ||
163 | |||
164 | // --------------------------------------------------------------------------- | ||
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 593e43cc5..04983155c 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts | |||
@@ -5,17 +5,18 @@ import { VideoModel } from '../../../models/video/video' | |||
5 | import { JobQueue } from '../job-queue' | 5 | import { JobQueue } from '../job-queue' |
6 | import { federateVideoIfNeeded } from '../../activitypub' | 6 | import { federateVideoIfNeeded } from '../../activitypub' |
7 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 7 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
8 | import { sequelizeTypescript } from '../../../initializers' | 8 | import { sequelizeTypescript, CONFIG } from '../../../initializers' |
9 | import * as Bluebird from 'bluebird' | 9 | import * as Bluebird from 'bluebird' |
10 | import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' | 10 | import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' |
11 | import { importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' | 11 | import { generateHlsPlaylist, importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' |
12 | import { Notifier } from '../../notifier' | 12 | import { Notifier } from '../../notifier' |
13 | 13 | ||
14 | export type VideoFilePayload = { | 14 | export type VideoFilePayload = { |
15 | videoUUID: string | 15 | videoUUID: string |
16 | isNewVideo?: boolean | ||
17 | resolution?: VideoResolution | 16 | resolution?: VideoResolution |
17 | isNewVideo?: boolean | ||
18 | isPortraitMode?: boolean | 18 | isPortraitMode?: boolean |
19 | generateHlsPlaylist?: boolean | ||
19 | } | 20 | } |
20 | 21 | ||
21 | export type VideoFileImportPayload = { | 22 | export type VideoFileImportPayload = { |
@@ -51,21 +52,38 @@ async function processVideoFile (job: Bull.Job) { | |||
51 | return undefined | 52 | return undefined |
52 | } | 53 | } |
53 | 54 | ||
54 | // Transcoding in other resolution | 55 | if (payload.generateHlsPlaylist) { |
55 | if (payload.resolution) { | 56 | await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false) |
57 | |||
58 | await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video) | ||
59 | } else if (payload.resolution) { // Transcoding in other resolution | ||
56 | await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) | 60 | await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) |
57 | 61 | ||
58 | await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) | 62 | await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video, payload) |
59 | } else { | 63 | } else { |
60 | await optimizeVideofile(video) | 64 | await optimizeVideofile(video) |
61 | 65 | ||
62 | await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) | 66 | await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload) |
63 | } | 67 | } |
64 | 68 | ||
65 | return video | 69 | return video |
66 | } | 70 | } |
67 | 71 | ||
68 | async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { | 72 | async function onHlsPlaylistGenerationSuccess (video: VideoModel) { |
73 | if (video === undefined) return undefined | ||
74 | |||
75 | await sequelizeTypescript.transaction(async t => { | ||
76 | // Maybe the video changed in database, refresh it | ||
77 | let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | ||
78 | // Video does not exist anymore | ||
79 | if (!videoDatabase) return undefined | ||
80 | |||
81 | // If the video was not published, we consider it is a new one for other instances | ||
82 | await federateVideoIfNeeded(videoDatabase, false, t) | ||
83 | }) | ||
84 | } | ||
85 | |||
86 | async function onVideoFileTranscoderOrImportSuccess (video: VideoModel, payload?: VideoFilePayload) { | ||
69 | if (video === undefined) return undefined | 87 | if (video === undefined) return undefined |
70 | 88 | ||
71 | const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { | 89 | const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { |
@@ -91,13 +109,16 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { | |||
91 | return { videoDatabase, videoPublished } | 109 | return { videoDatabase, videoPublished } |
92 | }) | 110 | }) |
93 | 111 | ||
94 | if (videoPublished) { | 112 | // don't notify prior to scheduled video update |
113 | if (videoPublished && !videoDatabase.ScheduleVideoUpdate) { | ||
95 | Notifier.Instance.notifyOnNewVideo(videoDatabase) | 114 | Notifier.Instance.notifyOnNewVideo(videoDatabase) |
96 | Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) | 115 | Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) |
97 | } | 116 | } |
117 | |||
118 | await createHlsJobIfEnabled(payload) | ||
98 | } | 119 | } |
99 | 120 | ||
100 | async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) { | 121 | async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoFilePayload) { |
101 | if (videoArg === undefined) return undefined | 122 | if (videoArg === undefined) return undefined |
102 | 123 | ||
103 | // Outside the transaction (IO on disk) | 124 | // Outside the transaction (IO on disk) |
@@ -144,13 +165,18 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo | |||
144 | logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) | 165 | logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) |
145 | } | 166 | } |
146 | 167 | ||
147 | await federateVideoIfNeeded(videoDatabase, isNewVideo, t) | 168 | await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t) |
148 | 169 | ||
149 | return { videoDatabase, videoPublished } | 170 | return { videoDatabase, videoPublished } |
150 | }) | 171 | }) |
151 | 172 | ||
152 | if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) | 173 | // don't notify prior to scheduled video update |
153 | if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) | 174 | if (!videoDatabase.ScheduleVideoUpdate) { |
175 | if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) | ||
176 | if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) | ||
177 | } | ||
178 | |||
179 | await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution })) | ||
154 | } | 180 | } |
155 | 181 | ||
156 | // --------------------------------------------------------------------------- | 182 | // --------------------------------------------------------------------------- |
@@ -159,3 +185,20 @@ export { | |||
159 | processVideoFile, | 185 | processVideoFile, |
160 | processVideoFileImport | 186 | processVideoFileImport |
161 | } | 187 | } |
188 | |||
189 | // --------------------------------------------------------------------------- | ||
190 | |||
191 | function createHlsJobIfEnabled (payload?: VideoFilePayload) { | ||
192 | // Generate HLS playlist? | ||
193 | if (payload && CONFIG.TRANSCODING.HLS.ENABLED) { | ||
194 | const hlsTranscodingPayload = { | ||
195 | videoUUID: payload.videoUUID, | ||
196 | resolution: payload.resolution, | ||
197 | isPortraitMode: payload.isPortraitMode, | ||
198 | |||
199 | generateHlsPlaylist: true | ||
200 | } | ||
201 | |||
202 | return JobQueue.Instance.createJob({ type: 'video-file', payload: hlsTranscodingPayload }) | ||
203 | } | ||
204 | } | ||
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index f643ee226..1a48f2bd0 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { AbstractScheduler } from './abstract-scheduler' | 1 | import { AbstractScheduler } from './abstract-scheduler' |
2 | import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' | 2 | import { CONFIG, HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { VideosRedundancy } from '../../../shared/models/redundancy' | 4 | import { VideosRedundancy } from '../../../shared/models/redundancy' |
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
@@ -9,9 +9,19 @@ import { join } from 'path' | |||
9 | import { move } from 'fs-extra' | 9 | import { move } from 'fs-extra' |
10 | import { getServerActor } from '../../helpers/utils' | 10 | import { getServerActor } from '../../helpers/utils' |
11 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | 11 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' |
12 | import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' | 12 | import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' |
13 | import { removeVideoRedundancy } from '../redundancy' | 13 | import { removeVideoRedundancy } from '../redundancy' |
14 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' | 14 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' |
15 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
16 | import { VideoModel } from '../../models/video/video' | ||
17 | import { downloadPlaylistSegments } from '../hls' | ||
18 | |||
19 | type CandidateToDuplicate = { | ||
20 | redundancy: VideosRedundancy, | ||
21 | video: VideoModel, | ||
22 | files: VideoFileModel[], | ||
23 | streamingPlaylists: VideoStreamingPlaylistModel[] | ||
24 | } | ||
15 | 25 | ||
16 | export class VideosRedundancyScheduler extends AbstractScheduler { | 26 | export class VideosRedundancyScheduler extends AbstractScheduler { |
17 | 27 | ||
@@ -24,28 +34,32 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
24 | } | 34 | } |
25 | 35 | ||
26 | protected async internalExecute () { | 36 | protected async internalExecute () { |
27 | for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { | 37 | for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { |
28 | logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) | 38 | logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) |
29 | 39 | ||
30 | try { | 40 | try { |
31 | const videoToDuplicate = await this.findVideoToDuplicate(obj) | 41 | const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig) |
32 | if (!videoToDuplicate) continue | 42 | if (!videoToDuplicate) continue |
33 | 43 | ||
34 | const videoFiles = videoToDuplicate.VideoFiles | 44 | const candidateToDuplicate = { |
35 | videoFiles.forEach(f => f.Video = videoToDuplicate) | 45 | video: videoToDuplicate, |
46 | redundancy: redundancyConfig, | ||
47 | files: videoToDuplicate.VideoFiles, | ||
48 | streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists | ||
49 | } | ||
36 | 50 | ||
37 | await this.purgeCacheIfNeeded(obj, videoFiles) | 51 | await this.purgeCacheIfNeeded(candidateToDuplicate) |
38 | 52 | ||
39 | if (await this.isTooHeavy(obj, videoFiles)) { | 53 | if (await this.isTooHeavy(candidateToDuplicate)) { |
40 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) | 54 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) |
41 | continue | 55 | continue |
42 | } | 56 | } |
43 | 57 | ||
44 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) | 58 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy) |
45 | 59 | ||
46 | await this.createVideoRedundancy(obj, videoFiles) | 60 | await this.createVideoRedundancies(candidateToDuplicate) |
47 | } catch (err) { | 61 | } catch (err) { |
48 | logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) | 62 | logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err }) |
49 | } | 63 | } |
50 | } | 64 | } |
51 | 65 | ||
@@ -63,25 +77,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
63 | 77 | ||
64 | for (const redundancyModel of expired) { | 78 | for (const redundancyModel of expired) { |
65 | try { | 79 | try { |
66 | await this.extendsOrDeleteRedundancy(redundancyModel) | 80 | const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) |
81 | const candidate = { | ||
82 | redundancy: redundancyConfig, | ||
83 | video: null, | ||
84 | files: [], | ||
85 | streamingPlaylists: [] | ||
86 | } | ||
87 | |||
88 | // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it | ||
89 | if (!redundancyConfig || await this.isTooHeavy(candidate)) { | ||
90 | logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) | ||
91 | await removeVideoRedundancy(redundancyModel) | ||
92 | } else { | ||
93 | await this.extendsRedundancy(redundancyModel) | ||
94 | } | ||
67 | } catch (err) { | 95 | } catch (err) { |
68 | logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel)) | 96 | logger.error( |
97 | 'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel), | ||
98 | { err } | ||
99 | ) | ||
69 | } | 100 | } |
70 | } | 101 | } |
71 | } | 102 | } |
72 | 103 | ||
73 | private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) { | 104 | private async extendsRedundancy (redundancyModel: VideoRedundancyModel) { |
74 | // Refresh the video, maybe it was deleted | ||
75 | const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url) | ||
76 | |||
77 | if (!video) { | ||
78 | logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url) | ||
79 | |||
80 | await redundancyModel.destroy() | ||
81 | return | ||
82 | } | ||
83 | |||
84 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) | 105 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) |
106 | // Redundancy strategy disabled, remove our redundancy instead of extending expiration | ||
107 | if (!redundancy) await removeVideoRedundancy(redundancyModel) | ||
108 | |||
85 | await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) | 109 | await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) |
86 | } | 110 | } |
87 | 111 | ||
@@ -112,49 +136,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
112 | } | 136 | } |
113 | } | 137 | } |
114 | 138 | ||
115 | private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 139 | private async createVideoRedundancies (data: CandidateToDuplicate) { |
116 | const serverActor = await getServerActor() | 140 | const video = await this.loadAndRefreshVideo(data.video.url) |
141 | |||
142 | if (!video) { | ||
143 | logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url) | ||
117 | 144 | ||
118 | for (const file of filesToDuplicate) { | 145 | return |
119 | const video = await this.loadAndRefreshVideo(file.Video.url) | 146 | } |
120 | 147 | ||
148 | for (const file of data.files) { | ||
121 | const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) | 149 | const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) |
122 | if (existingRedundancy) { | 150 | if (existingRedundancy) { |
123 | await this.extendsOrDeleteRedundancy(existingRedundancy) | 151 | await this.extendsRedundancy(existingRedundancy) |
124 | 152 | ||
125 | continue | 153 | continue |
126 | } | 154 | } |
127 | 155 | ||
128 | if (!video) { | 156 | await this.createVideoFileRedundancy(data.redundancy, video, file) |
129 | logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url) | 157 | } |
158 | |||
159 | for (const streamingPlaylist of data.streamingPlaylists) { | ||
160 | const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id) | ||
161 | if (existingRedundancy) { | ||
162 | await this.extendsRedundancy(existingRedundancy) | ||
130 | 163 | ||
131 | continue | 164 | continue |
132 | } | 165 | } |
133 | 166 | ||
134 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) | 167 | await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist) |
168 | } | ||
169 | } | ||
135 | 170 | ||
136 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | 171 | private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) { |
137 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) | 172 | file.Video = video |
138 | 173 | ||
139 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) | 174 | const serverActor = await getServerActor() |
140 | 175 | ||
141 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) | 176 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) |
142 | await move(tmpPath, destPath) | ||
143 | 177 | ||
144 | const createdModel = await VideoRedundancyModel.create({ | 178 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() |
145 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | 179 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) |
146 | url: getVideoCacheFileActivityPubUrl(file), | ||
147 | fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), | ||
148 | strategy: redundancy.strategy, | ||
149 | videoFileId: file.id, | ||
150 | actorId: serverActor.id | ||
151 | }) | ||
152 | createdModel.VideoFile = file | ||
153 | 180 | ||
154 | await sendCreateCacheFile(serverActor, createdModel) | 181 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) |
155 | 182 | ||
156 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) | 183 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) |
157 | } | 184 | await move(tmpPath, destPath) |
185 | |||
186 | const createdModel = await VideoRedundancyModel.create({ | ||
187 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | ||
188 | url: getVideoCacheFileActivityPubUrl(file), | ||
189 | fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), | ||
190 | strategy: redundancy.strategy, | ||
191 | videoFileId: file.id, | ||
192 | actorId: serverActor.id | ||
193 | }) | ||
194 | |||
195 | createdModel.VideoFile = file | ||
196 | |||
197 | await sendCreateCacheFile(serverActor, video, createdModel) | ||
198 | |||
199 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) | ||
200 | } | ||
201 | |||
202 | private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) { | ||
203 | playlist.Video = video | ||
204 | |||
205 | const serverActor = await getServerActor() | ||
206 | |||
207 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) | ||
208 | |||
209 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | ||
210 | await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) | ||
211 | |||
212 | const createdModel = await VideoRedundancyModel.create({ | ||
213 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | ||
214 | url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), | ||
215 | fileUrl: playlist.getVideoRedundancyUrl(CONFIG.WEBSERVER.URL), | ||
216 | strategy: redundancy.strategy, | ||
217 | videoStreamingPlaylistId: playlist.id, | ||
218 | actorId: serverActor.id | ||
219 | }) | ||
220 | |||
221 | createdModel.VideoStreamingPlaylist = playlist | ||
222 | |||
223 | await sendCreateCacheFile(serverActor, video, createdModel) | ||
224 | |||
225 | logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) | ||
158 | } | 226 | } |
159 | 227 | ||
160 | private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { | 228 | private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { |
@@ -168,8 +236,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
168 | await sendUpdateCacheFile(serverActor, redundancy) | 236 | await sendUpdateCacheFile(serverActor, redundancy) |
169 | } | 237 | } |
170 | 238 | ||
171 | private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 239 | private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { |
172 | while (this.isTooHeavy(redundancy, filesToDuplicate)) { | 240 | while (this.isTooHeavy(candidateToDuplicate)) { |
241 | const redundancy = candidateToDuplicate.redundancy | ||
173 | const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) | 242 | const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) |
174 | if (!toDelete) return | 243 | if (!toDelete) return |
175 | 244 | ||
@@ -177,11 +246,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
177 | } | 246 | } |
178 | } | 247 | } |
179 | 248 | ||
180 | private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 249 | private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { |
181 | const maxSize = redundancy.size | 250 | const maxSize = candidateToDuplicate.redundancy.size |
182 | 251 | ||
183 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) | 252 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy) |
184 | const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate) | 253 | const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) |
185 | 254 | ||
186 | return totalWillDuplicate > maxSize | 255 | return totalWillDuplicate > maxSize |
187 | } | 256 | } |
@@ -191,13 +260,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
191 | } | 260 | } |
192 | 261 | ||
193 | private buildEntryLogId (object: VideoRedundancyModel) { | 262 | private buildEntryLogId (object: VideoRedundancyModel) { |
194 | return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` | 263 | if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` |
264 | |||
265 | return `${object.VideoStreamingPlaylist.playlistUrl}` | ||
195 | } | 266 | } |
196 | 267 | ||
197 | private getTotalFileSizes (files: VideoFileModel[]) { | 268 | private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) { |
198 | const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size | 269 | const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size |
199 | 270 | ||
200 | return files.reduce(fileReducer, 0) | 271 | return files.reduce(fileReducer, 0) * playlists.length |
201 | } | 272 | } |
202 | 273 | ||
203 | private async loadAndRefreshVideo (videoUrl: string) { | 274 | private async loadAndRefreshVideo (videoUrl: string) { |
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index 4460f46e4..086b860a2 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts | |||
@@ -1,11 +1,14 @@ | |||
1 | import { CONFIG } from '../initializers' | 1 | import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers' |
2 | import { extname, join } from 'path' | 2 | import { extname, join } from 'path' |
3 | import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' | 3 | import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' |
4 | import { copy, remove, move, stat } from 'fs-extra' | 4 | import { copy, ensureDir, move, remove, stat } from 'fs-extra' |
5 | import { logger } from '../helpers/logger' | 5 | import { logger } from '../helpers/logger' |
6 | import { VideoResolution } from '../../shared/models/videos' | 6 | import { VideoResolution } from '../../shared/models/videos' |
7 | import { VideoFileModel } from '../models/video/video-file' | 7 | import { VideoFileModel } from '../models/video/video-file' |
8 | import { VideoModel } from '../models/video/video' | 8 | import { VideoModel } from '../models/video/video' |
9 | import { updateMasterHLSPlaylist, updateSha256Segments } from './hls' | ||
10 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
11 | import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type' | ||
9 | 12 | ||
10 | async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { | 13 | async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { |
11 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | 14 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR |
@@ -17,7 +20,8 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi | |||
17 | 20 | ||
18 | const transcodeOptions = { | 21 | const transcodeOptions = { |
19 | inputPath: videoInputPath, | 22 | inputPath: videoInputPath, |
20 | outputPath: videoTranscodedPath | 23 | outputPath: videoTranscodedPath, |
24 | resolution: inputVideoFile.resolution | ||
21 | } | 25 | } |
22 | 26 | ||
23 | // Could be very long! | 27 | // Could be very long! |
@@ -47,7 +51,7 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi | |||
47 | } | 51 | } |
48 | } | 52 | } |
49 | 53 | ||
50 | async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { | 54 | async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) { |
51 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | 55 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR |
52 | const extname = '.mp4' | 56 | const extname = '.mp4' |
53 | 57 | ||
@@ -60,13 +64,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR | |||
60 | size: 0, | 64 | size: 0, |
61 | videoId: video.id | 65 | videoId: video.id |
62 | }) | 66 | }) |
63 | const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) | 67 | const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile)) |
64 | 68 | ||
65 | const transcodeOptions = { | 69 | const transcodeOptions = { |
66 | inputPath: videoInputPath, | 70 | inputPath: videoInputPath, |
67 | outputPath: videoOutputPath, | 71 | outputPath: videoOutputPath, |
68 | resolution, | 72 | resolution, |
69 | isPortraitMode | 73 | isPortraitMode: isPortrait |
70 | } | 74 | } |
71 | 75 | ||
72 | await transcode(transcodeOptions) | 76 | await transcode(transcodeOptions) |
@@ -84,6 +88,41 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR | |||
84 | video.VideoFiles.push(newVideoFile) | 88 | video.VideoFiles.push(newVideoFile) |
85 | } | 89 | } |
86 | 90 | ||
91 | async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { | ||
92 | const baseHlsDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) | ||
93 | await ensureDir(join(HLS_PLAYLIST_DIRECTORY, video.uuid)) | ||
94 | |||
95 | const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile())) | ||
96 | const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) | ||
97 | |||
98 | const transcodeOptions = { | ||
99 | inputPath: videoInputPath, | ||
100 | outputPath, | ||
101 | resolution, | ||
102 | isPortraitMode, | ||
103 | |||
104 | hlsPlaylist: { | ||
105 | videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution) | ||
106 | } | ||
107 | } | ||
108 | |||
109 | await transcode(transcodeOptions) | ||
110 | |||
111 | await updateMasterHLSPlaylist(video) | ||
112 | await updateSha256Segments(video) | ||
113 | |||
114 | const playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) | ||
115 | |||
116 | await VideoStreamingPlaylistModel.upsert({ | ||
117 | videoId: video.id, | ||
118 | playlistUrl, | ||
119 | segmentsSha256Url: CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid), | ||
120 | p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles), | ||
121 | |||
122 | type: VideoStreamingPlaylistType.HLS | ||
123 | }) | ||
124 | } | ||
125 | |||
87 | async function importVideoFile (video: VideoModel, inputFilePath: string) { | 126 | async function importVideoFile (video: VideoModel, inputFilePath: string) { |
88 | const { videoFileResolution } = await getVideoFileResolution(inputFilePath) | 127 | const { videoFileResolution } = await getVideoFileResolution(inputFilePath) |
89 | const { size } = await stat(inputFilePath) | 128 | const { size } = await stat(inputFilePath) |
@@ -125,6 +164,7 @@ async function importVideoFile (video: VideoModel, inputFilePath: string) { | |||
125 | } | 164 | } |
126 | 165 | ||
127 | export { | 166 | export { |
167 | generateHlsPlaylist, | ||
128 | optimizeVideofile, | 168 | optimizeVideofile, |
129 | transcodeOriginalVideofile, | 169 | transcodeOriginalVideofile, |
130 | importVideoFile | 170 | importVideoFile |
diff --git a/server/middlewares/csp.ts b/server/middlewares/csp.ts index 8b919af0d..5fa9d1ab5 100644 --- a/server/middlewares/csp.ts +++ b/server/middlewares/csp.ts | |||
@@ -16,7 +16,7 @@ const baseDirectives = Object.assign({}, | |||
16 | baseUri: ["'self'"], | 16 | baseUri: ["'self'"], |
17 | manifestSrc: ["'self'"], | 17 | manifestSrc: ["'self'"], |
18 | frameSrc: ["'self'"], // instead of deprecated child-src / self because of test-embed | 18 | frameSrc: ["'self'"], // instead of deprecated child-src / self because of test-embed |
19 | workerSrc: ["'self'"] // instead of deprecated child-src | 19 | workerSrc: ["'self'", 'blob:'] // instead of deprecated child-src |
20 | }, | 20 | }, |
21 | CONFIG.SERVICES['CSP-LOGGER'] ? { reportUri: CONFIG.SERVICES['CSP-LOGGER'] } : {}, | 21 | CONFIG.SERVICES['CSP-LOGGER'] ? { reportUri: CONFIG.SERVICES['CSP-LOGGER'] } : {}, |
22 | CONFIG.WEBSERVER.SCHEME === 'https' ? { upgradeInsecureRequests: true } : {} | 22 | CONFIG.WEBSERVER.SCHEME === 'https' ? { upgradeInsecureRequests: true } : {} |
diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts index c72ab78b2..329322509 100644 --- a/server/middlewares/validators/redundancy.ts +++ b/server/middlewares/validators/redundancy.ts | |||
@@ -13,7 +13,7 @@ import { ActorFollowModel } from '../../models/activitypub/actor-follow' | |||
13 | import { SERVER_ACTOR_NAME } from '../../initializers' | 13 | import { SERVER_ACTOR_NAME } from '../../initializers' |
14 | import { ServerModel } from '../../models/server/server' | 14 | import { ServerModel } from '../../models/server/server' |
15 | 15 | ||
16 | const videoRedundancyGetValidator = [ | 16 | const videoFileRedundancyGetValidator = [ |
17 | param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), | 17 | param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), |
18 | param('resolution') | 18 | param('resolution') |
19 | .customSanitizer(toIntOrNull) | 19 | .customSanitizer(toIntOrNull) |
@@ -24,7 +24,7 @@ const videoRedundancyGetValidator = [ | |||
24 | .custom(exists).withMessage('Should have a valid fps'), | 24 | .custom(exists).withMessage('Should have a valid fps'), |
25 | 25 | ||
26 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | 26 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { |
27 | logger.debug('Checking videoRedundancyGetValidator parameters', { parameters: req.params }) | 27 | logger.debug('Checking videoFileRedundancyGetValidator parameters', { parameters: req.params }) |
28 | 28 | ||
29 | if (areValidationErrors(req, res)) return | 29 | if (areValidationErrors(req, res)) return |
30 | if (!await isVideoExist(req.params.videoId, res)) return | 30 | if (!await isVideoExist(req.params.videoId, res)) return |
@@ -38,7 +38,31 @@ const videoRedundancyGetValidator = [ | |||
38 | res.locals.videoFile = videoFile | 38 | res.locals.videoFile = videoFile |
39 | 39 | ||
40 | const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) | 40 | const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) |
41 | if (!videoRedundancy)return res.status(404).json({ error: 'Video redundancy not found.' }) | 41 | if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' }) |
42 | res.locals.videoRedundancy = videoRedundancy | ||
43 | |||
44 | return next() | ||
45 | } | ||
46 | ] | ||
47 | |||
48 | const videoPlaylistRedundancyGetValidator = [ | ||
49 | param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), | ||
50 | param('streamingPlaylistType').custom(exists).withMessage('Should have a valid streaming playlist type'), | ||
51 | |||
52 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
53 | logger.debug('Checking videoPlaylistRedundancyGetValidator parameters', { parameters: req.params }) | ||
54 | |||
55 | if (areValidationErrors(req, res)) return | ||
56 | if (!await isVideoExist(req.params.videoId, res)) return | ||
57 | |||
58 | const video: VideoModel = res.locals.video | ||
59 | const videoStreamingPlaylist = video.VideoStreamingPlaylists.find(p => p === req.params.streamingPlaylistType) | ||
60 | |||
61 | if (!videoStreamingPlaylist) return res.status(404).json({ error: 'Video playlist not found.' }) | ||
62 | res.locals.videoStreamingPlaylist = videoStreamingPlaylist | ||
63 | |||
64 | const videoRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(videoStreamingPlaylist.id) | ||
65 | if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' }) | ||
42 | res.locals.videoRedundancy = videoRedundancy | 66 | res.locals.videoRedundancy = videoRedundancy |
43 | 67 | ||
44 | return next() | 68 | return next() |
@@ -75,6 +99,7 @@ const updateServerRedundancyValidator = [ | |||
75 | // --------------------------------------------------------------------------- | 99 | // --------------------------------------------------------------------------- |
76 | 100 | ||
77 | export { | 101 | export { |
78 | videoRedundancyGetValidator, | 102 | videoFileRedundancyGetValidator, |
103 | videoPlaylistRedundancyGetValidator, | ||
79 | updateServerRedundancyValidator | 104 | updateServerRedundancyValidator |
80 | } | 105 | } |
diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts index 1bb0bfb1b..a52e3060a 100644 --- a/server/middlewares/validators/users.ts +++ b/server/middlewares/validators/users.ts | |||
@@ -113,6 +113,7 @@ const deleteMeValidator = [ | |||
113 | 113 | ||
114 | const usersUpdateValidator = [ | 114 | const usersUpdateValidator = [ |
115 | param('id').isInt().not().isEmpty().withMessage('Should have a valid id'), | 115 | param('id').isInt().not().isEmpty().withMessage('Should have a valid id'), |
116 | body('password').optional().custom(isUserPasswordValid).withMessage('Should have a valid password'), | ||
116 | body('email').optional().isEmail().withMessage('Should have a valid email attribute'), | 117 | body('email').optional().isEmail().withMessage('Should have a valid email attribute'), |
117 | body('emailVerified').optional().isBoolean().withMessage('Should have a valid email verified attribute'), | 118 | body('emailVerified').optional().isBoolean().withMessage('Should have a valid email verified attribute'), |
118 | body('videoQuota').optional().custom(isUserVideoQuotaValid).withMessage('Should have a valid user quota'), | 119 | body('videoQuota').optional().custom(isUserVideoQuotaValid).withMessage('Should have a valid user quota'), |
@@ -233,6 +234,7 @@ const usersAskResetPasswordValidator = [ | |||
233 | logger.debug('Checking usersAskResetPassword parameters', { parameters: req.body }) | 234 | logger.debug('Checking usersAskResetPassword parameters', { parameters: req.body }) |
234 | 235 | ||
235 | if (areValidationErrors(req, res)) return | 236 | if (areValidationErrors(req, res)) return |
237 | |||
236 | const exists = await checkUserEmailExist(req.body.email, res, false) | 238 | const exists = await checkUserEmailExist(req.body.email, res, false) |
237 | if (!exists) { | 239 | if (!exists) { |
238 | logger.debug('User with email %s does not exist (asking reset password).', req.body.email) | 240 | logger.debug('User with email %s does not exist (asking reset password).', req.body.email) |
diff --git a/server/models/account/user-notification.ts b/server/models/account/user-notification.ts index 9e4f982a3..6cdbb827b 100644 --- a/server/models/account/user-notification.ts +++ b/server/models/account/user-notification.ts | |||
@@ -27,11 +27,33 @@ import { VideoBlacklistModel } from '../video/video-blacklist' | |||
27 | import { VideoImportModel } from '../video/video-import' | 27 | import { VideoImportModel } from '../video/video-import' |
28 | import { ActorModel } from '../activitypub/actor' | 28 | import { ActorModel } from '../activitypub/actor' |
29 | import { ActorFollowModel } from '../activitypub/actor-follow' | 29 | import { ActorFollowModel } from '../activitypub/actor-follow' |
30 | import { AvatarModel } from '../avatar/avatar' | ||
31 | import { ServerModel } from '../server/server' | ||
30 | 32 | ||
31 | enum ScopeNames { | 33 | enum ScopeNames { |
32 | WITH_ALL = 'WITH_ALL' | 34 | WITH_ALL = 'WITH_ALL' |
33 | } | 35 | } |
34 | 36 | ||
37 | function buildActorWithAvatarInclude () { | ||
38 | return { | ||
39 | attributes: [ 'preferredUsername' ], | ||
40 | model: () => ActorModel.unscoped(), | ||
41 | required: true, | ||
42 | include: [ | ||
43 | { | ||
44 | attributes: [ 'filename' ], | ||
45 | model: () => AvatarModel.unscoped(), | ||
46 | required: false | ||
47 | }, | ||
48 | { | ||
49 | attributes: [ 'host' ], | ||
50 | model: () => ServerModel.unscoped(), | ||
51 | required: false | ||
52 | } | ||
53 | ] | ||
54 | } | ||
55 | } | ||
56 | |||
35 | function buildVideoInclude (required: boolean) { | 57 | function buildVideoInclude (required: boolean) { |
36 | return { | 58 | return { |
37 | attributes: [ 'id', 'uuid', 'name' ], | 59 | attributes: [ 'id', 'uuid', 'name' ], |
@@ -40,19 +62,21 @@ function buildVideoInclude (required: boolean) { | |||
40 | } | 62 | } |
41 | } | 63 | } |
42 | 64 | ||
43 | function buildChannelInclude (required: boolean) { | 65 | function buildChannelInclude (required: boolean, withActor = false) { |
44 | return { | 66 | return { |
45 | required, | 67 | required, |
46 | attributes: [ 'id', 'name' ], | 68 | attributes: [ 'id', 'name' ], |
47 | model: () => VideoChannelModel.unscoped() | 69 | model: () => VideoChannelModel.unscoped(), |
70 | include: withActor === true ? [ buildActorWithAvatarInclude() ] : [] | ||
48 | } | 71 | } |
49 | } | 72 | } |
50 | 73 | ||
51 | function buildAccountInclude (required: boolean) { | 74 | function buildAccountInclude (required: boolean, withActor = false) { |
52 | return { | 75 | return { |
53 | required, | 76 | required, |
54 | attributes: [ 'id', 'name' ], | 77 | attributes: [ 'id', 'name' ], |
55 | model: () => AccountModel.unscoped() | 78 | model: () => AccountModel.unscoped(), |
79 | include: withActor === true ? [ buildActorWithAvatarInclude() ] : [] | ||
56 | } | 80 | } |
57 | } | 81 | } |
58 | 82 | ||
@@ -60,47 +84,40 @@ function buildAccountInclude (required: boolean) { | |||
60 | [ScopeNames.WITH_ALL]: { | 84 | [ScopeNames.WITH_ALL]: { |
61 | include: [ | 85 | include: [ |
62 | Object.assign(buildVideoInclude(false), { | 86 | Object.assign(buildVideoInclude(false), { |
63 | include: [ buildChannelInclude(true) ] | 87 | include: [ buildChannelInclude(true, true) ] |
64 | }), | 88 | }), |
89 | |||
65 | { | 90 | { |
66 | attributes: [ 'id', 'originCommentId' ], | 91 | attributes: [ 'id', 'originCommentId' ], |
67 | model: () => VideoCommentModel.unscoped(), | 92 | model: () => VideoCommentModel.unscoped(), |
68 | required: false, | 93 | required: false, |
69 | include: [ | 94 | include: [ |
70 | buildAccountInclude(true), | 95 | buildAccountInclude(true, true), |
71 | buildVideoInclude(true) | 96 | buildVideoInclude(true) |
72 | ] | 97 | ] |
73 | }, | 98 | }, |
99 | |||
74 | { | 100 | { |
75 | attributes: [ 'id' ], | 101 | attributes: [ 'id' ], |
76 | model: () => VideoAbuseModel.unscoped(), | 102 | model: () => VideoAbuseModel.unscoped(), |
77 | required: false, | 103 | required: false, |
78 | include: [ buildVideoInclude(true) ] | 104 | include: [ buildVideoInclude(true) ] |
79 | }, | 105 | }, |
106 | |||
80 | { | 107 | { |
81 | attributes: [ 'id' ], | 108 | attributes: [ 'id' ], |
82 | model: () => VideoBlacklistModel.unscoped(), | 109 | model: () => VideoBlacklistModel.unscoped(), |
83 | required: false, | 110 | required: false, |
84 | include: [ buildVideoInclude(true) ] | 111 | include: [ buildVideoInclude(true) ] |
85 | }, | 112 | }, |
113 | |||
86 | { | 114 | { |
87 | attributes: [ 'id', 'magnetUri', 'targetUrl', 'torrentName' ], | 115 | attributes: [ 'id', 'magnetUri', 'targetUrl', 'torrentName' ], |
88 | model: () => VideoImportModel.unscoped(), | 116 | model: () => VideoImportModel.unscoped(), |
89 | required: false, | 117 | required: false, |
90 | include: [ buildVideoInclude(false) ] | 118 | include: [ buildVideoInclude(false) ] |
91 | }, | 119 | }, |
92 | { | 120 | |
93 | attributes: [ 'id', 'name' ], | ||
94 | model: () => AccountModel.unscoped(), | ||
95 | required: false, | ||
96 | include: [ | ||
97 | { | ||
98 | attributes: [ 'id', 'preferredUsername' ], | ||
99 | model: () => ActorModel.unscoped(), | ||
100 | required: true | ||
101 | } | ||
102 | ] | ||
103 | }, | ||
104 | { | 121 | { |
105 | attributes: [ 'id' ], | 122 | attributes: [ 'id' ], |
106 | model: () => ActorFollowModel.unscoped(), | 123 | model: () => ActorFollowModel.unscoped(), |
@@ -111,7 +128,23 @@ function buildAccountInclude (required: boolean) { | |||
111 | model: () => ActorModel.unscoped(), | 128 | model: () => ActorModel.unscoped(), |
112 | required: true, | 129 | required: true, |
113 | as: 'ActorFollower', | 130 | as: 'ActorFollower', |
114 | include: [ buildAccountInclude(true) ] | 131 | include: [ |
132 | { | ||
133 | attributes: [ 'id', 'name' ], | ||
134 | model: () => AccountModel.unscoped(), | ||
135 | required: true | ||
136 | }, | ||
137 | { | ||
138 | attributes: [ 'filename' ], | ||
139 | model: () => AvatarModel.unscoped(), | ||
140 | required: false | ||
141 | }, | ||
142 | { | ||
143 | attributes: [ 'host' ], | ||
144 | model: () => ServerModel.unscoped(), | ||
145 | required: false | ||
146 | } | ||
147 | ] | ||
115 | }, | 148 | }, |
116 | { | 149 | { |
117 | attributes: [ 'preferredUsername' ], | 150 | attributes: [ 'preferredUsername' ], |
@@ -124,7 +157,9 @@ function buildAccountInclude (required: boolean) { | |||
124 | ] | 157 | ] |
125 | } | 158 | } |
126 | ] | 159 | ] |
127 | } | 160 | }, |
161 | |||
162 | buildAccountInclude(false, true) | ||
128 | ] | 163 | ] |
129 | } | 164 | } |
130 | }) | 165 | }) |
@@ -132,10 +167,63 @@ function buildAccountInclude (required: boolean) { | |||
132 | tableName: 'userNotification', | 167 | tableName: 'userNotification', |
133 | indexes: [ | 168 | indexes: [ |
134 | { | 169 | { |
135 | fields: [ 'videoId' ] | 170 | fields: [ 'userId' ] |
171 | }, | ||
172 | { | ||
173 | fields: [ 'videoId' ], | ||
174 | where: { | ||
175 | videoId: { | ||
176 | [Op.ne]: null | ||
177 | } | ||
178 | } | ||
136 | }, | 179 | }, |
137 | { | 180 | { |
138 | fields: [ 'commentId' ] | 181 | fields: [ 'commentId' ], |
182 | where: { | ||
183 | commentId: { | ||
184 | [Op.ne]: null | ||
185 | } | ||
186 | } | ||
187 | }, | ||
188 | { | ||
189 | fields: [ 'videoAbuseId' ], | ||
190 | where: { | ||
191 | videoAbuseId: { | ||
192 | [Op.ne]: null | ||
193 | } | ||
194 | } | ||
195 | }, | ||
196 | { | ||
197 | fields: [ 'videoBlacklistId' ], | ||
198 | where: { | ||
199 | videoBlacklistId: { | ||
200 | [Op.ne]: null | ||
201 | } | ||
202 | } | ||
203 | }, | ||
204 | { | ||
205 | fields: [ 'videoImportId' ], | ||
206 | where: { | ||
207 | videoImportId: { | ||
208 | [Op.ne]: null | ||
209 | } | ||
210 | } | ||
211 | }, | ||
212 | { | ||
213 | fields: [ 'accountId' ], | ||
214 | where: { | ||
215 | accountId: { | ||
216 | [Op.ne]: null | ||
217 | } | ||
218 | } | ||
219 | }, | ||
220 | { | ||
221 | fields: [ 'actorFollowId' ], | ||
222 | where: { | ||
223 | actorFollowId: { | ||
224 | [Op.ne]: null | ||
225 | } | ||
226 | } | ||
139 | } | 227 | } |
140 | ] | 228 | ] |
141 | }) | 229 | }) |
@@ -297,12 +385,9 @@ export class UserNotificationModel extends Model<UserNotificationModel> { | |||
297 | } | 385 | } |
298 | 386 | ||
299 | toFormattedJSON (): UserNotification { | 387 | toFormattedJSON (): UserNotification { |
300 | const video = this.Video ? Object.assign(this.formatVideo(this.Video), { | 388 | const video = this.Video |
301 | channel: { | 389 | ? Object.assign(this.formatVideo(this.Video),{ channel: this.formatActor(this.Video.VideoChannel) }) |
302 | id: this.Video.VideoChannel.id, | 390 | : undefined |
303 | displayName: this.Video.VideoChannel.getDisplayName() | ||
304 | } | ||
305 | }) : undefined | ||
306 | 391 | ||
307 | const videoImport = this.VideoImport ? { | 392 | const videoImport = this.VideoImport ? { |
308 | id: this.VideoImport.id, | 393 | id: this.VideoImport.id, |
@@ -315,10 +400,7 @@ export class UserNotificationModel extends Model<UserNotificationModel> { | |||
315 | const comment = this.Comment ? { | 400 | const comment = this.Comment ? { |
316 | id: this.Comment.id, | 401 | id: this.Comment.id, |
317 | threadId: this.Comment.getThreadId(), | 402 | threadId: this.Comment.getThreadId(), |
318 | account: { | 403 | account: this.formatActor(this.Comment.Account), |
319 | id: this.Comment.Account.id, | ||
320 | displayName: this.Comment.Account.getDisplayName() | ||
321 | }, | ||
322 | video: this.formatVideo(this.Comment.Video) | 404 | video: this.formatVideo(this.Comment.Video) |
323 | } : undefined | 405 | } : undefined |
324 | 406 | ||
@@ -332,17 +414,16 @@ export class UserNotificationModel extends Model<UserNotificationModel> { | |||
332 | video: this.formatVideo(this.VideoBlacklist.Video) | 414 | video: this.formatVideo(this.VideoBlacklist.Video) |
333 | } : undefined | 415 | } : undefined |
334 | 416 | ||
335 | const account = this.Account ? { | 417 | const account = this.Account ? this.formatActor(this.Account) : undefined |
336 | id: this.Account.id, | ||
337 | displayName: this.Account.getDisplayName(), | ||
338 | name: this.Account.Actor.preferredUsername | ||
339 | } : undefined | ||
340 | 418 | ||
341 | const actorFollow = this.ActorFollow ? { | 419 | const actorFollow = this.ActorFollow ? { |
342 | id: this.ActorFollow.id, | 420 | id: this.ActorFollow.id, |
343 | follower: { | 421 | follower: { |
422 | id: this.ActorFollow.ActorFollower.Account.id, | ||
344 | displayName: this.ActorFollow.ActorFollower.Account.getDisplayName(), | 423 | displayName: this.ActorFollow.ActorFollower.Account.getDisplayName(), |
345 | name: this.ActorFollow.ActorFollower.preferredUsername | 424 | name: this.ActorFollow.ActorFollower.preferredUsername, |
425 | avatar: this.ActorFollow.ActorFollower.Avatar ? { path: this.ActorFollow.ActorFollower.Avatar.getWebserverPath() } : undefined, | ||
426 | host: this.ActorFollow.ActorFollower.getHost() | ||
346 | }, | 427 | }, |
347 | following: { | 428 | following: { |
348 | type: this.ActorFollow.ActorFollowing.VideoChannel ? 'channel' as 'channel' : 'account' as 'account', | 429 | type: this.ActorFollow.ActorFollowing.VideoChannel ? 'channel' as 'channel' : 'account' as 'account', |
@@ -374,4 +455,18 @@ export class UserNotificationModel extends Model<UserNotificationModel> { | |||
374 | name: video.name | 455 | name: video.name |
375 | } | 456 | } |
376 | } | 457 | } |
458 | |||
459 | private formatActor (accountOrChannel: AccountModel | VideoChannelModel) { | ||
460 | const avatar = accountOrChannel.Actor.Avatar | ||
461 | ? { path: accountOrChannel.Actor.Avatar.getWebserverPath() } | ||
462 | : undefined | ||
463 | |||
464 | return { | ||
465 | id: accountOrChannel.id, | ||
466 | displayName: accountOrChannel.getDisplayName(), | ||
467 | name: accountOrChannel.Actor.preferredUsername, | ||
468 | host: accountOrChannel.Actor.getHost(), | ||
469 | avatar | ||
470 | } | ||
471 | } | ||
377 | } | 472 | } |
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 8f2ef2d9a..b722bed14 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts | |||
@@ -28,6 +28,7 @@ import { sample } from 'lodash' | |||
28 | import { isTestInstance } from '../../helpers/core-utils' | 28 | import { isTestInstance } from '../../helpers/core-utils' |
29 | import * as Bluebird from 'bluebird' | 29 | import * as Bluebird from 'bluebird' |
30 | import * as Sequelize from 'sequelize' | 30 | import * as Sequelize from 'sequelize' |
31 | import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' | ||
31 | 32 | ||
32 | export enum ScopeNames { | 33 | export enum ScopeNames { |
33 | WITH_VIDEO = 'WITH_VIDEO' | 34 | WITH_VIDEO = 'WITH_VIDEO' |
@@ -38,7 +39,17 @@ export enum ScopeNames { | |||
38 | include: [ | 39 | include: [ |
39 | { | 40 | { |
40 | model: () => VideoFileModel, | 41 | model: () => VideoFileModel, |
41 | required: true, | 42 | required: false, |
43 | include: [ | ||
44 | { | ||
45 | model: () => VideoModel, | ||
46 | required: true | ||
47 | } | ||
48 | ] | ||
49 | }, | ||
50 | { | ||
51 | model: () => VideoStreamingPlaylistModel, | ||
52 | required: false, | ||
42 | include: [ | 53 | include: [ |
43 | { | 54 | { |
44 | model: () => VideoModel, | 55 | model: () => VideoModel, |
@@ -97,12 +108,24 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
97 | 108 | ||
98 | @BelongsTo(() => VideoFileModel, { | 109 | @BelongsTo(() => VideoFileModel, { |
99 | foreignKey: { | 110 | foreignKey: { |
100 | allowNull: false | 111 | allowNull: true |
101 | }, | 112 | }, |
102 | onDelete: 'cascade' | 113 | onDelete: 'cascade' |
103 | }) | 114 | }) |
104 | VideoFile: VideoFileModel | 115 | VideoFile: VideoFileModel |
105 | 116 | ||
117 | @ForeignKey(() => VideoStreamingPlaylistModel) | ||
118 | @Column | ||
119 | videoStreamingPlaylistId: number | ||
120 | |||
121 | @BelongsTo(() => VideoStreamingPlaylistModel, { | ||
122 | foreignKey: { | ||
123 | allowNull: true | ||
124 | }, | ||
125 | onDelete: 'cascade' | ||
126 | }) | ||
127 | VideoStreamingPlaylist: VideoStreamingPlaylistModel | ||
128 | |||
106 | @ForeignKey(() => ActorModel) | 129 | @ForeignKey(() => ActorModel) |
107 | @Column | 130 | @Column |
108 | actorId: number | 131 | actorId: number |
@@ -119,13 +142,25 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
119 | static async removeFile (instance: VideoRedundancyModel) { | 142 | static async removeFile (instance: VideoRedundancyModel) { |
120 | if (!instance.isOwned()) return | 143 | if (!instance.isOwned()) return |
121 | 144 | ||
122 | const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId) | 145 | if (instance.videoFileId) { |
146 | const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId) | ||
123 | 147 | ||
124 | const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` | 148 | const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` |
125 | logger.info('Removing duplicated video file %s.', logIdentifier) | 149 | logger.info('Removing duplicated video file %s.', logIdentifier) |
126 | 150 | ||
127 | videoFile.Video.removeFile(videoFile, true) | 151 | videoFile.Video.removeFile(videoFile, true) |
128 | .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) | 152 | .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) |
153 | } | ||
154 | |||
155 | if (instance.videoStreamingPlaylistId) { | ||
156 | const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId) | ||
157 | |||
158 | const videoUUID = videoStreamingPlaylist.Video.uuid | ||
159 | logger.info('Removing duplicated video streaming playlist %s.', videoUUID) | ||
160 | |||
161 | videoStreamingPlaylist.Video.removeStreamingPlaylist(true) | ||
162 | .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err })) | ||
163 | } | ||
129 | 164 | ||
130 | return undefined | 165 | return undefined |
131 | } | 166 | } |
@@ -143,6 +178,19 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
143 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) | 178 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) |
144 | } | 179 | } |
145 | 180 | ||
181 | static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number) { | ||
182 | const actor = await getServerActor() | ||
183 | |||
184 | const query = { | ||
185 | where: { | ||
186 | actorId: actor.id, | ||
187 | videoStreamingPlaylistId | ||
188 | } | ||
189 | } | ||
190 | |||
191 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) | ||
192 | } | ||
193 | |||
146 | static loadByUrl (url: string, transaction?: Sequelize.Transaction) { | 194 | static loadByUrl (url: string, transaction?: Sequelize.Transaction) { |
147 | const query = { | 195 | const query = { |
148 | where: { | 196 | where: { |
@@ -191,7 +239,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
191 | const ids = rows.map(r => r.id) | 239 | const ids = rows.map(r => r.id) |
192 | const id = sample(ids) | 240 | const id = sample(ids) |
193 | 241 | ||
194 | return VideoModel.loadWithFile(id, undefined, !isTestInstance()) | 242 | return VideoModel.loadWithFiles(id, undefined, !isTestInstance()) |
195 | } | 243 | } |
196 | 244 | ||
197 | static async findMostViewToDuplicate (randomizedFactor: number) { | 245 | static async findMostViewToDuplicate (randomizedFactor: number) { |
@@ -333,40 +381,44 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
333 | 381 | ||
334 | static async listLocalOfServer (serverId: number) { | 382 | static async listLocalOfServer (serverId: number) { |
335 | const actor = await getServerActor() | 383 | const actor = await getServerActor() |
336 | 384 | const buildVideoInclude = () => ({ | |
337 | const query = { | 385 | model: VideoModel, |
338 | where: { | 386 | required: true, |
339 | actorId: actor.id | ||
340 | }, | ||
341 | include: [ | 387 | include: [ |
342 | { | 388 | { |
343 | model: VideoFileModel, | 389 | attributes: [], |
390 | model: VideoChannelModel.unscoped(), | ||
344 | required: true, | 391 | required: true, |
345 | include: [ | 392 | include: [ |
346 | { | 393 | { |
347 | model: VideoModel, | 394 | attributes: [], |
395 | model: ActorModel.unscoped(), | ||
348 | required: true, | 396 | required: true, |
349 | include: [ | 397 | where: { |
350 | { | 398 | serverId |
351 | attributes: [], | 399 | } |
352 | model: VideoChannelModel.unscoped(), | ||
353 | required: true, | ||
354 | include: [ | ||
355 | { | ||
356 | attributes: [], | ||
357 | model: ActorModel.unscoped(), | ||
358 | required: true, | ||
359 | where: { | ||
360 | serverId | ||
361 | } | ||
362 | } | ||
363 | ] | ||
364 | } | ||
365 | ] | ||
366 | } | 400 | } |
367 | ] | 401 | ] |
368 | } | 402 | } |
369 | ] | 403 | ] |
404 | }) | ||
405 | |||
406 | const query = { | ||
407 | where: { | ||
408 | actorId: actor.id | ||
409 | }, | ||
410 | include: [ | ||
411 | { | ||
412 | model: VideoFileModel, | ||
413 | required: false, | ||
414 | include: [ buildVideoInclude() ] | ||
415 | }, | ||
416 | { | ||
417 | model: VideoStreamingPlaylistModel, | ||
418 | required: false, | ||
419 | include: [ buildVideoInclude() ] | ||
420 | } | ||
421 | ] | ||
370 | } | 422 | } |
371 | 423 | ||
372 | return VideoRedundancyModel.findAll(query) | 424 | return VideoRedundancyModel.findAll(query) |
@@ -403,11 +455,32 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
403 | })) | 455 | })) |
404 | } | 456 | } |
405 | 457 | ||
458 | getVideo () { | ||
459 | if (this.VideoFile) return this.VideoFile.Video | ||
460 | |||
461 | return this.VideoStreamingPlaylist.Video | ||
462 | } | ||
463 | |||
406 | isOwned () { | 464 | isOwned () { |
407 | return !!this.strategy | 465 | return !!this.strategy |
408 | } | 466 | } |
409 | 467 | ||
410 | toActivityPubObject (): CacheFileObject { | 468 | toActivityPubObject (): CacheFileObject { |
469 | if (this.VideoStreamingPlaylist) { | ||
470 | return { | ||
471 | id: this.url, | ||
472 | type: 'CacheFile' as 'CacheFile', | ||
473 | object: this.VideoStreamingPlaylist.Video.url, | ||
474 | expires: this.expiresOn.toISOString(), | ||
475 | url: { | ||
476 | type: 'Link', | ||
477 | mimeType: 'application/x-mpegURL', | ||
478 | mediaType: 'application/x-mpegURL', | ||
479 | href: this.fileUrl | ||
480 | } | ||
481 | } | ||
482 | } | ||
483 | |||
411 | return { | 484 | return { |
412 | id: this.url, | 485 | id: this.url, |
413 | type: 'CacheFile' as 'CacheFile', | 486 | type: 'CacheFile' as 'CacheFile', |
@@ -431,7 +504,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
431 | 504 | ||
432 | const notIn = Sequelize.literal( | 505 | const notIn = Sequelize.literal( |
433 | '(' + | 506 | '(' + |
434 | `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` + | 507 | `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` + |
435 | ')' | 508 | ')' |
436 | ) | 509 | ) |
437 | 510 | ||
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 1f1b76c1e..7d1e371b9 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts | |||
@@ -62,7 +62,7 @@ export class VideoFileModel extends Model<VideoFileModel> { | |||
62 | extname: string | 62 | extname: string |
63 | 63 | ||
64 | @AllowNull(false) | 64 | @AllowNull(false) |
65 | @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash')) | 65 | @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash')) |
66 | @Column | 66 | @Column |
67 | infoHash: string | 67 | infoHash: string |
68 | 68 | ||
@@ -86,14 +86,14 @@ export class VideoFileModel extends Model<VideoFileModel> { | |||
86 | 86 | ||
87 | @HasMany(() => VideoRedundancyModel, { | 87 | @HasMany(() => VideoRedundancyModel, { |
88 | foreignKey: { | 88 | foreignKey: { |
89 | allowNull: false | 89 | allowNull: true |
90 | }, | 90 | }, |
91 | onDelete: 'CASCADE', | 91 | onDelete: 'CASCADE', |
92 | hooks: true | 92 | hooks: true |
93 | }) | 93 | }) |
94 | RedundancyVideos: VideoRedundancyModel[] | 94 | RedundancyVideos: VideoRedundancyModel[] |
95 | 95 | ||
96 | static isInfohashExists (infoHash: string) { | 96 | static doesInfohashExist (infoHash: string) { |
97 | const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' | 97 | const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' |
98 | const options = { | 98 | const options = { |
99 | type: Sequelize.QueryTypes.SELECT, | 99 | type: Sequelize.QueryTypes.SELECT, |
diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index de0747f22..e49dbee30 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts | |||
@@ -1,7 +1,12 @@ | |||
1 | import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' | 1 | import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' |
2 | import { VideoModel } from './video' | 2 | import { VideoModel } from './video' |
3 | import { VideoFileModel } from './video-file' | 3 | import { VideoFileModel } from './video-file' |
4 | import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 4 | import { |
5 | ActivityPlaylistInfohashesObject, | ||
6 | ActivityPlaylistSegmentHashesObject, | ||
7 | ActivityUrlObject, | ||
8 | VideoTorrentObject | ||
9 | } from '../../../shared/models/activitypub/objects' | ||
5 | import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers' | 10 | import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers' |
6 | import { VideoCaptionModel } from './video-caption' | 11 | import { VideoCaptionModel } from './video-caption' |
7 | import { | 12 | import { |
@@ -11,6 +16,8 @@ import { | |||
11 | getVideoSharesActivityPubUrl | 16 | getVideoSharesActivityPubUrl |
12 | } from '../../lib/activitypub' | 17 | } from '../../lib/activitypub' |
13 | import { isArray } from '../../helpers/custom-validators/misc' | 18 | import { isArray } from '../../helpers/custom-validators/misc' |
19 | import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' | ||
20 | import { VideoStreamingPlaylistModel } from './video-streaming-playlist' | ||
14 | 21 | ||
15 | export type VideoFormattingJSONOptions = { | 22 | export type VideoFormattingJSONOptions = { |
16 | completeDescription?: boolean | 23 | completeDescription?: boolean |
@@ -120,7 +127,12 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { | |||
120 | } | 127 | } |
121 | }) | 128 | }) |
122 | 129 | ||
130 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | ||
131 | |||
123 | const tags = video.Tags ? video.Tags.map(t => t.name) : [] | 132 | const tags = video.Tags ? video.Tags.map(t => t.name) : [] |
133 | |||
134 | const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists) | ||
135 | |||
124 | const detailsJson = { | 136 | const detailsJson = { |
125 | support: video.support, | 137 | support: video.support, |
126 | descriptionPath: video.getDescriptionAPIPath(), | 138 | descriptionPath: video.getDescriptionAPIPath(), |
@@ -133,7 +145,11 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { | |||
133 | id: video.state, | 145 | id: video.state, |
134 | label: VideoModel.getStateLabel(video.state) | 146 | label: VideoModel.getStateLabel(video.state) |
135 | }, | 147 | }, |
136 | files: [] | 148 | |
149 | trackerUrls: video.getTrackerUrls(baseUrlHttp, baseUrlWs), | ||
150 | |||
151 | files: [], | ||
152 | streamingPlaylists | ||
137 | } | 153 | } |
138 | 154 | ||
139 | // Format and sort video files | 155 | // Format and sort video files |
@@ -142,6 +158,25 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { | |||
142 | return Object.assign(formattedJson, detailsJson) | 158 | return Object.assign(formattedJson, detailsJson) |
143 | } | 159 | } |
144 | 160 | ||
161 | function streamingPlaylistsModelToFormattedJSON (video: VideoModel, playlists: VideoStreamingPlaylistModel[]): VideoStreamingPlaylist[] { | ||
162 | if (isArray(playlists) === false) return [] | ||
163 | |||
164 | return playlists | ||
165 | .map(playlist => { | ||
166 | const redundancies = isArray(playlist.RedundancyVideos) | ||
167 | ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl })) | ||
168 | : [] | ||
169 | |||
170 | return { | ||
171 | id: playlist.id, | ||
172 | type: playlist.type, | ||
173 | playlistUrl: playlist.playlistUrl, | ||
174 | segmentsSha256Url: playlist.segmentsSha256Url, | ||
175 | redundancies | ||
176 | } as VideoStreamingPlaylist | ||
177 | }) | ||
178 | } | ||
179 | |||
145 | function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] { | 180 | function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] { |
146 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | 181 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() |
147 | 182 | ||
@@ -232,6 +267,28 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { | |||
232 | }) | 267 | }) |
233 | } | 268 | } |
234 | 269 | ||
270 | for (const playlist of (video.VideoStreamingPlaylists || [])) { | ||
271 | let tag: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[] | ||
272 | |||
273 | tag = playlist.p2pMediaLoaderInfohashes | ||
274 | .map(i => ({ type: 'Infohash' as 'Infohash', name: i })) | ||
275 | tag.push({ | ||
276 | type: 'Link', | ||
277 | name: 'sha256', | ||
278 | mimeType: 'application/json' as 'application/json', | ||
279 | mediaType: 'application/json' as 'application/json', | ||
280 | href: playlist.segmentsSha256Url | ||
281 | }) | ||
282 | |||
283 | url.push({ | ||
284 | type: 'Link', | ||
285 | mimeType: 'application/x-mpegURL' as 'application/x-mpegURL', | ||
286 | mediaType: 'application/x-mpegURL' as 'application/x-mpegURL', | ||
287 | href: playlist.playlistUrl, | ||
288 | tag | ||
289 | }) | ||
290 | } | ||
291 | |||
235 | // Add video url too | 292 | // Add video url too |
236 | url.push({ | 293 | url.push({ |
237 | type: 'Link', | 294 | type: 'Link', |
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts new file mode 100644 index 000000000..bf6f7b0c4 --- /dev/null +++ b/server/models/video/video-streaming-playlist.ts | |||
@@ -0,0 +1,158 @@ | |||
1 | import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' | ||
2 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' | ||
3 | import { throwIfNotValid } from '../utils' | ||
4 | import { VideoModel } from './video' | ||
5 | import * as Sequelize from 'sequelize' | ||
6 | import { VideoRedundancyModel } from '../redundancy/video-redundancy' | ||
7 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
8 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | ||
9 | import { CONSTRAINTS_FIELDS, STATIC_PATHS } from '../../initializers' | ||
10 | import { VideoFileModel } from './video-file' | ||
11 | import { join } from 'path' | ||
12 | import { sha1 } from '../../helpers/core-utils' | ||
13 | import { isArrayOf } from '../../helpers/custom-validators/misc' | ||
14 | |||
15 | @Table({ | ||
16 | tableName: 'videoStreamingPlaylist', | ||
17 | indexes: [ | ||
18 | { | ||
19 | fields: [ 'videoId' ] | ||
20 | }, | ||
21 | { | ||
22 | fields: [ 'videoId', 'type' ], | ||
23 | unique: true | ||
24 | }, | ||
25 | { | ||
26 | fields: [ 'p2pMediaLoaderInfohashes' ], | ||
27 | using: 'gin' | ||
28 | } | ||
29 | ] | ||
30 | }) | ||
31 | export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistModel> { | ||
32 | @CreatedAt | ||
33 | createdAt: Date | ||
34 | |||
35 | @UpdatedAt | ||
36 | updatedAt: Date | ||
37 | |||
38 | @AllowNull(false) | ||
39 | @Column | ||
40 | type: VideoStreamingPlaylistType | ||
41 | |||
42 | @AllowNull(false) | ||
43 | @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url')) | ||
44 | @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max)) | ||
45 | playlistUrl: string | ||
46 | |||
47 | @AllowNull(false) | ||
48 | @Is('VideoStreamingPlaylistInfoHashes', value => throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes')) | ||
49 | @Column(DataType.ARRAY(DataType.STRING)) | ||
50 | p2pMediaLoaderInfohashes: string[] | ||
51 | |||
52 | @AllowNull(false) | ||
53 | @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url')) | ||
54 | @Column | ||
55 | segmentsSha256Url: string | ||
56 | |||
57 | @ForeignKey(() => VideoModel) | ||
58 | @Column | ||
59 | videoId: number | ||
60 | |||
61 | @BelongsTo(() => VideoModel, { | ||
62 | foreignKey: { | ||
63 | allowNull: false | ||
64 | }, | ||
65 | onDelete: 'CASCADE' | ||
66 | }) | ||
67 | Video: VideoModel | ||
68 | |||
69 | @HasMany(() => VideoRedundancyModel, { | ||
70 | foreignKey: { | ||
71 | allowNull: false | ||
72 | }, | ||
73 | onDelete: 'CASCADE', | ||
74 | hooks: true | ||
75 | }) | ||
76 | RedundancyVideos: VideoRedundancyModel[] | ||
77 | |||
78 | static doesInfohashExist (infoHash: string) { | ||
79 | const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1' | ||
80 | const options = { | ||
81 | type: Sequelize.QueryTypes.SELECT, | ||
82 | bind: { infoHash }, | ||
83 | raw: true | ||
84 | } | ||
85 | |||
86 | return VideoModel.sequelize.query(query, options) | ||
87 | .then(results => { | ||
88 | return results.length === 1 | ||
89 | }) | ||
90 | } | ||
91 | |||
92 | static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: VideoFileModel[]) { | ||
93 | const hashes: string[] = [] | ||
94 | |||
95 | // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L97 | ||
96 | for (let i = 0; i < videoFiles.length; i++) { | ||
97 | hashes.push(sha1(`1${playlistUrl}+V${i}`)) | ||
98 | } | ||
99 | |||
100 | return hashes | ||
101 | } | ||
102 | |||
103 | static loadWithVideo (id: number) { | ||
104 | const options = { | ||
105 | include: [ | ||
106 | { | ||
107 | model: VideoModel.unscoped(), | ||
108 | required: true | ||
109 | } | ||
110 | ] | ||
111 | } | ||
112 | |||
113 | return VideoStreamingPlaylistModel.findById(id, options) | ||
114 | } | ||
115 | |||
116 | static getHlsPlaylistFilename (resolution: number) { | ||
117 | return resolution + '.m3u8' | ||
118 | } | ||
119 | |||
120 | static getMasterHlsPlaylistFilename () { | ||
121 | return 'master.m3u8' | ||
122 | } | ||
123 | |||
124 | static getHlsSha256SegmentsFilename () { | ||
125 | return 'segments-sha256.json' | ||
126 | } | ||
127 | |||
128 | static getHlsVideoName (uuid: string, resolution: number) { | ||
129 | return `${uuid}-${resolution}-fragmented.mp4` | ||
130 | } | ||
131 | |||
132 | static getHlsMasterPlaylistStaticPath (videoUUID: string) { | ||
133 | return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) | ||
134 | } | ||
135 | |||
136 | static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) { | ||
137 | return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) | ||
138 | } | ||
139 | |||
140 | static getHlsSha256SegmentsStaticPath (videoUUID: string) { | ||
141 | return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) | ||
142 | } | ||
143 | |||
144 | getStringType () { | ||
145 | if (this.type === VideoStreamingPlaylistType.HLS) return 'hls' | ||
146 | |||
147 | return 'unknown' | ||
148 | } | ||
149 | |||
150 | getVideoRedundancyUrl (baseUrlHttp: string) { | ||
151 | return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getStringType() + '/' + this.Video.uuid | ||
152 | } | ||
153 | |||
154 | hasSameUniqueKeysThan (other: VideoStreamingPlaylistModel) { | ||
155 | return this.type === other.type && | ||
156 | this.videoId === other.videoId | ||
157 | } | ||
158 | } | ||
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 80a6c7832..702260772 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -52,7 +52,7 @@ import { | |||
52 | ACTIVITY_PUB, | 52 | ACTIVITY_PUB, |
53 | API_VERSION, | 53 | API_VERSION, |
54 | CONFIG, | 54 | CONFIG, |
55 | CONSTRAINTS_FIELDS, | 55 | CONSTRAINTS_FIELDS, HLS_PLAYLIST_DIRECTORY, HLS_REDUNDANCY_DIRECTORY, |
56 | PREVIEWS_SIZE, | 56 | PREVIEWS_SIZE, |
57 | REMOTE_SCHEME, | 57 | REMOTE_SCHEME, |
58 | STATIC_DOWNLOAD_PATHS, | 58 | STATIC_DOWNLOAD_PATHS, |
@@ -95,6 +95,7 @@ import * as validator from 'validator' | |||
95 | import { UserVideoHistoryModel } from '../account/user-video-history' | 95 | import { UserVideoHistoryModel } from '../account/user-video-history' |
96 | import { UserModel } from '../account/user' | 96 | import { UserModel } from '../account/user' |
97 | import { VideoImportModel } from './video-import' | 97 | import { VideoImportModel } from './video-import' |
98 | import { VideoStreamingPlaylistModel } from './video-streaming-playlist' | ||
98 | 99 | ||
99 | // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation | 100 | // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation |
100 | const indexes: Sequelize.DefineIndexesOptions[] = [ | 101 | const indexes: Sequelize.DefineIndexesOptions[] = [ |
@@ -159,7 +160,9 @@ export enum ScopeNames { | |||
159 | WITH_FILES = 'WITH_FILES', | 160 | WITH_FILES = 'WITH_FILES', |
160 | WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', | 161 | WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', |
161 | WITH_BLACKLISTED = 'WITH_BLACKLISTED', | 162 | WITH_BLACKLISTED = 'WITH_BLACKLISTED', |
162 | WITH_USER_HISTORY = 'WITH_USER_HISTORY' | 163 | WITH_USER_HISTORY = 'WITH_USER_HISTORY', |
164 | WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS', | ||
165 | WITH_USER_ID = 'WITH_USER_ID' | ||
163 | } | 166 | } |
164 | 167 | ||
165 | type ForAPIOptions = { | 168 | type ForAPIOptions = { |
@@ -463,6 +466,22 @@ type AvailableForListIDsOptions = { | |||
463 | 466 | ||
464 | return query | 467 | return query |
465 | }, | 468 | }, |
469 | [ ScopeNames.WITH_USER_ID ]: { | ||
470 | include: [ | ||
471 | { | ||
472 | attributes: [ 'accountId' ], | ||
473 | model: () => VideoChannelModel.unscoped(), | ||
474 | required: true, | ||
475 | include: [ | ||
476 | { | ||
477 | attributes: [ 'userId' ], | ||
478 | model: () => AccountModel.unscoped(), | ||
479 | required: true | ||
480 | } | ||
481 | ] | ||
482 | } | ||
483 | ] | ||
484 | }, | ||
466 | [ ScopeNames.WITH_ACCOUNT_DETAILS ]: { | 485 | [ ScopeNames.WITH_ACCOUNT_DETAILS ]: { |
467 | include: [ | 486 | include: [ |
468 | { | 487 | { |
@@ -527,22 +546,55 @@ type AvailableForListIDsOptions = { | |||
527 | } | 546 | } |
528 | ] | 547 | ] |
529 | }, | 548 | }, |
530 | [ ScopeNames.WITH_FILES ]: { | 549 | [ ScopeNames.WITH_FILES ]: (withRedundancies = false) => { |
531 | include: [ | 550 | let subInclude: any[] = [] |
532 | { | 551 | |
533 | model: () => VideoFileModel.unscoped(), | 552 | if (withRedundancies === true) { |
534 | // FIXME: typings | 553 | subInclude = [ |
535 | [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join | 554 | { |
536 | required: false, | 555 | attributes: [ 'fileUrl' ], |
537 | include: [ | 556 | model: VideoRedundancyModel.unscoped(), |
538 | { | 557 | required: false |
539 | attributes: [ 'fileUrl' ], | 558 | } |
540 | model: () => VideoRedundancyModel.unscoped(), | 559 | ] |
541 | required: false | 560 | } |
542 | } | 561 | |
543 | ] | 562 | return { |
544 | } | 563 | include: [ |
545 | ] | 564 | { |
565 | model: VideoFileModel.unscoped(), | ||
566 | // FIXME: typings | ||
567 | [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join | ||
568 | required: false, | ||
569 | include: subInclude | ||
570 | } | ||
571 | ] | ||
572 | } | ||
573 | }, | ||
574 | [ ScopeNames.WITH_STREAMING_PLAYLISTS ]: (withRedundancies = false) => { | ||
575 | let subInclude: any[] = [] | ||
576 | |||
577 | if (withRedundancies === true) { | ||
578 | subInclude = [ | ||
579 | { | ||
580 | attributes: [ 'fileUrl' ], | ||
581 | model: VideoRedundancyModel.unscoped(), | ||
582 | required: false | ||
583 | } | ||
584 | ] | ||
585 | } | ||
586 | |||
587 | return { | ||
588 | include: [ | ||
589 | { | ||
590 | model: VideoStreamingPlaylistModel.unscoped(), | ||
591 | // FIXME: typings | ||
592 | [ 'separate' as any ]: true, // We may have multiple streaming playlists, having multiple redundancies so let's separate this join | ||
593 | required: false, | ||
594 | include: subInclude | ||
595 | } | ||
596 | ] | ||
597 | } | ||
546 | }, | 598 | }, |
547 | [ ScopeNames.WITH_SCHEDULED_UPDATE ]: { | 599 | [ ScopeNames.WITH_SCHEDULED_UPDATE ]: { |
548 | include: [ | 600 | include: [ |
@@ -722,6 +774,16 @@ export class VideoModel extends Model<VideoModel> { | |||
722 | }) | 774 | }) |
723 | VideoFiles: VideoFileModel[] | 775 | VideoFiles: VideoFileModel[] |
724 | 776 | ||
777 | @HasMany(() => VideoStreamingPlaylistModel, { | ||
778 | foreignKey: { | ||
779 | name: 'videoId', | ||
780 | allowNull: false | ||
781 | }, | ||
782 | hooks: true, | ||
783 | onDelete: 'cascade' | ||
784 | }) | ||
785 | VideoStreamingPlaylists: VideoStreamingPlaylistModel[] | ||
786 | |||
725 | @HasMany(() => VideoShareModel, { | 787 | @HasMany(() => VideoShareModel, { |
726 | foreignKey: { | 788 | foreignKey: { |
727 | name: 'videoId', | 789 | name: 'videoId', |
@@ -847,6 +909,9 @@ export class VideoModel extends Model<VideoModel> { | |||
847 | tasks.push(instance.removeFile(file)) | 909 | tasks.push(instance.removeFile(file)) |
848 | tasks.push(instance.removeTorrent(file)) | 910 | tasks.push(instance.removeTorrent(file)) |
849 | }) | 911 | }) |
912 | |||
913 | // Remove playlists file | ||
914 | tasks.push(instance.removeStreamingPlaylist()) | ||
850 | } | 915 | } |
851 | 916 | ||
852 | // Do not wait video deletion because we could be in a transaction | 917 | // Do not wait video deletion because we could be in a transaction |
@@ -858,10 +923,6 @@ export class VideoModel extends Model<VideoModel> { | |||
858 | return undefined | 923 | return undefined |
859 | } | 924 | } |
860 | 925 | ||
861 | static list () { | ||
862 | return VideoModel.scope(ScopeNames.WITH_FILES).findAll() | ||
863 | } | ||
864 | |||
865 | static listLocal () { | 926 | static listLocal () { |
866 | const query = { | 927 | const query = { |
867 | where: { | 928 | where: { |
@@ -869,7 +930,7 @@ export class VideoModel extends Model<VideoModel> { | |||
869 | } | 930 | } |
870 | } | 931 | } |
871 | 932 | ||
872 | return VideoModel.scope(ScopeNames.WITH_FILES).findAll(query) | 933 | return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]).findAll(query) |
873 | } | 934 | } |
874 | 935 | ||
875 | static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { | 936 | static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { |
@@ -1200,6 +1261,16 @@ export class VideoModel extends Model<VideoModel> { | |||
1200 | return VideoModel.findOne(options) | 1261 | return VideoModel.findOne(options) |
1201 | } | 1262 | } |
1202 | 1263 | ||
1264 | static loadWithRights (id: number | string, t?: Sequelize.Transaction) { | ||
1265 | const where = VideoModel.buildWhereIdOrUUID(id) | ||
1266 | const options = { | ||
1267 | where, | ||
1268 | transaction: t | ||
1269 | } | ||
1270 | |||
1271 | return VideoModel.scope([ ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_USER_ID ]).findOne(options) | ||
1272 | } | ||
1273 | |||
1203 | static loadOnlyId (id: number | string, t?: Sequelize.Transaction) { | 1274 | static loadOnlyId (id: number | string, t?: Sequelize.Transaction) { |
1204 | const where = VideoModel.buildWhereIdOrUUID(id) | 1275 | const where = VideoModel.buildWhereIdOrUUID(id) |
1205 | 1276 | ||
@@ -1212,8 +1283,8 @@ export class VideoModel extends Model<VideoModel> { | |||
1212 | return VideoModel.findOne(options) | 1283 | return VideoModel.findOne(options) |
1213 | } | 1284 | } |
1214 | 1285 | ||
1215 | static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) { | 1286 | static loadWithFiles (id: number, t?: Sequelize.Transaction, logging?: boolean) { |
1216 | return VideoModel.scope(ScopeNames.WITH_FILES) | 1287 | return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]) |
1217 | .findById(id, { transaction: t, logging }) | 1288 | .findById(id, { transaction: t, logging }) |
1218 | } | 1289 | } |
1219 | 1290 | ||
@@ -1224,9 +1295,7 @@ export class VideoModel extends Model<VideoModel> { | |||
1224 | } | 1295 | } |
1225 | } | 1296 | } |
1226 | 1297 | ||
1227 | return VideoModel | 1298 | return VideoModel.findOne(options) |
1228 | .scope([ ScopeNames.WITH_FILES ]) | ||
1229 | .findOne(options) | ||
1230 | } | 1299 | } |
1231 | 1300 | ||
1232 | static loadByUrl (url: string, transaction?: Sequelize.Transaction) { | 1301 | static loadByUrl (url: string, transaction?: Sequelize.Transaction) { |
@@ -1248,7 +1317,11 @@ export class VideoModel extends Model<VideoModel> { | |||
1248 | transaction | 1317 | transaction |
1249 | } | 1318 | } |
1250 | 1319 | ||
1251 | return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) | 1320 | return VideoModel.scope([ |
1321 | ScopeNames.WITH_ACCOUNT_DETAILS, | ||
1322 | ScopeNames.WITH_FILES, | ||
1323 | ScopeNames.WITH_STREAMING_PLAYLISTS | ||
1324 | ]).findOne(query) | ||
1252 | } | 1325 | } |
1253 | 1326 | ||
1254 | static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) { | 1327 | static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) { |
@@ -1263,9 +1336,37 @@ export class VideoModel extends Model<VideoModel> { | |||
1263 | const scopes = [ | 1336 | const scopes = [ |
1264 | ScopeNames.WITH_TAGS, | 1337 | ScopeNames.WITH_TAGS, |
1265 | ScopeNames.WITH_BLACKLISTED, | 1338 | ScopeNames.WITH_BLACKLISTED, |
1339 | ScopeNames.WITH_ACCOUNT_DETAILS, | ||
1340 | ScopeNames.WITH_SCHEDULED_UPDATE, | ||
1266 | ScopeNames.WITH_FILES, | 1341 | ScopeNames.WITH_FILES, |
1342 | ScopeNames.WITH_STREAMING_PLAYLISTS | ||
1343 | ] | ||
1344 | |||
1345 | if (userId) { | ||
1346 | scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings | ||
1347 | } | ||
1348 | |||
1349 | return VideoModel | ||
1350 | .scope(scopes) | ||
1351 | .findOne(options) | ||
1352 | } | ||
1353 | |||
1354 | static loadForGetAPI (id: number | string, t?: Sequelize.Transaction, userId?: number) { | ||
1355 | const where = VideoModel.buildWhereIdOrUUID(id) | ||
1356 | |||
1357 | const options = { | ||
1358 | order: [ [ 'Tags', 'name', 'ASC' ] ], | ||
1359 | where, | ||
1360 | transaction: t | ||
1361 | } | ||
1362 | |||
1363 | const scopes = [ | ||
1364 | ScopeNames.WITH_TAGS, | ||
1365 | ScopeNames.WITH_BLACKLISTED, | ||
1267 | ScopeNames.WITH_ACCOUNT_DETAILS, | 1366 | ScopeNames.WITH_ACCOUNT_DETAILS, |
1268 | ScopeNames.WITH_SCHEDULED_UPDATE | 1367 | ScopeNames.WITH_SCHEDULED_UPDATE, |
1368 | { method: [ ScopeNames.WITH_FILES, true ] } as any, // FIXME: typings | ||
1369 | { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } as any // FIXME: typings | ||
1269 | ] | 1370 | ] |
1270 | 1371 | ||
1271 | if (userId) { | 1372 | if (userId) { |
@@ -1612,6 +1713,14 @@ export class VideoModel extends Model<VideoModel> { | |||
1612 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) | 1713 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) |
1613 | } | 1714 | } |
1614 | 1715 | ||
1716 | removeStreamingPlaylist (isRedundancy = false) { | ||
1717 | const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_PLAYLIST_DIRECTORY | ||
1718 | |||
1719 | const filePath = join(baseDir, this.uuid) | ||
1720 | return remove(filePath) | ||
1721 | .catch(err => logger.warn('Cannot delete playlist directory %s.', filePath, { err })) | ||
1722 | } | ||
1723 | |||
1615 | isOutdated () { | 1724 | isOutdated () { |
1616 | if (this.isOwned()) return false | 1725 | if (this.isOwned()) return false |
1617 | 1726 | ||
@@ -1646,7 +1755,7 @@ export class VideoModel extends Model<VideoModel> { | |||
1646 | 1755 | ||
1647 | generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { | 1756 | generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { |
1648 | const xs = this.getTorrentUrl(videoFile, baseUrlHttp) | 1757 | const xs = this.getTorrentUrl(videoFile, baseUrlHttp) |
1649 | const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] | 1758 | const announce = this.getTrackerUrls(baseUrlHttp, baseUrlWs) |
1650 | let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] | 1759 | let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] |
1651 | 1760 | ||
1652 | const redundancies = videoFile.RedundancyVideos | 1761 | const redundancies = videoFile.RedundancyVideos |
@@ -1663,6 +1772,10 @@ export class VideoModel extends Model<VideoModel> { | |||
1663 | return magnetUtil.encode(magnetHash) | 1772 | return magnetUtil.encode(magnetHash) |
1664 | } | 1773 | } |
1665 | 1774 | ||
1775 | getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) { | ||
1776 | return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] | ||
1777 | } | ||
1778 | |||
1666 | getThumbnailUrl (baseUrlHttp: string) { | 1779 | getThumbnailUrl (baseUrlHttp: string) { |
1667 | return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() | 1780 | return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() |
1668 | } | 1781 | } |
@@ -1686,4 +1799,8 @@ export class VideoModel extends Model<VideoModel> { | |||
1686 | getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { | 1799 | getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { |
1687 | return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) | 1800 | return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) |
1688 | } | 1801 | } |
1802 | |||
1803 | getBandwidthBits (videoFile: VideoFileModel) { | ||
1804 | return Math.ceil((videoFile.size * 8) / this.duration) | ||
1805 | } | ||
1689 | } | 1806 | } |
diff --git a/server/tests/api/check-params/accounts.ts b/server/tests/api/check-params/accounts.ts index 567fd072c..68f9519c6 100644 --- a/server/tests/api/check-params/accounts.ts +++ b/server/tests/api/check-params/accounts.ts | |||
@@ -10,7 +10,7 @@ import { | |||
10 | } from '../../../../shared/utils/requests/check-api-params' | 10 | } from '../../../../shared/utils/requests/check-api-params' |
11 | import { getAccount } from '../../../../shared/utils/users/accounts' | 11 | import { getAccount } from '../../../../shared/utils/users/accounts' |
12 | 12 | ||
13 | describe('Test users API validators', function () { | 13 | describe('Test accounts API validators', function () { |
14 | const path = '/api/v1/accounts/' | 14 | const path = '/api/v1/accounts/' |
15 | let server: ServerInfo | 15 | let server: ServerInfo |
16 | 16 | ||
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 4038ecbf0..07de2b5a5 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts | |||
@@ -65,6 +65,9 @@ describe('Test config API validators', function () { | |||
65 | '480p': true, | 65 | '480p': true, |
66 | '720p': false, | 66 | '720p': false, |
67 | '1080p': false | 67 | '1080p': false |
68 | }, | ||
69 | hls: { | ||
70 | enabled: false | ||
68 | } | 71 | } |
69 | }, | 72 | }, |
70 | import: { | 73 | import: { |
diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts index a3e8e2e9c..13be8b460 100644 --- a/server/tests/api/check-params/users.ts +++ b/server/tests/api/check-params/users.ts | |||
@@ -464,6 +464,24 @@ describe('Test users API validators', function () { | |||
464 | await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) | 464 | await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) |
465 | }) | 465 | }) |
466 | 466 | ||
467 | it('Should fail with a too small password', async function () { | ||
468 | const fields = { | ||
469 | currentPassword: 'my super password', | ||
470 | password: 'bla' | ||
471 | } | ||
472 | |||
473 | await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) | ||
474 | }) | ||
475 | |||
476 | it('Should fail with a too long password', async function () { | ||
477 | const fields = { | ||
478 | currentPassword: 'my super password', | ||
479 | password: 'super'.repeat(61) | ||
480 | } | ||
481 | |||
482 | await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) | ||
483 | }) | ||
484 | |||
467 | it('Should fail with an non authenticated user', async function () { | 485 | it('Should fail with an non authenticated user', async function () { |
468 | const fields = { | 486 | const fields = { |
469 | videoQuota: 42 | 487 | videoQuota: 42 |
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 9d3ce8153..778611fff 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts | |||
@@ -17,7 +17,7 @@ import { | |||
17 | viewVideo, | 17 | viewVideo, |
18 | wait, | 18 | wait, |
19 | waitUntilLog, | 19 | waitUntilLog, |
20 | checkVideoFilesWereRemoved, removeVideo, getVideoWithToken | 20 | checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer, checkSegmentHash |
21 | } from '../../../../shared/utils' | 21 | } from '../../../../shared/utils' |
22 | import { waitJobs } from '../../../../shared/utils/server/jobs' | 22 | import { waitJobs } from '../../../../shared/utils/server/jobs' |
23 | 23 | ||
@@ -48,6 +48,11 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe | |||
48 | 48 | ||
49 | async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { | 49 | async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { |
50 | const config = { | 50 | const config = { |
51 | transcoding: { | ||
52 | hls: { | ||
53 | enabled: true | ||
54 | } | ||
55 | }, | ||
51 | redundancy: { | 56 | redundancy: { |
52 | videos: { | 57 | videos: { |
53 | check_interval: '5 seconds', | 58 | check_interval: '5 seconds', |
@@ -85,7 +90,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: | |||
85 | await waitJobs(servers) | 90 | await waitJobs(servers) |
86 | } | 91 | } |
87 | 92 | ||
88 | async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { | 93 | async function check1WebSeed (videoUUID?: string) { |
89 | if (!videoUUID) videoUUID = video1Server2UUID | 94 | if (!videoUUID) videoUUID = video1Server2UUID |
90 | 95 | ||
91 | const webseeds = [ | 96 | const webseeds = [ |
@@ -93,47 +98,17 @@ async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: str | |||
93 | ] | 98 | ] |
94 | 99 | ||
95 | for (const server of servers) { | 100 | for (const server of servers) { |
96 | { | 101 | // With token to avoid issues with video follow constraints |
97 | // With token to avoid issues with video follow constraints | 102 | const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) |
98 | const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) | ||
99 | 103 | ||
100 | const video: VideoDetails = res.body | 104 | const video: VideoDetails = res.body |
101 | for (const f of video.files) { | 105 | for (const f of video.files) { |
102 | checkMagnetWebseeds(f, webseeds, server) | 106 | checkMagnetWebseeds(f, webseeds, server) |
103 | } | ||
104 | } | 107 | } |
105 | } | 108 | } |
106 | } | 109 | } |
107 | 110 | ||
108 | async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { | 111 | async function check2Webseeds (videoUUID?: string) { |
109 | const res = await getStats(servers[0].url) | ||
110 | const data: ServerStats = res.body | ||
111 | |||
112 | expect(data.videosRedundancy).to.have.lengthOf(1) | ||
113 | const stat = data.videosRedundancy[0] | ||
114 | |||
115 | expect(stat.strategy).to.equal(strategy) | ||
116 | expect(stat.totalSize).to.equal(204800) | ||
117 | expect(stat.totalUsed).to.be.at.least(1).and.below(204801) | ||
118 | expect(stat.totalVideoFiles).to.equal(4) | ||
119 | expect(stat.totalVideos).to.equal(1) | ||
120 | } | ||
121 | |||
122 | async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { | ||
123 | const res = await getStats(servers[0].url) | ||
124 | const data: ServerStats = res.body | ||
125 | |||
126 | expect(data.videosRedundancy).to.have.lengthOf(1) | ||
127 | |||
128 | const stat = data.videosRedundancy[0] | ||
129 | expect(stat.strategy).to.equal(strategy) | ||
130 | expect(stat.totalSize).to.equal(204800) | ||
131 | expect(stat.totalUsed).to.equal(0) | ||
132 | expect(stat.totalVideoFiles).to.equal(0) | ||
133 | expect(stat.totalVideos).to.equal(0) | ||
134 | } | ||
135 | |||
136 | async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) { | ||
137 | if (!videoUUID) videoUUID = video1Server2UUID | 112 | if (!videoUUID) videoUUID = video1Server2UUID |
138 | 113 | ||
139 | const webseeds = [ | 114 | const webseeds = [ |
@@ -158,7 +133,7 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st | |||
158 | await makeGetRequest({ | 133 | await makeGetRequest({ |
159 | url: servers[1].url, | 134 | url: servers[1].url, |
160 | statusCodeExpected: 200, | 135 | statusCodeExpected: 200, |
161 | path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, | 136 | path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`, |
162 | contentType: null | 137 | contentType: null |
163 | }) | 138 | }) |
164 | } | 139 | } |
@@ -174,6 +149,85 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st | |||
174 | } | 149 | } |
175 | } | 150 | } |
176 | 151 | ||
152 | async function check0PlaylistRedundancies (videoUUID?: string) { | ||
153 | if (!videoUUID) videoUUID = video1Server2UUID | ||
154 | |||
155 | for (const server of servers) { | ||
156 | // With token to avoid issues with video follow constraints | ||
157 | const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) | ||
158 | const video: VideoDetails = res.body | ||
159 | |||
160 | expect(video.streamingPlaylists).to.be.an('array') | ||
161 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
162 | expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0) | ||
163 | } | ||
164 | } | ||
165 | |||
166 | async function check1PlaylistRedundancies (videoUUID?: string) { | ||
167 | if (!videoUUID) videoUUID = video1Server2UUID | ||
168 | |||
169 | for (const server of servers) { | ||
170 | const res = await getVideo(server.url, videoUUID) | ||
171 | const video: VideoDetails = res.body | ||
172 | |||
173 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
174 | expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1) | ||
175 | |||
176 | const redundancy = video.streamingPlaylists[0].redundancies[0] | ||
177 | |||
178 | expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) | ||
179 | } | ||
180 | |||
181 | const baseUrlPlaylist = servers[1].url + '/static/playlists/hls' | ||
182 | const baseUrlSegment = servers[0].url + '/static/redundancy/hls' | ||
183 | |||
184 | const res = await getVideo(servers[0].url, videoUUID) | ||
185 | const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0] | ||
186 | |||
187 | for (const resolution of [ 240, 360, 480, 720 ]) { | ||
188 | await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist) | ||
189 | } | ||
190 | |||
191 | for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) { | ||
192 | const files = await readdir(join(root(), directory, videoUUID)) | ||
193 | expect(files).to.have.length.at.least(4) | ||
194 | |||
195 | for (const resolution of [ 240, 360, 480, 720 ]) { | ||
196 | const filename = `${videoUUID}-${resolution}-fragmented.mp4` | ||
197 | |||
198 | expect(files.find(f => f === filename)).to.not.be.undefined | ||
199 | } | ||
200 | } | ||
201 | } | ||
202 | |||
203 | async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { | ||
204 | const res = await getStats(servers[0].url) | ||
205 | const data: ServerStats = res.body | ||
206 | |||
207 | expect(data.videosRedundancy).to.have.lengthOf(1) | ||
208 | const stat = data.videosRedundancy[0] | ||
209 | |||
210 | expect(stat.strategy).to.equal(strategy) | ||
211 | expect(stat.totalSize).to.equal(204800) | ||
212 | expect(stat.totalUsed).to.be.at.least(1).and.below(204801) | ||
213 | expect(stat.totalVideoFiles).to.equal(4) | ||
214 | expect(stat.totalVideos).to.equal(1) | ||
215 | } | ||
216 | |||
217 | async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { | ||
218 | const res = await getStats(servers[0].url) | ||
219 | const data: ServerStats = res.body | ||
220 | |||
221 | expect(data.videosRedundancy).to.have.lengthOf(1) | ||
222 | |||
223 | const stat = data.videosRedundancy[0] | ||
224 | expect(stat.strategy).to.equal(strategy) | ||
225 | expect(stat.totalSize).to.equal(204800) | ||
226 | expect(stat.totalUsed).to.equal(0) | ||
227 | expect(stat.totalVideoFiles).to.equal(0) | ||
228 | expect(stat.totalVideos).to.equal(0) | ||
229 | } | ||
230 | |||
177 | async function enableRedundancyOnServer1 () { | 231 | async function enableRedundancyOnServer1 () { |
178 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) | 232 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) |
179 | 233 | ||
@@ -220,7 +274,8 @@ describe('Test videos redundancy', function () { | |||
220 | }) | 274 | }) |
221 | 275 | ||
222 | it('Should have 1 webseed on the first video', async function () { | 276 | it('Should have 1 webseed on the first video', async function () { |
223 | await check1WebSeed(strategy) | 277 | await check1WebSeed() |
278 | await check0PlaylistRedundancies() | ||
224 | await checkStatsWith1Webseed(strategy) | 279 | await checkStatsWith1Webseed(strategy) |
225 | }) | 280 | }) |
226 | 281 | ||
@@ -229,27 +284,29 @@ describe('Test videos redundancy', function () { | |||
229 | }) | 284 | }) |
230 | 285 | ||
231 | it('Should have 2 webseeds on the first video', async function () { | 286 | it('Should have 2 webseeds on the first video', async function () { |
232 | this.timeout(40000) | 287 | this.timeout(80000) |
233 | 288 | ||
234 | await waitJobs(servers) | 289 | await waitJobs(servers) |
235 | await waitUntilLog(servers[0], 'Duplicated ', 4) | 290 | await waitUntilLog(servers[0], 'Duplicated ', 5) |
236 | await waitJobs(servers) | 291 | await waitJobs(servers) |
237 | 292 | ||
238 | await check2Webseeds(strategy) | 293 | await check2Webseeds() |
294 | await check1PlaylistRedundancies() | ||
239 | await checkStatsWith2Webseed(strategy) | 295 | await checkStatsWith2Webseed(strategy) |
240 | }) | 296 | }) |
241 | 297 | ||
242 | it('Should undo redundancy on server 1 and remove duplicated videos', async function () { | 298 | it('Should undo redundancy on server 1 and remove duplicated videos', async function () { |
243 | this.timeout(40000) | 299 | this.timeout(80000) |
244 | 300 | ||
245 | await disableRedundancyOnServer1() | 301 | await disableRedundancyOnServer1() |
246 | 302 | ||
247 | await waitJobs(servers) | 303 | await waitJobs(servers) |
248 | await wait(5000) | 304 | await wait(5000) |
249 | 305 | ||
250 | await check1WebSeed(strategy) | 306 | await check1WebSeed() |
307 | await check0PlaylistRedundancies() | ||
251 | 308 | ||
252 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) | 309 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ]) |
253 | }) | 310 | }) |
254 | 311 | ||
255 | after(function () { | 312 | after(function () { |
@@ -267,7 +324,8 @@ describe('Test videos redundancy', function () { | |||
267 | }) | 324 | }) |
268 | 325 | ||
269 | it('Should have 1 webseed on the first video', async function () { | 326 | it('Should have 1 webseed on the first video', async function () { |
270 | await check1WebSeed(strategy) | 327 | await check1WebSeed() |
328 | await check0PlaylistRedundancies() | ||
271 | await checkStatsWith1Webseed(strategy) | 329 | await checkStatsWith1Webseed(strategy) |
272 | }) | 330 | }) |
273 | 331 | ||
@@ -276,25 +334,27 @@ describe('Test videos redundancy', function () { | |||
276 | }) | 334 | }) |
277 | 335 | ||
278 | it('Should have 2 webseeds on the first video', async function () { | 336 | it('Should have 2 webseeds on the first video', async function () { |
279 | this.timeout(40000) | 337 | this.timeout(80000) |
280 | 338 | ||
281 | await waitJobs(servers) | 339 | await waitJobs(servers) |
282 | await waitUntilLog(servers[0], 'Duplicated ', 4) | 340 | await waitUntilLog(servers[0], 'Duplicated ', 5) |
283 | await waitJobs(servers) | 341 | await waitJobs(servers) |
284 | 342 | ||
285 | await check2Webseeds(strategy) | 343 | await check2Webseeds() |
344 | await check1PlaylistRedundancies() | ||
286 | await checkStatsWith2Webseed(strategy) | 345 | await checkStatsWith2Webseed(strategy) |
287 | }) | 346 | }) |
288 | 347 | ||
289 | it('Should unfollow on server 1 and remove duplicated videos', async function () { | 348 | it('Should unfollow on server 1 and remove duplicated videos', async function () { |
290 | this.timeout(40000) | 349 | this.timeout(80000) |
291 | 350 | ||
292 | await unfollow(servers[0].url, servers[0].accessToken, servers[1]) | 351 | await unfollow(servers[0].url, servers[0].accessToken, servers[1]) |
293 | 352 | ||
294 | await waitJobs(servers) | 353 | await waitJobs(servers) |
295 | await wait(5000) | 354 | await wait(5000) |
296 | 355 | ||
297 | await check1WebSeed(strategy) | 356 | await check1WebSeed() |
357 | await check0PlaylistRedundancies() | ||
298 | 358 | ||
299 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) | 359 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) |
300 | }) | 360 | }) |
@@ -314,7 +374,8 @@ describe('Test videos redundancy', function () { | |||
314 | }) | 374 | }) |
315 | 375 | ||
316 | it('Should have 1 webseed on the first video', async function () { | 376 | it('Should have 1 webseed on the first video', async function () { |
317 | await check1WebSeed(strategy) | 377 | await check1WebSeed() |
378 | await check0PlaylistRedundancies() | ||
318 | await checkStatsWith1Webseed(strategy) | 379 | await checkStatsWith1Webseed(strategy) |
319 | }) | 380 | }) |
320 | 381 | ||
@@ -323,18 +384,19 @@ describe('Test videos redundancy', function () { | |||
323 | }) | 384 | }) |
324 | 385 | ||
325 | it('Should still have 1 webseed on the first video', async function () { | 386 | it('Should still have 1 webseed on the first video', async function () { |
326 | this.timeout(40000) | 387 | this.timeout(80000) |
327 | 388 | ||
328 | await waitJobs(servers) | 389 | await waitJobs(servers) |
329 | await wait(15000) | 390 | await wait(15000) |
330 | await waitJobs(servers) | 391 | await waitJobs(servers) |
331 | 392 | ||
332 | await check1WebSeed(strategy) | 393 | await check1WebSeed() |
394 | await check0PlaylistRedundancies() | ||
333 | await checkStatsWith1Webseed(strategy) | 395 | await checkStatsWith1Webseed(strategy) |
334 | }) | 396 | }) |
335 | 397 | ||
336 | it('Should view 2 times the first video to have > min_views config', async function () { | 398 | it('Should view 2 times the first video to have > min_views config', async function () { |
337 | this.timeout(40000) | 399 | this.timeout(80000) |
338 | 400 | ||
339 | await viewVideo(servers[ 0 ].url, video1Server2UUID) | 401 | await viewVideo(servers[ 0 ].url, video1Server2UUID) |
340 | await viewVideo(servers[ 2 ].url, video1Server2UUID) | 402 | await viewVideo(servers[ 2 ].url, video1Server2UUID) |
@@ -344,13 +406,14 @@ describe('Test videos redundancy', function () { | |||
344 | }) | 406 | }) |
345 | 407 | ||
346 | it('Should have 2 webseeds on the first video', async function () { | 408 | it('Should have 2 webseeds on the first video', async function () { |
347 | this.timeout(40000) | 409 | this.timeout(80000) |
348 | 410 | ||
349 | await waitJobs(servers) | 411 | await waitJobs(servers) |
350 | await waitUntilLog(servers[0], 'Duplicated ', 4) | 412 | await waitUntilLog(servers[0], 'Duplicated ', 5) |
351 | await waitJobs(servers) | 413 | await waitJobs(servers) |
352 | 414 | ||
353 | await check2Webseeds(strategy) | 415 | await check2Webseeds() |
416 | await check1PlaylistRedundancies() | ||
354 | await checkStatsWith2Webseed(strategy) | 417 | await checkStatsWith2Webseed(strategy) |
355 | }) | 418 | }) |
356 | 419 | ||
@@ -405,7 +468,7 @@ describe('Test videos redundancy', function () { | |||
405 | }) | 468 | }) |
406 | 469 | ||
407 | it('Should still have 2 webseeds after 10 seconds', async function () { | 470 | it('Should still have 2 webseeds after 10 seconds', async function () { |
408 | this.timeout(40000) | 471 | this.timeout(80000) |
409 | 472 | ||
410 | await wait(10000) | 473 | await wait(10000) |
411 | 474 | ||
@@ -420,7 +483,7 @@ describe('Test videos redundancy', function () { | |||
420 | }) | 483 | }) |
421 | 484 | ||
422 | it('Should stop server 1 and expire video redundancy', async function () { | 485 | it('Should stop server 1 and expire video redundancy', async function () { |
423 | this.timeout(40000) | 486 | this.timeout(80000) |
424 | 487 | ||
425 | killallServers([ servers[0] ]) | 488 | killallServers([ servers[0] ]) |
426 | 489 | ||
@@ -446,10 +509,11 @@ describe('Test videos redundancy', function () { | |||
446 | await enableRedundancyOnServer1() | 509 | await enableRedundancyOnServer1() |
447 | 510 | ||
448 | await waitJobs(servers) | 511 | await waitJobs(servers) |
449 | await waitUntilLog(servers[0], 'Duplicated ', 4) | 512 | await waitUntilLog(servers[0], 'Duplicated ', 5) |
450 | await waitJobs(servers) | 513 | await waitJobs(servers) |
451 | 514 | ||
452 | await check2Webseeds(strategy) | 515 | await check2Webseeds() |
516 | await check1PlaylistRedundancies() | ||
453 | await checkStatsWith2Webseed(strategy) | 517 | await checkStatsWith2Webseed(strategy) |
454 | 518 | ||
455 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) | 519 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) |
@@ -467,8 +531,10 @@ describe('Test videos redundancy', function () { | |||
467 | await wait(1000) | 531 | await wait(1000) |
468 | 532 | ||
469 | try { | 533 | try { |
470 | await check1WebSeed(strategy, video1Server2UUID) | 534 | await check1WebSeed(video1Server2UUID) |
471 | await check2Webseeds(strategy, video2Server2UUID) | 535 | await check0PlaylistRedundancies(video1Server2UUID) |
536 | await check2Webseeds(video2Server2UUID) | ||
537 | await check1PlaylistRedundancies(video2Server2UUID) | ||
472 | 538 | ||
473 | checked = true | 539 | checked = true |
474 | } catch { | 540 | } catch { |
@@ -477,6 +543,26 @@ describe('Test videos redundancy', function () { | |||
477 | } | 543 | } |
478 | }) | 544 | }) |
479 | 545 | ||
546 | it('Should disable strategy and remove redundancies', async function () { | ||
547 | this.timeout(80000) | ||
548 | |||
549 | await waitJobs(servers) | ||
550 | |||
551 | killallServers([ servers[ 0 ] ]) | ||
552 | await reRunServer(servers[ 0 ], { | ||
553 | redundancy: { | ||
554 | videos: { | ||
555 | check_interval: '1 second', | ||
556 | strategies: [] | ||
557 | } | ||
558 | } | ||
559 | }) | ||
560 | |||
561 | await waitJobs(servers) | ||
562 | |||
563 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ]) | ||
564 | }) | ||
565 | |||
480 | after(function () { | 566 | after(function () { |
481 | return cleanServers() | 567 | return cleanServers() |
482 | }) | 568 | }) |
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index bebfc7398..0dfe6e4fe 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts | |||
@@ -57,6 +57,8 @@ function checkInitialConfig (data: CustomConfig) { | |||
57 | expect(data.transcoding.resolutions['480p']).to.be.true | 57 | expect(data.transcoding.resolutions['480p']).to.be.true |
58 | expect(data.transcoding.resolutions['720p']).to.be.true | 58 | expect(data.transcoding.resolutions['720p']).to.be.true |
59 | expect(data.transcoding.resolutions['1080p']).to.be.true | 59 | expect(data.transcoding.resolutions['1080p']).to.be.true |
60 | expect(data.transcoding.hls.enabled).to.be.true | ||
61 | |||
60 | expect(data.import.videos.http.enabled).to.be.true | 62 | expect(data.import.videos.http.enabled).to.be.true |
61 | expect(data.import.videos.torrent.enabled).to.be.true | 63 | expect(data.import.videos.torrent.enabled).to.be.true |
62 | } | 64 | } |
@@ -95,6 +97,7 @@ function checkUpdatedConfig (data: CustomConfig) { | |||
95 | expect(data.transcoding.resolutions['480p']).to.be.true | 97 | expect(data.transcoding.resolutions['480p']).to.be.true |
96 | expect(data.transcoding.resolutions['720p']).to.be.false | 98 | expect(data.transcoding.resolutions['720p']).to.be.false |
97 | expect(data.transcoding.resolutions['1080p']).to.be.false | 99 | expect(data.transcoding.resolutions['1080p']).to.be.false |
100 | expect(data.transcoding.hls.enabled).to.be.false | ||
98 | 101 | ||
99 | expect(data.import.videos.http.enabled).to.be.false | 102 | expect(data.import.videos.http.enabled).to.be.false |
100 | expect(data.import.videos.torrent.enabled).to.be.false | 103 | expect(data.import.videos.torrent.enabled).to.be.false |
@@ -205,6 +208,9 @@ describe('Test config', function () { | |||
205 | '480p': true, | 208 | '480p': true, |
206 | '720p': false, | 209 | '720p': false, |
207 | '1080p': false | 210 | '1080p': false |
211 | }, | ||
212 | hls: { | ||
213 | enabled: false | ||
208 | } | 214 | } |
209 | }, | 215 | }, |
210 | import: { | 216 | import: { |
diff --git a/server/tests/api/users/user-notifications.ts b/server/tests/api/users/user-notifications.ts index 5260d64cc..69e51677e 100644 --- a/server/tests/api/users/user-notifications.ts +++ b/server/tests/api/users/user-notifications.ts | |||
@@ -165,6 +165,8 @@ describe('Test users notifications', function () { | |||
165 | }) | 165 | }) |
166 | 166 | ||
167 | it('Should not send notifications if the user does not follow the video publisher', async function () { | 167 | it('Should not send notifications if the user does not follow the video publisher', async function () { |
168 | this.timeout(10000) | ||
169 | |||
168 | await uploadVideoByLocalAccount(servers) | 170 | await uploadVideoByLocalAccount(servers) |
169 | 171 | ||
170 | const notification = await getLastNotification(servers[ 0 ].url, userAccessToken) | 172 | const notification = await getLastNotification(servers[ 0 ].url, userAccessToken) |
@@ -644,6 +646,8 @@ describe('Test users notifications', function () { | |||
644 | }) | 646 | }) |
645 | 647 | ||
646 | it('Should not send a notification if transcoding is not enabled', async function () { | 648 | it('Should not send a notification if transcoding is not enabled', async function () { |
649 | this.timeout(10000) | ||
650 | |||
647 | const { name, uuid } = await uploadVideoByLocalAccount(servers) | 651 | const { name, uuid } = await uploadVideoByLocalAccount(servers) |
648 | await waitJobs(servers) | 652 | await waitJobs(servers) |
649 | 653 | ||
@@ -717,6 +721,24 @@ describe('Test users notifications', function () { | |||
717 | await wait(6000) | 721 | await wait(6000) |
718 | await checkVideoIsPublished(baseParams, name, uuid, 'presence') | 722 | await checkVideoIsPublished(baseParams, name, uuid, 'presence') |
719 | }) | 723 | }) |
724 | |||
725 | it('Should not send a notification before the video is published', async function () { | ||
726 | this.timeout(20000) | ||
727 | |||
728 | let updateAt = new Date(new Date().getTime() + 100000) | ||
729 | |||
730 | const data = { | ||
731 | privacy: VideoPrivacy.PRIVATE, | ||
732 | scheduleUpdate: { | ||
733 | updateAt: updateAt.toISOString(), | ||
734 | privacy: VideoPrivacy.PUBLIC | ||
735 | } | ||
736 | } | ||
737 | const { name, uuid } = await uploadVideoByRemoteAccount(servers, data) | ||
738 | |||
739 | await wait(6000) | ||
740 | await checkVideoIsPublished(baseParams, name, uuid, 'absence') | ||
741 | }) | ||
720 | }) | 742 | }) |
721 | 743 | ||
722 | describe('My video is imported', function () { | 744 | describe('My video is imported', function () { |
@@ -781,6 +803,8 @@ describe('Test users notifications', function () { | |||
781 | }) | 803 | }) |
782 | 804 | ||
783 | it('Should send a notification only to moderators when a user registers on the instance', async function () { | 805 | it('Should send a notification only to moderators when a user registers on the instance', async function () { |
806 | this.timeout(10000) | ||
807 | |||
784 | await registerUser(servers[0].url, 'user_45', 'password') | 808 | await registerUser(servers[0].url, 'user_45', 'password') |
785 | 809 | ||
786 | await waitJobs(servers) | 810 | await waitJobs(servers) |
@@ -849,6 +873,8 @@ describe('Test users notifications', function () { | |||
849 | }) | 873 | }) |
850 | 874 | ||
851 | it('Should notify when a local account is following one of our channel', async function () { | 875 | it('Should notify when a local account is following one of our channel', async function () { |
876 | this.timeout(10000) | ||
877 | |||
852 | await addUserSubscription(servers[0].url, servers[0].accessToken, 'user_1@localhost:9001') | 878 | await addUserSubscription(servers[0].url, servers[0].accessToken, 'user_1@localhost:9001') |
853 | 879 | ||
854 | await waitJobs(servers) | 880 | await waitJobs(servers) |
@@ -857,6 +883,8 @@ describe('Test users notifications', function () { | |||
857 | }) | 883 | }) |
858 | 884 | ||
859 | it('Should notify when a remote account is following one of our channel', async function () { | 885 | it('Should notify when a remote account is following one of our channel', async function () { |
886 | this.timeout(10000) | ||
887 | |||
860 | await addUserSubscription(servers[1].url, servers[1].accessToken, 'user_1@localhost:9001') | 888 | await addUserSubscription(servers[1].url, servers[1].accessToken, 'user_1@localhost:9001') |
861 | 889 | ||
862 | await waitJobs(servers) | 890 | await waitJobs(servers) |
@@ -926,6 +954,8 @@ describe('Test users notifications', function () { | |||
926 | }) | 954 | }) |
927 | 955 | ||
928 | it('Should not have notifications', async function () { | 956 | it('Should not have notifications', async function () { |
957 | this.timeout(10000) | ||
958 | |||
929 | await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { | 959 | await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { |
930 | newVideoFromSubscription: UserNotificationSettingValue.NONE | 960 | newVideoFromSubscription: UserNotificationSettingValue.NONE |
931 | })) | 961 | })) |
@@ -943,6 +973,8 @@ describe('Test users notifications', function () { | |||
943 | }) | 973 | }) |
944 | 974 | ||
945 | it('Should only have web notifications', async function () { | 975 | it('Should only have web notifications', async function () { |
976 | this.timeout(10000) | ||
977 | |||
946 | await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { | 978 | await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { |
947 | newVideoFromSubscription: UserNotificationSettingValue.WEB | 979 | newVideoFromSubscription: UserNotificationSettingValue.WEB |
948 | })) | 980 | })) |
@@ -967,6 +999,8 @@ describe('Test users notifications', function () { | |||
967 | }) | 999 | }) |
968 | 1000 | ||
969 | it('Should only have mail notifications', async function () { | 1001 | it('Should only have mail notifications', async function () { |
1002 | this.timeout(10000) | ||
1003 | |||
970 | await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { | 1004 | await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { |
971 | newVideoFromSubscription: UserNotificationSettingValue.EMAIL | 1005 | newVideoFromSubscription: UserNotificationSettingValue.EMAIL |
972 | })) | 1006 | })) |
@@ -991,6 +1025,8 @@ describe('Test users notifications', function () { | |||
991 | }) | 1025 | }) |
992 | 1026 | ||
993 | it('Should have email and web notifications', async function () { | 1027 | it('Should have email and web notifications', async function () { |
1028 | this.timeout(10000) | ||
1029 | |||
994 | await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { | 1030 | await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { |
995 | newVideoFromSubscription: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL | 1031 | newVideoFromSubscription: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL |
996 | })) | 1032 | })) |
diff --git a/server/tests/api/users/users.ts b/server/tests/api/users/users.ts index ad98ab1c7..c4465d541 100644 --- a/server/tests/api/users/users.ts +++ b/server/tests/api/users/users.ts | |||
@@ -501,6 +501,22 @@ describe('Test users', function () { | |||
501 | accessTokenUser = await userLogin(server, user) | 501 | accessTokenUser = await userLogin(server, user) |
502 | }) | 502 | }) |
503 | 503 | ||
504 | it('Should be able to update another user password', async function () { | ||
505 | await updateUser({ | ||
506 | url: server.url, | ||
507 | userId, | ||
508 | accessToken, | ||
509 | password: 'password updated' | ||
510 | }) | ||
511 | |||
512 | await getMyUserVideoQuotaUsed(server.url, accessTokenUser, 401) | ||
513 | |||
514 | await userLogin(server, user, 400) | ||
515 | |||
516 | user.password = 'password updated' | ||
517 | accessTokenUser = await userLogin(server, user) | ||
518 | }) | ||
519 | |||
504 | it('Should be able to list video blacklist by a moderator', async function () { | 520 | it('Should be able to list video blacklist by a moderator', async function () { |
505 | await getBlacklistedVideosList(server.url, accessTokenUser) | 521 | await getBlacklistedVideosList(server.url, accessTokenUser) |
506 | }) | 522 | }) |
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index 97f467aae..a501a80b2 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts | |||
@@ -8,6 +8,7 @@ import './video-change-ownership' | |||
8 | import './video-channels' | 8 | import './video-channels' |
9 | import './video-comments' | 9 | import './video-comments' |
10 | import './video-description' | 10 | import './video-description' |
11 | import './video-hls' | ||
11 | import './video-imports' | 12 | import './video-imports' |
12 | import './video-nsfw' | 13 | import './video-nsfw' |
13 | import './video-privacy' | 14 | import './video-privacy' |
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts new file mode 100644 index 000000000..a1214bad1 --- /dev/null +++ b/server/tests/api/videos/video-hls.ts | |||
@@ -0,0 +1,139 @@ | |||
1 | /* tslint:disable:no-unused-expression */ | ||
2 | |||
3 | import * as chai from 'chai' | ||
4 | import 'mocha' | ||
5 | import { | ||
6 | checkDirectoryIsEmpty, | ||
7 | checkSegmentHash, | ||
8 | checkTmpIsEmpty, | ||
9 | doubleFollow, | ||
10 | flushAndRunMultipleServers, | ||
11 | flushTests, | ||
12 | getPlaylist, | ||
13 | getVideo, | ||
14 | killallServers, | ||
15 | removeVideo, | ||
16 | ServerInfo, | ||
17 | setAccessTokensToServers, | ||
18 | updateVideo, | ||
19 | uploadVideo, | ||
20 | waitJobs | ||
21 | } from '../../../../shared/utils' | ||
22 | import { VideoDetails } from '../../../../shared/models/videos' | ||
23 | import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type' | ||
24 | import { join } from 'path' | ||
25 | |||
26 | const expect = chai.expect | ||
27 | |||
28 | async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string) { | ||
29 | const resolutions = [ 240, 360, 480, 720 ] | ||
30 | |||
31 | for (const server of servers) { | ||
32 | const res = await getVideo(server.url, videoUUID) | ||
33 | const videoDetails: VideoDetails = res.body | ||
34 | |||
35 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) | ||
36 | |||
37 | const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) | ||
38 | expect(hlsPlaylist).to.not.be.undefined | ||
39 | |||
40 | { | ||
41 | const res2 = await getPlaylist(hlsPlaylist.playlistUrl) | ||
42 | |||
43 | const masterPlaylist = res2.text | ||
44 | |||
45 | expect(masterPlaylist).to.contain('#EXT-X-STREAM-INF:BANDWIDTH=55472,RESOLUTION=640x360,FRAME-RATE=25') | ||
46 | |||
47 | for (const resolution of resolutions) { | ||
48 | expect(masterPlaylist).to.contain(`${resolution}.m3u8`) | ||
49 | } | ||
50 | } | ||
51 | |||
52 | { | ||
53 | for (const resolution of resolutions) { | ||
54 | const res2 = await getPlaylist(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}.m3u8`) | ||
55 | |||
56 | const subPlaylist = res2.text | ||
57 | expect(subPlaylist).to.contain(`${videoUUID}-${resolution}-fragmented.mp4`) | ||
58 | } | ||
59 | } | ||
60 | |||
61 | { | ||
62 | const baseUrl = 'http://localhost:9001/static/playlists/hls' | ||
63 | |||
64 | for (const resolution of resolutions) { | ||
65 | await checkSegmentHash(baseUrl, baseUrl, videoUUID, resolution, hlsPlaylist) | ||
66 | } | ||
67 | } | ||
68 | } | ||
69 | } | ||
70 | |||
71 | describe('Test HLS videos', function () { | ||
72 | let servers: ServerInfo[] = [] | ||
73 | let videoUUID = '' | ||
74 | |||
75 | before(async function () { | ||
76 | this.timeout(120000) | ||
77 | |||
78 | servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true, hls: { enabled: true } } }) | ||
79 | |||
80 | // Get the access tokens | ||
81 | await setAccessTokensToServers(servers) | ||
82 | |||
83 | // Server 1 and server 2 follow each other | ||
84 | await doubleFollow(servers[0], servers[1]) | ||
85 | }) | ||
86 | |||
87 | it('Should upload a video and transcode it to HLS', async function () { | ||
88 | this.timeout(120000) | ||
89 | |||
90 | { | ||
91 | const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1', fixture: 'video_short.webm' }) | ||
92 | videoUUID = res.body.video.uuid | ||
93 | } | ||
94 | |||
95 | await waitJobs(servers) | ||
96 | |||
97 | await checkHlsPlaylist(servers, videoUUID) | ||
98 | }) | ||
99 | |||
100 | it('Should update the video', async function () { | ||
101 | await updateVideo(servers[0].url, servers[0].accessToken, videoUUID, { name: 'video 1 updated' }) | ||
102 | |||
103 | await waitJobs(servers) | ||
104 | |||
105 | await checkHlsPlaylist(servers, videoUUID) | ||
106 | }) | ||
107 | |||
108 | it('Should delete the video', async function () { | ||
109 | await removeVideo(servers[0].url, servers[0].accessToken, videoUUID) | ||
110 | |||
111 | await waitJobs(servers) | ||
112 | |||
113 | for (const server of servers) { | ||
114 | await getVideo(server.url, videoUUID, 404) | ||
115 | } | ||
116 | }) | ||
117 | |||
118 | it('Should have the playlists/segment deleted from the disk', async function () { | ||
119 | for (const server of servers) { | ||
120 | await checkDirectoryIsEmpty(server, 'videos') | ||
121 | await checkDirectoryIsEmpty(server, join('playlists', 'hls')) | ||
122 | } | ||
123 | }) | ||
124 | |||
125 | it('Should have an empty tmp directory', async function () { | ||
126 | for (const server of servers) { | ||
127 | await checkTmpIsEmpty(server) | ||
128 | } | ||
129 | }) | ||
130 | |||
131 | after(async function () { | ||
132 | killallServers(servers) | ||
133 | |||
134 | // Keep the logs if the test failed | ||
135 | if (this['ok']) { | ||
136 | await flushTests() | ||
137 | } | ||
138 | }) | ||
139 | }) | ||
diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts index 811ea6a9f..d38bb4331 100644 --- a/server/tests/cli/update-host.ts +++ b/server/tests/cli/update-host.ts | |||
@@ -86,6 +86,13 @@ describe('Test update host scripts', function () { | |||
86 | const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid) | 86 | const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid) |
87 | 87 | ||
88 | expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid) | 88 | expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid) |
89 | |||
90 | const res = await getVideo(server.url, video.uuid) | ||
91 | const videoDetails: VideoDetails = res.body | ||
92 | |||
93 | expect(videoDetails.trackerUrls[0]).to.include(server.host) | ||
94 | expect(videoDetails.streamingPlaylists[0].playlistUrl).to.include(server.host) | ||
95 | expect(videoDetails.streamingPlaylists[0].segmentsSha256Url).to.include(server.host) | ||
89 | } | 96 | } |
90 | }) | 97 | }) |
91 | 98 | ||
@@ -100,7 +107,7 @@ describe('Test update host scripts', function () { | |||
100 | } | 107 | } |
101 | }) | 108 | }) |
102 | 109 | ||
103 | it('Should have update accounts url', async function () { | 110 | it('Should have updated accounts url', async function () { |
104 | const res = await getAccountsList(server.url) | 111 | const res = await getAccountsList(server.url) |
105 | expect(res.body.total).to.equal(3) | 112 | expect(res.body.total).to.equal(3) |
106 | 113 | ||
@@ -112,7 +119,7 @@ describe('Test update host scripts', function () { | |||
112 | } | 119 | } |
113 | }) | 120 | }) |
114 | 121 | ||
115 | it('Should update torrent hosts', async function () { | 122 | it('Should have updated torrent hosts', async function () { |
116 | this.timeout(30000) | 123 | this.timeout(30000) |
117 | 124 | ||
118 | const res = await getVideosList(server.url) | 125 | const res = await getVideosList(server.url) |