diff options
Diffstat (limited to 'server')
39 files changed, 1452 insertions, 272 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index 1a4e28dc8..32a83aa5f 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts | |||
@@ -37,7 +37,7 @@ import { | |||
37 | getVideoSharesActivityPubUrl | 37 | getVideoSharesActivityPubUrl |
38 | } from '../../lib/activitypub' | 38 | } from '../../lib/activitypub' |
39 | import { VideoCaptionModel } from '../../models/video/video-caption' | 39 | import { VideoCaptionModel } from '../../models/video/video-caption' |
40 | import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy' | 40 | import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy' |
41 | import { getServerActor } from '../../helpers/utils' | 41 | import { getServerActor } from '../../helpers/utils' |
42 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 42 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
43 | 43 | ||
@@ -66,11 +66,11 @@ activityPubClientRouter.get('/accounts?/:name/dislikes/:videoId', | |||
66 | 66 | ||
67 | activityPubClientRouter.get('/videos/watch/:id', | 67 | activityPubClientRouter.get('/videos/watch/:id', |
68 | executeIfActivityPub(asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS))), | 68 | executeIfActivityPub(asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS))), |
69 | executeIfActivityPub(asyncMiddleware(videosGetValidator)), | 69 | executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))), |
70 | executeIfActivityPub(asyncMiddleware(videoController)) | 70 | executeIfActivityPub(asyncMiddleware(videoController)) |
71 | ) | 71 | ) |
72 | activityPubClientRouter.get('/videos/watch/:id/activity', | 72 | activityPubClientRouter.get('/videos/watch/:id/activity', |
73 | executeIfActivityPub(asyncMiddleware(videosGetValidator)), | 73 | executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))), |
74 | executeIfActivityPub(asyncMiddleware(videoController)) | 74 | executeIfActivityPub(asyncMiddleware(videoController)) |
75 | ) | 75 | ) |
76 | activityPubClientRouter.get('/videos/watch/:id/announces', | 76 | activityPubClientRouter.get('/videos/watch/:id/announces', |
@@ -116,7 +116,11 @@ activityPubClientRouter.get('/video-channels/:name/following', | |||
116 | ) | 116 | ) |
117 | 117 | ||
118 | activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?', | 118 | activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?', |
119 | executeIfActivityPub(asyncMiddleware(videoRedundancyGetValidator)), | 119 | executeIfActivityPub(asyncMiddleware(videoFileRedundancyGetValidator)), |
120 | executeIfActivityPub(asyncMiddleware(videoRedundancyController)) | ||
121 | ) | ||
122 | activityPubClientRouter.get('/redundancy/video-playlists/:streamingPlaylistType/:videoId', | ||
123 | executeIfActivityPub(asyncMiddleware(videoPlaylistRedundancyGetValidator)), | ||
120 | executeIfActivityPub(asyncMiddleware(videoRedundancyController)) | 124 | executeIfActivityPub(asyncMiddleware(videoRedundancyController)) |
121 | ) | 125 | ) |
122 | 126 | ||
@@ -163,7 +167,8 @@ function getAccountVideoRate (rateType: VideoRateType) { | |||
163 | } | 167 | } |
164 | 168 | ||
165 | async function videoController (req: express.Request, res: express.Response) { | 169 | async function videoController (req: express.Request, res: express.Response) { |
166 | const video: VideoModel = res.locals.video | 170 | // We need more attributes |
171 | const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id) | ||
167 | 172 | ||
168 | if (video.url.startsWith(CONFIG.WEBSERVER.URL) === false) return res.redirect(video.url) | 173 | if (video.url.startsWith(CONFIG.WEBSERVER.URL) === false) return res.redirect(video.url) |
169 | 174 | ||
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 255026f46..1f3341bc0 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { omit, snakeCase } from 'lodash' | 2 | import { snakeCase } from 'lodash' |
3 | import { ServerConfig, UserRight } from '../../../shared' | 3 | import { ServerConfig, UserRight } from '../../../shared' |
4 | import { About } from '../../../shared/models/server/about.model' | 4 | import { About } from '../../../shared/models/server/about.model' |
5 | import { CustomConfig } from '../../../shared/models/server/custom-config.model' | 5 | import { CustomConfig } from '../../../shared/models/server/custom-config.model' |
@@ -78,6 +78,9 @@ async function getConfig (req: express.Request, res: express.Response) { | |||
78 | requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION | 78 | requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION |
79 | }, | 79 | }, |
80 | transcoding: { | 80 | transcoding: { |
81 | hls: { | ||
82 | enabled: CONFIG.TRANSCODING.HLS.ENABLED | ||
83 | }, | ||
81 | enabledResolutions | 84 | enabledResolutions |
82 | }, | 85 | }, |
83 | import: { | 86 | import: { |
@@ -246,6 +249,9 @@ function customConfig (): CustomConfig { | |||
246 | '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ], | 249 | '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ], |
247 | '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ], | 250 | '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ], |
248 | '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ] | 251 | '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ] |
252 | }, | ||
253 | hls: { | ||
254 | enabled: CONFIG.TRANSCODING.HLS.ENABLED | ||
249 | } | 255 | } |
250 | }, | 256 | }, |
251 | import: { | 257 | import: { |
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 2b2dfa7ca..e04fc8186 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts | |||
@@ -37,6 +37,7 @@ import { | |||
37 | setDefaultPagination, | 37 | setDefaultPagination, |
38 | setDefaultSort, | 38 | setDefaultSort, |
39 | videosAddValidator, | 39 | videosAddValidator, |
40 | videosCustomGetValidator, | ||
40 | videosGetValidator, | 41 | videosGetValidator, |
41 | videosRemoveValidator, | 42 | videosRemoveValidator, |
42 | videosSortValidator, | 43 | videosSortValidator, |
@@ -123,9 +124,9 @@ videosRouter.get('/:id/description', | |||
123 | ) | 124 | ) |
124 | videosRouter.get('/:id', | 125 | videosRouter.get('/:id', |
125 | optionalAuthenticate, | 126 | optionalAuthenticate, |
126 | asyncMiddleware(videosGetValidator), | 127 | asyncMiddleware(videosCustomGetValidator('only-video-with-rights')), |
127 | asyncMiddleware(checkVideoFollowConstraints), | 128 | asyncMiddleware(checkVideoFollowConstraints), |
128 | getVideo | 129 | asyncMiddleware(getVideo) |
129 | ) | 130 | ) |
130 | videosRouter.post('/:id/views', | 131 | videosRouter.post('/:id/views', |
131 | asyncMiddleware(videosGetValidator), | 132 | asyncMiddleware(videosGetValidator), |
@@ -395,15 +396,17 @@ async function updateVideo (req: express.Request, res: express.Response) { | |||
395 | return res.type('json').status(204).end() | 396 | return res.type('json').status(204).end() |
396 | } | 397 | } |
397 | 398 | ||
398 | function getVideo (req: express.Request, res: express.Response) { | 399 | async function getVideo (req: express.Request, res: express.Response) { |
399 | const videoInstance = res.locals.video | 400 | // We need more attributes |
401 | const userId: number = res.locals.oauth ? res.locals.oauth.token.User.id : null | ||
402 | const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id, undefined, userId) | ||
400 | 403 | ||
401 | if (videoInstance.isOutdated()) { | 404 | if (video.isOutdated()) { |
402 | JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoInstance.url } }) | 405 | JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } }) |
403 | .catch(err => logger.error('Cannot create AP refresher job for video %s.', videoInstance.url, { err })) | 406 | .catch(err => logger.error('Cannot create AP refresher job for video %s.', video.url, { err })) |
404 | } | 407 | } |
405 | 408 | ||
406 | return res.json(videoInstance.toFormattedDetailsJSON()) | 409 | return res.json(video.toFormattedDetailsJSON()) |
407 | } | 410 | } |
408 | 411 | ||
409 | async function viewVideo (req: express.Request, res: express.Response) { | 412 | async function viewVideo (req: express.Request, res: express.Response) { |
diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 4fd58f70c..b21f9da00 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import * as cors from 'cors' | 1 | import * as cors from 'cors' |
2 | import * as express from 'express' | 2 | import * as express from 'express' |
3 | import { CONFIG, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers' | 3 | import { CONFIG, HLS_PLAYLIST_DIRECTORY, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers' |
4 | import { VideosPreviewCache } from '../lib/cache' | 4 | import { VideosPreviewCache } from '../lib/cache' |
5 | import { cacheRoute } from '../middlewares/cache' | 5 | import { cacheRoute } from '../middlewares/cache' |
6 | import { asyncMiddleware, videosGetValidator } from '../middlewares' | 6 | import { asyncMiddleware, videosGetValidator } from '../middlewares' |
@@ -51,6 +51,13 @@ staticRouter.use( | |||
51 | asyncMiddleware(downloadVideoFile) | 51 | asyncMiddleware(downloadVideoFile) |
52 | ) | 52 | ) |
53 | 53 | ||
54 | // HLS | ||
55 | staticRouter.use( | ||
56 | STATIC_PATHS.PLAYLISTS.HLS, | ||
57 | cors(), | ||
58 | express.static(HLS_PLAYLIST_DIRECTORY, { fallthrough: false }) // 404 if the file does not exist | ||
59 | ) | ||
60 | |||
54 | // Thumbnails path for express | 61 | // Thumbnails path for express |
55 | const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR | 62 | const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR |
56 | staticRouter.use( | 63 | staticRouter.use( |
diff --git a/server/controllers/tracker.ts b/server/controllers/tracker.ts index 1deb8c402..8b77d9de7 100644 --- a/server/controllers/tracker.ts +++ b/server/controllers/tracker.ts | |||
@@ -7,6 +7,7 @@ import { Server as WebSocketServer } from 'ws' | |||
7 | import { CONFIG, TRACKER_RATE_LIMITS } from '../initializers/constants' | 7 | import { CONFIG, TRACKER_RATE_LIMITS } from '../initializers/constants' |
8 | import { VideoFileModel } from '../models/video/video-file' | 8 | import { VideoFileModel } from '../models/video/video-file' |
9 | import { parse } from 'url' | 9 | import { parse } from 'url' |
10 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
10 | 11 | ||
11 | const TrackerServer = bitTorrentTracker.Server | 12 | const TrackerServer = bitTorrentTracker.Server |
12 | 13 | ||
@@ -21,7 +22,7 @@ const trackerServer = new TrackerServer({ | |||
21 | udp: false, | 22 | udp: false, |
22 | ws: false, | 23 | ws: false, |
23 | dht: false, | 24 | dht: false, |
24 | filter: function (infoHash, params, cb) { | 25 | filter: async function (infoHash, params, cb) { |
25 | let ip: string | 26 | let ip: string |
26 | 27 | ||
27 | if (params.type === 'ws') { | 28 | if (params.type === 'ws') { |
@@ -32,19 +33,25 @@ const trackerServer = new TrackerServer({ | |||
32 | 33 | ||
33 | const key = ip + '-' + infoHash | 34 | const key = ip + '-' + infoHash |
34 | 35 | ||
35 | peersIps[ip] = peersIps[ip] ? peersIps[ip] + 1 : 1 | 36 | peersIps[ ip ] = peersIps[ ip ] ? peersIps[ ip ] + 1 : 1 |
36 | peersIpInfoHash[key] = peersIpInfoHash[key] ? peersIpInfoHash[key] + 1 : 1 | 37 | peersIpInfoHash[ key ] = peersIpInfoHash[ key ] ? peersIpInfoHash[ key ] + 1 : 1 |
37 | 38 | ||
38 | if (peersIpInfoHash[key] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) { | 39 | if (peersIpInfoHash[ key ] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) { |
39 | return cb(new Error(`Too many requests (${peersIpInfoHash[ key ]} of ip ${ip} for torrent ${infoHash}`)) | 40 | return cb(new Error(`Too many requests (${peersIpInfoHash[ key ]} of ip ${ip} for torrent ${infoHash}`)) |
40 | } | 41 | } |
41 | 42 | ||
42 | VideoFileModel.isInfohashExists(infoHash) | 43 | try { |
43 | .then(exists => { | 44 | const videoFileExists = await VideoFileModel.doesInfohashExist(infoHash) |
44 | if (exists === false) return cb(new Error(`Unknown infoHash ${infoHash}`)) | 45 | if (videoFileExists === true) return cb() |
45 | 46 | ||
46 | return cb() | 47 | const playlistExists = await VideoStreamingPlaylistModel.doesInfohashExist(infoHash) |
47 | }) | 48 | if (playlistExists === true) return cb() |
49 | |||
50 | return cb(new Error(`Unknown infoHash ${infoHash}`)) | ||
51 | } catch (err) { | ||
52 | logger.error('Error in tracker filter.', { err }) | ||
53 | return cb(err) | ||
54 | } | ||
48 | } | 55 | } |
49 | }) | 56 | }) |
50 | 57 | ||
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index f1430055f..eba552524 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts | |||
@@ -15,7 +15,7 @@ function activityPubContextify <T> (data: T) { | |||
15 | 'https://w3id.org/security/v1', | 15 | 'https://w3id.org/security/v1', |
16 | { | 16 | { |
17 | RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', | 17 | RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', |
18 | pt: 'https://joinpeertube.org/ns', | 18 | pt: 'https://joinpeertube.org/ns#', |
19 | sc: 'http://schema.org#', | 19 | sc: 'http://schema.org#', |
20 | Hashtag: 'as:Hashtag', | 20 | Hashtag: 'as:Hashtag', |
21 | uuid: 'sc:identifier', | 21 | uuid: 'sc:identifier', |
@@ -32,7 +32,8 @@ function activityPubContextify <T> (data: T) { | |||
32 | waitTranscoding: 'sc:Boolean', | 32 | waitTranscoding: 'sc:Boolean', |
33 | expires: 'sc:expires', | 33 | expires: 'sc:expires', |
34 | support: 'sc:Text', | 34 | support: 'sc:Text', |
35 | CacheFile: 'pt:CacheFile' | 35 | CacheFile: 'pt:CacheFile', |
36 | Infohash: 'pt:Infohash' | ||
36 | }, | 37 | }, |
37 | { | 38 | { |
38 | likes: { | 39 | likes: { |
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 3fb824e36..f38b82d97 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts | |||
@@ -193,10 +193,14 @@ function peertubeTruncate (str: string, maxLength: number) { | |||
193 | return truncate(str, options) | 193 | return truncate(str, options) |
194 | } | 194 | } |
195 | 195 | ||
196 | function sha256 (str: string, encoding: HexBase64Latin1Encoding = 'hex') { | 196 | function sha256 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') { |
197 | return createHash('sha256').update(str).digest(encoding) | 197 | return createHash('sha256').update(str).digest(encoding) |
198 | } | 198 | } |
199 | 199 | ||
200 | function sha1 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') { | ||
201 | return createHash('sha1').update(str).digest(encoding) | ||
202 | } | ||
203 | |||
200 | function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> { | 204 | function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> { |
201 | return function promisified (): Promise<A> { | 205 | return function promisified (): Promise<A> { |
202 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { | 206 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { |
@@ -262,7 +266,9 @@ export { | |||
262 | sanitizeHost, | 266 | sanitizeHost, |
263 | buildPath, | 267 | buildPath, |
264 | peertubeTruncate, | 268 | peertubeTruncate, |
269 | |||
265 | sha256, | 270 | sha256, |
271 | sha1, | ||
266 | 272 | ||
267 | promisify0, | 273 | promisify0, |
268 | promisify1, | 274 | promisify1, |
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts index e2bd0c55e..21d5c53ca 100644 --- a/server/helpers/custom-validators/activitypub/cache-file.ts +++ b/server/helpers/custom-validators/activitypub/cache-file.ts | |||
@@ -8,9 +8,19 @@ function isCacheFileObjectValid (object: CacheFileObject) { | |||
8 | object.type === 'CacheFile' && | 8 | object.type === 'CacheFile' && |
9 | isDateValid(object.expires) && | 9 | isDateValid(object.expires) && |
10 | isActivityPubUrlValid(object.object) && | 10 | isActivityPubUrlValid(object.object) && |
11 | isRemoteVideoUrlValid(object.url) | 11 | (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) |
12 | } | 12 | } |
13 | 13 | ||
14 | // --------------------------------------------------------------------------- | ||
15 | |||
14 | export { | 16 | export { |
15 | isCacheFileObjectValid | 17 | isCacheFileObjectValid |
16 | } | 18 | } |
19 | |||
20 | // --------------------------------------------------------------------------- | ||
21 | |||
22 | function isPlaylistRedundancyUrlValid (url: any) { | ||
23 | return url.type === 'Link' && | ||
24 | (url.mediaType || url.mimeType) === 'application/x-mpegURL' && | ||
25 | isActivityPubUrlValid(url.href) | ||
26 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 0f34aab21..ad99c2724 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import * as validator from 'validator' | 1 | import * as validator from 'validator' |
2 | import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers' | 2 | import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers' |
3 | import { peertubeTruncate } from '../../core-utils' | 3 | import { peertubeTruncate } from '../../core-utils' |
4 | import { exists, isBooleanValid, isDateValid, isUUIDValid } from '../misc' | 4 | import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' |
5 | import { | 5 | import { |
6 | isVideoDurationValid, | 6 | isVideoDurationValid, |
7 | isVideoNameValid, | 7 | isVideoNameValid, |
@@ -12,7 +12,6 @@ import { | |||
12 | } from '../videos' | 12 | } from '../videos' |
13 | import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' | 13 | import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' |
14 | import { VideoState } from '../../../../shared/models/videos' | 14 | import { VideoState } from '../../../../shared/models/videos' |
15 | import { isVideoAbuseReasonValid } from '../video-abuses' | ||
16 | 15 | ||
17 | function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { | 16 | function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { |
18 | return isBaseActivityValid(activity, 'Update') && | 17 | return isBaseActivityValid(activity, 'Update') && |
@@ -81,6 +80,11 @@ function isRemoteVideoUrlValid (url: any) { | |||
81 | ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType || url.mimeType) !== -1 && | 80 | ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType || url.mimeType) !== -1 && |
82 | validator.isLength(url.href, { min: 5 }) && | 81 | validator.isLength(url.href, { min: 5 }) && |
83 | validator.isInt(url.height + '', { min: 0 }) | 82 | validator.isInt(url.height + '', { min: 0 }) |
83 | ) || | ||
84 | ( | ||
85 | (url.mediaType || url.mimeType) === 'application/x-mpegURL' && | ||
86 | isActivityPubUrlValid(url.href) && | ||
87 | isArray(url.tag) | ||
84 | ) | 88 | ) |
85 | } | 89 | } |
86 | 90 | ||
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index b6f0ebe6f..76647fea2 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts | |||
@@ -13,6 +13,10 @@ function isNotEmptyIntArray (value: any) { | |||
13 | return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0 | 13 | return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0 |
14 | } | 14 | } |
15 | 15 | ||
16 | function isArrayOf (value: any, validator: (value: any) => boolean) { | ||
17 | return isArray(value) && value.every(v => validator(v)) | ||
18 | } | ||
19 | |||
16 | function isDateValid (value: string) { | 20 | function isDateValid (value: string) { |
17 | return exists(value) && validator.isISO8601(value) | 21 | return exists(value) && validator.isISO8601(value) |
18 | } | 22 | } |
@@ -82,6 +86,7 @@ function isFileValid ( | |||
82 | 86 | ||
83 | export { | 87 | export { |
84 | exists, | 88 | exists, |
89 | isArrayOf, | ||
85 | isNotEmptyIntArray, | 90 | isNotEmptyIntArray, |
86 | isArray, | 91 | isArray, |
87 | isIdValid, | 92 | isIdValid, |
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index 132f4690e..5ad8ed48e 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import * as ffmpeg from 'fluent-ffmpeg' | 1 | import * as ffmpeg from 'fluent-ffmpeg' |
2 | import { join } from 'path' | 2 | import { dirname, join } from 'path' |
3 | import { getTargetBitrate, VideoResolution } from '../../shared/models/videos' | 3 | import { getTargetBitrate, VideoResolution } from '../../shared/models/videos' |
4 | import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' | 4 | import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' |
5 | import { processImage } from './image-utils' | 5 | import { processImage } from './image-utils' |
@@ -29,12 +29,21 @@ function computeResolutionsToTranscode (videoFileHeight: number) { | |||
29 | return resolutionsEnabled | 29 | return resolutionsEnabled |
30 | } | 30 | } |
31 | 31 | ||
32 | async function getVideoFileResolution (path: string) { | 32 | async function getVideoFileSize (path: string) { |
33 | const videoStream = await getVideoFileStream(path) | 33 | const videoStream = await getVideoFileStream(path) |
34 | 34 | ||
35 | return { | 35 | return { |
36 | videoFileResolution: Math.min(videoStream.height, videoStream.width), | 36 | width: videoStream.width, |
37 | isPortraitMode: videoStream.height > videoStream.width | 37 | height: videoStream.height |
38 | } | ||
39 | } | ||
40 | |||
41 | async function getVideoFileResolution (path: string) { | ||
42 | const size = await getVideoFileSize(path) | ||
43 | |||
44 | return { | ||
45 | videoFileResolution: Math.min(size.height, size.width), | ||
46 | isPortraitMode: size.height > size.width | ||
38 | } | 47 | } |
39 | } | 48 | } |
40 | 49 | ||
@@ -110,8 +119,10 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima | |||
110 | type TranscodeOptions = { | 119 | type TranscodeOptions = { |
111 | inputPath: string | 120 | inputPath: string |
112 | outputPath: string | 121 | outputPath: string |
113 | resolution?: VideoResolution | 122 | resolution: VideoResolution |
114 | isPortraitMode?: boolean | 123 | isPortraitMode?: boolean |
124 | |||
125 | generateHlsPlaylist?: boolean | ||
115 | } | 126 | } |
116 | 127 | ||
117 | function transcode (options: TranscodeOptions) { | 128 | function transcode (options: TranscodeOptions) { |
@@ -150,6 +161,16 @@ function transcode (options: TranscodeOptions) { | |||
150 | command = command.withFPS(fps) | 161 | command = command.withFPS(fps) |
151 | } | 162 | } |
152 | 163 | ||
164 | if (options.generateHlsPlaylist) { | ||
165 | const segmentFilename = `${dirname(options.outputPath)}/${options.resolution}_%03d.ts` | ||
166 | |||
167 | command = command.outputOption('-hls_time 4') | ||
168 | .outputOption('-hls_list_size 0') | ||
169 | .outputOption('-hls_playlist_type vod') | ||
170 | .outputOption('-hls_segment_filename ' + segmentFilename) | ||
171 | .outputOption('-f hls') | ||
172 | } | ||
173 | |||
153 | command | 174 | command |
154 | .on('error', (err, stdout, stderr) => { | 175 | .on('error', (err, stdout, stderr) => { |
155 | logger.error('Error in transcoding job.', { stdout, stderr }) | 176 | logger.error('Error in transcoding job.', { stdout, stderr }) |
@@ -166,6 +187,7 @@ function transcode (options: TranscodeOptions) { | |||
166 | // --------------------------------------------------------------------------- | 187 | // --------------------------------------------------------------------------- |
167 | 188 | ||
168 | export { | 189 | export { |
190 | getVideoFileSize, | ||
169 | getVideoFileResolution, | 191 | getVideoFileResolution, |
170 | getDurationFromVideoFile, | 192 | getDurationFromVideoFile, |
171 | generateImageFromVideoFile, | 193 | generateImageFromVideoFile, |
diff --git a/server/helpers/video.ts b/server/helpers/video.ts index 1bd21467d..c90fe06c7 100644 --- a/server/helpers/video.ts +++ b/server/helpers/video.ts | |||
@@ -1,10 +1,12 @@ | |||
1 | import { VideoModel } from '../models/video/video' | 1 | import { VideoModel } from '../models/video/video' |
2 | 2 | ||
3 | type VideoFetchType = 'all' | 'only-video' | 'id' | 'none' | 3 | type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' |
4 | 4 | ||
5 | function fetchVideo (id: number | string, fetchType: VideoFetchType, userId?: number) { | 5 | function fetchVideo (id: number | string, fetchType: VideoFetchType, userId?: number) { |
6 | if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId) | 6 | if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId) |
7 | 7 | ||
8 | if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id) | ||
9 | |||
8 | if (fetchType === 'only-video') return VideoModel.load(id) | 10 | if (fetchType === 'only-video') return VideoModel.load(id) |
9 | 11 | ||
10 | if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) | 12 | if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) |
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 7905d9ffa..29fdb263e 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts | |||
@@ -12,7 +12,7 @@ function checkMissedConfig () { | |||
12 | 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max', | 12 | 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max', |
13 | 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', | 13 | 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', |
14 | 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', | 14 | 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', |
15 | 'storage.redundancy', 'storage.tmp', | 15 | 'storage.redundancy', 'storage.tmp', 'storage.playlists', |
16 | 'log.level', | 16 | 'log.level', |
17 | 'user.video_quota', 'user.video_quota_daily', | 17 | 'user.video_quota', 'user.video_quota_daily', |
18 | 'cache.previews.size', 'admin.email', 'contact_form.enabled', | 18 | 'cache.previews.size', 'admin.email', 'contact_form.enabled', |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6f3ebb9aa..98f8f8694 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -16,7 +16,7 @@ let config: IConfig = require('config') | |||
16 | 16 | ||
17 | // --------------------------------------------------------------------------- | 17 | // --------------------------------------------------------------------------- |
18 | 18 | ||
19 | const LAST_MIGRATION_VERSION = 325 | 19 | const LAST_MIGRATION_VERSION = 330 |
20 | 20 | ||
21 | // --------------------------------------------------------------------------- | 21 | // --------------------------------------------------------------------------- |
22 | 22 | ||
@@ -192,6 +192,7 @@ const CONFIG = { | |||
192 | AVATARS_DIR: buildPath(config.get<string>('storage.avatars')), | 192 | AVATARS_DIR: buildPath(config.get<string>('storage.avatars')), |
193 | LOG_DIR: buildPath(config.get<string>('storage.logs')), | 193 | LOG_DIR: buildPath(config.get<string>('storage.logs')), |
194 | VIDEOS_DIR: buildPath(config.get<string>('storage.videos')), | 194 | VIDEOS_DIR: buildPath(config.get<string>('storage.videos')), |
195 | PLAYLISTS_DIR: buildPath(config.get<string>('storage.playlists')), | ||
195 | REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')), | 196 | REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')), |
196 | THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')), | 197 | THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')), |
197 | PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')), | 198 | PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')), |
@@ -259,6 +260,9 @@ const CONFIG = { | |||
259 | get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') }, | 260 | get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') }, |
260 | get '720p' () { return config.get<boolean>('transcoding.resolutions.720p') }, | 261 | get '720p' () { return config.get<boolean>('transcoding.resolutions.720p') }, |
261 | get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') } | 262 | get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') } |
263 | }, | ||
264 | HLS: { | ||
265 | get ENABLED () { return config.get<boolean>('transcoding.hls.enabled') } | ||
262 | } | 266 | } |
263 | }, | 267 | }, |
264 | IMPORT: { | 268 | IMPORT: { |
@@ -590,6 +594,9 @@ const STATIC_PATHS = { | |||
590 | TORRENTS: '/static/torrents/', | 594 | TORRENTS: '/static/torrents/', |
591 | WEBSEED: '/static/webseed/', | 595 | WEBSEED: '/static/webseed/', |
592 | REDUNDANCY: '/static/redundancy/', | 596 | REDUNDANCY: '/static/redundancy/', |
597 | PLAYLISTS: { | ||
598 | HLS: '/static/playlists/hls' | ||
599 | }, | ||
593 | AVATARS: '/static/avatars/', | 600 | AVATARS: '/static/avatars/', |
594 | VIDEO_CAPTIONS: '/static/video-captions/' | 601 | VIDEO_CAPTIONS: '/static/video-captions/' |
595 | } | 602 | } |
@@ -632,6 +639,9 @@ const CACHE = { | |||
632 | } | 639 | } |
633 | } | 640 | } |
634 | 641 | ||
642 | const HLS_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.PLAYLISTS_DIR, 'hls') | ||
643 | const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') | ||
644 | |||
635 | const MEMOIZE_TTL = { | 645 | const MEMOIZE_TTL = { |
636 | OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours | 646 | OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours |
637 | } | 647 | } |
@@ -709,6 +719,7 @@ updateWebserverUrls() | |||
709 | 719 | ||
710 | export { | 720 | export { |
711 | API_VERSION, | 721 | API_VERSION, |
722 | HLS_REDUNDANCY_DIRECTORY, | ||
712 | AVATARS_SIZE, | 723 | AVATARS_SIZE, |
713 | ACCEPT_HEADERS, | 724 | ACCEPT_HEADERS, |
714 | BCRYPT_SALT_SIZE, | 725 | BCRYPT_SALT_SIZE, |
@@ -733,6 +744,7 @@ export { | |||
733 | PRIVATE_RSA_KEY_SIZE, | 744 | PRIVATE_RSA_KEY_SIZE, |
734 | ROUTE_CACHE_LIFETIME, | 745 | ROUTE_CACHE_LIFETIME, |
735 | SORTABLE_COLUMNS, | 746 | SORTABLE_COLUMNS, |
747 | HLS_PLAYLIST_DIRECTORY, | ||
736 | FEEDS, | 748 | FEEDS, |
737 | JOB_TTL, | 749 | JOB_TTL, |
738 | NSFW_POLICY_TYPES, | 750 | NSFW_POLICY_TYPES, |
diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 84ad2079b..fe296142d 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts | |||
@@ -33,6 +33,7 @@ import { AccountBlocklistModel } from '../models/account/account-blocklist' | |||
33 | import { ServerBlocklistModel } from '../models/server/server-blocklist' | 33 | import { ServerBlocklistModel } from '../models/server/server-blocklist' |
34 | import { UserNotificationModel } from '../models/account/user-notification' | 34 | import { UserNotificationModel } from '../models/account/user-notification' |
35 | import { UserNotificationSettingModel } from '../models/account/user-notification-setting' | 35 | import { UserNotificationSettingModel } from '../models/account/user-notification-setting' |
36 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
36 | 37 | ||
37 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string | 38 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string |
38 | 39 | ||
@@ -99,7 +100,8 @@ async function initDatabaseModels (silent: boolean) { | |||
99 | AccountBlocklistModel, | 100 | AccountBlocklistModel, |
100 | ServerBlocklistModel, | 101 | ServerBlocklistModel, |
101 | UserNotificationModel, | 102 | UserNotificationModel, |
102 | UserNotificationSettingModel | 103 | UserNotificationSettingModel, |
104 | VideoStreamingPlaylistModel | ||
103 | ]) | 105 | ]) |
104 | 106 | ||
105 | // Check extensions exist in the database | 107 | // Check extensions exist in the database |
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index b9a9da183..2b22e16fe 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts | |||
@@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user' | |||
6 | import { ApplicationModel } from '../models/application/application' | 6 | import { ApplicationModel } from '../models/application/application' |
7 | import { OAuthClientModel } from '../models/oauth/oauth-client' | 7 | import { OAuthClientModel } from '../models/oauth/oauth-client' |
8 | import { applicationExist, clientsExist, usersExist } from './checker-after-init' | 8 | import { applicationExist, clientsExist, usersExist } from './checker-after-init' |
9 | import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants' | 9 | import { CACHE, CONFIG, HLS_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants' |
10 | import { sequelizeTypescript } from './database' | 10 | import { sequelizeTypescript } from './database' |
11 | import { remove, ensureDir } from 'fs-extra' | 11 | import { remove, ensureDir } from 'fs-extra' |
12 | 12 | ||
@@ -73,6 +73,9 @@ function createDirectoriesIfNotExist () { | |||
73 | tasks.push(ensureDir(dir)) | 73 | tasks.push(ensureDir(dir)) |
74 | } | 74 | } |
75 | 75 | ||
76 | // Playlist directories | ||
77 | tasks.push(ensureDir(HLS_PLAYLIST_DIRECTORY)) | ||
78 | |||
76 | return Promise.all(tasks) | 79 | return Promise.all(tasks) |
77 | } | 80 | } |
78 | 81 | ||
diff --git a/server/initializers/migrations/0330-video-streaming-playlist.ts b/server/initializers/migrations/0330-video-streaming-playlist.ts new file mode 100644 index 000000000..c85a762ab --- /dev/null +++ b/server/initializers/migrations/0330-video-streaming-playlist.ts | |||
@@ -0,0 +1,51 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | |||
3 | async function up (utils: { | ||
4 | transaction: Sequelize.Transaction, | ||
5 | queryInterface: Sequelize.QueryInterface, | ||
6 | sequelize: Sequelize.Sequelize | ||
7 | }): Promise<void> { | ||
8 | |||
9 | { | ||
10 | const query = ` | ||
11 | CREATE TABLE IF NOT EXISTS "videoStreamingPlaylist" | ||
12 | ( | ||
13 | "id" SERIAL, | ||
14 | "type" INTEGER NOT NULL, | ||
15 | "playlistUrl" VARCHAR(2000) NOT NULL, | ||
16 | "p2pMediaLoaderInfohashes" VARCHAR(255)[] NOT NULL, | ||
17 | "segmentsSha256Url" VARCHAR(255) NOT NULL, | ||
18 | "videoId" INTEGER NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, | ||
19 | "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, | ||
20 | "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL, | ||
21 | PRIMARY KEY ("id") | ||
22 | );` | ||
23 | await utils.sequelize.query(query) | ||
24 | } | ||
25 | |||
26 | { | ||
27 | const data = { | ||
28 | type: Sequelize.INTEGER, | ||
29 | allowNull: true, | ||
30 | defaultValue: null | ||
31 | } | ||
32 | |||
33 | await utils.queryInterface.changeColumn('videoRedundancy', 'videoFileId', data) | ||
34 | } | ||
35 | |||
36 | { | ||
37 | const query = 'ALTER TABLE "videoRedundancy" ADD COLUMN "videoStreamingPlaylistId" INTEGER NULL ' + | ||
38 | 'REFERENCES "videoStreamingPlaylist" ("id") ON DELETE CASCADE ON UPDATE CASCADE' | ||
39 | |||
40 | await utils.sequelize.query(query) | ||
41 | } | ||
42 | } | ||
43 | |||
44 | function down (options) { | ||
45 | throw new Error('Not implemented.') | ||
46 | } | ||
47 | |||
48 | export { | ||
49 | up, | ||
50 | down | ||
51 | } | ||
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index f6f068b45..9a40414bb 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts | |||
@@ -1,11 +1,28 @@ | |||
1 | import { CacheFileObject } from '../../../shared/index' | 1 | import { ActivityPlaylistUrlObject, ActivityVideoUrlObject, CacheFileObject } from '../../../shared/index' |
2 | import { VideoModel } from '../../models/video/video' | 2 | import { VideoModel } from '../../models/video/video' |
3 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 3 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
4 | import { Transaction } from 'sequelize' | 4 | import { Transaction } from 'sequelize' |
5 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
5 | 6 | ||
6 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { | 7 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { |
7 | const url = cacheFileObject.url | ||
8 | 8 | ||
9 | if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { | ||
10 | const url = cacheFileObject.url | ||
11 | |||
12 | const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) | ||
13 | if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) | ||
14 | |||
15 | return { | ||
16 | expiresOn: new Date(cacheFileObject.expires), | ||
17 | url: cacheFileObject.id, | ||
18 | fileUrl: url.href, | ||
19 | strategy: null, | ||
20 | videoStreamingPlaylistId: playlist.id, | ||
21 | actorId: byActor.id | ||
22 | } | ||
23 | } | ||
24 | |||
25 | const url = cacheFileObject.url | ||
9 | const videoFile = video.VideoFiles.find(f => { | 26 | const videoFile = video.VideoFiles.find(f => { |
10 | return f.resolution === url.height && f.fps === url.fps | 27 | return f.resolution === url.height && f.fps === url.fps |
11 | }) | 28 | }) |
@@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject | |||
15 | return { | 32 | return { |
16 | expiresOn: new Date(cacheFileObject.expires), | 33 | expiresOn: new Date(cacheFileObject.expires), |
17 | url: cacheFileObject.id, | 34 | url: cacheFileObject.id, |
18 | fileUrl: cacheFileObject.url.href, | 35 | fileUrl: url.href, |
19 | strategy: null, | 36 | strategy: null, |
20 | videoFileId: videoFile.id, | 37 | videoFileId: videoFile.id, |
21 | actorId: byActor.id | 38 | actorId: byActor.id |
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index e3fca0a17..605aaba06 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
2 | import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub' | 2 | import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub' |
3 | import { VideoPrivacy } from '../../../../shared/models/videos' | 3 | import { Video, VideoPrivacy } from '../../../../shared/models/videos' |
4 | import { ActorModel } from '../../../models/activitypub/actor' | 4 | import { ActorModel } from '../../../models/activitypub/actor' |
5 | import { VideoModel } from '../../../models/video/video' | 5 | import { VideoModel } from '../../../models/video/video' |
6 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | 6 | import { VideoAbuseModel } from '../../../models/video/video-abuse' |
@@ -39,17 +39,14 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, | |||
39 | return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | 39 | return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) |
40 | } | 40 | } |
41 | 41 | ||
42 | async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { | 42 | async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) { |
43 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url) | 43 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url) |
44 | 44 | ||
45 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) | ||
46 | const redundancyObject = fileRedundancy.toActivityPubObject() | ||
47 | |||
48 | return sendVideoRelatedCreateActivity({ | 45 | return sendVideoRelatedCreateActivity({ |
49 | byActor, | 46 | byActor, |
50 | video, | 47 | video, |
51 | url: fileRedundancy.url, | 48 | url: fileRedundancy.url, |
52 | object: redundancyObject | 49 | object: fileRedundancy.toActivityPubObject() |
53 | }) | 50 | }) |
54 | } | 51 | } |
55 | 52 | ||
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index bf1b6e117..8976fcbc8 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts | |||
@@ -73,7 +73,8 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans | |||
73 | async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { | 73 | async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { |
74 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) | 74 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) |
75 | 75 | ||
76 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | 76 | const videoId = redundancyModel.getVideo().id |
77 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
77 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) | 78 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) |
78 | 79 | ||
79 | return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) | 80 | return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) |
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index a68f03edf..839f66470 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts | |||
@@ -61,7 +61,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod | |||
61 | async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { | 61 | async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { |
62 | logger.info('Creating job to update cache file %s.', redundancyModel.url) | 62 | logger.info('Creating job to update cache file %s.', redundancyModel.url) |
63 | 63 | ||
64 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | 64 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id) |
65 | 65 | ||
66 | const activityBuilder = (audience: ActivityAudience) => { | 66 | const activityBuilder = (audience: ActivityAudience) => { |
67 | const redundancyObject = redundancyModel.toActivityPubObject() | 67 | const redundancyObject = redundancyModel.toActivityPubObject() |
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts index 38f15448c..4229fe094 100644 --- a/server/lib/activitypub/url.ts +++ b/server/lib/activitypub/url.ts | |||
@@ -5,6 +5,8 @@ import { VideoModel } from '../../models/video/video' | |||
5 | import { VideoAbuseModel } from '../../models/video/video-abuse' | 5 | import { VideoAbuseModel } from '../../models/video/video-abuse' |
6 | import { VideoCommentModel } from '../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../models/video/video-comment' |
7 | import { VideoFileModel } from '../../models/video/video-file' | 7 | import { VideoFileModel } from '../../models/video/video-file' |
8 | import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' | ||
9 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
8 | 10 | ||
9 | function getVideoActivityPubUrl (video: VideoModel) { | 11 | function getVideoActivityPubUrl (video: VideoModel) { |
10 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid | 12 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid |
@@ -16,6 +18,10 @@ function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) { | |||
16 | return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` | 18 | return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` |
17 | } | 19 | } |
18 | 20 | ||
21 | function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) { | ||
22 | return `${CONFIG.WEBSERVER.URL}/redundancy/video-playlists/${playlist.getStringType()}/${video.uuid}` | ||
23 | } | ||
24 | |||
19 | function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { | 25 | function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { |
20 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id | 26 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id |
21 | } | 27 | } |
@@ -92,6 +98,7 @@ function getUndoActivityPubUrl (originalUrl: string) { | |||
92 | 98 | ||
93 | export { | 99 | export { |
94 | getVideoActivityPubUrl, | 100 | getVideoActivityPubUrl, |
101 | getVideoCacheStreamingPlaylistActivityPubUrl, | ||
95 | getVideoChannelActivityPubUrl, | 102 | getVideoChannelActivityPubUrl, |
96 | getAccountActivityPubUrl, | 103 | getAccountActivityPubUrl, |
97 | getVideoAbuseActivityPubUrl, | 104 | getVideoAbuseActivityPubUrl, |
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index e1e523499..edd01234f 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -2,7 +2,14 @@ import * as Bluebird from 'bluebird' | |||
2 | import * as sequelize from 'sequelize' | 2 | import * as sequelize from 'sequelize' |
3 | import * as magnetUtil from 'magnet-uri' | 3 | import * as magnetUtil from 'magnet-uri' |
4 | import * as request from 'request' | 4 | import * as request from 'request' |
5 | import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' | 5 | import { |
6 | ActivityIconObject, | ||
7 | ActivityPlaylistSegmentHashesObject, | ||
8 | ActivityPlaylistUrlObject, | ||
9 | ActivityUrlObject, | ||
10 | ActivityVideoUrlObject, | ||
11 | VideoState | ||
12 | } from '../../../shared/index' | ||
6 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 13 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
7 | import { VideoPrivacy } from '../../../shared/models/videos' | 14 | import { VideoPrivacy } from '../../../shared/models/videos' |
8 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' | 15 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' |
@@ -30,6 +37,9 @@ import { AccountModel } from '../../models/account/account' | |||
30 | import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' | 37 | import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' |
31 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | 38 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
32 | import { Notifier } from '../notifier' | 39 | import { Notifier } from '../notifier' |
40 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
41 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
42 | import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' | ||
33 | 43 | ||
34 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { | 44 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { |
35 | // If the video is not private and published, we federate it | 45 | // If the video is not private and published, we federate it |
@@ -264,6 +274,25 @@ async function updateVideoFromAP (options: { | |||
264 | } | 274 | } |
265 | 275 | ||
266 | { | 276 | { |
277 | const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject) | ||
278 | const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) | ||
279 | |||
280 | // Remove video files that do not exist anymore | ||
281 | const destroyTasks = options.video.VideoStreamingPlaylists | ||
282 | .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f))) | ||
283 | .map(f => f.destroy(sequelizeOptions)) | ||
284 | await Promise.all(destroyTasks) | ||
285 | |||
286 | // Update or add other one | ||
287 | const upsertTasks = streamingPlaylistAttributes.map(a => { | ||
288 | return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t }) | ||
289 | .then(([ streamingPlaylist ]) => streamingPlaylist) | ||
290 | }) | ||
291 | |||
292 | options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks) | ||
293 | } | ||
294 | |||
295 | { | ||
267 | // Update Tags | 296 | // Update Tags |
268 | const tags = options.videoObject.tag.map(tag => tag.name) | 297 | const tags = options.videoObject.tag.map(tag => tag.name) |
269 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 298 | const tagInstances = await TagModel.findOrCreateTags(tags, t) |
@@ -367,13 +396,25 @@ export { | |||
367 | 396 | ||
368 | // --------------------------------------------------------------------------- | 397 | // --------------------------------------------------------------------------- |
369 | 398 | ||
370 | function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { | 399 | function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { |
371 | const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) | 400 | const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) |
372 | 401 | ||
373 | const urlMediaType = url.mediaType || url.mimeType | 402 | const urlMediaType = url.mediaType || url.mimeType |
374 | return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') | 403 | return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') |
375 | } | 404 | } |
376 | 405 | ||
406 | function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject { | ||
407 | const urlMediaType = url.mediaType || url.mimeType | ||
408 | |||
409 | return urlMediaType === 'application/x-mpegURL' | ||
410 | } | ||
411 | |||
412 | function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { | ||
413 | const urlMediaType = tag.mediaType || tag.mimeType | ||
414 | |||
415 | return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json' | ||
416 | } | ||
417 | |||
377 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { | 418 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { |
378 | logger.debug('Adding remote video %s.', videoObject.id) | 419 | logger.debug('Adding remote video %s.', videoObject.id) |
379 | 420 | ||
@@ -394,8 +435,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor | |||
394 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) | 435 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) |
395 | await Promise.all(videoFilePromises) | 436 | await Promise.all(videoFilePromises) |
396 | 437 | ||
438 | const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject) | ||
439 | const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t })) | ||
440 | await Promise.all(playlistPromises) | ||
441 | |||
397 | // Process tags | 442 | // Process tags |
398 | const tags = videoObject.tag.map(t => t.name) | 443 | const tags = videoObject.tag |
444 | .filter(t => t.type === 'Hashtag') | ||
445 | .map(t => t.name) | ||
399 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 446 | const tagInstances = await TagModel.findOrCreateTags(tags, t) |
400 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) | 447 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) |
401 | 448 | ||
@@ -473,13 +520,13 @@ async function videoActivityObjectToDBAttributes ( | |||
473 | } | 520 | } |
474 | 521 | ||
475 | function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { | 522 | function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { |
476 | const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] | 523 | const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] |
477 | 524 | ||
478 | if (fileUrls.length === 0) { | 525 | if (fileUrls.length === 0) { |
479 | throw new Error('Cannot find video files for ' + video.url) | 526 | throw new Error('Cannot find video files for ' + video.url) |
480 | } | 527 | } |
481 | 528 | ||
482 | const attributes: VideoFileModel[] = [] | 529 | const attributes: FilteredModelAttributes<VideoFileModel>[] = [] |
483 | for (const fileUrl of fileUrls) { | 530 | for (const fileUrl of fileUrls) { |
484 | // Fetch associated magnet uri | 531 | // Fetch associated magnet uri |
485 | const magnet = videoObject.url.find(u => { | 532 | const magnet = videoObject.url.find(u => { |
@@ -502,7 +549,45 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid | |||
502 | size: fileUrl.size, | 549 | size: fileUrl.size, |
503 | videoId: video.id, | 550 | videoId: video.id, |
504 | fps: fileUrl.fps || -1 | 551 | fps: fileUrl.fps || -1 |
505 | } as VideoFileModel | 552 | } |
553 | |||
554 | attributes.push(attribute) | ||
555 | } | ||
556 | |||
557 | return attributes | ||
558 | } | ||
559 | |||
560 | function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { | ||
561 | const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] | ||
562 | if (playlistUrls.length === 0) return [] | ||
563 | |||
564 | const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = [] | ||
565 | for (const playlistUrlObject of playlistUrls) { | ||
566 | const p2pMediaLoaderInfohashes = playlistUrlObject.tag | ||
567 | .filter(t => t.type === 'Infohash') | ||
568 | .map(t => t.name) | ||
569 | if (p2pMediaLoaderInfohashes.length === 0) { | ||
570 | logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject }) | ||
571 | continue | ||
572 | } | ||
573 | |||
574 | const segmentsSha256UrlObject = playlistUrlObject.tag | ||
575 | .find(t => { | ||
576 | return isAPPlaylistSegmentHashesUrlObject(t) | ||
577 | }) as ActivityPlaylistSegmentHashesObject | ||
578 | if (!segmentsSha256UrlObject) { | ||
579 | logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) | ||
580 | continue | ||
581 | } | ||
582 | |||
583 | const attribute = { | ||
584 | type: VideoStreamingPlaylistType.HLS, | ||
585 | playlistUrl: playlistUrlObject.href, | ||
586 | segmentsSha256Url: segmentsSha256UrlObject.href, | ||
587 | p2pMediaLoaderInfohashes, | ||
588 | videoId: video.id | ||
589 | } | ||
590 | |||
506 | attributes.push(attribute) | 591 | attributes.push(attribute) |
507 | } | 592 | } |
508 | 593 | ||
diff --git a/server/lib/hls.ts b/server/lib/hls.ts new file mode 100644 index 000000000..10db6c3c3 --- /dev/null +++ b/server/lib/hls.ts | |||
@@ -0,0 +1,110 @@ | |||
1 | import { VideoModel } from '../models/video/video' | ||
2 | import { basename, dirname, join } from 'path' | ||
3 | import { HLS_PLAYLIST_DIRECTORY, CONFIG } from '../initializers' | ||
4 | import { outputJSON, pathExists, readdir, readFile, remove, writeFile, move } from 'fs-extra' | ||
5 | import { getVideoFileSize } from '../helpers/ffmpeg-utils' | ||
6 | import { sha256 } from '../helpers/core-utils' | ||
7 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
8 | import HLSDownloader from 'hlsdownloader' | ||
9 | import { logger } from '../helpers/logger' | ||
10 | import { parse } from 'url' | ||
11 | |||
12 | async function updateMasterHLSPlaylist (video: VideoModel) { | ||
13 | const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) | ||
14 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] | ||
15 | const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) | ||
16 | |||
17 | for (const file of video.VideoFiles) { | ||
18 | // If we did not generated a playlist for this resolution, skip | ||
19 | const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
20 | if (await pathExists(filePlaylistPath) === false) continue | ||
21 | |||
22 | const videoFilePath = video.getVideoFilePath(file) | ||
23 | |||
24 | const size = await getVideoFileSize(videoFilePath) | ||
25 | |||
26 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | ||
27 | const resolution = `RESOLUTION=${size.width}x${size.height}` | ||
28 | |||
29 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | ||
30 | if (file.fps) line += ',FRAME-RATE=' + file.fps | ||
31 | |||
32 | masterPlaylists.push(line) | ||
33 | masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
34 | } | ||
35 | |||
36 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | ||
37 | } | ||
38 | |||
39 | async function updateSha256Segments (video: VideoModel) { | ||
40 | const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) | ||
41 | const files = await readdir(directory) | ||
42 | const json: { [filename: string]: string} = {} | ||
43 | |||
44 | for (const file of files) { | ||
45 | if (file.endsWith('.ts') === false) continue | ||
46 | |||
47 | const buffer = await readFile(join(directory, file)) | ||
48 | const filename = basename(file) | ||
49 | |||
50 | json[filename] = sha256(buffer) | ||
51 | } | ||
52 | |||
53 | const outputPath = join(directory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) | ||
54 | await outputJSON(outputPath, json) | ||
55 | } | ||
56 | |||
57 | function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { | ||
58 | let timer | ||
59 | |||
60 | logger.info('Importing HLS playlist %s', playlistUrl) | ||
61 | |||
62 | const params = { | ||
63 | playlistURL: playlistUrl, | ||
64 | destination: CONFIG.STORAGE.TMP_DIR | ||
65 | } | ||
66 | const downloader = new HLSDownloader(params) | ||
67 | |||
68 | const hlsDestinationDir = join(CONFIG.STORAGE.TMP_DIR, dirname(parse(playlistUrl).pathname)) | ||
69 | |||
70 | return new Promise<string>(async (res, rej) => { | ||
71 | downloader.startDownload(err => { | ||
72 | clearTimeout(timer) | ||
73 | |||
74 | if (err) { | ||
75 | deleteTmpDirectory(hlsDestinationDir) | ||
76 | |||
77 | return rej(err) | ||
78 | } | ||
79 | |||
80 | move(hlsDestinationDir, destinationDir, { overwrite: true }) | ||
81 | .then(() => res()) | ||
82 | .catch(err => { | ||
83 | deleteTmpDirectory(hlsDestinationDir) | ||
84 | |||
85 | return rej(err) | ||
86 | }) | ||
87 | }) | ||
88 | |||
89 | timer = setTimeout(() => { | ||
90 | deleteTmpDirectory(hlsDestinationDir) | ||
91 | |||
92 | return rej(new Error('HLS download timeout.')) | ||
93 | }, timeout) | ||
94 | |||
95 | function deleteTmpDirectory (directory: string) { | ||
96 | remove(directory) | ||
97 | .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) | ||
98 | } | ||
99 | }) | ||
100 | } | ||
101 | |||
102 | // --------------------------------------------------------------------------- | ||
103 | |||
104 | export { | ||
105 | updateMasterHLSPlaylist, | ||
106 | updateSha256Segments, | ||
107 | downloadPlaylistSegments | ||
108 | } | ||
109 | |||
110 | // --------------------------------------------------------------------------- | ||
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 217d666b6..7119ce0ca 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts | |||
@@ -5,17 +5,18 @@ import { VideoModel } from '../../../models/video/video' | |||
5 | import { JobQueue } from '../job-queue' | 5 | import { JobQueue } from '../job-queue' |
6 | import { federateVideoIfNeeded } from '../../activitypub' | 6 | import { federateVideoIfNeeded } from '../../activitypub' |
7 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 7 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
8 | import { sequelizeTypescript } from '../../../initializers' | 8 | import { sequelizeTypescript, CONFIG } from '../../../initializers' |
9 | import * as Bluebird from 'bluebird' | 9 | import * as Bluebird from 'bluebird' |
10 | import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' | 10 | import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' |
11 | import { importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' | 11 | import { generateHlsPlaylist, importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' |
12 | import { Notifier } from '../../notifier' | 12 | import { Notifier } from '../../notifier' |
13 | 13 | ||
14 | export type VideoFilePayload = { | 14 | export type VideoFilePayload = { |
15 | videoUUID: string | 15 | videoUUID: string |
16 | isNewVideo?: boolean | ||
17 | resolution?: VideoResolution | 16 | resolution?: VideoResolution |
17 | isNewVideo?: boolean | ||
18 | isPortraitMode?: boolean | 18 | isPortraitMode?: boolean |
19 | generateHlsPlaylist?: boolean | ||
19 | } | 20 | } |
20 | 21 | ||
21 | export type VideoFileImportPayload = { | 22 | export type VideoFileImportPayload = { |
@@ -51,21 +52,38 @@ async function processVideoFile (job: Bull.Job) { | |||
51 | return undefined | 52 | return undefined |
52 | } | 53 | } |
53 | 54 | ||
54 | // Transcoding in other resolution | 55 | if (payload.generateHlsPlaylist) { |
55 | if (payload.resolution) { | 56 | await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false) |
57 | |||
58 | await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video) | ||
59 | } else if (payload.resolution) { // Transcoding in other resolution | ||
56 | await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) | 60 | await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) |
57 | 61 | ||
58 | await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) | 62 | await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video, payload) |
59 | } else { | 63 | } else { |
60 | await optimizeVideofile(video) | 64 | await optimizeVideofile(video) |
61 | 65 | ||
62 | await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) | 66 | await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload) |
63 | } | 67 | } |
64 | 68 | ||
65 | return video | 69 | return video |
66 | } | 70 | } |
67 | 71 | ||
68 | async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { | 72 | async function onHlsPlaylistGenerationSuccess (video: VideoModel) { |
73 | if (video === undefined) return undefined | ||
74 | |||
75 | await sequelizeTypescript.transaction(async t => { | ||
76 | // Maybe the video changed in database, refresh it | ||
77 | let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | ||
78 | // Video does not exist anymore | ||
79 | if (!videoDatabase) return undefined | ||
80 | |||
81 | // If the video was not published, we consider it is a new one for other instances | ||
82 | await federateVideoIfNeeded(videoDatabase, false, t) | ||
83 | }) | ||
84 | } | ||
85 | |||
86 | async function onVideoFileTranscoderOrImportSuccess (video: VideoModel, payload?: VideoFilePayload) { | ||
69 | if (video === undefined) return undefined | 87 | if (video === undefined) return undefined |
70 | 88 | ||
71 | const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { | 89 | const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { |
@@ -96,9 +114,11 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { | |||
96 | Notifier.Instance.notifyOnNewVideo(videoDatabase) | 114 | Notifier.Instance.notifyOnNewVideo(videoDatabase) |
97 | Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) | 115 | Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) |
98 | } | 116 | } |
117 | |||
118 | await createHlsJobIfEnabled(payload) | ||
99 | } | 119 | } |
100 | 120 | ||
101 | async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) { | 121 | async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoFilePayload) { |
102 | if (videoArg === undefined) return undefined | 122 | if (videoArg === undefined) return undefined |
103 | 123 | ||
104 | // Outside the transaction (IO on disk) | 124 | // Outside the transaction (IO on disk) |
@@ -145,7 +165,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo | |||
145 | logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) | 165 | logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) |
146 | } | 166 | } |
147 | 167 | ||
148 | await federateVideoIfNeeded(videoDatabase, isNewVideo, t) | 168 | await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t) |
149 | 169 | ||
150 | return { videoDatabase, videoPublished } | 170 | return { videoDatabase, videoPublished } |
151 | }) | 171 | }) |
@@ -155,6 +175,8 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo | |||
155 | if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) | 175 | if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) |
156 | if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) | 176 | if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) |
157 | } | 177 | } |
178 | |||
179 | await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution })) | ||
158 | } | 180 | } |
159 | 181 | ||
160 | // --------------------------------------------------------------------------- | 182 | // --------------------------------------------------------------------------- |
@@ -163,3 +185,20 @@ export { | |||
163 | processVideoFile, | 185 | processVideoFile, |
164 | processVideoFileImport | 186 | processVideoFileImport |
165 | } | 187 | } |
188 | |||
189 | // --------------------------------------------------------------------------- | ||
190 | |||
191 | function createHlsJobIfEnabled (payload?: VideoFilePayload) { | ||
192 | // Generate HLS playlist? | ||
193 | if (payload && CONFIG.TRANSCODING.HLS.ENABLED) { | ||
194 | const hlsTranscodingPayload = { | ||
195 | videoUUID: payload.videoUUID, | ||
196 | resolution: payload.resolution, | ||
197 | isPortraitMode: payload.isPortraitMode, | ||
198 | |||
199 | generateHlsPlaylist: true | ||
200 | } | ||
201 | |||
202 | return JobQueue.Instance.createJob({ type: 'video-file', payload: hlsTranscodingPayload }) | ||
203 | } | ||
204 | } | ||
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index f643ee226..1a48f2bd0 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { AbstractScheduler } from './abstract-scheduler' | 1 | import { AbstractScheduler } from './abstract-scheduler' |
2 | import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' | 2 | import { CONFIG, HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { VideosRedundancy } from '../../../shared/models/redundancy' | 4 | import { VideosRedundancy } from '../../../shared/models/redundancy' |
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
@@ -9,9 +9,19 @@ import { join } from 'path' | |||
9 | import { move } from 'fs-extra' | 9 | import { move } from 'fs-extra' |
10 | import { getServerActor } from '../../helpers/utils' | 10 | import { getServerActor } from '../../helpers/utils' |
11 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | 11 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' |
12 | import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' | 12 | import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' |
13 | import { removeVideoRedundancy } from '../redundancy' | 13 | import { removeVideoRedundancy } from '../redundancy' |
14 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' | 14 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' |
15 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
16 | import { VideoModel } from '../../models/video/video' | ||
17 | import { downloadPlaylistSegments } from '../hls' | ||
18 | |||
19 | type CandidateToDuplicate = { | ||
20 | redundancy: VideosRedundancy, | ||
21 | video: VideoModel, | ||
22 | files: VideoFileModel[], | ||
23 | streamingPlaylists: VideoStreamingPlaylistModel[] | ||
24 | } | ||
15 | 25 | ||
16 | export class VideosRedundancyScheduler extends AbstractScheduler { | 26 | export class VideosRedundancyScheduler extends AbstractScheduler { |
17 | 27 | ||
@@ -24,28 +34,32 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
24 | } | 34 | } |
25 | 35 | ||
26 | protected async internalExecute () { | 36 | protected async internalExecute () { |
27 | for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { | 37 | for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { |
28 | logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) | 38 | logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) |
29 | 39 | ||
30 | try { | 40 | try { |
31 | const videoToDuplicate = await this.findVideoToDuplicate(obj) | 41 | const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig) |
32 | if (!videoToDuplicate) continue | 42 | if (!videoToDuplicate) continue |
33 | 43 | ||
34 | const videoFiles = videoToDuplicate.VideoFiles | 44 | const candidateToDuplicate = { |
35 | videoFiles.forEach(f => f.Video = videoToDuplicate) | 45 | video: videoToDuplicate, |
46 | redundancy: redundancyConfig, | ||
47 | files: videoToDuplicate.VideoFiles, | ||
48 | streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists | ||
49 | } | ||
36 | 50 | ||
37 | await this.purgeCacheIfNeeded(obj, videoFiles) | 51 | await this.purgeCacheIfNeeded(candidateToDuplicate) |
38 | 52 | ||
39 | if (await this.isTooHeavy(obj, videoFiles)) { | 53 | if (await this.isTooHeavy(candidateToDuplicate)) { |
40 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) | 54 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) |
41 | continue | 55 | continue |
42 | } | 56 | } |
43 | 57 | ||
44 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) | 58 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy) |
45 | 59 | ||
46 | await this.createVideoRedundancy(obj, videoFiles) | 60 | await this.createVideoRedundancies(candidateToDuplicate) |
47 | } catch (err) { | 61 | } catch (err) { |
48 | logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) | 62 | logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err }) |
49 | } | 63 | } |
50 | } | 64 | } |
51 | 65 | ||
@@ -63,25 +77,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
63 | 77 | ||
64 | for (const redundancyModel of expired) { | 78 | for (const redundancyModel of expired) { |
65 | try { | 79 | try { |
66 | await this.extendsOrDeleteRedundancy(redundancyModel) | 80 | const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) |
81 | const candidate = { | ||
82 | redundancy: redundancyConfig, | ||
83 | video: null, | ||
84 | files: [], | ||
85 | streamingPlaylists: [] | ||
86 | } | ||
87 | |||
88 | // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it | ||
89 | if (!redundancyConfig || await this.isTooHeavy(candidate)) { | ||
90 | logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) | ||
91 | await removeVideoRedundancy(redundancyModel) | ||
92 | } else { | ||
93 | await this.extendsRedundancy(redundancyModel) | ||
94 | } | ||
67 | } catch (err) { | 95 | } catch (err) { |
68 | logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel)) | 96 | logger.error( |
97 | 'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel), | ||
98 | { err } | ||
99 | ) | ||
69 | } | 100 | } |
70 | } | 101 | } |
71 | } | 102 | } |
72 | 103 | ||
73 | private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) { | 104 | private async extendsRedundancy (redundancyModel: VideoRedundancyModel) { |
74 | // Refresh the video, maybe it was deleted | ||
75 | const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url) | ||
76 | |||
77 | if (!video) { | ||
78 | logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url) | ||
79 | |||
80 | await redundancyModel.destroy() | ||
81 | return | ||
82 | } | ||
83 | |||
84 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) | 105 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) |
106 | // Redundancy strategy disabled, remove our redundancy instead of extending expiration | ||
107 | if (!redundancy) await removeVideoRedundancy(redundancyModel) | ||
108 | |||
85 | await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) | 109 | await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) |
86 | } | 110 | } |
87 | 111 | ||
@@ -112,49 +136,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
112 | } | 136 | } |
113 | } | 137 | } |
114 | 138 | ||
115 | private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 139 | private async createVideoRedundancies (data: CandidateToDuplicate) { |
116 | const serverActor = await getServerActor() | 140 | const video = await this.loadAndRefreshVideo(data.video.url) |
141 | |||
142 | if (!video) { | ||
143 | logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url) | ||
117 | 144 | ||
118 | for (const file of filesToDuplicate) { | 145 | return |
119 | const video = await this.loadAndRefreshVideo(file.Video.url) | 146 | } |
120 | 147 | ||
148 | for (const file of data.files) { | ||
121 | const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) | 149 | const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) |
122 | if (existingRedundancy) { | 150 | if (existingRedundancy) { |
123 | await this.extendsOrDeleteRedundancy(existingRedundancy) | 151 | await this.extendsRedundancy(existingRedundancy) |
124 | 152 | ||
125 | continue | 153 | continue |
126 | } | 154 | } |
127 | 155 | ||
128 | if (!video) { | 156 | await this.createVideoFileRedundancy(data.redundancy, video, file) |
129 | logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url) | 157 | } |
158 | |||
159 | for (const streamingPlaylist of data.streamingPlaylists) { | ||
160 | const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id) | ||
161 | if (existingRedundancy) { | ||
162 | await this.extendsRedundancy(existingRedundancy) | ||
130 | 163 | ||
131 | continue | 164 | continue |
132 | } | 165 | } |
133 | 166 | ||
134 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) | 167 | await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist) |
168 | } | ||
169 | } | ||
135 | 170 | ||
136 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | 171 | private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) { |
137 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) | 172 | file.Video = video |
138 | 173 | ||
139 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) | 174 | const serverActor = await getServerActor() |
140 | 175 | ||
141 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) | 176 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) |
142 | await move(tmpPath, destPath) | ||
143 | 177 | ||
144 | const createdModel = await VideoRedundancyModel.create({ | 178 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() |
145 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | 179 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) |
146 | url: getVideoCacheFileActivityPubUrl(file), | ||
147 | fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), | ||
148 | strategy: redundancy.strategy, | ||
149 | videoFileId: file.id, | ||
150 | actorId: serverActor.id | ||
151 | }) | ||
152 | createdModel.VideoFile = file | ||
153 | 180 | ||
154 | await sendCreateCacheFile(serverActor, createdModel) | 181 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) |
155 | 182 | ||
156 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) | 183 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) |
157 | } | 184 | await move(tmpPath, destPath) |
185 | |||
186 | const createdModel = await VideoRedundancyModel.create({ | ||
187 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | ||
188 | url: getVideoCacheFileActivityPubUrl(file), | ||
189 | fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), | ||
190 | strategy: redundancy.strategy, | ||
191 | videoFileId: file.id, | ||
192 | actorId: serverActor.id | ||
193 | }) | ||
194 | |||
195 | createdModel.VideoFile = file | ||
196 | |||
197 | await sendCreateCacheFile(serverActor, video, createdModel) | ||
198 | |||
199 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) | ||
200 | } | ||
201 | |||
202 | private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) { | ||
203 | playlist.Video = video | ||
204 | |||
205 | const serverActor = await getServerActor() | ||
206 | |||
207 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) | ||
208 | |||
209 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | ||
210 | await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) | ||
211 | |||
212 | const createdModel = await VideoRedundancyModel.create({ | ||
213 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | ||
214 | url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), | ||
215 | fileUrl: playlist.getVideoRedundancyUrl(CONFIG.WEBSERVER.URL), | ||
216 | strategy: redundancy.strategy, | ||
217 | videoStreamingPlaylistId: playlist.id, | ||
218 | actorId: serverActor.id | ||
219 | }) | ||
220 | |||
221 | createdModel.VideoStreamingPlaylist = playlist | ||
222 | |||
223 | await sendCreateCacheFile(serverActor, video, createdModel) | ||
224 | |||
225 | logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) | ||
158 | } | 226 | } |
159 | 227 | ||
160 | private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { | 228 | private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { |
@@ -168,8 +236,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
168 | await sendUpdateCacheFile(serverActor, redundancy) | 236 | await sendUpdateCacheFile(serverActor, redundancy) |
169 | } | 237 | } |
170 | 238 | ||
171 | private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 239 | private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { |
172 | while (this.isTooHeavy(redundancy, filesToDuplicate)) { | 240 | while (this.isTooHeavy(candidateToDuplicate)) { |
241 | const redundancy = candidateToDuplicate.redundancy | ||
173 | const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) | 242 | const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) |
174 | if (!toDelete) return | 243 | if (!toDelete) return |
175 | 244 | ||
@@ -177,11 +246,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
177 | } | 246 | } |
178 | } | 247 | } |
179 | 248 | ||
180 | private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 249 | private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { |
181 | const maxSize = redundancy.size | 250 | const maxSize = candidateToDuplicate.redundancy.size |
182 | 251 | ||
183 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) | 252 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy) |
184 | const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate) | 253 | const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) |
185 | 254 | ||
186 | return totalWillDuplicate > maxSize | 255 | return totalWillDuplicate > maxSize |
187 | } | 256 | } |
@@ -191,13 +260,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
191 | } | 260 | } |
192 | 261 | ||
193 | private buildEntryLogId (object: VideoRedundancyModel) { | 262 | private buildEntryLogId (object: VideoRedundancyModel) { |
194 | return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` | 263 | if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` |
264 | |||
265 | return `${object.VideoStreamingPlaylist.playlistUrl}` | ||
195 | } | 266 | } |
196 | 267 | ||
197 | private getTotalFileSizes (files: VideoFileModel[]) { | 268 | private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) { |
198 | const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size | 269 | const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size |
199 | 270 | ||
200 | return files.reduce(fileReducer, 0) | 271 | return files.reduce(fileReducer, 0) * playlists.length |
201 | } | 272 | } |
202 | 273 | ||
203 | private async loadAndRefreshVideo (videoUrl: string) { | 274 | private async loadAndRefreshVideo (videoUrl: string) { |
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index 4460f46e4..608badfef 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts | |||
@@ -1,11 +1,14 @@ | |||
1 | import { CONFIG } from '../initializers' | 1 | import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers' |
2 | import { extname, join } from 'path' | 2 | import { extname, join } from 'path' |
3 | import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' | 3 | import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' |
4 | import { copy, remove, move, stat } from 'fs-extra' | 4 | import { copy, ensureDir, move, remove, stat } from 'fs-extra' |
5 | import { logger } from '../helpers/logger' | 5 | import { logger } from '../helpers/logger' |
6 | import { VideoResolution } from '../../shared/models/videos' | 6 | import { VideoResolution } from '../../shared/models/videos' |
7 | import { VideoFileModel } from '../models/video/video-file' | 7 | import { VideoFileModel } from '../models/video/video-file' |
8 | import { VideoModel } from '../models/video/video' | 8 | import { VideoModel } from '../models/video/video' |
9 | import { updateMasterHLSPlaylist, updateSha256Segments } from './hls' | ||
10 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
11 | import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type' | ||
9 | 12 | ||
10 | async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { | 13 | async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { |
11 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | 14 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR |
@@ -17,7 +20,8 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi | |||
17 | 20 | ||
18 | const transcodeOptions = { | 21 | const transcodeOptions = { |
19 | inputPath: videoInputPath, | 22 | inputPath: videoInputPath, |
20 | outputPath: videoTranscodedPath | 23 | outputPath: videoTranscodedPath, |
24 | resolution: inputVideoFile.resolution | ||
21 | } | 25 | } |
22 | 26 | ||
23 | // Could be very long! | 27 | // Could be very long! |
@@ -47,7 +51,7 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi | |||
47 | } | 51 | } |
48 | } | 52 | } |
49 | 53 | ||
50 | async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { | 54 | async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) { |
51 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | 55 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR |
52 | const extname = '.mp4' | 56 | const extname = '.mp4' |
53 | 57 | ||
@@ -60,13 +64,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR | |||
60 | size: 0, | 64 | size: 0, |
61 | videoId: video.id | 65 | videoId: video.id |
62 | }) | 66 | }) |
63 | const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) | 67 | const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile)) |
64 | 68 | ||
65 | const transcodeOptions = { | 69 | const transcodeOptions = { |
66 | inputPath: videoInputPath, | 70 | inputPath: videoInputPath, |
67 | outputPath: videoOutputPath, | 71 | outputPath: videoOutputPath, |
68 | resolution, | 72 | resolution, |
69 | isPortraitMode | 73 | isPortraitMode: isPortrait |
70 | } | 74 | } |
71 | 75 | ||
72 | await transcode(transcodeOptions) | 76 | await transcode(transcodeOptions) |
@@ -84,6 +88,38 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR | |||
84 | video.VideoFiles.push(newVideoFile) | 88 | video.VideoFiles.push(newVideoFile) |
85 | } | 89 | } |
86 | 90 | ||
91 | async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { | ||
92 | const baseHlsDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) | ||
93 | await ensureDir(join(HLS_PLAYLIST_DIRECTORY, video.uuid)) | ||
94 | |||
95 | const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile())) | ||
96 | const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) | ||
97 | |||
98 | const transcodeOptions = { | ||
99 | inputPath: videoInputPath, | ||
100 | outputPath, | ||
101 | resolution, | ||
102 | isPortraitMode, | ||
103 | generateHlsPlaylist: true | ||
104 | } | ||
105 | |||
106 | await transcode(transcodeOptions) | ||
107 | |||
108 | await updateMasterHLSPlaylist(video) | ||
109 | await updateSha256Segments(video) | ||
110 | |||
111 | const playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) | ||
112 | |||
113 | await VideoStreamingPlaylistModel.upsert({ | ||
114 | videoId: video.id, | ||
115 | playlistUrl, | ||
116 | segmentsSha256Url: CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid), | ||
117 | p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles), | ||
118 | |||
119 | type: VideoStreamingPlaylistType.HLS | ||
120 | }) | ||
121 | } | ||
122 | |||
87 | async function importVideoFile (video: VideoModel, inputFilePath: string) { | 123 | async function importVideoFile (video: VideoModel, inputFilePath: string) { |
88 | const { videoFileResolution } = await getVideoFileResolution(inputFilePath) | 124 | const { videoFileResolution } = await getVideoFileResolution(inputFilePath) |
89 | const { size } = await stat(inputFilePath) | 125 | const { size } = await stat(inputFilePath) |
@@ -125,6 +161,7 @@ async function importVideoFile (video: VideoModel, inputFilePath: string) { | |||
125 | } | 161 | } |
126 | 162 | ||
127 | export { | 163 | export { |
164 | generateHlsPlaylist, | ||
128 | optimizeVideofile, | 165 | optimizeVideofile, |
129 | transcodeOriginalVideofile, | 166 | transcodeOriginalVideofile, |
130 | importVideoFile | 167 | importVideoFile |
diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts index c72ab78b2..329322509 100644 --- a/server/middlewares/validators/redundancy.ts +++ b/server/middlewares/validators/redundancy.ts | |||
@@ -13,7 +13,7 @@ import { ActorFollowModel } from '../../models/activitypub/actor-follow' | |||
13 | import { SERVER_ACTOR_NAME } from '../../initializers' | 13 | import { SERVER_ACTOR_NAME } from '../../initializers' |
14 | import { ServerModel } from '../../models/server/server' | 14 | import { ServerModel } from '../../models/server/server' |
15 | 15 | ||
16 | const videoRedundancyGetValidator = [ | 16 | const videoFileRedundancyGetValidator = [ |
17 | param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), | 17 | param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), |
18 | param('resolution') | 18 | param('resolution') |
19 | .customSanitizer(toIntOrNull) | 19 | .customSanitizer(toIntOrNull) |
@@ -24,7 +24,7 @@ const videoRedundancyGetValidator = [ | |||
24 | .custom(exists).withMessage('Should have a valid fps'), | 24 | .custom(exists).withMessage('Should have a valid fps'), |
25 | 25 | ||
26 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | 26 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { |
27 | logger.debug('Checking videoRedundancyGetValidator parameters', { parameters: req.params }) | 27 | logger.debug('Checking videoFileRedundancyGetValidator parameters', { parameters: req.params }) |
28 | 28 | ||
29 | if (areValidationErrors(req, res)) return | 29 | if (areValidationErrors(req, res)) return |
30 | if (!await isVideoExist(req.params.videoId, res)) return | 30 | if (!await isVideoExist(req.params.videoId, res)) return |
@@ -38,7 +38,31 @@ const videoRedundancyGetValidator = [ | |||
38 | res.locals.videoFile = videoFile | 38 | res.locals.videoFile = videoFile |
39 | 39 | ||
40 | const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) | 40 | const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) |
41 | if (!videoRedundancy)return res.status(404).json({ error: 'Video redundancy not found.' }) | 41 | if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' }) |
42 | res.locals.videoRedundancy = videoRedundancy | ||
43 | |||
44 | return next() | ||
45 | } | ||
46 | ] | ||
47 | |||
48 | const videoPlaylistRedundancyGetValidator = [ | ||
49 | param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), | ||
50 | param('streamingPlaylistType').custom(exists).withMessage('Should have a valid streaming playlist type'), | ||
51 | |||
52 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
53 | logger.debug('Checking videoPlaylistRedundancyGetValidator parameters', { parameters: req.params }) | ||
54 | |||
55 | if (areValidationErrors(req, res)) return | ||
56 | if (!await isVideoExist(req.params.videoId, res)) return | ||
57 | |||
58 | const video: VideoModel = res.locals.video | ||
59 | const videoStreamingPlaylist = video.VideoStreamingPlaylists.find(p => p === req.params.streamingPlaylistType) | ||
60 | |||
61 | if (!videoStreamingPlaylist) return res.status(404).json({ error: 'Video playlist not found.' }) | ||
62 | res.locals.videoStreamingPlaylist = videoStreamingPlaylist | ||
63 | |||
64 | const videoRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(videoStreamingPlaylist.id) | ||
65 | if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' }) | ||
42 | res.locals.videoRedundancy = videoRedundancy | 66 | res.locals.videoRedundancy = videoRedundancy |
43 | 67 | ||
44 | return next() | 68 | return next() |
@@ -75,6 +99,7 @@ const updateServerRedundancyValidator = [ | |||
75 | // --------------------------------------------------------------------------- | 99 | // --------------------------------------------------------------------------- |
76 | 100 | ||
77 | export { | 101 | export { |
78 | videoRedundancyGetValidator, | 102 | videoFileRedundancyGetValidator, |
103 | videoPlaylistRedundancyGetValidator, | ||
79 | updateServerRedundancyValidator | 104 | updateServerRedundancyValidator |
80 | } | 105 | } |
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 8f2ef2d9a..b722bed14 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts | |||
@@ -28,6 +28,7 @@ import { sample } from 'lodash' | |||
28 | import { isTestInstance } from '../../helpers/core-utils' | 28 | import { isTestInstance } from '../../helpers/core-utils' |
29 | import * as Bluebird from 'bluebird' | 29 | import * as Bluebird from 'bluebird' |
30 | import * as Sequelize from 'sequelize' | 30 | import * as Sequelize from 'sequelize' |
31 | import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' | ||
31 | 32 | ||
32 | export enum ScopeNames { | 33 | export enum ScopeNames { |
33 | WITH_VIDEO = 'WITH_VIDEO' | 34 | WITH_VIDEO = 'WITH_VIDEO' |
@@ -38,7 +39,17 @@ export enum ScopeNames { | |||
38 | include: [ | 39 | include: [ |
39 | { | 40 | { |
40 | model: () => VideoFileModel, | 41 | model: () => VideoFileModel, |
41 | required: true, | 42 | required: false, |
43 | include: [ | ||
44 | { | ||
45 | model: () => VideoModel, | ||
46 | required: true | ||
47 | } | ||
48 | ] | ||
49 | }, | ||
50 | { | ||
51 | model: () => VideoStreamingPlaylistModel, | ||
52 | required: false, | ||
42 | include: [ | 53 | include: [ |
43 | { | 54 | { |
44 | model: () => VideoModel, | 55 | model: () => VideoModel, |
@@ -97,12 +108,24 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
97 | 108 | ||
98 | @BelongsTo(() => VideoFileModel, { | 109 | @BelongsTo(() => VideoFileModel, { |
99 | foreignKey: { | 110 | foreignKey: { |
100 | allowNull: false | 111 | allowNull: true |
101 | }, | 112 | }, |
102 | onDelete: 'cascade' | 113 | onDelete: 'cascade' |
103 | }) | 114 | }) |
104 | VideoFile: VideoFileModel | 115 | VideoFile: VideoFileModel |
105 | 116 | ||
117 | @ForeignKey(() => VideoStreamingPlaylistModel) | ||
118 | @Column | ||
119 | videoStreamingPlaylistId: number | ||
120 | |||
121 | @BelongsTo(() => VideoStreamingPlaylistModel, { | ||
122 | foreignKey: { | ||
123 | allowNull: true | ||
124 | }, | ||
125 | onDelete: 'cascade' | ||
126 | }) | ||
127 | VideoStreamingPlaylist: VideoStreamingPlaylistModel | ||
128 | |||
106 | @ForeignKey(() => ActorModel) | 129 | @ForeignKey(() => ActorModel) |
107 | @Column | 130 | @Column |
108 | actorId: number | 131 | actorId: number |
@@ -119,13 +142,25 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
119 | static async removeFile (instance: VideoRedundancyModel) { | 142 | static async removeFile (instance: VideoRedundancyModel) { |
120 | if (!instance.isOwned()) return | 143 | if (!instance.isOwned()) return |
121 | 144 | ||
122 | const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId) | 145 | if (instance.videoFileId) { |
146 | const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId) | ||
123 | 147 | ||
124 | const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` | 148 | const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` |
125 | logger.info('Removing duplicated video file %s.', logIdentifier) | 149 | logger.info('Removing duplicated video file %s.', logIdentifier) |
126 | 150 | ||
127 | videoFile.Video.removeFile(videoFile, true) | 151 | videoFile.Video.removeFile(videoFile, true) |
128 | .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) | 152 | .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) |
153 | } | ||
154 | |||
155 | if (instance.videoStreamingPlaylistId) { | ||
156 | const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId) | ||
157 | |||
158 | const videoUUID = videoStreamingPlaylist.Video.uuid | ||
159 | logger.info('Removing duplicated video streaming playlist %s.', videoUUID) | ||
160 | |||
161 | videoStreamingPlaylist.Video.removeStreamingPlaylist(true) | ||
162 | .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err })) | ||
163 | } | ||
129 | 164 | ||
130 | return undefined | 165 | return undefined |
131 | } | 166 | } |
@@ -143,6 +178,19 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
143 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) | 178 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) |
144 | } | 179 | } |
145 | 180 | ||
181 | static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number) { | ||
182 | const actor = await getServerActor() | ||
183 | |||
184 | const query = { | ||
185 | where: { | ||
186 | actorId: actor.id, | ||
187 | videoStreamingPlaylistId | ||
188 | } | ||
189 | } | ||
190 | |||
191 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) | ||
192 | } | ||
193 | |||
146 | static loadByUrl (url: string, transaction?: Sequelize.Transaction) { | 194 | static loadByUrl (url: string, transaction?: Sequelize.Transaction) { |
147 | const query = { | 195 | const query = { |
148 | where: { | 196 | where: { |
@@ -191,7 +239,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
191 | const ids = rows.map(r => r.id) | 239 | const ids = rows.map(r => r.id) |
192 | const id = sample(ids) | 240 | const id = sample(ids) |
193 | 241 | ||
194 | return VideoModel.loadWithFile(id, undefined, !isTestInstance()) | 242 | return VideoModel.loadWithFiles(id, undefined, !isTestInstance()) |
195 | } | 243 | } |
196 | 244 | ||
197 | static async findMostViewToDuplicate (randomizedFactor: number) { | 245 | static async findMostViewToDuplicate (randomizedFactor: number) { |
@@ -333,40 +381,44 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
333 | 381 | ||
334 | static async listLocalOfServer (serverId: number) { | 382 | static async listLocalOfServer (serverId: number) { |
335 | const actor = await getServerActor() | 383 | const actor = await getServerActor() |
336 | 384 | const buildVideoInclude = () => ({ | |
337 | const query = { | 385 | model: VideoModel, |
338 | where: { | 386 | required: true, |
339 | actorId: actor.id | ||
340 | }, | ||
341 | include: [ | 387 | include: [ |
342 | { | 388 | { |
343 | model: VideoFileModel, | 389 | attributes: [], |
390 | model: VideoChannelModel.unscoped(), | ||
344 | required: true, | 391 | required: true, |
345 | include: [ | 392 | include: [ |
346 | { | 393 | { |
347 | model: VideoModel, | 394 | attributes: [], |
395 | model: ActorModel.unscoped(), | ||
348 | required: true, | 396 | required: true, |
349 | include: [ | 397 | where: { |
350 | { | 398 | serverId |
351 | attributes: [], | 399 | } |
352 | model: VideoChannelModel.unscoped(), | ||
353 | required: true, | ||
354 | include: [ | ||
355 | { | ||
356 | attributes: [], | ||
357 | model: ActorModel.unscoped(), | ||
358 | required: true, | ||
359 | where: { | ||
360 | serverId | ||
361 | } | ||
362 | } | ||
363 | ] | ||
364 | } | ||
365 | ] | ||
366 | } | 400 | } |
367 | ] | 401 | ] |
368 | } | 402 | } |
369 | ] | 403 | ] |
404 | }) | ||
405 | |||
406 | const query = { | ||
407 | where: { | ||
408 | actorId: actor.id | ||
409 | }, | ||
410 | include: [ | ||
411 | { | ||
412 | model: VideoFileModel, | ||
413 | required: false, | ||
414 | include: [ buildVideoInclude() ] | ||
415 | }, | ||
416 | { | ||
417 | model: VideoStreamingPlaylistModel, | ||
418 | required: false, | ||
419 | include: [ buildVideoInclude() ] | ||
420 | } | ||
421 | ] | ||
370 | } | 422 | } |
371 | 423 | ||
372 | return VideoRedundancyModel.findAll(query) | 424 | return VideoRedundancyModel.findAll(query) |
@@ -403,11 +455,32 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
403 | })) | 455 | })) |
404 | } | 456 | } |
405 | 457 | ||
458 | getVideo () { | ||
459 | if (this.VideoFile) return this.VideoFile.Video | ||
460 | |||
461 | return this.VideoStreamingPlaylist.Video | ||
462 | } | ||
463 | |||
406 | isOwned () { | 464 | isOwned () { |
407 | return !!this.strategy | 465 | return !!this.strategy |
408 | } | 466 | } |
409 | 467 | ||
410 | toActivityPubObject (): CacheFileObject { | 468 | toActivityPubObject (): CacheFileObject { |
469 | if (this.VideoStreamingPlaylist) { | ||
470 | return { | ||
471 | id: this.url, | ||
472 | type: 'CacheFile' as 'CacheFile', | ||
473 | object: this.VideoStreamingPlaylist.Video.url, | ||
474 | expires: this.expiresOn.toISOString(), | ||
475 | url: { | ||
476 | type: 'Link', | ||
477 | mimeType: 'application/x-mpegURL', | ||
478 | mediaType: 'application/x-mpegURL', | ||
479 | href: this.fileUrl | ||
480 | } | ||
481 | } | ||
482 | } | ||
483 | |||
411 | return { | 484 | return { |
412 | id: this.url, | 485 | id: this.url, |
413 | type: 'CacheFile' as 'CacheFile', | 486 | type: 'CacheFile' as 'CacheFile', |
@@ -431,7 +504,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
431 | 504 | ||
432 | const notIn = Sequelize.literal( | 505 | const notIn = Sequelize.literal( |
433 | '(' + | 506 | '(' + |
434 | `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` + | 507 | `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` + |
435 | ')' | 508 | ')' |
436 | ) | 509 | ) |
437 | 510 | ||
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 1f1b76c1e..7d1e371b9 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts | |||
@@ -62,7 +62,7 @@ export class VideoFileModel extends Model<VideoFileModel> { | |||
62 | extname: string | 62 | extname: string |
63 | 63 | ||
64 | @AllowNull(false) | 64 | @AllowNull(false) |
65 | @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash')) | 65 | @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash')) |
66 | @Column | 66 | @Column |
67 | infoHash: string | 67 | infoHash: string |
68 | 68 | ||
@@ -86,14 +86,14 @@ export class VideoFileModel extends Model<VideoFileModel> { | |||
86 | 86 | ||
87 | @HasMany(() => VideoRedundancyModel, { | 87 | @HasMany(() => VideoRedundancyModel, { |
88 | foreignKey: { | 88 | foreignKey: { |
89 | allowNull: false | 89 | allowNull: true |
90 | }, | 90 | }, |
91 | onDelete: 'CASCADE', | 91 | onDelete: 'CASCADE', |
92 | hooks: true | 92 | hooks: true |
93 | }) | 93 | }) |
94 | RedundancyVideos: VideoRedundancyModel[] | 94 | RedundancyVideos: VideoRedundancyModel[] |
95 | 95 | ||
96 | static isInfohashExists (infoHash: string) { | 96 | static doesInfohashExist (infoHash: string) { |
97 | const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' | 97 | const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' |
98 | const options = { | 98 | const options = { |
99 | type: Sequelize.QueryTypes.SELECT, | 99 | type: Sequelize.QueryTypes.SELECT, |
diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index de0747f22..e49dbee30 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts | |||
@@ -1,7 +1,12 @@ | |||
1 | import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' | 1 | import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' |
2 | import { VideoModel } from './video' | 2 | import { VideoModel } from './video' |
3 | import { VideoFileModel } from './video-file' | 3 | import { VideoFileModel } from './video-file' |
4 | import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 4 | import { |
5 | ActivityPlaylistInfohashesObject, | ||
6 | ActivityPlaylistSegmentHashesObject, | ||
7 | ActivityUrlObject, | ||
8 | VideoTorrentObject | ||
9 | } from '../../../shared/models/activitypub/objects' | ||
5 | import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers' | 10 | import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers' |
6 | import { VideoCaptionModel } from './video-caption' | 11 | import { VideoCaptionModel } from './video-caption' |
7 | import { | 12 | import { |
@@ -11,6 +16,8 @@ import { | |||
11 | getVideoSharesActivityPubUrl | 16 | getVideoSharesActivityPubUrl |
12 | } from '../../lib/activitypub' | 17 | } from '../../lib/activitypub' |
13 | import { isArray } from '../../helpers/custom-validators/misc' | 18 | import { isArray } from '../../helpers/custom-validators/misc' |
19 | import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' | ||
20 | import { VideoStreamingPlaylistModel } from './video-streaming-playlist' | ||
14 | 21 | ||
15 | export type VideoFormattingJSONOptions = { | 22 | export type VideoFormattingJSONOptions = { |
16 | completeDescription?: boolean | 23 | completeDescription?: boolean |
@@ -120,7 +127,12 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { | |||
120 | } | 127 | } |
121 | }) | 128 | }) |
122 | 129 | ||
130 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | ||
131 | |||
123 | const tags = video.Tags ? video.Tags.map(t => t.name) : [] | 132 | const tags = video.Tags ? video.Tags.map(t => t.name) : [] |
133 | |||
134 | const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists) | ||
135 | |||
124 | const detailsJson = { | 136 | const detailsJson = { |
125 | support: video.support, | 137 | support: video.support, |
126 | descriptionPath: video.getDescriptionAPIPath(), | 138 | descriptionPath: video.getDescriptionAPIPath(), |
@@ -133,7 +145,11 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { | |||
133 | id: video.state, | 145 | id: video.state, |
134 | label: VideoModel.getStateLabel(video.state) | 146 | label: VideoModel.getStateLabel(video.state) |
135 | }, | 147 | }, |
136 | files: [] | 148 | |
149 | trackerUrls: video.getTrackerUrls(baseUrlHttp, baseUrlWs), | ||
150 | |||
151 | files: [], | ||
152 | streamingPlaylists | ||
137 | } | 153 | } |
138 | 154 | ||
139 | // Format and sort video files | 155 | // Format and sort video files |
@@ -142,6 +158,25 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { | |||
142 | return Object.assign(formattedJson, detailsJson) | 158 | return Object.assign(formattedJson, detailsJson) |
143 | } | 159 | } |
144 | 160 | ||
161 | function streamingPlaylistsModelToFormattedJSON (video: VideoModel, playlists: VideoStreamingPlaylistModel[]): VideoStreamingPlaylist[] { | ||
162 | if (isArray(playlists) === false) return [] | ||
163 | |||
164 | return playlists | ||
165 | .map(playlist => { | ||
166 | const redundancies = isArray(playlist.RedundancyVideos) | ||
167 | ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl })) | ||
168 | : [] | ||
169 | |||
170 | return { | ||
171 | id: playlist.id, | ||
172 | type: playlist.type, | ||
173 | playlistUrl: playlist.playlistUrl, | ||
174 | segmentsSha256Url: playlist.segmentsSha256Url, | ||
175 | redundancies | ||
176 | } as VideoStreamingPlaylist | ||
177 | }) | ||
178 | } | ||
179 | |||
145 | function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] { | 180 | function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] { |
146 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | 181 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() |
147 | 182 | ||
@@ -232,6 +267,28 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { | |||
232 | }) | 267 | }) |
233 | } | 268 | } |
234 | 269 | ||
270 | for (const playlist of (video.VideoStreamingPlaylists || [])) { | ||
271 | let tag: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[] | ||
272 | |||
273 | tag = playlist.p2pMediaLoaderInfohashes | ||
274 | .map(i => ({ type: 'Infohash' as 'Infohash', name: i })) | ||
275 | tag.push({ | ||
276 | type: 'Link', | ||
277 | name: 'sha256', | ||
278 | mimeType: 'application/json' as 'application/json', | ||
279 | mediaType: 'application/json' as 'application/json', | ||
280 | href: playlist.segmentsSha256Url | ||
281 | }) | ||
282 | |||
283 | url.push({ | ||
284 | type: 'Link', | ||
285 | mimeType: 'application/x-mpegURL' as 'application/x-mpegURL', | ||
286 | mediaType: 'application/x-mpegURL' as 'application/x-mpegURL', | ||
287 | href: playlist.playlistUrl, | ||
288 | tag | ||
289 | }) | ||
290 | } | ||
291 | |||
235 | // Add video url too | 292 | // Add video url too |
236 | url.push({ | 293 | url.push({ |
237 | type: 'Link', | 294 | type: 'Link', |
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts new file mode 100644 index 000000000..bce537781 --- /dev/null +++ b/server/models/video/video-streaming-playlist.ts | |||
@@ -0,0 +1,154 @@ | |||
1 | import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' | ||
2 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' | ||
3 | import { throwIfNotValid } from '../utils' | ||
4 | import { VideoModel } from './video' | ||
5 | import * as Sequelize from 'sequelize' | ||
6 | import { VideoRedundancyModel } from '../redundancy/video-redundancy' | ||
7 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
8 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | ||
9 | import { CONSTRAINTS_FIELDS, STATIC_PATHS } from '../../initializers' | ||
10 | import { VideoFileModel } from './video-file' | ||
11 | import { join } from 'path' | ||
12 | import { sha1 } from '../../helpers/core-utils' | ||
13 | import { isArrayOf } from '../../helpers/custom-validators/misc' | ||
14 | |||
15 | @Table({ | ||
16 | tableName: 'videoStreamingPlaylist', | ||
17 | indexes: [ | ||
18 | { | ||
19 | fields: [ 'videoId' ] | ||
20 | }, | ||
21 | { | ||
22 | fields: [ 'videoId', 'type' ], | ||
23 | unique: true | ||
24 | }, | ||
25 | { | ||
26 | fields: [ 'p2pMediaLoaderInfohashes' ], | ||
27 | using: 'gin' | ||
28 | } | ||
29 | ] | ||
30 | }) | ||
31 | export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistModel> { | ||
32 | @CreatedAt | ||
33 | createdAt: Date | ||
34 | |||
35 | @UpdatedAt | ||
36 | updatedAt: Date | ||
37 | |||
38 | @AllowNull(false) | ||
39 | @Column | ||
40 | type: VideoStreamingPlaylistType | ||
41 | |||
42 | @AllowNull(false) | ||
43 | @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url')) | ||
44 | @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max)) | ||
45 | playlistUrl: string | ||
46 | |||
47 | @AllowNull(false) | ||
48 | @Is('VideoStreamingPlaylistInfoHashes', value => throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes')) | ||
49 | @Column(DataType.ARRAY(DataType.STRING)) | ||
50 | p2pMediaLoaderInfohashes: string[] | ||
51 | |||
52 | @AllowNull(false) | ||
53 | @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url')) | ||
54 | @Column | ||
55 | segmentsSha256Url: string | ||
56 | |||
57 | @ForeignKey(() => VideoModel) | ||
58 | @Column | ||
59 | videoId: number | ||
60 | |||
61 | @BelongsTo(() => VideoModel, { | ||
62 | foreignKey: { | ||
63 | allowNull: false | ||
64 | }, | ||
65 | onDelete: 'CASCADE' | ||
66 | }) | ||
67 | Video: VideoModel | ||
68 | |||
69 | @HasMany(() => VideoRedundancyModel, { | ||
70 | foreignKey: { | ||
71 | allowNull: false | ||
72 | }, | ||
73 | onDelete: 'CASCADE', | ||
74 | hooks: true | ||
75 | }) | ||
76 | RedundancyVideos: VideoRedundancyModel[] | ||
77 | |||
78 | static doesInfohashExist (infoHash: string) { | ||
79 | const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1' | ||
80 | const options = { | ||
81 | type: Sequelize.QueryTypes.SELECT, | ||
82 | bind: { infoHash }, | ||
83 | raw: true | ||
84 | } | ||
85 | |||
86 | return VideoModel.sequelize.query(query, options) | ||
87 | .then(results => { | ||
88 | return results.length === 1 | ||
89 | }) | ||
90 | } | ||
91 | |||
92 | static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: VideoFileModel[]) { | ||
93 | const hashes: string[] = [] | ||
94 | |||
95 | // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L97 | ||
96 | for (let i = 0; i < videoFiles.length; i++) { | ||
97 | hashes.push(sha1(`1${playlistUrl}+V${i}`)) | ||
98 | } | ||
99 | |||
100 | return hashes | ||
101 | } | ||
102 | |||
103 | static loadWithVideo (id: number) { | ||
104 | const options = { | ||
105 | include: [ | ||
106 | { | ||
107 | model: VideoModel.unscoped(), | ||
108 | required: true | ||
109 | } | ||
110 | ] | ||
111 | } | ||
112 | |||
113 | return VideoStreamingPlaylistModel.findById(id, options) | ||
114 | } | ||
115 | |||
116 | static getHlsPlaylistFilename (resolution: number) { | ||
117 | return resolution + '.m3u8' | ||
118 | } | ||
119 | |||
120 | static getMasterHlsPlaylistFilename () { | ||
121 | return 'master.m3u8' | ||
122 | } | ||
123 | |||
124 | static getHlsSha256SegmentsFilename () { | ||
125 | return 'segments-sha256.json' | ||
126 | } | ||
127 | |||
128 | static getHlsMasterPlaylistStaticPath (videoUUID: string) { | ||
129 | return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) | ||
130 | } | ||
131 | |||
132 | static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) { | ||
133 | return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) | ||
134 | } | ||
135 | |||
136 | static getHlsSha256SegmentsStaticPath (videoUUID: string) { | ||
137 | return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) | ||
138 | } | ||
139 | |||
140 | getStringType () { | ||
141 | if (this.type === VideoStreamingPlaylistType.HLS) return 'hls' | ||
142 | |||
143 | return 'unknown' | ||
144 | } | ||
145 | |||
146 | getVideoRedundancyUrl (baseUrlHttp: string) { | ||
147 | return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getStringType() + '/' + this.Video.uuid | ||
148 | } | ||
149 | |||
150 | hasSameUniqueKeysThan (other: VideoStreamingPlaylistModel) { | ||
151 | return this.type === other.type && | ||
152 | this.videoId === other.videoId | ||
153 | } | ||
154 | } | ||
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 80a6c7832..702260772 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -52,7 +52,7 @@ import { | |||
52 | ACTIVITY_PUB, | 52 | ACTIVITY_PUB, |
53 | API_VERSION, | 53 | API_VERSION, |
54 | CONFIG, | 54 | CONFIG, |
55 | CONSTRAINTS_FIELDS, | 55 | CONSTRAINTS_FIELDS, HLS_PLAYLIST_DIRECTORY, HLS_REDUNDANCY_DIRECTORY, |
56 | PREVIEWS_SIZE, | 56 | PREVIEWS_SIZE, |
57 | REMOTE_SCHEME, | 57 | REMOTE_SCHEME, |
58 | STATIC_DOWNLOAD_PATHS, | 58 | STATIC_DOWNLOAD_PATHS, |
@@ -95,6 +95,7 @@ import * as validator from 'validator' | |||
95 | import { UserVideoHistoryModel } from '../account/user-video-history' | 95 | import { UserVideoHistoryModel } from '../account/user-video-history' |
96 | import { UserModel } from '../account/user' | 96 | import { UserModel } from '../account/user' |
97 | import { VideoImportModel } from './video-import' | 97 | import { VideoImportModel } from './video-import' |
98 | import { VideoStreamingPlaylistModel } from './video-streaming-playlist' | ||
98 | 99 | ||
99 | // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation | 100 | // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation |
100 | const indexes: Sequelize.DefineIndexesOptions[] = [ | 101 | const indexes: Sequelize.DefineIndexesOptions[] = [ |
@@ -159,7 +160,9 @@ export enum ScopeNames { | |||
159 | WITH_FILES = 'WITH_FILES', | 160 | WITH_FILES = 'WITH_FILES', |
160 | WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', | 161 | WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', |
161 | WITH_BLACKLISTED = 'WITH_BLACKLISTED', | 162 | WITH_BLACKLISTED = 'WITH_BLACKLISTED', |
162 | WITH_USER_HISTORY = 'WITH_USER_HISTORY' | 163 | WITH_USER_HISTORY = 'WITH_USER_HISTORY', |
164 | WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS', | ||
165 | WITH_USER_ID = 'WITH_USER_ID' | ||
163 | } | 166 | } |
164 | 167 | ||
165 | type ForAPIOptions = { | 168 | type ForAPIOptions = { |
@@ -463,6 +466,22 @@ type AvailableForListIDsOptions = { | |||
463 | 466 | ||
464 | return query | 467 | return query |
465 | }, | 468 | }, |
469 | [ ScopeNames.WITH_USER_ID ]: { | ||
470 | include: [ | ||
471 | { | ||
472 | attributes: [ 'accountId' ], | ||
473 | model: () => VideoChannelModel.unscoped(), | ||
474 | required: true, | ||
475 | include: [ | ||
476 | { | ||
477 | attributes: [ 'userId' ], | ||
478 | model: () => AccountModel.unscoped(), | ||
479 | required: true | ||
480 | } | ||
481 | ] | ||
482 | } | ||
483 | ] | ||
484 | }, | ||
466 | [ ScopeNames.WITH_ACCOUNT_DETAILS ]: { | 485 | [ ScopeNames.WITH_ACCOUNT_DETAILS ]: { |
467 | include: [ | 486 | include: [ |
468 | { | 487 | { |
@@ -527,22 +546,55 @@ type AvailableForListIDsOptions = { | |||
527 | } | 546 | } |
528 | ] | 547 | ] |
529 | }, | 548 | }, |
530 | [ ScopeNames.WITH_FILES ]: { | 549 | [ ScopeNames.WITH_FILES ]: (withRedundancies = false) => { |
531 | include: [ | 550 | let subInclude: any[] = [] |
532 | { | 551 | |
533 | model: () => VideoFileModel.unscoped(), | 552 | if (withRedundancies === true) { |
534 | // FIXME: typings | 553 | subInclude = [ |
535 | [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join | 554 | { |
536 | required: false, | 555 | attributes: [ 'fileUrl' ], |
537 | include: [ | 556 | model: VideoRedundancyModel.unscoped(), |
538 | { | 557 | required: false |
539 | attributes: [ 'fileUrl' ], | 558 | } |
540 | model: () => VideoRedundancyModel.unscoped(), | 559 | ] |
541 | required: false | 560 | } |
542 | } | 561 | |
543 | ] | 562 | return { |
544 | } | 563 | include: [ |
545 | ] | 564 | { |
565 | model: VideoFileModel.unscoped(), | ||
566 | // FIXME: typings | ||
567 | [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join | ||
568 | required: false, | ||
569 | include: subInclude | ||
570 | } | ||
571 | ] | ||
572 | } | ||
573 | }, | ||
574 | [ ScopeNames.WITH_STREAMING_PLAYLISTS ]: (withRedundancies = false) => { | ||
575 | let subInclude: any[] = [] | ||
576 | |||
577 | if (withRedundancies === true) { | ||
578 | subInclude = [ | ||
579 | { | ||
580 | attributes: [ 'fileUrl' ], | ||
581 | model: VideoRedundancyModel.unscoped(), | ||
582 | required: false | ||
583 | } | ||
584 | ] | ||
585 | } | ||
586 | |||
587 | return { | ||
588 | include: [ | ||
589 | { | ||
590 | model: VideoStreamingPlaylistModel.unscoped(), | ||
591 | // FIXME: typings | ||
592 | [ 'separate' as any ]: true, // We may have multiple streaming playlists, having multiple redundancies so let's separate this join | ||
593 | required: false, | ||
594 | include: subInclude | ||
595 | } | ||
596 | ] | ||
597 | } | ||
546 | }, | 598 | }, |
547 | [ ScopeNames.WITH_SCHEDULED_UPDATE ]: { | 599 | [ ScopeNames.WITH_SCHEDULED_UPDATE ]: { |
548 | include: [ | 600 | include: [ |
@@ -722,6 +774,16 @@ export class VideoModel extends Model<VideoModel> { | |||
722 | }) | 774 | }) |
723 | VideoFiles: VideoFileModel[] | 775 | VideoFiles: VideoFileModel[] |
724 | 776 | ||
777 | @HasMany(() => VideoStreamingPlaylistModel, { | ||
778 | foreignKey: { | ||
779 | name: 'videoId', | ||
780 | allowNull: false | ||
781 | }, | ||
782 | hooks: true, | ||
783 | onDelete: 'cascade' | ||
784 | }) | ||
785 | VideoStreamingPlaylists: VideoStreamingPlaylistModel[] | ||
786 | |||
725 | @HasMany(() => VideoShareModel, { | 787 | @HasMany(() => VideoShareModel, { |
726 | foreignKey: { | 788 | foreignKey: { |
727 | name: 'videoId', | 789 | name: 'videoId', |
@@ -847,6 +909,9 @@ export class VideoModel extends Model<VideoModel> { | |||
847 | tasks.push(instance.removeFile(file)) | 909 | tasks.push(instance.removeFile(file)) |
848 | tasks.push(instance.removeTorrent(file)) | 910 | tasks.push(instance.removeTorrent(file)) |
849 | }) | 911 | }) |
912 | |||
913 | // Remove playlists file | ||
914 | tasks.push(instance.removeStreamingPlaylist()) | ||
850 | } | 915 | } |
851 | 916 | ||
852 | // Do not wait video deletion because we could be in a transaction | 917 | // Do not wait video deletion because we could be in a transaction |
@@ -858,10 +923,6 @@ export class VideoModel extends Model<VideoModel> { | |||
858 | return undefined | 923 | return undefined |
859 | } | 924 | } |
860 | 925 | ||
861 | static list () { | ||
862 | return VideoModel.scope(ScopeNames.WITH_FILES).findAll() | ||
863 | } | ||
864 | |||
865 | static listLocal () { | 926 | static listLocal () { |
866 | const query = { | 927 | const query = { |
867 | where: { | 928 | where: { |
@@ -869,7 +930,7 @@ export class VideoModel extends Model<VideoModel> { | |||
869 | } | 930 | } |
870 | } | 931 | } |
871 | 932 | ||
872 | return VideoModel.scope(ScopeNames.WITH_FILES).findAll(query) | 933 | return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]).findAll(query) |
873 | } | 934 | } |
874 | 935 | ||
875 | static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { | 936 | static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { |
@@ -1200,6 +1261,16 @@ export class VideoModel extends Model<VideoModel> { | |||
1200 | return VideoModel.findOne(options) | 1261 | return VideoModel.findOne(options) |
1201 | } | 1262 | } |
1202 | 1263 | ||
1264 | static loadWithRights (id: number | string, t?: Sequelize.Transaction) { | ||
1265 | const where = VideoModel.buildWhereIdOrUUID(id) | ||
1266 | const options = { | ||
1267 | where, | ||
1268 | transaction: t | ||
1269 | } | ||
1270 | |||
1271 | return VideoModel.scope([ ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_USER_ID ]).findOne(options) | ||
1272 | } | ||
1273 | |||
1203 | static loadOnlyId (id: number | string, t?: Sequelize.Transaction) { | 1274 | static loadOnlyId (id: number | string, t?: Sequelize.Transaction) { |
1204 | const where = VideoModel.buildWhereIdOrUUID(id) | 1275 | const where = VideoModel.buildWhereIdOrUUID(id) |
1205 | 1276 | ||
@@ -1212,8 +1283,8 @@ export class VideoModel extends Model<VideoModel> { | |||
1212 | return VideoModel.findOne(options) | 1283 | return VideoModel.findOne(options) |
1213 | } | 1284 | } |
1214 | 1285 | ||
1215 | static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) { | 1286 | static loadWithFiles (id: number, t?: Sequelize.Transaction, logging?: boolean) { |
1216 | return VideoModel.scope(ScopeNames.WITH_FILES) | 1287 | return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]) |
1217 | .findById(id, { transaction: t, logging }) | 1288 | .findById(id, { transaction: t, logging }) |
1218 | } | 1289 | } |
1219 | 1290 | ||
@@ -1224,9 +1295,7 @@ export class VideoModel extends Model<VideoModel> { | |||
1224 | } | 1295 | } |
1225 | } | 1296 | } |
1226 | 1297 | ||
1227 | return VideoModel | 1298 | return VideoModel.findOne(options) |
1228 | .scope([ ScopeNames.WITH_FILES ]) | ||
1229 | .findOne(options) | ||
1230 | } | 1299 | } |
1231 | 1300 | ||
1232 | static loadByUrl (url: string, transaction?: Sequelize.Transaction) { | 1301 | static loadByUrl (url: string, transaction?: Sequelize.Transaction) { |
@@ -1248,7 +1317,11 @@ export class VideoModel extends Model<VideoModel> { | |||
1248 | transaction | 1317 | transaction |
1249 | } | 1318 | } |
1250 | 1319 | ||
1251 | return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) | 1320 | return VideoModel.scope([ |
1321 | ScopeNames.WITH_ACCOUNT_DETAILS, | ||
1322 | ScopeNames.WITH_FILES, | ||
1323 | ScopeNames.WITH_STREAMING_PLAYLISTS | ||
1324 | ]).findOne(query) | ||
1252 | } | 1325 | } |
1253 | 1326 | ||
1254 | static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) { | 1327 | static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) { |
@@ -1263,9 +1336,37 @@ export class VideoModel extends Model<VideoModel> { | |||
1263 | const scopes = [ | 1336 | const scopes = [ |
1264 | ScopeNames.WITH_TAGS, | 1337 | ScopeNames.WITH_TAGS, |
1265 | ScopeNames.WITH_BLACKLISTED, | 1338 | ScopeNames.WITH_BLACKLISTED, |
1339 | ScopeNames.WITH_ACCOUNT_DETAILS, | ||
1340 | ScopeNames.WITH_SCHEDULED_UPDATE, | ||
1266 | ScopeNames.WITH_FILES, | 1341 | ScopeNames.WITH_FILES, |
1342 | ScopeNames.WITH_STREAMING_PLAYLISTS | ||
1343 | ] | ||
1344 | |||
1345 | if (userId) { | ||
1346 | scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings | ||
1347 | } | ||
1348 | |||
1349 | return VideoModel | ||
1350 | .scope(scopes) | ||
1351 | .findOne(options) | ||
1352 | } | ||
1353 | |||
1354 | static loadForGetAPI (id: number | string, t?: Sequelize.Transaction, userId?: number) { | ||
1355 | const where = VideoModel.buildWhereIdOrUUID(id) | ||
1356 | |||
1357 | const options = { | ||
1358 | order: [ [ 'Tags', 'name', 'ASC' ] ], | ||
1359 | where, | ||
1360 | transaction: t | ||
1361 | } | ||
1362 | |||
1363 | const scopes = [ | ||
1364 | ScopeNames.WITH_TAGS, | ||
1365 | ScopeNames.WITH_BLACKLISTED, | ||
1267 | ScopeNames.WITH_ACCOUNT_DETAILS, | 1366 | ScopeNames.WITH_ACCOUNT_DETAILS, |
1268 | ScopeNames.WITH_SCHEDULED_UPDATE | 1367 | ScopeNames.WITH_SCHEDULED_UPDATE, |
1368 | { method: [ ScopeNames.WITH_FILES, true ] } as any, // FIXME: typings | ||
1369 | { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } as any // FIXME: typings | ||
1269 | ] | 1370 | ] |
1270 | 1371 | ||
1271 | if (userId) { | 1372 | if (userId) { |
@@ -1612,6 +1713,14 @@ export class VideoModel extends Model<VideoModel> { | |||
1612 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) | 1713 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) |
1613 | } | 1714 | } |
1614 | 1715 | ||
1716 | removeStreamingPlaylist (isRedundancy = false) { | ||
1717 | const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_PLAYLIST_DIRECTORY | ||
1718 | |||
1719 | const filePath = join(baseDir, this.uuid) | ||
1720 | return remove(filePath) | ||
1721 | .catch(err => logger.warn('Cannot delete playlist directory %s.', filePath, { err })) | ||
1722 | } | ||
1723 | |||
1615 | isOutdated () { | 1724 | isOutdated () { |
1616 | if (this.isOwned()) return false | 1725 | if (this.isOwned()) return false |
1617 | 1726 | ||
@@ -1646,7 +1755,7 @@ export class VideoModel extends Model<VideoModel> { | |||
1646 | 1755 | ||
1647 | generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { | 1756 | generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { |
1648 | const xs = this.getTorrentUrl(videoFile, baseUrlHttp) | 1757 | const xs = this.getTorrentUrl(videoFile, baseUrlHttp) |
1649 | const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] | 1758 | const announce = this.getTrackerUrls(baseUrlHttp, baseUrlWs) |
1650 | let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] | 1759 | let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] |
1651 | 1760 | ||
1652 | const redundancies = videoFile.RedundancyVideos | 1761 | const redundancies = videoFile.RedundancyVideos |
@@ -1663,6 +1772,10 @@ export class VideoModel extends Model<VideoModel> { | |||
1663 | return magnetUtil.encode(magnetHash) | 1772 | return magnetUtil.encode(magnetHash) |
1664 | } | 1773 | } |
1665 | 1774 | ||
1775 | getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) { | ||
1776 | return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] | ||
1777 | } | ||
1778 | |||
1666 | getThumbnailUrl (baseUrlHttp: string) { | 1779 | getThumbnailUrl (baseUrlHttp: string) { |
1667 | return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() | 1780 | return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() |
1668 | } | 1781 | } |
@@ -1686,4 +1799,8 @@ export class VideoModel extends Model<VideoModel> { | |||
1686 | getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { | 1799 | getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { |
1687 | return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) | 1800 | return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) |
1688 | } | 1801 | } |
1802 | |||
1803 | getBandwidthBits (videoFile: VideoFileModel) { | ||
1804 | return Math.ceil((videoFile.size * 8) / this.duration) | ||
1805 | } | ||
1689 | } | 1806 | } |
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 4038ecbf0..07de2b5a5 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts | |||
@@ -65,6 +65,9 @@ describe('Test config API validators', function () { | |||
65 | '480p': true, | 65 | '480p': true, |
66 | '720p': false, | 66 | '720p': false, |
67 | '1080p': false | 67 | '1080p': false |
68 | }, | ||
69 | hls: { | ||
70 | enabled: false | ||
68 | } | 71 | } |
69 | }, | 72 | }, |
70 | import: { | 73 | import: { |
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 9d3ce8153..5b99309fb 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts | |||
@@ -17,7 +17,7 @@ import { | |||
17 | viewVideo, | 17 | viewVideo, |
18 | wait, | 18 | wait, |
19 | waitUntilLog, | 19 | waitUntilLog, |
20 | checkVideoFilesWereRemoved, removeVideo, getVideoWithToken | 20 | checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer |
21 | } from '../../../../shared/utils' | 21 | } from '../../../../shared/utils' |
22 | import { waitJobs } from '../../../../shared/utils/server/jobs' | 22 | import { waitJobs } from '../../../../shared/utils/server/jobs' |
23 | 23 | ||
@@ -48,6 +48,11 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe | |||
48 | 48 | ||
49 | async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { | 49 | async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { |
50 | const config = { | 50 | const config = { |
51 | transcoding: { | ||
52 | hls: { | ||
53 | enabled: true | ||
54 | } | ||
55 | }, | ||
51 | redundancy: { | 56 | redundancy: { |
52 | videos: { | 57 | videos: { |
53 | check_interval: '5 seconds', | 58 | check_interval: '5 seconds', |
@@ -85,7 +90,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: | |||
85 | await waitJobs(servers) | 90 | await waitJobs(servers) |
86 | } | 91 | } |
87 | 92 | ||
88 | async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { | 93 | async function check1WebSeed (videoUUID?: string) { |
89 | if (!videoUUID) videoUUID = video1Server2UUID | 94 | if (!videoUUID) videoUUID = video1Server2UUID |
90 | 95 | ||
91 | const webseeds = [ | 96 | const webseeds = [ |
@@ -93,47 +98,17 @@ async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: str | |||
93 | ] | 98 | ] |
94 | 99 | ||
95 | for (const server of servers) { | 100 | for (const server of servers) { |
96 | { | 101 | // With token to avoid issues with video follow constraints |
97 | // With token to avoid issues with video follow constraints | 102 | const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) |
98 | const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) | ||
99 | 103 | ||
100 | const video: VideoDetails = res.body | 104 | const video: VideoDetails = res.body |
101 | for (const f of video.files) { | 105 | for (const f of video.files) { |
102 | checkMagnetWebseeds(f, webseeds, server) | 106 | checkMagnetWebseeds(f, webseeds, server) |
103 | } | ||
104 | } | 107 | } |
105 | } | 108 | } |
106 | } | 109 | } |
107 | 110 | ||
108 | async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { | 111 | async function check2Webseeds (videoUUID?: string) { |
109 | const res = await getStats(servers[0].url) | ||
110 | const data: ServerStats = res.body | ||
111 | |||
112 | expect(data.videosRedundancy).to.have.lengthOf(1) | ||
113 | const stat = data.videosRedundancy[0] | ||
114 | |||
115 | expect(stat.strategy).to.equal(strategy) | ||
116 | expect(stat.totalSize).to.equal(204800) | ||
117 | expect(stat.totalUsed).to.be.at.least(1).and.below(204801) | ||
118 | expect(stat.totalVideoFiles).to.equal(4) | ||
119 | expect(stat.totalVideos).to.equal(1) | ||
120 | } | ||
121 | |||
122 | async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { | ||
123 | const res = await getStats(servers[0].url) | ||
124 | const data: ServerStats = res.body | ||
125 | |||
126 | expect(data.videosRedundancy).to.have.lengthOf(1) | ||
127 | |||
128 | const stat = data.videosRedundancy[0] | ||
129 | expect(stat.strategy).to.equal(strategy) | ||
130 | expect(stat.totalSize).to.equal(204800) | ||
131 | expect(stat.totalUsed).to.equal(0) | ||
132 | expect(stat.totalVideoFiles).to.equal(0) | ||
133 | expect(stat.totalVideos).to.equal(0) | ||
134 | } | ||
135 | |||
136 | async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) { | ||
137 | if (!videoUUID) videoUUID = video1Server2UUID | 112 | if (!videoUUID) videoUUID = video1Server2UUID |
138 | 113 | ||
139 | const webseeds = [ | 114 | const webseeds = [ |
@@ -158,7 +133,7 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st | |||
158 | await makeGetRequest({ | 133 | await makeGetRequest({ |
159 | url: servers[1].url, | 134 | url: servers[1].url, |
160 | statusCodeExpected: 200, | 135 | statusCodeExpected: 200, |
161 | path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, | 136 | path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`, |
162 | contentType: null | 137 | contentType: null |
163 | }) | 138 | }) |
164 | } | 139 | } |
@@ -174,6 +149,81 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st | |||
174 | } | 149 | } |
175 | } | 150 | } |
176 | 151 | ||
152 | async function check0PlaylistRedundancies (videoUUID?: string) { | ||
153 | if (!videoUUID) videoUUID = video1Server2UUID | ||
154 | |||
155 | for (const server of servers) { | ||
156 | // With token to avoid issues with video follow constraints | ||
157 | const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) | ||
158 | const video: VideoDetails = res.body | ||
159 | |||
160 | expect(video.streamingPlaylists).to.be.an('array') | ||
161 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
162 | expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0) | ||
163 | } | ||
164 | } | ||
165 | |||
166 | async function check1PlaylistRedundancies (videoUUID?: string) { | ||
167 | if (!videoUUID) videoUUID = video1Server2UUID | ||
168 | |||
169 | for (const server of servers) { | ||
170 | const res = await getVideo(server.url, videoUUID) | ||
171 | const video: VideoDetails = res.body | ||
172 | |||
173 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
174 | expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1) | ||
175 | |||
176 | const redundancy = video.streamingPlaylists[0].redundancies[0] | ||
177 | |||
178 | expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) | ||
179 | } | ||
180 | |||
181 | await makeGetRequest({ | ||
182 | url: servers[0].url, | ||
183 | statusCodeExpected: 200, | ||
184 | path: `/static/redundancy/hls/${videoUUID}/360_000.ts`, | ||
185 | contentType: null | ||
186 | }) | ||
187 | |||
188 | for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) { | ||
189 | const files = await readdir(join(root(), directory, videoUUID)) | ||
190 | expect(files).to.have.length.at.least(4) | ||
191 | |||
192 | for (const resolution of [ 240, 360, 480, 720 ]) { | ||
193 | expect(files.find(f => f === `${resolution}_000.ts`)).to.not.be.undefined | ||
194 | expect(files.find(f => f === `${resolution}_001.ts`)).to.not.be.undefined | ||
195 | } | ||
196 | } | ||
197 | } | ||
198 | |||
199 | async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { | ||
200 | const res = await getStats(servers[0].url) | ||
201 | const data: ServerStats = res.body | ||
202 | |||
203 | expect(data.videosRedundancy).to.have.lengthOf(1) | ||
204 | const stat = data.videosRedundancy[0] | ||
205 | |||
206 | expect(stat.strategy).to.equal(strategy) | ||
207 | expect(stat.totalSize).to.equal(204800) | ||
208 | expect(stat.totalUsed).to.be.at.least(1).and.below(204801) | ||
209 | expect(stat.totalVideoFiles).to.equal(4) | ||
210 | expect(stat.totalVideos).to.equal(1) | ||
211 | } | ||
212 | |||
213 | async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { | ||
214 | const res = await getStats(servers[0].url) | ||
215 | const data: ServerStats = res.body | ||
216 | |||
217 | expect(data.videosRedundancy).to.have.lengthOf(1) | ||
218 | |||
219 | const stat = data.videosRedundancy[0] | ||
220 | expect(stat.strategy).to.equal(strategy) | ||
221 | expect(stat.totalSize).to.equal(204800) | ||
222 | expect(stat.totalUsed).to.equal(0) | ||
223 | expect(stat.totalVideoFiles).to.equal(0) | ||
224 | expect(stat.totalVideos).to.equal(0) | ||
225 | } | ||
226 | |||
177 | async function enableRedundancyOnServer1 () { | 227 | async function enableRedundancyOnServer1 () { |
178 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) | 228 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) |
179 | 229 | ||
@@ -220,7 +270,8 @@ describe('Test videos redundancy', function () { | |||
220 | }) | 270 | }) |
221 | 271 | ||
222 | it('Should have 1 webseed on the first video', async function () { | 272 | it('Should have 1 webseed on the first video', async function () { |
223 | await check1WebSeed(strategy) | 273 | await check1WebSeed() |
274 | await check0PlaylistRedundancies() | ||
224 | await checkStatsWith1Webseed(strategy) | 275 | await checkStatsWith1Webseed(strategy) |
225 | }) | 276 | }) |
226 | 277 | ||
@@ -229,27 +280,29 @@ describe('Test videos redundancy', function () { | |||
229 | }) | 280 | }) |
230 | 281 | ||
231 | it('Should have 2 webseeds on the first video', async function () { | 282 | it('Should have 2 webseeds on the first video', async function () { |
232 | this.timeout(40000) | 283 | this.timeout(80000) |
233 | 284 | ||
234 | await waitJobs(servers) | 285 | await waitJobs(servers) |
235 | await waitUntilLog(servers[0], 'Duplicated ', 4) | 286 | await waitUntilLog(servers[0], 'Duplicated ', 5) |
236 | await waitJobs(servers) | 287 | await waitJobs(servers) |
237 | 288 | ||
238 | await check2Webseeds(strategy) | 289 | await check2Webseeds() |
290 | await check1PlaylistRedundancies() | ||
239 | await checkStatsWith2Webseed(strategy) | 291 | await checkStatsWith2Webseed(strategy) |
240 | }) | 292 | }) |
241 | 293 | ||
242 | it('Should undo redundancy on server 1 and remove duplicated videos', async function () { | 294 | it('Should undo redundancy on server 1 and remove duplicated videos', async function () { |
243 | this.timeout(40000) | 295 | this.timeout(80000) |
244 | 296 | ||
245 | await disableRedundancyOnServer1() | 297 | await disableRedundancyOnServer1() |
246 | 298 | ||
247 | await waitJobs(servers) | 299 | await waitJobs(servers) |
248 | await wait(5000) | 300 | await wait(5000) |
249 | 301 | ||
250 | await check1WebSeed(strategy) | 302 | await check1WebSeed() |
303 | await check0PlaylistRedundancies() | ||
251 | 304 | ||
252 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) | 305 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ]) |
253 | }) | 306 | }) |
254 | 307 | ||
255 | after(function () { | 308 | after(function () { |
@@ -267,7 +320,8 @@ describe('Test videos redundancy', function () { | |||
267 | }) | 320 | }) |
268 | 321 | ||
269 | it('Should have 1 webseed on the first video', async function () { | 322 | it('Should have 1 webseed on the first video', async function () { |
270 | await check1WebSeed(strategy) | 323 | await check1WebSeed() |
324 | await check0PlaylistRedundancies() | ||
271 | await checkStatsWith1Webseed(strategy) | 325 | await checkStatsWith1Webseed(strategy) |
272 | }) | 326 | }) |
273 | 327 | ||
@@ -276,25 +330,27 @@ describe('Test videos redundancy', function () { | |||
276 | }) | 330 | }) |
277 | 331 | ||
278 | it('Should have 2 webseeds on the first video', async function () { | 332 | it('Should have 2 webseeds on the first video', async function () { |
279 | this.timeout(40000) | 333 | this.timeout(80000) |
280 | 334 | ||
281 | await waitJobs(servers) | 335 | await waitJobs(servers) |
282 | await waitUntilLog(servers[0], 'Duplicated ', 4) | 336 | await waitUntilLog(servers[0], 'Duplicated ', 5) |
283 | await waitJobs(servers) | 337 | await waitJobs(servers) |
284 | 338 | ||
285 | await check2Webseeds(strategy) | 339 | await check2Webseeds() |
340 | await check1PlaylistRedundancies() | ||
286 | await checkStatsWith2Webseed(strategy) | 341 | await checkStatsWith2Webseed(strategy) |
287 | }) | 342 | }) |
288 | 343 | ||
289 | it('Should unfollow on server 1 and remove duplicated videos', async function () { | 344 | it('Should unfollow on server 1 and remove duplicated videos', async function () { |
290 | this.timeout(40000) | 345 | this.timeout(80000) |
291 | 346 | ||
292 | await unfollow(servers[0].url, servers[0].accessToken, servers[1]) | 347 | await unfollow(servers[0].url, servers[0].accessToken, servers[1]) |
293 | 348 | ||
294 | await waitJobs(servers) | 349 | await waitJobs(servers) |
295 | await wait(5000) | 350 | await wait(5000) |
296 | 351 | ||
297 | await check1WebSeed(strategy) | 352 | await check1WebSeed() |
353 | await check0PlaylistRedundancies() | ||
298 | 354 | ||
299 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) | 355 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) |
300 | }) | 356 | }) |
@@ -314,7 +370,8 @@ describe('Test videos redundancy', function () { | |||
314 | }) | 370 | }) |
315 | 371 | ||
316 | it('Should have 1 webseed on the first video', async function () { | 372 | it('Should have 1 webseed on the first video', async function () { |
317 | await check1WebSeed(strategy) | 373 | await check1WebSeed() |
374 | await check0PlaylistRedundancies() | ||
318 | await checkStatsWith1Webseed(strategy) | 375 | await checkStatsWith1Webseed(strategy) |
319 | }) | 376 | }) |
320 | 377 | ||
@@ -323,18 +380,19 @@ describe('Test videos redundancy', function () { | |||
323 | }) | 380 | }) |
324 | 381 | ||
325 | it('Should still have 1 webseed on the first video', async function () { | 382 | it('Should still have 1 webseed on the first video', async function () { |
326 | this.timeout(40000) | 383 | this.timeout(80000) |
327 | 384 | ||
328 | await waitJobs(servers) | 385 | await waitJobs(servers) |
329 | await wait(15000) | 386 | await wait(15000) |
330 | await waitJobs(servers) | 387 | await waitJobs(servers) |
331 | 388 | ||
332 | await check1WebSeed(strategy) | 389 | await check1WebSeed() |
390 | await check0PlaylistRedundancies() | ||
333 | await checkStatsWith1Webseed(strategy) | 391 | await checkStatsWith1Webseed(strategy) |
334 | }) | 392 | }) |
335 | 393 | ||
336 | it('Should view 2 times the first video to have > min_views config', async function () { | 394 | it('Should view 2 times the first video to have > min_views config', async function () { |
337 | this.timeout(40000) | 395 | this.timeout(80000) |
338 | 396 | ||
339 | await viewVideo(servers[ 0 ].url, video1Server2UUID) | 397 | await viewVideo(servers[ 0 ].url, video1Server2UUID) |
340 | await viewVideo(servers[ 2 ].url, video1Server2UUID) | 398 | await viewVideo(servers[ 2 ].url, video1Server2UUID) |
@@ -344,13 +402,14 @@ describe('Test videos redundancy', function () { | |||
344 | }) | 402 | }) |
345 | 403 | ||
346 | it('Should have 2 webseeds on the first video', async function () { | 404 | it('Should have 2 webseeds on the first video', async function () { |
347 | this.timeout(40000) | 405 | this.timeout(80000) |
348 | 406 | ||
349 | await waitJobs(servers) | 407 | await waitJobs(servers) |
350 | await waitUntilLog(servers[0], 'Duplicated ', 4) | 408 | await waitUntilLog(servers[0], 'Duplicated ', 5) |
351 | await waitJobs(servers) | 409 | await waitJobs(servers) |
352 | 410 | ||
353 | await check2Webseeds(strategy) | 411 | await check2Webseeds() |
412 | await check1PlaylistRedundancies() | ||
354 | await checkStatsWith2Webseed(strategy) | 413 | await checkStatsWith2Webseed(strategy) |
355 | }) | 414 | }) |
356 | 415 | ||
@@ -405,7 +464,7 @@ describe('Test videos redundancy', function () { | |||
405 | }) | 464 | }) |
406 | 465 | ||
407 | it('Should still have 2 webseeds after 10 seconds', async function () { | 466 | it('Should still have 2 webseeds after 10 seconds', async function () { |
408 | this.timeout(40000) | 467 | this.timeout(80000) |
409 | 468 | ||
410 | await wait(10000) | 469 | await wait(10000) |
411 | 470 | ||
@@ -420,7 +479,7 @@ describe('Test videos redundancy', function () { | |||
420 | }) | 479 | }) |
421 | 480 | ||
422 | it('Should stop server 1 and expire video redundancy', async function () { | 481 | it('Should stop server 1 and expire video redundancy', async function () { |
423 | this.timeout(40000) | 482 | this.timeout(80000) |
424 | 483 | ||
425 | killallServers([ servers[0] ]) | 484 | killallServers([ servers[0] ]) |
426 | 485 | ||
@@ -446,10 +505,11 @@ describe('Test videos redundancy', function () { | |||
446 | await enableRedundancyOnServer1() | 505 | await enableRedundancyOnServer1() |
447 | 506 | ||
448 | await waitJobs(servers) | 507 | await waitJobs(servers) |
449 | await waitUntilLog(servers[0], 'Duplicated ', 4) | 508 | await waitUntilLog(servers[0], 'Duplicated ', 5) |
450 | await waitJobs(servers) | 509 | await waitJobs(servers) |
451 | 510 | ||
452 | await check2Webseeds(strategy) | 511 | await check2Webseeds() |
512 | await check1PlaylistRedundancies() | ||
453 | await checkStatsWith2Webseed(strategy) | 513 | await checkStatsWith2Webseed(strategy) |
454 | 514 | ||
455 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) | 515 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) |
@@ -467,8 +527,10 @@ describe('Test videos redundancy', function () { | |||
467 | await wait(1000) | 527 | await wait(1000) |
468 | 528 | ||
469 | try { | 529 | try { |
470 | await check1WebSeed(strategy, video1Server2UUID) | 530 | await check1WebSeed(video1Server2UUID) |
471 | await check2Webseeds(strategy, video2Server2UUID) | 531 | await check0PlaylistRedundancies(video1Server2UUID) |
532 | await check2Webseeds(video2Server2UUID) | ||
533 | await check1PlaylistRedundancies(video2Server2UUID) | ||
472 | 534 | ||
473 | checked = true | 535 | checked = true |
474 | } catch { | 536 | } catch { |
@@ -477,6 +539,26 @@ describe('Test videos redundancy', function () { | |||
477 | } | 539 | } |
478 | }) | 540 | }) |
479 | 541 | ||
542 | it('Should disable strategy and remove redundancies', async function () { | ||
543 | this.timeout(80000) | ||
544 | |||
545 | await waitJobs(servers) | ||
546 | |||
547 | killallServers([ servers[ 0 ] ]) | ||
548 | await reRunServer(servers[ 0 ], { | ||
549 | redundancy: { | ||
550 | videos: { | ||
551 | check_interval: '1 second', | ||
552 | strategies: [] | ||
553 | } | ||
554 | } | ||
555 | }) | ||
556 | |||
557 | await waitJobs(servers) | ||
558 | |||
559 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ]) | ||
560 | }) | ||
561 | |||
480 | after(function () { | 562 | after(function () { |
481 | return cleanServers() | 563 | return cleanServers() |
482 | }) | 564 | }) |
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index bebfc7398..0dfe6e4fe 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts | |||
@@ -57,6 +57,8 @@ function checkInitialConfig (data: CustomConfig) { | |||
57 | expect(data.transcoding.resolutions['480p']).to.be.true | 57 | expect(data.transcoding.resolutions['480p']).to.be.true |
58 | expect(data.transcoding.resolutions['720p']).to.be.true | 58 | expect(data.transcoding.resolutions['720p']).to.be.true |
59 | expect(data.transcoding.resolutions['1080p']).to.be.true | 59 | expect(data.transcoding.resolutions['1080p']).to.be.true |
60 | expect(data.transcoding.hls.enabled).to.be.true | ||
61 | |||
60 | expect(data.import.videos.http.enabled).to.be.true | 62 | expect(data.import.videos.http.enabled).to.be.true |
61 | expect(data.import.videos.torrent.enabled).to.be.true | 63 | expect(data.import.videos.torrent.enabled).to.be.true |
62 | } | 64 | } |
@@ -95,6 +97,7 @@ function checkUpdatedConfig (data: CustomConfig) { | |||
95 | expect(data.transcoding.resolutions['480p']).to.be.true | 97 | expect(data.transcoding.resolutions['480p']).to.be.true |
96 | expect(data.transcoding.resolutions['720p']).to.be.false | 98 | expect(data.transcoding.resolutions['720p']).to.be.false |
97 | expect(data.transcoding.resolutions['1080p']).to.be.false | 99 | expect(data.transcoding.resolutions['1080p']).to.be.false |
100 | expect(data.transcoding.hls.enabled).to.be.false | ||
98 | 101 | ||
99 | expect(data.import.videos.http.enabled).to.be.false | 102 | expect(data.import.videos.http.enabled).to.be.false |
100 | expect(data.import.videos.torrent.enabled).to.be.false | 103 | expect(data.import.videos.torrent.enabled).to.be.false |
@@ -205,6 +208,9 @@ describe('Test config', function () { | |||
205 | '480p': true, | 208 | '480p': true, |
206 | '720p': false, | 209 | '720p': false, |
207 | '1080p': false | 210 | '1080p': false |
211 | }, | ||
212 | hls: { | ||
213 | enabled: false | ||
208 | } | 214 | } |
209 | }, | 215 | }, |
210 | import: { | 216 | import: { |
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index 97f467aae..a501a80b2 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts | |||
@@ -8,6 +8,7 @@ import './video-change-ownership' | |||
8 | import './video-channels' | 8 | import './video-channels' |
9 | import './video-comments' | 9 | import './video-comments' |
10 | import './video-description' | 10 | import './video-description' |
11 | import './video-hls' | ||
11 | import './video-imports' | 12 | import './video-imports' |
12 | import './video-nsfw' | 13 | import './video-nsfw' |
13 | import './video-privacy' | 14 | import './video-privacy' |
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts new file mode 100644 index 000000000..71d863b12 --- /dev/null +++ b/server/tests/api/videos/video-hls.ts | |||
@@ -0,0 +1,145 @@ | |||
1 | /* tslint:disable:no-unused-expression */ | ||
2 | |||
3 | import * as chai from 'chai' | ||
4 | import 'mocha' | ||
5 | import { | ||
6 | checkDirectoryIsEmpty, | ||
7 | checkTmpIsEmpty, | ||
8 | doubleFollow, | ||
9 | flushAndRunMultipleServers, | ||
10 | flushTests, | ||
11 | getPlaylist, | ||
12 | getSegment, | ||
13 | getSegmentSha256, | ||
14 | getVideo, | ||
15 | killallServers, | ||
16 | removeVideo, | ||
17 | ServerInfo, | ||
18 | setAccessTokensToServers, | ||
19 | updateVideo, | ||
20 | uploadVideo, | ||
21 | waitJobs | ||
22 | } from '../../../../shared/utils' | ||
23 | import { VideoDetails } from '../../../../shared/models/videos' | ||
24 | import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type' | ||
25 | import { sha256 } from '../../../helpers/core-utils' | ||
26 | import { join } from 'path' | ||
27 | |||
28 | const expect = chai.expect | ||
29 | |||
30 | async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string) { | ||
31 | const resolutions = [ 240, 360, 480, 720 ] | ||
32 | |||
33 | for (const server of servers) { | ||
34 | const res = await getVideo(server.url, videoUUID) | ||
35 | const videoDetails: VideoDetails = res.body | ||
36 | |||
37 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) | ||
38 | |||
39 | const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) | ||
40 | expect(hlsPlaylist).to.not.be.undefined | ||
41 | |||
42 | { | ||
43 | const res2 = await getPlaylist(hlsPlaylist.playlistUrl) | ||
44 | |||
45 | const masterPlaylist = res2.text | ||
46 | |||
47 | expect(masterPlaylist).to.contain('#EXT-X-STREAM-INF:BANDWIDTH=55472,RESOLUTION=640x360,FRAME-RATE=25') | ||
48 | |||
49 | for (const resolution of resolutions) { | ||
50 | expect(masterPlaylist).to.contain(`${resolution}.m3u8`) | ||
51 | } | ||
52 | } | ||
53 | |||
54 | { | ||
55 | for (const resolution of resolutions) { | ||
56 | const res2 = await getPlaylist(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}.m3u8`) | ||
57 | |||
58 | const subPlaylist = res2.text | ||
59 | expect(subPlaylist).to.contain(resolution + '_000.ts') | ||
60 | } | ||
61 | } | ||
62 | |||
63 | { | ||
64 | for (const resolution of resolutions) { | ||
65 | |||
66 | const res2 = await getSegment(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}_000.ts`) | ||
67 | |||
68 | const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url) | ||
69 | |||
70 | const sha256Server = resSha.body[ resolution + '_000.ts' ] | ||
71 | expect(sha256(res2.body)).to.equal(sha256Server) | ||
72 | } | ||
73 | } | ||
74 | } | ||
75 | } | ||
76 | |||
77 | describe('Test HLS videos', function () { | ||
78 | let servers: ServerInfo[] = [] | ||
79 | let videoUUID = '' | ||
80 | |||
81 | before(async function () { | ||
82 | this.timeout(120000) | ||
83 | |||
84 | servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true, hls: { enabled: true } } }) | ||
85 | |||
86 | // Get the access tokens | ||
87 | await setAccessTokensToServers(servers) | ||
88 | |||
89 | // Server 1 and server 2 follow each other | ||
90 | await doubleFollow(servers[0], servers[1]) | ||
91 | }) | ||
92 | |||
93 | it('Should upload a video and transcode it to HLS', async function () { | ||
94 | this.timeout(120000) | ||
95 | |||
96 | { | ||
97 | const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1', fixture: 'video_short.webm' }) | ||
98 | videoUUID = res.body.video.uuid | ||
99 | } | ||
100 | |||
101 | await waitJobs(servers) | ||
102 | |||
103 | await checkHlsPlaylist(servers, videoUUID) | ||
104 | }) | ||
105 | |||
106 | it('Should update the video', async function () { | ||
107 | await updateVideo(servers[0].url, servers[0].accessToken, videoUUID, { name: 'video 1 updated' }) | ||
108 | |||
109 | await waitJobs(servers) | ||
110 | |||
111 | await checkHlsPlaylist(servers, videoUUID) | ||
112 | }) | ||
113 | |||
114 | it('Should delete the video', async function () { | ||
115 | await removeVideo(servers[0].url, servers[0].accessToken, videoUUID) | ||
116 | |||
117 | await waitJobs(servers) | ||
118 | |||
119 | for (const server of servers) { | ||
120 | await getVideo(server.url, videoUUID, 404) | ||
121 | } | ||
122 | }) | ||
123 | |||
124 | it('Should have the playlists/segment deleted from the disk', async function () { | ||
125 | for (const server of servers) { | ||
126 | await checkDirectoryIsEmpty(server, 'videos') | ||
127 | await checkDirectoryIsEmpty(server, join('playlists', 'hls')) | ||
128 | } | ||
129 | }) | ||
130 | |||
131 | it('Should have an empty tmp directory', async function () { | ||
132 | for (const server of servers) { | ||
133 | await checkTmpIsEmpty(server) | ||
134 | } | ||
135 | }) | ||
136 | |||
137 | after(async function () { | ||
138 | killallServers(servers) | ||
139 | |||
140 | // Keep the logs if the test failed | ||
141 | if (this['ok']) { | ||
142 | await flushTests() | ||
143 | } | ||
144 | }) | ||
145 | }) | ||
diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts index 811ea6a9f..d38bb4331 100644 --- a/server/tests/cli/update-host.ts +++ b/server/tests/cli/update-host.ts | |||
@@ -86,6 +86,13 @@ describe('Test update host scripts', function () { | |||
86 | const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid) | 86 | const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid) |
87 | 87 | ||
88 | expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid) | 88 | expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid) |
89 | |||
90 | const res = await getVideo(server.url, video.uuid) | ||
91 | const videoDetails: VideoDetails = res.body | ||
92 | |||
93 | expect(videoDetails.trackerUrls[0]).to.include(server.host) | ||
94 | expect(videoDetails.streamingPlaylists[0].playlistUrl).to.include(server.host) | ||
95 | expect(videoDetails.streamingPlaylists[0].segmentsSha256Url).to.include(server.host) | ||
89 | } | 96 | } |
90 | }) | 97 | }) |
91 | 98 | ||
@@ -100,7 +107,7 @@ describe('Test update host scripts', function () { | |||
100 | } | 107 | } |
101 | }) | 108 | }) |
102 | 109 | ||
103 | it('Should have update accounts url', async function () { | 110 | it('Should have updated accounts url', async function () { |
104 | const res = await getAccountsList(server.url) | 111 | const res = await getAccountsList(server.url) |
105 | expect(res.body.total).to.equal(3) | 112 | expect(res.body.total).to.equal(3) |
106 | 113 | ||
@@ -112,7 +119,7 @@ describe('Test update host scripts', function () { | |||
112 | } | 119 | } |
113 | }) | 120 | }) |
114 | 121 | ||
115 | it('Should update torrent hosts', async function () { | 122 | it('Should have updated torrent hosts', async function () { |
116 | this.timeout(30000) | 123 | this.timeout(30000) |
117 | 124 | ||
118 | const res = await getVideosList(server.url) | 125 | const res = await getVideosList(server.url) |