diff options
author | Chocobozzz <me@florianbigard.com> | 2020-09-25 16:19:35 +0200 |
---|---|---|
committer | Chocobozzz <chocobozzz@cpy.re> | 2020-11-09 15:33:04 +0100 |
commit | fb7194043d0486ce0a6a40b2ffbdf32878c33a6f (patch) | |
tree | 6ed304a5d730a75da0a4460b3009df88684fa598 /server | |
parent | a5cf76afa378aae81af2a9b0ce548e5d2582f832 (diff) | |
download | PeerTube-fb7194043d0486ce0a6a40b2ffbdf32878c33a6f.tar.gz PeerTube-fb7194043d0486ce0a6a40b2ffbdf32878c33a6f.tar.zst PeerTube-fb7194043d0486ce0a6a40b2ffbdf32878c33a6f.zip |
Check live duration and size
Diffstat (limited to 'server')
-rw-r--r-- | server/controllers/api/config.ts | 5 | ||||
-rw-r--r-- | server/controllers/api/users/me.ts | 6 | ||||
-rw-r--r-- | server/helpers/core-utils.ts | 1 | ||||
-rw-r--r-- | server/helpers/custom-validators/misc.ts | 5 | ||||
-rw-r--r-- | server/helpers/ffmpeg-utils.ts | 22 | ||||
-rw-r--r-- | server/initializers/checker-after-init.ts | 7 | ||||
-rw-r--r-- | server/initializers/checker-before-init.ts | 7 | ||||
-rw-r--r-- | server/initializers/config.ts | 3 | ||||
-rw-r--r-- | server/initializers/constants.ts | 7 | ||||
-rw-r--r-- | server/initializers/migrations/0535-video-live.ts | 2 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/video-import.ts | 3 | ||||
-rw-r--r-- | server/lib/live-manager.ts | 89 | ||||
-rw-r--r-- | server/lib/user.ts | 72 | ||||
-rw-r--r-- | server/middlewares/validators/config.ts | 40 | ||||
-rw-r--r-- | server/middlewares/validators/users.ts | 2 | ||||
-rw-r--r-- | server/middlewares/validators/videos/videos.ts | 5 | ||||
-rw-r--r-- | server/models/account/user.ts | 159 | ||||
-rw-r--r-- | server/models/video/video-live.ts | 10 | ||||
-rw-r--r-- | server/tests/api/check-params/config.ts | 3 | ||||
-rw-r--r-- | server/tests/api/server/config.ts | 6 |
20 files changed, 327 insertions, 127 deletions
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index bd100ef9c..99aabba62 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts | |||
@@ -118,6 +118,9 @@ async function getConfig (req: express.Request, res: express.Response) { | |||
118 | live: { | 118 | live: { |
119 | enabled: CONFIG.LIVE.ENABLED, | 119 | enabled: CONFIG.LIVE.ENABLED, |
120 | 120 | ||
121 | allowReplay: CONFIG.LIVE.ALLOW_REPLAY, | ||
122 | maxDuration: CONFIG.LIVE.MAX_DURATION, | ||
123 | |||
121 | transcoding: { | 124 | transcoding: { |
122 | enabled: CONFIG.LIVE.TRANSCODING.ENABLED, | 125 | enabled: CONFIG.LIVE.TRANSCODING.ENABLED, |
123 | enabledResolutions: getEnabledResolutions('live') | 126 | enabledResolutions: getEnabledResolutions('live') |
@@ -425,6 +428,8 @@ function customConfig (): CustomConfig { | |||
425 | }, | 428 | }, |
426 | live: { | 429 | live: { |
427 | enabled: CONFIG.LIVE.ENABLED, | 430 | enabled: CONFIG.LIVE.ENABLED, |
431 | allowReplay: CONFIG.LIVE.ALLOW_REPLAY, | ||
432 | maxDuration: CONFIG.LIVE.MAX_DURATION, | ||
428 | transcoding: { | 433 | transcoding: { |
429 | enabled: CONFIG.LIVE.TRANSCODING.ENABLED, | 434 | enabled: CONFIG.LIVE.TRANSCODING.ENABLED, |
430 | threads: CONFIG.LIVE.TRANSCODING.THREADS, | 435 | threads: CONFIG.LIVE.TRANSCODING.THREADS, |
diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index ba60a3d2a..b490518fc 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts | |||
@@ -9,7 +9,7 @@ import { MIMETYPES } from '../../../initializers/constants' | |||
9 | import { sequelizeTypescript } from '../../../initializers/database' | 9 | import { sequelizeTypescript } from '../../../initializers/database' |
10 | import { sendUpdateActor } from '../../../lib/activitypub/send' | 10 | import { sendUpdateActor } from '../../../lib/activitypub/send' |
11 | import { updateActorAvatarFile } from '../../../lib/avatar' | 11 | import { updateActorAvatarFile } from '../../../lib/avatar' |
12 | import { sendVerifyUserEmail } from '../../../lib/user' | 12 | import { getOriginalVideoFileTotalDailyFromUser, getOriginalVideoFileTotalFromUser, sendVerifyUserEmail } from '../../../lib/user' |
13 | import { | 13 | import { |
14 | asyncMiddleware, | 14 | asyncMiddleware, |
15 | asyncRetryTransactionMiddleware, | 15 | asyncRetryTransactionMiddleware, |
@@ -133,8 +133,8 @@ async function getUserInformation (req: express.Request, res: express.Response) | |||
133 | 133 | ||
134 | async function getUserVideoQuotaUsed (req: express.Request, res: express.Response) { | 134 | async function getUserVideoQuotaUsed (req: express.Request, res: express.Response) { |
135 | const user = res.locals.oauth.token.user | 135 | const user = res.locals.oauth.token.user |
136 | const videoQuotaUsed = await UserModel.getOriginalVideoFileTotalFromUser(user) | 136 | const videoQuotaUsed = await getOriginalVideoFileTotalFromUser(user) |
137 | const videoQuotaUsedDaily = await UserModel.getOriginalVideoFileTotalDailyFromUser(user) | 137 | const videoQuotaUsedDaily = await getOriginalVideoFileTotalDailyFromUser(user) |
138 | 138 | ||
139 | const data: UserVideoQuota = { | 139 | const data: UserVideoQuota = { |
140 | videoQuotaUsed, | 140 | videoQuotaUsed, |
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 49eee7c59..e1c15a6eb 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts | |||
@@ -41,6 +41,7 @@ const timeTable = { | |||
41 | } | 41 | } |
42 | 42 | ||
43 | export function parseDurationToMs (duration: number | string): number { | 43 | export function parseDurationToMs (duration: number | string): number { |
44 | if (duration === null) return null | ||
44 | if (typeof duration === 'number') return duration | 45 | if (typeof duration === 'number') return duration |
45 | 46 | ||
46 | if (typeof duration === 'string') { | 47 | if (typeof duration === 'string') { |
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index cf32201c4..61c03f0c9 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts | |||
@@ -45,6 +45,10 @@ function isBooleanValid (value: any) { | |||
45 | return typeof value === 'boolean' || (typeof value === 'string' && validator.isBoolean(value)) | 45 | return typeof value === 'boolean' || (typeof value === 'string' && validator.isBoolean(value)) |
46 | } | 46 | } |
47 | 47 | ||
48 | function isIntOrNull (value: any) { | ||
49 | return value === null || validator.isInt('' + value) | ||
50 | } | ||
51 | |||
48 | function toIntOrNull (value: string) { | 52 | function toIntOrNull (value: string) { |
49 | const v = toValueOrNull(value) | 53 | const v = toValueOrNull(value) |
50 | 54 | ||
@@ -116,6 +120,7 @@ export { | |||
116 | isArrayOf, | 120 | isArrayOf, |
117 | isNotEmptyIntArray, | 121 | isNotEmptyIntArray, |
118 | isArray, | 122 | isArray, |
123 | isIntOrNull, | ||
119 | isIdValid, | 124 | isIdValid, |
120 | isSafePath, | 125 | isSafePath, |
121 | isUUIDValid, | 126 | isUUIDValid, |
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index fac2595f1..b25dcaa90 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts | |||
@@ -5,7 +5,7 @@ import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata' | |||
5 | import { getMaxBitrate, getTargetBitrate, VideoResolution } from '../../shared/models/videos' | 5 | import { getMaxBitrate, getTargetBitrate, VideoResolution } from '../../shared/models/videos' |
6 | import { checkFFmpegEncoders } from '../initializers/checker-before-init' | 6 | import { checkFFmpegEncoders } from '../initializers/checker-before-init' |
7 | import { CONFIG } from '../initializers/config' | 7 | import { CONFIG } from '../initializers/config' |
8 | import { FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' | 8 | import { FFMPEG_NICE, VIDEO_LIVE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' |
9 | import { processImage } from './image-utils' | 9 | import { processImage } from './image-utils' |
10 | import { logger } from './logger' | 10 | import { logger } from './logger' |
11 | 11 | ||
@@ -353,7 +353,7 @@ function convertWebPToJPG (path: string, destination: string): Promise<void> { | |||
353 | }) | 353 | }) |
354 | } | 354 | } |
355 | 355 | ||
356 | function runLiveTranscoding (rtmpUrl: string, outPath: string, resolutions: number[]) { | 356 | function runLiveTranscoding (rtmpUrl: string, outPath: string, resolutions: number[], deleteSegments: boolean) { |
357 | const command = getFFmpeg(rtmpUrl) | 357 | const command = getFFmpeg(rtmpUrl) |
358 | command.inputOption('-fflags nobuffer') | 358 | command.inputOption('-fflags nobuffer') |
359 | 359 | ||
@@ -399,7 +399,7 @@ function runLiveTranscoding (rtmpUrl: string, outPath: string, resolutions: numb | |||
399 | varStreamMap.push(`v:${i},a:${i}`) | 399 | varStreamMap.push(`v:${i},a:${i}`) |
400 | } | 400 | } |
401 | 401 | ||
402 | addDefaultLiveHLSParams(command, outPath) | 402 | addDefaultLiveHLSParams(command, outPath, deleteSegments) |
403 | 403 | ||
404 | command.outputOption('-var_stream_map', varStreamMap.join(' ')) | 404 | command.outputOption('-var_stream_map', varStreamMap.join(' ')) |
405 | 405 | ||
@@ -408,7 +408,7 @@ function runLiveTranscoding (rtmpUrl: string, outPath: string, resolutions: numb | |||
408 | return command | 408 | return command |
409 | } | 409 | } |
410 | 410 | ||
411 | function runLiveMuxing (rtmpUrl: string, outPath: string) { | 411 | function runLiveMuxing (rtmpUrl: string, outPath: string, deleteSegments: boolean) { |
412 | const command = getFFmpeg(rtmpUrl) | 412 | const command = getFFmpeg(rtmpUrl) |
413 | command.inputOption('-fflags nobuffer') | 413 | command.inputOption('-fflags nobuffer') |
414 | 414 | ||
@@ -417,7 +417,7 @@ function runLiveMuxing (rtmpUrl: string, outPath: string) { | |||
417 | command.outputOption('-map 0:a?') | 417 | command.outputOption('-map 0:a?') |
418 | command.outputOption('-map 0:v?') | 418 | command.outputOption('-map 0:v?') |
419 | 419 | ||
420 | addDefaultLiveHLSParams(command, outPath) | 420 | addDefaultLiveHLSParams(command, outPath, deleteSegments) |
421 | 421 | ||
422 | command.run() | 422 | command.run() |
423 | 423 | ||
@@ -457,10 +457,14 @@ function addDefaultX264Params (command: ffmpeg.FfmpegCommand) { | |||
457 | .outputOption('-map_metadata -1') // strip all metadata | 457 | .outputOption('-map_metadata -1') // strip all metadata |
458 | } | 458 | } |
459 | 459 | ||
460 | function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string) { | 460 | function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string, deleteSegments: boolean) { |
461 | command.outputOption('-hls_time 4') | 461 | command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME) |
462 | command.outputOption('-hls_list_size 15') | 462 | command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE) |
463 | command.outputOption('-hls_flags delete_segments') | 463 | |
464 | if (deleteSegments === true) { | ||
465 | command.outputOption('-hls_flags delete_segments') | ||
466 | } | ||
467 | |||
464 | command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%d.ts')}`) | 468 | command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%d.ts')}`) |
465 | command.outputOption('-master_pl_name master.m3u8') | 469 | command.outputOption('-master_pl_name master.m3u8') |
466 | command.outputOption(`-f hls`) | 470 | command.outputOption(`-f hls`) |
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index b49ab6bca..979c97a8b 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts | |||
@@ -135,6 +135,13 @@ function checkConfig () { | |||
135 | } | 135 | } |
136 | } | 136 | } |
137 | 137 | ||
138 | // Live | ||
139 | if (CONFIG.LIVE.ENABLED === true) { | ||
140 | if (CONFIG.LIVE.ALLOW_REPLAY === true && CONFIG.TRANSCODING.ENABLED === false) { | ||
141 | return 'Live allow replay cannot be enabled if transcoding is not enabled.' | ||
142 | } | ||
143 | } | ||
144 | |||
138 | return null | 145 | return null |
139 | } | 146 | } |
140 | 147 | ||
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index e0819c4aa..d4140e3fa 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts | |||
@@ -37,8 +37,13 @@ function checkMissedConfig () { | |||
37 | 'remote_redundancy.videos.accept_from', | 37 | 'remote_redundancy.videos.accept_from', |
38 | 'federation.videos.federate_unlisted', | 38 | 'federation.videos.federate_unlisted', |
39 | 'search.remote_uri.users', 'search.remote_uri.anonymous', 'search.search_index.enabled', 'search.search_index.url', | 39 | 'search.remote_uri.users', 'search.remote_uri.anonymous', 'search.search_index.enabled', 'search.search_index.url', |
40 | 'search.search_index.disable_local_search', 'search.search_index.is_default_search' | 40 | 'search.search_index.disable_local_search', 'search.search_index.is_default_search', |
41 | 'live.enabled', 'live.allow_replay', 'live.max_duration', | ||
42 | 'live.transcoding.enabled', 'live.transcoding.threads', | ||
43 | 'live.transcoding.resolutions.240p', 'live.transcoding.resolutions.360p', 'live.transcoding.resolutions.480p', | ||
44 | 'live.transcoding.resolutions.720p', 'live.transcoding.resolutions.1080p', 'live.transcoding.resolutions.2160p' | ||
41 | ] | 45 | ] |
46 | |||
42 | const requiredAlternatives = [ | 47 | const requiredAlternatives = [ |
43 | [ // set | 48 | [ // set |
44 | [ 'redis.hostname', 'redis.port' ], // alternative | 49 | [ 'redis.hostname', 'redis.port' ], // alternative |
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 7a8200ed9..9e8927350 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -201,6 +201,9 @@ const CONFIG = { | |||
201 | LIVE: { | 201 | LIVE: { |
202 | get ENABLED () { return config.get<boolean>('live.enabled') }, | 202 | get ENABLED () { return config.get<boolean>('live.enabled') }, |
203 | 203 | ||
204 | get MAX_DURATION () { return parseDurationToMs(config.get<string>('live.max_duration')) }, | ||
205 | get ALLOW_REPLAY () { return config.get<boolean>('live.allow_replay') }, | ||
206 | |||
204 | RTMP: { | 207 | RTMP: { |
205 | get PORT () { return config.get<number>('live.rtmp.port') } | 208 | get PORT () { return config.get<number>('live.rtmp.port') } |
206 | }, | 209 | }, |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 82d04a94e..065012b32 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -608,7 +608,9 @@ const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') | |||
608 | 608 | ||
609 | const VIDEO_LIVE = { | 609 | const VIDEO_LIVE = { |
610 | EXTENSION: '.ts', | 610 | EXTENSION: '.ts', |
611 | CLEANUP_DELAY: 1000 * 60 * 5, // 5 mintues | 611 | CLEANUP_DELAY: 1000 * 60 * 5, // 5 minutes |
612 | SEGMENT_TIME: 4, // 4 seconds | ||
613 | SEGMENTS_LIST_SIZE: 15, // 15 maximum segments in live playlist | ||
612 | RTMP: { | 614 | RTMP: { |
613 | CHUNK_SIZE: 60000, | 615 | CHUNK_SIZE: 60000, |
614 | GOP_CACHE: true, | 616 | GOP_CACHE: true, |
@@ -620,7 +622,8 @@ const VIDEO_LIVE = { | |||
620 | 622 | ||
621 | const MEMOIZE_TTL = { | 623 | const MEMOIZE_TTL = { |
622 | OVERVIEWS_SAMPLE: 1000 * 3600 * 4, // 4 hours | 624 | OVERVIEWS_SAMPLE: 1000 * 3600 * 4, // 4 hours |
623 | INFO_HASH_EXISTS: 1000 * 3600 * 12 // 12 hours | 625 | INFO_HASH_EXISTS: 1000 * 3600 * 12, // 12 hours |
626 | LIVE_ABLE_TO_UPLOAD: 1000 * 60 // 1 minute | ||
624 | } | 627 | } |
625 | 628 | ||
626 | const MEMOIZE_LENGTH = { | 629 | const MEMOIZE_LENGTH = { |
diff --git a/server/initializers/migrations/0535-video-live.ts b/server/initializers/migrations/0535-video-live.ts index 35523efc4..7501e080b 100644 --- a/server/initializers/migrations/0535-video-live.ts +++ b/server/initializers/migrations/0535-video-live.ts | |||
@@ -9,7 +9,7 @@ async function up (utils: { | |||
9 | const query = ` | 9 | const query = ` |
10 | CREATE TABLE IF NOT EXISTS "videoLive" ( | 10 | CREATE TABLE IF NOT EXISTS "videoLive" ( |
11 | "id" SERIAL , | 11 | "id" SERIAL , |
12 | "streamKey" VARCHAR(255) NOT NULL, | 12 | "streamKey" VARCHAR(255), |
13 | "videoId" INTEGER NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, | 13 | "videoId" INTEGER NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, |
14 | "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, | 14 | "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, |
15 | "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL, | 15 | "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL, |
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 9b5f2bb2b..9210aec54 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -4,6 +4,7 @@ import { extname } from 'path' | |||
4 | import { addOptimizeOrMergeAudioJob } from '@server/helpers/video' | 4 | import { addOptimizeOrMergeAudioJob } from '@server/helpers/video' |
5 | import { isPostImportVideoAccepted } from '@server/lib/moderation' | 5 | import { isPostImportVideoAccepted } from '@server/lib/moderation' |
6 | import { Hooks } from '@server/lib/plugins/hooks' | 6 | import { Hooks } from '@server/lib/plugins/hooks' |
7 | import { isAbleToUploadVideo } from '@server/lib/user' | ||
7 | import { getVideoFilePath } from '@server/lib/video-paths' | 8 | import { getVideoFilePath } from '@server/lib/video-paths' |
8 | import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' | 9 | import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' |
9 | import { | 10 | import { |
@@ -108,7 +109,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
108 | 109 | ||
109 | // Get information about this video | 110 | // Get information about this video |
110 | const stats = await stat(tempVideoPath) | 111 | const stats = await stat(tempVideoPath) |
111 | const isAble = await videoImport.User.isAbleToUploadVideo({ size: stats.size }) | 112 | const isAble = await isAbleToUploadVideo(videoImport.User.id, stats.size) |
112 | if (isAble === false) { | 113 | if (isAble === false) { |
113 | throw new Error('The user video quota is exceeded with this video to import.') | 114 | throw new Error('The user video quota is exceeded with this video to import.') |
114 | } | 115 | } |
diff --git a/server/lib/live-manager.ts b/server/lib/live-manager.ts index 41176d197..3ff2434ff 100644 --- a/server/lib/live-manager.ts +++ b/server/lib/live-manager.ts | |||
@@ -2,24 +2,27 @@ | |||
2 | import { AsyncQueue, queue } from 'async' | 2 | import { AsyncQueue, queue } from 'async' |
3 | import * as chokidar from 'chokidar' | 3 | import * as chokidar from 'chokidar' |
4 | import { FfmpegCommand } from 'fluent-ffmpeg' | 4 | import { FfmpegCommand } from 'fluent-ffmpeg' |
5 | import { ensureDir } from 'fs-extra' | 5 | import { ensureDir, stat } from 'fs-extra' |
6 | import { basename } from 'path' | 6 | import { basename } from 'path' |
7 | import { computeResolutionsToTranscode, runLiveMuxing, runLiveTranscoding } from '@server/helpers/ffmpeg-utils' | 7 | import { computeResolutionsToTranscode, runLiveMuxing, runLiveTranscoding } from '@server/helpers/ffmpeg-utils' |
8 | import { logger } from '@server/helpers/logger' | 8 | import { logger } from '@server/helpers/logger' |
9 | import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' | 9 | import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' |
10 | import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, WEBSERVER } from '@server/initializers/constants' | 10 | import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, WEBSERVER } from '@server/initializers/constants' |
11 | import { UserModel } from '@server/models/account/user' | ||
11 | import { VideoModel } from '@server/models/video/video' | 12 | import { VideoModel } from '@server/models/video/video' |
12 | import { VideoFileModel } from '@server/models/video/video-file' | 13 | import { VideoFileModel } from '@server/models/video/video-file' |
13 | import { VideoLiveModel } from '@server/models/video/video-live' | 14 | import { VideoLiveModel } from '@server/models/video/video-live' |
14 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | 15 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' |
15 | import { MStreamingPlaylist, MVideoLiveVideo } from '@server/types/models' | 16 | import { MStreamingPlaylist, MUser, MUserId, MVideoLive, MVideoLiveVideo } from '@server/types/models' |
16 | import { VideoState, VideoStreamingPlaylistType } from '@shared/models' | 17 | import { VideoState, VideoStreamingPlaylistType } from '@shared/models' |
17 | import { federateVideoIfNeeded } from './activitypub/videos' | 18 | import { federateVideoIfNeeded } from './activitypub/videos' |
18 | import { buildSha256Segment } from './hls' | 19 | import { buildSha256Segment } from './hls' |
19 | import { JobQueue } from './job-queue' | 20 | import { JobQueue } from './job-queue' |
20 | import { PeerTubeSocket } from './peertube-socket' | 21 | import { PeerTubeSocket } from './peertube-socket' |
22 | import { isAbleToUploadVideo } from './user' | ||
21 | import { getHLSDirectory } from './video-paths' | 23 | import { getHLSDirectory } from './video-paths' |
22 | 24 | ||
25 | import memoizee = require('memoizee') | ||
23 | const NodeRtmpServer = require('node-media-server/node_rtmp_server') | 26 | const NodeRtmpServer = require('node-media-server/node_rtmp_server') |
24 | const context = require('node-media-server/node_core_ctx') | 27 | const context = require('node-media-server/node_core_ctx') |
25 | const nodeMediaServerLogger = require('node-media-server/node_core_logger') | 28 | const nodeMediaServerLogger = require('node-media-server/node_core_logger') |
@@ -53,6 +56,11 @@ class LiveManager { | |||
53 | private readonly transSessions = new Map<string, FfmpegCommand>() | 56 | private readonly transSessions = new Map<string, FfmpegCommand>() |
54 | private readonly videoSessions = new Map<number, string>() | 57 | private readonly videoSessions = new Map<number, string>() |
55 | private readonly segmentsSha256 = new Map<string, Map<string, string>>() | 58 | private readonly segmentsSha256 = new Map<string, Map<string, string>>() |
59 | private readonly livesPerUser = new Map<number, { liveId: number, videoId: number, size: number }[]>() | ||
60 | |||
61 | private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => { | ||
62 | return isAbleToUploadVideo(userId, 1000) | ||
63 | }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD }) | ||
56 | 64 | ||
57 | private segmentsSha256Queue: AsyncQueue<SegmentSha256QueueParam> | 65 | private segmentsSha256Queue: AsyncQueue<SegmentSha256QueueParam> |
58 | private rtmpServer: any | 66 | private rtmpServer: any |
@@ -127,7 +135,7 @@ class LiveManager { | |||
127 | 135 | ||
128 | this.abortSession(sessionId) | 136 | this.abortSession(sessionId) |
129 | 137 | ||
130 | this.onEndTransmuxing(videoId) | 138 | this.onEndTransmuxing(videoId, true) |
131 | .catch(err => logger.error('Cannot end transmuxing of video %d.', videoId, { err })) | 139 | .catch(err => logger.error('Cannot end transmuxing of video %d.', videoId, { err })) |
132 | } | 140 | } |
133 | 141 | ||
@@ -196,8 +204,18 @@ class LiveManager { | |||
196 | originalResolution: number | 204 | originalResolution: number |
197 | }) { | 205 | }) { |
198 | const { sessionId, videoLive, playlist, streamPath, resolutionsEnabled, originalResolution } = options | 206 | const { sessionId, videoLive, playlist, streamPath, resolutionsEnabled, originalResolution } = options |
207 | const startStreamDateTime = new Date().getTime() | ||
199 | const allResolutions = resolutionsEnabled.concat([ originalResolution ]) | 208 | const allResolutions = resolutionsEnabled.concat([ originalResolution ]) |
200 | 209 | ||
210 | const user = await UserModel.loadByLiveId(videoLive.id) | ||
211 | if (!this.livesPerUser.has(user.id)) { | ||
212 | this.livesPerUser.set(user.id, []) | ||
213 | } | ||
214 | |||
215 | const currentUserLive = { liveId: videoLive.id, videoId: videoLive.videoId, size: 0 } | ||
216 | const livesOfUser = this.livesPerUser.get(user.id) | ||
217 | livesOfUser.push(currentUserLive) | ||
218 | |||
201 | for (let i = 0; i < allResolutions.length; i++) { | 219 | for (let i = 0; i < allResolutions.length; i++) { |
202 | const resolution = allResolutions[i] | 220 | const resolution = allResolutions[i] |
203 | 221 | ||
@@ -216,26 +234,47 @@ class LiveManager { | |||
216 | const outPath = getHLSDirectory(videoLive.Video) | 234 | const outPath = getHLSDirectory(videoLive.Video) |
217 | await ensureDir(outPath) | 235 | await ensureDir(outPath) |
218 | 236 | ||
237 | const deleteSegments = videoLive.saveReplay === false | ||
238 | |||
219 | const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath | 239 | const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath |
220 | const ffmpegExec = CONFIG.LIVE.TRANSCODING.ENABLED | 240 | const ffmpegExec = CONFIG.LIVE.TRANSCODING.ENABLED |
221 | ? runLiveTranscoding(rtmpUrl, outPath, allResolutions) | 241 | ? runLiveTranscoding(rtmpUrl, outPath, allResolutions, deleteSegments) |
222 | : runLiveMuxing(rtmpUrl, outPath) | 242 | : runLiveMuxing(rtmpUrl, outPath, deleteSegments) |
223 | 243 | ||
224 | logger.info('Running live muxing/transcoding.') | 244 | logger.info('Running live muxing/transcoding.') |
225 | |||
226 | this.transSessions.set(sessionId, ffmpegExec) | 245 | this.transSessions.set(sessionId, ffmpegExec) |
227 | 246 | ||
228 | const videoUUID = videoLive.Video.uuid | 247 | const videoUUID = videoLive.Video.uuid |
229 | const tsWatcher = chokidar.watch(outPath + '/*.ts') | 248 | const tsWatcher = chokidar.watch(outPath + '/*.ts') |
230 | 249 | ||
231 | const updateHandler = segmentPath => { | 250 | const updateSegment = segmentPath => this.segmentsSha256Queue.push({ operation: 'update', segmentPath, videoUUID }) |
232 | this.segmentsSha256Queue.push({ operation: 'update', segmentPath, videoUUID }) | 251 | |
252 | const addHandler = segmentPath => { | ||
253 | updateSegment(segmentPath) | ||
254 | |||
255 | if (this.isDurationConstraintValid(startStreamDateTime) !== true) { | ||
256 | this.stopSessionOf(videoLive.videoId) | ||
257 | } | ||
258 | |||
259 | if (videoLive.saveReplay === true) { | ||
260 | stat(segmentPath) | ||
261 | .then(segmentStat => { | ||
262 | currentUserLive.size += segmentStat.size | ||
263 | }) | ||
264 | .then(() => this.isQuotaConstraintValid(user, videoLive)) | ||
265 | .then(quotaValid => { | ||
266 | if (quotaValid !== true) { | ||
267 | this.stopSessionOf(videoLive.videoId) | ||
268 | } | ||
269 | }) | ||
270 | .catch(err => logger.error('Cannot stat %s or check quota of %d.', segmentPath, user.id, { err })) | ||
271 | } | ||
233 | } | 272 | } |
234 | 273 | ||
235 | const deleteHandler = segmentPath => this.segmentsSha256Queue.push({ operation: 'delete', segmentPath, videoUUID }) | 274 | const deleteHandler = segmentPath => this.segmentsSha256Queue.push({ operation: 'delete', segmentPath, videoUUID }) |
236 | 275 | ||
237 | tsWatcher.on('add', p => updateHandler(p)) | 276 | tsWatcher.on('add', p => addHandler(p)) |
238 | tsWatcher.on('change', p => updateHandler(p)) | 277 | tsWatcher.on('change', p => updateSegment(p)) |
239 | tsWatcher.on('unlink', p => deleteHandler(p)) | 278 | tsWatcher.on('unlink', p => deleteHandler(p)) |
240 | 279 | ||
241 | const masterWatcher = chokidar.watch(outPath + '/master.m3u8') | 280 | const masterWatcher = chokidar.watch(outPath + '/master.m3u8') |
@@ -280,7 +319,14 @@ class LiveManager { | |||
280 | ffmpegExec.on('end', () => onFFmpegEnded()) | 319 | ffmpegExec.on('end', () => onFFmpegEnded()) |
281 | } | 320 | } |
282 | 321 | ||
283 | private async onEndTransmuxing (videoId: number) { | 322 | getLiveQuotaUsedByUser (userId: number) { |
323 | const currentLives = this.livesPerUser.get(userId) | ||
324 | if (!currentLives) return 0 | ||
325 | |||
326 | return currentLives.reduce((sum, obj) => sum + obj.size, 0) | ||
327 | } | ||
328 | |||
329 | private async onEndTransmuxing (videoId: number, cleanupNow = false) { | ||
284 | try { | 330 | try { |
285 | const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | 331 | const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) |
286 | if (!fullVideo) return | 332 | if (!fullVideo) return |
@@ -290,7 +336,7 @@ class LiveManager { | |||
290 | payload: { | 336 | payload: { |
291 | videoId: fullVideo.id | 337 | videoId: fullVideo.id |
292 | } | 338 | } |
293 | }, { delay: VIDEO_LIVE.CLEANUP_DELAY }) | 339 | }, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY }) |
294 | 340 | ||
295 | // FIXME: use end | 341 | // FIXME: use end |
296 | fullVideo.state = VideoState.WAITING_FOR_LIVE | 342 | fullVideo.state = VideoState.WAITING_FOR_LIVE |
@@ -337,6 +383,23 @@ class LiveManager { | |||
337 | filesMap.delete(segmentName) | 383 | filesMap.delete(segmentName) |
338 | } | 384 | } |
339 | 385 | ||
386 | private isDurationConstraintValid (streamingStartTime: number) { | ||
387 | const maxDuration = CONFIG.LIVE.MAX_DURATION | ||
388 | // No limit | ||
389 | if (maxDuration === null) return true | ||
390 | |||
391 | const now = new Date().getTime() | ||
392 | const max = streamingStartTime + maxDuration | ||
393 | |||
394 | return now <= max | ||
395 | } | ||
396 | |||
397 | private async isQuotaConstraintValid (user: MUserId, live: MVideoLive) { | ||
398 | if (live.saveReplay !== true) return true | ||
399 | |||
400 | return this.isAbleToUploadVideoWithCache(user.id) | ||
401 | } | ||
402 | |||
340 | static get Instance () { | 403 | static get Instance () { |
341 | return this.instance || (this.instance = new this()) | 404 | return this.instance || (this.instance = new this()) |
342 | } | 405 | } |
diff --git a/server/lib/user.ts b/server/lib/user.ts index aa14f0b54..d3338f329 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts | |||
@@ -1,20 +1,24 @@ | |||
1 | import { Transaction } from 'sequelize/types' | ||
1 | import { v4 as uuidv4 } from 'uuid' | 2 | import { v4 as uuidv4 } from 'uuid' |
3 | import { UserModel } from '@server/models/account/user' | ||
2 | import { ActivityPubActorType } from '../../shared/models/activitypub' | 4 | import { ActivityPubActorType } from '../../shared/models/activitypub' |
5 | import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users' | ||
3 | import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants' | 6 | import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants' |
7 | import { sequelizeTypescript } from '../initializers/database' | ||
4 | import { AccountModel } from '../models/account/account' | 8 | import { AccountModel } from '../models/account/account' |
5 | import { buildActorInstance, setAsyncActorKeys } from './activitypub/actor' | ||
6 | import { createLocalVideoChannel } from './video-channel' | ||
7 | import { ActorModel } from '../models/activitypub/actor' | ||
8 | import { UserNotificationSettingModel } from '../models/account/user-notification-setting' | 9 | import { UserNotificationSettingModel } from '../models/account/user-notification-setting' |
9 | import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users' | 10 | import { ActorModel } from '../models/activitypub/actor' |
10 | import { createWatchLaterPlaylist } from './video-playlist' | ||
11 | import { sequelizeTypescript } from '../initializers/database' | ||
12 | import { Transaction } from 'sequelize/types' | ||
13 | import { Redis } from './redis' | ||
14 | import { Emailer } from './emailer' | ||
15 | import { MAccountDefault, MActorDefault, MChannelActor } from '../types/models' | 11 | import { MAccountDefault, MActorDefault, MChannelActor } from '../types/models' |
16 | import { MUser, MUserDefault, MUserId } from '../types/models/user' | 12 | import { MUser, MUserDefault, MUserId } from '../types/models/user' |
13 | import { buildActorInstance, setAsyncActorKeys } from './activitypub/actor' | ||
17 | import { getAccountActivityPubUrl } from './activitypub/url' | 14 | import { getAccountActivityPubUrl } from './activitypub/url' |
15 | import { Emailer } from './emailer' | ||
16 | import { LiveManager } from './live-manager' | ||
17 | import { Redis } from './redis' | ||
18 | import { createLocalVideoChannel } from './video-channel' | ||
19 | import { createWatchLaterPlaylist } from './video-playlist' | ||
20 | |||
21 | import memoizee = require('memoizee') | ||
18 | 22 | ||
19 | type ChannelNames = { name: string, displayName: string } | 23 | type ChannelNames = { name: string, displayName: string } |
20 | 24 | ||
@@ -116,13 +120,61 @@ async function sendVerifyUserEmail (user: MUser, isPendingEmail = false) { | |||
116 | await Emailer.Instance.addVerifyEmailJob(username, email, url) | 120 | await Emailer.Instance.addVerifyEmailJob(username, email, url) |
117 | } | 121 | } |
118 | 122 | ||
123 | async function getOriginalVideoFileTotalFromUser (user: MUserId) { | ||
124 | // Don't use sequelize because we need to use a sub query | ||
125 | const query = UserModel.generateUserQuotaBaseSQL({ | ||
126 | withSelect: true, | ||
127 | whereUserId: '$userId' | ||
128 | }) | ||
129 | |||
130 | const base = await UserModel.getTotalRawQuery(query, user.id) | ||
131 | |||
132 | return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id) | ||
133 | } | ||
134 | |||
135 | // Returns cumulative size of all video files uploaded in the last 24 hours. | ||
136 | async function getOriginalVideoFileTotalDailyFromUser (user: MUserId) { | ||
137 | // Don't use sequelize because we need to use a sub query | ||
138 | const query = UserModel.generateUserQuotaBaseSQL({ | ||
139 | withSelect: true, | ||
140 | whereUserId: '$userId', | ||
141 | where: '"video"."createdAt" > now() - interval \'24 hours\'' | ||
142 | }) | ||
143 | |||
144 | const base = await UserModel.getTotalRawQuery(query, user.id) | ||
145 | |||
146 | return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id) | ||
147 | } | ||
148 | |||
149 | async function isAbleToUploadVideo (userId: number, size: number) { | ||
150 | const user = await UserModel.loadById(userId) | ||
151 | |||
152 | if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true) | ||
153 | |||
154 | const [ totalBytes, totalBytesDaily ] = await Promise.all([ | ||
155 | getOriginalVideoFileTotalFromUser(user.id), | ||
156 | getOriginalVideoFileTotalDailyFromUser(user.id) | ||
157 | ]) | ||
158 | |||
159 | const uploadedTotal = size + totalBytes | ||
160 | const uploadedDaily = size + totalBytesDaily | ||
161 | |||
162 | if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota | ||
163 | if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily | ||
164 | |||
165 | return uploadedTotal < user.videoQuota && uploadedDaily < user.videoQuotaDaily | ||
166 | } | ||
167 | |||
119 | // --------------------------------------------------------------------------- | 168 | // --------------------------------------------------------------------------- |
120 | 169 | ||
121 | export { | 170 | export { |
171 | getOriginalVideoFileTotalFromUser, | ||
172 | getOriginalVideoFileTotalDailyFromUser, | ||
122 | createApplicationActor, | 173 | createApplicationActor, |
123 | createUserAccountAndChannelAndPlaylist, | 174 | createUserAccountAndChannelAndPlaylist, |
124 | createLocalAccountWithoutKeys, | 175 | createLocalAccountWithoutKeys, |
125 | sendVerifyUserEmail | 176 | sendVerifyUserEmail, |
177 | isAbleToUploadVideo | ||
126 | } | 178 | } |
127 | 179 | ||
128 | // --------------------------------------------------------------------------- | 180 | // --------------------------------------------------------------------------- |
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index d3669f6be..41a6ae4f9 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts | |||
@@ -1,12 +1,13 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { body } from 'express-validator' | 2 | import { body } from 'express-validator' |
3 | import { isUserNSFWPolicyValid, isUserVideoQuotaDailyValid, isUserVideoQuotaValid } from '../../helpers/custom-validators/users' | 3 | import { isIntOrNull } from '@server/helpers/custom-validators/misc' |
4 | import { logger } from '../../helpers/logger' | 4 | import { isEmailEnabled } from '@server/initializers/config' |
5 | import { CustomConfig } from '../../../shared/models/server/custom-config.model' | 5 | import { CustomConfig } from '../../../shared/models/server/custom-config.model' |
6 | import { areValidationErrors } from './utils' | ||
7 | import { isThemeNameValid } from '../../helpers/custom-validators/plugins' | 6 | import { isThemeNameValid } from '../../helpers/custom-validators/plugins' |
7 | import { isUserNSFWPolicyValid, isUserVideoQuotaDailyValid, isUserVideoQuotaValid } from '../../helpers/custom-validators/users' | ||
8 | import { logger } from '../../helpers/logger' | ||
8 | import { isThemeRegistered } from '../../lib/plugins/theme-utils' | 9 | import { isThemeRegistered } from '../../lib/plugins/theme-utils' |
9 | import { isEmailEnabled } from '@server/initializers/config' | 10 | import { areValidationErrors } from './utils' |
10 | 11 | ||
11 | const customConfigUpdateValidator = [ | 12 | const customConfigUpdateValidator = [ |
12 | body('instance.name').exists().withMessage('Should have a valid instance name'), | 13 | body('instance.name').exists().withMessage('Should have a valid instance name'), |
@@ -43,6 +44,7 @@ const customConfigUpdateValidator = [ | |||
43 | body('transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'), | 44 | body('transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'), |
44 | body('transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'), | 45 | body('transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'), |
45 | body('transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'), | 46 | body('transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'), |
47 | body('transcoding.resolutions.2160p').isBoolean().withMessage('Should have a valid transcoding 2160p resolution enabled boolean'), | ||
46 | 48 | ||
47 | body('transcoding.webtorrent.enabled').isBoolean().withMessage('Should have a valid webtorrent transcoding enabled boolean'), | 49 | body('transcoding.webtorrent.enabled').isBoolean().withMessage('Should have a valid webtorrent transcoding enabled boolean'), |
48 | body('transcoding.hls.enabled').isBoolean().withMessage('Should have a valid hls transcoding enabled boolean'), | 50 | body('transcoding.hls.enabled').isBoolean().withMessage('Should have a valid hls transcoding enabled boolean'), |
@@ -60,6 +62,18 @@ const customConfigUpdateValidator = [ | |||
60 | body('broadcastMessage.level').exists().withMessage('Should have a valid broadcast level'), | 62 | body('broadcastMessage.level').exists().withMessage('Should have a valid broadcast level'), |
61 | body('broadcastMessage.dismissable').isBoolean().withMessage('Should have a valid broadcast dismissable boolean'), | 63 | body('broadcastMessage.dismissable').isBoolean().withMessage('Should have a valid broadcast dismissable boolean'), |
62 | 64 | ||
65 | body('live.enabled').isBoolean().withMessage('Should have a valid live enabled boolean'), | ||
66 | body('live.allowReplay').isBoolean().withMessage('Should have a valid live allow replay boolean'), | ||
67 | body('live.maxDuration').custom(isIntOrNull).withMessage('Should have a valid live max duration'), | ||
68 | body('live.transcoding.enabled').isBoolean().withMessage('Should have a valid live transcoding enabled boolean'), | ||
69 | body('live.transcoding.threads').isInt().withMessage('Should have a valid live transcoding threads'), | ||
70 | body('live.transcoding.resolutions.240p').isBoolean().withMessage('Should have a valid transcoding 240p resolution enabled boolean'), | ||
71 | body('live.transcoding.resolutions.360p').isBoolean().withMessage('Should have a valid transcoding 360p resolution enabled boolean'), | ||
72 | body('live.transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'), | ||
73 | body('live.transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'), | ||
74 | body('live.transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'), | ||
75 | body('live.transcoding.resolutions.2160p').isBoolean().withMessage('Should have a valid transcoding 2160p resolution enabled boolean'), | ||
76 | |||
63 | body('search.remoteUri.users').isBoolean().withMessage('Should have a remote URI search for users boolean'), | 77 | body('search.remoteUri.users').isBoolean().withMessage('Should have a remote URI search for users boolean'), |
64 | body('search.remoteUri.anonymous').isBoolean().withMessage('Should have a valid remote URI search for anonymous boolean'), | 78 | body('search.remoteUri.anonymous').isBoolean().withMessage('Should have a valid remote URI search for anonymous boolean'), |
65 | body('search.searchIndex.enabled').isBoolean().withMessage('Should have a valid search index enabled boolean'), | 79 | body('search.searchIndex.enabled').isBoolean().withMessage('Should have a valid search index enabled boolean'), |
@@ -71,8 +85,9 @@ const customConfigUpdateValidator = [ | |||
71 | logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body }) | 85 | logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body }) |
72 | 86 | ||
73 | if (areValidationErrors(req, res)) return | 87 | if (areValidationErrors(req, res)) return |
74 | if (!checkInvalidConfigIfEmailDisabled(req.body as CustomConfig, res)) return | 88 | if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return |
75 | if (!checkInvalidTranscodingConfig(req.body as CustomConfig, res)) return | 89 | if (!checkInvalidTranscodingConfig(req.body, res)) return |
90 | if (!checkInvalidLiveConfig(req.body, res)) return | ||
76 | 91 | ||
77 | return next() | 92 | return next() |
78 | } | 93 | } |
@@ -109,3 +124,16 @@ function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express | |||
109 | 124 | ||
110 | return true | 125 | return true |
111 | } | 126 | } |
127 | |||
128 | function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) { | ||
129 | if (customConfig.live.enabled === false) return true | ||
130 | |||
131 | if (customConfig.live.allowReplay === true && customConfig.transcoding.enabled === false) { | ||
132 | res.status(400) | ||
133 | .send({ error: 'You cannot allow live replay if transcoding is not enabled' }) | ||
134 | .end() | ||
135 | return false | ||
136 | } | ||
137 | |||
138 | return true | ||
139 | } | ||
diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts index 76ecff884..452c7fb93 100644 --- a/server/middlewares/validators/users.ts +++ b/server/middlewares/validators/users.ts | |||
@@ -497,7 +497,7 @@ export { | |||
497 | 497 | ||
498 | function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) { | 498 | function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) { |
499 | const id = parseInt(idArg + '', 10) | 499 | const id = parseInt(idArg + '', 10) |
500 | return checkUserExist(() => UserModel.loadById(id, withStats), res) | 500 | return checkUserExist(() => UserModel.loadByIdWithChannels(id, withStats), res) |
501 | } | 501 | } |
502 | 502 | ||
503 | function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) { | 503 | function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) { |
diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index b022b2c23..ff90e347a 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts | |||
@@ -1,5 +1,6 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { body, param, query, ValidationChain } from 'express-validator' | 2 | import { body, param, query, ValidationChain } from 'express-validator' |
3 | import { isAbleToUploadVideo } from '@server/lib/user' | ||
3 | import { getServerActor } from '@server/models/application/application' | 4 | import { getServerActor } from '@server/models/application/application' |
4 | import { MVideoFullLight } from '@server/types/models' | 5 | import { MVideoFullLight } from '@server/types/models' |
5 | import { ServerErrorCode, UserRight, VideoChangeOwnershipStatus, VideoPrivacy } from '../../../../shared' | 6 | import { ServerErrorCode, UserRight, VideoChangeOwnershipStatus, VideoPrivacy } from '../../../../shared' |
@@ -73,7 +74,7 @@ const videosAddValidator = getCommonVideoEditAttributes().concat([ | |||
73 | 74 | ||
74 | if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) | 75 | if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) |
75 | 76 | ||
76 | if (await user.isAbleToUploadVideo(videoFile) === false) { | 77 | if (await isAbleToUploadVideo(user.id, videoFile.size) === false) { |
77 | res.status(403) | 78 | res.status(403) |
78 | .json({ error: 'The user video quota is exceeded with this video.' }) | 79 | .json({ error: 'The user video quota is exceeded with this video.' }) |
79 | 80 | ||
@@ -291,7 +292,7 @@ const videosAcceptChangeOwnershipValidator = [ | |||
291 | 292 | ||
292 | const user = res.locals.oauth.token.User | 293 | const user = res.locals.oauth.token.User |
293 | const videoChangeOwnership = res.locals.videoChangeOwnership | 294 | const videoChangeOwnership = res.locals.videoChangeOwnership |
294 | const isAble = await user.isAbleToUploadVideo(videoChangeOwnership.Video.getMaxQualityFile()) | 295 | const isAble = await isAbleToUploadVideo(user.id, videoChangeOwnership.Video.getMaxQualityFile().size) |
295 | if (isAble === false) { | 296 | if (isAble === false) { |
296 | res.status(403) | 297 | res.status(403) |
297 | .json({ error: 'The user video quota is exceeded with this video.' }) | 298 | .json({ error: 'The user video quota is exceeded with this video.' }) |
diff --git a/server/models/account/user.ts b/server/models/account/user.ts index 22e6715b4..e850d1e6d 100644 --- a/server/models/account/user.ts +++ b/server/models/account/user.ts | |||
@@ -23,6 +23,7 @@ import { | |||
23 | } from 'sequelize-typescript' | 23 | } from 'sequelize-typescript' |
24 | import { | 24 | import { |
25 | MMyUserFormattable, | 25 | MMyUserFormattable, |
26 | MUser, | ||
26 | MUserDefault, | 27 | MUserDefault, |
27 | MUserFormattable, | 28 | MUserFormattable, |
28 | MUserId, | 29 | MUserId, |
@@ -70,6 +71,7 @@ import { VideoImportModel } from '../video/video-import' | |||
70 | import { VideoPlaylistModel } from '../video/video-playlist' | 71 | import { VideoPlaylistModel } from '../video/video-playlist' |
71 | import { AccountModel } from './account' | 72 | import { AccountModel } from './account' |
72 | import { UserNotificationSettingModel } from './user-notification-setting' | 73 | import { UserNotificationSettingModel } from './user-notification-setting' |
74 | import { VideoLiveModel } from '../video/video-live' | ||
73 | 75 | ||
74 | enum ScopeNames { | 76 | enum ScopeNames { |
75 | FOR_ME_API = 'FOR_ME_API', | 77 | FOR_ME_API = 'FOR_ME_API', |
@@ -540,7 +542,11 @@ export class UserModel extends Model<UserModel> { | |||
540 | return UserModel.findAll(query) | 542 | return UserModel.findAll(query) |
541 | } | 543 | } |
542 | 544 | ||
543 | static loadById (id: number, withStats = false): Bluebird<MUserDefault> { | 545 | static loadById (id: number): Bluebird<MUser> { |
546 | return UserModel.unscoped().findByPk(id) | ||
547 | } | ||
548 | |||
549 | static loadByIdWithChannels (id: number, withStats = false): Bluebird<MUserDefault> { | ||
544 | const scopes = [ | 550 | const scopes = [ |
545 | ScopeNames.WITH_VIDEOCHANNELS | 551 | ScopeNames.WITH_VIDEOCHANNELS |
546 | ] | 552 | ] |
@@ -685,26 +691,85 @@ export class UserModel extends Model<UserModel> { | |||
685 | return UserModel.findOne(query) | 691 | return UserModel.findOne(query) |
686 | } | 692 | } |
687 | 693 | ||
688 | static getOriginalVideoFileTotalFromUser (user: MUserId) { | 694 | static loadByLiveId (liveId: number): Bluebird<MUser> { |
689 | // Don't use sequelize because we need to use a sub query | 695 | const query = { |
690 | const query = UserModel.generateUserQuotaBaseSQL({ | 696 | include: [ |
691 | withSelect: true, | 697 | { |
692 | whereUserId: '$userId' | 698 | attributes: [ 'id' ], |
693 | }) | 699 | model: AccountModel.unscoped(), |
700 | required: true, | ||
701 | include: [ | ||
702 | { | ||
703 | attributes: [ 'id' ], | ||
704 | model: VideoChannelModel.unscoped(), | ||
705 | required: true, | ||
706 | include: [ | ||
707 | { | ||
708 | attributes: [ 'id' ], | ||
709 | model: VideoModel.unscoped(), | ||
710 | required: true, | ||
711 | include: [ | ||
712 | { | ||
713 | attributes: [ 'id', 'videoId' ], | ||
714 | model: VideoLiveModel.unscoped(), | ||
715 | required: true, | ||
716 | where: { | ||
717 | id: liveId | ||
718 | } | ||
719 | } | ||
720 | ] | ||
721 | } | ||
722 | ] | ||
723 | } | ||
724 | ] | ||
725 | } | ||
726 | ] | ||
727 | } | ||
728 | |||
729 | return UserModel.findOne(query) | ||
730 | } | ||
731 | |||
732 | static generateUserQuotaBaseSQL (options: { | ||
733 | whereUserId: '$userId' | '"UserModel"."id"' | ||
734 | withSelect: boolean | ||
735 | where?: string | ||
736 | }) { | ||
737 | const andWhere = options.where | ||
738 | ? 'AND ' + options.where | ||
739 | : '' | ||
740 | |||
741 | const videoChannelJoin = 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + | ||
742 | 'INNER JOIN "account" ON "videoChannel"."accountId" = "account"."id" ' + | ||
743 | `WHERE "account"."userId" = ${options.whereUserId} ${andWhere}` | ||
744 | |||
745 | const webtorrentFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' + | ||
746 | 'INNER JOIN "video" ON "videoFile"."videoId" = "video"."id" ' + | ||
747 | videoChannelJoin | ||
748 | |||
749 | const hlsFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' + | ||
750 | 'INNER JOIN "videoStreamingPlaylist" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id ' + | ||
751 | 'INNER JOIN "video" ON "videoStreamingPlaylist"."videoId" = "video"."id" ' + | ||
752 | videoChannelJoin | ||
694 | 753 | ||
695 | return UserModel.getTotalRawQuery(query, user.id) | 754 | return 'SELECT COALESCE(SUM("size"), 0) AS "total" ' + |
755 | 'FROM (' + | ||
756 | `SELECT MAX("t1"."size") AS "size" FROM (${webtorrentFiles} UNION ${hlsFiles}) t1 ` + | ||
757 | 'GROUP BY "t1"."videoId"' + | ||
758 | ') t2' | ||
696 | } | 759 | } |
697 | 760 | ||
698 | // Returns cumulative size of all video files uploaded in the last 24 hours. | 761 | static getTotalRawQuery (query: string, userId: number) { |
699 | static getOriginalVideoFileTotalDailyFromUser (user: MUserId) { | 762 | const options = { |
700 | // Don't use sequelize because we need to use a sub query | 763 | bind: { userId }, |
701 | const query = UserModel.generateUserQuotaBaseSQL({ | 764 | type: QueryTypes.SELECT as QueryTypes.SELECT |
702 | withSelect: true, | 765 | } |
703 | whereUserId: '$userId', | 766 | |
704 | where: '"video"."createdAt" > now() - interval \'24 hours\'' | 767 | return UserModel.sequelize.query<{ total: string }>(query, options) |
705 | }) | 768 | .then(([ { total } ]) => { |
769 | if (total === null) return 0 | ||
706 | 770 | ||
707 | return UserModel.getTotalRawQuery(query, user.id) | 771 | return parseInt(total, 10) |
772 | }) | ||
708 | } | 773 | } |
709 | 774 | ||
710 | static async getStats () { | 775 | static async getStats () { |
@@ -874,64 +939,4 @@ export class UserModel extends Model<UserModel> { | |||
874 | 939 | ||
875 | return Object.assign(formatted, { specialPlaylists }) | 940 | return Object.assign(formatted, { specialPlaylists }) |
876 | } | 941 | } |
877 | |||
878 | async isAbleToUploadVideo (videoFile: { size: number }) { | ||
879 | if (this.videoQuota === -1 && this.videoQuotaDaily === -1) return Promise.resolve(true) | ||
880 | |||
881 | const [ totalBytes, totalBytesDaily ] = await Promise.all([ | ||
882 | UserModel.getOriginalVideoFileTotalFromUser(this), | ||
883 | UserModel.getOriginalVideoFileTotalDailyFromUser(this) | ||
884 | ]) | ||
885 | |||
886 | const uploadedTotal = videoFile.size + totalBytes | ||
887 | const uploadedDaily = videoFile.size + totalBytesDaily | ||
888 | |||
889 | if (this.videoQuotaDaily === -1) return uploadedTotal < this.videoQuota | ||
890 | if (this.videoQuota === -1) return uploadedDaily < this.videoQuotaDaily | ||
891 | |||
892 | return uploadedTotal < this.videoQuota && uploadedDaily < this.videoQuotaDaily | ||
893 | } | ||
894 | |||
895 | private static generateUserQuotaBaseSQL (options: { | ||
896 | whereUserId: '$userId' | '"UserModel"."id"' | ||
897 | withSelect: boolean | ||
898 | where?: string | ||
899 | }) { | ||
900 | const andWhere = options.where | ||
901 | ? 'AND ' + options.where | ||
902 | : '' | ||
903 | |||
904 | const videoChannelJoin = 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + | ||
905 | 'INNER JOIN "account" ON "videoChannel"."accountId" = "account"."id" ' + | ||
906 | `WHERE "account"."userId" = ${options.whereUserId} ${andWhere}` | ||
907 | |||
908 | const webtorrentFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' + | ||
909 | 'INNER JOIN "video" ON "videoFile"."videoId" = "video"."id" ' + | ||
910 | videoChannelJoin | ||
911 | |||
912 | const hlsFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' + | ||
913 | 'INNER JOIN "videoStreamingPlaylist" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id ' + | ||
914 | 'INNER JOIN "video" ON "videoStreamingPlaylist"."videoId" = "video"."id" ' + | ||
915 | videoChannelJoin | ||
916 | |||
917 | return 'SELECT COALESCE(SUM("size"), 0) AS "total" ' + | ||
918 | 'FROM (' + | ||
919 | `SELECT MAX("t1"."size") AS "size" FROM (${webtorrentFiles} UNION ${hlsFiles}) t1 ` + | ||
920 | 'GROUP BY "t1"."videoId"' + | ||
921 | ') t2' | ||
922 | } | ||
923 | |||
924 | private static getTotalRawQuery (query: string, userId: number) { | ||
925 | const options = { | ||
926 | bind: { userId }, | ||
927 | type: QueryTypes.SELECT as QueryTypes.SELECT | ||
928 | } | ||
929 | |||
930 | return UserModel.sequelize.query<{ total: string }>(query, options) | ||
931 | .then(([ { total } ]) => { | ||
932 | if (total === null) return 0 | ||
933 | |||
934 | return parseInt(total, 10) | ||
935 | }) | ||
936 | } | ||
937 | } | 942 | } |
diff --git a/server/models/video/video-live.ts b/server/models/video/video-live.ts index 8608bc84c..a1dd80d3c 100644 --- a/server/models/video/video-live.ts +++ b/server/models/video/video-live.ts | |||
@@ -30,10 +30,18 @@ import { VideoBlacklistModel } from './video-blacklist' | |||
30 | }) | 30 | }) |
31 | export class VideoLiveModel extends Model<VideoLiveModel> { | 31 | export class VideoLiveModel extends Model<VideoLiveModel> { |
32 | 32 | ||
33 | @AllowNull(false) | 33 | @AllowNull(true) |
34 | @Column(DataType.STRING) | 34 | @Column(DataType.STRING) |
35 | streamKey: string | 35 | streamKey: string |
36 | 36 | ||
37 | @AllowNull(false) | ||
38 | @Column | ||
39 | perpetualLive: boolean | ||
40 | |||
41 | @AllowNull(false) | ||
42 | @Column | ||
43 | saveReplay: boolean | ||
44 | |||
37 | @CreatedAt | 45 | @CreatedAt |
38 | createdAt: Date | 46 | createdAt: Date |
39 | 47 | ||
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 35cb333ef..2882ceb7c 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts | |||
@@ -103,6 +103,9 @@ describe('Test config API validators', function () { | |||
103 | live: { | 103 | live: { |
104 | enabled: true, | 104 | enabled: true, |
105 | 105 | ||
106 | allowReplay: false, | ||
107 | maxDuration: null, | ||
108 | |||
106 | transcoding: { | 109 | transcoding: { |
107 | enabled: true, | 110 | enabled: true, |
108 | threads: 4, | 111 | threads: 4, |
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index a46e179c2..a7f035362 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts | |||
@@ -79,6 +79,8 @@ function checkInitialConfig (server: ServerInfo, data: CustomConfig) { | |||
79 | expect(data.transcoding.hls.enabled).to.be.true | 79 | expect(data.transcoding.hls.enabled).to.be.true |
80 | 80 | ||
81 | expect(data.live.enabled).to.be.false | 81 | expect(data.live.enabled).to.be.false |
82 | expect(data.live.allowReplay).to.be.true | ||
83 | expect(data.live.maxDuration).to.equal(1000 * 3600 * 5) | ||
82 | expect(data.live.transcoding.enabled).to.be.false | 84 | expect(data.live.transcoding.enabled).to.be.false |
83 | expect(data.live.transcoding.threads).to.equal(2) | 85 | expect(data.live.transcoding.threads).to.equal(2) |
84 | expect(data.live.transcoding.resolutions['240p']).to.be.false | 86 | expect(data.live.transcoding.resolutions['240p']).to.be.false |
@@ -162,6 +164,8 @@ function checkUpdatedConfig (data: CustomConfig) { | |||
162 | expect(data.transcoding.webtorrent.enabled).to.be.true | 164 | expect(data.transcoding.webtorrent.enabled).to.be.true |
163 | 165 | ||
164 | expect(data.live.enabled).to.be.true | 166 | expect(data.live.enabled).to.be.true |
167 | expect(data.live.allowReplay).to.be.false | ||
168 | expect(data.live.maxDuration).to.equal(5000) | ||
165 | expect(data.live.transcoding.enabled).to.be.true | 169 | expect(data.live.transcoding.enabled).to.be.true |
166 | expect(data.live.transcoding.threads).to.equal(4) | 170 | expect(data.live.transcoding.threads).to.equal(4) |
167 | expect(data.live.transcoding.resolutions['240p']).to.be.true | 171 | expect(data.live.transcoding.resolutions['240p']).to.be.true |
@@ -324,6 +328,8 @@ describe('Test config', function () { | |||
324 | }, | 328 | }, |
325 | live: { | 329 | live: { |
326 | enabled: true, | 330 | enabled: true, |
331 | allowReplay: false, | ||
332 | maxDuration: 5000, | ||
327 | transcoding: { | 333 | transcoding: { |
328 | enabled: true, | 334 | enabled: true, |
329 | threads: 4, | 335 | threads: 4, |