From fb7194043d0486ce0a6a40b2ffbdf32878c33a6f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 25 Sep 2020 16:19:35 +0200 Subject: [PATCH] Check live duration and size --- .../edit-custom-config.component.html | 102 +++++++---- .../edit-custom-config.component.ts | 30 +++- client/src/app/core/server/server.service.ts | 2 + config/test.yaml | 2 +- server/controllers/api/config.ts | 5 + server/controllers/api/users/me.ts | 6 +- server/helpers/core-utils.ts | 1 + server/helpers/custom-validators/misc.ts | 5 + server/helpers/ffmpeg-utils.ts | 22 ++- server/initializers/checker-after-init.ts | 7 + server/initializers/checker-before-init.ts | 7 +- server/initializers/config.ts | 3 + server/initializers/constants.ts | 7 +- .../migrations/0535-video-live.ts | 2 +- server/lib/job-queue/handlers/video-import.ts | 3 +- server/lib/live-manager.ts | 89 ++++++++-- server/lib/user.ts | 72 ++++++-- server/middlewares/validators/config.ts | 40 ++++- server/middlewares/validators/users.ts | 2 +- .../middlewares/validators/videos/videos.ts | 5 +- server/models/account/user.ts | 159 +++++++++--------- server/models/video/video-live.ts | 10 +- server/tests/api/check-params/config.ts | 3 + server/tests/api/server/config.ts | 6 + shared/extra-utils/server/config.ts | 2 + shared/models/server/custom-config.model.ts | 3 + shared/models/server/server-config.model.ts | 3 + 27 files changed, 433 insertions(+), 165 deletions(-) diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html index 8000f471f..2f3202e06 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html @@ -699,7 +699,7 @@ - +
Live streaming @@ -722,54 +722,78 @@ Allow live streaming - - Enabling live streaming requires trust in your users and extra moderation work - + + ⚠️ Enabling live streaming requires trust in your users and extra moderation work + - +
- - Requires a lot of CPU! + + If the user quota is reached, PeerTube will automatically terminate the live streaming
-
- +
+
- +
-
{{ formErrors.live.transcoding.threads }}
-
- - - -
- -
- - -
- - -
- + + +
+ + + Requires a lot of CPU! + +
-
+ +
+ +
+ +
+
{{ formErrors.live.transcoding.threads }}
+
+ +
+ + +
+ +
+ + +
+ + +
+ +
+
+
@@ -778,7 +802,7 @@
- +
Advanced configuration @@ -1026,9 +1050,15 @@
- It seems like the configuration is invalid. Please search for potential errors in the different tabs. + + It seems like the configuration is invalid. Please search for potential errors in the different tabs. + + + + You cannot allow live replay if you don't enable transcoding. + - +
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts index de800c87e..745238647 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts @@ -36,6 +36,7 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit, A resolutions: { id: string, label: string, description?: string }[] = [] liveResolutions: { id: string, label: string, description?: string }[] = [] transcodingThreadOptions: { label: string, value: number }[] = [] + liveMaxDurationOptions: { label: string, value: number }[] = [] languageItems: SelectOptionsItem[] = [] categoryItems: SelectOptionsItem[] = [] @@ -92,6 +93,14 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit, A { value: 4, label: '4' }, { value: 8, label: '8' } ] + + this.liveMaxDurationOptions = [ + { value: 0, label: $localize`No limit` }, + { value: 1000 * 3600, label: $localize`1 hour` }, + { value: 1000 * 3600 * 3, label: $localize`3 hours` }, + { value: 1000 * 3600 * 5, label: $localize`5 hours` }, + { value: 1000 * 3600 * 10, label: $localize`10 hours` } + ] } get videoQuotaOptions () { @@ -114,7 +123,9 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit, A ngOnInit () { this.serverConfig = this.serverService.getTmpConfig() this.serverService.getConfig() - .subscribe(config => this.serverConfig = config) + .subscribe(config => { + this.serverConfig = config + }) const formGroupData: { [key in keyof CustomConfig ]: any } = { instance: { @@ -204,6 +215,9 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit, A live: { enabled: null, + maxDuration: null, + allowReplay: null, + transcoding: { enabled: null, threads: TRANSCODING_THREADS_VALIDATOR, @@ -341,6 +355,20 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit, A } } + hasConsistentOptions () { + if (this.hasLiveAllowReplayConsistentOptions()) return true + + return false + } + + hasLiveAllowReplayConsistentOptions () { + if (this.isTranscodingEnabled() === false && this.isLiveEnabled() && this.form.value['live']['allowReplay'] === true) { + return false + } + + return true + } + private updateForm () { this.form.patchValue(this.customConfig) } diff --git a/client/src/app/core/server/server.service.ts b/client/src/app/core/server/server.service.ts index bc76bacfc..c19c3c12e 100644 --- a/client/src/app/core/server/server.service.ts +++ b/client/src/app/core/server/server.service.ts @@ -76,6 +76,8 @@ export class ServerService { }, live: { enabled: false, + allowReplay: true, + maxDuration: null, transcoding: { enabled: false, enabledResolutions: [] diff --git a/config/test.yaml b/config/test.yaml index 865ed5400..b9279b5e6 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -89,7 +89,7 @@ live: port: 1935 transcoding: - enabled: true + enabled: false threads: 2 resolutions: diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index bd100ef9c..99aabba62 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -118,6 +118,9 @@ async function getConfig (req: express.Request, res: express.Response) { live: { enabled: CONFIG.LIVE.ENABLED, + allowReplay: CONFIG.LIVE.ALLOW_REPLAY, + maxDuration: CONFIG.LIVE.MAX_DURATION, + transcoding: { enabled: CONFIG.LIVE.TRANSCODING.ENABLED, enabledResolutions: getEnabledResolutions('live') @@ -425,6 +428,8 @@ function customConfig (): CustomConfig { }, live: { enabled: CONFIG.LIVE.ENABLED, + allowReplay: CONFIG.LIVE.ALLOW_REPLAY, + maxDuration: CONFIG.LIVE.MAX_DURATION, transcoding: { enabled: CONFIG.LIVE.TRANSCODING.ENABLED, threads: CONFIG.LIVE.TRANSCODING.THREADS, diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index ba60a3d2a..b490518fc 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -9,7 +9,7 @@ import { MIMETYPES } from '../../../initializers/constants' import { sequelizeTypescript } from '../../../initializers/database' import { sendUpdateActor } from '../../../lib/activitypub/send' import { updateActorAvatarFile } from '../../../lib/avatar' -import { sendVerifyUserEmail } from '../../../lib/user' +import { getOriginalVideoFileTotalDailyFromUser, getOriginalVideoFileTotalFromUser, sendVerifyUserEmail } from '../../../lib/user' import { asyncMiddleware, asyncRetryTransactionMiddleware, @@ -133,8 +133,8 @@ async function getUserInformation (req: express.Request, res: express.Response) async function getUserVideoQuotaUsed (req: express.Request, res: express.Response) { const user = res.locals.oauth.token.user - const videoQuotaUsed = await UserModel.getOriginalVideoFileTotalFromUser(user) - const videoQuotaUsedDaily = await UserModel.getOriginalVideoFileTotalDailyFromUser(user) + const videoQuotaUsed = await getOriginalVideoFileTotalFromUser(user) + const videoQuotaUsedDaily = await getOriginalVideoFileTotalDailyFromUser(user) const data: UserVideoQuota = { videoQuotaUsed, diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 49eee7c59..e1c15a6eb 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -41,6 +41,7 @@ const timeTable = { } export function parseDurationToMs (duration: number | string): number { + if (duration === null) return null if (typeof duration === 'number') return duration if (typeof duration === 'string') { diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index cf32201c4..61c03f0c9 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts @@ -45,6 +45,10 @@ function isBooleanValid (value: any) { return typeof value === 'boolean' || (typeof value === 'string' && validator.isBoolean(value)) } +function isIntOrNull (value: any) { + return value === null || validator.isInt('' + value) +} + function toIntOrNull (value: string) { const v = toValueOrNull(value) @@ -116,6 +120,7 @@ export { isArrayOf, isNotEmptyIntArray, isArray, + isIntOrNull, isIdValid, isSafePath, isUUIDValid, diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index fac2595f1..b25dcaa90 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -5,7 +5,7 @@ import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata' import { getMaxBitrate, getTargetBitrate, VideoResolution } from '../../shared/models/videos' import { checkFFmpegEncoders } from '../initializers/checker-before-init' import { CONFIG } from '../initializers/config' -import { FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' +import { FFMPEG_NICE, VIDEO_LIVE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' import { processImage } from './image-utils' import { logger } from './logger' @@ -353,7 +353,7 @@ function convertWebPToJPG (path: string, destination: string): Promise { }) } -function runLiveTranscoding (rtmpUrl: string, outPath: string, resolutions: number[]) { +function runLiveTranscoding (rtmpUrl: string, outPath: string, resolutions: number[], deleteSegments: boolean) { const command = getFFmpeg(rtmpUrl) command.inputOption('-fflags nobuffer') @@ -399,7 +399,7 @@ function runLiveTranscoding (rtmpUrl: string, outPath: string, resolutions: numb varStreamMap.push(`v:${i},a:${i}`) } - addDefaultLiveHLSParams(command, outPath) + addDefaultLiveHLSParams(command, outPath, deleteSegments) command.outputOption('-var_stream_map', varStreamMap.join(' ')) @@ -408,7 +408,7 @@ function runLiveTranscoding (rtmpUrl: string, outPath: string, resolutions: numb return command } -function runLiveMuxing (rtmpUrl: string, outPath: string) { +function runLiveMuxing (rtmpUrl: string, outPath: string, deleteSegments: boolean) { const command = getFFmpeg(rtmpUrl) command.inputOption('-fflags nobuffer') @@ -417,7 +417,7 @@ function runLiveMuxing (rtmpUrl: string, outPath: string) { command.outputOption('-map 0:a?') command.outputOption('-map 0:v?') - addDefaultLiveHLSParams(command, outPath) + addDefaultLiveHLSParams(command, outPath, deleteSegments) command.run() @@ -457,10 +457,14 @@ function addDefaultX264Params (command: ffmpeg.FfmpegCommand) { .outputOption('-map_metadata -1') // strip all metadata } -function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string) { - command.outputOption('-hls_time 4') - command.outputOption('-hls_list_size 15') - command.outputOption('-hls_flags delete_segments') +function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string, deleteSegments: boolean) { + command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME) + command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE) + + if (deleteSegments === true) { + command.outputOption('-hls_flags delete_segments') + } + command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%d.ts')}`) command.outputOption('-master_pl_name master.m3u8') command.outputOption(`-f hls`) diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index b49ab6bca..979c97a8b 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts @@ -135,6 +135,13 @@ function checkConfig () { } } + // Live + if (CONFIG.LIVE.ENABLED === true) { + if (CONFIG.LIVE.ALLOW_REPLAY === true && CONFIG.TRANSCODING.ENABLED === false) { + return 'Live allow replay cannot be enabled if transcoding is not enabled.' + } + } + return null } diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index e0819c4aa..d4140e3fa 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -37,8 +37,13 @@ function checkMissedConfig () { 'remote_redundancy.videos.accept_from', 'federation.videos.federate_unlisted', 'search.remote_uri.users', 'search.remote_uri.anonymous', 'search.search_index.enabled', 'search.search_index.url', - 'search.search_index.disable_local_search', 'search.search_index.is_default_search' + 'search.search_index.disable_local_search', 'search.search_index.is_default_search', + 'live.enabled', 'live.allow_replay', 'live.max_duration', + 'live.transcoding.enabled', 'live.transcoding.threads', + 'live.transcoding.resolutions.240p', 'live.transcoding.resolutions.360p', 'live.transcoding.resolutions.480p', + 'live.transcoding.resolutions.720p', 'live.transcoding.resolutions.1080p', 'live.transcoding.resolutions.2160p' ] + const requiredAlternatives = [ [ // set [ 'redis.hostname', 'redis.port' ], // alternative diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 7a8200ed9..9e8927350 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -201,6 +201,9 @@ const CONFIG = { LIVE: { get ENABLED () { return config.get('live.enabled') }, + get MAX_DURATION () { return parseDurationToMs(config.get('live.max_duration')) }, + get ALLOW_REPLAY () { return config.get('live.allow_replay') }, + RTMP: { get PORT () { return config.get('live.rtmp.port') } }, diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 82d04a94e..065012b32 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -608,7 +608,9 @@ const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') const VIDEO_LIVE = { EXTENSION: '.ts', - CLEANUP_DELAY: 1000 * 60 * 5, // 5 mintues + CLEANUP_DELAY: 1000 * 60 * 5, // 5 minutes + SEGMENT_TIME: 4, // 4 seconds + SEGMENTS_LIST_SIZE: 15, // 15 maximum segments in live playlist RTMP: { CHUNK_SIZE: 60000, GOP_CACHE: true, @@ -620,7 +622,8 @@ const VIDEO_LIVE = { const MEMOIZE_TTL = { OVERVIEWS_SAMPLE: 1000 * 3600 * 4, // 4 hours - INFO_HASH_EXISTS: 1000 * 3600 * 12 // 12 hours + INFO_HASH_EXISTS: 1000 * 3600 * 12, // 12 hours + LIVE_ABLE_TO_UPLOAD: 1000 * 60 // 1 minute } const MEMOIZE_LENGTH = { diff --git a/server/initializers/migrations/0535-video-live.ts b/server/initializers/migrations/0535-video-live.ts index 35523efc4..7501e080b 100644 --- a/server/initializers/migrations/0535-video-live.ts +++ b/server/initializers/migrations/0535-video-live.ts @@ -9,7 +9,7 @@ async function up (utils: { const query = ` CREATE TABLE IF NOT EXISTS "videoLive" ( "id" SERIAL , - "streamKey" VARCHAR(255) NOT NULL, + "streamKey" VARCHAR(255), "videoId" INTEGER NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL, diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 9b5f2bb2b..9210aec54 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -4,6 +4,7 @@ import { extname } from 'path' import { addOptimizeOrMergeAudioJob } from '@server/helpers/video' import { isPostImportVideoAccepted } from '@server/lib/moderation' import { Hooks } from '@server/lib/plugins/hooks' +import { isAbleToUploadVideo } from '@server/lib/user' import { getVideoFilePath } from '@server/lib/video-paths' import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' import { @@ -108,7 +109,7 @@ async function processFile (downloader: () => Promise, videoImport: MVid // Get information about this video const stats = await stat(tempVideoPath) - const isAble = await videoImport.User.isAbleToUploadVideo({ size: stats.size }) + const isAble = await isAbleToUploadVideo(videoImport.User.id, stats.size) if (isAble === false) { throw new Error('The user video quota is exceeded with this video to import.') } diff --git a/server/lib/live-manager.ts b/server/lib/live-manager.ts index 41176d197..3ff2434ff 100644 --- a/server/lib/live-manager.ts +++ b/server/lib/live-manager.ts @@ -2,24 +2,27 @@ import { AsyncQueue, queue } from 'async' import * as chokidar from 'chokidar' import { FfmpegCommand } from 'fluent-ffmpeg' -import { ensureDir } from 'fs-extra' +import { ensureDir, stat } from 'fs-extra' import { basename } from 'path' import { computeResolutionsToTranscode, runLiveMuxing, runLiveTranscoding } from '@server/helpers/ffmpeg-utils' import { logger } from '@server/helpers/logger' import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' -import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, WEBSERVER } from '@server/initializers/constants' +import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, WEBSERVER } from '@server/initializers/constants' +import { UserModel } from '@server/models/account/user' import { VideoModel } from '@server/models/video/video' import { VideoFileModel } from '@server/models/video/video-file' import { VideoLiveModel } from '@server/models/video/video-live' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MStreamingPlaylist, MVideoLiveVideo } from '@server/types/models' +import { MStreamingPlaylist, MUser, MUserId, MVideoLive, MVideoLiveVideo } from '@server/types/models' import { VideoState, VideoStreamingPlaylistType } from '@shared/models' import { federateVideoIfNeeded } from './activitypub/videos' import { buildSha256Segment } from './hls' import { JobQueue } from './job-queue' import { PeerTubeSocket } from './peertube-socket' +import { isAbleToUploadVideo } from './user' import { getHLSDirectory } from './video-paths' +import memoizee = require('memoizee') const NodeRtmpServer = require('node-media-server/node_rtmp_server') const context = require('node-media-server/node_core_ctx') const nodeMediaServerLogger = require('node-media-server/node_core_logger') @@ -53,6 +56,11 @@ class LiveManager { private readonly transSessions = new Map() private readonly videoSessions = new Map() private readonly segmentsSha256 = new Map>() + private readonly livesPerUser = new Map() + + private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => { + return isAbleToUploadVideo(userId, 1000) + }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD }) private segmentsSha256Queue: AsyncQueue private rtmpServer: any @@ -127,7 +135,7 @@ class LiveManager { this.abortSession(sessionId) - this.onEndTransmuxing(videoId) + this.onEndTransmuxing(videoId, true) .catch(err => logger.error('Cannot end transmuxing of video %d.', videoId, { err })) } @@ -196,8 +204,18 @@ class LiveManager { originalResolution: number }) { const { sessionId, videoLive, playlist, streamPath, resolutionsEnabled, originalResolution } = options + const startStreamDateTime = new Date().getTime() const allResolutions = resolutionsEnabled.concat([ originalResolution ]) + const user = await UserModel.loadByLiveId(videoLive.id) + if (!this.livesPerUser.has(user.id)) { + this.livesPerUser.set(user.id, []) + } + + const currentUserLive = { liveId: videoLive.id, videoId: videoLive.videoId, size: 0 } + const livesOfUser = this.livesPerUser.get(user.id) + livesOfUser.push(currentUserLive) + for (let i = 0; i < allResolutions.length; i++) { const resolution = allResolutions[i] @@ -216,26 +234,47 @@ class LiveManager { const outPath = getHLSDirectory(videoLive.Video) await ensureDir(outPath) + const deleteSegments = videoLive.saveReplay === false + const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath const ffmpegExec = CONFIG.LIVE.TRANSCODING.ENABLED - ? runLiveTranscoding(rtmpUrl, outPath, allResolutions) - : runLiveMuxing(rtmpUrl, outPath) + ? runLiveTranscoding(rtmpUrl, outPath, allResolutions, deleteSegments) + : runLiveMuxing(rtmpUrl, outPath, deleteSegments) logger.info('Running live muxing/transcoding.') - this.transSessions.set(sessionId, ffmpegExec) const videoUUID = videoLive.Video.uuid const tsWatcher = chokidar.watch(outPath + '/*.ts') - const updateHandler = segmentPath => { - this.segmentsSha256Queue.push({ operation: 'update', segmentPath, videoUUID }) + const updateSegment = segmentPath => this.segmentsSha256Queue.push({ operation: 'update', segmentPath, videoUUID }) + + const addHandler = segmentPath => { + updateSegment(segmentPath) + + if (this.isDurationConstraintValid(startStreamDateTime) !== true) { + this.stopSessionOf(videoLive.videoId) + } + + if (videoLive.saveReplay === true) { + stat(segmentPath) + .then(segmentStat => { + currentUserLive.size += segmentStat.size + }) + .then(() => this.isQuotaConstraintValid(user, videoLive)) + .then(quotaValid => { + if (quotaValid !== true) { + this.stopSessionOf(videoLive.videoId) + } + }) + .catch(err => logger.error('Cannot stat %s or check quota of %d.', segmentPath, user.id, { err })) + } } const deleteHandler = segmentPath => this.segmentsSha256Queue.push({ operation: 'delete', segmentPath, videoUUID }) - tsWatcher.on('add', p => updateHandler(p)) - tsWatcher.on('change', p => updateHandler(p)) + tsWatcher.on('add', p => addHandler(p)) + tsWatcher.on('change', p => updateSegment(p)) tsWatcher.on('unlink', p => deleteHandler(p)) const masterWatcher = chokidar.watch(outPath + '/master.m3u8') @@ -280,7 +319,14 @@ class LiveManager { ffmpegExec.on('end', () => onFFmpegEnded()) } - private async onEndTransmuxing (videoId: number) { + getLiveQuotaUsedByUser (userId: number) { + const currentLives = this.livesPerUser.get(userId) + if (!currentLives) return 0 + + return currentLives.reduce((sum, obj) => sum + obj.size, 0) + } + + private async onEndTransmuxing (videoId: number, cleanupNow = false) { try { const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) if (!fullVideo) return @@ -290,7 +336,7 @@ class LiveManager { payload: { videoId: fullVideo.id } - }, { delay: VIDEO_LIVE.CLEANUP_DELAY }) + }, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY }) // FIXME: use end fullVideo.state = VideoState.WAITING_FOR_LIVE @@ -337,6 +383,23 @@ class LiveManager { filesMap.delete(segmentName) } + private isDurationConstraintValid (streamingStartTime: number) { + const maxDuration = CONFIG.LIVE.MAX_DURATION + // No limit + if (maxDuration === null) return true + + const now = new Date().getTime() + const max = streamingStartTime + maxDuration + + return now <= max + } + + private async isQuotaConstraintValid (user: MUserId, live: MVideoLive) { + if (live.saveReplay !== true) return true + + return this.isAbleToUploadVideoWithCache(user.id) + } + static get Instance () { return this.instance || (this.instance = new this()) } diff --git a/server/lib/user.ts b/server/lib/user.ts index aa14f0b54..d3338f329 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts @@ -1,20 +1,24 @@ +import { Transaction } from 'sequelize/types' import { v4 as uuidv4 } from 'uuid' +import { UserModel } from '@server/models/account/user' import { ActivityPubActorType } from '../../shared/models/activitypub' +import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users' import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants' +import { sequelizeTypescript } from '../initializers/database' import { AccountModel } from '../models/account/account' -import { buildActorInstance, setAsyncActorKeys } from './activitypub/actor' -import { createLocalVideoChannel } from './video-channel' -import { ActorModel } from '../models/activitypub/actor' import { UserNotificationSettingModel } from '../models/account/user-notification-setting' -import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users' -import { createWatchLaterPlaylist } from './video-playlist' -import { sequelizeTypescript } from '../initializers/database' -import { Transaction } from 'sequelize/types' -import { Redis } from './redis' -import { Emailer } from './emailer' +import { ActorModel } from '../models/activitypub/actor' import { MAccountDefault, MActorDefault, MChannelActor } from '../types/models' import { MUser, MUserDefault, MUserId } from '../types/models/user' +import { buildActorInstance, setAsyncActorKeys } from './activitypub/actor' import { getAccountActivityPubUrl } from './activitypub/url' +import { Emailer } from './emailer' +import { LiveManager } from './live-manager' +import { Redis } from './redis' +import { createLocalVideoChannel } from './video-channel' +import { createWatchLaterPlaylist } from './video-playlist' + +import memoizee = require('memoizee') type ChannelNames = { name: string, displayName: string } @@ -116,13 +120,61 @@ async function sendVerifyUserEmail (user: MUser, isPendingEmail = false) { await Emailer.Instance.addVerifyEmailJob(username, email, url) } +async function getOriginalVideoFileTotalFromUser (user: MUserId) { + // Don't use sequelize because we need to use a sub query + const query = UserModel.generateUserQuotaBaseSQL({ + withSelect: true, + whereUserId: '$userId' + }) + + const base = await UserModel.getTotalRawQuery(query, user.id) + + return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id) +} + +// Returns cumulative size of all video files uploaded in the last 24 hours. +async function getOriginalVideoFileTotalDailyFromUser (user: MUserId) { + // Don't use sequelize because we need to use a sub query + const query = UserModel.generateUserQuotaBaseSQL({ + withSelect: true, + whereUserId: '$userId', + where: '"video"."createdAt" > now() - interval \'24 hours\'' + }) + + const base = await UserModel.getTotalRawQuery(query, user.id) + + return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id) +} + +async function isAbleToUploadVideo (userId: number, size: number) { + const user = await UserModel.loadById(userId) + + if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true) + + const [ totalBytes, totalBytesDaily ] = await Promise.all([ + getOriginalVideoFileTotalFromUser(user.id), + getOriginalVideoFileTotalDailyFromUser(user.id) + ]) + + const uploadedTotal = size + totalBytes + const uploadedDaily = size + totalBytesDaily + + if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota + if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily + + return uploadedTotal < user.videoQuota && uploadedDaily < user.videoQuotaDaily +} + // --------------------------------------------------------------------------- export { + getOriginalVideoFileTotalFromUser, + getOriginalVideoFileTotalDailyFromUser, createApplicationActor, createUserAccountAndChannelAndPlaylist, createLocalAccountWithoutKeys, - sendVerifyUserEmail + sendVerifyUserEmail, + isAbleToUploadVideo } // --------------------------------------------------------------------------- diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index d3669f6be..41a6ae4f9 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts @@ -1,12 +1,13 @@ import * as express from 'express' import { body } from 'express-validator' -import { isUserNSFWPolicyValid, isUserVideoQuotaDailyValid, isUserVideoQuotaValid } from '../../helpers/custom-validators/users' -import { logger } from '../../helpers/logger' +import { isIntOrNull } from '@server/helpers/custom-validators/misc' +import { isEmailEnabled } from '@server/initializers/config' import { CustomConfig } from '../../../shared/models/server/custom-config.model' -import { areValidationErrors } from './utils' import { isThemeNameValid } from '../../helpers/custom-validators/plugins' +import { isUserNSFWPolicyValid, isUserVideoQuotaDailyValid, isUserVideoQuotaValid } from '../../helpers/custom-validators/users' +import { logger } from '../../helpers/logger' import { isThemeRegistered } from '../../lib/plugins/theme-utils' -import { isEmailEnabled } from '@server/initializers/config' +import { areValidationErrors } from './utils' const customConfigUpdateValidator = [ body('instance.name').exists().withMessage('Should have a valid instance name'), @@ -43,6 +44,7 @@ const customConfigUpdateValidator = [ body('transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'), body('transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'), body('transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'), + body('transcoding.resolutions.2160p').isBoolean().withMessage('Should have a valid transcoding 2160p resolution enabled boolean'), body('transcoding.webtorrent.enabled').isBoolean().withMessage('Should have a valid webtorrent transcoding enabled boolean'), body('transcoding.hls.enabled').isBoolean().withMessage('Should have a valid hls transcoding enabled boolean'), @@ -60,6 +62,18 @@ const customConfigUpdateValidator = [ body('broadcastMessage.level').exists().withMessage('Should have a valid broadcast level'), body('broadcastMessage.dismissable').isBoolean().withMessage('Should have a valid broadcast dismissable boolean'), + body('live.enabled').isBoolean().withMessage('Should have a valid live enabled boolean'), + body('live.allowReplay').isBoolean().withMessage('Should have a valid live allow replay boolean'), + body('live.maxDuration').custom(isIntOrNull).withMessage('Should have a valid live max duration'), + body('live.transcoding.enabled').isBoolean().withMessage('Should have a valid live transcoding enabled boolean'), + body('live.transcoding.threads').isInt().withMessage('Should have a valid live transcoding threads'), + body('live.transcoding.resolutions.240p').isBoolean().withMessage('Should have a valid transcoding 240p resolution enabled boolean'), + body('live.transcoding.resolutions.360p').isBoolean().withMessage('Should have a valid transcoding 360p resolution enabled boolean'), + body('live.transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'), + body('live.transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'), + body('live.transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'), + body('live.transcoding.resolutions.2160p').isBoolean().withMessage('Should have a valid transcoding 2160p resolution enabled boolean'), + body('search.remoteUri.users').isBoolean().withMessage('Should have a remote URI search for users boolean'), body('search.remoteUri.anonymous').isBoolean().withMessage('Should have a valid remote URI search for anonymous boolean'), body('search.searchIndex.enabled').isBoolean().withMessage('Should have a valid search index enabled boolean'), @@ -71,8 +85,9 @@ const customConfigUpdateValidator = [ logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body }) if (areValidationErrors(req, res)) return - if (!checkInvalidConfigIfEmailDisabled(req.body as CustomConfig, res)) return - if (!checkInvalidTranscodingConfig(req.body as CustomConfig, res)) return + if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return + if (!checkInvalidTranscodingConfig(req.body, res)) return + if (!checkInvalidLiveConfig(req.body, res)) return return next() } @@ -109,3 +124,16 @@ function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express return true } + +function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) { + if (customConfig.live.enabled === false) return true + + if (customConfig.live.allowReplay === true && customConfig.transcoding.enabled === false) { + res.status(400) + .send({ error: 'You cannot allow live replay if transcoding is not enabled' }) + .end() + return false + } + + return true +} diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts index 76ecff884..452c7fb93 100644 --- a/server/middlewares/validators/users.ts +++ b/server/middlewares/validators/users.ts @@ -497,7 +497,7 @@ export { function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) { const id = parseInt(idArg + '', 10) - return checkUserExist(() => UserModel.loadById(id, withStats), res) + return checkUserExist(() => UserModel.loadByIdWithChannels(id, withStats), res) } function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) { diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index b022b2c23..ff90e347a 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -1,5 +1,6 @@ import * as express from 'express' import { body, param, query, ValidationChain } from 'express-validator' +import { isAbleToUploadVideo } from '@server/lib/user' import { getServerActor } from '@server/models/application/application' import { MVideoFullLight } from '@server/types/models' import { ServerErrorCode, UserRight, VideoChangeOwnershipStatus, VideoPrivacy } from '../../../../shared' @@ -73,7 +74,7 @@ const videosAddValidator = getCommonVideoEditAttributes().concat([ if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) - if (await user.isAbleToUploadVideo(videoFile) === false) { + if (await isAbleToUploadVideo(user.id, videoFile.size) === false) { res.status(403) .json({ error: 'The user video quota is exceeded with this video.' }) @@ -291,7 +292,7 @@ const videosAcceptChangeOwnershipValidator = [ const user = res.locals.oauth.token.User const videoChangeOwnership = res.locals.videoChangeOwnership - const isAble = await user.isAbleToUploadVideo(videoChangeOwnership.Video.getMaxQualityFile()) + const isAble = await isAbleToUploadVideo(user.id, videoChangeOwnership.Video.getMaxQualityFile().size) if (isAble === false) { res.status(403) .json({ error: 'The user video quota is exceeded with this video.' }) diff --git a/server/models/account/user.ts b/server/models/account/user.ts index 22e6715b4..e850d1e6d 100644 --- a/server/models/account/user.ts +++ b/server/models/account/user.ts @@ -23,6 +23,7 @@ import { } from 'sequelize-typescript' import { MMyUserFormattable, + MUser, MUserDefault, MUserFormattable, MUserId, @@ -70,6 +71,7 @@ import { VideoImportModel } from '../video/video-import' import { VideoPlaylistModel } from '../video/video-playlist' import { AccountModel } from './account' import { UserNotificationSettingModel } from './user-notification-setting' +import { VideoLiveModel } from '../video/video-live' enum ScopeNames { FOR_ME_API = 'FOR_ME_API', @@ -540,7 +542,11 @@ export class UserModel extends Model { return UserModel.findAll(query) } - static loadById (id: number, withStats = false): Bluebird { + static loadById (id: number): Bluebird { + return UserModel.unscoped().findByPk(id) + } + + static loadByIdWithChannels (id: number, withStats = false): Bluebird { const scopes = [ ScopeNames.WITH_VIDEOCHANNELS ] @@ -685,26 +691,85 @@ export class UserModel extends Model { return UserModel.findOne(query) } - static getOriginalVideoFileTotalFromUser (user: MUserId) { - // Don't use sequelize because we need to use a sub query - const query = UserModel.generateUserQuotaBaseSQL({ - withSelect: true, - whereUserId: '$userId' - }) + static loadByLiveId (liveId: number): Bluebird { + const query = { + include: [ + { + attributes: [ 'id' ], + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id' ], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id' ], + model: VideoModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id', 'videoId' ], + model: VideoLiveModel.unscoped(), + required: true, + where: { + id: liveId + } + } + ] + } + ] + } + ] + } + ] + } + + return UserModel.findOne(query) + } + + static generateUserQuotaBaseSQL (options: { + whereUserId: '$userId' | '"UserModel"."id"' + withSelect: boolean + where?: string + }) { + const andWhere = options.where + ? 'AND ' + options.where + : '' + + const videoChannelJoin = 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + + 'INNER JOIN "account" ON "videoChannel"."accountId" = "account"."id" ' + + `WHERE "account"."userId" = ${options.whereUserId} ${andWhere}` + + const webtorrentFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' + + 'INNER JOIN "video" ON "videoFile"."videoId" = "video"."id" ' + + videoChannelJoin + + const hlsFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' + + 'INNER JOIN "videoStreamingPlaylist" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id ' + + 'INNER JOIN "video" ON "videoStreamingPlaylist"."videoId" = "video"."id" ' + + videoChannelJoin - return UserModel.getTotalRawQuery(query, user.id) + return 'SELECT COALESCE(SUM("size"), 0) AS "total" ' + + 'FROM (' + + `SELECT MAX("t1"."size") AS "size" FROM (${webtorrentFiles} UNION ${hlsFiles}) t1 ` + + 'GROUP BY "t1"."videoId"' + + ') t2' } - // Returns cumulative size of all video files uploaded in the last 24 hours. - static getOriginalVideoFileTotalDailyFromUser (user: MUserId) { - // Don't use sequelize because we need to use a sub query - const query = UserModel.generateUserQuotaBaseSQL({ - withSelect: true, - whereUserId: '$userId', - where: '"video"."createdAt" > now() - interval \'24 hours\'' - }) + static getTotalRawQuery (query: string, userId: number) { + const options = { + bind: { userId }, + type: QueryTypes.SELECT as QueryTypes.SELECT + } + + return UserModel.sequelize.query<{ total: string }>(query, options) + .then(([ { total } ]) => { + if (total === null) return 0 - return UserModel.getTotalRawQuery(query, user.id) + return parseInt(total, 10) + }) } static async getStats () { @@ -874,64 +939,4 @@ export class UserModel extends Model { return Object.assign(formatted, { specialPlaylists }) } - - async isAbleToUploadVideo (videoFile: { size: number }) { - if (this.videoQuota === -1 && this.videoQuotaDaily === -1) return Promise.resolve(true) - - const [ totalBytes, totalBytesDaily ] = await Promise.all([ - UserModel.getOriginalVideoFileTotalFromUser(this), - UserModel.getOriginalVideoFileTotalDailyFromUser(this) - ]) - - const uploadedTotal = videoFile.size + totalBytes - const uploadedDaily = videoFile.size + totalBytesDaily - - if (this.videoQuotaDaily === -1) return uploadedTotal < this.videoQuota - if (this.videoQuota === -1) return uploadedDaily < this.videoQuotaDaily - - return uploadedTotal < this.videoQuota && uploadedDaily < this.videoQuotaDaily - } - - private static generateUserQuotaBaseSQL (options: { - whereUserId: '$userId' | '"UserModel"."id"' - withSelect: boolean - where?: string - }) { - const andWhere = options.where - ? 'AND ' + options.where - : '' - - const videoChannelJoin = 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + - 'INNER JOIN "account" ON "videoChannel"."accountId" = "account"."id" ' + - `WHERE "account"."userId" = ${options.whereUserId} ${andWhere}` - - const webtorrentFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' + - 'INNER JOIN "video" ON "videoFile"."videoId" = "video"."id" ' + - videoChannelJoin - - const hlsFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' + - 'INNER JOIN "videoStreamingPlaylist" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id ' + - 'INNER JOIN "video" ON "videoStreamingPlaylist"."videoId" = "video"."id" ' + - videoChannelJoin - - return 'SELECT COALESCE(SUM("size"), 0) AS "total" ' + - 'FROM (' + - `SELECT MAX("t1"."size") AS "size" FROM (${webtorrentFiles} UNION ${hlsFiles}) t1 ` + - 'GROUP BY "t1"."videoId"' + - ') t2' - } - - private static getTotalRawQuery (query: string, userId: number) { - const options = { - bind: { userId }, - type: QueryTypes.SELECT as QueryTypes.SELECT - } - - return UserModel.sequelize.query<{ total: string }>(query, options) - .then(([ { total } ]) => { - if (total === null) return 0 - - return parseInt(total, 10) - }) - } } diff --git a/server/models/video/video-live.ts b/server/models/video/video-live.ts index 8608bc84c..a1dd80d3c 100644 --- a/server/models/video/video-live.ts +++ b/server/models/video/video-live.ts @@ -30,10 +30,18 @@ import { VideoBlacklistModel } from './video-blacklist' }) export class VideoLiveModel extends Model { - @AllowNull(false) + @AllowNull(true) @Column(DataType.STRING) streamKey: string + @AllowNull(false) + @Column + perpetualLive: boolean + + @AllowNull(false) + @Column + saveReplay: boolean + @CreatedAt createdAt: Date diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 35cb333ef..2882ceb7c 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -103,6 +103,9 @@ describe('Test config API validators', function () { live: { enabled: true, + allowReplay: false, + maxDuration: null, + transcoding: { enabled: true, threads: 4, diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index a46e179c2..a7f035362 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -79,6 +79,8 @@ function checkInitialConfig (server: ServerInfo, data: CustomConfig) { expect(data.transcoding.hls.enabled).to.be.true expect(data.live.enabled).to.be.false + expect(data.live.allowReplay).to.be.true + expect(data.live.maxDuration).to.equal(1000 * 3600 * 5) expect(data.live.transcoding.enabled).to.be.false expect(data.live.transcoding.threads).to.equal(2) expect(data.live.transcoding.resolutions['240p']).to.be.false @@ -162,6 +164,8 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.transcoding.webtorrent.enabled).to.be.true expect(data.live.enabled).to.be.true + expect(data.live.allowReplay).to.be.false + expect(data.live.maxDuration).to.equal(5000) expect(data.live.transcoding.enabled).to.be.true expect(data.live.transcoding.threads).to.equal(4) expect(data.live.transcoding.resolutions['240p']).to.be.true @@ -324,6 +328,8 @@ describe('Test config', function () { }, live: { enabled: true, + allowReplay: false, + maxDuration: 5000, transcoding: { enabled: true, threads: 4, diff --git a/shared/extra-utils/server/config.ts b/shared/extra-utils/server/config.ts index 3606976bd..bb7e23d54 100644 --- a/shared/extra-utils/server/config.ts +++ b/shared/extra-utils/server/config.ts @@ -128,6 +128,8 @@ function updateCustomSubConfig (url: string, token: string, newConfig: DeepParti }, live: { enabled: true, + allowReplay: false, + maxDuration: null, transcoding: { enabled: true, threads: 4, diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts index e609d1a33..11b2ef2eb 100644 --- a/shared/models/server/custom-config.model.ts +++ b/shared/models/server/custom-config.model.ts @@ -98,6 +98,9 @@ export interface CustomConfig { live: { enabled: boolean + allowReplay: boolean + maxDuration: number + transcoding: { enabled: boolean threads: number diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts index 77694a627..1563d848e 100644 --- a/shared/models/server/server-config.model.ts +++ b/shared/models/server/server-config.model.ts @@ -101,6 +101,9 @@ export interface ServerConfig { live: { enabled: boolean + maxDuration: number + allowReplay: boolean + transcoding: { enabled: boolean -- 2.41.0