From bee0abffff73804d816b90c7fd599e0a51c09d61 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 28 Aug 2018 02:01:35 -0500 Subject: Implement daily upload limit (#956) * Implement daily upload limit (ref #652) * remove duplicate code * review fixes * fix tests? * whitespace fixes, finish leftover todo * fix tests * added some new tests * use different config value for tests * remove todo --- server/controllers/api/config.ts | 7 ++- server/controllers/api/users/index.ts | 7 ++- server/controllers/api/users/me.ts | 4 +- server/helpers/custom-validators/users.ts | 5 ++ server/initializers/checker.ts | 2 +- server/initializers/constants.ts | 4 +- server/initializers/installer.ts | 3 +- .../migrations/0260-upload_quota_daily.ts | 23 ++++++++ server/middlewares/validators/users.ts | 5 +- server/models/account/user.ts | 68 +++++++++++++++++++--- server/tests/api/check-params/config.ts | 3 +- server/tests/api/check-params/users.ts | 56 +++++++++++++++++- server/tests/api/server/config.ts | 5 +- server/tests/utils/server/config.ts | 3 +- server/tests/utils/users/users.ts | 6 +- 15 files changed, 177 insertions(+), 24 deletions(-) create mode 100644 server/initializers/migrations/0260-upload_quota_daily.ts (limited to 'server') diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index b25f739bb..3fd355e6d 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -103,7 +103,8 @@ async function getConfig (req: express.Request, res: express.Response, next: exp } }, user: { - videoQuota: CONFIG.USER.VIDEO_QUOTA + videoQuota: CONFIG.USER.VIDEO_QUOTA, + videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY } } @@ -154,6 +155,7 @@ async function updateCustomConfig (req: express.Request, res: express.Response, toUpdate.cache.captions.size = parseInt('' + toUpdate.cache.captions.size, 10) toUpdate.signup.limit = parseInt('' + toUpdate.signup.limit, 10) toUpdate.user.videoQuota = parseInt('' + toUpdate.user.videoQuota, 10) + toUpdate.user.videoQuotaDaily = parseInt('' + toUpdate.user.videoQuotaDaily, 10) toUpdate.transcoding.threads = parseInt('' + toUpdate.transcoding.threads, 10) // camelCase to snake_case key @@ -223,7 +225,8 @@ function customConfig (): CustomConfig { email: CONFIG.ADMIN.EMAIL }, user: { - videoQuota: CONFIG.USER.VIDEO_QUOTA + videoQuota: CONFIG.USER.VIDEO_QUOTA, + videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY }, transcoding: { enabled: CONFIG.TRANSCODING.ENABLED, diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts index 608d439ac..25d51ae5e 100644 --- a/server/controllers/api/users/index.ts +++ b/server/controllers/api/users/index.ts @@ -134,7 +134,8 @@ async function createUser (req: express.Request, res: express.Response) { nsfwPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, autoPlayVideo: true, role: body.role, - videoQuota: body.videoQuota + videoQuota: body.videoQuota, + videoQuotaDaily: body.videoQuotaDaily }) const { user, account } = await createUserAccountAndChannel(userToCreate) @@ -163,7 +164,8 @@ async function registerUser (req: express.Request, res: express.Response) { nsfwPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, autoPlayVideo: true, role: UserRole.USER, - videoQuota: CONFIG.USER.VIDEO_QUOTA + videoQuota: CONFIG.USER.VIDEO_QUOTA, + videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY }) const { user } = await createUserAccountAndChannel(userToCreate) @@ -219,6 +221,7 @@ async function updateUser (req: express.Request, res: express.Response, next: ex if (body.email !== undefined) userToUpdate.email = body.email if (body.videoQuota !== undefined) userToUpdate.videoQuota = body.videoQuota + if (body.videoQuotaDaily !== undefined) userToUpdate.videoQuotaDaily = body.videoQuotaDaily if (body.role !== undefined) userToUpdate.role = body.role const user = await userToUpdate.save() diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index 000c706b5..0f18b42f9 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -283,9 +283,11 @@ async function getUserVideoQuotaUsed (req: express.Request, res: express.Respons // We did not load channels in res.locals.user const user = await UserModel.loadByUsernameAndPopulateChannels(res.locals.oauth.token.user.username) const videoQuotaUsed = await UserModel.getOriginalVideoFileTotalFromUser(user) + const videoQuotaUsedDaily = await UserModel.getOriginalVideoFileTotalDailyFromUser(user) const data: UserVideoQuota = { - videoQuotaUsed + videoQuotaUsed, + videoQuotaUsedDaily } return res.json(data) } diff --git a/server/helpers/custom-validators/users.ts b/server/helpers/custom-validators/users.ts index c3cdefd4e..8d6247e41 100644 --- a/server/helpers/custom-validators/users.ts +++ b/server/helpers/custom-validators/users.ts @@ -15,6 +15,10 @@ function isUserVideoQuotaValid (value: string) { return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA) } +function isUserVideoQuotaDailyValid (value: string) { + return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA_DAILY) +} + function isUserUsernameValid (value: string) { const max = USERS_CONSTRAINTS_FIELDS.USERNAME.max const min = USERS_CONSTRAINTS_FIELDS.USERNAME.min @@ -66,6 +70,7 @@ export { isUserBlockedReasonValid, isUserRoleValid, isUserVideoQuotaValid, + isUserVideoQuotaDailyValid, isUserUsernameValid, isUserNSFWPolicyValid, isUserAutoPlayVideoValid, diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 608123607..916e9067e 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts @@ -47,7 +47,7 @@ function checkMissedConfig () { 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', 'log.level', - 'user.video_quota', + 'user.video_quota', 'user.video_quota_daily', 'cache.previews.size', 'admin.email', 'signup.enabled', 'signup.limit', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', 'transcoding.enabled', 'transcoding.threads', diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index a0dd78f42..4111d04ec 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -202,7 +202,8 @@ const CONFIG = { } }, USER: { - get VIDEO_QUOTA () { return config.get('user.video_quota') } + get VIDEO_QUOTA () { return config.get('user.video_quota') }, + get VIDEO_QUOTA_DAILY () { return config.get('user.video_quota_daily') } }, TRANSCODING: { get ENABLED () { return config.get('transcoding.enabled') }, @@ -263,6 +264,7 @@ const CONSTRAINTS_FIELDS = { USERNAME: { min: 3, max: 20 }, // Length PASSWORD: { min: 6, max: 255 }, // Length VIDEO_QUOTA: { min: -1 }, + VIDEO_QUOTA_DAILY: { min: -1 }, BLOCKED_REASON: { min: 3, max: 250 } // Length }, VIDEO_ABUSES: { diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index e319164e4..d4aaec8fe 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts @@ -123,7 +123,8 @@ async function createOAuthAdminIfNotExist () { password, role, nsfwPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, - videoQuota: -1 + videoQuota: -1, + videoQuotaDaily: -1 } const user = new UserModel(userData) diff --git a/server/initializers/migrations/0260-upload_quota_daily.ts b/server/initializers/migrations/0260-upload_quota_daily.ts new file mode 100644 index 000000000..d25154ba6 --- /dev/null +++ b/server/initializers/migrations/0260-upload_quota_daily.ts @@ -0,0 +1,23 @@ +import * as Sequelize from 'sequelize' +import { CONSTRAINTS_FIELDS } from '../constants' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + { + const data = { + type: Sequelize.BIGINT, + allowNull: false, + defaultValue: -1 + } + await utils.queryInterface.addColumn('user', 'videoQuotaDaily', data) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { up, down } diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts index c8baf22e2..6c5e783e9 100644 --- a/server/middlewares/validators/users.ts +++ b/server/middlewares/validators/users.ts @@ -12,7 +12,8 @@ import { isUserPasswordValid, isUserRoleValid, isUserUsernameValid, - isUserVideoQuotaValid + isUserVideoQuotaValid, + isUserVideoQuotaDailyValid } from '../../helpers/custom-validators/users' import { isVideoExist } from '../../helpers/custom-validators/videos' import { logger } from '../../helpers/logger' @@ -27,6 +28,7 @@ const usersAddValidator = [ body('password').custom(isUserPasswordValid).withMessage('Should have a valid password'), body('email').isEmail().withMessage('Should have a valid email'), body('videoQuota').custom(isUserVideoQuotaValid).withMessage('Should have a valid user quota'), + body('videoQuotaDaily').custom(isUserVideoQuotaDailyValid).withMessage('Should have a valid daily user quota'), body('role').custom(isUserRoleValid).withMessage('Should have a valid role'), async (req: express.Request, res: express.Response, next: express.NextFunction) => { @@ -112,6 +114,7 @@ const usersUpdateValidator = [ param('id').isInt().not().isEmpty().withMessage('Should have a valid id'), body('email').optional().isEmail().withMessage('Should have a valid email attribute'), body('videoQuota').optional().custom(isUserVideoQuotaValid).withMessage('Should have a valid user quota'), + body('videoQuotaDaily').optional().custom(isUserVideoQuotaDailyValid).withMessage('Should have a valid daily user quota'), body('role').optional().custom(isUserRoleValid).withMessage('Should have a valid role'), async (req: express.Request, res: express.Response, next: express.NextFunction) => { diff --git a/server/models/account/user.ts b/server/models/account/user.ts index 0150df4ce..178012eae 100644 --- a/server/models/account/user.ts +++ b/server/models/account/user.ts @@ -27,7 +27,8 @@ import { isUserPasswordValid, isUserRoleValid, isUserUsernameValid, - isUserVideoQuotaValid + isUserVideoQuotaValid, + isUserVideoQuotaDailyValid } from '../../helpers/custom-validators/users' import { comparePassword, cryptPassword } from '../../helpers/peertube-crypto' import { OAuthTokenModel } from '../oauth/oauth-token' @@ -124,6 +125,11 @@ export class UserModel extends Model { @Column(DataType.BIGINT) videoQuota: number + @AllowNull(false) + @Is('UserVideoQuotaDaily', value => throwIfNotValid(value, isUserVideoQuotaDailyValid, 'video quota daily')) + @Column(DataType.BIGINT) + videoQuotaDaily: number + @CreatedAt createdAt: Date @@ -271,7 +277,32 @@ export class UserModel extends Model { 'INNER JOIN "video" ON "videoFile"."videoId" = "video"."id" ' + 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + 'INNER JOIN "account" ON "videoChannel"."accountId" = "account"."id" ' + - 'WHERE "account"."userId" = $userId GROUP BY "video"."id") t' + 'WHERE "account"."userId" = $userId ' + + 'GROUP BY "video"."id") t' + + const options = { + bind: { userId: user.id }, + type: Sequelize.QueryTypes.SELECT + } + return UserModel.sequelize.query(query, options) + .then(([ { total } ]) => { + if (total === null) return 0 + + return parseInt(total, 10) + }) + } + + // Returns comulative size of all video files uploaded in the last 24 hours. + static getOriginalVideoFileTotalDailyFromUser (user: UserModel) { + // Don't use sequelize because we need to use a sub query + const query = 'SELECT SUM("size") AS "total" FROM ' + + '(SELECT MAX("videoFile"."size") AS "size" FROM "videoFile" ' + + 'INNER JOIN "video" ON "videoFile"."videoId" = "video"."id" ' + + 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + + 'INNER JOIN "account" ON "videoChannel"."accountId" = "account"."id" ' + + 'WHERE "account"."userId" = $userId ' + + 'AND "video"."createdAt" > now() - interval \'24 hours\'' + + 'GROUP BY "video"."id") t' const options = { bind: { userId: user.id }, @@ -303,6 +334,7 @@ export class UserModel extends Model { toFormattedJSON (): User { const videoQuotaUsed = this.get('videoQuotaUsed') + const videoQuotaUsedDaily = this.get('videoQuotaUsedDaily') const json = { id: this.id, @@ -313,12 +345,18 @@ export class UserModel extends Model { role: this.role, roleLabel: USER_ROLE_LABELS[ this.role ], videoQuota: this.videoQuota, + videoQuotaDaily: this.videoQuotaDaily, createdAt: this.createdAt, blocked: this.blocked, blockedReason: this.blockedReason, account: this.Account.toFormattedJSON(), videoChannels: [], - videoQuotaUsed: videoQuotaUsed !== undefined ? parseInt(videoQuotaUsed, 10) : undefined + videoQuotaUsed: videoQuotaUsed !== undefined + ? parseInt(videoQuotaUsed, 10) + : undefined, + videoQuotaUsedDaily: videoQuotaUsedDaily !== undefined + ? parseInt(videoQuotaUsedDaily, 10) + : undefined } if (Array.isArray(this.Account.VideoChannels) === true) { @@ -335,12 +373,24 @@ export class UserModel extends Model { return json } - isAbleToUploadVideo (videoFile: { size: number }) { - if (this.videoQuota === -1) return Promise.resolve(true) + async isAbleToUploadVideo (videoFile: { size: number }) { + if (this.videoQuota === -1 && this.videoQuotaDaily === -1) return Promise.resolve(true) - return UserModel.getOriginalVideoFileTotalFromUser(this) - .then(totalBytes => { - return (videoFile.size + totalBytes) < this.videoQuota - }) + const [ totalBytes, totalBytesDaily ] = await Promise.all([ + UserModel.getOriginalVideoFileTotalFromUser(this), + UserModel.getOriginalVideoFileTotalDailyFromUser(this) + ]) + + const uploadedTotal = videoFile.size + totalBytes + const uploadedDaily = videoFile.size + totalBytesDaily + if (this.videoQuotaDaily === -1) { + return uploadedTotal < this.videoQuota + } + if (this.videoQuota === -1) { + return uploadedDaily < this.videoQuotaDaily + } + + return (uploadedTotal < this.videoQuota) && + (uploadedDaily < this.videoQuotaDaily) } } diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index b26dfa252..ecfb76d47 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -48,7 +48,8 @@ describe('Test config API validators', function () { email: 'superadmin1@example.com' }, user: { - videoQuota: 5242881 + videoQuota: 5242881, + videoQuotaDaily: 318742 }, transcoding: { enabled: true, diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts index b3fb61f6c..8b2ed1b04 100644 --- a/server/tests/api/check-params/users.ts +++ b/server/tests/api/check-params/users.ts @@ -94,6 +94,7 @@ describe('Test users API validators', function () { email: 'test@example.com', password: 'my super password', videoQuota: -1, + videoQuotaDaily: -1, role: UserRole.USER } @@ -173,12 +174,24 @@ describe('Test users API validators', function () { await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) }) + it('Should fail without a videoQuotaDaily', async function () { + const fields = omit(baseCorrectParams, 'videoQuotaDaily') + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + it('Should fail with an invalid videoQuota', async function () { const fields = immutableAssign(baseCorrectParams, { videoQuota: -5 }) await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) }) + it('Should fail with an invalid videoQuotaDaily', async function () { + const fields = immutableAssign(baseCorrectParams, { videoQuotaDaily: -7 }) + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + it('Should fail without a user role', async function () { const fields = omit(baseCorrectParams, 'role') @@ -607,7 +620,7 @@ describe('Test users API validators', function () { }) describe('When having a video quota', function () { - it('Should fail with a user having too many video', async function () { + it('Should fail with a user having too many videos', async function () { await updateUser({ url: server.url, userId: rootId, @@ -618,7 +631,7 @@ describe('Test users API validators', function () { await uploadVideo(server.url, server.accessToken, {}, 403) }) - it('Should fail with a registered user having too many video', async function () { + it('Should fail with a registered user having too many videos', async function () { this.timeout(30000) const user = { @@ -663,6 +676,45 @@ describe('Test users API validators', function () { }) }) + describe('When having a daily video quota', function () { + it('Should fail with a user having too many videos', async function () { + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuotaDaily: 42 + }) + + await uploadVideo(server.url, server.accessToken, {}, 403) + }) + }) + + describe('When having an absolute and daily video quota', function () { + it('Should fail if exceeding total quota', async function () { + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 42, + videoQuotaDaily: 1024 * 1024 * 1024 + }) + + await uploadVideo(server.url, server.accessToken, {}, 403) + }) + + it('Should fail if exceeding daily quota', async function () { + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 1024 * 1024 * 1024, + videoQuotaDaily: 42 + }) + + await uploadVideo(server.url, server.accessToken, {}, 403) + }) + }) + describe('When asking a password reset', function () { const path = '/api/v1/users/ask-reset-password' diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index f9805b6ea..8a5f27c34 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -37,6 +37,7 @@ function checkInitialConfig (data: CustomConfig) { expect(data.signup.limit).to.equal(4) expect(data.admin.email).to.equal('admin1@example.com') expect(data.user.videoQuota).to.equal(5242880) + expect(data.user.videoQuotaDaily).to.equal(318742) expect(data.transcoding.enabled).to.be.false expect(data.transcoding.threads).to.equal(2) expect(data.transcoding.resolutions['240p']).to.be.true @@ -65,6 +66,7 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.signup.limit).to.equal(5) expect(data.admin.email).to.equal('superadmin1@example.com') expect(data.user.videoQuota).to.equal(5242881) + expect(data.user.videoQuotaDaily).to.equal(318742) expect(data.transcoding.enabled).to.be.true expect(data.transcoding.threads).to.equal(1) expect(data.transcoding.resolutions['240p']).to.be.false @@ -152,7 +154,8 @@ describe('Test config', function () { email: 'superadmin1@example.com' }, user: { - videoQuota: 5242881 + videoQuota: 5242881, + videoQuotaDaily: 318742 }, transcoding: { enabled: true, diff --git a/server/tests/utils/server/config.ts b/server/tests/utils/server/config.ts index d6ac3ef8a..799c31ae5 100644 --- a/server/tests/utils/server/config.ts +++ b/server/tests/utils/server/config.ts @@ -80,7 +80,8 @@ function updateCustomSubConfig (url: string, token: string, newConfig: any) { email: 'superadmin1@example.com' }, user: { - videoQuota: 5242881 + videoQuota: 5242881, + videoQuotaDaily: 318742 }, transcoding: { enabled: true, diff --git a/server/tests/utils/users/users.ts b/server/tests/utils/users/users.ts index f786de6e3..5dba34b69 100644 --- a/server/tests/utils/users/users.ts +++ b/server/tests/utils/users/users.ts @@ -10,6 +10,7 @@ function createUser ( username: string, password: string, videoQuota = 1000000, + videoQuotaDaily = -1, role: UserRole = UserRole.USER, specialStatus = 200 ) { @@ -19,7 +20,8 @@ function createUser ( password, role, email: username + '@example.com', - videoQuota + videoQuota, + videoQuotaDaily } return request(url) @@ -202,6 +204,7 @@ function updateUser (options: { accessToken: string, email?: string, videoQuota?: number, + videoQuotaDaily?: number, role?: UserRole }) { const path = '/api/v1/users/' + options.userId @@ -209,6 +212,7 @@ function updateUser (options: { const toSend = {} if (options.email !== undefined && options.email !== null) toSend['email'] = options.email if (options.videoQuota !== undefined && options.videoQuota !== null) toSend['videoQuota'] = options.videoQuota + if (options.videoQuotaDaily !== undefined && options.videoQuotaDaily !== null) toSend['videoQuotaDaily'] = options.videoQuotaDaily if (options.role !== undefined && options.role !== null) toSend['role'] = options.role return makePutBodyRequest({ -- cgit v1.2.3