Implement daily upload limit (#956)

* Implement daily upload limit (ref #652)

* remove duplicate code

* review fixes

* fix tests?

* whitespace fixes, finish leftover todo

* fix tests

* added some new tests

* use different config value for tests

* remove todo

15 files changed, 177 insertions, 24 deletions

diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index b25f739bb..3fd355e6d 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -103,7 +103,8 @@ async function getConfig (req: express.Request, res: express.Response, next: exp
       }
     },
     user: {
-      videoQuota: CONFIG.USER.VIDEO_QUOTA
+      videoQuota: CONFIG.USER.VIDEO_QUOTA,
+      videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY
     }
   }
 
@@ -154,6 +155,7 @@ async function updateCustomConfig (req: express.Request, res: express.Response,
   toUpdate.cache.captions.size = parseInt('' + toUpdate.cache.captions.size, 10)
   toUpdate.signup.limit = parseInt('' + toUpdate.signup.limit, 10)
   toUpdate.user.videoQuota = parseInt('' + toUpdate.user.videoQuota, 10)
+  toUpdate.user.videoQuotaDaily = parseInt('' + toUpdate.user.videoQuotaDaily, 10)
   toUpdate.transcoding.threads = parseInt('' + toUpdate.transcoding.threads, 10)
 
   // camelCase to snake_case key
@@ -223,7 +225,8 @@ function customConfig (): CustomConfig {
       email: CONFIG.ADMIN.EMAIL
     },
     user: {
-      videoQuota: CONFIG.USER.VIDEO_QUOTA
+      videoQuota: CONFIG.USER.VIDEO_QUOTA,
+      videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY
     },
     transcoding: {
       enabled: CONFIG.TRANSCODING.ENABLED,
diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts
index 608d439ac..25d51ae5e 100644
--- a/server/controllers/api/users/index.ts
+++ b/server/controllers/api/users/index.ts
@@ -134,7 +134,8 @@ async function createUser (req: express.Request, res: express.Response) {
     nsfwPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY,
     autoPlayVideo: true,
     role: body.role,
-    videoQuota: body.videoQuota
+    videoQuota: body.videoQuota,
+    videoQuotaDaily: body.videoQuotaDaily
   })
 
   const { user, account } = await createUserAccountAndChannel(userToCreate)
@@ -163,7 +164,8 @@ async function registerUser (req: express.Request, res: express.Response) {
     nsfwPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY,
     autoPlayVideo: true,
     role: UserRole.USER,
-    videoQuota: CONFIG.USER.VIDEO_QUOTA
+    videoQuota: CONFIG.USER.VIDEO_QUOTA,
+    videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY
   })
 
   const { user } = await createUserAccountAndChannel(userToCreate)
@@ -219,6 +221,7 @@ async function updateUser (req: express.Request, res: express.Response, next: ex
 
   if (body.email !== undefined) userToUpdate.email = body.email
   if (body.videoQuota !== undefined) userToUpdate.videoQuota = body.videoQuota
+  if (body.videoQuotaDaily !== undefined) userToUpdate.videoQuotaDaily = body.videoQuotaDaily
   if (body.role !== undefined) userToUpdate.role = body.role
 
   const user = await userToUpdate.save()
diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts
index 000c706b5..0f18b42f9 100644
--- a/server/controllers/api/users/me.ts
+++ b/server/controllers/api/users/me.ts
@@ -283,9 +283,11 @@ async function getUserVideoQuotaUsed (req: express.Request, res: express.Respons
   // We did not load channels in res.locals.user
   const user = await UserModel.loadByUsernameAndPopulateChannels(res.locals.oauth.token.user.username)
   const videoQuotaUsed = await UserModel.getOriginalVideoFileTotalFromUser(user)
+  const videoQuotaUsedDaily = await UserModel.getOriginalVideoFileTotalDailyFromUser(user)
 
   const data: UserVideoQuota = {
-    videoQuotaUsed
+    videoQuotaUsed,
+    videoQuotaUsedDaily
   }
   return res.json(data)
 }
diff --git a/server/helpers/custom-validators/users.ts b/server/helpers/custom-validators/users.ts
index c3cdefd4e..8d6247e41 100644
--- a/server/helpers/custom-validators/users.ts
+++ b/server/helpers/custom-validators/users.ts
@@ -15,6 +15,10 @@ function isUserVideoQuotaValid (value: string) {
   return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA)
 }
 
+function isUserVideoQuotaDailyValid (value: string) {
+  return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA_DAILY)
+}
+
 function isUserUsernameValid (value: string) {
   const max = USERS_CONSTRAINTS_FIELDS.USERNAME.max
   const min = USERS_CONSTRAINTS_FIELDS.USERNAME.min
@@ -66,6 +70,7 @@ export {
   isUserBlockedReasonValid,
   isUserRoleValid,
   isUserVideoQuotaValid,
+  isUserVideoQuotaDailyValid,
   isUserUsernameValid,
   isUserNSFWPolicyValid,
   isUserAutoPlayVideoValid,
diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts
index 608123607..916e9067e 100644
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker.ts
@@ -47,7 +47,7 @@ function checkMissedConfig () {
     'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
     'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
     'log.level',
-    'user.video_quota',
+    'user.video_quota', 'user.video_quota_daily',
     'cache.previews.size', 'admin.email',
     'signup.enabled', 'signup.limit', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
     'transcoding.enabled', 'transcoding.threads',
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index a0dd78f42..4111d04ec 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -202,7 +202,8 @@ const CONFIG = {
     }
   },
   USER: {
-    get VIDEO_QUOTA () { return config.get<number>('user.video_quota') }
+    get VIDEO_QUOTA () { return config.get<number>('user.video_quota') },
+    get VIDEO_QUOTA_DAILY () { return config.get<number>('user.video_quota_daily') }
   },
   TRANSCODING: {
     get ENABLED () { return config.get<boolean>('transcoding.enabled') },
@@ -263,6 +264,7 @@ const CONSTRAINTS_FIELDS = {
     USERNAME: { min: 3, max: 20 }, // Length
     PASSWORD: { min: 6, max: 255 }, // Length
     VIDEO_QUOTA: { min: -1 },
+    VIDEO_QUOTA_DAILY: { min: -1 },
     BLOCKED_REASON: { min: 3, max: 250 } // Length
   },
   VIDEO_ABUSES: {
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index e319164e4..d4aaec8fe 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -123,7 +123,8 @@ async function createOAuthAdminIfNotExist () {
     password,
     role,
     nsfwPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY,
-    videoQuota: -1
+    videoQuota: -1,
+    videoQuotaDaily: -1
   }
   const user = new UserModel(userData)
 
diff --git a/server/initializers/migrations/0260-upload_quota_daily.ts b/server/initializers/migrations/0260-upload_quota_daily.ts
new file mode 100644
index 000000000..d25154ba6
--- /dev/null
+++ b/server/initializers/migrations/0260-upload_quota_daily.ts
@@ -0,0 +1,23 @@
+import * as Sequelize from 'sequelize'
+import { CONSTRAINTS_FIELDS } from '../constants'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+}): Promise<any> {
+  {
+    const data = {
+      type: Sequelize.BIGINT,
+      allowNull: false,
+      defaultValue: -1
+    }
+    await utils.queryInterface.addColumn('user', 'videoQuotaDaily', data)
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export { up, down }
diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts
index c8baf22e2..6c5e783e9 100644
--- a/server/middlewares/validators/users.ts
+++ b/server/middlewares/validators/users.ts
@@ -12,7 +12,8 @@ import {
   isUserPasswordValid,
   isUserRoleValid,
   isUserUsernameValid,
-  isUserVideoQuotaValid
+  isUserVideoQuotaValid,
+  isUserVideoQuotaDailyValid
 } from '../../helpers/custom-validators/users'
 import { isVideoExist } from '../../helpers/custom-validators/videos'
 import { logger } from '../../helpers/logger'
@@ -27,6 +28,7 @@ const usersAddValidator = [
   body('password').custom(isUserPasswordValid).withMessage('Should have a valid password'),
   body('email').isEmail().withMessage('Should have a valid email'),
   body('videoQuota').custom(isUserVideoQuotaValid).withMessage('Should have a valid user quota'),
+  body('videoQuotaDaily').custom(isUserVideoQuotaDailyValid).withMessage('Should have a valid daily user quota'),
   body('role').custom(isUserRoleValid).withMessage('Should have a valid role'),
 
   async (req: express.Request, res: express.Response, next: express.NextFunction) => {
@@ -112,6 +114,7 @@ const usersUpdateValidator = [
   param('id').isInt().not().isEmpty().withMessage('Should have a valid id'),
   body('email').optional().isEmail().withMessage('Should have a valid email attribute'),
   body('videoQuota').optional().custom(isUserVideoQuotaValid).withMessage('Should have a valid user quota'),
+  body('videoQuotaDaily').optional().custom(isUserVideoQuotaDailyValid).withMessage('Should have a valid daily user quota'),
   body('role').optional().custom(isUserRoleValid).withMessage('Should have a valid role'),
 
   async (req: express.Request, res: express.Response, next: express.NextFunction) => {
diff --git a/server/models/account/user.ts b/server/models/account/user.ts
index 0150df4ce..178012eae 100644
--- a/server/models/account/user.ts
+++ b/server/models/account/user.ts
@@ -27,7 +27,8 @@ import {
   isUserPasswordValid,
   isUserRoleValid,
   isUserUsernameValid,
-  isUserVideoQuotaValid
+  isUserVideoQuotaValid,
+  isUserVideoQuotaDailyValid
 } from '../../helpers/custom-validators/users'
 import { comparePassword, cryptPassword } from '../../helpers/peertube-crypto'
 import { OAuthTokenModel } from '../oauth/oauth-token'
@@ -124,6 +125,11 @@ export class UserModel extends Model<UserModel> {
   @Column(DataType.BIGINT)
   videoQuota: number
 
+  @AllowNull(false)
+  @Is('UserVideoQuotaDaily', value => throwIfNotValid(value, isUserVideoQuotaDailyValid, 'video quota daily'))
+  @Column(DataType.BIGINT)
+  videoQuotaDaily: number
+
   @CreatedAt
   createdAt: Date
 
@@ -271,7 +277,32 @@ export class UserModel extends Model<UserModel> {
       'INNER JOIN "video" ON "videoFile"."videoId" = "video"."id" ' +
       'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' +
       'INNER JOIN "account" ON "videoChannel"."accountId" = "account"."id" ' +
-      'WHERE "account"."userId" = $userId GROUP BY "video"."id") t'
+      'WHERE "account"."userId" = $userId ' +
+      'GROUP BY "video"."id") t'
+
+    const options = {
+      bind: { userId: user.id },
+      type: Sequelize.QueryTypes.SELECT
+    }
+    return UserModel.sequelize.query(query, options)
+      .then(([ { total } ]) => {
+        if (total === null) return 0
+
+        return parseInt(total, 10)
+      })
+  }
+
+  // Returns comulative size of all video files uploaded in the last 24 hours.
+  static getOriginalVideoFileTotalDailyFromUser (user: UserModel) {
+    // Don't use sequelize because we need to use a sub query
+    const query = 'SELECT SUM("size") AS "total" FROM ' +
+      '(SELECT MAX("videoFile"."size") AS "size" FROM "videoFile" ' +
+      'INNER JOIN "video" ON "videoFile"."videoId" = "video"."id" ' +
+      'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' +
+      'INNER JOIN "account" ON "videoChannel"."accountId" = "account"."id" ' +
+      'WHERE "account"."userId" = $userId ' +
+      'AND "video"."createdAt" > now() - interval \'24 hours\'' +
+      'GROUP BY "video"."id") t'
 
     const options = {
       bind: { userId: user.id },
@@ -303,6 +334,7 @@ export class UserModel extends Model<UserModel> {
 
   toFormattedJSON (): User {
     const videoQuotaUsed = this.get('videoQuotaUsed')
+    const videoQuotaUsedDaily = this.get('videoQuotaUsedDaily')
 
     const json = {
       id: this.id,
@@ -313,12 +345,18 @@ export class UserModel extends Model<UserModel> {
       role: this.role,
       roleLabel: USER_ROLE_LABELS[ this.role ],
       videoQuota: this.videoQuota,
+      videoQuotaDaily: this.videoQuotaDaily,
       createdAt: this.createdAt,
       blocked: this.blocked,
       blockedReason: this.blockedReason,
       account: this.Account.toFormattedJSON(),
       videoChannels: [],
-      videoQuotaUsed: videoQuotaUsed !== undefined ? parseInt(videoQuotaUsed, 10) : undefined
+      videoQuotaUsed: videoQuotaUsed !== undefined
+            ? parseInt(videoQuotaUsed, 10)
+            : undefined,
+      videoQuotaUsedDaily: videoQuotaUsedDaily !== undefined
+            ? parseInt(videoQuotaUsedDaily, 10)
+            : undefined
     }
 
     if (Array.isArray(this.Account.VideoChannels) === true) {
@@ -335,12 +373,24 @@ export class UserModel extends Model<UserModel> {
     return json
   }
 
-  isAbleToUploadVideo (videoFile: { size: number }) {
-    if (this.videoQuota === -1) return Promise.resolve(true)
+  async isAbleToUploadVideo (videoFile: { size: number }) {
+    if (this.videoQuota === -1 && this.videoQuotaDaily === -1) return Promise.resolve(true)
 
-    return UserModel.getOriginalVideoFileTotalFromUser(this)
-      .then(totalBytes => {
-        return (videoFile.size + totalBytes) < this.videoQuota
-      })
+    const [ totalBytes, totalBytesDaily ] = await Promise.all([
+      UserModel.getOriginalVideoFileTotalFromUser(this),
+      UserModel.getOriginalVideoFileTotalDailyFromUser(this)
+    ])
+
+    const uploadedTotal = videoFile.size + totalBytes
+    const uploadedDaily = videoFile.size + totalBytesDaily
+    if (this.videoQuotaDaily === -1) {
+      return uploadedTotal < this.videoQuota
+    }
+    if (this.videoQuota === -1) {
+      return uploadedDaily < this.videoQuotaDaily
+    }
+
+    return (uploadedTotal < this.videoQuota) &&
+        (uploadedDaily < this.videoQuotaDaily)
   }
 }
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index b26dfa252..ecfb76d47 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -48,7 +48,8 @@ describe('Test config API validators', function () {
       email: 'superadmin1@example.com'
     },
     user: {
-      videoQuota: 5242881
+      videoQuota: 5242881,
+      videoQuotaDaily: 318742
     },
     transcoding: {
       enabled: true,
diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts
index b3fb61f6c..8b2ed1b04 100644
--- a/server/tests/api/check-params/users.ts
+++ b/server/tests/api/check-params/users.ts
@@ -94,6 +94,7 @@ describe('Test users API validators', function () {
       email: 'test@example.com',
       password: 'my super password',
       videoQuota: -1,
+      videoQuotaDaily: -1,
       role: UserRole.USER
     }
 
@@ -173,12 +174,24 @@ describe('Test users API validators', function () {
       await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
     })
 
+    it('Should fail without a videoQuotaDaily', async function () {
+      const fields = omit(baseCorrectParams, 'videoQuotaDaily')
+
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
     it('Should fail with an invalid videoQuota', async function () {
       const fields = immutableAssign(baseCorrectParams, { videoQuota: -5 })
 
       await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
     })
 
+    it('Should fail with an invalid videoQuotaDaily', async function () {
+      const fields = immutableAssign(baseCorrectParams, { videoQuotaDaily: -7 })
+
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
     it('Should fail without a user role', async function () {
       const fields = omit(baseCorrectParams, 'role')
 
@@ -607,7 +620,7 @@ describe('Test users API validators', function () {
   })
 
   describe('When having a video quota', function () {
-    it('Should fail with a user having too many video', async function () {
+    it('Should fail with a user having too many videos', async function () {
       await updateUser({
         url: server.url,
         userId: rootId,
@@ -618,7 +631,7 @@ describe('Test users API validators', function () {
       await uploadVideo(server.url, server.accessToken, {}, 403)
     })
 
-    it('Should fail with a registered user having too many video', async function () {
+    it('Should fail with a registered user having too many videos', async function () {
       this.timeout(30000)
 
       const user = {
@@ -663,6 +676,45 @@ describe('Test users API validators', function () {
     })
   })
 
+  describe('When having a daily video quota', function () {
+    it('Should fail with a user having too many videos', async function () {
+      await updateUser({
+        url: server.url,
+        userId: rootId,
+        accessToken: server.accessToken,
+        videoQuotaDaily: 42
+      })
+
+      await uploadVideo(server.url, server.accessToken, {}, 403)
+    })
+  })
+
+  describe('When having an absolute and daily video quota', function () {
+    it('Should fail if exceeding total quota', async function () {
+      await updateUser({
+        url: server.url,
+        userId: rootId,
+        accessToken: server.accessToken,
+        videoQuota: 42,
+        videoQuotaDaily: 1024 * 1024 * 1024
+      })
+
+      await uploadVideo(server.url, server.accessToken, {}, 403)
+    })
+
+    it('Should fail if exceeding daily quota', async function () {
+      await updateUser({
+        url: server.url,
+        userId: rootId,
+        accessToken: server.accessToken,
+        videoQuota: 1024 * 1024 * 1024,
+        videoQuotaDaily: 42
+      })
+
+      await uploadVideo(server.url, server.accessToken, {}, 403)
+    })
+  })
+
   describe('When asking a password reset', function () {
     const path = '/api/v1/users/ask-reset-password'
 
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index f9805b6ea..8a5f27c34 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -37,6 +37,7 @@ function checkInitialConfig (data: CustomConfig) {
   expect(data.signup.limit).to.equal(4)
   expect(data.admin.email).to.equal('admin1@example.com')
   expect(data.user.videoQuota).to.equal(5242880)
+  expect(data.user.videoQuotaDaily).to.equal(318742)
   expect(data.transcoding.enabled).to.be.false
   expect(data.transcoding.threads).to.equal(2)
   expect(data.transcoding.resolutions['240p']).to.be.true
@@ -65,6 +66,7 @@ function checkUpdatedConfig (data: CustomConfig) {
   expect(data.signup.limit).to.equal(5)
   expect(data.admin.email).to.equal('superadmin1@example.com')
   expect(data.user.videoQuota).to.equal(5242881)
+  expect(data.user.videoQuotaDaily).to.equal(318742)
   expect(data.transcoding.enabled).to.be.true
   expect(data.transcoding.threads).to.equal(1)
   expect(data.transcoding.resolutions['240p']).to.be.false
@@ -152,7 +154,8 @@ describe('Test config', function () {
         email: 'superadmin1@example.com'
       },
       user: {
-        videoQuota: 5242881
+        videoQuota: 5242881,
+        videoQuotaDaily: 318742
       },
       transcoding: {
         enabled: true,
diff --git a/server/tests/utils/server/config.ts b/server/tests/utils/server/config.ts
index d6ac3ef8a..799c31ae5 100644
--- a/server/tests/utils/server/config.ts
+++ b/server/tests/utils/server/config.ts
@@ -80,7 +80,8 @@ function updateCustomSubConfig (url: string, token: string, newConfig: any) {
       email: 'superadmin1@example.com'
     },
     user: {
-      videoQuota: 5242881
+      videoQuota: 5242881,
+      videoQuotaDaily: 318742
     },
     transcoding: {
       enabled: true,
diff --git a/server/tests/utils/users/users.ts b/server/tests/utils/users/users.ts
index f786de6e3..5dba34b69 100644
--- a/server/tests/utils/users/users.ts
+++ b/server/tests/utils/users/users.ts
@@ -10,6 +10,7 @@ function createUser (
   username: string,
   password: string,
   videoQuota = 1000000,
+  videoQuotaDaily = -1,
   role: UserRole = UserRole.USER,
   specialStatus = 200
 ) {
@@ -19,7 +20,8 @@ function createUser (
     password,
     role,
     email: username + '@example.com',
-    videoQuota
+    videoQuota,
+    videoQuotaDaily
   }
 
   return request(url)
@@ -202,6 +204,7 @@ function updateUser (options: {
   accessToken: string,
   email?: string,
   videoQuota?: number,
+  videoQuotaDaily?: number,
   role?: UserRole
 }) {
   const path = '/api/v1/users/' + options.userId
@@ -209,6 +212,7 @@ function updateUser (options: {
   const toSend = {}
   if (options.email !== undefined && options.email !== null) toSend['email'] = options.email
   if (options.videoQuota !== undefined && options.videoQuota !== null) toSend['videoQuota'] = options.videoQuota
+  if (options.videoQuotaDaily !== undefined && options.videoQuotaDaily !== null) toSend['videoQuotaDaily'] = options.videoQuotaDaily
   if (options.role !== undefined && options.role !== null) toSend['role'] = options.role
 
   return makePutBodyRequest({