Add history on server side

Add ability to disable, clear and list user videos history

16 files changed, 341 insertions, 17 deletions

diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts
index 87fab4a40..bc24792a2 100644
--- a/server/controllers/api/users/index.ts
+++ b/server/controllers/api/users/index.ts
@@ -38,6 +38,7 @@ import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '../../../h
 import { meRouter } from './me'
 import { deleteUserToken } from '../../../lib/oauth-model'
 import { myBlocklistRouter } from './my-blocklist'
+import { myVideosHistoryRouter } from './my-history'
 
 const auditLogger = auditLoggerFactory('users')
 
@@ -55,6 +56,7 @@ const askSendEmailLimiter = new RateLimit({
 
 const usersRouter = express.Router()
 usersRouter.use('/', myBlocklistRouter)
+usersRouter.use('/', myVideosHistoryRouter)
 usersRouter.use('/', meRouter)
 
 usersRouter.get('/autocomplete',
diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts
index f712b0f0b..8a3208160 100644
--- a/server/controllers/api/users/me.ts
+++ b/server/controllers/api/users/me.ts
@@ -330,6 +330,7 @@ async function updateMe (req: express.Request, res: express.Response, next: expr
   if (body.nsfwPolicy !== undefined) user.nsfwPolicy = body.nsfwPolicy
   if (body.webTorrentEnabled !== undefined) user.webTorrentEnabled = body.webTorrentEnabled
   if (body.autoPlayVideo !== undefined) user.autoPlayVideo = body.autoPlayVideo
+  if (body.videosHistoryEnabled !== undefined) user.videosHistoryEnabled = body.videosHistoryEnabled
 
   await sequelizeTypescript.transaction(async t => {
     const userAccount = await AccountModel.load(user.Account.id)
diff --git a/server/controllers/api/users/my-history.ts b/server/controllers/api/users/my-history.ts
new file mode 100644
index 000000000..6cd782c47
--- /dev/null
+++ b/server/controllers/api/users/my-history.ts
@@ -0,0 +1,57 @@
+import * as express from 'express'
+import {
+  asyncMiddleware,
+  asyncRetryTransactionMiddleware,
+  authenticate,
+  paginationValidator,
+  setDefaultPagination,
+  userHistoryRemoveValidator
+} from '../../../middlewares'
+import { UserModel } from '../../../models/account/user'
+import { getFormattedObjects } from '../../../helpers/utils'
+import { UserVideoHistoryModel } from '../../../models/account/user-video-history'
+import { sequelizeTypescript } from '../../../initializers'
+
+const myVideosHistoryRouter = express.Router()
+
+myVideosHistoryRouter.get('/me/history/videos',
+  authenticate,
+  paginationValidator,
+  setDefaultPagination,
+  asyncMiddleware(listMyVideosHistory)
+)
+
+myVideosHistoryRouter.post('/me/history/videos/remove',
+  authenticate,
+  userHistoryRemoveValidator,
+  asyncRetryTransactionMiddleware(removeUserHistory)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  myVideosHistoryRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function listMyVideosHistory (req: express.Request, res: express.Response) {
+  const user: UserModel = res.locals.oauth.token.User
+
+  const resultList = await UserVideoHistoryModel.listForApi(user, req.query.start, req.query.count)
+
+  return res.json(getFormattedObjects(resultList.data, resultList.total))
+}
+
+async function removeUserHistory (req: express.Request, res: express.Response) {
+  const user: UserModel = res.locals.oauth.token.User
+  const beforeDate = req.body.beforeDate || null
+
+  await sequelizeTypescript.transaction(t => {
+    return UserVideoHistoryModel.removeHistoryBefore(user, beforeDate, t)
+  })
+
+  // Do not send the delete to other instances, we delete OUR copy of this video abuse
+
+  return res.type('json').status(204).end()
+}
diff --git a/server/helpers/custom-validators/users.ts b/server/helpers/custom-validators/users.ts
index 1cb5e5b0f..80652b479 100644
--- a/server/helpers/custom-validators/users.ts
+++ b/server/helpers/custom-validators/users.ts
@@ -46,6 +46,10 @@ function isUserWebTorrentEnabledValid (value: any) {
   return isBooleanValid(value)
 }
 
+function isUserVideosHistoryEnabledValid (value: any) {
+  return isBooleanValid(value)
+}
+
 function isUserAutoPlayVideoValid (value: any) {
   return isBooleanValid(value)
 }
@@ -73,6 +77,7 @@ function isAvatarFile (files: { [ fieldname: string ]: Express.Multer.File[] } |
 // ---------------------------------------------------------------------------
 
 export {
+  isUserVideosHistoryEnabledValid,
   isUserBlockedValid,
   isUserPasswordValid,
   isUserBlockedReasonValid,
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 6971ab775..6e463a1d6 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -16,7 +16,7 @@ let config: IConfig = require('config')
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 295
+const LAST_MIGRATION_VERSION = 300
 
 // ---------------------------------------------------------------------------
 
diff --git a/server/initializers/migrations/0300-user-videos-history-enabled.ts b/server/initializers/migrations/0300-user-videos-history-enabled.ts
new file mode 100644
index 000000000..aa5fc21fb
--- /dev/null
+++ b/server/initializers/migrations/0300-user-videos-history-enabled.ts
@@ -0,0 +1,27 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction,
+  queryInterface: Sequelize.QueryInterface,
+  sequelize: Sequelize.Sequelize,
+  db: any
+}): Promise<void> {
+  {
+    const data = {
+      type: Sequelize.BOOLEAN,
+      allowNull: false,
+      defaultValue: true
+    }
+
+    await utils.queryInterface.addColumn('user', 'videosHistoryEnabled', data)
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}
diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts
index 46c7f0f3a..65dd00335 100644
--- a/server/middlewares/validators/index.ts
+++ b/server/middlewares/validators/index.ts
@@ -12,3 +12,4 @@ export * from './videos'
 export * from './webfinger'
 export * from './search'
 export * from './server'
+export * from './user-history'
diff --git a/server/middlewares/validators/user-history.ts b/server/middlewares/validators/user-history.ts
new file mode 100644
index 000000000..3c8971ea1
--- /dev/null
+++ b/server/middlewares/validators/user-history.ts
@@ -0,0 +1,30 @@
+import * as express from 'express'
+import 'express-validator'
+import { body, param, query } from 'express-validator/check'
+import { logger } from '../../helpers/logger'
+import { areValidationErrors } from './utils'
+import { ActorFollowModel } from '../../models/activitypub/actor-follow'
+import { areValidActorHandles, isValidActorHandle } from '../../helpers/custom-validators/activitypub/actor'
+import { UserModel } from '../../models/account/user'
+import { CONFIG } from '../../initializers'
+import { isDateValid, toArray } from '../../helpers/custom-validators/misc'
+
+const userHistoryRemoveValidator = [
+  body('beforeDate')
+    .optional()
+    .custom(isDateValid).withMessage('Should have a valid before date'),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking userHistoryRemoveValidator parameters', { parameters: req.body })
+
+    if (areValidationErrors(req, res)) return
+
+    return next()
+  }
+]
+
+// ---------------------------------------------------------------------------
+
+export {
+  userHistoryRemoveValidator
+}
diff --git a/server/middlewares/validators/videos/video-watch.ts b/server/middlewares/validators/videos/video-watch.ts
index bca64662f..c38ad8a10 100644
--- a/server/middlewares/validators/videos/video-watch.ts
+++ b/server/middlewares/validators/videos/video-watch.ts
@@ -4,6 +4,7 @@ import { isIdOrUUIDValid } from '../../../helpers/custom-validators/misc'
 import { isVideoExist } from '../../../helpers/custom-validators/videos'
 import { areValidationErrors } from '../utils'
 import { logger } from '../../../helpers/logger'
+import { UserModel } from '../../../models/account/user'
 
 const videoWatchingValidator = [
   param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'),
@@ -17,6 +18,12 @@ const videoWatchingValidator = [
     if (areValidationErrors(req, res)) return
     if (!await isVideoExist(req.params.videoId, res, 'id')) return
 
+    const user = res.locals.oauth.token.User as UserModel
+    if (user.videosHistoryEnabled === false) {
+      logger.warn('Cannot set videos to watch by user %d: videos history is disabled.', user.id)
+      return res.status(409).end()
+    }
+
     return next()
   }
 ]
diff --git a/server/models/account/user-video-history.ts b/server/models/account/user-video-history.ts
index 0476cad9d..15cb399c9 100644
--- a/server/models/account/user-video-history.ts
+++ b/server/models/account/user-video-history.ts
@@ -1,6 +1,7 @@
-import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, IsInt, Min, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, IsInt, Model, Table, UpdatedAt } from 'sequelize-typescript'
 import { VideoModel } from '../video/video'
 import { UserModel } from './user'
+import { Transaction, Op, DestroyOptions } from 'sequelize'
 
 @Table({
   tableName: 'userVideoHistory',
@@ -52,4 +53,34 @@ export class UserVideoHistoryModel extends Model<UserVideoHistoryModel> {
     onDelete: 'CASCADE'
   })
   User: UserModel
+
+  static listForApi (user: UserModel, start: number, count: number) {
+    return VideoModel.listForApi({
+      start,
+      count,
+      sort: '-UserVideoHistories.updatedAt',
+      nsfw: null, // All
+      includeLocalVideos: true,
+      withFiles: false,
+      user,
+      historyOfUser: user
+    })
+  }
+
+  static removeHistoryBefore (user: UserModel, beforeDate: string, t: Transaction) {
+    const query: DestroyOptions = {
+      where: {
+        userId: user.id
+      },
+      transaction: t
+    }
+
+    if (beforeDate) {
+      query.where.updatedAt = {
+        [Op.lt]: beforeDate
+      }
+    }
+
+    return UserVideoHistoryModel.destroy(query)
+  }
 }
diff --git a/server/models/account/user.ts b/server/models/account/user.ts
index 1843603f1..ea017c338 100644
--- a/server/models/account/user.ts
+++ b/server/models/account/user.ts
@@ -32,7 +32,8 @@ import {
   isUserUsernameValid,
   isUserVideoQuotaDailyValid,
   isUserVideoQuotaValid,
-  isUserWebTorrentEnabledValid
+  isUserWebTorrentEnabledValid,
+  isUserVideosHistoryEnabledValid
 } from '../../helpers/custom-validators/users'
 import { comparePassword, cryptPassword } from '../../helpers/peertube-crypto'
 import { OAuthTokenModel } from '../oauth/oauth-token'
@@ -116,6 +117,12 @@ export class UserModel extends Model<UserModel> {
 
   @AllowNull(false)
   @Default(true)
+  @Is('UserVideosHistoryEnabled', value => throwIfNotValid(value, isUserVideosHistoryEnabledValid, 'Videos history enabled'))
+  @Column
+  videosHistoryEnabled: boolean
+
+  @AllowNull(false)
+  @Default(true)
   @Is('UserAutoPlayVideo', value => throwIfNotValid(value, isUserAutoPlayVideoValid, 'auto play video boolean'))
   @Column
   autoPlayVideo: boolean
diff --git a/server/models/utils.ts b/server/models/utils.ts
index 60b0906e8..6694eda69 100644
--- a/server/models/utils.ts
+++ b/server/models/utils.ts
@@ -29,7 +29,7 @@ function getVideoSort (value: string, lastSort: string[] = [ 'id', 'ASC' ]) {
     ]
   }
 
-  return [ [ field, direction ], lastSort ]
+  return [ field.split('.').concat([ direction ]), lastSort ]
 }
 
 function getSortOnModel (model: any, value: string, lastSort: string[] = [ 'id', 'ASC' ]) {
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index adef37937..199ea9ea4 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -153,7 +153,8 @@ type AvailableForListIDsOptions = {
   accountId?: number
   videoChannelId?: number
   trendingDays?: number
-  user?: UserModel
+  user?: UserModel,
+  historyOfUser?: UserModel
 }
 
 @Scopes({
@@ -416,6 +417,16 @@ type AvailableForListIDsOptions = {
       query.subQuery = false
     }
 
+    if (options.historyOfUser) {
+      query.include.push({
+        model: UserVideoHistoryModel,
+        required: true,
+        where: {
+          userId: options.historyOfUser.id
+        }
+      })
+    }
+
     return query
   },
   [ ScopeNames.WITH_ACCOUNT_DETAILS ]: {
@@ -987,7 +998,8 @@ export class VideoModel extends Model<VideoModel> {
     videoChannelId?: number,
     followerActorId?: number
     trendingDays?: number,
-    user?: UserModel
+    user?: UserModel,
+    historyOfUser?: UserModel
   }, countVideos = true) {
     if (options.filter && options.filter === 'all-local' && !options.user.hasRight(UserRight.SEE_ALL_VIDEOS)) {
       throw new Error('Try to filter all-local but no user has not the see all videos right')
@@ -1026,6 +1038,7 @@ export class VideoModel extends Model<VideoModel> {
       videoChannelId: options.videoChannelId,
       includeLocalVideos: options.includeLocalVideos,
       user: options.user,
+      historyOfUser: options.historyOfUser,
       trendingDays
     }
 
@@ -1341,7 +1354,7 @@ export class VideoModel extends Model<VideoModel> {
     }
 
     const [ count, rowsId ] = await Promise.all([
-      countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined),
+      countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve<number>(undefined),
       VideoModel.scope(idsScope).findAll(query)
     ])
     const ids = rowsId.map(r => r.id)
diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts
index ec35429d3..f8044cbd4 100644
--- a/server/tests/api/check-params/users.ts
+++ b/server/tests/api/check-params/users.ts
@@ -308,6 +308,14 @@ describe('Test users API validators', function () {
       await makePutBodyRequest({ url: server.url, path: path + 'me', token: userAccessToken, fields })
     })
 
+    it('Should fail with an invalid videosHistoryEnabled attribute', async function () {
+      const fields = {
+        videosHistoryEnabled: -1
+      }
+
+      await makePutBodyRequest({ url: server.url, path: path + 'me', token: userAccessToken, fields })
+    })
+
     it('Should fail with an non authenticated user', async function () {
       const fields = {
         currentPassword: 'my super password',
diff --git a/server/tests/api/check-params/videos-history.ts b/server/tests/api/check-params/videos-history.ts
index 09c6f7861..8c079a956 100644
--- a/server/tests/api/check-params/videos-history.ts
+++ b/server/tests/api/check-params/videos-history.ts
@@ -3,8 +3,11 @@
 import * as chai from 'chai'
 import 'mocha'
 import {
+  checkBadCountPagination,
+  checkBadStartPagination,
   flushTests,
   killallServers,
+  makeGetRequest,
   makePostBodyRequest,
   makePutBodyRequest,
   runServer,
@@ -16,7 +19,9 @@ import {
 const expect = chai.expect
 
 describe('Test videos history API validator', function () {
-  let path: string
+  let watchingPath: string
+  let myHistoryPath = '/api/v1/users/me/history/videos'
+  let myHistoryRemove = myHistoryPath + '/remove'
   let server: ServerInfo
 
   // ---------------------------------------------------------------
@@ -33,14 +38,14 @@ describe('Test videos history API validator', function () {
     const res = await uploadVideo(server.url, server.accessToken, {})
     const videoUUID = res.body.video.uuid
 
-    path = '/api/v1/videos/' + videoUUID + '/watching'
+    watchingPath = '/api/v1/videos/' + videoUUID + '/watching'
   })
 
   describe('When notifying a user is watching a video', function () {
 
     it('Should fail with an unauthenticated user', async function () {
       const fields = { currentTime: 5 }
-      await makePutBodyRequest({ url: server.url, path, fields, statusCodeExpected: 401 })
+      await makePutBodyRequest({ url: server.url, path: watchingPath, fields, statusCodeExpected: 401 })
     })
 
     it('Should fail with an incorrect video id', async function () {
@@ -58,13 +63,68 @@ describe('Test videos history API validator', function () {
 
     it('Should fail with a bad current time', async function () {
       const fields = { currentTime: 'hello' }
-      await makePutBodyRequest({ url: server.url, path, fields, token: server.accessToken, statusCodeExpected: 400 })
+      await makePutBodyRequest({ url: server.url, path: watchingPath, fields, token: server.accessToken, statusCodeExpected: 400 })
     })
 
     it('Should succeed with the correct parameters', async function () {
       const fields = { currentTime: 5 }
 
-      await makePutBodyRequest({ url: server.url, path, fields, token: server.accessToken, statusCodeExpected: 204 })
+      await makePutBodyRequest({ url: server.url, path: watchingPath, fields, token: server.accessToken, statusCodeExpected: 204 })
+    })
+  })
+
+  describe('When listing user videos history', function () {
+    it('Should fail with a bad start pagination', async function () {
+      await checkBadStartPagination(server.url, myHistoryPath, server.accessToken)
+    })
+
+    it('Should fail with a bad count pagination', async function () {
+      await checkBadCountPagination(server.url, myHistoryPath, server.accessToken)
+    })
+
+    it('Should fail with an unauthenticated user', async function () {
+      await makeGetRequest({ url: server.url, path: myHistoryPath, statusCodeExpected: 401 })
+    })
+
+    it('Should succeed with the correct params', async function () {
+      await makeGetRequest({ url: server.url, token: server.accessToken, path: myHistoryPath, statusCodeExpected: 200 })
+    })
+  })
+
+  describe('When removing user videos history', function () {
+    it('Should fail with an unauthenticated user', async function () {
+      await makePostBodyRequest({ url: server.url, path: myHistoryPath + '/remove', statusCodeExpected: 401 })
+    })
+
+    it('Should fail with a bad beforeDate parameter', async function () {
+      const body = { beforeDate: '15' }
+      await makePostBodyRequest({
+        url: server.url,
+        token: server.accessToken,
+        path: myHistoryRemove,
+        fields: body,
+        statusCodeExpected: 400
+      })
+    })
+
+    it('Should succeed with a valid beforeDate param', async function () {
+      const body = { beforeDate: new Date().toISOString() }
+      await makePostBodyRequest({
+        url: server.url,
+        token: server.accessToken,
+        path: myHistoryRemove,
+        fields: body,
+        statusCodeExpected: 204
+      })
+    })
+
+    it('Should succeed without body', async function () {
+      await makePostBodyRequest({
+        url: server.url,
+        token: server.accessToken,
+        path: myHistoryRemove,
+        statusCodeExpected: 204
+      })
     })
   })
 
diff --git a/server/tests/api/videos/videos-history.ts b/server/tests/api/videos/videos-history.ts
index 40ae94f79..f654a422b 100644
--- a/server/tests/api/videos/videos-history.ts
+++ b/server/tests/api/videos/videos-history.ts
@@ -3,17 +3,21 @@
 import * as chai from 'chai'
 import 'mocha'
 import {
+  createUser,
   flushTests,
   getVideosListWithToken,
   getVideoWithToken,
-  killallServers, makePutBodyRequest,
-  runServer, searchVideoWithToken,
+  killallServers,
+  runServer,
+  searchVideoWithToken,
   ServerInfo,
   setAccessTokensToServers,
-  uploadVideo
+  updateMyUser,
+  uploadVideo,
+  userLogin
 } from '../../../../shared/utils'
 import { Video, VideoDetails } from '../../../../shared/models/videos'
-import { userWatchVideo } from '../../../../shared/utils/videos/video-history'
+import { listMyVideosHistory, removeMyVideosHistory, userWatchVideo } from '../../../../shared/utils/videos/video-history'
 
 const expect = chai.expect
 
@@ -22,6 +26,8 @@ describe('Test videos history', function () {
   let video1UUID: string
   let video2UUID: string
   let video3UUID: string
+  let video3WatchedDate: Date
+  let userAccessToken: string
 
   before(async function () {
     this.timeout(30000)
@@ -46,6 +52,13 @@ describe('Test videos history', function () {
       const res = await uploadVideo(server.url, server.accessToken, { name: 'video 3' })
       video3UUID = res.body.video.uuid
     }
+
+    const user = {
+      username: 'user_1',
+      password: 'super password'
+    }
+    await createUser(server.url, server.accessToken, user.username, user.password)
+    userAccessToken = await userLogin(server, user)
   })
 
   it('Should get videos, without watching history', async function () {
@@ -62,8 +75,8 @@ describe('Test videos history', function () {
   })
 
   it('Should watch the first and second video', async function () {
-    await userWatchVideo(server.url, server.accessToken, video1UUID, 3)
     await userWatchVideo(server.url, server.accessToken, video2UUID, 8)
+    await userWatchVideo(server.url, server.accessToken, video1UUID, 3)
   })
 
   it('Should return the correct history when listing, searching and getting videos', async function () {
@@ -117,6 +130,68 @@ describe('Test videos history', function () {
     }
   })
 
+  it('Should have these videos when listing my history', async function () {
+    video3WatchedDate = new Date()
+    await userWatchVideo(server.url, server.accessToken, video3UUID, 2)
+
+    const res = await listMyVideosHistory(server.url, server.accessToken)
+
+    expect(res.body.total).to.equal(3)
+
+    const videos: Video[] = res.body.data
+    expect(videos[0].name).to.equal('video 3')
+    expect(videos[1].name).to.equal('video 1')
+    expect(videos[2].name).to.equal('video 2')
+  })
+
+  it('Should not have videos history on another user', async function () {
+    const res = await listMyVideosHistory(server.url, userAccessToken)
+
+    expect(res.body.total).to.equal(0)
+    expect(res.body.data).to.have.lengthOf(0)
+  })
+
+  it('Should clear my history', async function () {
+    await removeMyVideosHistory(server.url, server.accessToken, video3WatchedDate.toISOString())
+  })
+
+  it('Should have my history cleared', async function () {
+    const res = await listMyVideosHistory(server.url, server.accessToken)
+
+    expect(res.body.total).to.equal(1)
+
+    const videos: Video[] = res.body.data
+    expect(videos[0].name).to.equal('video 3')
+  })
+
+  it('Should disable videos history', async function () {
+    await updateMyUser({
+      url: server.url,
+      accessToken: server.accessToken,
+      videosHistoryEnabled: false
+    })
+
+    await userWatchVideo(server.url, server.accessToken, video2UUID, 8, 409)
+  })
+
+  it('Should re-enable videos history', async function () {
+    await updateMyUser({
+      url: server.url,
+      accessToken: server.accessToken,
+      videosHistoryEnabled: true
+    })
+
+    await userWatchVideo(server.url, server.accessToken, video1UUID, 8)
+
+    const res = await listMyVideosHistory(server.url, server.accessToken)
+
+    expect(res.body.total).to.equal(2)
+
+    const videos: Video[] = res.body.data
+    expect(videos[0].name).to.equal('video 1')
+    expect(videos[1].name).to.equal('video 3')
+  })
+
   after(async function () {
     killallServers([ server ])