8 files changed, 194 insertions, 79 deletions

diff --git a/server/models/video/video-abuse.ts b/server/models/video/video-abuse.ts
index dbb88ca45..cc47644f2 100644
--- a/server/models/video/video-abuse.ts
+++ b/server/models/video/video-abuse.ts
@@ -1,17 +1,4 @@
-import {
-  AfterCreate,
-  AllowNull,
-  BelongsTo,
-  Column,
-  CreatedAt,
-  DataType,
-  Default,
-  ForeignKey,
-  Is,
-  Model,
-  Table,
-  UpdatedAt
-} from 'sequelize-typescript'
+import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
 import { VideoAbuseObject } from '../../../shared/models/activitypub/objects'
 import { VideoAbuse } from '../../../shared/models/videos'
 import {
@@ -19,7 +6,6 @@ import {
   isVideoAbuseReasonValid,
   isVideoAbuseStateValid
 } from '../../helpers/custom-validators/video-abuses'
-import { Emailer } from '../../lib/emailer'
 import { AccountModel } from '../account/account'
 import { getSort, throwIfNotValid } from '../utils'
 import { VideoModel } from './video'
@@ -40,8 +26,9 @@ import { CONSTRAINTS_FIELDS, VIDEO_ABUSE_STATES } from '../../initializers'
 export class VideoAbuseModel extends Model<VideoAbuseModel> {
 
   @AllowNull(false)
+  @Default(null)
   @Is('VideoAbuseReason', value => throwIfNotValid(value, isVideoAbuseReasonValid, 'reason'))
-  @Column
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_ABUSES.REASON.max))
   reason: string
 
   @AllowNull(false)
@@ -86,11 +73,6 @@ export class VideoAbuseModel extends Model<VideoAbuseModel> {
   })
   Video: VideoModel
 
-  @AfterCreate
-  static sendEmailNotification (instance: VideoAbuseModel) {
-    return Emailer.Instance.addVideoAbuseReportJob(instance.videoId)
-  }
-
   static loadByIdAndVideoId (id: number, videoId: number) {
     const query = {
       where: {
diff --git a/server/models/video/video-blacklist.ts b/server/models/video/video-blacklist.ts
index 67f7cd487..3b567e488 100644
--- a/server/models/video/video-blacklist.ts
+++ b/server/models/video/video-blacklist.ts
@@ -1,21 +1,7 @@
-import {
-  AfterCreate,
-  AfterDestroy,
-  AllowNull,
-  BelongsTo,
-  Column,
-  CreatedAt,
-  DataType,
-  ForeignKey,
-  Is,
-  Model,
-  Table,
-  UpdatedAt
-} from 'sequelize-typescript'
+import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
 import { getSortOnModel, SortType, throwIfNotValid } from '../utils'
 import { VideoModel } from './video'
 import { isVideoBlacklistReasonValid } from '../../helpers/custom-validators/video-blacklist'
-import { Emailer } from '../../lib/emailer'
 import { VideoBlacklist } from '../../../shared/models/videos'
 import { CONSTRAINTS_FIELDS } from '../../initializers'
 
@@ -35,6 +21,10 @@ export class VideoBlacklistModel extends Model<VideoBlacklistModel> {
   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_BLACKLIST.REASON.max))
   reason: string
 
+  @AllowNull(false)
+  @Column
+  unfederated: boolean
+
   @CreatedAt
   createdAt: Date
 
@@ -53,16 +43,6 @@ export class VideoBlacklistModel extends Model<VideoBlacklistModel> {
   })
   Video: VideoModel
 
-  @AfterCreate
-  static sendBlacklistEmailNotification (instance: VideoBlacklistModel) {
-    return Emailer.Instance.addVideoBlacklistReportJob(instance.videoId, instance.reason)
-  }
-
-  @AfterDestroy
-  static sendUnblacklistEmailNotification (instance: VideoBlacklistModel) {
-    return Emailer.Instance.addVideoUnblacklistReportJob(instance.videoId)
-  }
-
   static listForApi (start: number, count: number, sort: SortType) {
     const query = {
       offset: start,
@@ -103,6 +83,7 @@ export class VideoBlacklistModel extends Model<VideoBlacklistModel> {
       createdAt: this.createdAt,
       updatedAt: this.updatedAt,
       reason: this.reason,
+      unfederated: this.unfederated,
 
       video: {
         id: video.id,
diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts
index f4586917e..5598d80f6 100644
--- a/server/models/video/video-channel.ts
+++ b/server/models/video/video-channel.ts
@@ -233,6 +233,27 @@ export class VideoChannelModel extends Model<VideoChannelModel> {
       })
   }
 
+  static listLocalsForSitemap (sort: string) {
+    const query = {
+      attributes: [ ],
+      offset: 0,
+      order: getSort(sort),
+      include: [
+        {
+          attributes: [ 'preferredUsername', 'serverId' ],
+          model: ActorModel.unscoped(),
+          where: {
+            serverId: null
+          }
+        }
+      ]
+    }
+
+    return VideoChannelModel
+      .unscoped()
+      .findAll(query)
+  }
+
   static searchForApi (options: {
     actorId: number
     search: string
@@ -449,4 +470,8 @@ export class VideoChannelModel extends Model<VideoChannelModel> {
   getDisplayName () {
     return this.name
   }
+
+  isOutdated () {
+    return this.Actor.isOutdated()
+  }
 }
diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts
index dd6d08139..cf6278da7 100644
--- a/server/models/video/video-comment.ts
+++ b/server/models/video/video-comment.ts
@@ -18,7 +18,7 @@ import { ActivityTagObject } from '../../../shared/models/activitypub/objects/co
 import { VideoCommentObject } from '../../../shared/models/activitypub/objects/video-comment-object'
 import { VideoComment } from '../../../shared/models/videos/video-comment.model'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
-import { CONSTRAINTS_FIELDS } from '../../initializers'
+import { CONFIG, CONSTRAINTS_FIELDS } from '../../initializers'
 import { sendDeleteVideoComment } from '../../lib/activitypub/send'
 import { AccountModel } from '../account/account'
 import { ActorModel } from '../activitypub/actor'
@@ -29,6 +29,9 @@ import { VideoModel } from './video'
 import { VideoChannelModel } from './video-channel'
 import { getServerActor } from '../../helpers/utils'
 import { UserModel } from '../account/user'
+import { actorNameAlphabet } from '../../helpers/custom-validators/activitypub/actor'
+import { regexpCapture } from '../../helpers/regexp'
+import { uniq } from 'lodash'
 
 enum ScopeNames {
   WITH_ACCOUNT = 'WITH_ACCOUNT',
@@ -370,9 +373,11 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
         id: {
           [ Sequelize.Op.in ]: Sequelize.literal('(' +
             'WITH RECURSIVE children (id, "inReplyToCommentId") AS ( ' +
-            'SELECT id, "inReplyToCommentId" FROM "videoComment" WHERE id = ' + comment.id + ' UNION ' +
-            'SELECT p.id, p."inReplyToCommentId" from "videoComment" p ' +
-            'INNER JOIN children c ON c."inReplyToCommentId" = p.id) ' +
+              `SELECT id, "inReplyToCommentId" FROM "videoComment" WHERE id = ${comment.id} ` +
+              'UNION ' +
+              'SELECT "parent"."id", "parent"."inReplyToCommentId" FROM "videoComment" "parent" ' +
+              'INNER JOIN "children" ON "children"."inReplyToCommentId" = "parent"."id"' +
+            ') ' +
             'SELECT id FROM children' +
           ')'),
           [ Sequelize.Op.ne ]: comment.id
@@ -448,6 +453,10 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
     }
   }
 
+  getCommentStaticPath () {
+    return this.Video.getWatchStaticPath() + ';threadId=' + this.getThreadId()
+  }
+
   getThreadId (): number {
     return this.originCommentId || this.id
   }
@@ -456,6 +465,34 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
     return this.Account.isOwned()
   }
 
+  extractMentions () {
+    if (!this.text) return []
+
+    const localMention = `@(${actorNameAlphabet}+)`
+    const remoteMention = `${localMention}@${CONFIG.WEBSERVER.HOST}`
+
+    const remoteMentionsRegex = new RegExp(' ' + remoteMention + ' ', 'g')
+    const localMentionsRegex = new RegExp(' ' + localMention + ' ', 'g')
+    const firstMentionRegex = new RegExp('^(?:(?:' + remoteMention + ')|(?:' + localMention + ')) ', 'g')
+    const endMentionRegex = new RegExp(' (?:(?:' + remoteMention + ')|(?:' + localMention + '))$', 'g')
+
+    return uniq(
+      [].concat(
+        regexpCapture(this.text, remoteMentionsRegex)
+          .map(([ , username ]) => username),
+
+        regexpCapture(this.text, localMentionsRegex)
+          .map(([ , username ]) => username),
+
+        regexpCapture(this.text, firstMentionRegex)
+          .map(([ , username1, username2 ]) => username1 || username2),
+
+        regexpCapture(this.text, endMentionRegex)
+          .map(([ , username1, username2 ]) => username1 || username2)
+      )
+    )
+  }
+
   toFormattedJSON () {
     return {
       id: this.id,
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index adebdf0c7..1f1b76c1e 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -1,4 +1,3 @@
-import { values } from 'lodash'
 import {
   AllowNull,
   BelongsTo,
@@ -14,12 +13,12 @@ import {
   UpdatedAt
 } from 'sequelize-typescript'
 import {
+  isVideoFileExtnameValid,
   isVideoFileInfoHashValid,
   isVideoFileResolutionValid,
   isVideoFileSizeValid,
   isVideoFPSResolutionValid
 } from '../../helpers/custom-validators/videos'
-import { CONSTRAINTS_FIELDS } from '../../initializers'
 import { throwIfNotValid } from '../utils'
 import { VideoModel } from './video'
 import * as Sequelize from 'sequelize'
@@ -58,7 +57,8 @@ export class VideoFileModel extends Model<VideoFileModel> {
   size: number
 
   @AllowNull(false)
-  @Column(DataType.ENUM(values(CONSTRAINTS_FIELDS.VIDEOS.EXTNAME)))
+  @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
+  @Column
   extname: string
 
   @AllowNull(false)
@@ -120,6 +120,26 @@ export class VideoFileModel extends Model<VideoFileModel> {
     return VideoFileModel.findById(id, options)
   }
 
+  static async getStats () {
+    let totalLocalVideoFilesSize = await VideoFileModel.sum('size', {
+      include: [
+        {
+          attributes: [],
+          model: VideoModel.unscoped(),
+          where: {
+            remote: false
+          }
+        }
+      ]
+    } as any)
+    // Sequelize could return null...
+    if (!totalLocalVideoFilesSize) totalLocalVideoFilesSize = 0
+
+    return {
+      totalLocalVideoFilesSize
+    }
+  }
+
   hasSameUniqueKeysThan (other: VideoFileModel) {
     return this.fps === other.fps &&
       this.resolution === other.resolution &&
diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts
index e3f8d525b..de0747f22 100644
--- a/server/models/video/video-format-utils.ts
+++ b/server/models/video/video-format-utils.ts
@@ -2,7 +2,7 @@ import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
 import { VideoModel } from './video'
 import { VideoFileModel } from './video-file'
 import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects'
-import { CONFIG, THUMBNAILS_SIZE, VIDEO_EXT_MIMETYPE } from '../../initializers'
+import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers'
 import { VideoCaptionModel } from './video-caption'
 import {
   getVideoCommentsActivityPubUrl,
@@ -207,8 +207,8 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject {
   for (const file of video.VideoFiles) {
     url.push({
       type: 'Link',
-      mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any,
-      mediaType: VIDEO_EXT_MIMETYPE[ file.extname ] as any,
+      mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ file.extname ] as any,
+      mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ file.extname ] as any,
       href: video.getVideoFileUrl(file, baseUrlHttp),
       height: file.resolution,
       size: file.size,
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts
index 8d442b3f8..c723e57c0 100644
--- a/server/models/video/video-import.ts
+++ b/server/models/video/video-import.ts
@@ -144,6 +144,10 @@ export class VideoImportModel extends Model<VideoImportModel> {
                            })
   }
 
+  getTargetIdentifier () {
+    return this.targetUrl || this.magnetUri || this.torrentName
+  }
+
   toFormattedJSON (): VideoImport {
     const videoFormatOptions = {
       completeDescription: true,
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 0f18d9f0c..80a6c7832 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -94,6 +94,7 @@ import {
 import * as validator from 'validator'
 import { UserVideoHistoryModel } from '../account/user-video-history'
 import { UserModel } from '../account/user'
+import { VideoImportModel } from './video-import'
 
 // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
 const indexes: Sequelize.DefineIndexesOptions[] = [
@@ -102,17 +103,45 @@ const indexes: Sequelize.DefineIndexesOptions[] = [
   { fields: [ 'createdAt' ] },
   { fields: [ 'publishedAt' ] },
   { fields: [ 'duration' ] },
-  { fields: [ 'category' ] },
-  { fields: [ 'licence' ] },
-  { fields: [ 'nsfw' ] },
-  { fields: [ 'language' ] },
-  { fields: [ 'waitTranscoding' ] },
-  { fields: [ 'state' ] },
-  { fields: [ 'remote' ] },
   { fields: [ 'views' ] },
-  { fields: [ 'likes' ] },
   { fields: [ 'channelId' ] },
   {
+    fields: [ 'category' ], // We don't care videos with an unknown category
+    where: {
+      category: {
+        [Sequelize.Op.ne]: null
+      }
+    }
+  },
+  {
+    fields: [ 'licence' ], // We don't care videos with an unknown licence
+    where: {
+      licence: {
+        [Sequelize.Op.ne]: null
+      }
+    }
+  },
+  {
+    fields: [ 'language' ], // We don't care videos with an unknown language
+    where: {
+      language: {
+        [Sequelize.Op.ne]: null
+      }
+    }
+  },
+  {
+    fields: [ 'nsfw' ], // Most of the videos are not NSFW
+    where: {
+      nsfw: true
+    }
+  },
+  {
+    fields: [ 'remote' ], // Only index local videos
+    where: {
+      remote: false
+    }
+  },
+  {
     fields: [ 'uuid' ],
     unique: true
   },
@@ -140,7 +169,7 @@ type ForAPIOptions = {
 
 type AvailableForListIDsOptions = {
   serverAccountId: number
-  actorId: number
+  followerActorId: number
   includeLocalVideos: boolean
   filter?: VideoFilter
   categoryOneOf?: number[]
@@ -153,7 +182,8 @@ type AvailableForListIDsOptions = {
   accountId?: number
   videoChannelId?: number
   trendingDays?: number
-  user?: UserModel
+  user?: UserModel,
+  historyOfUser?: UserModel
 }
 
 @Scopes({
@@ -315,7 +345,7 @@ type AvailableForListIDsOptions = {
       query.include.push(videoChannelInclude)
     }
 
-    if (options.actorId) {
+    if (options.followerActorId) {
       let localVideosReq = ''
       if (options.includeLocalVideos === true) {
         localVideosReq = ' UNION ALL ' +
@@ -327,7 +357,7 @@ type AvailableForListIDsOptions = {
       }
 
       // Force actorId to be a number to avoid SQL injections
-      const actorIdNumber = parseInt(options.actorId.toString(), 10)
+      const actorIdNumber = parseInt(options.followerActorId.toString(), 10)
       query.where[ 'id' ][ Sequelize.Op.and ].push({
         [ Sequelize.Op.in ]: Sequelize.literal(
           '(' +
@@ -416,6 +446,21 @@ type AvailableForListIDsOptions = {
       query.subQuery = false
     }
 
+    if (options.historyOfUser) {
+      query.include.push({
+        model: UserVideoHistoryModel,
+        required: true,
+        where: {
+          userId: options.historyOfUser.id
+        }
+      })
+
+      // Even if the relation is n:m, we know that a user only have 0..1 video history
+      // So we won't have multiple rows for the same video
+      // Without this, we would not be able to sort on "updatedAt" column of UserVideoHistoryModel
+      query.subQuery = false
+    }
+
     return query
   },
   [ ScopeNames.WITH_ACCOUNT_DETAILS ]: {
@@ -741,6 +786,15 @@ export class VideoModel extends Model<VideoModel> {
   })
   VideoBlacklist: VideoBlacklistModel
 
+  @HasOne(() => VideoImportModel, {
+    foreignKey: {
+      name: 'videoId',
+      allowNull: true
+    },
+    onDelete: 'set null'
+  })
+  VideoImport: VideoImportModel
+
   @HasMany(() => VideoCaptionModel, {
     foreignKey: {
       name: 'videoId',
@@ -985,9 +1039,10 @@ export class VideoModel extends Model<VideoModel> {
     filter?: VideoFilter,
     accountId?: number,
     videoChannelId?: number,
-    actorId?: number
+    followerActorId?: number
     trendingDays?: number,
-    user?: UserModel
+    user?: UserModel,
+    historyOfUser?: UserModel
   }, countVideos = true) {
     if (options.filter && options.filter === 'all-local' && !options.user.hasRight(UserRight.SEE_ALL_VIDEOS)) {
       throw new Error('Try to filter all-local but no user has not the see all videos right')
@@ -1008,11 +1063,11 @@ export class VideoModel extends Model<VideoModel> {
 
     const serverActor = await getServerActor()
 
-    // actorId === null has a meaning, so just check undefined
-    const actorId = options.actorId !== undefined ? options.actorId : serverActor.id
+    // followerActorId === null has a meaning, so just check undefined
+    const followerActorId = options.followerActorId !== undefined ? options.followerActorId : serverActor.id
 
     const queryOptions = {
-      actorId,
+      followerActorId,
       serverAccountId: serverActor.Account.id,
       nsfw: options.nsfw,
       categoryOneOf: options.categoryOneOf,
@@ -1026,6 +1081,7 @@ export class VideoModel extends Model<VideoModel> {
       videoChannelId: options.videoChannelId,
       includeLocalVideos: options.includeLocalVideos,
       user: options.user,
+      historyOfUser: options.historyOfUser,
       trendingDays
     }
 
@@ -1118,7 +1174,7 @@ export class VideoModel extends Model<VideoModel> {
 
     const serverActor = await getServerActor()
     const queryOptions = {
-      actorId: serverActor.id,
+      followerActorId: serverActor.id,
       serverAccountId: serverActor.Account.id,
       includeLocalVideos: options.includeLocalVideos,
       nsfw: options.nsfw,
@@ -1273,11 +1329,11 @@ export class VideoModel extends Model<VideoModel> {
   // threshold corresponds to how many video the field should have to be returned
   static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) {
     const serverActor = await getServerActor()
-    const actorId = serverActor.id
+    const followerActorId = serverActor.id
 
     const scopeOptions: AvailableForListIDsOptions = {
       serverAccountId: serverActor.Account.id,
-      actorId,
+      followerActorId,
       includeLocalVideos: true
     }
 
@@ -1341,7 +1397,7 @@ export class VideoModel extends Model<VideoModel> {
     }
 
     const [ count, rowsId ] = await Promise.all([
-      countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined),
+      countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve<number>(undefined),
       VideoModel.scope(idsScope).findAll(query)
     ])
     const ids = rowsId.map(r => r.id)
@@ -1481,6 +1537,10 @@ export class VideoModel extends Model<VideoModel> {
     videoFile.infoHash = parsedTorrent.infoHash
   }
 
+  getWatchStaticPath () {
+    return '/videos/watch/' + this.uuid
+  }
+
   getEmbedStaticPath () {
     return '/videos/embed/' + this.uuid
   }
@@ -1538,8 +1598,10 @@ export class VideoModel extends Model<VideoModel> {
       .catch(err => logger.warn('Cannot delete preview %s.', previewPath, { err }))
   }
 
-  removeFile (videoFile: VideoFileModel) {
-    const filePath = join(CONFIG.STORAGE.VIDEOS_DIR, this.getVideoFilename(videoFile))
+  removeFile (videoFile: VideoFileModel, isRedundancy = false) {
+    const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR
+
+    const filePath = join(baseDir, this.getVideoFilename(videoFile))
     return remove(filePath)
       .catch(err => logger.warn('Cannot delete file %s.', filePath, { err }))
   }
@@ -1617,6 +1679,10 @@ export class VideoModel extends Model<VideoModel> {
     return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
   }
 
+  getVideoRedundancyUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+    return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getVideoFilename(videoFile)
+  }
+
   getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
     return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
   }