9 files changed, 293 insertions, 116 deletions

diff --git a/server/models/account/account-blocklist.ts b/server/models/account/account-blocklist.ts
index 6ebe32556..e2f66d733 100644
--- a/server/models/account/account-blocklist.ts
+++ b/server/models/account/account-blocklist.ts
@@ -80,7 +80,7 @@ export class AccountBlocklistModel extends Model<AccountBlocklistModel> {
       attributes: [ 'accountId', 'id' ],
       where: {
         accountId: {
-          [Op.in]: accountIds // FIXME: sequelize ANY seems broken
+          [Op.in]: accountIds
         },
         targetAccountId
       },
diff --git a/server/models/account/user-notification.ts b/server/models/account/user-notification.ts
index a05f30175..f649b8f95 100644
--- a/server/models/account/user-notification.ts
+++ b/server/models/account/user-notification.ts
@@ -363,7 +363,7 @@ export class UserNotificationModel extends Model<UserNotificationModel> {
       where: {
         userId,
         id: {
-          [Op.in]: notificationIds // FIXME: sequelize ANY seems broken
+          [Op.in]: notificationIds
         }
       }
     }
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index 007647ced..d651a281a 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -43,7 +43,7 @@ import {
   MActorFull,
   MActorHost,
   MActorServer,
-  MActorSummaryFormattable,
+  MActorSummaryFormattable, MActorUrl,
   MActorWithInboxes
 } from '../../typings/models'
 import * as Bluebird from 'bluebird'
@@ -276,7 +276,8 @@ export class ActorModel extends Model<ActorModel> {
   })
   VideoChannel: VideoChannelModel
 
-  private static cache: { [ id: string ]: any } = {}
+  private static localNameCache: { [ id: string ]: any } = {}
+  private static localUrlCache: { [ id: string ]: any } = {}
 
   static load (id: number): Bluebird<MActor> {
     return ActorModel.unscoped().findByPk(id)
@@ -345,8 +346,8 @@ export class ActorModel extends Model<ActorModel> {
 
   static loadLocalByName (preferredUsername: string, transaction?: Transaction): Bluebird<MActorFull> {
     // The server actor never change, so we can easily cache it
-    if (preferredUsername === SERVER_ACTOR_NAME && ActorModel.cache[preferredUsername]) {
-      return Bluebird.resolve(ActorModel.cache[preferredUsername])
+    if (preferredUsername === SERVER_ACTOR_NAME && ActorModel.localNameCache[preferredUsername]) {
+      return Bluebird.resolve(ActorModel.localNameCache[preferredUsername])
     }
 
     const query = {
@@ -361,7 +362,33 @@ export class ActorModel extends Model<ActorModel> {
                      .findOne(query)
                      .then(actor => {
                        if (preferredUsername === SERVER_ACTOR_NAME) {
-                         ActorModel.cache[ preferredUsername ] = actor
+                         ActorModel.localNameCache[ preferredUsername ] = actor
+                       }
+
+                       return actor
+                     })
+  }
+
+  static loadLocalUrlByName (preferredUsername: string, transaction?: Transaction): Bluebird<MActorUrl> {
+    // The server actor never change, so we can easily cache it
+    if (preferredUsername === SERVER_ACTOR_NAME && ActorModel.localUrlCache[preferredUsername]) {
+      return Bluebird.resolve(ActorModel.localUrlCache[preferredUsername])
+    }
+
+    const query = {
+      attributes: [ 'url' ],
+      where: {
+        preferredUsername,
+        serverId: null
+      },
+      transaction
+    }
+
+    return ActorModel.unscoped()
+                     .findOne(query)
+                     .then(actor => {
+                       if (preferredUsername === SERVER_ACTOR_NAME) {
+                         ActorModel.localUrlCache[ preferredUsername ] = actor
                        }
 
                        return actor
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index 8c9a7eabf..4e66d72e3 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -13,13 +13,13 @@ import {
   UpdatedAt
 } from 'sequelize-typescript'
 import { ActorModel } from '../activitypub/actor'
-import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
+import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
 import { VideoFileModel } from '../video/video-file'
 import { getServerActor } from '../../helpers/utils'
 import { VideoModel } from '../video/video'
-import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
+import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
 import { logger } from '../../helpers/logger'
 import { CacheFileObject, VideoPrivacy } from '../../../shared'
 import { VideoChannelModel } from '../video/video-channel'
@@ -27,10 +27,16 @@ import { ServerModel } from '../server/server'
 import { sample } from 'lodash'
 import { isTestInstance } from '../../helpers/core-utils'
 import * as Bluebird from 'bluebird'
-import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize'
+import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
 import { CONFIG } from '../../initializers/config'
-import { MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models'
+import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models'
+import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
+import {
+  FileRedundancyInformation,
+  StreamingPlaylistRedundancyInformation,
+  VideoRedundancy
+} from '@shared/models/redundancy/video-redundancy.model'
 
 export enum ScopeNames {
   WITH_VIDEO = 'WITH_VIDEO'
@@ -86,7 +92,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
   @UpdatedAt
   updatedAt: Date
 
-  @AllowNull(false)
+  @AllowNull(true)
   @Column
   expiresOn: Date
 
@@ -193,6 +199,15 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
   }
 
+  static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird<MVideoRedundancyVideo> {
+    const query = {
+      where: { id },
+      transaction
+    }
+
+    return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
+  }
+
   static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> {
     const query = {
       where: {
@@ -394,7 +409,8 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
           [Op.ne]: actor.id
         },
         expiresOn: {
-          [ Op.lt ]: new Date()
+          [ Op.lt ]: new Date(),
+          [ Op.ne ]: null
         }
       }
     }
@@ -447,7 +463,112 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
     return VideoRedundancyModel.findAll(query)
   }
 
-  static async getStats (strategy: VideoRedundancyStrategy) {
+  static listForApi (options: {
+    start: number,
+    count: number,
+    sort: string,
+    target: VideoRedundanciesTarget,
+    strategy?: string
+  }) {
+    const { start, count, sort, target, strategy } = options
+    let redundancyWhere: WhereOptions = {}
+    let videosWhere: WhereOptions = {}
+    let redundancySqlSuffix = ''
+
+    if (target === 'my-videos') {
+      Object.assign(videosWhere, { remote: false })
+    } else if (target === 'remote-videos') {
+      Object.assign(videosWhere, { remote: true })
+      Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
+      redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
+    }
+
+    if (strategy) {
+      Object.assign(redundancyWhere, { strategy: strategy })
+    }
+
+    const videoFilterWhere = {
+      [Op.and]: [
+        {
+          [ Op.or ]: [
+            {
+              id: {
+                [ Op.in ]: literal(
+                  '(' +
+                  'SELECT "videoId" FROM "videoFile" ' +
+                  'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
+                  redundancySqlSuffix +
+                  ')'
+                )
+              }
+            },
+            {
+              id: {
+                [ Op.in ]: literal(
+                  '(' +
+                  'select "videoId" FROM "videoStreamingPlaylist" ' +
+                  'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
+                  redundancySqlSuffix +
+                  ')'
+                )
+              }
+            }
+          ]
+        },
+
+        videosWhere
+      ]
+    }
+
+    // /!\ On video model /!\
+    const findOptions = {
+      offset: start,
+      limit: count,
+      order: getSort(sort),
+      include: [
+        {
+          required: false,
+          model: VideoFileModel.unscoped(),
+          include: [
+            {
+              model: VideoRedundancyModel.unscoped(),
+              required: false,
+              where: redundancyWhere
+            }
+          ]
+        },
+        {
+          required: false,
+          model: VideoStreamingPlaylistModel.unscoped(),
+          include: [
+            {
+              model: VideoRedundancyModel.unscoped(),
+              required: false,
+              where: redundancyWhere
+            },
+            {
+              model: VideoFileModel.unscoped(),
+              required: false
+            }
+          ]
+        }
+      ],
+      where: videoFilterWhere
+    }
+
+    // /!\ On video model /!\
+    const countOptions = {
+      where: videoFilterWhere
+    }
+
+    return Promise.all([
+      VideoModel.findAll(findOptions),
+
+      VideoModel.count(countOptions)
+    ]).then(([ data, total ]) => ({ total, data }))
+  }
+
+  static async getStats (strategy: VideoRedundancyStrategyWithManual) {
     const actor = await getServerActor()
 
     const query: FindOptions = {
@@ -478,6 +599,53 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
       }))
   }
 
+  static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
+    let filesRedundancies: FileRedundancyInformation[] = []
+    let streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
+
+    for (const file of video.VideoFiles) {
+      for (const redundancy of file.RedundancyVideos) {
+        filesRedundancies.push({
+          id: redundancy.id,
+          fileUrl: redundancy.fileUrl,
+          strategy: redundancy.strategy,
+          createdAt: redundancy.createdAt,
+          updatedAt: redundancy.updatedAt,
+          expiresOn: redundancy.expiresOn,
+          size: file.size
+        })
+      }
+    }
+
+    for (const playlist of video.VideoStreamingPlaylists) {
+      const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
+
+      for (const redundancy of playlist.RedundancyVideos) {
+        streamingPlaylistsRedundancies.push({
+          id: redundancy.id,
+          fileUrl: redundancy.fileUrl,
+          strategy: redundancy.strategy,
+          createdAt: redundancy.createdAt,
+          updatedAt: redundancy.updatedAt,
+          expiresOn: redundancy.expiresOn,
+          size
+        })
+      }
+    }
+
+    return {
+      id: video.id,
+      name: video.name,
+      url: video.url,
+      uuid: video.uuid,
+
+      redundancies: {
+        files: filesRedundancies,
+        streamingPlaylists: streamingPlaylistsRedundancies
+      }
+    }
+  }
+
   getVideo () {
     if (this.VideoFile) return this.VideoFile.Video
 
@@ -494,7 +662,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
         id: this.url,
         type: 'CacheFile' as 'CacheFile',
         object: this.VideoStreamingPlaylist.Video.url,
-        expires: this.expiresOn.toISOString(),
+        expires: this.expiresOn ? this.expiresOn.toISOString() : null,
         url: {
           type: 'Link',
           mediaType: 'application/x-mpegURL',
@@ -507,7 +675,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
       id: this.url,
       type: 'CacheFile' as 'CacheFile',
       object: this.VideoFile.Video.url,
-      expires: this.expiresOn.toISOString(),
+      expires: this.expiresOn ? this.expiresOn.toISOString() : null,
       url: {
         type: 'Link',
         mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
diff --git a/server/models/server/server-blocklist.ts b/server/models/server/server-blocklist.ts
index b88df4fd5..883ae47ab 100644
--- a/server/models/server/server-blocklist.ts
+++ b/server/models/server/server-blocklist.ts
@@ -81,7 +81,7 @@ export class ServerBlocklistModel extends Model<ServerBlocklistModel> {
       attributes: [ 'accountId', 'id' ],
       where: {
         accountId: {
-          [Op.in]: accountIds // FIXME: sequelize ANY seems broken
+          [Op.in]: accountIds
         },
         targetServerId
       },
diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts
index eeb2a4afd..6335d44e4 100644
--- a/server/models/video/video-caption.ts
+++ b/server/models/video/video-caption.ts
@@ -120,7 +120,7 @@ export class VideoCaptionModel extends Model<VideoCaptionModel> {
       language
     }
 
-    return (VideoCaptionModel.upsert<VideoCaptionModel>(values, { transaction, returning: true }) as any) // FIXME: typings
+    return VideoCaptionModel.upsert(values, { transaction, returning: true })
       .then(([ caption ]) => caption)
   }
 
diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts
index e10adcb3a..4e98e7ba3 100644
--- a/server/models/video/video-channel.ts
+++ b/server/models/video/video-channel.ts
@@ -43,18 +43,6 @@ import {
   MChannelSummaryFormattable
 } from '../../typings/models/video'
 
-// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
-const indexes: ModelIndexesOptions[] = [
-  buildTrigramSearchIndex('video_channel_name_trigram', 'name'),
-
-  {
-    fields: [ 'accountId' ]
-  },
-  {
-    fields: [ 'actorId' ]
-  }
-]
-
 export enum ScopeNames {
   FOR_API = 'FOR_API',
   WITH_ACCOUNT = 'WITH_ACCOUNT',
@@ -176,7 +164,16 @@ export type SummaryOptions = {
 }))
 @Table({
   tableName: 'videoChannel',
-  indexes
+  indexes: [
+    buildTrigramSearchIndex('video_channel_name_trigram', 'name'),
+
+    {
+      fields: [ 'accountId' ]
+    },
+    {
+      fields: [ 'actorId' ]
+    }
+  ]
 })
 export class VideoChannelModel extends Model<VideoChannelModel> {
 
diff --git a/server/models/video/video-playlist.ts b/server/models/video/video-playlist.ts
index bcdda36e5..ba1fc23ea 100644
--- a/server/models/video/video-playlist.ts
+++ b/server/models/video/video-playlist.ts
@@ -369,7 +369,7 @@ export class VideoPlaylistModel extends Model<VideoPlaylistModel> {
           model: VideoPlaylistElementModel.unscoped(),
           where: {
             videoId: {
-              [Op.in]: videoIds // FIXME: sequelize ANY seems broken
+              [Op.in]: videoIds
             }
           },
           required: true
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index eacffe186..20e1f1c4a 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -1,18 +1,7 @@
 import * as Bluebird from 'bluebird'
 import { maxBy, minBy } from 'lodash'
 import { join } from 'path'
-import {
-  CountOptions,
-  FindOptions,
-  IncludeOptions,
-  ModelIndexesOptions,
-  Op,
-  QueryTypes,
-  ScopeOptions,
-  Sequelize,
-  Transaction,
-  WhereOptions
-} from 'sequelize'
+import { CountOptions, FindOptions, IncludeOptions, Op, QueryTypes, ScopeOptions, Sequelize, Transaction, WhereOptions } from 'sequelize'
 import {
   AllowNull,
   BeforeDestroy,
@@ -136,8 +125,7 @@ import {
   MVideoThumbnailBlacklist,
   MVideoWithAllFiles,
   MVideoWithFile,
-  MVideoWithRights,
-  MStreamingPlaylistFiles
+  MVideoWithRights
 } from '../../typings/models'
 import { MVideoFile, MVideoFileStreamingPlaylistVideo } from '../../typings/models/video/video-file'
 import { MThumbnail } from '../../typings/models/video/thumbnail'
@@ -145,74 +133,6 @@ import { VideoFile } from '@shared/models/videos/video-file.model'
 import { getHLSDirectory, getTorrentFileName, getTorrentFilePath, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
 import validator from 'validator'
 
-// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
-const indexes: (ModelIndexesOptions & { where?: WhereOptions })[] = [
-  buildTrigramSearchIndex('video_name_trigram', 'name'),
-
-  { fields: [ 'createdAt' ] },
-  {
-    fields: [
-      { name: 'publishedAt', order: 'DESC' },
-      { name: 'id', order: 'ASC' }
-    ]
-  },
-  { fields: [ 'duration' ] },
-  { fields: [ 'views' ] },
-  { fields: [ 'channelId' ] },
-  {
-    fields: [ 'originallyPublishedAt' ],
-    where: {
-      originallyPublishedAt: {
-        [Op.ne]: null
-      }
-    }
-  },
-  {
-    fields: [ 'category' ], // We don't care videos with an unknown category
-    where: {
-      category: {
-        [Op.ne]: null
-      }
-    }
-  },
-  {
-    fields: [ 'licence' ], // We don't care videos with an unknown licence
-    where: {
-      licence: {
-        [Op.ne]: null
-      }
-    }
-  },
-  {
-    fields: [ 'language' ], // We don't care videos with an unknown language
-    where: {
-      language: {
-        [Op.ne]: null
-      }
-    }
-  },
-  {
-    fields: [ 'nsfw' ], // Most of the videos are not NSFW
-    where: {
-      nsfw: true
-    }
-  },
-  {
-    fields: [ 'remote' ], // Only index local videos
-    where: {
-      remote: false
-    }
-  },
-  {
-    fields: [ 'uuid' ],
-    unique: true
-  },
-  {
-    fields: [ 'url' ],
-    unique: true
-  }
-]
-
 export enum ScopeNames {
   AVAILABLE_FOR_LIST_IDS = 'AVAILABLE_FOR_LIST_IDS',
   FOR_API = 'FOR_API',
@@ -292,7 +212,7 @@ export type AvailableForListIDsOptions = {
     if (options.ids) {
       query.where = {
         id: {
-          [ Op.in ]: options.ids // FIXME: sequelize ANY seems broken
+          [ Op.in ]: options.ids
         }
       }
     }
@@ -760,7 +680,72 @@ export type AvailableForListIDsOptions = {
 }))
 @Table({
   tableName: 'video',
-  indexes
+  indexes: [
+    buildTrigramSearchIndex('video_name_trigram', 'name'),
+
+    { fields: [ 'createdAt' ] },
+    {
+      fields: [
+        { name: 'publishedAt', order: 'DESC' },
+        { name: 'id', order: 'ASC' }
+      ]
+    },
+    { fields: [ 'duration' ] },
+    { fields: [ 'views' ] },
+    { fields: [ 'channelId' ] },
+    {
+      fields: [ 'originallyPublishedAt' ],
+      where: {
+        originallyPublishedAt: {
+          [Op.ne]: null
+        }
+      }
+    },
+    {
+      fields: [ 'category' ], // We don't care videos with an unknown category
+      where: {
+        category: {
+          [Op.ne]: null
+        }
+      }
+    },
+    {
+      fields: [ 'licence' ], // We don't care videos with an unknown licence
+      where: {
+        licence: {
+          [Op.ne]: null
+        }
+      }
+    },
+    {
+      fields: [ 'language' ], // We don't care videos with an unknown language
+      where: {
+        language: {
+          [Op.ne]: null
+        }
+      }
+    },
+    {
+      fields: [ 'nsfw' ], // Most of the videos are not NSFW
+      where: {
+        nsfw: true
+      }
+    },
+    {
+      fields: [ 'remote' ], // Only index local videos
+      where: {
+        remote: false
+      }
+    },
+    {
+      fields: [ 'uuid' ],
+      unique: true
+    },
+    {
+      fields: [ 'url' ],
+      unique: true
+    }
+  ]
 })
 export class VideoModel extends Model<VideoModel> {