Basic video redundancy implementation

6 files changed, 344 insertions, 37 deletions

diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts
index 8bc095997..27bb43dae 100644
--- a/server/models/activitypub/actor-follow.ts
+++ b/server/models/activitypub/actor-follow.ts
@@ -19,7 +19,7 @@ import {
   UpdatedAt
 } from 'sequelize-typescript'
 import { FollowState } from '../../../shared/models/actors'
-import { AccountFollow } from '../../../shared/models/actors/follow.model'
+import { ActorFollow } from '../../../shared/models/actors/follow.model'
 import { logger } from '../../helpers/logger'
 import { getServerActor } from '../../helpers/utils'
 import { ACTOR_FOLLOW_SCORE } from '../../initializers'
@@ -529,7 +529,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
     return ActorFollowModel.findAll(query)
   }
 
-  toFormattedJSON (): AccountFollow {
+  toFormattedJSON (): ActorFollow {
     const follower = this.ActorFollower.toFormattedJSON()
     const following = this.ActorFollowing.toFormattedJSON()
 
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index 119d0c1da..ef8dd9f7c 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -76,7 +76,13 @@ export const unusedActorAttributesForAPI = [
       },
       {
         model: () => VideoChannelModel.unscoped(),
-        required: false
+        required: false,
+        include: [
+          {
+            model: () => AccountModel,
+            required: true
+          }
+        ]
       },
       {
         model: () => ServerModel,
@@ -337,6 +343,7 @@ export class ActorModel extends Model<ActorModel> {
       uuid: this.uuid,
       name: this.preferredUsername,
       host: this.getHost(),
+      hostRedundancyAllowed: this.getRedundancyAllowed(),
       followingCount: this.followingCount,
       followersCount: this.followersCount,
       avatar,
@@ -440,6 +447,10 @@ export class ActorModel extends Model<ActorModel> {
     return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST
   }
 
+  getRedundancyAllowed () {
+    return this.Server ? this.Server.redundancyAllowed : false
+  }
+
   getAvatarUrl () {
     if (!this.avatarId) return undefined
 
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
new file mode 100644
index 000000000..48ec77206
--- /dev/null
+++ b/server/models/redundancy/video-redundancy.ts
@@ -0,0 +1,249 @@
+import {
+  AfterDestroy,
+  AllowNull,
+  BelongsTo,
+  Column,
+  CreatedAt,
+  DataType,
+  ForeignKey,
+  Is,
+  Model,
+  Scopes,
+  Sequelize,
+  Table,
+  UpdatedAt
+} from 'sequelize-typescript'
+import { ActorModel } from '../activitypub/actor'
+import { throwIfNotValid } from '../utils'
+import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
+import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers'
+import { VideoFileModel } from '../video/video-file'
+import { isDateValid } from '../../helpers/custom-validators/misc'
+import { getServerActor } from '../../helpers/utils'
+import { VideoModel } from '../video/video'
+import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
+import { logger } from '../../helpers/logger'
+import { CacheFileObject } from '../../../shared'
+import { VideoChannelModel } from '../video/video-channel'
+import { ServerModel } from '../server/server'
+import { sample } from 'lodash'
+import { isTestInstance } from '../../helpers/core-utils'
+
+export enum ScopeNames {
+  WITH_VIDEO = 'WITH_VIDEO'
+}
+
+@Scopes({
+  [ ScopeNames.WITH_VIDEO ]: {
+    include: [
+      {
+        model: () => VideoFileModel,
+        required: true,
+        include: [
+          {
+            model: () => VideoModel,
+            required: true
+          }
+        ]
+      }
+    ]
+  }
+})
+
+@Table({
+  tableName: 'videoRedundancy',
+  indexes: [
+    {
+      fields: [ 'videoFileId' ]
+    },
+    {
+      fields: [ 'actorId' ]
+    },
+    {
+      fields: [ 'url' ],
+      unique: true
+    }
+  ]
+})
+export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
+
+  @CreatedAt
+  createdAt: Date
+
+  @UpdatedAt
+  updatedAt: Date
+
+  @AllowNull(false)
+  @Column
+  expiresOn: Date
+
+  @AllowNull(false)
+  @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
+  fileUrl: string
+
+  @AllowNull(false)
+  @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
+  url: string
+
+  @AllowNull(true)
+  @Column
+  strategy: string // Only used by us
+
+  @ForeignKey(() => VideoFileModel)
+  @Column
+  videoFileId: number
+
+  @BelongsTo(() => VideoFileModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+  VideoFile: VideoFileModel
+
+  @ForeignKey(() => ActorModel)
+  @Column
+  actorId: number
+
+  @BelongsTo(() => ActorModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+  Actor: ActorModel
+
+  @AfterDestroy
+  static removeFilesAndSendDelete (instance: VideoRedundancyModel) {
+    // Not us
+    if (!instance.strategy) return
+
+    logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution)
+
+    return instance.VideoFile.Video.removeFile(instance.VideoFile)
+  }
+
+  static loadByFileId (videoFileId: number) {
+    const query = {
+      where: {
+        videoFileId
+      }
+    }
+
+    return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
+  }
+
+  static loadByUrl (url: string) {
+    const query = {
+      where: {
+        url
+      }
+    }
+
+    return VideoRedundancyModel.findOne(query)
+  }
+
+  static async findMostViewToDuplicate (randomizedFactor: number) {
+    // On VideoModel!
+    const query = {
+      logging: !isTestInstance(),
+      limit: randomizedFactor,
+      order: [ [ 'views', 'DESC' ] ],
+      include: [
+        {
+          model: VideoFileModel.unscoped(),
+          required: true,
+          where: {
+            id: {
+              [ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn()
+            }
+          }
+        },
+        {
+          attributes: [],
+          model: VideoChannelModel.unscoped(),
+          required: true,
+          include: [
+            {
+              attributes: [],
+              model: ActorModel.unscoped(),
+              required: true,
+              include: [
+                {
+                  attributes: [],
+                  model: ServerModel.unscoped(),
+                  required: true,
+                  where: {
+                    redundancyAllowed: true
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+
+    const rows = await VideoModel.unscoped().findAll(query)
+
+    return sample(rows)
+  }
+
+  static async getVideoFiles (strategy: VideoRedundancyStrategy) {
+    const actor = await getServerActor()
+
+    const queryVideoFiles = {
+      logging: !isTestInstance(),
+      where: {
+        actorId: actor.id,
+        strategy
+      }
+    }
+
+    return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
+                               .findAll(queryVideoFiles)
+  }
+
+  static listAllExpired () {
+    const query = {
+      logging: !isTestInstance(),
+      where: {
+        expiresOn: {
+          [Sequelize.Op.lt]: new Date()
+        }
+      }
+    }
+
+    return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
+                               .findAll(query)
+  }
+
+  toActivityPubObject (): CacheFileObject {
+    return {
+      id: this.url,
+      type: 'CacheFile' as 'CacheFile',
+      object: this.VideoFile.Video.url,
+      expires: this.expiresOn.toISOString(),
+      url: {
+        type: 'Link',
+        mimeType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any,
+        href: this.fileUrl,
+        height: this.VideoFile.resolution,
+        size: this.VideoFile.size,
+        fps: this.VideoFile.fps
+      }
+    }
+  }
+
+  private static async buildExcludeIn () {
+    const actor = await getServerActor()
+
+    return Sequelize.literal(
+      '(' +
+        `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` +
+      ')'
+    )
+  }
+}
diff --git a/server/models/server/server.ts b/server/models/server/server.ts
index 9749f503e..ca3b24d51 100644
--- a/server/models/server/server.ts
+++ b/server/models/server/server.ts
@@ -1,4 +1,4 @@
-import { AllowNull, Column, CreatedAt, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { AllowNull, Column, CreatedAt, Default, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
 import { isHostValid } from '../../helpers/custom-validators/servers'
 import { ActorModel } from '../activitypub/actor'
 import { throwIfNotValid } from '../utils'
@@ -19,6 +19,11 @@ export class ServerModel extends Model<ServerModel> {
   @Column
   host: string
 
+  @AllowNull(false)
+  @Default(false)
+  @Column
+  redundancyAllowed: boolean
+
   @CreatedAt
   createdAt: Date
 
@@ -34,4 +39,14 @@ export class ServerModel extends Model<ServerModel> {
     hooks: true
   })
   Actors: ActorModel[]
+
+  static loadByHost (host: string) {
+    const query = {
+      where: {
+        host
+      }
+    }
+
+    return ServerModel.findOne(query)
+  }
 }
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 3bc4855f3..0907ea569 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -1,5 +1,18 @@
 import { values } from 'lodash'
-import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import {
+  AllowNull,
+  BelongsTo,
+  Column,
+  CreatedAt,
+  DataType,
+  Default,
+  ForeignKey,
+  HasMany,
+  Is,
+  Model,
+  Table,
+  UpdatedAt
+} from 'sequelize-typescript'
 import {
   isVideoFileInfoHashValid,
   isVideoFileResolutionValid,
@@ -10,6 +23,7 @@ import { CONSTRAINTS_FIELDS } from '../../initializers'
 import { throwIfNotValid } from '../utils'
 import { VideoModel } from './video'
 import * as Sequelize from 'sequelize'
+import { VideoRedundancyModel } from '../redundancy/video-redundancy'
 
 @Table({
   tableName: 'videoFile',
@@ -70,6 +84,15 @@ export class VideoFileModel extends Model<VideoFileModel> {
   })
   Video: VideoModel
 
+  @HasMany(() => VideoRedundancyModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'CASCADE',
+    hooks: true
+  })
+  RedundancyVideos: VideoRedundancyModel[]
+
   static isInfohashExists (infoHash: string) {
     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
     const options = {
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 86316653f..27c631dcd 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -27,13 +27,13 @@ import {
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
-import { VideoPrivacy, VideoResolution, VideoState } from '../../../shared'
+import { ActivityUrlObject, VideoPrivacy, VideoResolution, VideoState } from '../../../shared'
 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
 import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
 import { VideoFilter } from '../../../shared/models/videos/video-query.type'
 import { createTorrentPromise, peertubeTruncate } from '../../helpers/core-utils'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
-import { isBooleanValid } from '../../helpers/custom-validators/misc'
+import { isArray, isBooleanValid } from '../../helpers/custom-validators/misc'
 import {
   isVideoCategoryValid,
   isVideoDescriptionValid,
@@ -90,6 +90,7 @@ import { VideoCaptionModel } from './video-caption'
 import { VideoBlacklistModel } from './video-blacklist'
 import { copy, remove, rename, stat, writeFile } from 'fs-extra'
 import { VideoViewModel } from './video-views'
+import { VideoRedundancyModel } from '../redundancy/video-redundancy'
 
 // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
 const indexes: Sequelize.DefineIndexesOptions[] = [
@@ -470,7 +471,13 @@ type AvailableForListIDsOptions = {
     include: [
       {
         model: () => VideoFileModel.unscoped(),
-        required: false
+        required: false,
+        include: [
+          {
+            model: () => VideoRedundancyModel.unscoped(),
+            required: false
+          }
+        ]
       }
     ]
   },
@@ -633,6 +640,7 @@ export class VideoModel extends Model<VideoModel> {
       name: 'videoId',
       allowNull: false
     },
+    hooks: true,
     onDelete: 'cascade'
   })
   VideoFiles: VideoFileModel[]
@@ -1325,9 +1333,7 @@ export class VideoModel extends Model<VideoModel> {
         [ CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + '/tracker/socket' ],
         [ CONFIG.WEBSERVER.URL + '/tracker/announce' ]
       ],
-      urlList: [
-        CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
-      ]
+      urlList: [ CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) ]
     }
 
     const torrent = await createTorrentPromise(this.getVideoFilePath(videoFile), options)
@@ -1535,11 +1541,11 @@ export class VideoModel extends Model<VideoModel> {
       }
     }
 
-    const url = []
+    const url: ActivityUrlObject[] = []
     for (const file of this.VideoFiles) {
       url.push({
         type: 'Link',
-        mimeType: VIDEO_EXT_MIMETYPE[ file.extname ],
+        mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any,
         href: this.getVideoFileUrl(file, baseUrlHttp),
         height: file.resolution,
         size: file.size,
@@ -1548,14 +1554,14 @@ export class VideoModel extends Model<VideoModel> {
 
       url.push({
         type: 'Link',
-        mimeType: 'application/x-bittorrent',
+        mimeType: 'application/x-bittorrent' as 'application/x-bittorrent',
         href: this.getTorrentUrl(file, baseUrlHttp),
         height: file.resolution
       })
 
       url.push({
         type: 'Link',
-        mimeType: 'application/x-bittorrent;x-scheme-handler/magnet',
+        mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet',
         href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs),
         height: file.resolution
       })
@@ -1796,7 +1802,7 @@ export class VideoModel extends Model<VideoModel> {
       (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL
   }
 
-  private getBaseUrls () {
+  getBaseUrls () {
     let baseUrlHttp
     let baseUrlWs
 
@@ -1811,39 +1817,42 @@ export class VideoModel extends Model<VideoModel> {
     return { baseUrlHttp, baseUrlWs }
   }
 
-  private getThumbnailUrl (baseUrlHttp: string) {
+  generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
+    const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
+    const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
+    let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
+
+    const redundancies = videoFile.RedundancyVideos
+    if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl))
+
+    const magnetHash = {
+      xs,
+      announce,
+      urlList,
+      infoHash: videoFile.infoHash,
+      name: this.name
+    }
+
+    return magnetUtil.encode(magnetHash)
+  }
+
+  getThumbnailUrl (baseUrlHttp: string) {
     return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName()
   }
 
-  private getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+  getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
     return baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
   }
 
-  private getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+  getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
     return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
   }
 
-  private getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+  getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
     return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
   }
 
-  private getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+  getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
     return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
   }
-
-  private generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
-    const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
-    const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
-    const urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
-
-    const magnetHash = {
-      xs,
-      announce,
-      urlList,
-      infoHash: videoFile.infoHash,
-      name: this.name
-    }
-
-    return magnetUtil.encode(magnetHash)
-  }
 }