server/models/redundancy/video-redundancy.ts

   1 import {
   2   AfterDestroy,
   3   AllowNull,
   4   BelongsTo,
   5   Column,
   6   CreatedAt,
   7   DataType,
   8   ForeignKey,
   9   Is,
  10   Model,
  11   Scopes,
  12   Sequelize,
  13   Table,
  14   UpdatedAt
  15 } from 'sequelize-typescript'
  16 import { ActorModel } from '../activitypub/actor'
  17 import { getVideoSort, throwIfNotValid } from '../utils'
  18 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  19 import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers'
  20 import { VideoFileModel } from '../video/video-file'
  21 import { getServerActor } from '../../helpers/utils'
  22 import { VideoModel } from '../video/video'
  23 import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
  24 import { logger } from '../../helpers/logger'
  25 import { CacheFileObject } from '../../../shared'
  26 import { VideoChannelModel } from '../video/video-channel'
  27 import { ServerModel } from '../server/server'
  28 import { sample } from 'lodash'
  29 import { isTestInstance } from '../../helpers/core-utils'
  30
  31 export enum ScopeNames {
  32   WITH_VIDEO = 'WITH_VIDEO'
  33 }
  34
  35 @Scopes({
  36   [ ScopeNames.WITH_VIDEO ]: {
  37     include: [
  38       {
  39         model: () => VideoFileModel,
  40         required: true,
  41         include: [
  42           {
  43             model: () => VideoModel,
  44             required: true
  45           }
  46         ]
  47       }
  48     ]
  49   }
  50 })
  51
  52 @Table({
  53   tableName: 'videoRedundancy',
  54   indexes: [
  55     {
  56       fields: [ 'videoFileId' ]
  57     },
  58     {
  59       fields: [ 'actorId' ]
  60     },
  61     {
  62       fields: [ 'url' ],
  63       unique: true
  64     }
  65   ]
  66 })
  67 export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
  68
  69   @CreatedAt
  70   createdAt: Date
  71
  72   @UpdatedAt
  73   updatedAt: Date
  74
  75   @AllowNull(false)
  76   @Column
  77   expiresOn: Date
  78
  79   @AllowNull(false)
  80   @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
  81   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
  82   fileUrl: string
  83
  84   @AllowNull(false)
  85   @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
  86   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
  87   url: string
  88
  89   @AllowNull(true)
  90   @Column
  91   strategy: string // Only used by us
  92
  93   @ForeignKey(() => VideoFileModel)
  94   @Column
  95   videoFileId: number
  96
  97   @BelongsTo(() => VideoFileModel, {
  98     foreignKey: {
  99       allowNull: false
 100     },
 101     onDelete: 'cascade'
 102   })
 103   VideoFile: VideoFileModel
 104
 105   @ForeignKey(() => ActorModel)
 106   @Column
 107   actorId: number
 108
 109   @BelongsTo(() => ActorModel, {
 110     foreignKey: {
 111       allowNull: false
 112     },
 113     onDelete: 'cascade'
 114   })
 115   Actor: ActorModel
 116
 117   @AfterDestroy
 118   static removeFilesAndSendDelete (instance: VideoRedundancyModel) {
 119     // Not us
 120     if (!instance.strategy) return
 121
 122     logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution)
 123
 124     return instance.VideoFile.Video.removeFile(instance.VideoFile)
 125   }
 126
 127   static loadByFileId (videoFileId: number) {
 128     const query = {
 129       where: {
 130         videoFileId
 131       }
 132     }
 133
 134     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 135   }
 136
 137   static loadByUrl (url: string) {
 138     const query = {
 139       where: {
 140         url
 141       }
 142     }
 143
 144     return VideoRedundancyModel.findOne(query)
 145   }
 146
 147   static getVideoSample (rows: { id: number }[]) {
 148     const ids = rows.map(r => r.id)
 149     const id = sample(ids)
 150
 151     return VideoModel.loadWithFile(id, undefined, !isTestInstance())
 152   }
 153
 154   static async findMostViewToDuplicate (randomizedFactor: number) {
 155     // On VideoModel!
 156     const query = {
 157       attributes: [ 'id', 'views' ],
 158       logging: !isTestInstance(),
 159       limit: randomizedFactor,
 160       order: getVideoSort('-views'),
 161       include: [
 162         await VideoRedundancyModel.buildVideoFileForDuplication(),
 163         VideoRedundancyModel.buildServerRedundancyInclude()
 164       ]
 165     }
 166
 167     const rows = await VideoModel.unscoped().findAll(query)
 168
 169     return VideoRedundancyModel.getVideoSample(rows as { id: number }[])
 170   }
 171
 172   static async findTrendingToDuplicate (randomizedFactor: number) {
 173     // On VideoModel!
 174     const query = {
 175       attributes: [ 'id', 'views' ],
 176       subQuery: false,
 177       logging: !isTestInstance(),
 178       group: 'VideoModel.id',
 179       limit: randomizedFactor,
 180       order: getVideoSort('-trending'),
 181       include: [
 182         await VideoRedundancyModel.buildVideoFileForDuplication(),
 183         VideoRedundancyModel.buildServerRedundancyInclude(),
 184
 185         VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
 186       ]
 187     }
 188
 189     const rows = await VideoModel.unscoped().findAll(query)
 190
 191     return VideoRedundancyModel.getVideoSample(rows as { id: number }[])
 192   }
 193
 194   static async getVideoFiles (strategy: VideoRedundancyStrategy) {
 195     const actor = await getServerActor()
 196
 197     const queryVideoFiles = {
 198       logging: !isTestInstance(),
 199       where: {
 200         actorId: actor.id,
 201         strategy
 202       }
 203     }
 204
 205     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
 206                                .findAll(queryVideoFiles)
 207   }
 208
 209   static listAllExpired () {
 210     const query = {
 211       logging: !isTestInstance(),
 212       where: {
 213         expiresOn: {
 214           [ Sequelize.Op.lt ]: new Date()
 215         }
 216       }
 217     }
 218
 219     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
 220                                .findAll(query)
 221   }
 222
 223   toActivityPubObject (): CacheFileObject {
 224     return {
 225       id: this.url,
 226       type: 'CacheFile' as 'CacheFile',
 227       object: this.VideoFile.Video.url,
 228       expires: this.expiresOn.toISOString(),
 229       url: {
 230         type: 'Link',
 231         mimeType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any,
 232         href: this.fileUrl,
 233         height: this.VideoFile.resolution,
 234         size: this.VideoFile.size,
 235         fps: this.VideoFile.fps
 236       }
 237     }
 238   }
 239
 240   // Don't include video files we already duplicated
 241   private static async buildVideoFileForDuplication () {
 242     const actor = await getServerActor()
 243
 244     const notIn = Sequelize.literal(
 245       '(' +
 246         `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` +
 247       ')'
 248     )
 249
 250     return {
 251       attributes: [],
 252       model: VideoFileModel.unscoped(),
 253       required: true,
 254       where: {
 255         id: {
 256           [ Sequelize.Op.notIn ]: notIn
 257         }
 258       }
 259     }
 260   }
 261
 262   private static buildServerRedundancyInclude () {
 263     return {
 264       attributes: [],
 265       model: VideoChannelModel.unscoped(),
 266       required: true,
 267       include: [
 268         {
 269           attributes: [],
 270           model: ActorModel.unscoped(),
 271           required: true,
 272           include: [
 273             {
 274               attributes: [],
 275               model: ServerModel.unscoped(),
 276               required: true,
 277               where: {
 278                 redundancyAllowed: true
 279               }
 280             }
 281           ]
 282         }
 283       ]
 284     }
 285   }
 286 }