server/models/redundancy/video-redundancy.ts

   1 import {
   2   AllowNull,
   3   BeforeDestroy,
   4   BelongsTo,
   5   Column,
   6   CreatedAt,
   7   DataType,
   8   ForeignKey,
   9   Is,
  10   Model,
  11   Scopes,
  12   Table,
  13   UpdatedAt
  14 } from 'sequelize-typescript'
  15 import { ActorModel } from '../activitypub/actor'
  16 import { getVideoSort, throwIfNotValid } from '../utils'
  17 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  18 import { CONFIG, CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers'
  19 import { VideoFileModel } from '../video/video-file'
  20 import { getServerActor } from '../../helpers/utils'
  21 import { VideoModel } from '../video/video'
  22 import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
  23 import { logger } from '../../helpers/logger'
  24 import { CacheFileObject, VideoPrivacy } from '../../../shared'
  25 import { VideoChannelModel } from '../video/video-channel'
  26 import { ServerModel } from '../server/server'
  27 import { sample } from 'lodash'
  28 import { isTestInstance } from '../../helpers/core-utils'
  29 import * as Bluebird from 'bluebird'
  30 import * as Sequelize from 'sequelize'
  31
  32 export enum ScopeNames {
  33   WITH_VIDEO = 'WITH_VIDEO'
  34 }
  35
  36 @Scopes({
  37   [ ScopeNames.WITH_VIDEO ]: {
  38     include: [
  39       {
  40         model: () => VideoFileModel,
  41         required: true,
  42         include: [
  43           {
  44             model: () => VideoModel,
  45             required: true
  46           }
  47         ]
  48       }
  49     ]
  50   }
  51 })
  52
  53 @Table({
  54   tableName: 'videoRedundancy',
  55   indexes: [
  56     {
  57       fields: [ 'videoFileId' ]
  58     },
  59     {
  60       fields: [ 'actorId' ]
  61     },
  62     {
  63       fields: [ 'url' ],
  64       unique: true
  65     }
  66   ]
  67 })
  68 export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
  69
  70   @CreatedAt
  71   createdAt: Date
  72
  73   @UpdatedAt
  74   updatedAt: Date
  75
  76   @AllowNull(false)
  77   @Column
  78   expiresOn: Date
  79
  80   @AllowNull(false)
  81   @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
  82   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
  83   fileUrl: string
  84
  85   @AllowNull(false)
  86   @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
  87   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
  88   url: string
  89
  90   @AllowNull(true)
  91   @Column
  92   strategy: string // Only used by us
  93
  94   @ForeignKey(() => VideoFileModel)
  95   @Column
  96   videoFileId: number
  97
  98   @BelongsTo(() => VideoFileModel, {
  99     foreignKey: {
 100       allowNull: false
 101     },
 102     onDelete: 'cascade'
 103   })
 104   VideoFile: VideoFileModel
 105
 106   @ForeignKey(() => ActorModel)
 107   @Column
 108   actorId: number
 109
 110   @BelongsTo(() => ActorModel, {
 111     foreignKey: {
 112       allowNull: false
 113     },
 114     onDelete: 'cascade'
 115   })
 116   Actor: ActorModel
 117
 118   @BeforeDestroy
 119   static async removeFile (instance: VideoRedundancyModel) {
 120     if (!instance.isOwned()) return
 121
 122     const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
 123
 124     const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
 125     logger.info('Removing duplicated video file %s.', logIdentifier)
 126
 127     videoFile.Video.removeFile(videoFile, true)
 128              .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
 129
 130     return undefined
 131   }
 132
 133   static async loadLocalByFileId (videoFileId: number) {
 134     const actor = await getServerActor()
 135
 136     const query = {
 137       where: {
 138         actorId: actor.id,
 139         videoFileId
 140       }
 141     }
 142
 143     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 144   }
 145
 146   static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
 147     const query = {
 148       where: {
 149         url
 150       },
 151       transaction
 152     }
 153
 154     return VideoRedundancyModel.findOne(query)
 155   }
 156
 157   static async isLocalByVideoUUIDExists (uuid: string) {
 158     const actor = await getServerActor()
 159
 160     const query = {
 161       raw: true,
 162       attributes: [ 'id' ],
 163       where: {
 164         actorId: actor.id
 165       },
 166       include: [
 167         {
 168           attributes: [ ],
 169           model: VideoFileModel,
 170           required: true,
 171           include: [
 172             {
 173               attributes: [ ],
 174               model: VideoModel,
 175               required: true,
 176               where: {
 177                 uuid
 178               }
 179             }
 180           ]
 181         }
 182       ]
 183     }
 184
 185     return VideoRedundancyModel.findOne(query)
 186       .then(r => !!r)
 187   }
 188
 189   static async getVideoSample (p: Bluebird<VideoModel[]>) {
 190     const rows = await p
 191     const ids = rows.map(r => r.id)
 192     const id = sample(ids)
 193
 194     return VideoModel.loadWithFile(id, undefined, !isTestInstance())
 195   }
 196
 197   static async findMostViewToDuplicate (randomizedFactor: number) {
 198     // On VideoModel!
 199     const query = {
 200       attributes: [ 'id', 'views' ],
 201       limit: randomizedFactor,
 202       order: getVideoSort('-views'),
 203       where: {
 204         privacy: VideoPrivacy.PUBLIC
 205       },
 206       include: [
 207         await VideoRedundancyModel.buildVideoFileForDuplication(),
 208         VideoRedundancyModel.buildServerRedundancyInclude()
 209       ]
 210     }
 211
 212     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 213   }
 214
 215   static async findTrendingToDuplicate (randomizedFactor: number) {
 216     // On VideoModel!
 217     const query = {
 218       attributes: [ 'id', 'views' ],
 219       subQuery: false,
 220       group: 'VideoModel.id',
 221       limit: randomizedFactor,
 222       order: getVideoSort('-trending'),
 223       where: {
 224         privacy: VideoPrivacy.PUBLIC
 225       },
 226       include: [
 227         await VideoRedundancyModel.buildVideoFileForDuplication(),
 228         VideoRedundancyModel.buildServerRedundancyInclude(),
 229
 230         VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
 231       ]
 232     }
 233
 234     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 235   }
 236
 237   static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
 238     // On VideoModel!
 239     const query = {
 240       attributes: [ 'id', 'publishedAt' ],
 241       limit: randomizedFactor,
 242       order: getVideoSort('-publishedAt'),
 243       where: {
 244         privacy: VideoPrivacy.PUBLIC,
 245         views: {
 246           [ Sequelize.Op.gte ]: minViews
 247         }
 248       },
 249       include: [
 250         await VideoRedundancyModel.buildVideoFileForDuplication(),
 251         VideoRedundancyModel.buildServerRedundancyInclude()
 252       ]
 253     }
 254
 255     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 256   }
 257
 258   static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) {
 259     const expiredDate = new Date()
 260     expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
 261
 262     const actor = await getServerActor()
 263
 264     const query = {
 265       where: {
 266         actorId: actor.id,
 267         strategy,
 268         createdAt: {
 269           [ Sequelize.Op.lt ]: expiredDate
 270         }
 271       }
 272     }
 273
 274     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
 275   }
 276
 277   static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
 278     const actor = await getServerActor()
 279
 280     const options = {
 281       include: [
 282         {
 283           attributes: [],
 284           model: VideoRedundancyModel,
 285           required: true,
 286           where: {
 287             actorId: actor.id,
 288             strategy
 289           }
 290         }
 291       ]
 292     }
 293
 294     return VideoFileModel.sum('size', options as any) // FIXME: typings
 295       .then(v => {
 296         if (!v || isNaN(v)) return 0
 297
 298         return v
 299       })
 300   }
 301
 302   static async listLocalExpired () {
 303     const actor = await getServerActor()
 304
 305     const query = {
 306       where: {
 307         actorId: actor.id,
 308         expiresOn: {
 309           [ Sequelize.Op.lt ]: new Date()
 310         }
 311       }
 312     }
 313
 314     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
 315   }
 316
 317   static async listRemoteExpired () {
 318     const actor = await getServerActor()
 319
 320     const query = {
 321       where: {
 322         actorId: {
 323           [Sequelize.Op.ne]: actor.id
 324         },
 325         expiresOn: {
 326           [ Sequelize.Op.lt ]: new Date()
 327         }
 328       }
 329     }
 330
 331     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
 332   }
 333
 334   static async listLocalOfServer (serverId: number) {
 335     const actor = await getServerActor()
 336
 337     const query = {
 338       where: {
 339         actorId: actor.id
 340       },
 341       include: [
 342         {
 343           model: VideoFileModel,
 344           required: true,
 345           include: [
 346             {
 347               model: VideoModel,
 348               required: true,
 349               include: [
 350                 {
 351                   attributes: [],
 352                   model: VideoChannelModel.unscoped(),
 353                   required: true,
 354                   include: [
 355                     {
 356                       attributes: [],
 357                       model: ActorModel.unscoped(),
 358                       required: true,
 359                       where: {
 360                         serverId
 361                       }
 362                     }
 363                   ]
 364                 }
 365               ]
 366             }
 367           ]
 368         }
 369       ]
 370     }
 371
 372     return VideoRedundancyModel.findAll(query)
 373   }
 374
 375   static async getStats (strategy: VideoRedundancyStrategy) {
 376     const actor = await getServerActor()
 377
 378     const query = {
 379       raw: true,
 380       attributes: [
 381         [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ],
 382         [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', Sequelize.col('videoId'))), 'totalVideos' ],
 383         [ Sequelize.fn('COUNT', Sequelize.col('videoFileId')), 'totalVideoFiles' ]
 384       ],
 385       where: {
 386         strategy,
 387         actorId: actor.id
 388       },
 389       include: [
 390         {
 391           attributes: [],
 392           model: VideoFileModel,
 393           required: true
 394         }
 395       ]
 396     }
 397
 398     return VideoRedundancyModel.find(query as any) // FIXME: typings
 399       .then((r: any) => ({
 400         totalUsed: parseInt(r.totalUsed.toString(), 10),
 401         totalVideos: r.totalVideos,
 402         totalVideoFiles: r.totalVideoFiles
 403       }))
 404   }
 405
 406   isOwned () {
 407     return !!this.strategy
 408   }
 409
 410   toActivityPubObject (): CacheFileObject {
 411     return {
 412       id: this.url,
 413       type: 'CacheFile' as 'CacheFile',
 414       object: this.VideoFile.Video.url,
 415       expires: this.expiresOn.toISOString(),
 416       url: {
 417         type: 'Link',
 418         mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
 419         mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
 420         href: this.fileUrl,
 421         height: this.VideoFile.resolution,
 422         size: this.VideoFile.size,
 423         fps: this.VideoFile.fps
 424       }
 425     }
 426   }
 427
 428   // Don't include video files we already duplicated
 429   private static async buildVideoFileForDuplication () {
 430     const actor = await getServerActor()
 431
 432     const notIn = Sequelize.literal(
 433       '(' +
 434         `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` +
 435       ')'
 436     )
 437
 438     return {
 439       attributes: [],
 440       model: VideoFileModel.unscoped(),
 441       required: true,
 442       where: {
 443         id: {
 444           [ Sequelize.Op.notIn ]: notIn
 445         }
 446       }
 447     }
 448   }
 449
 450   private static buildServerRedundancyInclude () {
 451     return {
 452       attributes: [],
 453       model: VideoChannelModel.unscoped(),
 454       required: true,
 455       include: [
 456         {
 457           attributes: [],
 458           model: ActorModel.unscoped(),
 459           required: true,
 460           include: [
 461             {
 462               attributes: [],
 463               model: ServerModel.unscoped(),
 464               required: true,
 465               where: {
 466                 redundancyAllowed: true
 467               }
 468             }
 469           ]
 470         }
 471       ]
 472     }
 473   }
 474 }