server/models/redundancy/video-redundancy.ts

   1 import {
   2   AllowNull,
   3   BeforeDestroy,
   4   BelongsTo,
   5   Column,
   6   CreatedAt,
   7   DataType,
   8   ForeignKey,
   9   Is,
  10   Model,
  11   Scopes,
  12   Table,
  13   UpdatedAt
  14 } from 'sequelize-typescript'
  15 import { ActorModel } from '../activitypub/actor'
  16 import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
  17 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  18 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
  19 import { VideoFileModel } from '../video/video-file'
  20 import { getServerActor } from '../../helpers/utils'
  21 import { VideoModel } from '../video/video'
  22 import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
  23 import { logger } from '../../helpers/logger'
  24 import { CacheFileObject, VideoPrivacy } from '../../../shared'
  25 import { VideoChannelModel } from '../video/video-channel'
  26 import { ServerModel } from '../server/server'
  27 import { sample } from 'lodash'
  28 import { isTestInstance } from '../../helpers/core-utils'
  29 import * as Bluebird from 'bluebird'
  30 import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize'
  31 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
  32 import { CONFIG } from '../../initializers/config'
  33
  34 export enum ScopeNames {
  35   WITH_VIDEO = 'WITH_VIDEO'
  36 }
  37
  38 @Scopes(() => ({
  39   [ ScopeNames.WITH_VIDEO ]: {
  40     include: [
  41       {
  42         model: VideoFileModel,
  43         required: false,
  44         include: [
  45           {
  46             model: VideoModel,
  47             required: true
  48           }
  49         ]
  50       },
  51       {
  52         model: VideoStreamingPlaylistModel,
  53         required: false,
  54         include: [
  55           {
  56             model: VideoModel,
  57             required: true
  58           }
  59         ]
  60       }
  61     ]
  62   }
  63 }))
  64
  65 @Table({
  66   tableName: 'videoRedundancy',
  67   indexes: [
  68     {
  69       fields: [ 'videoFileId' ]
  70     },
  71     {
  72       fields: [ 'actorId' ]
  73     },
  74     {
  75       fields: [ 'url' ],
  76       unique: true
  77     }
  78   ]
  79 })
  80 export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
  81
  82   @CreatedAt
  83   createdAt: Date
  84
  85   @UpdatedAt
  86   updatedAt: Date
  87
  88   @AllowNull(false)
  89   @Column
  90   expiresOn: Date
  91
  92   @AllowNull(false)
  93   @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
  94   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
  95   fileUrl: string
  96
  97   @AllowNull(false)
  98   @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
  99   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
 100   url: string
 101
 102   @AllowNull(true)
 103   @Column
 104   strategy: string // Only used by us
 105
 106   @ForeignKey(() => VideoFileModel)
 107   @Column
 108   videoFileId: number
 109
 110   @BelongsTo(() => VideoFileModel, {
 111     foreignKey: {
 112       allowNull: true
 113     },
 114     onDelete: 'cascade'
 115   })
 116   VideoFile: VideoFileModel
 117
 118   @ForeignKey(() => VideoStreamingPlaylistModel)
 119   @Column
 120   videoStreamingPlaylistId: number
 121
 122   @BelongsTo(() => VideoStreamingPlaylistModel, {
 123     foreignKey: {
 124       allowNull: true
 125     },
 126     onDelete: 'cascade'
 127   })
 128   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 129
 130   @ForeignKey(() => ActorModel)
 131   @Column
 132   actorId: number
 133
 134   @BelongsTo(() => ActorModel, {
 135     foreignKey: {
 136       allowNull: false
 137     },
 138     onDelete: 'cascade'
 139   })
 140   Actor: ActorModel
 141
 142   @BeforeDestroy
 143   static async removeFile (instance: VideoRedundancyModel) {
 144     if (!instance.isOwned()) return
 145
 146     if (instance.videoFileId) {
 147       const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
 148
 149       const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
 150       logger.info('Removing duplicated video file %s.', logIdentifier)
 151
 152       videoFile.Video.removeFile(videoFile, true)
 153                .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
 154     }
 155
 156     if (instance.videoStreamingPlaylistId) {
 157       const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
 158
 159       const videoUUID = videoStreamingPlaylist.Video.uuid
 160       logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
 161
 162       videoStreamingPlaylist.Video.removeStreamingPlaylist(true)
 163                .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
 164     }
 165
 166     return undefined
 167   }
 168
 169   static async loadLocalByFileId (videoFileId: number) {
 170     const actor = await getServerActor()
 171
 172     const query = {
 173       where: {
 174         actorId: actor.id,
 175         videoFileId
 176       }
 177     }
 178
 179     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 180   }
 181
 182   static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number) {
 183     const actor = await getServerActor()
 184
 185     const query = {
 186       where: {
 187         actorId: actor.id,
 188         videoStreamingPlaylistId
 189       }
 190     }
 191
 192     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 193   }
 194
 195   static loadByUrl (url: string, transaction?: Transaction) {
 196     const query = {
 197       where: {
 198         url
 199       },
 200       transaction
 201     }
 202
 203     return VideoRedundancyModel.findOne(query)
 204   }
 205
 206   static async isLocalByVideoUUIDExists (uuid: string) {
 207     const actor = await getServerActor()
 208
 209     const query = {
 210       raw: true,
 211       attributes: [ 'id' ],
 212       where: {
 213         actorId: actor.id
 214       },
 215       include: [
 216         {
 217           attributes: [ ],
 218           model: VideoFileModel,
 219           required: true,
 220           include: [
 221             {
 222               attributes: [ ],
 223               model: VideoModel,
 224               required: true,
 225               where: {
 226                 uuid
 227               }
 228             }
 229           ]
 230         }
 231       ]
 232     }
 233
 234     return VideoRedundancyModel.findOne(query)
 235       .then(r => !!r)
 236   }
 237
 238   static async getVideoSample (p: Bluebird<VideoModel[]>) {
 239     const rows = await p
 240     if (rows.length === 0) return undefined
 241
 242     const ids = rows.map(r => r.id)
 243     const id = sample(ids)
 244
 245     return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
 246   }
 247
 248   static async findMostViewToDuplicate (randomizedFactor: number) {
 249     // On VideoModel!
 250     const query = {
 251       attributes: [ 'id', 'views' ],
 252       limit: randomizedFactor,
 253       order: getVideoSort('-views'),
 254       where: {
 255         privacy: VideoPrivacy.PUBLIC
 256       },
 257       include: [
 258         await VideoRedundancyModel.buildVideoFileForDuplication(),
 259         VideoRedundancyModel.buildServerRedundancyInclude()
 260       ]
 261     }
 262
 263     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 264   }
 265
 266   static async findTrendingToDuplicate (randomizedFactor: number) {
 267     // On VideoModel!
 268     const query = {
 269       attributes: [ 'id', 'views' ],
 270       subQuery: false,
 271       group: 'VideoModel.id',
 272       limit: randomizedFactor,
 273       order: getVideoSort('-trending'),
 274       where: {
 275         privacy: VideoPrivacy.PUBLIC
 276       },
 277       include: [
 278         await VideoRedundancyModel.buildVideoFileForDuplication(),
 279         VideoRedundancyModel.buildServerRedundancyInclude(),
 280
 281         VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
 282       ]
 283     }
 284
 285     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 286   }
 287
 288   static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
 289     // On VideoModel!
 290     const query = {
 291       attributes: [ 'id', 'publishedAt' ],
 292       limit: randomizedFactor,
 293       order: getVideoSort('-publishedAt'),
 294       where: {
 295         privacy: VideoPrivacy.PUBLIC,
 296         views: {
 297           [ Op.gte ]: minViews
 298         }
 299       },
 300       include: [
 301         await VideoRedundancyModel.buildVideoFileForDuplication(),
 302         VideoRedundancyModel.buildServerRedundancyInclude()
 303       ]
 304     }
 305
 306     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 307   }
 308
 309   static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) {
 310     const expiredDate = new Date()
 311     expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
 312
 313     const actor = await getServerActor()
 314
 315     const query = {
 316       where: {
 317         actorId: actor.id,
 318         strategy,
 319         createdAt: {
 320           [ Op.lt ]: expiredDate
 321         }
 322       }
 323     }
 324
 325     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
 326   }
 327
 328   static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
 329     const actor = await getServerActor()
 330     const redundancyInclude = {
 331       attributes: [],
 332       model: VideoRedundancyModel,
 333       required: true,
 334       where: {
 335         actorId: actor.id,
 336         strategy
 337       }
 338     }
 339
 340     const queryFiles: FindOptions = {
 341       include: [ redundancyInclude ]
 342     }
 343
 344     const queryStreamingPlaylists: FindOptions = {
 345       include: [
 346         {
 347           attributes: [],
 348           model: VideoModel.unscoped(),
 349           required: true,
 350           include: [
 351             {
 352               required: true,
 353               attributes: [],
 354               model: VideoStreamingPlaylistModel.unscoped(),
 355               include: [
 356                 redundancyInclude
 357               ]
 358             }
 359           ]
 360         }
 361       ]
 362     }
 363
 364     return Promise.all([
 365       VideoFileModel.aggregate('size', 'SUM', queryFiles),
 366       VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
 367     ]).then(([ r1, r2 ]) => {
 368       return parseAggregateResult(r1) + parseAggregateResult(r2)
 369     })
 370   }
 371
 372   static async listLocalExpired () {
 373     const actor = await getServerActor()
 374
 375     const query = {
 376       where: {
 377         actorId: actor.id,
 378         expiresOn: {
 379           [ Op.lt ]: new Date()
 380         }
 381       }
 382     }
 383
 384     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
 385   }
 386
 387   static async listRemoteExpired () {
 388     const actor = await getServerActor()
 389
 390     const query = {
 391       where: {
 392         actorId: {
 393           [Op.ne]: actor.id
 394         },
 395         expiresOn: {
 396           [ Op.lt ]: new Date()
 397         }
 398       }
 399     }
 400
 401     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
 402   }
 403
 404   static async listLocalOfServer (serverId: number) {
 405     const actor = await getServerActor()
 406     const buildVideoInclude = () => ({
 407       model: VideoModel,
 408       required: true,
 409       include: [
 410         {
 411           attributes: [],
 412           model: VideoChannelModel.unscoped(),
 413           required: true,
 414           include: [
 415             {
 416               attributes: [],
 417               model: ActorModel.unscoped(),
 418               required: true,
 419               where: {
 420                 serverId
 421               }
 422             }
 423           ]
 424         }
 425       ]
 426     })
 427
 428     const query = {
 429       where: {
 430         actorId: actor.id
 431       },
 432       include: [
 433         {
 434           model: VideoFileModel,
 435           required: false,
 436           include: [ buildVideoInclude() ]
 437         },
 438         {
 439           model: VideoStreamingPlaylistModel,
 440           required: false,
 441           include: [ buildVideoInclude() ]
 442         }
 443       ]
 444     }
 445
 446     return VideoRedundancyModel.findAll(query)
 447   }
 448
 449   static async getStats (strategy: VideoRedundancyStrategy) {
 450     const actor = await getServerActor()
 451
 452     const query: FindOptions = {
 453       raw: true,
 454       attributes: [
 455         [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
 456         [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
 457         [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
 458       ],
 459       where: {
 460         strategy,
 461         actorId: actor.id
 462       },
 463       include: [
 464         {
 465           attributes: [],
 466           model: VideoFileModel,
 467           required: true
 468         }
 469       ]
 470     }
 471
 472     return VideoRedundancyModel.findOne(query)
 473       .then((r: any) => ({
 474         totalUsed: parseAggregateResult(r.totalUsed),
 475         totalVideos: r.totalVideos,
 476         totalVideoFiles: r.totalVideoFiles
 477       }))
 478   }
 479
 480   getVideo () {
 481     if (this.VideoFile) return this.VideoFile.Video
 482
 483     return this.VideoStreamingPlaylist.Video
 484   }
 485
 486   isOwned () {
 487     return !!this.strategy
 488   }
 489
 490   toActivityPubObject (): CacheFileObject {
 491     if (this.VideoStreamingPlaylist) {
 492       return {
 493         id: this.url,
 494         type: 'CacheFile' as 'CacheFile',
 495         object: this.VideoStreamingPlaylist.Video.url,
 496         expires: this.expiresOn.toISOString(),
 497         url: {
 498           type: 'Link',
 499           mimeType: 'application/x-mpegURL',
 500           mediaType: 'application/x-mpegURL',
 501           href: this.fileUrl
 502         }
 503       }
 504     }
 505
 506     return {
 507       id: this.url,
 508       type: 'CacheFile' as 'CacheFile',
 509       object: this.VideoFile.Video.url,
 510       expires: this.expiresOn.toISOString(),
 511       url: {
 512         type: 'Link',
 513         mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
 514         mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
 515         href: this.fileUrl,
 516         height: this.VideoFile.resolution,
 517         size: this.VideoFile.size,
 518         fps: this.VideoFile.fps
 519       }
 520     }
 521   }
 522
 523   // Don't include video files we already duplicated
 524   private static async buildVideoFileForDuplication () {
 525     const actor = await getServerActor()
 526
 527     const notIn = literal(
 528       '(' +
 529         `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
 530       ')'
 531     )
 532
 533     return {
 534       attributes: [],
 535       model: VideoFileModel.unscoped(),
 536       required: true,
 537       where: {
 538         id: {
 539           [ Op.notIn ]: notIn
 540         }
 541       }
 542     }
 543   }
 544
 545   private static buildServerRedundancyInclude () {
 546     return {
 547       attributes: [],
 548       model: VideoChannelModel.unscoped(),
 549       required: true,
 550       include: [
 551         {
 552           attributes: [],
 553           model: ActorModel.unscoped(),
 554           required: true,
 555           include: [
 556             {
 557               attributes: [],
 558               model: ServerModel.unscoped(),
 559               required: true,
 560               where: {
 561                 redundancyAllowed: true
 562               }
 563             }
 564           ]
 565         }
 566       ]
 567     }
 568   }
 569 }