server/models/redundancy/video-redundancy.ts

   1 import {
   2   AllowNull,
   3   BeforeDestroy,
   4   BelongsTo,
   5   Column,
   6   CreatedAt,
   7   DataType,
   8   ForeignKey,
   9   Is,
  10   Model,
  11   Scopes,
  12   Table,
  13   UpdatedAt
  14 } from 'sequelize-typescript'
  15 import { ActorModel } from '../activitypub/actor'
  16 import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
  17 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  18 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
  19 import { VideoFileModel } from '../video/video-file'
  20 import { getServerActor } from '../../helpers/utils'
  21 import { VideoModel } from '../video/video'
  22 import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
  23 import { logger } from '../../helpers/logger'
  24 import { CacheFileObject, VideoPrivacy } from '../../../shared'
  25 import { VideoChannelModel } from '../video/video-channel'
  26 import { ServerModel } from '../server/server'
  27 import { sample } from 'lodash'
  28 import { isTestInstance } from '../../helpers/core-utils'
  29 import * as Bluebird from 'bluebird'
  30 import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize'
  31 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
  32 import { CONFIG } from '../../initializers/config'
  33
  34 export enum ScopeNames {
  35   WITH_VIDEO = 'WITH_VIDEO'
  36 }
  37
  38 @Scopes(() => ({
  39   [ ScopeNames.WITH_VIDEO ]: {
  40     include: [
  41       {
  42         model: VideoFileModel,
  43         required: false,
  44         include: [
  45           {
  46             model: VideoModel,
  47             required: true
  48           }
  49         ]
  50       },
  51       {
  52         model: VideoStreamingPlaylistModel,
  53         required: false,
  54         include: [
  55           {
  56             model: VideoModel,
  57             required: true
  58           }
  59         ]
  60       }
  61     ]
  62   }
  63 }))
  64
  65 @Table({
  66   tableName: 'videoRedundancy',
  67   indexes: [
  68     {
  69       fields: [ 'videoFileId' ]
  70     },
  71     {
  72       fields: [ 'actorId' ]
  73     },
  74     {
  75       fields: [ 'url' ],
  76       unique: true
  77     }
  78   ]
  79 })
  80 export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
  81
  82   @CreatedAt
  83   createdAt: Date
  84
  85   @UpdatedAt
  86   updatedAt: Date
  87
  88   @AllowNull(false)
  89   @Column
  90   expiresOn: Date
  91
  92   @AllowNull(false)
  93   @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
  94   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
  95   fileUrl: string
  96
  97   @AllowNull(false)
  98   @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
  99   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
 100   url: string
 101
 102   @AllowNull(true)
 103   @Column
 104   strategy: string // Only used by us
 105
 106   @ForeignKey(() => VideoFileModel)
 107   @Column
 108   videoFileId: number
 109
 110   @BelongsTo(() => VideoFileModel, {
 111     foreignKey: {
 112       allowNull: true
 113     },
 114     onDelete: 'cascade'
 115   })
 116   VideoFile: VideoFileModel
 117
 118   @ForeignKey(() => VideoStreamingPlaylistModel)
 119   @Column
 120   videoStreamingPlaylistId: number
 121
 122   @BelongsTo(() => VideoStreamingPlaylistModel, {
 123     foreignKey: {
 124       allowNull: true
 125     },
 126     onDelete: 'cascade'
 127   })
 128   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 129
 130   @ForeignKey(() => ActorModel)
 131   @Column
 132   actorId: number
 133
 134   @BelongsTo(() => ActorModel, {
 135     foreignKey: {
 136       allowNull: false
 137     },
 138     onDelete: 'cascade'
 139   })
 140   Actor: ActorModel
 141
 142   @BeforeDestroy
 143   static async removeFile (instance: VideoRedundancyModel) {
 144     if (!instance.isOwned()) return
 145
 146     if (instance.videoFileId) {
 147       const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
 148
 149       const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
 150       logger.info('Removing duplicated video file %s.', logIdentifier)
 151
 152       videoFile.Video.removeFile(videoFile, true)
 153                .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
 154     }
 155
 156     if (instance.videoStreamingPlaylistId) {
 157       const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
 158
 159       const videoUUID = videoStreamingPlaylist.Video.uuid
 160       logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
 161
 162       videoStreamingPlaylist.Video.removeStreamingPlaylist(true)
 163                .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
 164     }
 165
 166     return undefined
 167   }
 168
 169   static async loadLocalByFileId (videoFileId: number) {
 170     const actor = await getServerActor()
 171
 172     const query = {
 173       where: {
 174         actorId: actor.id,
 175         videoFileId
 176       }
 177     }
 178
 179     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 180   }
 181
 182   static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number) {
 183     const actor = await getServerActor()
 184
 185     const query = {
 186       where: {
 187         actorId: actor.id,
 188         videoStreamingPlaylistId
 189       }
 190     }
 191
 192     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 193   }
 194
 195   static loadByUrl (url: string, transaction?: Transaction) {
 196     const query = {
 197       where: {
 198         url
 199       },
 200       transaction
 201     }
 202
 203     return VideoRedundancyModel.findOne(query)
 204   }
 205
 206   static async isLocalByVideoUUIDExists (uuid: string) {
 207     const actor = await getServerActor()
 208
 209     const query = {
 210       raw: true,
 211       attributes: [ 'id' ],
 212       where: {
 213         actorId: actor.id
 214       },
 215       include: [
 216         {
 217           attributes: [ ],
 218           model: VideoFileModel,
 219           required: true,
 220           include: [
 221             {
 222               attributes: [ ],
 223               model: VideoModel,
 224               required: true,
 225               where: {
 226                 uuid
 227               }
 228             }
 229           ]
 230         }
 231       ]
 232     }
 233
 234     return VideoRedundancyModel.findOne(query)
 235       .then(r => !!r)
 236   }
 237
 238   static async getVideoSample (p: Bluebird<VideoModel[]>) {
 239     const rows = await p
 240     const ids = rows.map(r => r.id)
 241     const id = sample(ids)
 242
 243     return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
 244   }
 245
 246   static async findMostViewToDuplicate (randomizedFactor: number) {
 247     // On VideoModel!
 248     const query = {
 249       attributes: [ 'id', 'views' ],
 250       limit: randomizedFactor,
 251       order: getVideoSort('-views'),
 252       where: {
 253         privacy: VideoPrivacy.PUBLIC
 254       },
 255       include: [
 256         await VideoRedundancyModel.buildVideoFileForDuplication(),
 257         VideoRedundancyModel.buildServerRedundancyInclude()
 258       ]
 259     }
 260
 261     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 262   }
 263
 264   static async findTrendingToDuplicate (randomizedFactor: number) {
 265     // On VideoModel!
 266     const query = {
 267       attributes: [ 'id', 'views' ],
 268       subQuery: false,
 269       group: 'VideoModel.id',
 270       limit: randomizedFactor,
 271       order: getVideoSort('-trending'),
 272       where: {
 273         privacy: VideoPrivacy.PUBLIC
 274       },
 275       include: [
 276         await VideoRedundancyModel.buildVideoFileForDuplication(),
 277         VideoRedundancyModel.buildServerRedundancyInclude(),
 278
 279         VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
 280       ]
 281     }
 282
 283     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 284   }
 285
 286   static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
 287     // On VideoModel!
 288     const query = {
 289       attributes: [ 'id', 'publishedAt' ],
 290       limit: randomizedFactor,
 291       order: getVideoSort('-publishedAt'),
 292       where: {
 293         privacy: VideoPrivacy.PUBLIC,
 294         views: {
 295           [ Op.gte ]: minViews
 296         }
 297       },
 298       include: [
 299         await VideoRedundancyModel.buildVideoFileForDuplication(),
 300         VideoRedundancyModel.buildServerRedundancyInclude()
 301       ]
 302     }
 303
 304     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 305   }
 306
 307   static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) {
 308     const expiredDate = new Date()
 309     expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
 310
 311     const actor = await getServerActor()
 312
 313     const query = {
 314       where: {
 315         actorId: actor.id,
 316         strategy,
 317         createdAt: {
 318           [ Op.lt ]: expiredDate
 319         }
 320       }
 321     }
 322
 323     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
 324   }
 325
 326   static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
 327     const actor = await getServerActor()
 328
 329     const query: FindOptions = {
 330       include: [
 331         {
 332           attributes: [],
 333           model: VideoRedundancyModel,
 334           required: true,
 335           where: {
 336             actorId: actor.id,
 337             strategy
 338           }
 339         }
 340       ]
 341     }
 342
 343     return VideoFileModel.aggregate('size', 'SUM', query)
 344       .then(result => parseAggregateResult(result))
 345   }
 346
 347   static async listLocalExpired () {
 348     const actor = await getServerActor()
 349
 350     const query = {
 351       where: {
 352         actorId: actor.id,
 353         expiresOn: {
 354           [ Op.lt ]: new Date()
 355         }
 356       }
 357     }
 358
 359     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
 360   }
 361
 362   static async listRemoteExpired () {
 363     const actor = await getServerActor()
 364
 365     const query = {
 366       where: {
 367         actorId: {
 368           [Op.ne]: actor.id
 369         },
 370         expiresOn: {
 371           [ Op.lt ]: new Date()
 372         }
 373       }
 374     }
 375
 376     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
 377   }
 378
 379   static async listLocalOfServer (serverId: number) {
 380     const actor = await getServerActor()
 381     const buildVideoInclude = () => ({
 382       model: VideoModel,
 383       required: true,
 384       include: [
 385         {
 386           attributes: [],
 387           model: VideoChannelModel.unscoped(),
 388           required: true,
 389           include: [
 390             {
 391               attributes: [],
 392               model: ActorModel.unscoped(),
 393               required: true,
 394               where: {
 395                 serverId
 396               }
 397             }
 398           ]
 399         }
 400       ]
 401     })
 402
 403     const query = {
 404       where: {
 405         actorId: actor.id
 406       },
 407       include: [
 408         {
 409           model: VideoFileModel,
 410           required: false,
 411           include: [ buildVideoInclude() ]
 412         },
 413         {
 414           model: VideoStreamingPlaylistModel,
 415           required: false,
 416           include: [ buildVideoInclude() ]
 417         }
 418       ]
 419     }
 420
 421     return VideoRedundancyModel.findAll(query)
 422   }
 423
 424   static async getStats (strategy: VideoRedundancyStrategy) {
 425     const actor = await getServerActor()
 426
 427     const query: FindOptions = {
 428       raw: true,
 429       attributes: [
 430         [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
 431         [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
 432         [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
 433       ],
 434       where: {
 435         strategy,
 436         actorId: actor.id
 437       },
 438       include: [
 439         {
 440           attributes: [],
 441           model: VideoFileModel,
 442           required: true
 443         }
 444       ]
 445     }
 446
 447     return VideoRedundancyModel.findOne(query)
 448       .then((r: any) => ({
 449         totalUsed: parseAggregateResult(r.totalUsed),
 450         totalVideos: r.totalVideos,
 451         totalVideoFiles: r.totalVideoFiles
 452       }))
 453   }
 454
 455   getVideo () {
 456     if (this.VideoFile) return this.VideoFile.Video
 457
 458     return this.VideoStreamingPlaylist.Video
 459   }
 460
 461   isOwned () {
 462     return !!this.strategy
 463   }
 464
 465   toActivityPubObject (): CacheFileObject {
 466     if (this.VideoStreamingPlaylist) {
 467       return {
 468         id: this.url,
 469         type: 'CacheFile' as 'CacheFile',
 470         object: this.VideoStreamingPlaylist.Video.url,
 471         expires: this.expiresOn.toISOString(),
 472         url: {
 473           type: 'Link',
 474           mimeType: 'application/x-mpegURL',
 475           mediaType: 'application/x-mpegURL',
 476           href: this.fileUrl
 477         }
 478       }
 479     }
 480
 481     return {
 482       id: this.url,
 483       type: 'CacheFile' as 'CacheFile',
 484       object: this.VideoFile.Video.url,
 485       expires: this.expiresOn.toISOString(),
 486       url: {
 487         type: 'Link',
 488         mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
 489         mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
 490         href: this.fileUrl,
 491         height: this.VideoFile.resolution,
 492         size: this.VideoFile.size,
 493         fps: this.VideoFile.fps
 494       }
 495     }
 496   }
 497
 498   // Don't include video files we already duplicated
 499   private static async buildVideoFileForDuplication () {
 500     const actor = await getServerActor()
 501
 502     const notIn = literal(
 503       '(' +
 504         `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
 505       ')'
 506     )
 507
 508     return {
 509       attributes: [],
 510       model: VideoFileModel.unscoped(),
 511       required: true,
 512       where: {
 513         id: {
 514           [ Op.notIn ]: notIn
 515         }
 516       }
 517     }
 518   }
 519
 520   private static buildServerRedundancyInclude () {
 521     return {
 522       attributes: [],
 523       model: VideoChannelModel.unscoped(),
 524       required: true,
 525       include: [
 526         {
 527           attributes: [],
 528           model: ActorModel.unscoped(),
 529           required: true,
 530           include: [
 531             {
 532               attributes: [],
 533               model: ServerModel.unscoped(),
 534               required: true,
 535               where: {
 536                 redundancyAllowed: true
 537               }
 538             }
 539           ]
 540         }
 541       ]
 542     }
 543   }
 544 }