server/models/redundancy/video-redundancy.ts

   1 import {
   2   AllowNull,
   3   BeforeDestroy,
   4   BelongsTo,
   5   Column,
   6   CreatedAt,
   7   DataType,
   8   ForeignKey,
   9   Is,
  10   Model,
  11   Scopes,
  12   Table,
  13   UpdatedAt
  14 } from 'sequelize-typescript'
  15 import { ActorModel } from '../activitypub/actor'
  16 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
  17 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  18 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
  19 import { VideoFileModel } from '../video/video-file'
  20 import { getServerActor } from '../../helpers/utils'
  21 import { VideoModel } from '../video/video'
  22 import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
  23 import { logger } from '../../helpers/logger'
  24 import { CacheFileObject, VideoPrivacy } from '../../../shared'
  25 import { VideoChannelModel } from '../video/video-channel'
  26 import { ServerModel } from '../server/server'
  27 import { sample } from 'lodash'
  28 import { isTestInstance } from '../../helpers/core-utils'
  29 import * as Bluebird from 'bluebird'
  30 import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
  31 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
  32 import { CONFIG } from '../../initializers/config'
  33 import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models'
  34 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
  35 import {
  36   FileRedundancyInformation,
  37   StreamingPlaylistRedundancyInformation,
  38   VideoRedundancy
  39 } from '@shared/models/redundancy/video-redundancy.model'
  40
  41 export enum ScopeNames {
  42   WITH_VIDEO = 'WITH_VIDEO'
  43 }
  44
  45 @Scopes(() => ({
  46   [ScopeNames.WITH_VIDEO]: {
  47     include: [
  48       {
  49         model: VideoFileModel,
  50         required: false,
  51         include: [
  52           {
  53             model: VideoModel,
  54             required: true
  55           }
  56         ]
  57       },
  58       {
  59         model: VideoStreamingPlaylistModel,
  60         required: false,
  61         include: [
  62           {
  63             model: VideoModel,
  64             required: true
  65           }
  66         ]
  67       }
  68     ]
  69   }
  70 }))
  71
  72 @Table({
  73   tableName: 'videoRedundancy',
  74   indexes: [
  75     {
  76       fields: [ 'videoFileId' ]
  77     },
  78     {
  79       fields: [ 'actorId' ]
  80     },
  81     {
  82       fields: [ 'url' ],
  83       unique: true
  84     }
  85   ]
  86 })
  87 export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
  88
  89   @CreatedAt
  90   createdAt: Date
  91
  92   @UpdatedAt
  93   updatedAt: Date
  94
  95   @AllowNull(true)
  96   @Column
  97   expiresOn: Date
  98
  99   @AllowNull(false)
 100   @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
 101   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
 102   fileUrl: string
 103
 104   @AllowNull(false)
 105   @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
 106   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
 107   url: string
 108
 109   @AllowNull(true)
 110   @Column
 111   strategy: string // Only used by us
 112
 113   @ForeignKey(() => VideoFileModel)
 114   @Column
 115   videoFileId: number
 116
 117   @BelongsTo(() => VideoFileModel, {
 118     foreignKey: {
 119       allowNull: true
 120     },
 121     onDelete: 'cascade'
 122   })
 123   VideoFile: VideoFileModel
 124
 125   @ForeignKey(() => VideoStreamingPlaylistModel)
 126   @Column
 127   videoStreamingPlaylistId: number
 128
 129   @BelongsTo(() => VideoStreamingPlaylistModel, {
 130     foreignKey: {
 131       allowNull: true
 132     },
 133     onDelete: 'cascade'
 134   })
 135   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 136
 137   @ForeignKey(() => ActorModel)
 138   @Column
 139   actorId: number
 140
 141   @BelongsTo(() => ActorModel, {
 142     foreignKey: {
 143       allowNull: false
 144     },
 145     onDelete: 'cascade'
 146   })
 147   Actor: ActorModel
 148
 149   @BeforeDestroy
 150   static async removeFile (instance: VideoRedundancyModel) {
 151     if (!instance.isOwned()) return
 152
 153     if (instance.videoFileId) {
 154       const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
 155
 156       const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
 157       logger.info('Removing duplicated video file %s.', logIdentifier)
 158
 159       videoFile.Video.removeFile(videoFile, true)
 160                .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
 161     }
 162
 163     if (instance.videoStreamingPlaylistId) {
 164       const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
 165
 166       const videoUUID = videoStreamingPlaylist.Video.uuid
 167       logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
 168
 169       videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
 170                             .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
 171     }
 172
 173     return undefined
 174   }
 175
 176   static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
 177     const actor = await getServerActor()
 178
 179     const query = {
 180       where: {
 181         actorId: actor.id,
 182         videoFileId
 183       }
 184     }
 185
 186     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 187   }
 188
 189   static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
 190     const actor = await getServerActor()
 191
 192     const query = {
 193       where: {
 194         actorId: actor.id,
 195         videoStreamingPlaylistId
 196       }
 197     }
 198
 199     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 200   }
 201
 202   static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird<MVideoRedundancyVideo> {
 203     const query = {
 204       where: { id },
 205       transaction
 206     }
 207
 208     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 209   }
 210
 211   static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> {
 212     const query = {
 213       where: {
 214         url
 215       },
 216       transaction
 217     }
 218
 219     return VideoRedundancyModel.findOne(query)
 220   }
 221
 222   static async isLocalByVideoUUIDExists (uuid: string) {
 223     const actor = await getServerActor()
 224
 225     const query = {
 226       raw: true,
 227       attributes: [ 'id' ],
 228       where: {
 229         actorId: actor.id
 230       },
 231       include: [
 232         {
 233           attributes: [],
 234           model: VideoFileModel,
 235           required: true,
 236           include: [
 237             {
 238               attributes: [],
 239               model: VideoModel,
 240               required: true,
 241               where: {
 242                 uuid
 243               }
 244             }
 245           ]
 246         }
 247       ]
 248     }
 249
 250     return VideoRedundancyModel.findOne(query)
 251                                .then(r => !!r)
 252   }
 253
 254   static async getVideoSample (p: Bluebird<VideoModel[]>) {
 255     const rows = await p
 256     if (rows.length === 0) return undefined
 257
 258     const ids = rows.map(r => r.id)
 259     const id = sample(ids)
 260
 261     return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
 262   }
 263
 264   static async findMostViewToDuplicate (randomizedFactor: number) {
 265     // On VideoModel!
 266     const query = {
 267       attributes: [ 'id', 'views' ],
 268       limit: randomizedFactor,
 269       order: getVideoSort('-views'),
 270       where: {
 271         privacy: VideoPrivacy.PUBLIC
 272       },
 273       include: [
 274         await VideoRedundancyModel.buildVideoFileForDuplication(),
 275         VideoRedundancyModel.buildServerRedundancyInclude()
 276       ]
 277     }
 278
 279     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 280   }
 281
 282   static async findTrendingToDuplicate (randomizedFactor: number) {
 283     // On VideoModel!
 284     const query = {
 285       attributes: [ 'id', 'views' ],
 286       subQuery: false,
 287       group: 'VideoModel.id',
 288       limit: randomizedFactor,
 289       order: getVideoSort('-trending'),
 290       where: {
 291         privacy: VideoPrivacy.PUBLIC
 292       },
 293       include: [
 294         await VideoRedundancyModel.buildVideoFileForDuplication(),
 295         VideoRedundancyModel.buildServerRedundancyInclude(),
 296
 297         VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
 298       ]
 299     }
 300
 301     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 302   }
 303
 304   static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
 305     // On VideoModel!
 306     const query = {
 307       attributes: [ 'id', 'publishedAt' ],
 308       limit: randomizedFactor,
 309       order: getVideoSort('-publishedAt'),
 310       where: {
 311         privacy: VideoPrivacy.PUBLIC,
 312         views: {
 313           [Op.gte]: minViews
 314         }
 315       },
 316       include: [
 317         await VideoRedundancyModel.buildVideoFileForDuplication(),
 318         VideoRedundancyModel.buildServerRedundancyInclude()
 319       ]
 320     }
 321
 322     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 323   }
 324
 325   static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
 326     const expiredDate = new Date()
 327     expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
 328
 329     const actor = await getServerActor()
 330
 331     const query = {
 332       where: {
 333         actorId: actor.id,
 334         strategy,
 335         createdAt: {
 336           [Op.lt]: expiredDate
 337         }
 338       }
 339     }
 340
 341     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
 342   }
 343
 344   static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
 345     const actor = await getServerActor()
 346     const redundancyInclude = {
 347       attributes: [],
 348       model: VideoRedundancyModel,
 349       required: true,
 350       where: {
 351         actorId: actor.id,
 352         strategy
 353       }
 354     }
 355
 356     const queryFiles: FindOptions = {
 357       include: [ redundancyInclude ]
 358     }
 359
 360     const queryStreamingPlaylists: FindOptions = {
 361       include: [
 362         {
 363           attributes: [],
 364           model: VideoModel.unscoped(),
 365           required: true,
 366           include: [
 367             {
 368               required: true,
 369               attributes: [],
 370               model: VideoStreamingPlaylistModel.unscoped(),
 371               include: [
 372                 redundancyInclude
 373               ]
 374             }
 375           ]
 376         }
 377       ]
 378     }
 379
 380     return Promise.all([
 381       VideoFileModel.aggregate('size', 'SUM', queryFiles),
 382       VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
 383     ]).then(([ r1, r2 ]) => {
 384       return parseAggregateResult(r1) + parseAggregateResult(r2)
 385     })
 386   }
 387
 388   static async listLocalExpired () {
 389     const actor = await getServerActor()
 390
 391     const query = {
 392       where: {
 393         actorId: actor.id,
 394         expiresOn: {
 395           [Op.lt]: new Date()
 396         }
 397       }
 398     }
 399
 400     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
 401   }
 402
 403   static async listRemoteExpired () {
 404     const actor = await getServerActor()
 405
 406     const query = {
 407       where: {
 408         actorId: {
 409           [Op.ne]: actor.id
 410         },
 411         expiresOn: {
 412           [Op.lt]: new Date(),
 413           [Op.ne]: null
 414         }
 415       }
 416     }
 417
 418     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
 419   }
 420
 421   static async listLocalOfServer (serverId: number) {
 422     const actor = await getServerActor()
 423     const buildVideoInclude = () => ({
 424       model: VideoModel,
 425       required: true,
 426       include: [
 427         {
 428           attributes: [],
 429           model: VideoChannelModel.unscoped(),
 430           required: true,
 431           include: [
 432             {
 433               attributes: [],
 434               model: ActorModel.unscoped(),
 435               required: true,
 436               where: {
 437                 serverId
 438               }
 439             }
 440           ]
 441         }
 442       ]
 443     })
 444
 445     const query = {
 446       where: {
 447         actorId: actor.id
 448       },
 449       include: [
 450         {
 451           model: VideoFileModel,
 452           required: false,
 453           include: [ buildVideoInclude() ]
 454         },
 455         {
 456           model: VideoStreamingPlaylistModel,
 457           required: false,
 458           include: [ buildVideoInclude() ]
 459         }
 460       ]
 461     }
 462
 463     return VideoRedundancyModel.findAll(query)
 464   }
 465
 466   static listForApi (options: {
 467     start: number
 468     count: number
 469     sort: string
 470     target: VideoRedundanciesTarget
 471     strategy?: string
 472   }) {
 473     const { start, count, sort, target, strategy } = options
 474     const redundancyWhere: WhereOptions = {}
 475     const videosWhere: WhereOptions = {}
 476     let redundancySqlSuffix = ''
 477
 478     if (target === 'my-videos') {
 479       Object.assign(videosWhere, { remote: false })
 480     } else if (target === 'remote-videos') {
 481       Object.assign(videosWhere, { remote: true })
 482       Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
 483       redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
 484     }
 485
 486     if (strategy) {
 487       Object.assign(redundancyWhere, { strategy: strategy })
 488     }
 489
 490     const videoFilterWhere = {
 491       [Op.and]: [
 492         {
 493           [Op.or]: [
 494             {
 495               id: {
 496                 [Op.in]: literal(
 497                   '(' +
 498                   'SELECT "videoId" FROM "videoFile" ' +
 499                   'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
 500                   redundancySqlSuffix +
 501                   ')'
 502                 )
 503               }
 504             },
 505             {
 506               id: {
 507                 [Op.in]: literal(
 508                   '(' +
 509                   'select "videoId" FROM "videoStreamingPlaylist" ' +
 510                   'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
 511                   redundancySqlSuffix +
 512                   ')'
 513                 )
 514               }
 515             }
 516           ]
 517         },
 518
 519         videosWhere
 520       ]
 521     }
 522
 523     // /!\ On video model /!\
 524     const findOptions = {
 525       offset: start,
 526       limit: count,
 527       order: getSort(sort),
 528       include: [
 529         {
 530           required: false,
 531           model: VideoFileModel,
 532           include: [
 533             {
 534               model: VideoRedundancyModel.unscoped(),
 535               required: false,
 536               where: redundancyWhere
 537             }
 538           ]
 539         },
 540         {
 541           required: false,
 542           model: VideoStreamingPlaylistModel.unscoped(),
 543           include: [
 544             {
 545               model: VideoRedundancyModel.unscoped(),
 546               required: false,
 547               where: redundancyWhere
 548             },
 549             {
 550               model: VideoFileModel,
 551               required: false
 552             }
 553           ]
 554         }
 555       ],
 556       where: videoFilterWhere
 557     }
 558
 559     // /!\ On video model /!\
 560     const countOptions = {
 561       where: videoFilterWhere
 562     }
 563
 564     return Promise.all([
 565       VideoModel.findAll(findOptions),
 566
 567       VideoModel.count(countOptions)
 568     ]).then(([ data, total ]) => ({ total, data }))
 569   }
 570
 571   static async getStats (strategy: VideoRedundancyStrategyWithManual) {
 572     const actor = await getServerActor()
 573
 574     const query: FindOptions = {
 575       raw: true,
 576       attributes: [
 577         [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
 578         [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
 579         [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
 580       ],
 581       where: {
 582         strategy,
 583         actorId: actor.id
 584       },
 585       include: [
 586         {
 587           attributes: [],
 588           model: VideoFileModel,
 589           required: true
 590         }
 591       ]
 592     }
 593
 594     return VideoRedundancyModel.findOne(query)
 595                                .then((r: any) => ({
 596                                  totalUsed: parseAggregateResult(r.totalUsed),
 597                                  totalVideos: r.totalVideos,
 598                                  totalVideoFiles: r.totalVideoFiles
 599                                }))
 600   }
 601
 602   static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
 603     const filesRedundancies: FileRedundancyInformation[] = []
 604     const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
 605
 606     for (const file of video.VideoFiles) {
 607       for (const redundancy of file.RedundancyVideos) {
 608         filesRedundancies.push({
 609           id: redundancy.id,
 610           fileUrl: redundancy.fileUrl,
 611           strategy: redundancy.strategy,
 612           createdAt: redundancy.createdAt,
 613           updatedAt: redundancy.updatedAt,
 614           expiresOn: redundancy.expiresOn,
 615           size: file.size
 616         })
 617       }
 618     }
 619
 620     for (const playlist of video.VideoStreamingPlaylists) {
 621       const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
 622
 623       for (const redundancy of playlist.RedundancyVideos) {
 624         streamingPlaylistsRedundancies.push({
 625           id: redundancy.id,
 626           fileUrl: redundancy.fileUrl,
 627           strategy: redundancy.strategy,
 628           createdAt: redundancy.createdAt,
 629           updatedAt: redundancy.updatedAt,
 630           expiresOn: redundancy.expiresOn,
 631           size
 632         })
 633       }
 634     }
 635
 636     return {
 637       id: video.id,
 638       name: video.name,
 639       url: video.url,
 640       uuid: video.uuid,
 641
 642       redundancies: {
 643         files: filesRedundancies,
 644         streamingPlaylists: streamingPlaylistsRedundancies
 645       }
 646     }
 647   }
 648
 649   getVideo () {
 650     if (this.VideoFile) return this.VideoFile.Video
 651
 652     return this.VideoStreamingPlaylist.Video
 653   }
 654
 655   isOwned () {
 656     return !!this.strategy
 657   }
 658
 659   toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
 660     if (this.VideoStreamingPlaylist) {
 661       return {
 662         id: this.url,
 663         type: 'CacheFile' as 'CacheFile',
 664         object: this.VideoStreamingPlaylist.Video.url,
 665         expires: this.expiresOn ? this.expiresOn.toISOString() : null,
 666         url: {
 667           type: 'Link',
 668           mediaType: 'application/x-mpegURL',
 669           href: this.fileUrl
 670         }
 671       }
 672     }
 673
 674     return {
 675       id: this.url,
 676       type: 'CacheFile' as 'CacheFile',
 677       object: this.VideoFile.Video.url,
 678       expires: this.expiresOn ? this.expiresOn.toISOString() : null,
 679       url: {
 680         type: 'Link',
 681         mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
 682         href: this.fileUrl,
 683         height: this.VideoFile.resolution,
 684         size: this.VideoFile.size,
 685         fps: this.VideoFile.fps
 686       }
 687     }
 688   }
 689
 690   // Don't include video files we already duplicated
 691   private static async buildVideoFileForDuplication () {
 692     const actor = await getServerActor()
 693
 694     const notIn = literal(
 695       '(' +
 696       `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
 697       ')'
 698     )
 699
 700     return {
 701       attributes: [],
 702       model: VideoFileModel,
 703       required: true,
 704       where: {
 705         id: {
 706           [Op.notIn]: notIn
 707         }
 708       }
 709     }
 710   }
 711
 712   private static buildServerRedundancyInclude () {
 713     return {
 714       attributes: [],
 715       model: VideoChannelModel.unscoped(),
 716       required: true,
 717       include: [
 718         {
 719           attributes: [],
 720           model: ActorModel.unscoped(),
 721           required: true,
 722           include: [
 723             {
 724               attributes: [],
 725               model: ServerModel.unscoped(),
 726               required: true,
 727               where: {
 728                 redundancyAllowed: true
 729               }
 730             }
 731           ]
 732         }
 733       ]
 734     }
 735   }
 736 }