server/models/redundancy/video-redundancy.ts

   1 import { sample } from 'lodash'
   2 import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
   3 import {
   4   AllowNull,
   5   BeforeDestroy,
   6   BelongsTo,
   7   Column,
   8   CreatedAt,
   9   DataType,
  10   ForeignKey,
  11   Is,
  12   Model,
  13   Scopes,
  14   Table,
  15   UpdatedAt
  16 } from 'sequelize-typescript'
  17 import { getServerActor } from '@server/models/application/application'
  18 import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
  19 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
  20 import {
  21   FileRedundancyInformation,
  22   StreamingPlaylistRedundancyInformation,
  23   VideoRedundancy
  24 } from '@shared/models/redundancy/video-redundancy.model'
  25 import { CacheFileObject, VideoPrivacy } from '../../../shared'
  26 import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
  27 import { isTestInstance } from '../../helpers/core-utils'
  28 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  29 import { logger } from '../../helpers/logger'
  30 import { CONFIG } from '../../initializers/config'
  31 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
  32 import { ActorModel } from '../activitypub/actor'
  33 import { ServerModel } from '../server/server'
  34 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
  35 import { VideoModel } from '../video/video'
  36 import { VideoChannelModel } from '../video/video-channel'
  37 import { VideoFileModel } from '../video/video-file'
  38 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
  39
  40 export enum ScopeNames {
  41   WITH_VIDEO = 'WITH_VIDEO'
  42 }
  43
  44 @Scopes(() => ({
  45   [ScopeNames.WITH_VIDEO]: {
  46     include: [
  47       {
  48         model: VideoFileModel,
  49         required: false,
  50         include: [
  51           {
  52             model: VideoModel,
  53             required: true
  54           }
  55         ]
  56       },
  57       {
  58         model: VideoStreamingPlaylistModel,
  59         required: false,
  60         include: [
  61           {
  62             model: VideoModel,
  63             required: true
  64           }
  65         ]
  66       }
  67     ]
  68   }
  69 }))
  70
  71 @Table({
  72   tableName: 'videoRedundancy',
  73   indexes: [
  74     {
  75       fields: [ 'videoFileId' ]
  76     },
  77     {
  78       fields: [ 'actorId' ]
  79     },
  80     {
  81       fields: [ 'url' ],
  82       unique: true
  83     }
  84   ]
  85 })
  86 export class VideoRedundancyModel extends Model {
  87
  88   @CreatedAt
  89   createdAt: Date
  90
  91   @UpdatedAt
  92   updatedAt: Date
  93
  94   @AllowNull(true)
  95   @Column
  96   expiresOn: Date
  97
  98   @AllowNull(false)
  99   @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
 100   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
 101   fileUrl: string
 102
 103   @AllowNull(false)
 104   @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
 105   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
 106   url: string
 107
 108   @AllowNull(true)
 109   @Column
 110   strategy: string // Only used by us
 111
 112   @ForeignKey(() => VideoFileModel)
 113   @Column
 114   videoFileId: number
 115
 116   @BelongsTo(() => VideoFileModel, {
 117     foreignKey: {
 118       allowNull: true
 119     },
 120     onDelete: 'cascade'
 121   })
 122   VideoFile: VideoFileModel
 123
 124   @ForeignKey(() => VideoStreamingPlaylistModel)
 125   @Column
 126   videoStreamingPlaylistId: number
 127
 128   @BelongsTo(() => VideoStreamingPlaylistModel, {
 129     foreignKey: {
 130       allowNull: true
 131     },
 132     onDelete: 'cascade'
 133   })
 134   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 135
 136   @ForeignKey(() => ActorModel)
 137   @Column
 138   actorId: number
 139
 140   @BelongsTo(() => ActorModel, {
 141     foreignKey: {
 142       allowNull: false
 143     },
 144     onDelete: 'cascade'
 145   })
 146   Actor: ActorModel
 147
 148   @BeforeDestroy
 149   static async removeFile (instance: VideoRedundancyModel) {
 150     if (!instance.isOwned()) return
 151
 152     if (instance.videoFileId) {
 153       const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
 154
 155       const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
 156       logger.info('Removing duplicated video file %s.', logIdentifier)
 157
 158       videoFile.Video.removeFile(videoFile, true)
 159                .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
 160     }
 161
 162     if (instance.videoStreamingPlaylistId) {
 163       const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
 164
 165       const videoUUID = videoStreamingPlaylist.Video.uuid
 166       logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
 167
 168       videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
 169                             .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
 170     }
 171
 172     return undefined
 173   }
 174
 175   static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
 176     const actor = await getServerActor()
 177
 178     const query = {
 179       where: {
 180         actorId: actor.id,
 181         videoFileId
 182       }
 183     }
 184
 185     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 186   }
 187
 188   static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
 189     const actor = await getServerActor()
 190
 191     const query = {
 192       where: {
 193         actorId: actor.id,
 194         videoStreamingPlaylistId
 195       }
 196     }
 197
 198     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 199   }
 200
 201   static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MVideoRedundancyVideo> {
 202     const query = {
 203       where: { id },
 204       transaction
 205     }
 206
 207     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 208   }
 209
 210   static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoRedundancy> {
 211     const query = {
 212       where: {
 213         url
 214       },
 215       transaction
 216     }
 217
 218     return VideoRedundancyModel.findOne(query)
 219   }
 220
 221   static async isLocalByVideoUUIDExists (uuid: string) {
 222     const actor = await getServerActor()
 223
 224     const query = {
 225       raw: true,
 226       attributes: [ 'id' ],
 227       where: {
 228         actorId: actor.id
 229       },
 230       include: [
 231         {
 232           attributes: [],
 233           model: VideoFileModel,
 234           required: true,
 235           include: [
 236             {
 237               attributes: [],
 238               model: VideoModel,
 239               required: true,
 240               where: {
 241                 uuid
 242               }
 243             }
 244           ]
 245         }
 246       ]
 247     }
 248
 249     return VideoRedundancyModel.findOne(query)
 250                                .then(r => !!r)
 251   }
 252
 253   static async getVideoSample (p: Promise<VideoModel[]>) {
 254     const rows = await p
 255     if (rows.length === 0) return undefined
 256
 257     const ids = rows.map(r => r.id)
 258     const id = sample(ids)
 259
 260     return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
 261   }
 262
 263   static async findMostViewToDuplicate (randomizedFactor: number) {
 264     // On VideoModel!
 265     const query = {
 266       attributes: [ 'id', 'views' ],
 267       limit: randomizedFactor,
 268       order: getVideoSort('-views'),
 269       where: {
 270         privacy: VideoPrivacy.PUBLIC,
 271         isLive: false
 272       },
 273       include: [
 274         await VideoRedundancyModel.buildVideoFileForDuplication(),
 275         VideoRedundancyModel.buildServerRedundancyInclude()
 276       ]
 277     }
 278
 279     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 280   }
 281
 282   static async findTrendingToDuplicate (randomizedFactor: number) {
 283     // On VideoModel!
 284     const query = {
 285       attributes: [ 'id', 'views' ],
 286       subQuery: false,
 287       group: 'VideoModel.id',
 288       limit: randomizedFactor,
 289       order: getVideoSort('-trending'),
 290       where: {
 291         privacy: VideoPrivacy.PUBLIC,
 292         isLive: false
 293       },
 294       include: [
 295         await VideoRedundancyModel.buildVideoFileForDuplication(),
 296         VideoRedundancyModel.buildServerRedundancyInclude(),
 297
 298         VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
 299       ]
 300     }
 301
 302     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 303   }
 304
 305   static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
 306     // On VideoModel!
 307     const query = {
 308       attributes: [ 'id', 'publishedAt' ],
 309       limit: randomizedFactor,
 310       order: getVideoSort('-publishedAt'),
 311       where: {
 312         privacy: VideoPrivacy.PUBLIC,
 313         isLive: false,
 314         views: {
 315           [Op.gte]: minViews
 316         }
 317       },
 318       include: [
 319         await VideoRedundancyModel.buildVideoFileForDuplication(),
 320         VideoRedundancyModel.buildServerRedundancyInclude()
 321       ]
 322     }
 323
 324     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 325   }
 326
 327   static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
 328     const expiredDate = new Date()
 329     expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
 330
 331     const actor = await getServerActor()
 332
 333     const query = {
 334       where: {
 335         actorId: actor.id,
 336         strategy,
 337         createdAt: {
 338           [Op.lt]: expiredDate
 339         }
 340       }
 341     }
 342
 343     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
 344   }
 345
 346   static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
 347     const actor = await getServerActor()
 348     const redundancyInclude = {
 349       attributes: [],
 350       model: VideoRedundancyModel,
 351       required: true,
 352       where: {
 353         actorId: actor.id,
 354         strategy
 355       }
 356     }
 357
 358     const queryFiles: FindOptions = {
 359       include: [ redundancyInclude ]
 360     }
 361
 362     const queryStreamingPlaylists: FindOptions = {
 363       include: [
 364         {
 365           attributes: [],
 366           model: VideoModel.unscoped(),
 367           required: true,
 368           include: [
 369             {
 370               required: true,
 371               attributes: [],
 372               model: VideoStreamingPlaylistModel.unscoped(),
 373               include: [
 374                 redundancyInclude
 375               ]
 376             }
 377           ]
 378         }
 379       ]
 380     }
 381
 382     return Promise.all([
 383       VideoFileModel.aggregate('size', 'SUM', queryFiles),
 384       VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
 385     ]).then(([ r1, r2 ]) => {
 386       return parseAggregateResult(r1) + parseAggregateResult(r2)
 387     })
 388   }
 389
 390   static async listLocalExpired () {
 391     const actor = await getServerActor()
 392
 393     const query = {
 394       where: {
 395         actorId: actor.id,
 396         expiresOn: {
 397           [Op.lt]: new Date()
 398         }
 399       }
 400     }
 401
 402     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
 403   }
 404
 405   static async listRemoteExpired () {
 406     const actor = await getServerActor()
 407
 408     const query = {
 409       where: {
 410         actorId: {
 411           [Op.ne]: actor.id
 412         },
 413         expiresOn: {
 414           [Op.lt]: new Date(),
 415           [Op.ne]: null
 416         }
 417       }
 418     }
 419
 420     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
 421   }
 422
 423   static async listLocalOfServer (serverId: number) {
 424     const actor = await getServerActor()
 425     const buildVideoInclude = () => ({
 426       model: VideoModel,
 427       required: true,
 428       include: [
 429         {
 430           attributes: [],
 431           model: VideoChannelModel.unscoped(),
 432           required: true,
 433           include: [
 434             {
 435               attributes: [],
 436               model: ActorModel.unscoped(),
 437               required: true,
 438               where: {
 439                 serverId
 440               }
 441             }
 442           ]
 443         }
 444       ]
 445     })
 446
 447     const query = {
 448       where: {
 449         actorId: actor.id
 450       },
 451       include: [
 452         {
 453           model: VideoFileModel,
 454           required: false,
 455           include: [ buildVideoInclude() ]
 456         },
 457         {
 458           model: VideoStreamingPlaylistModel,
 459           required: false,
 460           include: [ buildVideoInclude() ]
 461         }
 462       ]
 463     }
 464
 465     return VideoRedundancyModel.findAll(query)
 466   }
 467
 468   static listForApi (options: {
 469     start: number
 470     count: number
 471     sort: string
 472     target: VideoRedundanciesTarget
 473     strategy?: string
 474   }) {
 475     const { start, count, sort, target, strategy } = options
 476     const redundancyWhere: WhereOptions = {}
 477     const videosWhere: WhereOptions = {}
 478     let redundancySqlSuffix = ''
 479
 480     if (target === 'my-videos') {
 481       Object.assign(videosWhere, { remote: false })
 482     } else if (target === 'remote-videos') {
 483       Object.assign(videosWhere, { remote: true })
 484       Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
 485       redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
 486     }
 487
 488     if (strategy) {
 489       Object.assign(redundancyWhere, { strategy: strategy })
 490     }
 491
 492     const videoFilterWhere = {
 493       [Op.and]: [
 494         {
 495           [Op.or]: [
 496             {
 497               id: {
 498                 [Op.in]: literal(
 499                   '(' +
 500                   'SELECT "videoId" FROM "videoFile" ' +
 501                   'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
 502                   redundancySqlSuffix +
 503                   ')'
 504                 )
 505               }
 506             },
 507             {
 508               id: {
 509                 [Op.in]: literal(
 510                   '(' +
 511                   'select "videoId" FROM "videoStreamingPlaylist" ' +
 512                   'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
 513                   redundancySqlSuffix +
 514                   ')'
 515                 )
 516               }
 517             }
 518           ]
 519         },
 520
 521         videosWhere
 522       ]
 523     }
 524
 525     // /!\ On video model /!\
 526     const findOptions = {
 527       offset: start,
 528       limit: count,
 529       order: getSort(sort),
 530       include: [
 531         {
 532           required: false,
 533           model: VideoFileModel,
 534           include: [
 535             {
 536               model: VideoRedundancyModel.unscoped(),
 537               required: false,
 538               where: redundancyWhere
 539             }
 540           ]
 541         },
 542         {
 543           required: false,
 544           model: VideoStreamingPlaylistModel.unscoped(),
 545           include: [
 546             {
 547               model: VideoRedundancyModel.unscoped(),
 548               required: false,
 549               where: redundancyWhere
 550             },
 551             {
 552               model: VideoFileModel,
 553               required: false
 554             }
 555           ]
 556         }
 557       ],
 558       where: videoFilterWhere
 559     }
 560
 561     // /!\ On video model /!\
 562     const countOptions = {
 563       where: videoFilterWhere
 564     }
 565
 566     return Promise.all([
 567       VideoModel.findAll(findOptions),
 568
 569       VideoModel.count(countOptions)
 570     ]).then(([ data, total ]) => ({ total, data }))
 571   }
 572
 573   static async getStats (strategy: VideoRedundancyStrategyWithManual) {
 574     const actor = await getServerActor()
 575
 576     const query: FindOptions = {
 577       raw: true,
 578       attributes: [
 579         [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
 580         [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
 581         [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
 582       ],
 583       where: {
 584         strategy,
 585         actorId: actor.id
 586       },
 587       include: [
 588         {
 589           attributes: [],
 590           model: VideoFileModel,
 591           required: true
 592         }
 593       ]
 594     }
 595
 596     return VideoRedundancyModel.findOne(query)
 597                                .then((r: any) => ({
 598                                  totalUsed: parseAggregateResult(r.totalUsed),
 599                                  totalVideos: r.totalVideos,
 600                                  totalVideoFiles: r.totalVideoFiles
 601                                }))
 602   }
 603
 604   static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
 605     const filesRedundancies: FileRedundancyInformation[] = []
 606     const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
 607
 608     for (const file of video.VideoFiles) {
 609       for (const redundancy of file.RedundancyVideos) {
 610         filesRedundancies.push({
 611           id: redundancy.id,
 612           fileUrl: redundancy.fileUrl,
 613           strategy: redundancy.strategy,
 614           createdAt: redundancy.createdAt,
 615           updatedAt: redundancy.updatedAt,
 616           expiresOn: redundancy.expiresOn,
 617           size: file.size
 618         })
 619       }
 620     }
 621
 622     for (const playlist of video.VideoStreamingPlaylists) {
 623       const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
 624
 625       for (const redundancy of playlist.RedundancyVideos) {
 626         streamingPlaylistsRedundancies.push({
 627           id: redundancy.id,
 628           fileUrl: redundancy.fileUrl,
 629           strategy: redundancy.strategy,
 630           createdAt: redundancy.createdAt,
 631           updatedAt: redundancy.updatedAt,
 632           expiresOn: redundancy.expiresOn,
 633           size
 634         })
 635       }
 636     }
 637
 638     return {
 639       id: video.id,
 640       name: video.name,
 641       url: video.url,
 642       uuid: video.uuid,
 643
 644       redundancies: {
 645         files: filesRedundancies,
 646         streamingPlaylists: streamingPlaylistsRedundancies
 647       }
 648     }
 649   }
 650
 651   getVideo () {
 652     if (this.VideoFile) return this.VideoFile.Video
 653
 654     if (this.VideoStreamingPlaylist.Video) return this.VideoStreamingPlaylist.Video
 655
 656     return undefined
 657   }
 658
 659   isOwned () {
 660     return !!this.strategy
 661   }
 662
 663   toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
 664     if (this.VideoStreamingPlaylist) {
 665       return {
 666         id: this.url,
 667         type: 'CacheFile' as 'CacheFile',
 668         object: this.VideoStreamingPlaylist.Video.url,
 669         expires: this.expiresOn ? this.expiresOn.toISOString() : null,
 670         url: {
 671           type: 'Link',
 672           mediaType: 'application/x-mpegURL',
 673           href: this.fileUrl
 674         }
 675       }
 676     }
 677
 678     return {
 679       id: this.url,
 680       type: 'CacheFile' as 'CacheFile',
 681       object: this.VideoFile.Video.url,
 682       expires: this.expiresOn ? this.expiresOn.toISOString() : null,
 683       url: {
 684         type: 'Link',
 685         mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
 686         href: this.fileUrl,
 687         height: this.VideoFile.resolution,
 688         size: this.VideoFile.size,
 689         fps: this.VideoFile.fps
 690       }
 691     }
 692   }
 693
 694   // Don't include video files we already duplicated
 695   private static async buildVideoFileForDuplication () {
 696     const actor = await getServerActor()
 697
 698     const notIn = literal(
 699       '(' +
 700         `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
 701       ')'
 702     )
 703
 704     return {
 705       attributes: [],
 706       model: VideoFileModel,
 707       required: true,
 708       where: {
 709         id: {
 710           [Op.notIn]: notIn
 711         }
 712       }
 713     }
 714   }
 715
 716   private static buildServerRedundancyInclude () {
 717     return {
 718       attributes: [],
 719       model: VideoChannelModel.unscoped(),
 720       required: true,
 721       include: [
 722         {
 723           attributes: [],
 724           model: ActorModel.unscoped(),
 725           required: true,
 726           include: [
 727             {
 728               attributes: [],
 729               model: ServerModel.unscoped(),
 730               required: true,
 731               where: {
 732                 redundancyAllowed: true
 733               }
 734             }
 735           ]
 736         }
 737       ]
 738     }
 739   }
 740 }