server/models/redundancy/video-redundancy.ts

   1 import {
   2   AllowNull,
   3   BeforeDestroy,
   4   BelongsTo,
   5   Column,
   6   CreatedAt,
   7   DataType,
   8   ForeignKey,
   9   Is,
  10   Model,
  11   Scopes,
  12   Table,
  13   UpdatedAt
  14 } from 'sequelize-typescript'
  15 import { ActorModel } from '../activitypub/actor'
  16 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
  17 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  18 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
  19 import { VideoFileModel } from '../video/video-file'
  20 import { VideoModel } from '../video/video'
  21 import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
  22 import { logger } from '../../helpers/logger'
  23 import { CacheFileObject, VideoPrivacy } from '../../../shared'
  24 import { VideoChannelModel } from '../video/video-channel'
  25 import { ServerModel } from '../server/server'
  26 import { sample } from 'lodash'
  27 import { isTestInstance } from '../../helpers/core-utils'
  28 import * as Bluebird from 'bluebird'
  29 import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
  30 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
  31 import { CONFIG } from '../../initializers/config'
  32 import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
  33 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
  34 import {
  35   FileRedundancyInformation,
  36   StreamingPlaylistRedundancyInformation,
  37   VideoRedundancy
  38 } from '@shared/models/redundancy/video-redundancy.model'
  39 import { getServerActor } from '@server/models/application/application'
  40
  41 export enum ScopeNames {
  42   WITH_VIDEO = 'WITH_VIDEO'
  43 }
  44
  45 @Scopes(() => ({
  46   [ScopeNames.WITH_VIDEO]: {
  47     include: [
  48       {
  49         model: VideoFileModel,
  50         required: false,
  51         include: [
  52           {
  53             model: VideoModel,
  54             required: true
  55           }
  56         ]
  57       },
  58       {
  59         model: VideoStreamingPlaylistModel,
  60         required: false,
  61         include: [
  62           {
  63             model: VideoModel,
  64             required: true
  65           }
  66         ]
  67       }
  68     ]
  69   }
  70 }))
  71
  72 @Table({
  73   tableName: 'videoRedundancy',
  74   indexes: [
  75     {
  76       fields: [ 'videoFileId' ]
  77     },
  78     {
  79       fields: [ 'actorId' ]
  80     },
  81     {
  82       fields: [ 'url' ],
  83       unique: true
  84     }
  85   ]
  86 })
  87 export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
  88
  89   @CreatedAt
  90   createdAt: Date
  91
  92   @UpdatedAt
  93   updatedAt: Date
  94
  95   @AllowNull(true)
  96   @Column
  97   expiresOn: Date
  98
  99   @AllowNull(false)
 100   @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
 101   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
 102   fileUrl: string
 103
 104   @AllowNull(false)
 105   @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
 106   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
 107   url: string
 108
 109   @AllowNull(true)
 110   @Column
 111   strategy: string // Only used by us
 112
 113   @ForeignKey(() => VideoFileModel)
 114   @Column
 115   videoFileId: number
 116
 117   @BelongsTo(() => VideoFileModel, {
 118     foreignKey: {
 119       allowNull: true
 120     },
 121     onDelete: 'cascade'
 122   })
 123   VideoFile: VideoFileModel
 124
 125   @ForeignKey(() => VideoStreamingPlaylistModel)
 126   @Column
 127   videoStreamingPlaylistId: number
 128
 129   @BelongsTo(() => VideoStreamingPlaylistModel, {
 130     foreignKey: {
 131       allowNull: true
 132     },
 133     onDelete: 'cascade'
 134   })
 135   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 136
 137   @ForeignKey(() => ActorModel)
 138   @Column
 139   actorId: number
 140
 141   @BelongsTo(() => ActorModel, {
 142     foreignKey: {
 143       allowNull: false
 144     },
 145     onDelete: 'cascade'
 146   })
 147   Actor: ActorModel
 148
 149   @BeforeDestroy
 150   static async removeFile (instance: VideoRedundancyModel) {
 151     if (!instance.isOwned()) return
 152
 153     if (instance.videoFileId) {
 154       const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
 155
 156       const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
 157       logger.info('Removing duplicated video file %s.', logIdentifier)
 158
 159       videoFile.Video.removeFile(videoFile, true)
 160                .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
 161     }
 162
 163     if (instance.videoStreamingPlaylistId) {
 164       const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
 165
 166       const videoUUID = videoStreamingPlaylist.Video.uuid
 167       logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
 168
 169       videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
 170                             .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
 171     }
 172
 173     return undefined
 174   }
 175
 176   static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
 177     const actor = await getServerActor()
 178
 179     const query = {
 180       where: {
 181         actorId: actor.id,
 182         videoFileId
 183       }
 184     }
 185
 186     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 187   }
 188
 189   static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
 190     const actor = await getServerActor()
 191
 192     const query = {
 193       where: {
 194         actorId: actor.id,
 195         videoStreamingPlaylistId
 196       }
 197     }
 198
 199     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 200   }
 201
 202   static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird<MVideoRedundancyVideo> {
 203     const query = {
 204       where: { id },
 205       transaction
 206     }
 207
 208     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
 209   }
 210
 211   static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> {
 212     const query = {
 213       where: {
 214         url
 215       },
 216       transaction
 217     }
 218
 219     return VideoRedundancyModel.findOne(query)
 220   }
 221
 222   static async isLocalByVideoUUIDExists (uuid: string) {
 223     const actor = await getServerActor()
 224
 225     const query = {
 226       raw: true,
 227       attributes: [ 'id' ],
 228       where: {
 229         actorId: actor.id
 230       },
 231       include: [
 232         {
 233           attributes: [],
 234           model: VideoFileModel,
 235           required: true,
 236           include: [
 237             {
 238               attributes: [],
 239               model: VideoModel,
 240               required: true,
 241               where: {
 242                 uuid
 243               }
 244             }
 245           ]
 246         }
 247       ]
 248     }
 249
 250     return VideoRedundancyModel.findOne(query)
 251                                .then(r => !!r)
 252   }
 253
 254   static async getVideoSample (p: Bluebird<VideoModel[]>) {
 255     const rows = await p
 256     if (rows.length === 0) return undefined
 257
 258     const ids = rows.map(r => r.id)
 259     const id = sample(ids)
 260
 261     return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
 262   }
 263
 264   static async findMostViewToDuplicate (randomizedFactor: number) {
 265     // On VideoModel!
 266     const query = {
 267       attributes: [ 'id', 'views' ],
 268       limit: randomizedFactor,
 269       order: getVideoSort('-views'),
 270       where: {
 271         privacy: VideoPrivacy.PUBLIC,
 272         isLive: false
 273       },
 274       include: [
 275         await VideoRedundancyModel.buildVideoFileForDuplication(),
 276         VideoRedundancyModel.buildServerRedundancyInclude()
 277       ]
 278     }
 279
 280     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 281   }
 282
 283   static async findTrendingToDuplicate (randomizedFactor: number) {
 284     // On VideoModel!
 285     const query = {
 286       attributes: [ 'id', 'views' ],
 287       subQuery: false,
 288       group: 'VideoModel.id',
 289       limit: randomizedFactor,
 290       order: getVideoSort('-trending'),
 291       where: {
 292         privacy: VideoPrivacy.PUBLIC,
 293         isLive: false
 294       },
 295       include: [
 296         await VideoRedundancyModel.buildVideoFileForDuplication(),
 297         VideoRedundancyModel.buildServerRedundancyInclude(),
 298
 299         VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
 300       ]
 301     }
 302
 303     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 304   }
 305
 306   static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
 307     // On VideoModel!
 308     const query = {
 309       attributes: [ 'id', 'publishedAt' ],
 310       limit: randomizedFactor,
 311       order: getVideoSort('-publishedAt'),
 312       where: {
 313         privacy: VideoPrivacy.PUBLIC,
 314         isLive: false,
 315         views: {
 316           [Op.gte]: minViews
 317         }
 318       },
 319       include: [
 320         await VideoRedundancyModel.buildVideoFileForDuplication(),
 321         VideoRedundancyModel.buildServerRedundancyInclude()
 322       ]
 323     }
 324
 325     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
 326   }
 327
 328   static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
 329     const expiredDate = new Date()
 330     expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
 331
 332     const actor = await getServerActor()
 333
 334     const query = {
 335       where: {
 336         actorId: actor.id,
 337         strategy,
 338         createdAt: {
 339           [Op.lt]: expiredDate
 340         }
 341       }
 342     }
 343
 344     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
 345   }
 346
 347   static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
 348     const actor = await getServerActor()
 349     const redundancyInclude = {
 350       attributes: [],
 351       model: VideoRedundancyModel,
 352       required: true,
 353       where: {
 354         actorId: actor.id,
 355         strategy
 356       }
 357     }
 358
 359     const queryFiles: FindOptions = {
 360       include: [ redundancyInclude ]
 361     }
 362
 363     const queryStreamingPlaylists: FindOptions = {
 364       include: [
 365         {
 366           attributes: [],
 367           model: VideoModel.unscoped(),
 368           required: true,
 369           include: [
 370             {
 371               required: true,
 372               attributes: [],
 373               model: VideoStreamingPlaylistModel.unscoped(),
 374               include: [
 375                 redundancyInclude
 376               ]
 377             }
 378           ]
 379         }
 380       ]
 381     }
 382
 383     return Promise.all([
 384       VideoFileModel.aggregate('size', 'SUM', queryFiles),
 385       VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
 386     ]).then(([ r1, r2 ]) => {
 387       return parseAggregateResult(r1) + parseAggregateResult(r2)
 388     })
 389   }
 390
 391   static async listLocalExpired () {
 392     const actor = await getServerActor()
 393
 394     const query = {
 395       where: {
 396         actorId: actor.id,
 397         expiresOn: {
 398           [Op.lt]: new Date()
 399         }
 400       }
 401     }
 402
 403     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
 404   }
 405
 406   static async listRemoteExpired () {
 407     const actor = await getServerActor()
 408
 409     const query = {
 410       where: {
 411         actorId: {
 412           [Op.ne]: actor.id
 413         },
 414         expiresOn: {
 415           [Op.lt]: new Date(),
 416           [Op.ne]: null
 417         }
 418       }
 419     }
 420
 421     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
 422   }
 423
 424   static async listLocalOfServer (serverId: number) {
 425     const actor = await getServerActor()
 426     const buildVideoInclude = () => ({
 427       model: VideoModel,
 428       required: true,
 429       include: [
 430         {
 431           attributes: [],
 432           model: VideoChannelModel.unscoped(),
 433           required: true,
 434           include: [
 435             {
 436               attributes: [],
 437               model: ActorModel.unscoped(),
 438               required: true,
 439               where: {
 440                 serverId
 441               }
 442             }
 443           ]
 444         }
 445       ]
 446     })
 447
 448     const query = {
 449       where: {
 450         actorId: actor.id
 451       },
 452       include: [
 453         {
 454           model: VideoFileModel,
 455           required: false,
 456           include: [ buildVideoInclude() ]
 457         },
 458         {
 459           model: VideoStreamingPlaylistModel,
 460           required: false,
 461           include: [ buildVideoInclude() ]
 462         }
 463       ]
 464     }
 465
 466     return VideoRedundancyModel.findAll(query)
 467   }
 468
 469   static listForApi (options: {
 470     start: number
 471     count: number
 472     sort: string
 473     target: VideoRedundanciesTarget
 474     strategy?: string
 475   }) {
 476     const { start, count, sort, target, strategy } = options
 477     const redundancyWhere: WhereOptions = {}
 478     const videosWhere: WhereOptions = {}
 479     let redundancySqlSuffix = ''
 480
 481     if (target === 'my-videos') {
 482       Object.assign(videosWhere, { remote: false })
 483     } else if (target === 'remote-videos') {
 484       Object.assign(videosWhere, { remote: true })
 485       Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
 486       redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
 487     }
 488
 489     if (strategy) {
 490       Object.assign(redundancyWhere, { strategy: strategy })
 491     }
 492
 493     const videoFilterWhere = {
 494       [Op.and]: [
 495         {
 496           [Op.or]: [
 497             {
 498               id: {
 499                 [Op.in]: literal(
 500                   '(' +
 501                   'SELECT "videoId" FROM "videoFile" ' +
 502                   'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
 503                   redundancySqlSuffix +
 504                   ')'
 505                 )
 506               }
 507             },
 508             {
 509               id: {
 510                 [Op.in]: literal(
 511                   '(' +
 512                   'select "videoId" FROM "videoStreamingPlaylist" ' +
 513                   'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
 514                   redundancySqlSuffix +
 515                   ')'
 516                 )
 517               }
 518             }
 519           ]
 520         },
 521
 522         videosWhere
 523       ]
 524     }
 525
 526     // /!\ On video model /!\
 527     const findOptions = {
 528       offset: start,
 529       limit: count,
 530       order: getSort(sort),
 531       include: [
 532         {
 533           required: false,
 534           model: VideoFileModel,
 535           include: [
 536             {
 537               model: VideoRedundancyModel.unscoped(),
 538               required: false,
 539               where: redundancyWhere
 540             }
 541           ]
 542         },
 543         {
 544           required: false,
 545           model: VideoStreamingPlaylistModel.unscoped(),
 546           include: [
 547             {
 548               model: VideoRedundancyModel.unscoped(),
 549               required: false,
 550               where: redundancyWhere
 551             },
 552             {
 553               model: VideoFileModel,
 554               required: false
 555             }
 556           ]
 557         }
 558       ],
 559       where: videoFilterWhere
 560     }
 561
 562     // /!\ On video model /!\
 563     const countOptions = {
 564       where: videoFilterWhere
 565     }
 566
 567     return Promise.all([
 568       VideoModel.findAll(findOptions),
 569
 570       VideoModel.count(countOptions)
 571     ]).then(([ data, total ]) => ({ total, data }))
 572   }
 573
 574   static async getStats (strategy: VideoRedundancyStrategyWithManual) {
 575     const actor = await getServerActor()
 576
 577     const query: FindOptions = {
 578       raw: true,
 579       attributes: [
 580         [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
 581         [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
 582         [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
 583       ],
 584       where: {
 585         strategy,
 586         actorId: actor.id
 587       },
 588       include: [
 589         {
 590           attributes: [],
 591           model: VideoFileModel,
 592           required: true
 593         }
 594       ]
 595     }
 596
 597     return VideoRedundancyModel.findOne(query)
 598                                .then((r: any) => ({
 599                                  totalUsed: parseAggregateResult(r.totalUsed),
 600                                  totalVideos: r.totalVideos,
 601                                  totalVideoFiles: r.totalVideoFiles
 602                                }))
 603   }
 604
 605   static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
 606     const filesRedundancies: FileRedundancyInformation[] = []
 607     const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
 608
 609     for (const file of video.VideoFiles) {
 610       for (const redundancy of file.RedundancyVideos) {
 611         filesRedundancies.push({
 612           id: redundancy.id,
 613           fileUrl: redundancy.fileUrl,
 614           strategy: redundancy.strategy,
 615           createdAt: redundancy.createdAt,
 616           updatedAt: redundancy.updatedAt,
 617           expiresOn: redundancy.expiresOn,
 618           size: file.size
 619         })
 620       }
 621     }
 622
 623     for (const playlist of video.VideoStreamingPlaylists) {
 624       const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
 625
 626       for (const redundancy of playlist.RedundancyVideos) {
 627         streamingPlaylistsRedundancies.push({
 628           id: redundancy.id,
 629           fileUrl: redundancy.fileUrl,
 630           strategy: redundancy.strategy,
 631           createdAt: redundancy.createdAt,
 632           updatedAt: redundancy.updatedAt,
 633           expiresOn: redundancy.expiresOn,
 634           size
 635         })
 636       }
 637     }
 638
 639     return {
 640       id: video.id,
 641       name: video.name,
 642       url: video.url,
 643       uuid: video.uuid,
 644
 645       redundancies: {
 646         files: filesRedundancies,
 647         streamingPlaylists: streamingPlaylistsRedundancies
 648       }
 649     }
 650   }
 651
 652   getVideo () {
 653     if (this.VideoFile) return this.VideoFile.Video
 654
 655     return this.VideoStreamingPlaylist.Video
 656   }
 657
 658   isOwned () {
 659     return !!this.strategy
 660   }
 661
 662   toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
 663     if (this.VideoStreamingPlaylist) {
 664       return {
 665         id: this.url,
 666         type: 'CacheFile' as 'CacheFile',
 667         object: this.VideoStreamingPlaylist.Video.url,
 668         expires: this.expiresOn ? this.expiresOn.toISOString() : null,
 669         url: {
 670           type: 'Link',
 671           mediaType: 'application/x-mpegURL',
 672           href: this.fileUrl
 673         }
 674       }
 675     }
 676
 677     return {
 678       id: this.url,
 679       type: 'CacheFile' as 'CacheFile',
 680       object: this.VideoFile.Video.url,
 681       expires: this.expiresOn ? this.expiresOn.toISOString() : null,
 682       url: {
 683         type: 'Link',
 684         mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
 685         href: this.fileUrl,
 686         height: this.VideoFile.resolution,
 687         size: this.VideoFile.size,
 688         fps: this.VideoFile.fps
 689       }
 690     }
 691   }
 692
 693   // Don't include video files we already duplicated
 694   private static async buildVideoFileForDuplication () {
 695     const actor = await getServerActor()
 696
 697     const notIn = literal(
 698       '(' +
 699       `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
 700       ')'
 701     )
 702
 703     return {
 704       attributes: [],
 705       model: VideoFileModel,
 706       required: true,
 707       where: {
 708         id: {
 709           [Op.notIn]: notIn
 710         }
 711       }
 712     }
 713   }
 714
 715   private static buildServerRedundancyInclude () {
 716     return {
 717       attributes: [],
 718       model: VideoChannelModel.unscoped(),
 719       required: true,
 720       include: [
 721         {
 722           attributes: [],
 723           model: ActorModel.unscoped(),
 724           required: true,
 725           include: [
 726             {
 727               attributes: [],
 728               model: ServerModel.unscoped(),
 729               required: true,
 730               where: {
 731                 redundancyAllowed: true
 732               }
 733             }
 734           ]
 735         }
 736       ]
 737     }
 738   }
 739 }