server/models/video/video-file.ts

   1 import { remove } from 'fs-extra'
   2 import memoizee from 'memoizee'
   3 import { join } from 'path'
   4 import { FindOptions, Op, Transaction, WhereOptions } from 'sequelize'
   5 import {
   6   AllowNull,
   7   BelongsTo,
   8   Column,
   9   CreatedAt,
  10   DataType,
  11   Default,
  12   DefaultScope,
  13   ForeignKey,
  14   HasMany,
  15   Is,
  16   Model,
  17   Scopes,
  18   Table,
  19   UpdatedAt
  20 } from 'sequelize-typescript'
  21 import validator from 'validator'
  22 import { logger } from '@server/helpers/logger'
  23 import { extractVideo } from '@server/helpers/video'
  24 import { CONFIG } from '@server/initializers/config'
  25 import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url'
  26 import {
  27   getHLSPrivateFileUrl,
  28   getHLSPublicFileUrl,
  29   getWebTorrentPrivateFileUrl,
  30   getWebTorrentPublicFileUrl
  31 } from '@server/lib/object-storage'
  32 import { getFSTorrentFilePath } from '@server/lib/paths'
  33 import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
  34 import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
  35 import { VideoResolution, VideoStorage } from '@shared/models'
  36 import { AttributesOnly } from '@shared/typescript-utils'
  37 import {
  38   isVideoFileExtnameValid,
  39   isVideoFileInfoHashValid,
  40   isVideoFileResolutionValid,
  41   isVideoFileSizeValid,
  42   isVideoFPSResolutionValid
  43 } from '../../helpers/custom-validators/videos'
  44 import {
  45   LAZY_STATIC_PATHS,
  46   MEMOIZE_LENGTH,
  47   MEMOIZE_TTL,
  48   STATIC_DOWNLOAD_PATHS,
  49   STATIC_PATHS,
  50   WEBSERVER
  51 } from '../../initializers/constants'
  52 import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file'
  53 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
  54 import { doesExist, parseAggregateResult, throwIfNotValid } from '../shared'
  55 import { VideoModel } from './video'
  56 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
  57
  58 export enum ScopeNames {
  59   WITH_VIDEO = 'WITH_VIDEO',
  60   WITH_METADATA = 'WITH_METADATA',
  61   WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
  62 }
  63
  64 @DefaultScope(() => ({
  65   attributes: {
  66     exclude: [ 'metadata' ]
  67   }
  68 }))
  69 @Scopes(() => ({
  70   [ScopeNames.WITH_VIDEO]: {
  71     include: [
  72       {
  73         model: VideoModel.unscoped(),
  74         required: true
  75       }
  76     ]
  77   },
  78   [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: WhereOptions } = {}) => {
  79     return {
  80       include: [
  81         {
  82           model: VideoModel.unscoped(),
  83           required: false,
  84           where: options.whereVideo
  85         },
  86         {
  87           model: VideoStreamingPlaylistModel.unscoped(),
  88           required: false,
  89           include: [
  90             {
  91               model: VideoModel.unscoped(),
  92               required: true,
  93               where: options.whereVideo
  94             }
  95           ]
  96         }
  97       ]
  98     }
  99   },
 100   [ScopeNames.WITH_METADATA]: {
 101     attributes: {
 102       include: [ 'metadata' ]
 103     }
 104   }
 105 }))
 106 @Table({
 107   tableName: 'videoFile',
 108   indexes: [
 109     {
 110       fields: [ 'videoId' ],
 111       where: {
 112         videoId: {
 113           [Op.ne]: null
 114         }
 115       }
 116     },
 117     {
 118       fields: [ 'videoStreamingPlaylistId' ],
 119       where: {
 120         videoStreamingPlaylistId: {
 121           [Op.ne]: null
 122         }
 123       }
 124     },
 125
 126     {
 127       fields: [ 'infoHash' ]
 128     },
 129
 130     {
 131       fields: [ 'torrentFilename' ],
 132       unique: true
 133     },
 134
 135     {
 136       fields: [ 'filename' ],
 137       unique: true
 138     },
 139
 140     {
 141       fields: [ 'videoId', 'resolution', 'fps' ],
 142       unique: true,
 143       where: {
 144         videoId: {
 145           [Op.ne]: null
 146         }
 147       }
 148     },
 149     {
 150       fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
 151       unique: true,
 152       where: {
 153         videoStreamingPlaylistId: {
 154           [Op.ne]: null
 155         }
 156       }
 157     }
 158   ]
 159 })
 160 export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>>> {
 161   @CreatedAt
 162   createdAt: Date
 163
 164   @UpdatedAt
 165   updatedAt: Date
 166
 167   @AllowNull(false)
 168   @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution'))
 169   @Column
 170   resolution: number
 171
 172   @AllowNull(false)
 173   @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size'))
 174   @Column(DataType.BIGINT)
 175   size: number
 176
 177   @AllowNull(false)
 178   @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
 179   @Column
 180   extname: string
 181
 182   @AllowNull(true)
 183   @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
 184   @Column
 185   infoHash: string
 186
 187   @AllowNull(false)
 188   @Default(-1)
 189   @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
 190   @Column
 191   fps: number
 192
 193   @AllowNull(true)
 194   @Column(DataType.JSONB)
 195   metadata: any
 196
 197   @AllowNull(true)
 198   @Column
 199   metadataUrl: string
 200
 201   // Could be null for remote files
 202   @AllowNull(true)
 203   @Column
 204   fileUrl: string
 205
 206   // Could be null for live files
 207   @AllowNull(true)
 208   @Column
 209   filename: string
 210
 211   // Could be null for remote files
 212   @AllowNull(true)
 213   @Column
 214   torrentUrl: string
 215
 216   // Could be null for live files
 217   @AllowNull(true)
 218   @Column
 219   torrentFilename: string
 220
 221   @ForeignKey(() => VideoModel)
 222   @Column
 223   videoId: number
 224
 225   @AllowNull(false)
 226   @Default(VideoStorage.FILE_SYSTEM)
 227   @Column
 228   storage: VideoStorage
 229
 230   @BelongsTo(() => VideoModel, {
 231     foreignKey: {
 232       allowNull: true
 233     },
 234     onDelete: 'CASCADE'
 235   })
 236   Video: VideoModel
 237
 238   @ForeignKey(() => VideoStreamingPlaylistModel)
 239   @Column
 240   videoStreamingPlaylistId: number
 241
 242   @BelongsTo(() => VideoStreamingPlaylistModel, {
 243     foreignKey: {
 244       allowNull: true
 245     },
 246     onDelete: 'CASCADE'
 247   })
 248   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 249
 250   @HasMany(() => VideoRedundancyModel, {
 251     foreignKey: {
 252       allowNull: true
 253     },
 254     onDelete: 'CASCADE',
 255     hooks: true
 256   })
 257   RedundancyVideos: VideoRedundancyModel[]
 258
 259   static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, {
 260     promise: true,
 261     max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
 262     maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
 263   })
 264
 265   static doesInfohashExist (infoHash: string) {
 266     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
 267
 268     return doesExist(this.sequelize, query, { infoHash })
 269   }
 270
 271   static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
 272     const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
 273
 274     return !!videoFile
 275   }
 276
 277   static async doesOwnedTorrentFileExist (filename: string) {
 278     const query = 'SELECT 1 FROM "videoFile" ' +
 279                   'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
 280                   'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
 281                   'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
 282                   'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
 283
 284     return doesExist(this.sequelize, query, { filename })
 285   }
 286
 287   static async doesOwnedWebTorrentVideoFileExist (filename: string) {
 288     const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
 289                   `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1`
 290
 291     return doesExist(this.sequelize, query, { filename })
 292   }
 293
 294   static loadByFilename (filename: string) {
 295     const query = {
 296       where: {
 297         filename
 298       }
 299     }
 300
 301     return VideoFileModel.findOne(query)
 302   }
 303
 304   static loadWithVideoByFilename (filename: string): Promise<MVideoFileVideo | MVideoFileStreamingPlaylistVideo> {
 305     const query = {
 306       where: {
 307         filename
 308       }
 309     }
 310
 311     return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
 312   }
 313
 314   static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
 315     const query = {
 316       where: {
 317         torrentFilename: filename
 318       }
 319     }
 320
 321     return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
 322   }
 323
 324   static load (id: number): Promise<MVideoFile> {
 325     return VideoFileModel.findByPk(id)
 326   }
 327
 328   static loadWithMetadata (id: number) {
 329     return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
 330   }
 331
 332   static loadWithVideo (id: number) {
 333     return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
 334   }
 335
 336   static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
 337     const whereVideo = validator.isUUID(videoIdOrUUID + '')
 338       ? { uuid: videoIdOrUUID }
 339       : { id: videoIdOrUUID }
 340
 341     const options = {
 342       where: {
 343         id
 344       }
 345     }
 346
 347     return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
 348       .findOne(options)
 349       .then(file => {
 350         // We used `required: false` so check we have at least a video or a streaming playlist
 351         if (!file.Video && !file.VideoStreamingPlaylist) return null
 352
 353         return file
 354       })
 355   }
 356
 357   static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
 358     const query = {
 359       include: [
 360         {
 361           model: VideoModel.unscoped(),
 362           required: true,
 363           include: [
 364             {
 365               model: VideoStreamingPlaylistModel.unscoped(),
 366               required: true,
 367               where: {
 368                 id: streamingPlaylistId
 369               }
 370             }
 371           ]
 372         }
 373       ],
 374       transaction
 375     }
 376
 377     return VideoFileModel.findAll(query)
 378   }
 379
 380   static getStats () {
 381     const webtorrentFilesQuery: FindOptions = {
 382       include: [
 383         {
 384           attributes: [],
 385           required: true,
 386           model: VideoModel.unscoped(),
 387           where: {
 388             remote: false
 389           }
 390         }
 391       ]
 392     }
 393
 394     const hlsFilesQuery: FindOptions = {
 395       include: [
 396         {
 397           attributes: [],
 398           required: true,
 399           model: VideoStreamingPlaylistModel.unscoped(),
 400           include: [
 401             {
 402               attributes: [],
 403               model: VideoModel.unscoped(),
 404               required: true,
 405               where: {
 406                 remote: false
 407               }
 408             }
 409           ]
 410         }
 411       ]
 412     }
 413
 414     return Promise.all([
 415       VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery),
 416       VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
 417     ]).then(([ webtorrentResult, hlsResult ]) => ({
 418       totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult)
 419     }))
 420   }
 421
 422   // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
 423   static async customUpsert (
 424     videoFile: MVideoFile,
 425     mode: 'streaming-playlist' | 'video',
 426     transaction: Transaction
 427   ) {
 428     const baseFind = {
 429       fps: videoFile.fps,
 430       resolution: videoFile.resolution,
 431       transaction
 432     }
 433
 434     const element = mode === 'streaming-playlist'
 435       ? await VideoFileModel.loadHLSFile({ ...baseFind, playlistId: videoFile.videoStreamingPlaylistId })
 436       : await VideoFileModel.loadWebTorrentFile({ ...baseFind, videoId: videoFile.videoId })
 437
 438     if (!element) return videoFile.save({ transaction })
 439
 440     for (const k of Object.keys(videoFile.toJSON())) {
 441       element.set(k, videoFile[k])
 442     }
 443
 444     return element.save({ transaction })
 445   }
 446
 447   static async loadWebTorrentFile (options: {
 448     videoId: number
 449     fps: number
 450     resolution: number
 451     transaction?: Transaction
 452   }) {
 453     const where = {
 454       fps: options.fps,
 455       resolution: options.resolution,
 456       videoId: options.videoId
 457     }
 458
 459     return VideoFileModel.findOne({ where, transaction: options.transaction })
 460   }
 461
 462   static async loadHLSFile (options: {
 463     playlistId: number
 464     fps: number
 465     resolution: number
 466     transaction?: Transaction
 467   }) {
 468     const where = {
 469       fps: options.fps,
 470       resolution: options.resolution,
 471       videoStreamingPlaylistId: options.playlistId
 472     }
 473
 474     return VideoFileModel.findOne({ where, transaction: options.transaction })
 475   }
 476
 477   static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
 478     const options = {
 479       where: { videoStreamingPlaylistId }
 480     }
 481
 482     return VideoFileModel.destroy(options)
 483   }
 484
 485   hasTorrent () {
 486     return this.infoHash && this.torrentFilename
 487   }
 488
 489   getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
 490     if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video
 491
 492     return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
 493   }
 494
 495   getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
 496     return extractVideo(this.getVideoOrStreamingPlaylist())
 497   }
 498
 499   isAudio () {
 500     return this.resolution === VideoResolution.H_NOVIDEO
 501   }
 502
 503   isLive () {
 504     return this.size === -1
 505   }
 506
 507   isHLS () {
 508     return !!this.videoStreamingPlaylistId
 509   }
 510
 511   // ---------------------------------------------------------------------------
 512
 513   getObjectStorageUrl (video: MVideo) {
 514     if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
 515       return this.getPrivateObjectStorageUrl(video)
 516     }
 517
 518     return this.getPublicObjectStorageUrl()
 519   }
 520
 521   private getPrivateObjectStorageUrl (video: MVideo) {
 522     if (this.isHLS()) {
 523       return getHLSPrivateFileUrl(video, this.filename)
 524     }
 525
 526     return getWebTorrentPrivateFileUrl(this.filename)
 527   }
 528
 529   private getPublicObjectStorageUrl () {
 530     if (this.isHLS()) {
 531       return getHLSPublicFileUrl(this.fileUrl)
 532     }
 533
 534     return getWebTorrentPublicFileUrl(this.fileUrl)
 535   }
 536
 537   // ---------------------------------------------------------------------------
 538
 539   getFileUrl (video: MVideo) {
 540     if (video.isOwned()) {
 541       if (this.storage === VideoStorage.OBJECT_STORAGE) {
 542         return this.getObjectStorageUrl(video)
 543       }
 544
 545       return WEBSERVER.URL + this.getFileStaticPath(video)
 546     }
 547
 548     return this.fileUrl
 549   }
 550
 551   // ---------------------------------------------------------------------------
 552
 553   getFileStaticPath (video: MVideo) {
 554     if (this.isHLS()) return this.getHLSFileStaticPath(video)
 555
 556     return this.getWebTorrentFileStaticPath(video)
 557   }
 558
 559   private getWebTorrentFileStaticPath (video: MVideo) {
 560     if (isVideoInPrivateDirectory(video.privacy)) {
 561       return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename)
 562     }
 563
 564     return join(STATIC_PATHS.WEBSEED, this.filename)
 565   }
 566
 567   private getHLSFileStaticPath (video: MVideo) {
 568     if (isVideoInPrivateDirectory(video.privacy)) {
 569       return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
 570     }
 571
 572     return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
 573   }
 574
 575   // ---------------------------------------------------------------------------
 576
 577   getFileDownloadUrl (video: MVideoWithHost) {
 578     const path = this.isHLS()
 579       ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
 580       : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
 581
 582     if (video.isOwned()) return WEBSERVER.URL + path
 583
 584     // FIXME: don't guess remote URL
 585     return buildRemoteVideoBaseUrl(video, path)
 586   }
 587
 588   getRemoteTorrentUrl (video: MVideo) {
 589     if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
 590
 591     return this.torrentUrl
 592   }
 593
 594   // We proxify torrent requests so use a local URL
 595   getTorrentUrl () {
 596     if (!this.torrentFilename) return null
 597
 598     return WEBSERVER.URL + this.getTorrentStaticPath()
 599   }
 600
 601   getTorrentStaticPath () {
 602     if (!this.torrentFilename) return null
 603
 604     return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
 605   }
 606
 607   getTorrentDownloadUrl () {
 608     if (!this.torrentFilename) return null
 609
 610     return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
 611   }
 612
 613   removeTorrent () {
 614     if (!this.torrentFilename) return null
 615
 616     const torrentPath = getFSTorrentFilePath(this)
 617     return remove(torrentPath)
 618       .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
 619   }
 620
 621   hasSameUniqueKeysThan (other: MVideoFile) {
 622     return this.fps === other.fps &&
 623       this.resolution === other.resolution &&
 624       (
 625         (this.videoId !== null && this.videoId === other.videoId) ||
 626         (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
 627       )
 628   }
 629
 630   withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
 631     if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist })
 632
 633     return Object.assign(this, { Video: videoOrPlaylist })
 634   }
 635 }