server/models/video/video-file.ts

   1 import { remove } from 'fs-extra'
   2 import memoizee from 'memoizee'
   3 import { join } from 'path'
   4 import { FindOptions, Op, Transaction, WhereOptions } from 'sequelize'
   5 import {
   6   AllowNull,
   7   BelongsTo,
   8   Column,
   9   CreatedAt,
  10   DataType,
  11   Default,
  12   DefaultScope,
  13   ForeignKey,
  14   HasMany,
  15   Is,
  16   Model,
  17   Scopes,
  18   Table,
  19   UpdatedAt
  20 } from 'sequelize-typescript'
  21 import validator from 'validator'
  22 import { logger } from '@server/helpers/logger'
  23 import { extractVideo } from '@server/helpers/video'
  24 import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url'
  25 import {
  26   getHLSPrivateFileUrl,
  27   getHLSPublicFileUrl,
  28   getWebTorrentPrivateFileUrl,
  29   getWebTorrentPublicFileUrl
  30 } from '@server/lib/object-storage'
  31 import { getFSTorrentFilePath } from '@server/lib/paths'
  32 import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
  33 import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
  34 import { VideoResolution, VideoStorage } from '@shared/models'
  35 import { AttributesOnly } from '@shared/typescript-utils'
  36 import {
  37   isVideoFileExtnameValid,
  38   isVideoFileInfoHashValid,
  39   isVideoFileResolutionValid,
  40   isVideoFileSizeValid,
  41   isVideoFPSResolutionValid
  42 } from '../../helpers/custom-validators/videos'
  43 import {
  44   LAZY_STATIC_PATHS,
  45   MEMOIZE_LENGTH,
  46   MEMOIZE_TTL,
  47   STATIC_DOWNLOAD_PATHS,
  48   STATIC_PATHS,
  49   WEBSERVER
  50 } from '../../initializers/constants'
  51 import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file'
  52 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
  53 import { doesExist } from '../shared'
  54 import { parseAggregateResult, throwIfNotValid } from '../utils'
  55 import { VideoModel } from './video'
  56 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
  57 import { CONFIG } from '@server/initializers/config'
  58
  59 export enum ScopeNames {
  60   WITH_VIDEO = 'WITH_VIDEO',
  61   WITH_METADATA = 'WITH_METADATA',
  62   WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
  63 }
  64
  65 @DefaultScope(() => ({
  66   attributes: {
  67     exclude: [ 'metadata' ]
  68   }
  69 }))
  70 @Scopes(() => ({
  71   [ScopeNames.WITH_VIDEO]: {
  72     include: [
  73       {
  74         model: VideoModel.unscoped(),
  75         required: true
  76       }
  77     ]
  78   },
  79   [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: WhereOptions } = {}) => {
  80     return {
  81       include: [
  82         {
  83           model: VideoModel.unscoped(),
  84           required: false,
  85           where: options.whereVideo
  86         },
  87         {
  88           model: VideoStreamingPlaylistModel.unscoped(),
  89           required: false,
  90           include: [
  91             {
  92               model: VideoModel.unscoped(),
  93               required: true,
  94               where: options.whereVideo
  95             }
  96           ]
  97         }
  98       ]
  99     }
 100   },
 101   [ScopeNames.WITH_METADATA]: {
 102     attributes: {
 103       include: [ 'metadata' ]
 104     }
 105   }
 106 }))
 107 @Table({
 108   tableName: 'videoFile',
 109   indexes: [
 110     {
 111       fields: [ 'videoId' ],
 112       where: {
 113         videoId: {
 114           [Op.ne]: null
 115         }
 116       }
 117     },
 118     {
 119       fields: [ 'videoStreamingPlaylistId' ],
 120       where: {
 121         videoStreamingPlaylistId: {
 122           [Op.ne]: null
 123         }
 124       }
 125     },
 126
 127     {
 128       fields: [ 'infoHash' ]
 129     },
 130
 131     {
 132       fields: [ 'torrentFilename' ],
 133       unique: true
 134     },
 135
 136     {
 137       fields: [ 'filename' ],
 138       unique: true
 139     },
 140
 141     {
 142       fields: [ 'videoId', 'resolution', 'fps' ],
 143       unique: true,
 144       where: {
 145         videoId: {
 146           [Op.ne]: null
 147         }
 148       }
 149     },
 150     {
 151       fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
 152       unique: true,
 153       where: {
 154         videoStreamingPlaylistId: {
 155           [Op.ne]: null
 156         }
 157       }
 158     }
 159   ]
 160 })
 161 export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>>> {
 162   @CreatedAt
 163   createdAt: Date
 164
 165   @UpdatedAt
 166   updatedAt: Date
 167
 168   @AllowNull(false)
 169   @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution'))
 170   @Column
 171   resolution: number
 172
 173   @AllowNull(false)
 174   @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size'))
 175   @Column(DataType.BIGINT)
 176   size: number
 177
 178   @AllowNull(false)
 179   @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
 180   @Column
 181   extname: string
 182
 183   @AllowNull(true)
 184   @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
 185   @Column
 186   infoHash: string
 187
 188   @AllowNull(false)
 189   @Default(-1)
 190   @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
 191   @Column
 192   fps: number
 193
 194   @AllowNull(true)
 195   @Column(DataType.JSONB)
 196   metadata: any
 197
 198   @AllowNull(true)
 199   @Column
 200   metadataUrl: string
 201
 202   // Could be null for remote files
 203   @AllowNull(true)
 204   @Column
 205   fileUrl: string
 206
 207   // Could be null for live files
 208   @AllowNull(true)
 209   @Column
 210   filename: string
 211
 212   // Could be null for remote files
 213   @AllowNull(true)
 214   @Column
 215   torrentUrl: string
 216
 217   // Could be null for live files
 218   @AllowNull(true)
 219   @Column
 220   torrentFilename: string
 221
 222   @ForeignKey(() => VideoModel)
 223   @Column
 224   videoId: number
 225
 226   @AllowNull(false)
 227   @Default(VideoStorage.FILE_SYSTEM)
 228   @Column
 229   storage: VideoStorage
 230
 231   @BelongsTo(() => VideoModel, {
 232     foreignKey: {
 233       allowNull: true
 234     },
 235     onDelete: 'CASCADE'
 236   })
 237   Video: VideoModel
 238
 239   @ForeignKey(() => VideoStreamingPlaylistModel)
 240   @Column
 241   videoStreamingPlaylistId: number
 242
 243   @BelongsTo(() => VideoStreamingPlaylistModel, {
 244     foreignKey: {
 245       allowNull: true
 246     },
 247     onDelete: 'CASCADE'
 248   })
 249   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 250
 251   @HasMany(() => VideoRedundancyModel, {
 252     foreignKey: {
 253       allowNull: true
 254     },
 255     onDelete: 'CASCADE',
 256     hooks: true
 257   })
 258   RedundancyVideos: VideoRedundancyModel[]
 259
 260   static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, {
 261     promise: true,
 262     max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
 263     maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
 264   })
 265
 266   static doesInfohashExist (infoHash: string) {
 267     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
 268
 269     return doesExist(query, { infoHash })
 270   }
 271
 272   static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
 273     const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
 274
 275     return !!videoFile
 276   }
 277
 278   static async doesOwnedTorrentFileExist (filename: string) {
 279     const query = 'SELECT 1 FROM "videoFile" ' +
 280                   'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
 281                   'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
 282                   'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
 283                   'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
 284
 285     return doesExist(query, { filename })
 286   }
 287
 288   static async doesOwnedWebTorrentVideoFileExist (filename: string) {
 289     const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
 290                   `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1`
 291
 292     return doesExist(query, { filename })
 293   }
 294
 295   static loadByFilename (filename: string) {
 296     const query = {
 297       where: {
 298         filename
 299       }
 300     }
 301
 302     return VideoFileModel.findOne(query)
 303   }
 304
 305   static loadWithVideoByFilename (filename: string): Promise<MVideoFileVideo | MVideoFileStreamingPlaylistVideo> {
 306     const query = {
 307       where: {
 308         filename
 309       }
 310     }
 311
 312     return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
 313   }
 314
 315   static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
 316     const query = {
 317       where: {
 318         torrentFilename: filename
 319       }
 320     }
 321
 322     return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
 323   }
 324
 325   static load (id: number): Promise<MVideoFile> {
 326     return VideoFileModel.findByPk(id)
 327   }
 328
 329   static loadWithMetadata (id: number) {
 330     return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
 331   }
 332
 333   static loadWithVideo (id: number) {
 334     return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
 335   }
 336
 337   static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
 338     const whereVideo = validator.isUUID(videoIdOrUUID + '')
 339       ? { uuid: videoIdOrUUID }
 340       : { id: videoIdOrUUID }
 341
 342     const options = {
 343       where: {
 344         id
 345       }
 346     }
 347
 348     return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
 349       .findOne(options)
 350       .then(file => {
 351         // We used `required: false` so check we have at least a video or a streaming playlist
 352         if (!file.Video && !file.VideoStreamingPlaylist) return null
 353
 354         return file
 355       })
 356   }
 357
 358   static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
 359     const query = {
 360       include: [
 361         {
 362           model: VideoModel.unscoped(),
 363           required: true,
 364           include: [
 365             {
 366               model: VideoStreamingPlaylistModel.unscoped(),
 367               required: true,
 368               where: {
 369                 id: streamingPlaylistId
 370               }
 371             }
 372           ]
 373         }
 374       ],
 375       transaction
 376     }
 377
 378     return VideoFileModel.findAll(query)
 379   }
 380
 381   static getStats () {
 382     const webtorrentFilesQuery: FindOptions = {
 383       include: [
 384         {
 385           attributes: [],
 386           required: true,
 387           model: VideoModel.unscoped(),
 388           where: {
 389             remote: false
 390           }
 391         }
 392       ]
 393     }
 394
 395     const hlsFilesQuery: FindOptions = {
 396       include: [
 397         {
 398           attributes: [],
 399           required: true,
 400           model: VideoStreamingPlaylistModel.unscoped(),
 401           include: [
 402             {
 403               attributes: [],
 404               model: VideoModel.unscoped(),
 405               required: true,
 406               where: {
 407                 remote: false
 408               }
 409             }
 410           ]
 411         }
 412       ]
 413     }
 414
 415     return Promise.all([
 416       VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery),
 417       VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
 418     ]).then(([ webtorrentResult, hlsResult ]) => ({
 419       totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult)
 420     }))
 421   }
 422
 423   // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
 424   static async customUpsert (
 425     videoFile: MVideoFile,
 426     mode: 'streaming-playlist' | 'video',
 427     transaction: Transaction
 428   ) {
 429     const baseFind = {
 430       fps: videoFile.fps,
 431       resolution: videoFile.resolution,
 432       transaction
 433     }
 434
 435     const element = mode === 'streaming-playlist'
 436       ? await VideoFileModel.loadHLSFile({ ...baseFind, playlistId: videoFile.videoStreamingPlaylistId })
 437       : await VideoFileModel.loadWebTorrentFile({ ...baseFind, videoId: videoFile.videoId })
 438
 439     if (!element) return videoFile.save({ transaction })
 440
 441     for (const k of Object.keys(videoFile.toJSON())) {
 442       element.set(k, videoFile[k])
 443     }
 444
 445     return element.save({ transaction })
 446   }
 447
 448   static async loadWebTorrentFile (options: {
 449     videoId: number
 450     fps: number
 451     resolution: number
 452     transaction?: Transaction
 453   }) {
 454     const where = {
 455       fps: options.fps,
 456       resolution: options.resolution,
 457       videoId: options.videoId
 458     }
 459
 460     return VideoFileModel.findOne({ where, transaction: options.transaction })
 461   }
 462
 463   static async loadHLSFile (options: {
 464     playlistId: number
 465     fps: number
 466     resolution: number
 467     transaction?: Transaction
 468   }) {
 469     const where = {
 470       fps: options.fps,
 471       resolution: options.resolution,
 472       videoStreamingPlaylistId: options.playlistId
 473     }
 474
 475     return VideoFileModel.findOne({ where, transaction: options.transaction })
 476   }
 477
 478   static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
 479     const options = {
 480       where: { videoStreamingPlaylistId }
 481     }
 482
 483     return VideoFileModel.destroy(options)
 484   }
 485
 486   hasTorrent () {
 487     return this.infoHash && this.torrentFilename
 488   }
 489
 490   getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
 491     if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video
 492
 493     return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
 494   }
 495
 496   getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
 497     return extractVideo(this.getVideoOrStreamingPlaylist())
 498   }
 499
 500   isAudio () {
 501     return this.resolution === VideoResolution.H_NOVIDEO
 502   }
 503
 504   isLive () {
 505     return this.size === -1
 506   }
 507
 508   isHLS () {
 509     return !!this.videoStreamingPlaylistId
 510   }
 511
 512   // ---------------------------------------------------------------------------
 513
 514   getObjectStorageUrl (video: MVideo) {
 515     if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
 516       return this.getPrivateObjectStorageUrl(video)
 517     }
 518
 519     return this.getPublicObjectStorageUrl()
 520   }
 521
 522   private getPrivateObjectStorageUrl (video: MVideo) {
 523     if (this.isHLS()) {
 524       return getHLSPrivateFileUrl(video, this.filename)
 525     }
 526
 527     return getWebTorrentPrivateFileUrl(this.filename)
 528   }
 529
 530   private getPublicObjectStorageUrl () {
 531     if (this.isHLS()) {
 532       return getHLSPublicFileUrl(this.fileUrl)
 533     }
 534
 535     return getWebTorrentPublicFileUrl(this.fileUrl)
 536   }
 537
 538   // ---------------------------------------------------------------------------
 539
 540   getFileUrl (video: MVideo) {
 541     if (video.isOwned()) {
 542       if (this.storage === VideoStorage.OBJECT_STORAGE) {
 543         return this.getObjectStorageUrl(video)
 544       }
 545
 546       return WEBSERVER.URL + this.getFileStaticPath(video)
 547     }
 548
 549     return this.fileUrl
 550   }
 551
 552   // ---------------------------------------------------------------------------
 553
 554   getFileStaticPath (video: MVideo) {
 555     if (this.isHLS()) return this.getHLSFileStaticPath(video)
 556
 557     return this.getWebTorrentFileStaticPath(video)
 558   }
 559
 560   private getWebTorrentFileStaticPath (video: MVideo) {
 561     if (isVideoInPrivateDirectory(video.privacy)) {
 562       return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename)
 563     }
 564
 565     return join(STATIC_PATHS.WEBSEED, this.filename)
 566   }
 567
 568   private getHLSFileStaticPath (video: MVideo) {
 569     if (isVideoInPrivateDirectory(video.privacy)) {
 570       return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
 571     }
 572
 573     return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
 574   }
 575
 576   // ---------------------------------------------------------------------------
 577
 578   getFileDownloadUrl (video: MVideoWithHost) {
 579     const path = this.isHLS()
 580       ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
 581       : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
 582
 583     if (video.isOwned()) return WEBSERVER.URL + path
 584
 585     // FIXME: don't guess remote URL
 586     return buildRemoteVideoBaseUrl(video, path)
 587   }
 588
 589   getRemoteTorrentUrl (video: MVideo) {
 590     if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
 591
 592     return this.torrentUrl
 593   }
 594
 595   // We proxify torrent requests so use a local URL
 596   getTorrentUrl () {
 597     if (!this.torrentFilename) return null
 598
 599     return WEBSERVER.URL + this.getTorrentStaticPath()
 600   }
 601
 602   getTorrentStaticPath () {
 603     if (!this.torrentFilename) return null
 604
 605     return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
 606   }
 607
 608   getTorrentDownloadUrl () {
 609     if (!this.torrentFilename) return null
 610
 611     return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
 612   }
 613
 614   removeTorrent () {
 615     if (!this.torrentFilename) return null
 616
 617     const torrentPath = getFSTorrentFilePath(this)
 618     return remove(torrentPath)
 619       .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
 620   }
 621
 622   hasSameUniqueKeysThan (other: MVideoFile) {
 623     return this.fps === other.fps &&
 624       this.resolution === other.resolution &&
 625       (
 626         (this.videoId !== null && this.videoId === other.videoId) ||
 627         (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
 628       )
 629   }
 630
 631   withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
 632     if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist })
 633
 634     return Object.assign(this, { Video: videoOrPlaylist })
 635   }
 636 }