server/models/video/video-file.ts

   1 import { remove } from 'fs-extra'
   2 import memoizee from 'memoizee'
   3 import { join } from 'path'
   4 import { FindOptions, Op, Transaction, WhereOptions } from 'sequelize'
   5 import {
   6   AllowNull,
   7   BelongsTo,
   8   Column,
   9   CreatedAt,
  10   DataType,
  11   Default,
  12   DefaultScope,
  13   ForeignKey,
  14   HasMany,
  15   Is,
  16   Model,
  17   Scopes,
  18   Table,
  19   UpdatedAt
  20 } from 'sequelize-typescript'
  21 import validator from 'validator'
  22 import { logger } from '@server/helpers/logger'
  23 import { extractVideo } from '@server/helpers/video'
  24 import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url'
  25 import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
  26 import { getFSTorrentFilePath } from '@server/lib/paths'
  27 import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
  28 import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
  29 import { VideoResolution, VideoStorage } from '@shared/models'
  30 import { AttributesOnly } from '@shared/typescript-utils'
  31 import {
  32   isVideoFileExtnameValid,
  33   isVideoFileInfoHashValid,
  34   isVideoFileResolutionValid,
  35   isVideoFileSizeValid,
  36   isVideoFPSResolutionValid
  37 } from '../../helpers/custom-validators/videos'
  38 import {
  39   LAZY_STATIC_PATHS,
  40   MEMOIZE_LENGTH,
  41   MEMOIZE_TTL,
  42   STATIC_DOWNLOAD_PATHS,
  43   STATIC_PATHS,
  44   WEBSERVER
  45 } from '../../initializers/constants'
  46 import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file'
  47 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
  48 import { doesExist } from '../shared'
  49 import { parseAggregateResult, throwIfNotValid } from '../utils'
  50 import { VideoModel } from './video'
  51 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
  52
  53 export enum ScopeNames {
  54   WITH_VIDEO = 'WITH_VIDEO',
  55   WITH_METADATA = 'WITH_METADATA',
  56   WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
  57 }
  58
  59 @DefaultScope(() => ({
  60   attributes: {
  61     exclude: [ 'metadata' ]
  62   }
  63 }))
  64 @Scopes(() => ({
  65   [ScopeNames.WITH_VIDEO]: {
  66     include: [
  67       {
  68         model: VideoModel.unscoped(),
  69         required: true
  70       }
  71     ]
  72   },
  73   [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: WhereOptions } = {}) => {
  74     return {
  75       include: [
  76         {
  77           model: VideoModel.unscoped(),
  78           required: false,
  79           where: options.whereVideo
  80         },
  81         {
  82           model: VideoStreamingPlaylistModel.unscoped(),
  83           required: false,
  84           include: [
  85             {
  86               model: VideoModel.unscoped(),
  87               required: true,
  88               where: options.whereVideo
  89             }
  90           ]
  91         }
  92       ]
  93     }
  94   },
  95   [ScopeNames.WITH_METADATA]: {
  96     attributes: {
  97       include: [ 'metadata' ]
  98     }
  99   }
 100 }))
 101 @Table({
 102   tableName: 'videoFile',
 103   indexes: [
 104     {
 105       fields: [ 'videoId' ],
 106       where: {
 107         videoId: {
 108           [Op.ne]: null
 109         }
 110       }
 111     },
 112     {
 113       fields: [ 'videoStreamingPlaylistId' ],
 114       where: {
 115         videoStreamingPlaylistId: {
 116           [Op.ne]: null
 117         }
 118       }
 119     },
 120
 121     {
 122       fields: [ 'infoHash' ]
 123     },
 124
 125     {
 126       fields: [ 'torrentFilename' ],
 127       unique: true
 128     },
 129
 130     {
 131       fields: [ 'filename' ],
 132       unique: true
 133     },
 134
 135     {
 136       fields: [ 'videoId', 'resolution', 'fps' ],
 137       unique: true,
 138       where: {
 139         videoId: {
 140           [Op.ne]: null
 141         }
 142       }
 143     },
 144     {
 145       fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
 146       unique: true,
 147       where: {
 148         videoStreamingPlaylistId: {
 149           [Op.ne]: null
 150         }
 151       }
 152     }
 153   ]
 154 })
 155 export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>>> {
 156   @CreatedAt
 157   createdAt: Date
 158
 159   @UpdatedAt
 160   updatedAt: Date
 161
 162   @AllowNull(false)
 163   @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution'))
 164   @Column
 165   resolution: number
 166
 167   @AllowNull(false)
 168   @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size'))
 169   @Column(DataType.BIGINT)
 170   size: number
 171
 172   @AllowNull(false)
 173   @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
 174   @Column
 175   extname: string
 176
 177   @AllowNull(true)
 178   @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
 179   @Column
 180   infoHash: string
 181
 182   @AllowNull(false)
 183   @Default(-1)
 184   @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
 185   @Column
 186   fps: number
 187
 188   @AllowNull(true)
 189   @Column(DataType.JSONB)
 190   metadata: any
 191
 192   @AllowNull(true)
 193   @Column
 194   metadataUrl: string
 195
 196   // Could be null for remote files
 197   @AllowNull(true)
 198   @Column
 199   fileUrl: string
 200
 201   // Could be null for live files
 202   @AllowNull(true)
 203   @Column
 204   filename: string
 205
 206   // Could be null for remote files
 207   @AllowNull(true)
 208   @Column
 209   torrentUrl: string
 210
 211   // Could be null for live files
 212   @AllowNull(true)
 213   @Column
 214   torrentFilename: string
 215
 216   @ForeignKey(() => VideoModel)
 217   @Column
 218   videoId: number
 219
 220   @AllowNull(false)
 221   @Default(VideoStorage.FILE_SYSTEM)
 222   @Column
 223   storage: VideoStorage
 224
 225   @BelongsTo(() => VideoModel, {
 226     foreignKey: {
 227       allowNull: true
 228     },
 229     onDelete: 'CASCADE'
 230   })
 231   Video: VideoModel
 232
 233   @ForeignKey(() => VideoStreamingPlaylistModel)
 234   @Column
 235   videoStreamingPlaylistId: number
 236
 237   @BelongsTo(() => VideoStreamingPlaylistModel, {
 238     foreignKey: {
 239       allowNull: true
 240     },
 241     onDelete: 'CASCADE'
 242   })
 243   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 244
 245   @HasMany(() => VideoRedundancyModel, {
 246     foreignKey: {
 247       allowNull: true
 248     },
 249     onDelete: 'CASCADE',
 250     hooks: true
 251   })
 252   RedundancyVideos: VideoRedundancyModel[]
 253
 254   static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, {
 255     promise: true,
 256     max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
 257     maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
 258   })
 259
 260   static doesInfohashExist (infoHash: string) {
 261     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
 262
 263     return doesExist(query, { infoHash })
 264   }
 265
 266   static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
 267     const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
 268
 269     return !!videoFile
 270   }
 271
 272   static async doesOwnedTorrentFileExist (filename: string) {
 273     const query = 'SELECT 1 FROM "videoFile" ' +
 274                   'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
 275                   'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
 276                   'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
 277                   'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
 278
 279     return doesExist(query, { filename })
 280   }
 281
 282   static async doesOwnedWebTorrentVideoFileExist (filename: string) {
 283     const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
 284                   `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1`
 285
 286     return doesExist(query, { filename })
 287   }
 288
 289   static loadByFilename (filename: string) {
 290     const query = {
 291       where: {
 292         filename
 293       }
 294     }
 295
 296     return VideoFileModel.findOne(query)
 297   }
 298
 299   static loadWithVideoByFilename (filename: string): Promise<MVideoFileVideo | MVideoFileStreamingPlaylistVideo> {
 300     const query = {
 301       where: {
 302         filename
 303       }
 304     }
 305
 306     return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
 307   }
 308
 309   static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
 310     const query = {
 311       where: {
 312         torrentFilename: filename
 313       }
 314     }
 315
 316     return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
 317   }
 318
 319   static load (id: number): Promise<MVideoFile> {
 320     return VideoFileModel.findByPk(id)
 321   }
 322
 323   static loadWithMetadata (id: number) {
 324     return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
 325   }
 326
 327   static loadWithVideo (id: number) {
 328     return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
 329   }
 330
 331   static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
 332     const whereVideo = validator.isUUID(videoIdOrUUID + '')
 333       ? { uuid: videoIdOrUUID }
 334       : { id: videoIdOrUUID }
 335
 336     const options = {
 337       where: {
 338         id
 339       }
 340     }
 341
 342     return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
 343       .findOne(options)
 344       .then(file => {
 345         // We used `required: false` so check we have at least a video or a streaming playlist
 346         if (!file.Video && !file.VideoStreamingPlaylist) return null
 347
 348         return file
 349       })
 350   }
 351
 352   static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
 353     const query = {
 354       include: [
 355         {
 356           model: VideoModel.unscoped(),
 357           required: true,
 358           include: [
 359             {
 360               model: VideoStreamingPlaylistModel.unscoped(),
 361               required: true,
 362               where: {
 363                 id: streamingPlaylistId
 364               }
 365             }
 366           ]
 367         }
 368       ],
 369       transaction
 370     }
 371
 372     return VideoFileModel.findAll(query)
 373   }
 374
 375   static getStats () {
 376     const webtorrentFilesQuery: FindOptions = {
 377       include: [
 378         {
 379           attributes: [],
 380           required: true,
 381           model: VideoModel.unscoped(),
 382           where: {
 383             remote: false
 384           }
 385         }
 386       ]
 387     }
 388
 389     const hlsFilesQuery: FindOptions = {
 390       include: [
 391         {
 392           attributes: [],
 393           required: true,
 394           model: VideoStreamingPlaylistModel.unscoped(),
 395           include: [
 396             {
 397               attributes: [],
 398               model: VideoModel.unscoped(),
 399               required: true,
 400               where: {
 401                 remote: false
 402               }
 403             }
 404           ]
 405         }
 406       ]
 407     }
 408
 409     return Promise.all([
 410       VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery),
 411       VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
 412     ]).then(([ webtorrentResult, hlsResult ]) => ({
 413       totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult)
 414     }))
 415   }
 416
 417   // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
 418   static async customUpsert (
 419     videoFile: MVideoFile,
 420     mode: 'streaming-playlist' | 'video',
 421     transaction: Transaction
 422   ) {
 423     const baseFind = {
 424       fps: videoFile.fps,
 425       resolution: videoFile.resolution,
 426       transaction
 427     }
 428
 429     const element = mode === 'streaming-playlist'
 430       ? await VideoFileModel.loadHLSFile({ ...baseFind, playlistId: videoFile.videoStreamingPlaylistId })
 431       : await VideoFileModel.loadWebTorrentFile({ ...baseFind, videoId: videoFile.videoId })
 432
 433     if (!element) return videoFile.save({ transaction })
 434
 435     for (const k of Object.keys(videoFile.toJSON())) {
 436       element[k] = videoFile[k]
 437     }
 438
 439     return element.save({ transaction })
 440   }
 441
 442   static async loadWebTorrentFile (options: {
 443     videoId: number
 444     fps: number
 445     resolution: number
 446     transaction?: Transaction
 447   }) {
 448     const where = {
 449       fps: options.fps,
 450       resolution: options.resolution,
 451       videoId: options.videoId
 452     }
 453
 454     return VideoFileModel.findOne({ where, transaction: options.transaction })
 455   }
 456
 457   static async loadHLSFile (options: {
 458     playlistId: number
 459     fps: number
 460     resolution: number
 461     transaction?: Transaction
 462   }) {
 463     const where = {
 464       fps: options.fps,
 465       resolution: options.resolution,
 466       videoStreamingPlaylistId: options.playlistId
 467     }
 468
 469     return VideoFileModel.findOne({ where, transaction: options.transaction })
 470   }
 471
 472   static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
 473     const options = {
 474       where: { videoStreamingPlaylistId }
 475     }
 476
 477     return VideoFileModel.destroy(options)
 478   }
 479
 480   hasTorrent () {
 481     return this.infoHash && this.torrentFilename
 482   }
 483
 484   getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
 485     if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video
 486
 487     return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
 488   }
 489
 490   getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
 491     return extractVideo(this.getVideoOrStreamingPlaylist())
 492   }
 493
 494   isAudio () {
 495     return this.resolution === VideoResolution.H_NOVIDEO
 496   }
 497
 498   isLive () {
 499     return this.size === -1
 500   }
 501
 502   isHLS () {
 503     return !!this.videoStreamingPlaylistId
 504   }
 505
 506   getObjectStorageUrl () {
 507     if (this.isHLS()) {
 508       return getHLSPublicFileUrl(this.fileUrl)
 509     }
 510
 511     return getWebTorrentPublicFileUrl(this.fileUrl)
 512   }
 513
 514   getFileUrl (video: MVideo) {
 515     if (this.storage === VideoStorage.OBJECT_STORAGE) {
 516       return this.getObjectStorageUrl()
 517     }
 518
 519     if (!this.Video) this.Video = video as VideoModel
 520     if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
 521
 522     return this.fileUrl
 523   }
 524
 525   getFileStaticPath (video: MVideo) {
 526     if (this.isHLS()) {
 527       if (isVideoInPrivateDirectory(video.privacy)) {
 528         return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
 529       }
 530
 531       return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
 532     }
 533
 534     if (isVideoInPrivateDirectory(video.privacy)) {
 535       return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename)
 536     }
 537
 538     return join(STATIC_PATHS.WEBSEED, this.filename)
 539   }
 540
 541   getFileDownloadUrl (video: MVideoWithHost) {
 542     const path = this.isHLS()
 543       ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
 544       : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
 545
 546     if (video.isOwned()) return WEBSERVER.URL + path
 547
 548     // FIXME: don't guess remote URL
 549     return buildRemoteVideoBaseUrl(video, path)
 550   }
 551
 552   getRemoteTorrentUrl (video: MVideo) {
 553     if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
 554
 555     return this.torrentUrl
 556   }
 557
 558   // We proxify torrent requests so use a local URL
 559   getTorrentUrl () {
 560     if (!this.torrentFilename) return null
 561
 562     return WEBSERVER.URL + this.getTorrentStaticPath()
 563   }
 564
 565   getTorrentStaticPath () {
 566     if (!this.torrentFilename) return null
 567
 568     return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
 569   }
 570
 571   getTorrentDownloadUrl () {
 572     if (!this.torrentFilename) return null
 573
 574     return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
 575   }
 576
 577   removeTorrent () {
 578     if (!this.torrentFilename) return null
 579
 580     const torrentPath = getFSTorrentFilePath(this)
 581     return remove(torrentPath)
 582       .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
 583   }
 584
 585   hasSameUniqueKeysThan (other: MVideoFile) {
 586     return this.fps === other.fps &&
 587       this.resolution === other.resolution &&
 588       (
 589         (this.videoId !== null && this.videoId === other.videoId) ||
 590         (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
 591       )
 592   }
 593
 594   withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
 595     if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist })
 596
 597     return Object.assign(this, { Video: videoOrPlaylist })
 598   }
 599 }