server/models/video/video-file.ts

   1 import { remove } from 'fs-extra'
   2 import * as memoizee from 'memoizee'
   3 import { join } from 'path'
   4 import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize'
   5 import {
   6   AllowNull,
   7   BelongsTo,
   8   Column,
   9   CreatedAt,
  10   DataType,
  11   Default,
  12   DefaultScope,
  13   ForeignKey,
  14   HasMany,
  15   Is,
  16   Model,
  17   Scopes,
  18   Table,
  19   UpdatedAt
  20 } from 'sequelize-typescript'
  21 import { Where } from 'sequelize/types/lib/utils'
  22 import validator from 'validator'
  23 import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
  24 import { logger } from '@server/helpers/logger'
  25 import { extractVideo } from '@server/helpers/video'
  26 import { getTorrentFilePath } from '@server/lib/video-paths'
  27 import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
  28 import {
  29   isVideoFileExtnameValid,
  30   isVideoFileInfoHashValid,
  31   isVideoFileResolutionValid,
  32   isVideoFileSizeValid,
  33   isVideoFPSResolutionValid
  34 } from '../../helpers/custom-validators/videos'
  35 import {
  36   LAZY_STATIC_PATHS,
  37   MEMOIZE_LENGTH,
  38   MEMOIZE_TTL,
  39   MIMETYPES,
  40   STATIC_DOWNLOAD_PATHS,
  41   STATIC_PATHS,
  42   WEBSERVER
  43 } from '../../initializers/constants'
  44 import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file'
  45 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
  46 import { parseAggregateResult, throwIfNotValid } from '../utils'
  47 import { VideoModel } from './video'
  48 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
  49
  50 export enum ScopeNames {
  51   WITH_VIDEO = 'WITH_VIDEO',
  52   WITH_METADATA = 'WITH_METADATA',
  53   WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
  54 }
  55
  56 @DefaultScope(() => ({
  57   attributes: {
  58     exclude: [ 'metadata' ]
  59   }
  60 }))
  61 @Scopes(() => ({
  62   [ScopeNames.WITH_VIDEO]: {
  63     include: [
  64       {
  65         model: VideoModel.unscoped(),
  66         required: true
  67       }
  68     ]
  69   },
  70   [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: Where } = {}) => {
  71     return {
  72       include: [
  73         {
  74           model: VideoModel.unscoped(),
  75           required: false,
  76           where: options.whereVideo
  77         },
  78         {
  79           model: VideoStreamingPlaylistModel.unscoped(),
  80           required: false,
  81           include: [
  82             {
  83               model: VideoModel.unscoped(),
  84               required: true,
  85               where: options.whereVideo
  86             }
  87           ]
  88         }
  89       ]
  90     }
  91   },
  92   [ScopeNames.WITH_METADATA]: {
  93     attributes: {
  94       include: [ 'metadata' ]
  95     }
  96   }
  97 }))
  98 @Table({
  99   tableName: 'videoFile',
 100   indexes: [
 101     {
 102       fields: [ 'videoId' ],
 103       where: {
 104         videoId: {
 105           [Op.ne]: null
 106         }
 107       }
 108     },
 109     {
 110       fields: [ 'videoStreamingPlaylistId' ],
 111       where: {
 112         videoStreamingPlaylistId: {
 113           [Op.ne]: null
 114         }
 115       }
 116     },
 117
 118     {
 119       fields: [ 'infoHash' ]
 120     },
 121
 122     {
 123       fields: [ 'torrentFilename' ],
 124       unique: true
 125     },
 126
 127     {
 128       fields: [ 'filename' ],
 129       unique: true
 130     },
 131
 132     {
 133       fields: [ 'videoId', 'resolution', 'fps' ],
 134       unique: true,
 135       where: {
 136         videoId: {
 137           [Op.ne]: null
 138         }
 139       }
 140     },
 141     {
 142       fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
 143       unique: true,
 144       where: {
 145         videoStreamingPlaylistId: {
 146           [Op.ne]: null
 147         }
 148       }
 149     }
 150   ]
 151 })
 152 export class VideoFileModel extends Model {
 153   @CreatedAt
 154   createdAt: Date
 155
 156   @UpdatedAt
 157   updatedAt: Date
 158
 159   @AllowNull(false)
 160   @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution'))
 161   @Column
 162   resolution: number
 163
 164   @AllowNull(false)
 165   @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size'))
 166   @Column(DataType.BIGINT)
 167   size: number
 168
 169   @AllowNull(false)
 170   @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
 171   @Column
 172   extname: string
 173
 174   @AllowNull(true)
 175   @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
 176   @Column
 177   infoHash: string
 178
 179   @AllowNull(false)
 180   @Default(-1)
 181   @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
 182   @Column
 183   fps: number
 184
 185   @AllowNull(true)
 186   @Column(DataType.JSONB)
 187   metadata: any
 188
 189   @AllowNull(true)
 190   @Column
 191   metadataUrl: string
 192
 193   @AllowNull(true)
 194   @Column
 195   fileUrl: string
 196
 197   // Could be null for live files
 198   @AllowNull(true)
 199   @Column
 200   filename: string
 201
 202   @AllowNull(true)
 203   @Column
 204   torrentUrl: string
 205
 206   // Could be null for live files
 207   @AllowNull(true)
 208   @Column
 209   torrentFilename: string
 210
 211   @ForeignKey(() => VideoModel)
 212   @Column
 213   videoId: number
 214
 215   @BelongsTo(() => VideoModel, {
 216     foreignKey: {
 217       allowNull: true
 218     },
 219     onDelete: 'CASCADE'
 220   })
 221   Video: VideoModel
 222
 223   @ForeignKey(() => VideoStreamingPlaylistModel)
 224   @Column
 225   videoStreamingPlaylistId: number
 226
 227   @BelongsTo(() => VideoStreamingPlaylistModel, {
 228     foreignKey: {
 229       allowNull: true
 230     },
 231     onDelete: 'CASCADE'
 232   })
 233   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 234
 235   @HasMany(() => VideoRedundancyModel, {
 236     foreignKey: {
 237       allowNull: true
 238     },
 239     onDelete: 'CASCADE',
 240     hooks: true
 241   })
 242   RedundancyVideos: VideoRedundancyModel[]
 243
 244   static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, {
 245     promise: true,
 246     max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
 247     maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
 248   })
 249
 250   static doesInfohashExist (infoHash: string) {
 251     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
 252     const options = {
 253       type: QueryTypes.SELECT as QueryTypes.SELECT,
 254       bind: { infoHash },
 255       raw: true
 256     }
 257
 258     return VideoModel.sequelize.query(query, options)
 259               .then(results => results.length === 1)
 260   }
 261
 262   static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
 263     const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
 264
 265     return !!videoFile
 266   }
 267
 268   static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
 269     const query = {
 270       where: {
 271         torrentFilename: filename
 272       }
 273     }
 274
 275     return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
 276   }
 277
 278   static loadWithMetadata (id: number) {
 279     return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
 280   }
 281
 282   static loadWithVideo (id: number) {
 283     return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
 284   }
 285
 286   static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
 287     const whereVideo = validator.isUUID(videoIdOrUUID + '')
 288       ? { uuid: videoIdOrUUID }
 289       : { id: videoIdOrUUID }
 290
 291     const options = {
 292       where: {
 293         id
 294       }
 295     }
 296
 297     return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
 298       .findOne(options)
 299       .then(file => {
 300         // We used `required: false` so check we have at least a video or a streaming playlist
 301         if (!file.Video && !file.VideoStreamingPlaylist) return null
 302
 303         return file
 304       })
 305   }
 306
 307   static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
 308     const query = {
 309       include: [
 310         {
 311           model: VideoModel.unscoped(),
 312           required: true,
 313           include: [
 314             {
 315               model: VideoStreamingPlaylistModel.unscoped(),
 316               required: true,
 317               where: {
 318                 id: streamingPlaylistId
 319               }
 320             }
 321           ]
 322         }
 323       ],
 324       transaction
 325     }
 326
 327     return VideoFileModel.findAll(query)
 328   }
 329
 330   static getStats () {
 331     const webtorrentFilesQuery: FindOptions = {
 332       include: [
 333         {
 334           attributes: [],
 335           required: true,
 336           model: VideoModel.unscoped(),
 337           where: {
 338             remote: false
 339           }
 340         }
 341       ]
 342     }
 343
 344     const hlsFilesQuery: FindOptions = {
 345       include: [
 346         {
 347           attributes: [],
 348           required: true,
 349           model: VideoStreamingPlaylistModel.unscoped(),
 350           include: [
 351             {
 352               attributes: [],
 353               model: VideoModel.unscoped(),
 354               required: true,
 355               where: {
 356                 remote: false
 357               }
 358             }
 359           ]
 360         }
 361       ]
 362     }
 363
 364     return Promise.all([
 365       VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery),
 366       VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
 367     ]).then(([ webtorrentResult, hlsResult ]) => ({
 368       totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult)
 369     }))
 370   }
 371
 372   // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
 373   static async customUpsert (
 374     videoFile: MVideoFile,
 375     mode: 'streaming-playlist' | 'video',
 376     transaction: Transaction
 377   ) {
 378     const baseWhere = {
 379       fps: videoFile.fps,
 380       resolution: videoFile.resolution
 381     }
 382
 383     if (mode === 'streaming-playlist') Object.assign(baseWhere, { videoStreamingPlaylistId: videoFile.videoStreamingPlaylistId })
 384     else Object.assign(baseWhere, { videoId: videoFile.videoId })
 385
 386     const element = await VideoFileModel.findOne({ where: baseWhere, transaction })
 387     if (!element) return videoFile.save({ transaction })
 388
 389     for (const k of Object.keys(videoFile.toJSON())) {
 390       element[k] = videoFile[k]
 391     }
 392
 393     return element.save({ transaction })
 394   }
 395
 396   static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
 397     const options = {
 398       where: { videoStreamingPlaylistId }
 399     }
 400
 401     return VideoFileModel.destroy(options)
 402   }
 403
 404   hasTorrent () {
 405     return this.infoHash && this.torrentFilename
 406   }
 407
 408   getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
 409     if (this.videoId) return (this as MVideoFileVideo).Video
 410
 411     return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
 412   }
 413
 414   getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
 415     return extractVideo(this.getVideoOrStreamingPlaylist())
 416   }
 417
 418   isAudio () {
 419     return !!MIMETYPES.AUDIO.EXT_MIMETYPE[this.extname]
 420   }
 421
 422   isLive () {
 423     return this.size === -1
 424   }
 425
 426   isHLS () {
 427     return !!this.videoStreamingPlaylistId
 428   }
 429
 430   getFileUrl (video: MVideo) {
 431     if (!this.Video) this.Video = video as VideoModel
 432
 433     if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
 434
 435     return this.fileUrl
 436   }
 437
 438   getFileStaticPath (video: MVideo) {
 439     if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
 440
 441     return join(STATIC_PATHS.WEBSEED, this.filename)
 442   }
 443
 444   getFileDownloadUrl (video: MVideoWithHost) {
 445     const basePath = this.isHLS()
 446       ? STATIC_DOWNLOAD_PATHS.HLS_VIDEOS
 447       : STATIC_DOWNLOAD_PATHS.VIDEOS
 448     const path = join(basePath, this.filename)
 449
 450     if (video.isOwned()) return WEBSERVER.URL + path
 451
 452     // FIXME: don't guess remote URL
 453     return buildRemoteVideoBaseUrl(video, path)
 454   }
 455
 456   getRemoteTorrentUrl (video: MVideo) {
 457     if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
 458
 459     return this.torrentUrl
 460   }
 461
 462   // We proxify torrent requests so use a local URL
 463   getTorrentUrl () {
 464     if (!this.torrentFilename) return null
 465
 466     return WEBSERVER.URL + this.getTorrentStaticPath()
 467   }
 468
 469   getTorrentStaticPath () {
 470     if (!this.torrentFilename) return null
 471
 472     return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
 473   }
 474
 475   getTorrentDownloadUrl () {
 476     if (!this.torrentFilename) return null
 477
 478     return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
 479   }
 480
 481   removeTorrent () {
 482     if (!this.torrentFilename) return null
 483
 484     const torrentPath = getTorrentFilePath(this)
 485     return remove(torrentPath)
 486       .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
 487   }
 488
 489   hasSameUniqueKeysThan (other: MVideoFile) {
 490     return this.fps === other.fps &&
 491       this.resolution === other.resolution &&
 492       (
 493         (this.videoId !== null && this.videoId === other.videoId) ||
 494         (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
 495       )
 496   }
 497 }