server/models/video/video-file.ts

   1 import { remove } from 'fs-extra'
   2 import * as memoizee from 'memoizee'
   3 import { join } from 'path'
   4 import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize'
   5 import {
   6   AllowNull,
   7   BelongsTo,
   8   Column,
   9   CreatedAt,
  10   DataType,
  11   Default,
  12   DefaultScope,
  13   ForeignKey,
  14   HasMany,
  15   Is,
  16   Model,
  17   Scopes,
  18   Table,
  19   UpdatedAt
  20 } from 'sequelize-typescript'
  21 import { Where } from 'sequelize/types/lib/utils'
  22 import validator from 'validator'
  23 import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
  24 import { logger } from '@server/helpers/logger'
  25 import { extractVideo } from '@server/helpers/video'
  26 import { getTorrentFilePath } from '@server/lib/video-paths'
  27 import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
  28 import { AttributesOnly } from '@shared/core-utils'
  29 import {
  30   isVideoFileExtnameValid,
  31   isVideoFileInfoHashValid,
  32   isVideoFileResolutionValid,
  33   isVideoFileSizeValid,
  34   isVideoFPSResolutionValid
  35 } from '../../helpers/custom-validators/videos'
  36 import {
  37   LAZY_STATIC_PATHS,
  38   MEMOIZE_LENGTH,
  39   MEMOIZE_TTL,
  40   MIMETYPES,
  41   STATIC_DOWNLOAD_PATHS,
  42   STATIC_PATHS,
  43   WEBSERVER
  44 } from '../../initializers/constants'
  45 import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file'
  46 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
  47 import { parseAggregateResult, throwIfNotValid } from '../utils'
  48 import { VideoModel } from './video'
  49 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
  50
  51 export enum ScopeNames {
  52   WITH_VIDEO = 'WITH_VIDEO',
  53   WITH_METADATA = 'WITH_METADATA',
  54   WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
  55 }
  56
  57 @DefaultScope(() => ({
  58   attributes: {
  59     exclude: [ 'metadata' ]
  60   }
  61 }))
  62 @Scopes(() => ({
  63   [ScopeNames.WITH_VIDEO]: {
  64     include: [
  65       {
  66         model: VideoModel.unscoped(),
  67         required: true
  68       }
  69     ]
  70   },
  71   [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: Where } = {}) => {
  72     return {
  73       include: [
  74         {
  75           model: VideoModel.unscoped(),
  76           required: false,
  77           where: options.whereVideo
  78         },
  79         {
  80           model: VideoStreamingPlaylistModel.unscoped(),
  81           required: false,
  82           include: [
  83             {
  84               model: VideoModel.unscoped(),
  85               required: true,
  86               where: options.whereVideo
  87             }
  88           ]
  89         }
  90       ]
  91     }
  92   },
  93   [ScopeNames.WITH_METADATA]: {
  94     attributes: {
  95       include: [ 'metadata' ]
  96     }
  97   }
  98 }))
  99 @Table({
 100   tableName: 'videoFile',
 101   indexes: [
 102     {
 103       fields: [ 'videoId' ],
 104       where: {
 105         videoId: {
 106           [Op.ne]: null
 107         }
 108       }
 109     },
 110     {
 111       fields: [ 'videoStreamingPlaylistId' ],
 112       where: {
 113         videoStreamingPlaylistId: {
 114           [Op.ne]: null
 115         }
 116       }
 117     },
 118
 119     {
 120       fields: [ 'infoHash' ]
 121     },
 122
 123     {
 124       fields: [ 'torrentFilename' ],
 125       unique: true
 126     },
 127
 128     {
 129       fields: [ 'filename' ],
 130       unique: true
 131     },
 132
 133     {
 134       fields: [ 'videoId', 'resolution', 'fps' ],
 135       unique: true,
 136       where: {
 137         videoId: {
 138           [Op.ne]: null
 139         }
 140       }
 141     },
 142     {
 143       fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
 144       unique: true,
 145       where: {
 146         videoStreamingPlaylistId: {
 147           [Op.ne]: null
 148         }
 149       }
 150     }
 151   ]
 152 })
 153 export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>>> {
 154   @CreatedAt
 155   createdAt: Date
 156
 157   @UpdatedAt
 158   updatedAt: Date
 159
 160   @AllowNull(false)
 161   @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution'))
 162   @Column
 163   resolution: number
 164
 165   @AllowNull(false)
 166   @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size'))
 167   @Column(DataType.BIGINT)
 168   size: number
 169
 170   @AllowNull(false)
 171   @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
 172   @Column
 173   extname: string
 174
 175   @AllowNull(true)
 176   @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
 177   @Column
 178   infoHash: string
 179
 180   @AllowNull(false)
 181   @Default(-1)
 182   @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
 183   @Column
 184   fps: number
 185
 186   @AllowNull(true)
 187   @Column(DataType.JSONB)
 188   metadata: any
 189
 190   @AllowNull(true)
 191   @Column
 192   metadataUrl: string
 193
 194   @AllowNull(true)
 195   @Column
 196   fileUrl: string
 197
 198   // Could be null for live files
 199   @AllowNull(true)
 200   @Column
 201   filename: string
 202
 203   @AllowNull(true)
 204   @Column
 205   torrentUrl: string
 206
 207   // Could be null for live files
 208   @AllowNull(true)
 209   @Column
 210   torrentFilename: string
 211
 212   @ForeignKey(() => VideoModel)
 213   @Column
 214   videoId: number
 215
 216   @BelongsTo(() => VideoModel, {
 217     foreignKey: {
 218       allowNull: true
 219     },
 220     onDelete: 'CASCADE'
 221   })
 222   Video: VideoModel
 223
 224   @ForeignKey(() => VideoStreamingPlaylistModel)
 225   @Column
 226   videoStreamingPlaylistId: number
 227
 228   @BelongsTo(() => VideoStreamingPlaylistModel, {
 229     foreignKey: {
 230       allowNull: true
 231     },
 232     onDelete: 'CASCADE'
 233   })
 234   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 235
 236   @HasMany(() => VideoRedundancyModel, {
 237     foreignKey: {
 238       allowNull: true
 239     },
 240     onDelete: 'CASCADE',
 241     hooks: true
 242   })
 243   RedundancyVideos: VideoRedundancyModel[]
 244
 245   static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, {
 246     promise: true,
 247     max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
 248     maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
 249   })
 250
 251   static doesInfohashExist (infoHash: string) {
 252     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
 253     const options = {
 254       type: QueryTypes.SELECT as QueryTypes.SELECT,
 255       bind: { infoHash },
 256       raw: true
 257     }
 258
 259     return VideoModel.sequelize.query(query, options)
 260               .then(results => results.length === 1)
 261   }
 262
 263   static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
 264     const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
 265
 266     return !!videoFile
 267   }
 268
 269   static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
 270     const query = {
 271       where: {
 272         torrentFilename: filename
 273       }
 274     }
 275
 276     return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
 277   }
 278
 279   static loadWithMetadata (id: number) {
 280     return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
 281   }
 282
 283   static loadWithVideo (id: number) {
 284     return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
 285   }
 286
 287   static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
 288     const whereVideo = validator.isUUID(videoIdOrUUID + '')
 289       ? { uuid: videoIdOrUUID }
 290       : { id: videoIdOrUUID }
 291
 292     const options = {
 293       where: {
 294         id
 295       }
 296     }
 297
 298     return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
 299       .findOne(options)
 300       .then(file => {
 301         // We used `required: false` so check we have at least a video or a streaming playlist
 302         if (!file.Video && !file.VideoStreamingPlaylist) return null
 303
 304         return file
 305       })
 306   }
 307
 308   static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
 309     const query = {
 310       include: [
 311         {
 312           model: VideoModel.unscoped(),
 313           required: true,
 314           include: [
 315             {
 316               model: VideoStreamingPlaylistModel.unscoped(),
 317               required: true,
 318               where: {
 319                 id: streamingPlaylistId
 320               }
 321             }
 322           ]
 323         }
 324       ],
 325       transaction
 326     }
 327
 328     return VideoFileModel.findAll(query)
 329   }
 330
 331   static getStats () {
 332     const webtorrentFilesQuery: FindOptions = {
 333       include: [
 334         {
 335           attributes: [],
 336           required: true,
 337           model: VideoModel.unscoped(),
 338           where: {
 339             remote: false
 340           }
 341         }
 342       ]
 343     }
 344
 345     const hlsFilesQuery: FindOptions = {
 346       include: [
 347         {
 348           attributes: [],
 349           required: true,
 350           model: VideoStreamingPlaylistModel.unscoped(),
 351           include: [
 352             {
 353               attributes: [],
 354               model: VideoModel.unscoped(),
 355               required: true,
 356               where: {
 357                 remote: false
 358               }
 359             }
 360           ]
 361         }
 362       ]
 363     }
 364
 365     return Promise.all([
 366       VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery),
 367       VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
 368     ]).then(([ webtorrentResult, hlsResult ]) => ({
 369       totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult)
 370     }))
 371   }
 372
 373   // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
 374   static async customUpsert (
 375     videoFile: MVideoFile,
 376     mode: 'streaming-playlist' | 'video',
 377     transaction: Transaction
 378   ) {
 379     const baseWhere = {
 380       fps: videoFile.fps,
 381       resolution: videoFile.resolution
 382     }
 383
 384     if (mode === 'streaming-playlist') Object.assign(baseWhere, { videoStreamingPlaylistId: videoFile.videoStreamingPlaylistId })
 385     else Object.assign(baseWhere, { videoId: videoFile.videoId })
 386
 387     const element = await VideoFileModel.findOne({ where: baseWhere, transaction })
 388     if (!element) return videoFile.save({ transaction })
 389
 390     for (const k of Object.keys(videoFile.toJSON())) {
 391       element[k] = videoFile[k]
 392     }
 393
 394     return element.save({ transaction })
 395   }
 396
 397   static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
 398     const options = {
 399       where: { videoStreamingPlaylistId }
 400     }
 401
 402     return VideoFileModel.destroy(options)
 403   }
 404
 405   hasTorrent () {
 406     return this.infoHash && this.torrentFilename
 407   }
 408
 409   getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
 410     if (this.videoId) return (this as MVideoFileVideo).Video
 411
 412     return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
 413   }
 414
 415   getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
 416     return extractVideo(this.getVideoOrStreamingPlaylist())
 417   }
 418
 419   isAudio () {
 420     return !!MIMETYPES.AUDIO.EXT_MIMETYPE[this.extname]
 421   }
 422
 423   isLive () {
 424     return this.size === -1
 425   }
 426
 427   isHLS () {
 428     return !!this.videoStreamingPlaylistId
 429   }
 430
 431   getFileUrl (video: MVideo) {
 432     if (!this.Video) this.Video = video as VideoModel
 433
 434     if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
 435
 436     return this.fileUrl
 437   }
 438
 439   getFileStaticPath (video: MVideo) {
 440     if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
 441
 442     return join(STATIC_PATHS.WEBSEED, this.filename)
 443   }
 444
 445   getFileDownloadUrl (video: MVideoWithHost) {
 446     const basePath = this.isHLS()
 447       ? STATIC_DOWNLOAD_PATHS.HLS_VIDEOS
 448       : STATIC_DOWNLOAD_PATHS.VIDEOS
 449     const path = join(basePath, this.filename)
 450
 451     if (video.isOwned()) return WEBSERVER.URL + path
 452
 453     // FIXME: don't guess remote URL
 454     return buildRemoteVideoBaseUrl(video, path)
 455   }
 456
 457   getRemoteTorrentUrl (video: MVideo) {
 458     if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
 459
 460     return this.torrentUrl
 461   }
 462
 463   // We proxify torrent requests so use a local URL
 464   getTorrentUrl () {
 465     if (!this.torrentFilename) return null
 466
 467     return WEBSERVER.URL + this.getTorrentStaticPath()
 468   }
 469
 470   getTorrentStaticPath () {
 471     if (!this.torrentFilename) return null
 472
 473     return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
 474   }
 475
 476   getTorrentDownloadUrl () {
 477     if (!this.torrentFilename) return null
 478
 479     return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
 480   }
 481
 482   removeTorrent () {
 483     if (!this.torrentFilename) return null
 484
 485     const torrentPath = getTorrentFilePath(this)
 486     return remove(torrentPath)
 487       .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
 488   }
 489
 490   hasSameUniqueKeysThan (other: MVideoFile) {
 491     return this.fps === other.fps &&
 492       this.resolution === other.resolution &&
 493       (
 494         (this.videoId !== null && this.videoId === other.videoId) ||
 495         (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
 496       )
 497   }
 498 }