server/models/video/video-file.ts

   1 import { remove } from 'fs-extra'
   2 import memoizee from 'memoizee'
   3 import { join } from 'path'
   4 import { FindOptions, Op, Transaction } from 'sequelize'
   5 import {
   6   AllowNull,
   7   BelongsTo,
   8   Column,
   9   CreatedAt,
  10   DataType,
  11   Default,
  12   DefaultScope,
  13   ForeignKey,
  14   HasMany,
  15   Is,
  16   Model,
  17   Scopes,
  18   Table,
  19   UpdatedAt
  20 } from 'sequelize-typescript'
  21 import { Where } from 'sequelize/types/lib/utils'
  22 import validator from 'validator'
  23 import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
  24 import { logger } from '@server/helpers/logger'
  25 import { extractVideo } from '@server/helpers/video'
  26 import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
  27 import { getFSTorrentFilePath } from '@server/lib/paths'
  28 import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
  29 import { AttributesOnly } from '@shared/core-utils'
  30 import { VideoStorage } from '@shared/models'
  31 import {
  32   isVideoFileExtnameValid,
  33   isVideoFileInfoHashValid,
  34   isVideoFileResolutionValid,
  35   isVideoFileSizeValid,
  36   isVideoFPSResolutionValid
  37 } from '../../helpers/custom-validators/videos'
  38 import {
  39   LAZY_STATIC_PATHS,
  40   MEMOIZE_LENGTH,
  41   MEMOIZE_TTL,
  42   MIMETYPES,
  43   STATIC_DOWNLOAD_PATHS,
  44   STATIC_PATHS,
  45   WEBSERVER
  46 } from '../../initializers/constants'
  47 import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file'
  48 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
  49 import { doesExist } from '../shared'
  50 import { parseAggregateResult, throwIfNotValid } from '../utils'
  51 import { VideoModel } from './video'
  52 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
  53
  54 export enum ScopeNames {
  55   WITH_VIDEO = 'WITH_VIDEO',
  56   WITH_METADATA = 'WITH_METADATA',
  57   WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
  58 }
  59
  60 @DefaultScope(() => ({
  61   attributes: {
  62     exclude: [ 'metadata' ]
  63   }
  64 }))
  65 @Scopes(() => ({
  66   [ScopeNames.WITH_VIDEO]: {
  67     include: [
  68       {
  69         model: VideoModel.unscoped(),
  70         required: true
  71       }
  72     ]
  73   },
  74   [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: Where } = {}) => {
  75     return {
  76       include: [
  77         {
  78           model: VideoModel.unscoped(),
  79           required: false,
  80           where: options.whereVideo
  81         },
  82         {
  83           model: VideoStreamingPlaylistModel.unscoped(),
  84           required: false,
  85           include: [
  86             {
  87               model: VideoModel.unscoped(),
  88               required: true,
  89               where: options.whereVideo
  90             }
  91           ]
  92         }
  93       ]
  94     }
  95   },
  96   [ScopeNames.WITH_METADATA]: {
  97     attributes: {
  98       include: [ 'metadata' ]
  99     }
 100   }
 101 }))
 102 @Table({
 103   tableName: 'videoFile',
 104   indexes: [
 105     {
 106       fields: [ 'videoId' ],
 107       where: {
 108         videoId: {
 109           [Op.ne]: null
 110         }
 111       }
 112     },
 113     {
 114       fields: [ 'videoStreamingPlaylistId' ],
 115       where: {
 116         videoStreamingPlaylistId: {
 117           [Op.ne]: null
 118         }
 119       }
 120     },
 121
 122     {
 123       fields: [ 'infoHash' ]
 124     },
 125
 126     {
 127       fields: [ 'torrentFilename' ],
 128       unique: true
 129     },
 130
 131     {
 132       fields: [ 'filename' ],
 133       unique: true
 134     },
 135
 136     {
 137       fields: [ 'videoId', 'resolution', 'fps' ],
 138       unique: true,
 139       where: {
 140         videoId: {
 141           [Op.ne]: null
 142         }
 143       }
 144     },
 145     {
 146       fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
 147       unique: true,
 148       where: {
 149         videoStreamingPlaylistId: {
 150           [Op.ne]: null
 151         }
 152       }
 153     }
 154   ]
 155 })
 156 export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>>> {
 157   @CreatedAt
 158   createdAt: Date
 159
 160   @UpdatedAt
 161   updatedAt: Date
 162
 163   @AllowNull(false)
 164   @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution'))
 165   @Column
 166   resolution: number
 167
 168   @AllowNull(false)
 169   @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size'))
 170   @Column(DataType.BIGINT)
 171   size: number
 172
 173   @AllowNull(false)
 174   @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
 175   @Column
 176   extname: string
 177
 178   @AllowNull(true)
 179   @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
 180   @Column
 181   infoHash: string
 182
 183   @AllowNull(false)
 184   @Default(-1)
 185   @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
 186   @Column
 187   fps: number
 188
 189   @AllowNull(true)
 190   @Column(DataType.JSONB)
 191   metadata: any
 192
 193   @AllowNull(true)
 194   @Column
 195   metadataUrl: string
 196
 197   @AllowNull(true)
 198   @Column
 199   fileUrl: string
 200
 201   // Could be null for live files
 202   @AllowNull(true)
 203   @Column
 204   filename: string
 205
 206   @AllowNull(true)
 207   @Column
 208   torrentUrl: string
 209
 210   // Could be null for live files
 211   @AllowNull(true)
 212   @Column
 213   torrentFilename: string
 214
 215   @ForeignKey(() => VideoModel)
 216   @Column
 217   videoId: number
 218
 219   @AllowNull(false)
 220   @Default(VideoStorage.FILE_SYSTEM)
 221   @Column
 222   storage: VideoStorage
 223
 224   @BelongsTo(() => VideoModel, {
 225     foreignKey: {
 226       allowNull: true
 227     },
 228     onDelete: 'CASCADE'
 229   })
 230   Video: VideoModel
 231
 232   @ForeignKey(() => VideoStreamingPlaylistModel)
 233   @Column
 234   videoStreamingPlaylistId: number
 235
 236   @BelongsTo(() => VideoStreamingPlaylistModel, {
 237     foreignKey: {
 238       allowNull: true
 239     },
 240     onDelete: 'CASCADE'
 241   })
 242   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 243
 244   @HasMany(() => VideoRedundancyModel, {
 245     foreignKey: {
 246       allowNull: true
 247     },
 248     onDelete: 'CASCADE',
 249     hooks: true
 250   })
 251   RedundancyVideos: VideoRedundancyModel[]
 252
 253   static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, {
 254     promise: true,
 255     max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
 256     maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
 257   })
 258
 259   static doesInfohashExist (infoHash: string) {
 260     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
 261
 262     return doesExist(query, { infoHash })
 263   }
 264
 265   static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
 266     const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
 267
 268     return !!videoFile
 269   }
 270
 271   static async doesOwnedTorrentFileExist (filename: string) {
 272     const query = 'SELECT 1 FROM "videoFile" ' +
 273                   'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
 274                   'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
 275                   'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
 276                   'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
 277
 278     return doesExist(query, { filename })
 279   }
 280
 281   static async doesOwnedWebTorrentVideoFileExist (filename: string) {
 282     const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
 283                   `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1`
 284
 285     return doesExist(query, { filename })
 286   }
 287
 288   static loadByFilename (filename: string) {
 289     const query = {
 290       where: {
 291         filename
 292       }
 293     }
 294
 295     return VideoFileModel.findOne(query)
 296   }
 297
 298   static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
 299     const query = {
 300       where: {
 301         torrentFilename: filename
 302       }
 303     }
 304
 305     return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
 306   }
 307
 308   static loadWithMetadata (id: number) {
 309     return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
 310   }
 311
 312   static loadWithVideo (id: number) {
 313     return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
 314   }
 315
 316   static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
 317     const whereVideo = validator.isUUID(videoIdOrUUID + '')
 318       ? { uuid: videoIdOrUUID }
 319       : { id: videoIdOrUUID }
 320
 321     const options = {
 322       where: {
 323         id
 324       }
 325     }
 326
 327     return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
 328       .findOne(options)
 329       .then(file => {
 330         // We used `required: false` so check we have at least a video or a streaming playlist
 331         if (!file.Video && !file.VideoStreamingPlaylist) return null
 332
 333         return file
 334       })
 335   }
 336
 337   static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
 338     const query = {
 339       include: [
 340         {
 341           model: VideoModel.unscoped(),
 342           required: true,
 343           include: [
 344             {
 345               model: VideoStreamingPlaylistModel.unscoped(),
 346               required: true,
 347               where: {
 348                 id: streamingPlaylistId
 349               }
 350             }
 351           ]
 352         }
 353       ],
 354       transaction
 355     }
 356
 357     return VideoFileModel.findAll(query)
 358   }
 359
 360   static getStats () {
 361     const webtorrentFilesQuery: FindOptions = {
 362       include: [
 363         {
 364           attributes: [],
 365           required: true,
 366           model: VideoModel.unscoped(),
 367           where: {
 368             remote: false
 369           }
 370         }
 371       ]
 372     }
 373
 374     const hlsFilesQuery: FindOptions = {
 375       include: [
 376         {
 377           attributes: [],
 378           required: true,
 379           model: VideoStreamingPlaylistModel.unscoped(),
 380           include: [
 381             {
 382               attributes: [],
 383               model: VideoModel.unscoped(),
 384               required: true,
 385               where: {
 386                 remote: false
 387               }
 388             }
 389           ]
 390         }
 391       ]
 392     }
 393
 394     return Promise.all([
 395       VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery),
 396       VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
 397     ]).then(([ webtorrentResult, hlsResult ]) => ({
 398       totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult)
 399     }))
 400   }
 401
 402   // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
 403   static async customUpsert (
 404     videoFile: MVideoFile,
 405     mode: 'streaming-playlist' | 'video',
 406     transaction: Transaction
 407   ) {
 408     const baseWhere = {
 409       fps: videoFile.fps,
 410       resolution: videoFile.resolution
 411     }
 412
 413     if (mode === 'streaming-playlist') Object.assign(baseWhere, { videoStreamingPlaylistId: videoFile.videoStreamingPlaylistId })
 414     else Object.assign(baseWhere, { videoId: videoFile.videoId })
 415
 416     const element = await VideoFileModel.findOne({ where: baseWhere, transaction })
 417     if (!element) return videoFile.save({ transaction })
 418
 419     for (const k of Object.keys(videoFile.toJSON())) {
 420       element[k] = videoFile[k]
 421     }
 422
 423     return element.save({ transaction })
 424   }
 425
 426   static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
 427     const options = {
 428       where: { videoStreamingPlaylistId }
 429     }
 430
 431     return VideoFileModel.destroy(options)
 432   }
 433
 434   hasTorrent () {
 435     return this.infoHash && this.torrentFilename
 436   }
 437
 438   getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
 439     if (this.videoId) return (this as MVideoFileVideo).Video
 440
 441     return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
 442   }
 443
 444   getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
 445     return extractVideo(this.getVideoOrStreamingPlaylist())
 446   }
 447
 448   isAudio () {
 449     return !!MIMETYPES.AUDIO.EXT_MIMETYPE[this.extname]
 450   }
 451
 452   isLive () {
 453     return this.size === -1
 454   }
 455
 456   isHLS () {
 457     return !!this.videoStreamingPlaylistId
 458   }
 459
 460   getObjectStorageUrl () {
 461     if (this.isHLS()) {
 462       return getHLSPublicFileUrl(this.fileUrl)
 463     }
 464
 465     return getWebTorrentPublicFileUrl(this.fileUrl)
 466   }
 467
 468   getFileUrl (video: MVideo) {
 469     if (this.storage === VideoStorage.OBJECT_STORAGE) {
 470       return this.getObjectStorageUrl()
 471     }
 472
 473     if (!this.Video) this.Video = video as VideoModel
 474     if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
 475
 476     return this.fileUrl
 477   }
 478
 479   getFileStaticPath (video: MVideo) {
 480     if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
 481
 482     return join(STATIC_PATHS.WEBSEED, this.filename)
 483   }
 484
 485   getFileDownloadUrl (video: MVideoWithHost) {
 486     const path = this.isHLS()
 487       ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
 488       : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
 489
 490     if (video.isOwned()) return WEBSERVER.URL + path
 491
 492     // FIXME: don't guess remote URL
 493     return buildRemoteVideoBaseUrl(video, path)
 494   }
 495
 496   getRemoteTorrentUrl (video: MVideo) {
 497     if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
 498
 499     return this.torrentUrl
 500   }
 501
 502   // We proxify torrent requests so use a local URL
 503   getTorrentUrl () {
 504     if (!this.torrentFilename) return null
 505
 506     return WEBSERVER.URL + this.getTorrentStaticPath()
 507   }
 508
 509   getTorrentStaticPath () {
 510     if (!this.torrentFilename) return null
 511
 512     return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
 513   }
 514
 515   getTorrentDownloadUrl () {
 516     if (!this.torrentFilename) return null
 517
 518     return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
 519   }
 520
 521   removeTorrent () {
 522     if (!this.torrentFilename) return null
 523
 524     const torrentPath = getFSTorrentFilePath(this)
 525     return remove(torrentPath)
 526       .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
 527   }
 528
 529   hasSameUniqueKeysThan (other: MVideoFile) {
 530     return this.fps === other.fps &&
 531       this.resolution === other.resolution &&
 532       (
 533         (this.videoId !== null && this.videoId === other.videoId) ||
 534         (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
 535       )
 536   }
 537 }