server/models/video/video-file.ts

   1 import { remove } from 'fs-extra'
   2 import memoizee from 'memoizee'
   3 import { join } from 'path'
   4 import { FindOptions, Op, Transaction, WhereOptions } from 'sequelize'
   5 import {
   6   AllowNull,
   7   BelongsTo,
   8   Column,
   9   CreatedAt,
  10   DataType,
  11   Default,
  12   DefaultScope,
  13   ForeignKey,
  14   HasMany,
  15   Is,
  16   Model,
  17   Scopes,
  18   Table,
  19   UpdatedAt
  20 } from 'sequelize-typescript'
  21 import validator from 'validator'
  22 import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
  23 import { logger } from '@server/helpers/logger'
  24 import { extractVideo } from '@server/helpers/video'
  25 import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
  26 import { getFSTorrentFilePath } from '@server/lib/paths'
  27 import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
  28 import { VideoResolution, VideoStorage } from '@shared/models'
  29 import { AttributesOnly } from '@shared/typescript-utils'
  30 import {
  31   isVideoFileExtnameValid,
  32   isVideoFileInfoHashValid,
  33   isVideoFileResolutionValid,
  34   isVideoFileSizeValid,
  35   isVideoFPSResolutionValid
  36 } from '../../helpers/custom-validators/videos'
  37 import {
  38   LAZY_STATIC_PATHS,
  39   MEMOIZE_LENGTH,
  40   MEMOIZE_TTL,
  41   STATIC_DOWNLOAD_PATHS,
  42   STATIC_PATHS,
  43   WEBSERVER
  44 } from '../../initializers/constants'
  45 import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file'
  46 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
  47 import { doesExist } from '../shared'
  48 import { parseAggregateResult, throwIfNotValid } from '../utils'
  49 import { VideoModel } from './video'
  50 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
  51
  52 export enum ScopeNames {
  53   WITH_VIDEO = 'WITH_VIDEO',
  54   WITH_METADATA = 'WITH_METADATA',
  55   WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
  56 }
  57
  58 @DefaultScope(() => ({
  59   attributes: {
  60     exclude: [ 'metadata' ]
  61   }
  62 }))
  63 @Scopes(() => ({
  64   [ScopeNames.WITH_VIDEO]: {
  65     include: [
  66       {
  67         model: VideoModel.unscoped(),
  68         required: true
  69       }
  70     ]
  71   },
  72   [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: WhereOptions } = {}) => {
  73     return {
  74       include: [
  75         {
  76           model: VideoModel.unscoped(),
  77           required: false,
  78           where: options.whereVideo
  79         },
  80         {
  81           model: VideoStreamingPlaylistModel.unscoped(),
  82           required: false,
  83           include: [
  84             {
  85               model: VideoModel.unscoped(),
  86               required: true,
  87               where: options.whereVideo
  88             }
  89           ]
  90         }
  91       ]
  92     }
  93   },
  94   [ScopeNames.WITH_METADATA]: {
  95     attributes: {
  96       include: [ 'metadata' ]
  97     }
  98   }
  99 }))
 100 @Table({
 101   tableName: 'videoFile',
 102   indexes: [
 103     {
 104       fields: [ 'videoId' ],
 105       where: {
 106         videoId: {
 107           [Op.ne]: null
 108         }
 109       }
 110     },
 111     {
 112       fields: [ 'videoStreamingPlaylistId' ],
 113       where: {
 114         videoStreamingPlaylistId: {
 115           [Op.ne]: null
 116         }
 117       }
 118     },
 119
 120     {
 121       fields: [ 'infoHash' ]
 122     },
 123
 124     {
 125       fields: [ 'torrentFilename' ],
 126       unique: true
 127     },
 128
 129     {
 130       fields: [ 'filename' ],
 131       unique: true
 132     },
 133
 134     {
 135       fields: [ 'videoId', 'resolution', 'fps' ],
 136       unique: true,
 137       where: {
 138         videoId: {
 139           [Op.ne]: null
 140         }
 141       }
 142     },
 143     {
 144       fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
 145       unique: true,
 146       where: {
 147         videoStreamingPlaylistId: {
 148           [Op.ne]: null
 149         }
 150       }
 151     }
 152   ]
 153 })
 154 export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>>> {
 155   @CreatedAt
 156   createdAt: Date
 157
 158   @UpdatedAt
 159   updatedAt: Date
 160
 161   @AllowNull(false)
 162   @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution'))
 163   @Column
 164   resolution: number
 165
 166   @AllowNull(false)
 167   @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size'))
 168   @Column(DataType.BIGINT)
 169   size: number
 170
 171   @AllowNull(false)
 172   @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
 173   @Column
 174   extname: string
 175
 176   @AllowNull(true)
 177   @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
 178   @Column
 179   infoHash: string
 180
 181   @AllowNull(false)
 182   @Default(-1)
 183   @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
 184   @Column
 185   fps: number
 186
 187   @AllowNull(true)
 188   @Column(DataType.JSONB)
 189   metadata: any
 190
 191   @AllowNull(true)
 192   @Column
 193   metadataUrl: string
 194
 195   // Could be null for remote files
 196   @AllowNull(true)
 197   @Column
 198   fileUrl: string
 199
 200   // Could be null for live files
 201   @AllowNull(true)
 202   @Column
 203   filename: string
 204
 205   // Could be null for remote files
 206   @AllowNull(true)
 207   @Column
 208   torrentUrl: string
 209
 210   // Could be null for live files
 211   @AllowNull(true)
 212   @Column
 213   torrentFilename: string
 214
 215   @ForeignKey(() => VideoModel)
 216   @Column
 217   videoId: number
 218
 219   @AllowNull(false)
 220   @Default(VideoStorage.FILE_SYSTEM)
 221   @Column
 222   storage: VideoStorage
 223
 224   @BelongsTo(() => VideoModel, {
 225     foreignKey: {
 226       allowNull: true
 227     },
 228     onDelete: 'CASCADE'
 229   })
 230   Video: VideoModel
 231
 232   @ForeignKey(() => VideoStreamingPlaylistModel)
 233   @Column
 234   videoStreamingPlaylistId: number
 235
 236   @BelongsTo(() => VideoStreamingPlaylistModel, {
 237     foreignKey: {
 238       allowNull: true
 239     },
 240     onDelete: 'CASCADE'
 241   })
 242   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 243
 244   @HasMany(() => VideoRedundancyModel, {
 245     foreignKey: {
 246       allowNull: true
 247     },
 248     onDelete: 'CASCADE',
 249     hooks: true
 250   })
 251   RedundancyVideos: VideoRedundancyModel[]
 252
 253   static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, {
 254     promise: true,
 255     max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
 256     maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
 257   })
 258
 259   static doesInfohashExist (infoHash: string) {
 260     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
 261
 262     return doesExist(query, { infoHash })
 263   }
 264
 265   static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
 266     const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
 267
 268     return !!videoFile
 269   }
 270
 271   static async doesOwnedTorrentFileExist (filename: string) {
 272     const query = 'SELECT 1 FROM "videoFile" ' +
 273                   'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
 274                   'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
 275                   'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
 276                   'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
 277
 278     return doesExist(query, { filename })
 279   }
 280
 281   static async doesOwnedWebTorrentVideoFileExist (filename: string) {
 282     const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
 283                   `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1`
 284
 285     return doesExist(query, { filename })
 286   }
 287
 288   static loadByFilename (filename: string) {
 289     const query = {
 290       where: {
 291         filename
 292       }
 293     }
 294
 295     return VideoFileModel.findOne(query)
 296   }
 297
 298   static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
 299     const query = {
 300       where: {
 301         torrentFilename: filename
 302       }
 303     }
 304
 305     return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
 306   }
 307
 308   static loadWithMetadata (id: number) {
 309     return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
 310   }
 311
 312   static loadWithVideo (id: number) {
 313     return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
 314   }
 315
 316   static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
 317     const whereVideo = validator.isUUID(videoIdOrUUID + '')
 318       ? { uuid: videoIdOrUUID }
 319       : { id: videoIdOrUUID }
 320
 321     const options = {
 322       where: {
 323         id
 324       }
 325     }
 326
 327     return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
 328       .findOne(options)
 329       .then(file => {
 330         // We used `required: false` so check we have at least a video or a streaming playlist
 331         if (!file.Video && !file.VideoStreamingPlaylist) return null
 332
 333         return file
 334       })
 335   }
 336
 337   static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
 338     const query = {
 339       include: [
 340         {
 341           model: VideoModel.unscoped(),
 342           required: true,
 343           include: [
 344             {
 345               model: VideoStreamingPlaylistModel.unscoped(),
 346               required: true,
 347               where: {
 348                 id: streamingPlaylistId
 349               }
 350             }
 351           ]
 352         }
 353       ],
 354       transaction
 355     }
 356
 357     return VideoFileModel.findAll(query)
 358   }
 359
 360   static getStats () {
 361     const webtorrentFilesQuery: FindOptions = {
 362       include: [
 363         {
 364           attributes: [],
 365           required: true,
 366           model: VideoModel.unscoped(),
 367           where: {
 368             remote: false
 369           }
 370         }
 371       ]
 372     }
 373
 374     const hlsFilesQuery: FindOptions = {
 375       include: [
 376         {
 377           attributes: [],
 378           required: true,
 379           model: VideoStreamingPlaylistModel.unscoped(),
 380           include: [
 381             {
 382               attributes: [],
 383               model: VideoModel.unscoped(),
 384               required: true,
 385               where: {
 386                 remote: false
 387               }
 388             }
 389           ]
 390         }
 391       ]
 392     }
 393
 394     return Promise.all([
 395       VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery),
 396       VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
 397     ]).then(([ webtorrentResult, hlsResult ]) => ({
 398       totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult)
 399     }))
 400   }
 401
 402   // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
 403   static async customUpsert (
 404     videoFile: MVideoFile,
 405     mode: 'streaming-playlist' | 'video',
 406     transaction: Transaction
 407   ) {
 408     const baseWhere = {
 409       fps: videoFile.fps,
 410       resolution: videoFile.resolution
 411     }
 412
 413     if (mode === 'streaming-playlist') Object.assign(baseWhere, { videoStreamingPlaylistId: videoFile.videoStreamingPlaylistId })
 414     else Object.assign(baseWhere, { videoId: videoFile.videoId })
 415
 416     const element = await VideoFileModel.findOne({ where: baseWhere, transaction })
 417     if (!element) return videoFile.save({ transaction })
 418
 419     for (const k of Object.keys(videoFile.toJSON())) {
 420       element[k] = videoFile[k]
 421     }
 422
 423     return element.save({ transaction })
 424   }
 425
 426   static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
 427     const options = {
 428       where: { videoStreamingPlaylistId }
 429     }
 430
 431     return VideoFileModel.destroy(options)
 432   }
 433
 434   hasTorrent () {
 435     return this.infoHash && this.torrentFilename
 436   }
 437
 438   getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
 439     if (this.videoId) return (this as MVideoFileVideo).Video
 440
 441     return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
 442   }
 443
 444   getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
 445     return extractVideo(this.getVideoOrStreamingPlaylist())
 446   }
 447
 448   isAudio () {
 449     return this.resolution === VideoResolution.H_NOVIDEO
 450   }
 451
 452   isLive () {
 453     return this.size === -1
 454   }
 455
 456   isHLS () {
 457     return !!this.videoStreamingPlaylistId
 458   }
 459
 460   getObjectStorageUrl () {
 461     if (this.isHLS()) {
 462       return getHLSPublicFileUrl(this.fileUrl)
 463     }
 464
 465     return getWebTorrentPublicFileUrl(this.fileUrl)
 466   }
 467
 468   getFileUrl (video: MVideo) {
 469     if (this.storage === VideoStorage.OBJECT_STORAGE) {
 470       return this.getObjectStorageUrl()
 471     }
 472
 473     if (!this.Video) this.Video = video as VideoModel
 474     if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
 475
 476     return this.fileUrl
 477   }
 478
 479   getFileStaticPath (video: MVideo) {
 480     if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
 481
 482     return join(STATIC_PATHS.WEBSEED, this.filename)
 483   }
 484
 485   getFileDownloadUrl (video: MVideoWithHost) {
 486     const path = this.isHLS()
 487       ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
 488       : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
 489
 490     if (video.isOwned()) return WEBSERVER.URL + path
 491
 492     // FIXME: don't guess remote URL
 493     return buildRemoteVideoBaseUrl(video, path)
 494   }
 495
 496   getRemoteTorrentUrl (video: MVideo) {
 497     if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
 498
 499     return this.torrentUrl
 500   }
 501
 502   // We proxify torrent requests so use a local URL
 503   getTorrentUrl () {
 504     if (!this.torrentFilename) return null
 505
 506     return WEBSERVER.URL + this.getTorrentStaticPath()
 507   }
 508
 509   getTorrentStaticPath () {
 510     if (!this.torrentFilename) return null
 511
 512     return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
 513   }
 514
 515   getTorrentDownloadUrl () {
 516     if (!this.torrentFilename) return null
 517
 518     return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
 519   }
 520
 521   removeTorrent () {
 522     if (!this.torrentFilename) return null
 523
 524     const torrentPath = getFSTorrentFilePath(this)
 525     return remove(torrentPath)
 526       .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
 527   }
 528
 529   hasSameUniqueKeysThan (other: MVideoFile) {
 530     return this.fps === other.fps &&
 531       this.resolution === other.resolution &&
 532       (
 533         (this.videoId !== null && this.videoId === other.videoId) ||
 534         (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
 535       )
 536   }
 537
 538   withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
 539     if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist })
 540
 541     return Object.assign(this, { Video: videoOrPlaylist })
 542   }
 543 }