server/models/video/video-file.ts

   1 import { remove } from 'fs-extra'
   2 import memoizee from 'memoizee'
   3 import { join } from 'path'
   4 import { FindOptions, Op, Transaction } from 'sequelize'
   5 import {
   6   AllowNull,
   7   BelongsTo,
   8   Column,
   9   CreatedAt,
  10   DataType,
  11   Default,
  12   DefaultScope,
  13   ForeignKey,
  14   HasMany,
  15   Is,
  16   Model,
  17   Scopes,
  18   Table,
  19   UpdatedAt
  20 } from 'sequelize-typescript'
  21 import { Where } from 'sequelize/types/utils'
  22 import validator from 'validator'
  23 import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
  24 import { logger } from '@server/helpers/logger'
  25 import { extractVideo } from '@server/helpers/video'
  26 import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
  27 import { getFSTorrentFilePath } from '@server/lib/paths'
  28 import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
  29 import { VideoResolution, VideoStorage } from '@shared/models'
  30 import { AttributesOnly } from '@shared/typescript-utils'
  31 import {
  32   isVideoFileExtnameValid,
  33   isVideoFileInfoHashValid,
  34   isVideoFileResolutionValid,
  35   isVideoFileSizeValid,
  36   isVideoFPSResolutionValid
  37 } from '../../helpers/custom-validators/videos'
  38 import {
  39   LAZY_STATIC_PATHS,
  40   MEMOIZE_LENGTH,
  41   MEMOIZE_TTL,
  42   STATIC_DOWNLOAD_PATHS,
  43   STATIC_PATHS,
  44   WEBSERVER
  45 } from '../../initializers/constants'
  46 import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file'
  47 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
  48 import { doesExist } from '../shared'
  49 import { parseAggregateResult, throwIfNotValid } from '../utils'
  50 import { VideoModel } from './video'
  51 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
  52
  53 export enum ScopeNames {
  54   WITH_VIDEO = 'WITH_VIDEO',
  55   WITH_METADATA = 'WITH_METADATA',
  56   WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
  57 }
  58
  59 @DefaultScope(() => ({
  60   attributes: {
  61     exclude: [ 'metadata' ]
  62   }
  63 }))
  64 @Scopes(() => ({
  65   [ScopeNames.WITH_VIDEO]: {
  66     include: [
  67       {
  68         model: VideoModel.unscoped(),
  69         required: true
  70       }
  71     ]
  72   },
  73   [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: Where } = {}) => {
  74     return {
  75       include: [
  76         {
  77           model: VideoModel.unscoped(),
  78           required: false,
  79           where: options.whereVideo
  80         },
  81         {
  82           model: VideoStreamingPlaylistModel.unscoped(),
  83           required: false,
  84           include: [
  85             {
  86               model: VideoModel.unscoped(),
  87               required: true,
  88               where: options.whereVideo
  89             }
  90           ]
  91         }
  92       ]
  93     }
  94   },
  95   [ScopeNames.WITH_METADATA]: {
  96     attributes: {
  97       include: [ 'metadata' ]
  98     }
  99   }
 100 }))
 101 @Table({
 102   tableName: 'videoFile',
 103   indexes: [
 104     {
 105       fields: [ 'videoId' ],
 106       where: {
 107         videoId: {
 108           [Op.ne]: null
 109         }
 110       }
 111     },
 112     {
 113       fields: [ 'videoStreamingPlaylistId' ],
 114       where: {
 115         videoStreamingPlaylistId: {
 116           [Op.ne]: null
 117         }
 118       }
 119     },
 120
 121     {
 122       fields: [ 'infoHash' ]
 123     },
 124
 125     {
 126       fields: [ 'torrentFilename' ],
 127       unique: true
 128     },
 129
 130     {
 131       fields: [ 'filename' ],
 132       unique: true
 133     },
 134
 135     {
 136       fields: [ 'videoId', 'resolution', 'fps' ],
 137       unique: true,
 138       where: {
 139         videoId: {
 140           [Op.ne]: null
 141         }
 142       }
 143     },
 144     {
 145       fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
 146       unique: true,
 147       where: {
 148         videoStreamingPlaylistId: {
 149           [Op.ne]: null
 150         }
 151       }
 152     }
 153   ]
 154 })
 155 export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>>> {
 156   @CreatedAt
 157   createdAt: Date
 158
 159   @UpdatedAt
 160   updatedAt: Date
 161
 162   @AllowNull(false)
 163   @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution'))
 164   @Column
 165   resolution: number
 166
 167   @AllowNull(false)
 168   @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size'))
 169   @Column(DataType.BIGINT)
 170   size: number
 171
 172   @AllowNull(false)
 173   @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
 174   @Column
 175   extname: string
 176
 177   @AllowNull(true)
 178   @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
 179   @Column
 180   infoHash: string
 181
 182   @AllowNull(false)
 183   @Default(-1)
 184   @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
 185   @Column
 186   fps: number
 187
 188   @AllowNull(true)
 189   @Column(DataType.JSONB)
 190   metadata: any
 191
 192   @AllowNull(true)
 193   @Column
 194   metadataUrl: string
 195
 196   // Could be null for remote files
 197   @AllowNull(true)
 198   @Column
 199   fileUrl: string
 200
 201   // Could be null for live files
 202   @AllowNull(true)
 203   @Column
 204   filename: string
 205
 206   // Could be null for remote files
 207   @AllowNull(true)
 208   @Column
 209   torrentUrl: string
 210
 211   // Could be null for live files
 212   @AllowNull(true)
 213   @Column
 214   torrentFilename: string
 215
 216   @ForeignKey(() => VideoModel)
 217   @Column
 218   videoId: number
 219
 220   @AllowNull(false)
 221   @Default(VideoStorage.FILE_SYSTEM)
 222   @Column
 223   storage: VideoStorage
 224
 225   @BelongsTo(() => VideoModel, {
 226     foreignKey: {
 227       allowNull: true
 228     },
 229     onDelete: 'CASCADE'
 230   })
 231   Video: VideoModel
 232
 233   @ForeignKey(() => VideoStreamingPlaylistModel)
 234   @Column
 235   videoStreamingPlaylistId: number
 236
 237   @BelongsTo(() => VideoStreamingPlaylistModel, {
 238     foreignKey: {
 239       allowNull: true
 240     },
 241     onDelete: 'CASCADE'
 242   })
 243   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 244
 245   @HasMany(() => VideoRedundancyModel, {
 246     foreignKey: {
 247       allowNull: true
 248     },
 249     onDelete: 'CASCADE',
 250     hooks: true
 251   })
 252   RedundancyVideos: VideoRedundancyModel[]
 253
 254   static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, {
 255     promise: true,
 256     max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
 257     maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
 258   })
 259
 260   static doesInfohashExist (infoHash: string) {
 261     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
 262
 263     return doesExist(query, { infoHash })
 264   }
 265
 266   static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
 267     const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
 268
 269     return !!videoFile
 270   }
 271
 272   static async doesOwnedTorrentFileExist (filename: string) {
 273     const query = 'SELECT 1 FROM "videoFile" ' +
 274                   'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
 275                   'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
 276                   'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
 277                   'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
 278
 279     return doesExist(query, { filename })
 280   }
 281
 282   static async doesOwnedWebTorrentVideoFileExist (filename: string) {
 283     const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
 284                   `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1`
 285
 286     return doesExist(query, { filename })
 287   }
 288
 289   static loadByFilename (filename: string) {
 290     const query = {
 291       where: {
 292         filename
 293       }
 294     }
 295
 296     return VideoFileModel.findOne(query)
 297   }
 298
 299   static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
 300     const query = {
 301       where: {
 302         torrentFilename: filename
 303       }
 304     }
 305
 306     return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
 307   }
 308
 309   static loadWithMetadata (id: number) {
 310     return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
 311   }
 312
 313   static loadWithVideo (id: number) {
 314     return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
 315   }
 316
 317   static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
 318     const whereVideo = validator.isUUID(videoIdOrUUID + '')
 319       ? { uuid: videoIdOrUUID }
 320       : { id: videoIdOrUUID }
 321
 322     const options = {
 323       where: {
 324         id
 325       }
 326     }
 327
 328     return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
 329       .findOne(options)
 330       .then(file => {
 331         // We used `required: false` so check we have at least a video or a streaming playlist
 332         if (!file.Video && !file.VideoStreamingPlaylist) return null
 333
 334         return file
 335       })
 336   }
 337
 338   static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
 339     const query = {
 340       include: [
 341         {
 342           model: VideoModel.unscoped(),
 343           required: true,
 344           include: [
 345             {
 346               model: VideoStreamingPlaylistModel.unscoped(),
 347               required: true,
 348               where: {
 349                 id: streamingPlaylistId
 350               }
 351             }
 352           ]
 353         }
 354       ],
 355       transaction
 356     }
 357
 358     return VideoFileModel.findAll(query)
 359   }
 360
 361   static getStats () {
 362     const webtorrentFilesQuery: FindOptions = {
 363       include: [
 364         {
 365           attributes: [],
 366           required: true,
 367           model: VideoModel.unscoped(),
 368           where: {
 369             remote: false
 370           }
 371         }
 372       ]
 373     }
 374
 375     const hlsFilesQuery: FindOptions = {
 376       include: [
 377         {
 378           attributes: [],
 379           required: true,
 380           model: VideoStreamingPlaylistModel.unscoped(),
 381           include: [
 382             {
 383               attributes: [],
 384               model: VideoModel.unscoped(),
 385               required: true,
 386               where: {
 387                 remote: false
 388               }
 389             }
 390           ]
 391         }
 392       ]
 393     }
 394
 395     return Promise.all([
 396       VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery),
 397       VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
 398     ]).then(([ webtorrentResult, hlsResult ]) => ({
 399       totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult)
 400     }))
 401   }
 402
 403   // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
 404   static async customUpsert (
 405     videoFile: MVideoFile,
 406     mode: 'streaming-playlist' | 'video',
 407     transaction: Transaction
 408   ) {
 409     const baseWhere = {
 410       fps: videoFile.fps,
 411       resolution: videoFile.resolution
 412     }
 413
 414     if (mode === 'streaming-playlist') Object.assign(baseWhere, { videoStreamingPlaylistId: videoFile.videoStreamingPlaylistId })
 415     else Object.assign(baseWhere, { videoId: videoFile.videoId })
 416
 417     const element = await VideoFileModel.findOne({ where: baseWhere, transaction })
 418     if (!element) return videoFile.save({ transaction })
 419
 420     for (const k of Object.keys(videoFile.toJSON())) {
 421       element[k] = videoFile[k]
 422     }
 423
 424     return element.save({ transaction })
 425   }
 426
 427   static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
 428     const options = {
 429       where: { videoStreamingPlaylistId }
 430     }
 431
 432     return VideoFileModel.destroy(options)
 433   }
 434
 435   hasTorrent () {
 436     return this.infoHash && this.torrentFilename
 437   }
 438
 439   getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
 440     if (this.videoId) return (this as MVideoFileVideo).Video
 441
 442     return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
 443   }
 444
 445   getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
 446     return extractVideo(this.getVideoOrStreamingPlaylist())
 447   }
 448
 449   isAudio () {
 450     return this.resolution === VideoResolution.H_NOVIDEO
 451   }
 452
 453   isLive () {
 454     return this.size === -1
 455   }
 456
 457   isHLS () {
 458     return !!this.videoStreamingPlaylistId
 459   }
 460
 461   getObjectStorageUrl () {
 462     if (this.isHLS()) {
 463       return getHLSPublicFileUrl(this.fileUrl)
 464     }
 465
 466     return getWebTorrentPublicFileUrl(this.fileUrl)
 467   }
 468
 469   getFileUrl (video: MVideo) {
 470     if (this.storage === VideoStorage.OBJECT_STORAGE) {
 471       return this.getObjectStorageUrl()
 472     }
 473
 474     if (!this.Video) this.Video = video as VideoModel
 475     if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
 476
 477     return this.fileUrl
 478   }
 479
 480   getFileStaticPath (video: MVideo) {
 481     if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
 482
 483     return join(STATIC_PATHS.WEBSEED, this.filename)
 484   }
 485
 486   getFileDownloadUrl (video: MVideoWithHost) {
 487     const path = this.isHLS()
 488       ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
 489       : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
 490
 491     if (video.isOwned()) return WEBSERVER.URL + path
 492
 493     // FIXME: don't guess remote URL
 494     return buildRemoteVideoBaseUrl(video, path)
 495   }
 496
 497   getRemoteTorrentUrl (video: MVideo) {
 498     if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
 499
 500     return this.torrentUrl
 501   }
 502
 503   // We proxify torrent requests so use a local URL
 504   getTorrentUrl () {
 505     if (!this.torrentFilename) return null
 506
 507     return WEBSERVER.URL + this.getTorrentStaticPath()
 508   }
 509
 510   getTorrentStaticPath () {
 511     if (!this.torrentFilename) return null
 512
 513     return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
 514   }
 515
 516   getTorrentDownloadUrl () {
 517     if (!this.torrentFilename) return null
 518
 519     return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
 520   }
 521
 522   removeTorrent () {
 523     if (!this.torrentFilename) return null
 524
 525     const torrentPath = getFSTorrentFilePath(this)
 526     return remove(torrentPath)
 527       .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
 528   }
 529
 530   hasSameUniqueKeysThan (other: MVideoFile) {
 531     return this.fps === other.fps &&
 532       this.resolution === other.resolution &&
 533       (
 534         (this.videoId !== null && this.videoId === other.videoId) ||
 535         (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
 536       )
 537   }
 538
 539   withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
 540     if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist })
 541
 542     return Object.assign(this, { Video: videoOrPlaylist })
 543   }
 544 }