server/models/video/video-file.ts

   1 import { remove } from 'fs-extra'
   2 import * as memoizee from 'memoizee'
   3 import { join } from 'path'
   4 import { FindOptions, Op, Transaction } from 'sequelize'
   5 import {
   6   AllowNull,
   7   BelongsTo,
   8   Column,
   9   CreatedAt,
  10   DataType,
  11   Default,
  12   DefaultScope,
  13   ForeignKey,
  14   HasMany,
  15   Is,
  16   Model,
  17   Scopes,
  18   Table,
  19   UpdatedAt
  20 } from 'sequelize-typescript'
  21 import { Where } from 'sequelize/types/lib/utils'
  22 import validator from 'validator'
  23 import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
  24 import { doesExist } from '@server/helpers/database-utils'
  25 import { logger } from '@server/helpers/logger'
  26 import { extractVideo } from '@server/helpers/video'
  27 import { getTorrentFilePath } from '@server/lib/video-paths'
  28 import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
  29 import { AttributesOnly } from '@shared/core-utils'
  30 import {
  31   isVideoFileExtnameValid,
  32   isVideoFileInfoHashValid,
  33   isVideoFileResolutionValid,
  34   isVideoFileSizeValid,
  35   isVideoFPSResolutionValid
  36 } from '../../helpers/custom-validators/videos'
  37 import {
  38   LAZY_STATIC_PATHS,
  39   MEMOIZE_LENGTH,
  40   MEMOIZE_TTL,
  41   MIMETYPES,
  42   STATIC_DOWNLOAD_PATHS,
  43   STATIC_PATHS,
  44   WEBSERVER
  45 } from '../../initializers/constants'
  46 import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file'
  47 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
  48 import { parseAggregateResult, throwIfNotValid } from '../utils'
  49 import { VideoModel } from './video'
  50 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
  51
  52 export enum ScopeNames {
  53   WITH_VIDEO = 'WITH_VIDEO',
  54   WITH_METADATA = 'WITH_METADATA',
  55   WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
  56 }
  57
  58 @DefaultScope(() => ({
  59   attributes: {
  60     exclude: [ 'metadata' ]
  61   }
  62 }))
  63 @Scopes(() => ({
  64   [ScopeNames.WITH_VIDEO]: {
  65     include: [
  66       {
  67         model: VideoModel.unscoped(),
  68         required: true
  69       }
  70     ]
  71   },
  72   [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: Where } = {}) => {
  73     return {
  74       include: [
  75         {
  76           model: VideoModel.unscoped(),
  77           required: false,
  78           where: options.whereVideo
  79         },
  80         {
  81           model: VideoStreamingPlaylistModel.unscoped(),
  82           required: false,
  83           include: [
  84             {
  85               model: VideoModel.unscoped(),
  86               required: true,
  87               where: options.whereVideo
  88             }
  89           ]
  90         }
  91       ]
  92     }
  93   },
  94   [ScopeNames.WITH_METADATA]: {
  95     attributes: {
  96       include: [ 'metadata' ]
  97     }
  98   }
  99 }))
 100 @Table({
 101   tableName: 'videoFile',
 102   indexes: [
 103     {
 104       fields: [ 'videoId' ],
 105       where: {
 106         videoId: {
 107           [Op.ne]: null
 108         }
 109       }
 110     },
 111     {
 112       fields: [ 'videoStreamingPlaylistId' ],
 113       where: {
 114         videoStreamingPlaylistId: {
 115           [Op.ne]: null
 116         }
 117       }
 118     },
 119
 120     {
 121       fields: [ 'infoHash' ]
 122     },
 123
 124     {
 125       fields: [ 'torrentFilename' ],
 126       unique: true
 127     },
 128
 129     {
 130       fields: [ 'filename' ],
 131       unique: true
 132     },
 133
 134     {
 135       fields: [ 'videoId', 'resolution', 'fps' ],
 136       unique: true,
 137       where: {
 138         videoId: {
 139           [Op.ne]: null
 140         }
 141       }
 142     },
 143     {
 144       fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
 145       unique: true,
 146       where: {
 147         videoStreamingPlaylistId: {
 148           [Op.ne]: null
 149         }
 150       }
 151     }
 152   ]
 153 })
 154 export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>>> {
 155   @CreatedAt
 156   createdAt: Date
 157
 158   @UpdatedAt
 159   updatedAt: Date
 160
 161   @AllowNull(false)
 162   @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution'))
 163   @Column
 164   resolution: number
 165
 166   @AllowNull(false)
 167   @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size'))
 168   @Column(DataType.BIGINT)
 169   size: number
 170
 171   @AllowNull(false)
 172   @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
 173   @Column
 174   extname: string
 175
 176   @AllowNull(true)
 177   @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
 178   @Column
 179   infoHash: string
 180
 181   @AllowNull(false)
 182   @Default(-1)
 183   @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
 184   @Column
 185   fps: number
 186
 187   @AllowNull(true)
 188   @Column(DataType.JSONB)
 189   metadata: any
 190
 191   @AllowNull(true)
 192   @Column
 193   metadataUrl: string
 194
 195   @AllowNull(true)
 196   @Column
 197   fileUrl: string
 198
 199   // Could be null for live files
 200   @AllowNull(true)
 201   @Column
 202   filename: string
 203
 204   @AllowNull(true)
 205   @Column
 206   torrentUrl: string
 207
 208   // Could be null for live files
 209   @AllowNull(true)
 210   @Column
 211   torrentFilename: string
 212
 213   @ForeignKey(() => VideoModel)
 214   @Column
 215   videoId: number
 216
 217   @BelongsTo(() => VideoModel, {
 218     foreignKey: {
 219       allowNull: true
 220     },
 221     onDelete: 'CASCADE'
 222   })
 223   Video: VideoModel
 224
 225   @ForeignKey(() => VideoStreamingPlaylistModel)
 226   @Column
 227   videoStreamingPlaylistId: number
 228
 229   @BelongsTo(() => VideoStreamingPlaylistModel, {
 230     foreignKey: {
 231       allowNull: true
 232     },
 233     onDelete: 'CASCADE'
 234   })
 235   VideoStreamingPlaylist: VideoStreamingPlaylistModel
 236
 237   @HasMany(() => VideoRedundancyModel, {
 238     foreignKey: {
 239       allowNull: true
 240     },
 241     onDelete: 'CASCADE',
 242     hooks: true
 243   })
 244   RedundancyVideos: VideoRedundancyModel[]
 245
 246   static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, {
 247     promise: true,
 248     max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
 249     maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
 250   })
 251
 252   static doesInfohashExist (infoHash: string) {
 253     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
 254
 255     return doesExist(query, { infoHash })
 256   }
 257
 258   static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
 259     const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
 260
 261     return !!videoFile
 262   }
 263
 264   static async doesOwnedTorrentFileExist (filename: string) {
 265     const query = 'SELECT 1 FROM "videoFile" ' +
 266                   'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
 267                   'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
 268                   'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
 269                   'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
 270
 271     return doesExist(query, { filename })
 272   }
 273
 274   static async doesOwnedWebTorrentVideoFileExist (filename: string) {
 275     const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
 276                   'WHERE "filename" = $filename LIMIT 1'
 277
 278     return doesExist(query, { filename })
 279   }
 280
 281   static loadByFilename (filename: string) {
 282     const query = {
 283       where: {
 284         filename
 285       }
 286     }
 287
 288     return VideoFileModel.findOne(query)
 289   }
 290
 291   static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
 292     const query = {
 293       where: {
 294         torrentFilename: filename
 295       }
 296     }
 297
 298     return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
 299   }
 300
 301   static loadWithMetadata (id: number) {
 302     return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
 303   }
 304
 305   static loadWithVideo (id: number) {
 306     return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
 307   }
 308
 309   static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
 310     const whereVideo = validator.isUUID(videoIdOrUUID + '')
 311       ? { uuid: videoIdOrUUID }
 312       : { id: videoIdOrUUID }
 313
 314     const options = {
 315       where: {
 316         id
 317       }
 318     }
 319
 320     return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
 321       .findOne(options)
 322       .then(file => {
 323         // We used `required: false` so check we have at least a video or a streaming playlist
 324         if (!file.Video && !file.VideoStreamingPlaylist) return null
 325
 326         return file
 327       })
 328   }
 329
 330   static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
 331     const query = {
 332       include: [
 333         {
 334           model: VideoModel.unscoped(),
 335           required: true,
 336           include: [
 337             {
 338               model: VideoStreamingPlaylistModel.unscoped(),
 339               required: true,
 340               where: {
 341                 id: streamingPlaylistId
 342               }
 343             }
 344           ]
 345         }
 346       ],
 347       transaction
 348     }
 349
 350     return VideoFileModel.findAll(query)
 351   }
 352
 353   static getStats () {
 354     const webtorrentFilesQuery: FindOptions = {
 355       include: [
 356         {
 357           attributes: [],
 358           required: true,
 359           model: VideoModel.unscoped(),
 360           where: {
 361             remote: false
 362           }
 363         }
 364       ]
 365     }
 366
 367     const hlsFilesQuery: FindOptions = {
 368       include: [
 369         {
 370           attributes: [],
 371           required: true,
 372           model: VideoStreamingPlaylistModel.unscoped(),
 373           include: [
 374             {
 375               attributes: [],
 376               model: VideoModel.unscoped(),
 377               required: true,
 378               where: {
 379                 remote: false
 380               }
 381             }
 382           ]
 383         }
 384       ]
 385     }
 386
 387     return Promise.all([
 388       VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery),
 389       VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
 390     ]).then(([ webtorrentResult, hlsResult ]) => ({
 391       totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult)
 392     }))
 393   }
 394
 395   // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
 396   static async customUpsert (
 397     videoFile: MVideoFile,
 398     mode: 'streaming-playlist' | 'video',
 399     transaction: Transaction
 400   ) {
 401     const baseWhere = {
 402       fps: videoFile.fps,
 403       resolution: videoFile.resolution
 404     }
 405
 406     if (mode === 'streaming-playlist') Object.assign(baseWhere, { videoStreamingPlaylistId: videoFile.videoStreamingPlaylistId })
 407     else Object.assign(baseWhere, { videoId: videoFile.videoId })
 408
 409     const element = await VideoFileModel.findOne({ where: baseWhere, transaction })
 410     if (!element) return videoFile.save({ transaction })
 411
 412     for (const k of Object.keys(videoFile.toJSON())) {
 413       element[k] = videoFile[k]
 414     }
 415
 416     return element.save({ transaction })
 417   }
 418
 419   static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
 420     const options = {
 421       where: { videoStreamingPlaylistId }
 422     }
 423
 424     return VideoFileModel.destroy(options)
 425   }
 426
 427   hasTorrent () {
 428     return this.infoHash && this.torrentFilename
 429   }
 430
 431   getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
 432     if (this.videoId) return (this as MVideoFileVideo).Video
 433
 434     return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
 435   }
 436
 437   getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
 438     return extractVideo(this.getVideoOrStreamingPlaylist())
 439   }
 440
 441   isAudio () {
 442     return !!MIMETYPES.AUDIO.EXT_MIMETYPE[this.extname]
 443   }
 444
 445   isLive () {
 446     return this.size === -1
 447   }
 448
 449   isHLS () {
 450     return !!this.videoStreamingPlaylistId
 451   }
 452
 453   getFileUrl (video: MVideo) {
 454     if (!this.Video) this.Video = video as VideoModel
 455
 456     if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
 457
 458     return this.fileUrl
 459   }
 460
 461   getFileStaticPath (video: MVideo) {
 462     if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
 463
 464     return join(STATIC_PATHS.WEBSEED, this.filename)
 465   }
 466
 467   getFileDownloadUrl (video: MVideoWithHost) {
 468     const path = this.isHLS()
 469       ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
 470       : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
 471
 472     if (video.isOwned()) return WEBSERVER.URL + path
 473
 474     // FIXME: don't guess remote URL
 475     return buildRemoteVideoBaseUrl(video, path)
 476   }
 477
 478   getRemoteTorrentUrl (video: MVideo) {
 479     if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
 480
 481     return this.torrentUrl
 482   }
 483
 484   // We proxify torrent requests so use a local URL
 485   getTorrentUrl () {
 486     if (!this.torrentFilename) return null
 487
 488     return WEBSERVER.URL + this.getTorrentStaticPath()
 489   }
 490
 491   getTorrentStaticPath () {
 492     if (!this.torrentFilename) return null
 493
 494     return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
 495   }
 496
 497   getTorrentDownloadUrl () {
 498     if (!this.torrentFilename) return null
 499
 500     return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
 501   }
 502
 503   removeTorrent () {
 504     if (!this.torrentFilename) return null
 505
 506     const torrentPath = getTorrentFilePath(this)
 507     return remove(torrentPath)
 508       .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
 509   }
 510
 511   hasSameUniqueKeysThan (other: MVideoFile) {
 512     return this.fps === other.fps &&
 513       this.resolution === other.resolution &&
 514       (
 515         (this.videoId !== null && this.videoId === other.videoId) ||
 516         (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
 517       )
 518   }
 519 }