X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Fmodels%2Fvideo%2Fvideo-file.ts;h=1a9f4561af51747cfe11e388fbb185598e99b52a;hb=c9f27d9881ea0cc05e953117380b4b22a1376d21;hp=05c4907594fbbd36c7d86e0e8a5373c3fe24a664;hpb=53da06304957c7980c759c0db3e93010f4de7db3;p=github%2FChocobozzz%2FPeerTube.git

diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 05c490759..1a9f4561a 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -1,3 +1,7 @@
+import { remove } from 'fs-extra'
+import memoizee from 'memoizee'
+import { join } from 'path'
+import { FindOptions, Op, Transaction } from 'sequelize'
 import {
   AllowNull,
   BelongsTo,
@@ -5,13 +9,25 @@ import {
   CreatedAt,
   DataType,
   Default,
+  DefaultScope,
   ForeignKey,
   HasMany,
   Is,
   Model,
+  Scopes,
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
+import { Where } from 'sequelize/types/utils'
+import validator from 'validator'
+import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
+import { logger } from '@server/helpers/logger'
+import { extractVideo } from '@server/helpers/video'
+import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
+import { getFSTorrentFilePath } from '@server/lib/paths'
+import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
+import { VideoResolution, VideoStorage } from '@shared/models'
+import { AttributesOnly } from '@shared/typescript-utils'
 import {
   isVideoFileExtnameValid,
   isVideoFileInfoHashValid,
@@ -19,29 +35,124 @@ import {
   isVideoFileSizeValid,
   isVideoFPSResolutionValid
 } from '../../helpers/custom-validators/videos'
+import {
+  LAZY_STATIC_PATHS,
+  MEMOIZE_LENGTH,
+  MEMOIZE_TTL,
+  STATIC_DOWNLOAD_PATHS,
+  STATIC_PATHS,
+  WEBSERVER
+} from '../../initializers/constants'
+import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file'
+import { VideoRedundancyModel } from '../redundancy/video-redundancy'
+import { doesExist } from '../shared'
 import { parseAggregateResult, throwIfNotValid } from '../utils'
 import { VideoModel } from './video'
-import { VideoRedundancyModel } from '../redundancy/video-redundancy'
 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
-import { FindOptions, QueryTypes, Transaction } from 'sequelize'
-import { MIMETYPES } from '../../initializers/constants'
 
+export enum ScopeNames {
+  WITH_VIDEO = 'WITH_VIDEO',
+  WITH_METADATA = 'WITH_METADATA',
+  WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
+}
+
+@DefaultScope(() => ({
+  attributes: {
+    exclude: [ 'metadata' ]
+  }
+}))
+@Scopes(() => ({
+  [ScopeNames.WITH_VIDEO]: {
+    include: [
+      {
+        model: VideoModel.unscoped(),
+        required: true
+      }
+    ]
+  },
+  [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: Where } = {}) => {
+    return {
+      include: [
+        {
+          model: VideoModel.unscoped(),
+          required: false,
+          where: options.whereVideo
+        },
+        {
+          model: VideoStreamingPlaylistModel.unscoped(),
+          required: false,
+          include: [
+            {
+              model: VideoModel.unscoped(),
+              required: true,
+              where: options.whereVideo
+            }
+          ]
+        }
+      ]
+    }
+  },
+  [ScopeNames.WITH_METADATA]: {
+    attributes: {
+      include: [ 'metadata' ]
+    }
+  }
+}))
 @Table({
   tableName: 'videoFile',
   indexes: [
     {
-      fields: [ 'videoId' ]
+      fields: [ 'videoId' ],
+      where: {
+        videoId: {
+          [Op.ne]: null
+        }
+      }
     },
+    {
+      fields: [ 'videoStreamingPlaylistId' ],
+      where: {
+        videoStreamingPlaylistId: {
+          [Op.ne]: null
+        }
+      }
+    },
+
     {
       fields: [ 'infoHash' ]
     },
+
     {
-      fields: [ 'videoId', 'resolution', 'fps' ],
+      fields: [ 'torrentFilename' ],
       unique: true
+    },
+
+    {
+      fields: [ 'filename' ],
+      unique: true
+    },
+
+    {
+      fields: [ 'videoId', 'resolution', 'fps' ],
+      unique: true,
+      where: {
+        videoId: {
+          [Op.ne]: null
+        }
+      }
+    },
+    {
+      fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
+      unique: true,
+      where: {
+        videoStreamingPlaylistId: {
+          [Op.ne]: null
+        }
+      }
     }
   ]
 })
-export class VideoFileModel extends Model<VideoFileModel> {
+export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>>> {
   @CreatedAt
   createdAt: Date
 
@@ -63,8 +174,8 @@ export class VideoFileModel extends Model<VideoFileModel> {
   @Column
   extname: string
 
-  @AllowNull(false)
-  @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash'))
+  @AllowNull(true)
+  @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
   @Column
   infoHash: string
 
@@ -74,18 +185,63 @@ export class VideoFileModel extends Model<VideoFileModel> {
   @Column
   fps: number
 
+  @AllowNull(true)
+  @Column(DataType.JSONB)
+  metadata: any
+
+  @AllowNull(true)
+  @Column
+  metadataUrl: string
+
+  // Could be null for remote files
+  @AllowNull(true)
+  @Column
+  fileUrl: string
+
+  // Could be null for live files
+  @AllowNull(true)
+  @Column
+  filename: string
+
+  // Could be null for remote files
+  @AllowNull(true)
+  @Column
+  torrentUrl: string
+
+  // Could be null for live files
+  @AllowNull(true)
+  @Column
+  torrentFilename: string
+
   @ForeignKey(() => VideoModel)
   @Column
   videoId: number
 
+  @AllowNull(false)
+  @Default(VideoStorage.FILE_SYSTEM)
+  @Column
+  storage: VideoStorage
+
   @BelongsTo(() => VideoModel, {
     foreignKey: {
-      allowNull: false
+      allowNull: true
     },
     onDelete: 'CASCADE'
   })
   Video: VideoModel
 
+  @ForeignKey(() => VideoStreamingPlaylistModel)
+  @Column
+  videoStreamingPlaylistId: number
+
+  @BelongsTo(() => VideoStreamingPlaylistModel, {
+    foreignKey: {
+      allowNull: true
+    },
+    onDelete: 'CASCADE'
+  })
+  VideoStreamingPlaylist: VideoStreamingPlaylistModel
+
   @HasMany(() => VideoRedundancyModel, {
     foreignKey: {
       allowNull: true
@@ -95,29 +251,88 @@ export class VideoFileModel extends Model<VideoFileModel> {
   })
   RedundancyVideos: VideoRedundancyModel[]
 
+  static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, {
+    promise: true,
+    max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
+    maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
+  })
+
   static doesInfohashExist (infoHash: string) {
     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
-    const options = {
-      type: QueryTypes.SELECT,
-      bind: { infoHash },
-      raw: true
+
+    return doesExist(query, { infoHash })
+  }
+
+  static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
+    const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
+
+    return !!videoFile
+  }
+
+  static async doesOwnedTorrentFileExist (filename: string) {
+    const query = 'SELECT 1 FROM "videoFile" ' +
+                  'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
+                  'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
+                  'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
+                  'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
+
+    return doesExist(query, { filename })
+  }
+
+  static async doesOwnedWebTorrentVideoFileExist (filename: string) {
+    const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
+                  `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1`
+
+    return doesExist(query, { filename })
+  }
+
+  static loadByFilename (filename: string) {
+    const query = {
+      where: {
+        filename
+      }
+    }
+
+    return VideoFileModel.findOne(query)
+  }
+
+  static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
+    const query = {
+      where: {
+        torrentFilename: filename
+      }
     }
 
-    return VideoModel.sequelize.query(query, options)
-              .then(results => results.length === 1)
+    return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
+  }
+
+  static loadWithMetadata (id: number) {
+    return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
   }
 
   static loadWithVideo (id: number) {
+    return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
+  }
+
+  static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
+    const whereVideo = validator.isUUID(videoIdOrUUID + '')
+      ? { uuid: videoIdOrUUID }
+      : { id: videoIdOrUUID }
+
     const options = {
-      include: [
-        {
-          model: VideoModel.unscoped(),
-          required: true
-        }
-      ]
+      where: {
+        id
+      }
     }
 
-    return VideoFileModel.findByPk(id, options)
+    return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
+      .findOne(options)
+      .then(file => {
+        // We used `required: false` so check we have at least a video or a streaming playlist
+        if (!file.Video && !file.VideoStreamingPlaylist) return null
+
+        return file
+      })
   }
 
   static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
@@ -144,10 +359,11 @@ export class VideoFileModel extends Model<VideoFileModel> {
   }
 
   static getStats () {
-    const query: FindOptions = {
+    const webtorrentFilesQuery: FindOptions = {
       include: [
         {
           attributes: [],
+          required: true,
           model: VideoModel.unscoped(),
           where: {
             remote: false
@@ -156,19 +372,173 @@ export class VideoFileModel extends Model<VideoFileModel> {
       ]
     }
 
-    return VideoFileModel.aggregate('size', 'SUM', query)
-      .then(result => ({
-        totalLocalVideoFilesSize: parseAggregateResult(result)
-      }))
+    const hlsFilesQuery: FindOptions = {
+      include: [
+        {
+          attributes: [],
+          required: true,
+          model: VideoStreamingPlaylistModel.unscoped(),
+          include: [
+            {
+              attributes: [],
+              model: VideoModel.unscoped(),
+              required: true,
+              where: {
+                remote: false
+              }
+            }
+          ]
+        }
+      ]
+    }
+
+    return Promise.all([
+      VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery),
+      VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
+    ]).then(([ webtorrentResult, hlsResult ]) => ({
+      totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult)
+    }))
+  }
+
+  // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
+  static async customUpsert (
+    videoFile: MVideoFile,
+    mode: 'streaming-playlist' | 'video',
+    transaction: Transaction
+  ) {
+    const baseWhere = {
+      fps: videoFile.fps,
+      resolution: videoFile.resolution
+    }
+
+    if (mode === 'streaming-playlist') Object.assign(baseWhere, { videoStreamingPlaylistId: videoFile.videoStreamingPlaylistId })
+    else Object.assign(baseWhere, { videoId: videoFile.videoId })
+
+    const element = await VideoFileModel.findOne({ where: baseWhere, transaction })
+    if (!element) return videoFile.save({ transaction })
+
+    for (const k of Object.keys(videoFile.toJSON())) {
+      element[k] = videoFile[k]
+    }
+
+    return element.save({ transaction })
+  }
+
+  static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
+    const options = {
+      where: { videoStreamingPlaylistId }
+    }
+
+    return VideoFileModel.destroy(options)
+  }
+
+  hasTorrent () {
+    return this.infoHash && this.torrentFilename
+  }
+
+  getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
+    if (this.videoId) return (this as MVideoFileVideo).Video
+
+    return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
+  }
+
+  getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
+    return extractVideo(this.getVideoOrStreamingPlaylist())
   }
 
   isAudio () {
-    return !!MIMETYPES.AUDIO.EXT_MIMETYPE[this.extname]
+    return this.resolution === VideoResolution.H_NOVIDEO
+  }
+
+  isLive () {
+    return this.size === -1
+  }
+
+  isHLS () {
+    return !!this.videoStreamingPlaylistId
+  }
+
+  getObjectStorageUrl () {
+    if (this.isHLS()) {
+      return getHLSPublicFileUrl(this.fileUrl)
+    }
+
+    return getWebTorrentPublicFileUrl(this.fileUrl)
+  }
+
+  getFileUrl (video: MVideo) {
+    if (this.storage === VideoStorage.OBJECT_STORAGE) {
+      return this.getObjectStorageUrl()
+    }
+
+    if (!this.Video) this.Video = video as VideoModel
+    if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
+
+    return this.fileUrl
+  }
+
+  getFileStaticPath (video: MVideo) {
+    if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
+
+    return join(STATIC_PATHS.WEBSEED, this.filename)
+  }
+
+  getFileDownloadUrl (video: MVideoWithHost) {
+    const path = this.isHLS()
+      ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
+      : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
+
+    if (video.isOwned()) return WEBSERVER.URL + path
+
+    // FIXME: don't guess remote URL
+    return buildRemoteVideoBaseUrl(video, path)
+  }
+
+  getRemoteTorrentUrl (video: MVideo) {
+    if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
+
+    return this.torrentUrl
   }
 
-  hasSameUniqueKeysThan (other: VideoFileModel) {
+  // We proxify torrent requests so use a local URL
+  getTorrentUrl () {
+    if (!this.torrentFilename) return null
+
+    return WEBSERVER.URL + this.getTorrentStaticPath()
+  }
+
+  getTorrentStaticPath () {
+    if (!this.torrentFilename) return null
+
+    return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
+  }
+
+  getTorrentDownloadUrl () {
+    if (!this.torrentFilename) return null
+
+    return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
+  }
+
+  removeTorrent () {
+    if (!this.torrentFilename) return null
+
+    const torrentPath = getFSTorrentFilePath(this)
+    return remove(torrentPath)
+      .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
+  }
+
+  hasSameUniqueKeysThan (other: MVideoFile) {
     return this.fps === other.fps &&
       this.resolution === other.resolution &&
-      this.videoId === other.videoId
+      (
+        (this.videoId !== null && this.videoId === other.videoId) ||
+        (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
+      )
+  }
+
+  withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
+    if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist })
+
+    return Object.assign(this, { Video: videoOrPlaylist })
   }
 }