1 import { remove } from 'fs-extra'
2 import memoizee from 'memoizee'
3 import { join } from 'path'
4 import { FindOptions, Op, Transaction, WhereOptions } from 'sequelize'
20 } from 'sequelize-typescript'
21 import validator from 'validator'
22 import { logger } from '@server/helpers/logger'
23 import { extractVideo } from '@server/helpers/video'
24 import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url'
25 import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
26 import { getFSTorrentFilePath } from '@server/lib/paths'
27 import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
28 import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
29 import { VideoResolution, VideoStorage } from '@shared/models'
30 import { AttributesOnly } from '@shared/typescript-utils'
32 isVideoFileExtnameValid,
33 isVideoFileInfoHashValid,
34 isVideoFileResolutionValid,
36 isVideoFPSResolutionValid
37 } from '../../helpers/custom-validators/videos'
42 STATIC_DOWNLOAD_PATHS,
45 } from '../../initializers/constants'
46 import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file'
47 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
48 import { doesExist } from '../shared'
49 import { parseAggregateResult, throwIfNotValid } from '../utils'
50 import { VideoModel } from './video'
51 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
53 export enum ScopeNames {
54 WITH_VIDEO = 'WITH_VIDEO',
55 WITH_METADATA = 'WITH_METADATA',
56 WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
59 @DefaultScope(() => ({
61 exclude: [ 'metadata' ]
65 [ScopeNames.WITH_VIDEO]: {
68 model: VideoModel.unscoped(),
73 [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: WhereOptions } = {}) => {
77 model: VideoModel.unscoped(),
79 where: options.whereVideo
82 model: VideoStreamingPlaylistModel.unscoped(),
86 model: VideoModel.unscoped(),
88 where: options.whereVideo
95 [ScopeNames.WITH_METADATA]: {
97 include: [ 'metadata' ]
102 tableName: 'videoFile',
105 fields: [ 'videoId' ],
113 fields: [ 'videoStreamingPlaylistId' ],
115 videoStreamingPlaylistId: {
122 fields: [ 'infoHash' ]
126 fields: [ 'torrentFilename' ],
131 fields: [ 'filename' ],
136 fields: [ 'videoId', 'resolution', 'fps' ],
145 fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
148 videoStreamingPlaylistId: {
155 export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>>> {
163 @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution'))
168 @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size'))
169 @Column(DataType.BIGINT)
173 @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
178 @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
184 @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
189 @Column(DataType.JSONB)
196 // Could be null for remote files
201 // Could be null for live files
206 // Could be null for remote files
211 // Could be null for live files
214 torrentFilename: string
216 @ForeignKey(() => VideoModel)
221 @Default(VideoStorage.FILE_SYSTEM)
223 storage: VideoStorage
225 @BelongsTo(() => VideoModel, {
233 @ForeignKey(() => VideoStreamingPlaylistModel)
235 videoStreamingPlaylistId: number
237 @BelongsTo(() => VideoStreamingPlaylistModel, {
243 VideoStreamingPlaylist: VideoStreamingPlaylistModel
245 @HasMany(() => VideoRedundancyModel, {
252 RedundancyVideos: VideoRedundancyModel[]
254 static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, {
256 max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
257 maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
260 static doesInfohashExist (infoHash: string) {
261 const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
263 return doesExist(query, { infoHash })
266 static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
267 const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
272 static async doesOwnedTorrentFileExist (filename: string) {
273 const query = 'SELECT 1 FROM "videoFile" ' +
274 'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
275 'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
276 'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
277 'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
279 return doesExist(query, { filename })
282 static async doesOwnedWebTorrentVideoFileExist (filename: string) {
283 const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
284 `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1`
286 return doesExist(query, { filename })
289 static loadByFilename (filename: string) {
296 return VideoFileModel.findOne(query)
299 static loadWithVideoByFilename (filename: string): Promise<MVideoFileVideo | MVideoFileStreamingPlaylistVideo> {
306 return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
309 static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
312 torrentFilename: filename
316 return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
319 static load (id: number): Promise<MVideoFile> {
320 return VideoFileModel.findByPk(id)
323 static loadWithMetadata (id: number) {
324 return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
327 static loadWithVideo (id: number) {
328 return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
331 static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
332 const whereVideo = validator.isUUID(videoIdOrUUID + '')
333 ? { uuid: videoIdOrUUID }
334 : { id: videoIdOrUUID }
342 return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
345 // We used `required: false` so check we have at least a video or a streaming playlist
346 if (!file.Video && !file.VideoStreamingPlaylist) return null
352 static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
356 model: VideoModel.unscoped(),
360 model: VideoStreamingPlaylistModel.unscoped(),
363 id: streamingPlaylistId
372 return VideoFileModel.findAll(query)
376 const webtorrentFilesQuery: FindOptions = {
381 model: VideoModel.unscoped(),
389 const hlsFilesQuery: FindOptions = {
394 model: VideoStreamingPlaylistModel.unscoped(),
398 model: VideoModel.unscoped(),
410 VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery),
411 VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
412 ]).then(([ webtorrentResult, hlsResult ]) => ({
413 totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult)
417 // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
418 static async customUpsert (
419 videoFile: MVideoFile,
420 mode: 'streaming-playlist' | 'video',
421 transaction: Transaction
425 resolution: videoFile.resolution,
429 const element = mode === 'streaming-playlist'
430 ? await VideoFileModel.loadHLSFile({ ...baseFind, playlistId: videoFile.videoStreamingPlaylistId })
431 : await VideoFileModel.loadWebTorrentFile({ ...baseFind, videoId: videoFile.videoId })
433 if (!element) return videoFile.save({ transaction })
435 for (const k of Object.keys(videoFile.toJSON())) {
436 element[k] = videoFile[k]
439 return element.save({ transaction })
442 static async loadWebTorrentFile (options: {
446 transaction?: Transaction
450 resolution: options.resolution,
451 videoId: options.videoId
454 return VideoFileModel.findOne({ where, transaction: options.transaction })
457 static async loadHLSFile (options: {
461 transaction?: Transaction
465 resolution: options.resolution,
466 videoStreamingPlaylistId: options.playlistId
469 return VideoFileModel.findOne({ where, transaction: options.transaction })
472 static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
474 where: { videoStreamingPlaylistId }
477 return VideoFileModel.destroy(options)
481 return this.infoHash && this.torrentFilename
484 getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
485 if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video
487 return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
490 getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
491 return extractVideo(this.getVideoOrStreamingPlaylist())
495 return this.resolution === VideoResolution.H_NOVIDEO
499 return this.size === -1
503 return !!this.videoStreamingPlaylistId
506 getObjectStorageUrl () {
508 return getHLSPublicFileUrl(this.fileUrl)
511 return getWebTorrentPublicFileUrl(this.fileUrl)
514 getFileUrl (video: MVideo) {
515 if (this.storage === VideoStorage.OBJECT_STORAGE) {
516 return this.getObjectStorageUrl()
519 if (!this.Video) this.Video = video as VideoModel
520 if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
525 getFileStaticPath (video: MVideo) {
527 if (isVideoInPrivateDirectory(video.privacy)) {
528 return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
531 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
534 if (isVideoInPrivateDirectory(video.privacy)) {
535 return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename)
538 return join(STATIC_PATHS.WEBSEED, this.filename)
541 getFileDownloadUrl (video: MVideoWithHost) {
542 const path = this.isHLS()
543 ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
544 : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
546 if (video.isOwned()) return WEBSERVER.URL + path
548 // FIXME: don't guess remote URL
549 return buildRemoteVideoBaseUrl(video, path)
552 getRemoteTorrentUrl (video: MVideo) {
553 if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
555 return this.torrentUrl
558 // We proxify torrent requests so use a local URL
560 if (!this.torrentFilename) return null
562 return WEBSERVER.URL + this.getTorrentStaticPath()
565 getTorrentStaticPath () {
566 if (!this.torrentFilename) return null
568 return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
571 getTorrentDownloadUrl () {
572 if (!this.torrentFilename) return null
574 return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
578 if (!this.torrentFilename) return null
580 const torrentPath = getFSTorrentFilePath(this)
581 return remove(torrentPath)
582 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
585 hasSameUniqueKeysThan (other: MVideoFile) {
586 return this.fps === other.fps &&
587 this.resolution === other.resolution &&
589 (this.videoId !== null && this.videoId === other.videoId) ||
590 (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
594 withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
595 if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist })
597 return Object.assign(this, { Video: videoOrPlaylist })