1 import { remove } from 'fs-extra'
2 import memoizee from 'memoizee'
3 import { join } from 'path'
4 import { FindOptions, Op, Transaction, WhereOptions } from 'sequelize'
20 } from 'sequelize-typescript'
21 import validator from 'validator'
22 import { logger } from '@server/helpers/logger'
23 import { extractVideo } from '@server/helpers/video'
24 import { CONFIG } from '@server/initializers/config'
25 import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url'
29 getWebTorrentPrivateFileUrl,
30 getWebTorrentPublicFileUrl
31 } from '@server/lib/object-storage'
32 import { getFSTorrentFilePath } from '@server/lib/paths'
33 import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
34 import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
35 import { VideoResolution, VideoStorage } from '@shared/models'
36 import { AttributesOnly } from '@shared/typescript-utils'
38 isVideoFileExtnameValid,
39 isVideoFileInfoHashValid,
40 isVideoFileResolutionValid,
42 isVideoFPSResolutionValid
43 } from '../../helpers/custom-validators/videos'
48 STATIC_DOWNLOAD_PATHS,
51 } from '../../initializers/constants'
52 import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file'
53 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
54 import { doesExist, parseAggregateResult, throwIfNotValid } from '../shared'
55 import { VideoModel } from './video'
56 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
58 export enum ScopeNames {
59 WITH_VIDEO = 'WITH_VIDEO',
60 WITH_METADATA = 'WITH_METADATA',
61 WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
64 @DefaultScope(() => ({
66 exclude: [ 'metadata' ]
70 [ScopeNames.WITH_VIDEO]: {
73 model: VideoModel.unscoped(),
78 [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: WhereOptions } = {}) => {
82 model: VideoModel.unscoped(),
84 where: options.whereVideo
87 model: VideoStreamingPlaylistModel.unscoped(),
91 model: VideoModel.unscoped(),
93 where: options.whereVideo
100 [ScopeNames.WITH_METADATA]: {
102 include: [ 'metadata' ]
107 tableName: 'videoFile',
110 fields: [ 'videoId' ],
118 fields: [ 'videoStreamingPlaylistId' ],
120 videoStreamingPlaylistId: {
127 fields: [ 'infoHash' ]
131 fields: [ 'torrentFilename' ],
136 fields: [ 'filename' ],
141 fields: [ 'videoId', 'resolution', 'fps' ],
150 fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
153 videoStreamingPlaylistId: {
160 export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>>> {
168 @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution'))
173 @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size'))
174 @Column(DataType.BIGINT)
178 @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
183 @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
189 @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
194 @Column(DataType.JSONB)
201 // Could be null for remote files
206 // Could be null for live files
211 // Could be null for remote files
216 // Could be null for live files
219 torrentFilename: string
221 @ForeignKey(() => VideoModel)
226 @Default(VideoStorage.FILE_SYSTEM)
228 storage: VideoStorage
230 @BelongsTo(() => VideoModel, {
238 @ForeignKey(() => VideoStreamingPlaylistModel)
240 videoStreamingPlaylistId: number
242 @BelongsTo(() => VideoStreamingPlaylistModel, {
248 VideoStreamingPlaylist: VideoStreamingPlaylistModel
250 @HasMany(() => VideoRedundancyModel, {
257 RedundancyVideos: VideoRedundancyModel[]
259 static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, {
261 max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
262 maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
265 static doesInfohashExist (infoHash: string) {
266 const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
268 return doesExist(this.sequelize, query, { infoHash })
271 static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
272 const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
277 static async doesOwnedTorrentFileExist (filename: string) {
278 const query = 'SELECT 1 FROM "videoFile" ' +
279 'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
280 'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
281 'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
282 'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
284 return doesExist(this.sequelize, query, { filename })
287 static async doesOwnedWebTorrentVideoFileExist (filename: string) {
288 const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
289 `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1`
291 return doesExist(this.sequelize, query, { filename })
294 static loadByFilename (filename: string) {
301 return VideoFileModel.findOne(query)
304 static loadWithVideoByFilename (filename: string): Promise<MVideoFileVideo | MVideoFileStreamingPlaylistVideo> {
311 return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
314 static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
317 torrentFilename: filename
321 return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
324 static load (id: number): Promise<MVideoFile> {
325 return VideoFileModel.findByPk(id)
328 static loadWithMetadata (id: number) {
329 return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
332 static loadWithVideo (id: number) {
333 return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
336 static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
337 const whereVideo = validator.isUUID(videoIdOrUUID + '')
338 ? { uuid: videoIdOrUUID }
339 : { id: videoIdOrUUID }
347 return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
350 // We used `required: false` so check we have at least a video or a streaming playlist
351 if (!file.Video && !file.VideoStreamingPlaylist) return null
357 static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
361 model: VideoModel.unscoped(),
365 model: VideoStreamingPlaylistModel.unscoped(),
368 id: streamingPlaylistId
377 return VideoFileModel.findAll(query)
381 const webtorrentFilesQuery: FindOptions = {
386 model: VideoModel.unscoped(),
394 const hlsFilesQuery: FindOptions = {
399 model: VideoStreamingPlaylistModel.unscoped(),
403 model: VideoModel.unscoped(),
415 VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery),
416 VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
417 ]).then(([ webtorrentResult, hlsResult ]) => ({
418 totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult)
422 // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
423 static async customUpsert (
424 videoFile: MVideoFile,
425 mode: 'streaming-playlist' | 'video',
426 transaction: Transaction
430 resolution: videoFile.resolution,
434 const element = mode === 'streaming-playlist'
435 ? await VideoFileModel.loadHLSFile({ ...baseFind, playlistId: videoFile.videoStreamingPlaylistId })
436 : await VideoFileModel.loadWebTorrentFile({ ...baseFind, videoId: videoFile.videoId })
438 if (!element) return videoFile.save({ transaction })
440 for (const k of Object.keys(videoFile.toJSON())) {
441 element.set(k, videoFile[k])
444 return element.save({ transaction })
447 static async loadWebTorrentFile (options: {
451 transaction?: Transaction
455 resolution: options.resolution,
456 videoId: options.videoId
459 return VideoFileModel.findOne({ where, transaction: options.transaction })
462 static async loadHLSFile (options: {
466 transaction?: Transaction
470 resolution: options.resolution,
471 videoStreamingPlaylistId: options.playlistId
474 return VideoFileModel.findOne({ where, transaction: options.transaction })
477 static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
479 where: { videoStreamingPlaylistId }
482 return VideoFileModel.destroy(options)
486 return this.infoHash && this.torrentFilename
489 getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
490 if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video
492 return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
495 getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
496 return extractVideo(this.getVideoOrStreamingPlaylist())
500 return this.resolution === VideoResolution.H_NOVIDEO
504 return this.size === -1
508 return !!this.videoStreamingPlaylistId
511 // ---------------------------------------------------------------------------
513 getObjectStorageUrl (video: MVideo) {
514 if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
515 return this.getPrivateObjectStorageUrl(video)
518 return this.getPublicObjectStorageUrl()
521 private getPrivateObjectStorageUrl (video: MVideo) {
523 return getHLSPrivateFileUrl(video, this.filename)
526 return getWebTorrentPrivateFileUrl(this.filename)
529 private getPublicObjectStorageUrl () {
531 return getHLSPublicFileUrl(this.fileUrl)
534 return getWebTorrentPublicFileUrl(this.fileUrl)
537 // ---------------------------------------------------------------------------
539 getFileUrl (video: MVideo) {
540 if (video.isOwned()) {
541 if (this.storage === VideoStorage.OBJECT_STORAGE) {
542 return this.getObjectStorageUrl(video)
545 return WEBSERVER.URL + this.getFileStaticPath(video)
551 // ---------------------------------------------------------------------------
553 getFileStaticPath (video: MVideo) {
554 if (this.isHLS()) return this.getHLSFileStaticPath(video)
556 return this.getWebTorrentFileStaticPath(video)
559 private getWebTorrentFileStaticPath (video: MVideo) {
560 if (isVideoInPrivateDirectory(video.privacy)) {
561 return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename)
564 return join(STATIC_PATHS.WEBSEED, this.filename)
567 private getHLSFileStaticPath (video: MVideo) {
568 if (isVideoInPrivateDirectory(video.privacy)) {
569 return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
572 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
575 // ---------------------------------------------------------------------------
577 getFileDownloadUrl (video: MVideoWithHost) {
578 const path = this.isHLS()
579 ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
580 : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
582 if (video.isOwned()) return WEBSERVER.URL + path
584 // FIXME: don't guess remote URL
585 return buildRemoteVideoBaseUrl(video, path)
588 getRemoteTorrentUrl (video: MVideo) {
589 if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
591 return this.torrentUrl
594 // We proxify torrent requests so use a local URL
596 if (!this.torrentFilename) return null
598 return WEBSERVER.URL + this.getTorrentStaticPath()
601 getTorrentStaticPath () {
602 if (!this.torrentFilename) return null
604 return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
607 getTorrentDownloadUrl () {
608 if (!this.torrentFilename) return null
610 return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
614 if (!this.torrentFilename) return null
616 const torrentPath = getFSTorrentFilePath(this)
617 return remove(torrentPath)
618 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
621 hasSameUniqueKeysThan (other: MVideoFile) {
622 return this.fps === other.fps &&
623 this.resolution === other.resolution &&
625 (this.videoId !== null && this.videoId === other.videoId) ||
626 (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
630 withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
631 if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist })
633 return Object.assign(this, { Video: videoOrPlaylist })