1 import { remove } from 'fs-extra'
2 import memoizee from 'memoizee'
3 import { join } from 'path'
4 import { FindOptions, Op, Transaction, WhereOptions } from 'sequelize'
20 } from 'sequelize-typescript'
21 import validator from 'validator'
22 import { logger } from '@server/helpers/logger'
23 import { extractVideo } from '@server/helpers/video'
24 import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url'
28 getWebTorrentPrivateFileUrl,
29 getWebTorrentPublicFileUrl
30 } from '@server/lib/object-storage'
31 import { getFSTorrentFilePath } from '@server/lib/paths'
32 import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
33 import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
34 import { VideoResolution, VideoStorage } from '@shared/models'
35 import { AttributesOnly } from '@shared/typescript-utils'
37 isVideoFileExtnameValid,
38 isVideoFileInfoHashValid,
39 isVideoFileResolutionValid,
41 isVideoFPSResolutionValid
42 } from '../../helpers/custom-validators/videos'
47 STATIC_DOWNLOAD_PATHS,
50 } from '../../initializers/constants'
51 import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file'
52 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
53 import { doesExist } from '../shared'
54 import { parseAggregateResult, throwIfNotValid } from '../utils'
55 import { VideoModel } from './video'
56 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
57 import { CONFIG } from '@server/initializers/config'
59 export enum ScopeNames {
60 WITH_VIDEO = 'WITH_VIDEO',
61 WITH_METADATA = 'WITH_METADATA',
62 WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
65 @DefaultScope(() => ({
67 exclude: [ 'metadata' ]
71 [ScopeNames.WITH_VIDEO]: {
74 model: VideoModel.unscoped(),
79 [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: WhereOptions } = {}) => {
83 model: VideoModel.unscoped(),
85 where: options.whereVideo
88 model: VideoStreamingPlaylistModel.unscoped(),
92 model: VideoModel.unscoped(),
94 where: options.whereVideo
101 [ScopeNames.WITH_METADATA]: {
103 include: [ 'metadata' ]
108 tableName: 'videoFile',
111 fields: [ 'videoId' ],
119 fields: [ 'videoStreamingPlaylistId' ],
121 videoStreamingPlaylistId: {
128 fields: [ 'infoHash' ]
132 fields: [ 'torrentFilename' ],
137 fields: [ 'filename' ],
142 fields: [ 'videoId', 'resolution', 'fps' ],
151 fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
154 videoStreamingPlaylistId: {
161 export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>>> {
169 @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution'))
174 @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size'))
175 @Column(DataType.BIGINT)
179 @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
184 @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
190 @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
195 @Column(DataType.JSONB)
202 // Could be null for remote files
207 // Could be null for live files
212 // Could be null for remote files
217 // Could be null for live files
220 torrentFilename: string
222 @ForeignKey(() => VideoModel)
227 @Default(VideoStorage.FILE_SYSTEM)
229 storage: VideoStorage
231 @BelongsTo(() => VideoModel, {
239 @ForeignKey(() => VideoStreamingPlaylistModel)
241 videoStreamingPlaylistId: number
243 @BelongsTo(() => VideoStreamingPlaylistModel, {
249 VideoStreamingPlaylist: VideoStreamingPlaylistModel
251 @HasMany(() => VideoRedundancyModel, {
258 RedundancyVideos: VideoRedundancyModel[]
260 static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, {
262 max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
263 maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
266 static doesInfohashExist (infoHash: string) {
267 const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
269 return doesExist(query, { infoHash })
272 static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
273 const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
278 static async doesOwnedTorrentFileExist (filename: string) {
279 const query = 'SELECT 1 FROM "videoFile" ' +
280 'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
281 'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
282 'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
283 'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
285 return doesExist(query, { filename })
288 static async doesOwnedWebTorrentVideoFileExist (filename: string) {
289 const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
290 `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1`
292 return doesExist(query, { filename })
295 static loadByFilename (filename: string) {
302 return VideoFileModel.findOne(query)
305 static loadWithVideoByFilename (filename: string): Promise<MVideoFileVideo | MVideoFileStreamingPlaylistVideo> {
312 return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
315 static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
318 torrentFilename: filename
322 return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
325 static load (id: number): Promise<MVideoFile> {
326 return VideoFileModel.findByPk(id)
329 static loadWithMetadata (id: number) {
330 return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
333 static loadWithVideo (id: number) {
334 return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
337 static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
338 const whereVideo = validator.isUUID(videoIdOrUUID + '')
339 ? { uuid: videoIdOrUUID }
340 : { id: videoIdOrUUID }
348 return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
351 // We used `required: false` so check we have at least a video or a streaming playlist
352 if (!file.Video && !file.VideoStreamingPlaylist) return null
358 static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
362 model: VideoModel.unscoped(),
366 model: VideoStreamingPlaylistModel.unscoped(),
369 id: streamingPlaylistId
378 return VideoFileModel.findAll(query)
382 const webtorrentFilesQuery: FindOptions = {
387 model: VideoModel.unscoped(),
395 const hlsFilesQuery: FindOptions = {
400 model: VideoStreamingPlaylistModel.unscoped(),
404 model: VideoModel.unscoped(),
416 VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery),
417 VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
418 ]).then(([ webtorrentResult, hlsResult ]) => ({
419 totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult)
423 // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
424 static async customUpsert (
425 videoFile: MVideoFile,
426 mode: 'streaming-playlist' | 'video',
427 transaction: Transaction
431 resolution: videoFile.resolution,
435 const element = mode === 'streaming-playlist'
436 ? await VideoFileModel.loadHLSFile({ ...baseFind, playlistId: videoFile.videoStreamingPlaylistId })
437 : await VideoFileModel.loadWebTorrentFile({ ...baseFind, videoId: videoFile.videoId })
439 if (!element) return videoFile.save({ transaction })
441 for (const k of Object.keys(videoFile.toJSON())) {
442 element[k] = videoFile[k]
445 return element.save({ transaction })
448 static async loadWebTorrentFile (options: {
452 transaction?: Transaction
456 resolution: options.resolution,
457 videoId: options.videoId
460 return VideoFileModel.findOne({ where, transaction: options.transaction })
463 static async loadHLSFile (options: {
467 transaction?: Transaction
471 resolution: options.resolution,
472 videoStreamingPlaylistId: options.playlistId
475 return VideoFileModel.findOne({ where, transaction: options.transaction })
478 static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
480 where: { videoStreamingPlaylistId }
483 return VideoFileModel.destroy(options)
487 return this.infoHash && this.torrentFilename
490 getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
491 if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video
493 return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
496 getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
497 return extractVideo(this.getVideoOrStreamingPlaylist())
501 return this.resolution === VideoResolution.H_NOVIDEO
505 return this.size === -1
509 return !!this.videoStreamingPlaylistId
512 // ---------------------------------------------------------------------------
514 getObjectStorageUrl (video: MVideo) {
515 if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
516 return this.getPrivateObjectStorageUrl(video)
519 return this.getPublicObjectStorageUrl()
522 private getPrivateObjectStorageUrl (video: MVideo) {
524 return getHLSPrivateFileUrl(video, this.filename)
527 return getWebTorrentPrivateFileUrl(this.filename)
530 private getPublicObjectStorageUrl () {
532 return getHLSPublicFileUrl(this.fileUrl)
535 return getWebTorrentPublicFileUrl(this.fileUrl)
538 // ---------------------------------------------------------------------------
540 getFileUrl (video: MVideo) {
541 if (video.isOwned()) {
542 if (this.storage === VideoStorage.OBJECT_STORAGE) {
543 return this.getObjectStorageUrl(video)
546 return WEBSERVER.URL + this.getFileStaticPath(video)
552 // ---------------------------------------------------------------------------
554 getFileStaticPath (video: MVideo) {
555 if (this.isHLS()) return this.getHLSFileStaticPath(video)
557 return this.getWebTorrentFileStaticPath(video)
560 private getWebTorrentFileStaticPath (video: MVideo) {
561 if (isVideoInPrivateDirectory(video.privacy)) {
562 return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename)
565 return join(STATIC_PATHS.WEBSEED, this.filename)
568 private getHLSFileStaticPath (video: MVideo) {
569 if (isVideoInPrivateDirectory(video.privacy)) {
570 return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
573 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
576 // ---------------------------------------------------------------------------
578 getFileDownloadUrl (video: MVideoWithHost) {
579 const path = this.isHLS()
580 ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
581 : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
583 if (video.isOwned()) return WEBSERVER.URL + path
585 // FIXME: don't guess remote URL
586 return buildRemoteVideoBaseUrl(video, path)
589 getRemoteTorrentUrl (video: MVideo) {
590 if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
592 return this.torrentUrl
595 // We proxify torrent requests so use a local URL
597 if (!this.torrentFilename) return null
599 return WEBSERVER.URL + this.getTorrentStaticPath()
602 getTorrentStaticPath () {
603 if (!this.torrentFilename) return null
605 return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
608 getTorrentDownloadUrl () {
609 if (!this.torrentFilename) return null
611 return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
615 if (!this.torrentFilename) return null
617 const torrentPath = getFSTorrentFilePath(this)
618 return remove(torrentPath)
619 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
622 hasSameUniqueKeysThan (other: MVideoFile) {
623 return this.fps === other.fps &&
624 this.resolution === other.resolution &&
626 (this.videoId !== null && this.videoId === other.videoId) ||
627 (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
631 withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
632 if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist })
634 return Object.assign(this, { Video: videoOrPlaylist })