X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Fmodels%2Fvideo%2Fvideo-file.ts;h=1a9f4561af51747cfe11e388fbb185598e99b52a;hb=c9f27d9881ea0cc05e953117380b4b22a1376d21;hp=cacef0106e71e144a536d80bb79208972491afd5;hpb=d7a25329f9e607894d29ab342b9cb66638b56dc0;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index cacef0106..1a9f4561a 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -1,3 +1,7 @@ +import { remove } from 'fs-extra' +import memoizee from 'memoizee' +import { join } from 'path' +import { FindOptions, Op, Transaction } from 'sequelize' import { AllowNull, BelongsTo, @@ -5,13 +9,25 @@ import { CreatedAt, DataType, Default, + DefaultScope, ForeignKey, HasMany, Is, Model, + Scopes, Table, UpdatedAt } from 'sequelize-typescript' +import { Where } from 'sequelize/types/utils' +import validator from 'validator' +import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' +import { logger } from '@server/helpers/logger' +import { extractVideo } from '@server/helpers/video' +import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage' +import { getFSTorrentFilePath } from '@server/lib/paths' +import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' +import { VideoResolution, VideoStorage } from '@shared/models' +import { AttributesOnly } from '@shared/typescript-utils' import { isVideoFileExtnameValid, isVideoFileInfoHashValid, @@ -19,15 +35,69 @@ import { isVideoFileSizeValid, isVideoFPSResolutionValid } from '../../helpers/custom-validators/videos' +import { + LAZY_STATIC_PATHS, + MEMOIZE_LENGTH, + MEMOIZE_TTL, + STATIC_DOWNLOAD_PATHS, + STATIC_PATHS, + WEBSERVER +} from '../../initializers/constants' +import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file' +import { VideoRedundancyModel } from '../redundancy/video-redundancy' +import { doesExist } from '../shared' import { parseAggregateResult, throwIfNotValid } from '../utils' import { VideoModel } from './video' -import { VideoRedundancyModel } from '../redundancy/video-redundancy' import { VideoStreamingPlaylistModel } from './video-streaming-playlist' -import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize' -import { MIMETYPES } from '../../initializers/constants' -import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../typings/models/video/video-file' -import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/typings/models' +export enum ScopeNames { + WITH_VIDEO = 'WITH_VIDEO', + WITH_METADATA = 'WITH_METADATA', + WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST' +} + +@DefaultScope(() => ({ + attributes: { + exclude: [ 'metadata' ] + } +})) +@Scopes(() => ({ + [ScopeNames.WITH_VIDEO]: { + include: [ + { + model: VideoModel.unscoped(), + required: true + } + ] + }, + [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: Where } = {}) => { + return { + include: [ + { + model: VideoModel.unscoped(), + required: false, + where: options.whereVideo + }, + { + model: VideoStreamingPlaylistModel.unscoped(), + required: false, + include: [ + { + model: VideoModel.unscoped(), + required: true, + where: options.whereVideo + } + ] + } + ] + } + }, + [ScopeNames.WITH_METADATA]: { + attributes: { + include: [ 'metadata' ] + } + } +})) @Table({ tableName: 'videoFile', indexes: [ @@ -52,6 +122,16 @@ import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/typ fields: [ 'infoHash' ] }, + { + fields: [ 'torrentFilename' ], + unique: true + }, + + { + fields: [ 'filename' ], + unique: true + }, + { fields: [ 'videoId', 'resolution', 'fps' ], unique: true, @@ -72,7 +152,7 @@ import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/typ } ] }) -export class VideoFileModel extends Model { +export class VideoFileModel extends Model>> { @CreatedAt createdAt: Date @@ -94,8 +174,8 @@ export class VideoFileModel extends Model { @Column extname: string - @AllowNull(false) - @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash')) + @AllowNull(true) + @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true)) @Column infoHash: string @@ -105,10 +185,43 @@ export class VideoFileModel extends Model { @Column fps: number + @AllowNull(true) + @Column(DataType.JSONB) + metadata: any + + @AllowNull(true) + @Column + metadataUrl: string + + // Could be null for remote files + @AllowNull(true) + @Column + fileUrl: string + + // Could be null for live files + @AllowNull(true) + @Column + filename: string + + // Could be null for remote files + @AllowNull(true) + @Column + torrentUrl: string + + // Could be null for live files + @AllowNull(true) + @Column + torrentFilename: string + @ForeignKey(() => VideoModel) @Column videoId: number + @AllowNull(false) + @Default(VideoStorage.FILE_SYSTEM) + @Column + storage: VideoStorage + @BelongsTo(() => VideoModel, { foreignKey: { allowNull: true @@ -138,29 +251,88 @@ export class VideoFileModel extends Model { }) RedundancyVideos: VideoRedundancyModel[] + static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, { + promise: true, + max: MEMOIZE_LENGTH.INFO_HASH_EXISTS, + maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS + }) + static doesInfohashExist (infoHash: string) { const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' - const options = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - bind: { infoHash }, - raw: true + + return doesExist(query, { infoHash }) + } + + static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) { + const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID) + + return !!videoFile + } + + static async doesOwnedTorrentFileExist (filename: string) { + const query = 'SELECT 1 FROM "videoFile" ' + + 'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' + + 'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' + + 'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' + + 'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1' + + return doesExist(query, { filename }) + } + + static async doesOwnedWebTorrentVideoFileExist (filename: string) { + const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + + `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1` + + return doesExist(query, { filename }) + } + + static loadByFilename (filename: string) { + const query = { + where: { + filename + } + } + + return VideoFileModel.findOne(query) + } + + static loadWithVideoOrPlaylistByTorrentFilename (filename: string) { + const query = { + where: { + torrentFilename: filename + } } - return VideoModel.sequelize.query(query, options) - .then(results => results.length === 1) + return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query) + } + + static loadWithMetadata (id: number) { + return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id) } static loadWithVideo (id: number) { + return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id) + } + + static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) { + const whereVideo = validator.isUUID(videoIdOrUUID + '') + ? { uuid: videoIdOrUUID } + : { id: videoIdOrUUID } + const options = { - include: [ - { - model: VideoModel.unscoped(), - required: true - } - ] + where: { + id + } } - return VideoFileModel.findByPk(id, options) + return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] }) + .findOne(options) + .then(file => { + // We used `required: false` so check we have at least a video or a streaming playlist + if (!file.Video && !file.VideoStreamingPlaylist) return null + + return file + }) } static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) { @@ -187,10 +359,11 @@ export class VideoFileModel extends Model { } static getStats () { - const query: FindOptions = { + const webtorrentFilesQuery: FindOptions = { include: [ { attributes: [], + required: true, model: VideoModel.unscoped(), where: { remote: false @@ -199,10 +372,32 @@ export class VideoFileModel extends Model { ] } - return VideoFileModel.aggregate('size', 'SUM', query) - .then(result => ({ - totalLocalVideoFilesSize: parseAggregateResult(result) - })) + const hlsFilesQuery: FindOptions = { + include: [ + { + attributes: [], + required: true, + model: VideoStreamingPlaylistModel.unscoped(), + include: [ + { + attributes: [], + model: VideoModel.unscoped(), + required: true, + where: { + remote: false + } + } + ] + } + ] + } + + return Promise.all([ + VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery), + VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery) + ]).then(([ webtorrentResult, hlsResult ]) => ({ + totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult) + })) } // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes @@ -229,14 +424,107 @@ export class VideoFileModel extends Model { return element.save({ transaction }) } + static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) { + const options = { + where: { videoStreamingPlaylistId } + } + + return VideoFileModel.destroy(options) + } + + hasTorrent () { + return this.infoHash && this.torrentFilename + } + getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo { if (this.videoId) return (this as MVideoFileVideo).Video return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist } + getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo { + return extractVideo(this.getVideoOrStreamingPlaylist()) + } + isAudio () { - return !!MIMETYPES.AUDIO.EXT_MIMETYPE[this.extname] + return this.resolution === VideoResolution.H_NOVIDEO + } + + isLive () { + return this.size === -1 + } + + isHLS () { + return !!this.videoStreamingPlaylistId + } + + getObjectStorageUrl () { + if (this.isHLS()) { + return getHLSPublicFileUrl(this.fileUrl) + } + + return getWebTorrentPublicFileUrl(this.fileUrl) + } + + getFileUrl (video: MVideo) { + if (this.storage === VideoStorage.OBJECT_STORAGE) { + return this.getObjectStorageUrl() + } + + if (!this.Video) this.Video = video as VideoModel + if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) + + return this.fileUrl + } + + getFileStaticPath (video: MVideo) { + if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename) + + return join(STATIC_PATHS.WEBSEED, this.filename) + } + + getFileDownloadUrl (video: MVideoWithHost) { + const path = this.isHLS() + ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`) + : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`) + + if (video.isOwned()) return WEBSERVER.URL + path + + // FIXME: don't guess remote URL + return buildRemoteVideoBaseUrl(video, path) + } + + getRemoteTorrentUrl (video: MVideo) { + if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`) + + return this.torrentUrl + } + + // We proxify torrent requests so use a local URL + getTorrentUrl () { + if (!this.torrentFilename) return null + + return WEBSERVER.URL + this.getTorrentStaticPath() + } + + getTorrentStaticPath () { + if (!this.torrentFilename) return null + + return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename) + } + + getTorrentDownloadUrl () { + if (!this.torrentFilename) return null + + return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename) + } + + removeTorrent () { + if (!this.torrentFilename) return null + + const torrentPath = getFSTorrentFilePath(this) + return remove(torrentPath) + .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) } hasSameUniqueKeysThan (other: MVideoFile) { @@ -247,4 +535,10 @@ export class VideoFileModel extends Model { (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId) ) } + + withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) { + if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist }) + + return Object.assign(this, { Video: videoOrPlaylist }) + } }