X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Fmodels%2Fvideo%2Fvideo.ts;h=06c63e87c42a0a372e3c372230d9f4bdde1e98b8;hb=9c6ca37fc1512a99d420ea90707cebcd06cdc970;hp=adef37937436f0eb769bfbd4244d5d96bdc2f889;hpb=4e74e8032be8293ffe3cb3c30528d4ef7c11a798;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/models/video/video.ts b/server/models/video/video.ts index adef37937..06c63e87c 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -53,6 +53,8 @@ import { API_VERSION, CONFIG, CONSTRAINTS_FIELDS, + HLS_STREAMING_PLAYLIST_DIRECTORY, + HLS_REDUNDANCY_DIRECTORY, PREVIEWS_SIZE, REMOTE_SCHEME, STATIC_DOWNLOAD_PATHS, @@ -70,10 +72,17 @@ import { AccountVideoRateModel } from '../account/account-video-rate' import { ActorModel } from '../activitypub/actor' import { AvatarModel } from '../avatar/avatar' import { ServerModel } from '../server/server' -import { buildBlockedAccountSQL, buildTrigramSearchIndex, createSimilarityAttribute, getVideoSort, throwIfNotValid } from '../utils' +import { + buildBlockedAccountSQL, + buildTrigramSearchIndex, + buildWhereIdOrUUID, + createSimilarityAttribute, + getVideoSort, + throwIfNotValid +} from '../utils' import { TagModel } from './tag' import { VideoAbuseModel } from './video-abuse' -import { VideoChannelModel } from './video-channel' +import { VideoChannelModel, ScopeNames as VideoChannelScopeNames } from './video-channel' import { VideoCommentModel } from './video-comment' import { VideoFileModel } from './video-file' import { VideoShareModel } from './video-share' @@ -91,9 +100,11 @@ import { videoModelToFormattedDetailsJSON, videoModelToFormattedJSON } from './video-format-utils' -import * as validator from 'validator' import { UserVideoHistoryModel } from '../account/user-video-history' import { UserModel } from '../account/user' +import { VideoImportModel } from './video-import' +import { VideoStreamingPlaylistModel } from './video-streaming-playlist' +import { VideoPlaylistElementModel } from './video-playlist-element' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: Sequelize.DefineIndexesOptions[] = [ @@ -102,16 +113,52 @@ const indexes: Sequelize.DefineIndexesOptions[] = [ { fields: [ 'createdAt' ] }, { fields: [ 'publishedAt' ] }, { fields: [ 'duration' ] }, - { fields: [ 'category' ] }, - { fields: [ 'licence' ] }, - { fields: [ 'nsfw' ] }, - { fields: [ 'language' ] }, - { fields: [ 'waitTranscoding' ] }, - { fields: [ 'state' ] }, - { fields: [ 'remote' ] }, { fields: [ 'views' ] }, - { fields: [ 'likes' ] }, { fields: [ 'channelId' ] }, + { + fields: [ 'originallyPublishedAt' ], + where: { + originallyPublishedAt: { + [Sequelize.Op.ne]: null + } + } + }, + { + fields: [ 'category' ], // We don't care videos with an unknown category + where: { + category: { + [Sequelize.Op.ne]: null + } + } + }, + { + fields: [ 'licence' ], // We don't care videos with an unknown licence + where: { + licence: { + [Sequelize.Op.ne]: null + } + } + }, + { + fields: [ 'language' ], // We don't care videos with an unknown language + where: { + language: { + [Sequelize.Op.ne]: null + } + } + }, + { + fields: [ 'nsfw' ], // Most of the videos are not NSFW + where: { + nsfw: true + } + }, + { + fields: [ 'remote' ], // Only index local videos + where: { + remote: false + } + }, { fields: [ 'uuid' ], unique: true @@ -130,11 +177,16 @@ export enum ScopeNames { WITH_FILES = 'WITH_FILES', WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', WITH_BLACKLISTED = 'WITH_BLACKLISTED', - WITH_USER_HISTORY = 'WITH_USER_HISTORY' + WITH_USER_HISTORY = 'WITH_USER_HISTORY', + WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS', + WITH_USER_ID = 'WITH_USER_ID' } type ForAPIOptions = { ids: number[] + + videoPlaylistId?: number + withFiles?: boolean } @@ -142,6 +194,7 @@ type AvailableForListIDsOptions = { serverAccountId: number followerActorId: number includeLocalVideos: boolean + filter?: VideoFilter categoryOneOf?: number[] nsfw?: boolean @@ -149,71 +202,32 @@ type AvailableForListIDsOptions = { languageOneOf?: string[] tagsOneOf?: string[] tagsAllOf?: string[] + withFiles?: boolean + accountId?: number videoChannelId?: number + + videoPlaylistId?: number + trendingDays?: number - user?: UserModel + user?: UserModel, + historyOfUser?: UserModel } @Scopes({ [ ScopeNames.FOR_API ]: (options: ForAPIOptions) => { - const accountInclude = { - attributes: [ 'id', 'name' ], - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id', 'uuid', 'preferredUsername', 'url', 'serverId', 'avatarId' ], - model: ActorModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: false - }, - { - model: AvatarModel.unscoped(), - required: false - } - ] - } - ] - } - - const videoChannelInclude = { - attributes: [ 'name', 'description', 'id' ], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'uuid', 'preferredUsername', 'url', 'serverId', 'avatarId' ], - model: ActorModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: false - }, - { - model: AvatarModel.unscoped(), - required: false - } - ] - }, - accountInclude - ] - } - const query: IFindOptions = { where: { id: { [ Sequelize.Op.any ]: options.ids } }, - include: [ videoChannelInclude ] + include: [ + { + model: VideoChannelModel.scope({ method: [ VideoChannelScopeNames.SUMMARY, true ] }) + } + ] } if (options.withFiles === true) { @@ -223,6 +237,13 @@ type AvailableForListIDsOptions = { }) } + if (options.videoPlaylistId) { + query.include.push({ + model: VideoPlaylistElementModel.unscoped(), + required: true + }) + } + return query }, [ ScopeNames.AVAILABLE_FOR_LIST_IDS ]: (options: AvailableForListIDsOptions) => { @@ -274,6 +295,17 @@ type AvailableForListIDsOptions = { Object.assign(query.where, privacyWhere) } + if (options.videoPlaylistId) { + query.include.push({ + attributes: [], + model: VideoPlaylistElementModel.unscoped(), + required: true, + where: { + videoPlaylistId: options.videoPlaylistId + } + }) + } + if (options.filter || options.accountId || options.videoChannelId) { const videoChannelInclude: IIncludeOptions = { attributes: [], @@ -416,8 +448,39 @@ type AvailableForListIDsOptions = { query.subQuery = false } + if (options.historyOfUser) { + query.include.push({ + model: UserVideoHistoryModel, + required: true, + where: { + userId: options.historyOfUser.id + } + }) + + // Even if the relation is n:m, we know that a user only have 0..1 video history + // So we won't have multiple rows for the same video + // Without this, we would not be able to sort on "updatedAt" column of UserVideoHistoryModel + query.subQuery = false + } + return query }, + [ ScopeNames.WITH_USER_ID ]: { + include: [ + { + attributes: [ 'accountId' ], + model: () => VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'userId' ], + model: () => AccountModel.unscoped(), + required: true + } + ] + } + ] + }, [ ScopeNames.WITH_ACCOUNT_DETAILS ]: { include: [ { @@ -482,22 +545,55 @@ type AvailableForListIDsOptions = { } ] }, - [ ScopeNames.WITH_FILES ]: { - include: [ - { - model: () => VideoFileModel.unscoped(), - // FIXME: typings - [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join - required: false, - include: [ - { - attributes: [ 'fileUrl' ], - model: () => VideoRedundancyModel.unscoped(), - required: false - } - ] - } - ] + [ ScopeNames.WITH_FILES ]: (withRedundancies = false) => { + let subInclude: any[] = [] + + if (withRedundancies === true) { + subInclude = [ + { + attributes: [ 'fileUrl' ], + model: VideoRedundancyModel.unscoped(), + required: false + } + ] + } + + return { + include: [ + { + model: VideoFileModel.unscoped(), + // FIXME: typings + [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join + required: false, + include: subInclude + } + ] + } + }, + [ ScopeNames.WITH_STREAMING_PLAYLISTS ]: (withRedundancies = false) => { + let subInclude: any[] = [] + + if (withRedundancies === true) { + subInclude = [ + { + attributes: [ 'fileUrl' ], + model: VideoRedundancyModel.unscoped(), + required: false + } + ] + } + + return { + include: [ + { + model: VideoStreamingPlaylistModel.unscoped(), + // FIXME: typings + [ 'separate' as any ]: true, // We may have multiple streaming playlists, having multiple redundancies so let's separate this join + required: false, + include: subInclude + } + ] + } }, [ ScopeNames.WITH_SCHEDULED_UPDATE ]: { include: [ @@ -618,6 +714,10 @@ export class VideoModel extends Model { @Column commentsEnabled: boolean + @AllowNull(false) + @Column + downloadEnabled: boolean + @AllowNull(false) @Column waitTranscoding: boolean @@ -639,6 +739,11 @@ export class VideoModel extends Model { @Column publishedAt: Date + @AllowNull(true) + @Default(null) + @Column + originallyPublishedAt: Date + @ForeignKey(() => VideoChannelModel) @Column channelId: number @@ -658,6 +763,15 @@ export class VideoModel extends Model { }) Tags: TagModel[] + @HasMany(() => VideoPlaylistElementModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) + VideoPlaylistElements: VideoPlaylistElementModel[] + @HasMany(() => VideoAbuseModel, { foreignKey: { name: 'videoId', @@ -677,6 +791,16 @@ export class VideoModel extends Model { }) VideoFiles: VideoFileModel[] + @HasMany(() => VideoStreamingPlaylistModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + hooks: true, + onDelete: 'cascade' + }) + VideoStreamingPlaylists: VideoStreamingPlaylistModel[] + @HasMany(() => VideoShareModel, { foreignKey: { name: 'videoId', @@ -741,6 +865,15 @@ export class VideoModel extends Model { }) VideoBlacklist: VideoBlacklistModel + @HasOne(() => VideoImportModel, { + foreignKey: { + name: 'videoId', + allowNull: true + }, + onDelete: 'set null' + }) + VideoImport: VideoImportModel + @HasMany(() => VideoCaptionModel, { foreignKey: { name: 'videoId', @@ -793,6 +926,9 @@ export class VideoModel extends Model { tasks.push(instance.removeFile(file)) tasks.push(instance.removeTorrent(file)) }) + + // Remove playlists file + tasks.push(instance.removeStreamingPlaylist()) } // Do not wait video deletion because we could be in a transaction @@ -804,10 +940,6 @@ export class VideoModel extends Model { return undefined } - static list () { - return VideoModel.scope(ScopeNames.WITH_FILES).findAll() - } - static listLocal () { const query = { where: { @@ -815,7 +947,7 @@ export class VideoModel extends Model { } } - return VideoModel.scope(ScopeNames.WITH_FILES).findAll(query) + return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]).findAll(query) } static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { @@ -986,8 +1118,10 @@ export class VideoModel extends Model { accountId?: number, videoChannelId?: number, followerActorId?: number + videoPlaylistId?: number, trendingDays?: number, - user?: UserModel + user?: UserModel, + historyOfUser?: UserModel }, countVideos = true) { if (options.filter && options.filter === 'all-local' && !options.user.hasRight(UserRight.SEE_ALL_VIDEOS)) { throw new Error('Try to filter all-local but no user has not the see all videos right') @@ -1024,8 +1158,10 @@ export class VideoModel extends Model { withFiles: options.withFiles, accountId: options.accountId, videoChannelId: options.videoChannelId, + videoPlaylistId: options.videoPlaylistId, includeLocalVideos: options.includeLocalVideos, user: options.user, + historyOfUser: options.historyOfUser, trendingDays } @@ -1040,6 +1176,8 @@ export class VideoModel extends Model { sort?: string startDate?: string // ISO 8601 endDate?: string // ISO 8601 + originallyPublishedStartDate?: string + originallyPublishedEndDate?: string nsfw?: boolean categoryOneOf?: number[] licenceOneOf?: number[] @@ -1062,6 +1200,15 @@ export class VideoModel extends Model { whereAnd.push({ publishedAt: publishedAtRange }) } + if (options.originallyPublishedStartDate || options.originallyPublishedEndDate) { + const originallyPublishedAtRange = {} + + if (options.originallyPublishedStartDate) originallyPublishedAtRange[ Sequelize.Op.gte ] = options.originallyPublishedStartDate + if (options.originallyPublishedEndDate) originallyPublishedAtRange[ Sequelize.Op.lte ] = options.originallyPublishedEndDate + + whereAnd.push({ originallyPublishedAt: originallyPublishedAtRange }) + } + if (options.durationMin || options.durationMax) { const durationRange = {} @@ -1135,7 +1282,7 @@ export class VideoModel extends Model { } static load (id: number | string, t?: Sequelize.Transaction) { - const where = VideoModel.buildWhereIdOrUUID(id) + const where = buildWhereIdOrUUID(id) const options = { where, transaction: t @@ -1144,8 +1291,18 @@ export class VideoModel extends Model { return VideoModel.findOne(options) } + static loadWithRights (id: number | string, t?: Sequelize.Transaction) { + const where = buildWhereIdOrUUID(id) + const options = { + where, + transaction: t + } + + return VideoModel.scope([ ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_USER_ID ]).findOne(options) + } + static loadOnlyId (id: number | string, t?: Sequelize.Transaction) { - const where = VideoModel.buildWhereIdOrUUID(id) + const where = buildWhereIdOrUUID(id) const options = { attributes: [ 'id' ], @@ -1156,9 +1313,9 @@ export class VideoModel extends Model { return VideoModel.findOne(options) } - static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) { - return VideoModel.scope(ScopeNames.WITH_FILES) - .findById(id, { transaction: t, logging }) + static loadWithFiles (id: number, t?: Sequelize.Transaction, logging?: boolean) { + return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]) + .findByPk(id, { transaction: t, logging }) } static loadByUUIDWithFile (uuid: string) { @@ -1168,9 +1325,7 @@ export class VideoModel extends Model { } } - return VideoModel - .scope([ ScopeNames.WITH_FILES ]) - .findOne(options) + return VideoModel.findOne(options) } static loadByUrl (url: string, transaction?: Sequelize.Transaction) { @@ -1192,11 +1347,15 @@ export class VideoModel extends Model { transaction } - return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) + return VideoModel.scope([ + ScopeNames.WITH_ACCOUNT_DETAILS, + ScopeNames.WITH_FILES, + ScopeNames.WITH_STREAMING_PLAYLISTS + ]).findOne(query) } static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) { - const where = VideoModel.buildWhereIdOrUUID(id) + const where = buildWhereIdOrUUID(id) const options = { order: [ [ 'Tags', 'name', 'ASC' ] ], @@ -1207,9 +1366,37 @@ export class VideoModel extends Model { const scopes = [ ScopeNames.WITH_TAGS, ScopeNames.WITH_BLACKLISTED, + ScopeNames.WITH_ACCOUNT_DETAILS, + ScopeNames.WITH_SCHEDULED_UPDATE, ScopeNames.WITH_FILES, + ScopeNames.WITH_STREAMING_PLAYLISTS + ] + + if (userId) { + scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings + } + + return VideoModel + .scope(scopes) + .findOne(options) + } + + static loadForGetAPI (id: number | string, t?: Sequelize.Transaction, userId?: number) { + const where = buildWhereIdOrUUID(id) + + const options = { + order: [ [ 'Tags', 'name', 'ASC' ] ], + where, + transaction: t + } + + const scopes = [ + ScopeNames.WITH_TAGS, + ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_ACCOUNT_DETAILS, - ScopeNames.WITH_SCHEDULED_UPDATE + ScopeNames.WITH_SCHEDULED_UPDATE, + { method: [ ScopeNames.WITH_FILES, true ] } as any, // FIXME: typings + { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } as any // FIXME: typings ] if (userId) { @@ -1341,25 +1528,14 @@ export class VideoModel extends Model { } const [ count, rowsId ] = await Promise.all([ - countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined), + countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined), VideoModel.scope(idsScope).findAll(query) ]) const ids = rowsId.map(r => r.id) if (ids.length === 0) return { data: [], total: count } - // FIXME: typings - const apiScope: any[] = [ - { - method: [ ScopeNames.FOR_API, { ids, withFiles: options.withFiles } as ForAPIOptions ] - } - ] - - if (options.user) { - apiScope.push({ method: [ ScopeNames.WITH_USER_HISTORY, options.user.id ] }) - } - - const secondQuery = { + const secondQuery: IFindOptions = { offset: 0, limit: query.limit, attributes: query.attributes, @@ -1369,6 +1545,29 @@ export class VideoModel extends Model { ) ] } + + // FIXME: typing + const apiScope: any[] = [] + + if (options.user) { + apiScope.push({ method: [ ScopeNames.WITH_USER_HISTORY, options.user.id ] }) + + // Even if the relation is n:m, we know that a user only have 0..1 video history + // So we won't have multiple rows for the same video + // A subquery adds some bugs in our query so disable it + secondQuery.subQuery = false + } + + apiScope.push({ + method: [ + ScopeNames.FOR_API, { + ids, withFiles: + options.withFiles, + videoPlaylistId: options.videoPlaylistId + } as ForAPIOptions + ] + }) + const rows = await VideoModel.scope(apiScope).findAll(secondQuery) return { @@ -1397,10 +1596,6 @@ export class VideoModel extends Model { return VIDEO_STATES[ id ] || 'Unknown' } - static buildWhereIdOrUUID (id: number | string) { - return validator.isInt('' + id) ? { id } : { uuid: id } - } - getOriginalFile () { if (Array.isArray(this.VideoFiles) === false) return undefined @@ -1413,7 +1608,6 @@ export class VideoModel extends Model { } getThumbnailName () { - // We always have a copy of the thumbnail const extension = '.jpg' return this.uuid + extension } @@ -1481,6 +1675,10 @@ export class VideoModel extends Model { videoFile.infoHash = parsedTorrent.infoHash } + getWatchStaticPath () { + return '/videos/watch/' + this.uuid + } + getEmbedStaticPath () { return '/videos/embed/' + this.uuid } @@ -1552,6 +1750,14 @@ export class VideoModel extends Model { .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) } + removeStreamingPlaylist (isRedundancy = false) { + const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_STREAMING_PLAYLIST_DIRECTORY + + const filePath = join(baseDir, this.uuid) + return remove(filePath) + .catch(err => logger.warn('Cannot delete playlist directory %s.', filePath, { err })) + } + isOutdated () { if (this.isOwned()) return false @@ -1586,7 +1792,7 @@ export class VideoModel extends Model { generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { const xs = this.getTorrentUrl(videoFile, baseUrlHttp) - const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] + const announce = this.getTrackerUrls(baseUrlHttp, baseUrlWs) let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] const redundancies = videoFile.RedundancyVideos @@ -1603,6 +1809,10 @@ export class VideoModel extends Model { return magnetUtil.encode(magnetHash) } + getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) { + return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] + } + getThumbnailUrl (baseUrlHttp: string) { return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() } @@ -1626,4 +1836,8 @@ export class VideoModel extends Model { getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) } + + getBandwidthBits (videoFile: VideoFileModel) { + return Math.ceil((videoFile.size * 8) / this.duration) + } }