X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Fmodels%2Fvideo%2Fvideo.ts;h=06c63e87c42a0a372e3c372230d9f4bdde1e98b8;hb=9c6ca37fc1512a99d420ea90707cebcd06cdc970;hp=27c631dcd7e071a8f9c9a163feafb4ca0518d803;hpb=c48e82b5e0478434de30626d14594a97f2402e7c;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 27c631dcd..06c63e87c 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1,8 +1,8 @@ import * as Bluebird from 'bluebird' -import { map, maxBy } from 'lodash' +import { maxBy } from 'lodash' import * as magnetUtil from 'magnet-uri' import * as parseTorrent from 'parse-torrent' -import { extname, join } from 'path' +import { join } from 'path' import * as Sequelize from 'sequelize' import { AllowNull, @@ -27,7 +27,7 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { ActivityUrlObject, VideoPrivacy, VideoResolution, VideoState } from '../../../shared' +import { UserRight, VideoPrivacy, VideoState } from '../../../shared' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' import { VideoFilter } from '../../../shared/models/videos/video-query.type' @@ -45,7 +45,7 @@ import { isVideoStateValid, isVideoSupportValid } from '../../helpers/custom-validators/videos' -import { generateImageFromVideoFile, getVideoFileFPS, getVideoFileResolution, transcode } from '../../helpers/ffmpeg-utils' +import { generateImageFromVideoFile, getVideoFileResolution } from '../../helpers/ffmpeg-utils' import { logger } from '../../helpers/logger' import { getServerActor } from '../../helpers/utils' import { @@ -53,34 +53,36 @@ import { API_VERSION, CONFIG, CONSTRAINTS_FIELDS, + HLS_STREAMING_PLAYLIST_DIRECTORY, + HLS_REDUNDANCY_DIRECTORY, PREVIEWS_SIZE, REMOTE_SCHEME, STATIC_DOWNLOAD_PATHS, STATIC_PATHS, THUMBNAILS_SIZE, VIDEO_CATEGORIES, - VIDEO_EXT_MIMETYPE, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES, VIDEO_STATES } from '../../initializers' -import { - getVideoCommentsActivityPubUrl, - getVideoDislikesActivityPubUrl, - getVideoLikesActivityPubUrl, - getVideoSharesActivityPubUrl -} from '../../lib/activitypub' import { sendDeleteVideo } from '../../lib/activitypub/send' import { AccountModel } from '../account/account' import { AccountVideoRateModel } from '../account/account-video-rate' import { ActorModel } from '../activitypub/actor' import { AvatarModel } from '../avatar/avatar' import { ServerModel } from '../server/server' -import { buildTrigramSearchIndex, createSimilarityAttribute, getVideoSort, throwIfNotValid } from '../utils' +import { + buildBlockedAccountSQL, + buildTrigramSearchIndex, + buildWhereIdOrUUID, + createSimilarityAttribute, + getVideoSort, + throwIfNotValid +} from '../utils' import { TagModel } from './tag' import { VideoAbuseModel } from './video-abuse' -import { VideoChannelModel } from './video-channel' +import { VideoChannelModel, ScopeNames as VideoChannelScopeNames } from './video-channel' import { VideoCommentModel } from './video-comment' import { VideoFileModel } from './video-file' import { VideoShareModel } from './video-share' @@ -88,9 +90,21 @@ import { VideoTagModel } from './video-tag' import { ScheduleVideoUpdateModel } from './schedule-video-update' import { VideoCaptionModel } from './video-caption' import { VideoBlacklistModel } from './video-blacklist' -import { copy, remove, rename, stat, writeFile } from 'fs-extra' +import { remove, writeFile } from 'fs-extra' import { VideoViewModel } from './video-views' import { VideoRedundancyModel } from '../redundancy/video-redundancy' +import { + videoFilesModelToFormattedJSON, + VideoFormattingJSONOptions, + videoModelToActivityPubObject, + videoModelToFormattedDetailsJSON, + videoModelToFormattedJSON +} from './video-format-utils' +import { UserVideoHistoryModel } from '../account/user-video-history' +import { UserModel } from '../account/user' +import { VideoImportModel } from './video-import' +import { VideoStreamingPlaylistModel } from './video-streaming-playlist' +import { VideoPlaylistElementModel } from './video-playlist-element' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: Sequelize.DefineIndexesOptions[] = [ @@ -99,16 +113,52 @@ const indexes: Sequelize.DefineIndexesOptions[] = [ { fields: [ 'createdAt' ] }, { fields: [ 'publishedAt' ] }, { fields: [ 'duration' ] }, - { fields: [ 'category' ] }, - { fields: [ 'licence' ] }, - { fields: [ 'nsfw' ] }, - { fields: [ 'language' ] }, - { fields: [ 'waitTranscoding' ] }, - { fields: [ 'state' ] }, - { fields: [ 'remote' ] }, { fields: [ 'views' ] }, - { fields: [ 'likes' ] }, { fields: [ 'channelId' ] }, + { + fields: [ 'originallyPublishedAt' ], + where: { + originallyPublishedAt: { + [Sequelize.Op.ne]: null + } + } + }, + { + fields: [ 'category' ], // We don't care videos with an unknown category + where: { + category: { + [Sequelize.Op.ne]: null + } + } + }, + { + fields: [ 'licence' ], // We don't care videos with an unknown licence + where: { + licence: { + [Sequelize.Op.ne]: null + } + } + }, + { + fields: [ 'language' ], // We don't care videos with an unknown language + where: { + language: { + [Sequelize.Op.ne]: null + } + } + }, + { + fields: [ 'nsfw' ], // Most of the videos are not NSFW + where: { + nsfw: true + } + }, + { + fields: [ 'remote' ], // Only index local videos + where: { + remote: false + } + }, { fields: [ 'uuid' ], unique: true @@ -126,17 +176,25 @@ export enum ScopeNames { WITH_TAGS = 'WITH_TAGS', WITH_FILES = 'WITH_FILES', WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', - WITH_BLACKLISTED = 'WITH_BLACKLISTED' + WITH_BLACKLISTED = 'WITH_BLACKLISTED', + WITH_USER_HISTORY = 'WITH_USER_HISTORY', + WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS', + WITH_USER_ID = 'WITH_USER_ID' } type ForAPIOptions = { ids: number[] + + videoPlaylistId?: number + withFiles?: boolean } type AvailableForListIDsOptions = { - actorId: number + serverAccountId: number + followerActorId: number includeLocalVideos: boolean + filter?: VideoFilter categoryOneOf?: number[] nsfw?: boolean @@ -144,70 +202,32 @@ type AvailableForListIDsOptions = { languageOneOf?: string[] tagsOneOf?: string[] tagsAllOf?: string[] + withFiles?: boolean + accountId?: number videoChannelId?: number + + videoPlaylistId?: number + trendingDays?: number + user?: UserModel, + historyOfUser?: UserModel } @Scopes({ [ ScopeNames.FOR_API ]: (options: ForAPIOptions) => { - const accountInclude = { - attributes: [ 'id', 'name' ], - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id', 'uuid', 'preferredUsername', 'url', 'serverId', 'avatarId' ], - model: ActorModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: false - }, - { - model: AvatarModel.unscoped(), - required: false - } - ] - } - ] - } - - const videoChannelInclude = { - attributes: [ 'name', 'description', 'id' ], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'uuid', 'preferredUsername', 'url', 'serverId', 'avatarId' ], - model: ActorModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: false - }, - { - model: AvatarModel.unscoped(), - required: false - } - ] - }, - accountInclude - ] - } - const query: IFindOptions = { where: { id: { [ Sequelize.Op.any ]: options.ids } }, - include: [ videoChannelInclude ] + include: [ + { + model: VideoChannelModel.scope({ method: [ VideoChannelScopeNames.SUMMARY, true ] }) + } + ] } if (options.withFiles === true) { @@ -217,10 +237,18 @@ type AvailableForListIDsOptions = { }) } + if (options.videoPlaylistId) { + query.include.push({ + model: VideoPlaylistElementModel.unscoped(), + required: true + }) + } + return query }, [ ScopeNames.AVAILABLE_FOR_LIST_IDS ]: (options: AvailableForListIDsOptions) => { const query: IFindOptions = { + raw: true, attributes: [ 'id' ], where: { id: { @@ -232,6 +260,22 @@ type AvailableForListIDsOptions = { } ] }, + channelId: { + [ Sequelize.Op.notIn ]: Sequelize.literal( + '(' + + 'SELECT id FROM "videoChannel" WHERE "accountId" IN (' + + buildBlockedAccountSQL(options.serverAccountId, options.user ? options.user.Account.id : undefined) + + ')' + + ')' + ) + } + }, + include: [] + } + + // Only list public/published videos + if (!options.filter || options.filter !== 'all-local') { + const privacyWhere = { // Always list public videos privacy: VideoPrivacy.PUBLIC, // Always list published videos, or videos that are being transcoded but on which we don't want to wait for transcoding @@ -246,8 +290,20 @@ type AvailableForListIDsOptions = { } } ] - }, - include: [] + } + + Object.assign(query.where, privacyWhere) + } + + if (options.videoPlaylistId) { + query.include.push({ + attributes: [], + model: VideoPlaylistElementModel.unscoped(), + required: true, + where: { + videoPlaylistId: options.videoPlaylistId + } + }) } if (options.filter || options.accountId || options.videoChannelId) { @@ -291,7 +347,7 @@ type AvailableForListIDsOptions = { query.include.push(videoChannelInclude) } - if (options.actorId) { + if (options.followerActorId) { let localVideosReq = '' if (options.includeLocalVideos === true) { localVideosReq = ' UNION ALL ' + @@ -303,7 +359,7 @@ type AvailableForListIDsOptions = { } // Force actorId to be a number to avoid SQL injections - const actorIdNumber = parseInt(options.actorId.toString(), 10) + const actorIdNumber = parseInt(options.followerActorId.toString(), 10) query.where[ 'id' ][ Sequelize.Op.and ].push({ [ Sequelize.Op.in ]: Sequelize.literal( '(' + @@ -387,22 +443,44 @@ type AvailableForListIDsOptions = { } if (options.trendingDays) { + query.include.push(VideoModel.buildTrendingQuery(options.trendingDays)) + + query.subQuery = false + } + + if (options.historyOfUser) { query.include.push({ - attributes: [], - model: VideoViewModel, - required: false, + model: UserVideoHistoryModel, + required: true, where: { - startDate: { - [ Sequelize.Op.gte ]: new Date(new Date().getTime() - (24 * 3600 * 1000) * options.trendingDays) - } + userId: options.historyOfUser.id } }) + // Even if the relation is n:m, we know that a user only have 0..1 video history + // So we won't have multiple rows for the same video + // Without this, we would not be able to sort on "updatedAt" column of UserVideoHistoryModel query.subQuery = false } return query }, + [ ScopeNames.WITH_USER_ID ]: { + include: [ + { + attributes: [ 'accountId' ], + model: () => VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'userId' ], + model: () => AccountModel.unscoped(), + required: true + } + ] + } + ] + }, [ ScopeNames.WITH_ACCOUNT_DETAILS ]: { include: [ { @@ -467,19 +545,55 @@ type AvailableForListIDsOptions = { } ] }, - [ ScopeNames.WITH_FILES ]: { - include: [ - { - model: () => VideoFileModel.unscoped(), - required: false, - include: [ - { - model: () => VideoRedundancyModel.unscoped(), - required: false - } - ] - } - ] + [ ScopeNames.WITH_FILES ]: (withRedundancies = false) => { + let subInclude: any[] = [] + + if (withRedundancies === true) { + subInclude = [ + { + attributes: [ 'fileUrl' ], + model: VideoRedundancyModel.unscoped(), + required: false + } + ] + } + + return { + include: [ + { + model: VideoFileModel.unscoped(), + // FIXME: typings + [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join + required: false, + include: subInclude + } + ] + } + }, + [ ScopeNames.WITH_STREAMING_PLAYLISTS ]: (withRedundancies = false) => { + let subInclude: any[] = [] + + if (withRedundancies === true) { + subInclude = [ + { + attributes: [ 'fileUrl' ], + model: VideoRedundancyModel.unscoped(), + required: false + } + ] + } + + return { + include: [ + { + model: VideoStreamingPlaylistModel.unscoped(), + // FIXME: typings + [ 'separate' as any ]: true, // We may have multiple streaming playlists, having multiple redundancies so let's separate this join + required: false, + include: subInclude + } + ] + } }, [ ScopeNames.WITH_SCHEDULED_UPDATE ]: { include: [ @@ -488,6 +602,20 @@ type AvailableForListIDsOptions = { required: false } ] + }, + [ ScopeNames.WITH_USER_HISTORY ]: (userId: number) => { + return { + include: [ + { + attributes: [ 'currentTime' ], + model: UserVideoHistoryModel.unscoped(), + required: false, + where: { + userId + } + } + ] + } } }) @Table({ @@ -586,6 +714,10 @@ export class VideoModel extends Model { @Column commentsEnabled: boolean + @AllowNull(false) + @Column + downloadEnabled: boolean + @AllowNull(false) @Column waitTranscoding: boolean @@ -607,6 +739,11 @@ export class VideoModel extends Model { @Column publishedAt: Date + @AllowNull(true) + @Default(null) + @Column + originallyPublishedAt: Date + @ForeignKey(() => VideoChannelModel) @Column channelId: number @@ -626,6 +763,15 @@ export class VideoModel extends Model { }) Tags: TagModel[] + @HasMany(() => VideoPlaylistElementModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) + VideoPlaylistElements: VideoPlaylistElementModel[] + @HasMany(() => VideoAbuseModel, { foreignKey: { name: 'videoId', @@ -645,6 +791,16 @@ export class VideoModel extends Model { }) VideoFiles: VideoFileModel[] + @HasMany(() => VideoStreamingPlaylistModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + hooks: true, + onDelete: 'cascade' + }) + VideoStreamingPlaylists: VideoStreamingPlaylistModel[] + @HasMany(() => VideoShareModel, { foreignKey: { name: 'videoId', @@ -678,11 +834,19 @@ export class VideoModel extends Model { name: 'videoId', allowNull: false }, - onDelete: 'cascade', - hooks: true + onDelete: 'cascade' }) VideoViews: VideoViewModel[] + @HasMany(() => UserVideoHistoryModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) + UserVideoHistories: UserVideoHistoryModel[] + @HasOne(() => ScheduleVideoUpdateModel, { foreignKey: { name: 'videoId', @@ -701,6 +865,15 @@ export class VideoModel extends Model { }) VideoBlacklist: VideoBlacklistModel + @HasOne(() => VideoImportModel, { + foreignKey: { + name: 'videoId', + allowNull: true + }, + onDelete: 'set null' + }) + VideoImport: VideoImportModel + @HasMany(() => VideoCaptionModel, { foreignKey: { name: 'videoId', @@ -753,6 +926,9 @@ export class VideoModel extends Model { tasks.push(instance.removeFile(file)) tasks.push(instance.removeTorrent(file)) }) + + // Remove playlists file + tasks.push(instance.removeStreamingPlaylist()) } // Do not wait video deletion because we could be in a transaction @@ -764,8 +940,14 @@ export class VideoModel extends Model { return undefined } - static list () { - return VideoModel.scope(ScopeNames.WITH_FILES).findAll() + static listLocal () { + const query = { + where: { + remote: false + } + } + + return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]).findAll(query) } static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { @@ -935,9 +1117,16 @@ export class VideoModel extends Model { filter?: VideoFilter, accountId?: number, videoChannelId?: number, - actorId?: number - trendingDays?: number - }) { + followerActorId?: number + videoPlaylistId?: number, + trendingDays?: number, + user?: UserModel, + historyOfUser?: UserModel + }, countVideos = true) { + if (options.filter && options.filter === 'all-local' && !options.user.hasRight(UserRight.SEE_ALL_VIDEOS)) { + throw new Error('Try to filter all-local but no user has not the see all videos right') + } + const query: IFindOptions = { offset: options.start, limit: options.count, @@ -951,11 +1140,14 @@ export class VideoModel extends Model { query.group = 'VideoModel.id' } - // actorId === null has a meaning, so just check undefined - const actorId = options.actorId !== undefined ? options.actorId : (await getServerActor()).id + const serverActor = await getServerActor() + + // followerActorId === null has a meaning, so just check undefined + const followerActorId = options.followerActorId !== undefined ? options.followerActorId : serverActor.id const queryOptions = { - actorId, + followerActorId, + serverAccountId: serverActor.Account.id, nsfw: options.nsfw, categoryOneOf: options.categoryOneOf, licenceOneOf: options.licenceOneOf, @@ -966,11 +1158,14 @@ export class VideoModel extends Model { withFiles: options.withFiles, accountId: options.accountId, videoChannelId: options.videoChannelId, + videoPlaylistId: options.videoPlaylistId, includeLocalVideos: options.includeLocalVideos, + user: options.user, + historyOfUser: options.historyOfUser, trendingDays } - return VideoModel.getAvailableForApi(query, queryOptions) + return VideoModel.getAvailableForApi(query, queryOptions, countVideos) } static async searchAndPopulateAccountAndServer (options: { @@ -981,6 +1176,8 @@ export class VideoModel extends Model { sort?: string startDate?: string // ISO 8601 endDate?: string // ISO 8601 + originallyPublishedStartDate?: string + originallyPublishedEndDate?: string nsfw?: boolean categoryOneOf?: number[] licenceOneOf?: number[] @@ -989,6 +1186,8 @@ export class VideoModel extends Model { tagsAllOf?: string[] durationMin?: number // seconds durationMax?: number // seconds + user?: UserModel, + filter?: VideoFilter }) { const whereAnd = [] @@ -1001,6 +1200,15 @@ export class VideoModel extends Model { whereAnd.push({ publishedAt: publishedAtRange }) } + if (options.originallyPublishedStartDate || options.originallyPublishedEndDate) { + const originallyPublishedAtRange = {} + + if (options.originallyPublishedStartDate) originallyPublishedAtRange[ Sequelize.Op.gte ] = options.originallyPublishedStartDate + if (options.originallyPublishedEndDate) originallyPublishedAtRange[ Sequelize.Op.lte ] = options.originallyPublishedEndDate + + whereAnd.push({ originallyPublishedAt: originallyPublishedAtRange }) + } + if (options.durationMin || options.durationMax) { const durationRange = {} @@ -1057,82 +1265,146 @@ export class VideoModel extends Model { const serverActor = await getServerActor() const queryOptions = { - actorId: serverActor.id, + followerActorId: serverActor.id, + serverAccountId: serverActor.Account.id, includeLocalVideos: options.includeLocalVideos, nsfw: options.nsfw, categoryOneOf: options.categoryOneOf, licenceOneOf: options.licenceOneOf, languageOneOf: options.languageOneOf, tagsOneOf: options.tagsOneOf, - tagsAllOf: options.tagsAllOf + tagsAllOf: options.tagsAllOf, + user: options.user, + filter: options.filter } return VideoModel.getAvailableForApi(query, queryOptions) } - static load (id: number, t?: Sequelize.Transaction) { - const options = t ? { transaction: t } : undefined + static load (id: number | string, t?: Sequelize.Transaction) { + const where = buildWhereIdOrUUID(id) + const options = { + where, + transaction: t + } - return VideoModel.findById(id, options) + return VideoModel.findOne(options) } - static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) { - const query: IFindOptions = { - where: { - url - } + static loadWithRights (id: number | string, t?: Sequelize.Transaction) { + const where = buildWhereIdOrUUID(id) + const options = { + where, + transaction: t } - if (t !== undefined) query.transaction = t - - return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) + return VideoModel.scope([ ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_USER_ID ]).findOne(options) } - static loadAndPopulateAccountAndServerAndTags (id: number) { + static loadOnlyId (id: number | string, t?: Sequelize.Transaction) { + const where = buildWhereIdOrUUID(id) + const options = { - order: [ [ 'Tags', 'name', 'ASC' ] ] + attributes: [ 'id' ], + where, + transaction: t } - return VideoModel - .scope([ - ScopeNames.WITH_TAGS, - ScopeNames.WITH_BLACKLISTED, - ScopeNames.WITH_FILES, - ScopeNames.WITH_ACCOUNT_DETAILS, - ScopeNames.WITH_SCHEDULED_UPDATE - ]) - .findById(id, options) + return VideoModel.findOne(options) + } + + static loadWithFiles (id: number, t?: Sequelize.Transaction, logging?: boolean) { + return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]) + .findByPk(id, { transaction: t, logging }) } - static loadByUUID (uuid: string) { + static loadByUUIDWithFile (uuid: string) { const options = { where: { uuid } } + return VideoModel.findOne(options) + } + + static loadByUrl (url: string, transaction?: Sequelize.Transaction) { + const query: IFindOptions = { + where: { + url + }, + transaction + } + + return VideoModel.findOne(query) + } + + static loadByUrlAndPopulateAccount (url: string, transaction?: Sequelize.Transaction) { + const query: IFindOptions = { + where: { + url + }, + transaction + } + + return VideoModel.scope([ + ScopeNames.WITH_ACCOUNT_DETAILS, + ScopeNames.WITH_FILES, + ScopeNames.WITH_STREAMING_PLAYLISTS + ]).findOne(query) + } + + static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) { + const where = buildWhereIdOrUUID(id) + + const options = { + order: [ [ 'Tags', 'name', 'ASC' ] ], + where, + transaction: t + } + + const scopes = [ + ScopeNames.WITH_TAGS, + ScopeNames.WITH_BLACKLISTED, + ScopeNames.WITH_ACCOUNT_DETAILS, + ScopeNames.WITH_SCHEDULED_UPDATE, + ScopeNames.WITH_FILES, + ScopeNames.WITH_STREAMING_PLAYLISTS + ] + + if (userId) { + scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings + } + return VideoModel - .scope([ ScopeNames.WITH_FILES ]) + .scope(scopes) .findOne(options) } - static loadByUUIDAndPopulateAccountAndServerAndTags (uuid: string, t?: Sequelize.Transaction) { + static loadForGetAPI (id: number | string, t?: Sequelize.Transaction, userId?: number) { + const where = buildWhereIdOrUUID(id) + const options = { order: [ [ 'Tags', 'name', 'ASC' ] ], - where: { - uuid - }, + where, transaction: t } + const scopes = [ + ScopeNames.WITH_TAGS, + ScopeNames.WITH_BLACKLISTED, + ScopeNames.WITH_ACCOUNT_DETAILS, + ScopeNames.WITH_SCHEDULED_UPDATE, + { method: [ ScopeNames.WITH_FILES, true ] } as any, // FIXME: typings + { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } as any // FIXME: typings + ] + + if (userId) { + scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings + } + return VideoModel - .scope([ - ScopeNames.WITH_TAGS, - ScopeNames.WITH_BLACKLISTED, - ScopeNames.WITH_FILES, - ScopeNames.WITH_ACCOUNT_DETAILS, - ScopeNames.WITH_SCHEDULED_UPDATE - ]) + .scope(scopes) .findOne(options) } @@ -1168,8 +1440,34 @@ export class VideoModel extends Model { }) } + static checkVideoHasInstanceFollow (videoId: number, followerActorId: number) { + // Instances only share videos + const query = 'SELECT 1 FROM "videoShare" ' + + 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' + + 'WHERE "actorFollow"."actorId" = $followerActorId AND "videoShare"."videoId" = $videoId ' + + 'LIMIT 1' + + const options = { + type: Sequelize.QueryTypes.SELECT, + bind: { followerActorId, videoId }, + raw: true + } + + return VideoModel.sequelize.query(query, options) + .then(results => results.length === 1) + } + // threshold corresponds to how many video the field should have to be returned - static getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { + static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { + const serverActor = await getServerActor() + const followerActorId = serverActor.id + + const scopeOptions: AvailableForListIDsOptions = { + serverAccountId: serverActor.Account.id, + followerActorId, + includeLocalVideos: true + } + const query: IFindOptions = { attributes: [ field ], limit: count, @@ -1177,22 +1475,30 @@ export class VideoModel extends Model { having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), { [ Sequelize.Op.gte ]: threshold }) as any, // FIXME: typings - where: { - [ field ]: { - [ Sequelize.Op.not ]: null - }, - privacy: VideoPrivacy.PUBLIC, - state: VideoState.PUBLISHED - }, order: [ this.sequelize.random() ] } - return VideoModel.findAll(query) + return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, scopeOptions ] }) + .findAll(query) .then(rows => rows.map(r => r[ field ])) } + static buildTrendingQuery (trendingDays: number) { + return { + attributes: [], + subQuery: false, + model: VideoViewModel, + required: false, + where: { + startDate: { + [ Sequelize.Op.gte ]: new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays) + } + } + } + } + private static buildActorWhereWithFilter (filter?: VideoFilter) { - if (filter && filter === 'local') { + if (filter && (filter === 'local' || filter === 'all-local')) { return { serverId: null } @@ -1201,7 +1507,11 @@ export class VideoModel extends Model { return {} } - private static async getAvailableForApi (query: IFindOptions, options: AvailableForListIDsOptions) { + private static async getAvailableForApi ( + query: IFindOptions, + options: AvailableForListIDsOptions, + countVideos = true + ) { const idsScope = { method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, options @@ -1218,18 +1528,14 @@ export class VideoModel extends Model { } const [ count, rowsId ] = await Promise.all([ - VideoModel.scope(countScope).count(countQuery), + countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined), VideoModel.scope(idsScope).findAll(query) ]) const ids = rowsId.map(r => r.id) if (ids.length === 0) return { data: [], total: count } - const apiScope = { - method: [ ScopeNames.FOR_API, { ids, withFiles: options.withFiles } as ForAPIOptions ] - } - - const secondQuery = { + const secondQuery: IFindOptions = { offset: 0, limit: query.limit, attributes: query.attributes, @@ -1239,6 +1545,29 @@ export class VideoModel extends Model { ) ] } + + // FIXME: typing + const apiScope: any[] = [] + + if (options.user) { + apiScope.push({ method: [ ScopeNames.WITH_USER_HISTORY, options.user.id ] }) + + // Even if the relation is n:m, we know that a user only have 0..1 video history + // So we won't have multiple rows for the same video + // A subquery adds some bugs in our query so disable it + secondQuery.subQuery = false + } + + apiScope.push({ + method: [ + ScopeNames.FOR_API, { + ids, withFiles: + options.withFiles, + videoPlaylistId: options.videoPlaylistId + } as ForAPIOptions + ] + }) + const rows = await VideoModel.scope(apiScope).findAll(secondQuery) return { @@ -1247,23 +1576,23 @@ export class VideoModel extends Model { } } - private static getCategoryLabel (id: number) { + static getCategoryLabel (id: number) { return VIDEO_CATEGORIES[ id ] || 'Misc' } - private static getLicenceLabel (id: number) { + static getLicenceLabel (id: number) { return VIDEO_LICENCES[ id ] || 'Unknown' } - private static getLanguageLabel (id: string) { + static getLanguageLabel (id: string) { return VIDEO_LANGUAGES[ id ] || 'Unknown' } - private static getPrivacyLabel (id: number) { + static getPrivacyLabel (id: number) { return VIDEO_PRIVACIES[ id ] || 'Unknown' } - private static getStateLabel (id: number) { + static getStateLabel (id: number) { return VIDEO_STATES[ id ] || 'Unknown' } @@ -1279,7 +1608,6 @@ export class VideoModel extends Model { } getThumbnailName () { - // We always have a copy of the thumbnail const extension = '.jpg' return this.uuid + extension } @@ -1347,6 +1675,10 @@ export class VideoModel extends Model { videoFile.infoHash = parsedTorrent.infoHash } + getWatchStaticPath () { + return '/videos/watch/' + this.uuid + } + getEmbedStaticPath () { return '/videos/embed/' + this.uuid } @@ -1359,273 +1691,20 @@ export class VideoModel extends Model { return join(STATIC_PATHS.PREVIEWS, this.getPreviewName()) } - toFormattedJSON (options?: { - additionalAttributes: { - state?: boolean, - waitTranscoding?: boolean, - scheduledUpdate?: boolean, - blacklistInfo?: boolean - } - }): Video { - const formattedAccount = this.VideoChannel.Account.toFormattedJSON() - const formattedVideoChannel = this.VideoChannel.toFormattedJSON() - - const videoObject: Video = { - id: this.id, - uuid: this.uuid, - name: this.name, - category: { - id: this.category, - label: VideoModel.getCategoryLabel(this.category) - }, - licence: { - id: this.licence, - label: VideoModel.getLicenceLabel(this.licence) - }, - language: { - id: this.language, - label: VideoModel.getLanguageLabel(this.language) - }, - privacy: { - id: this.privacy, - label: VideoModel.getPrivacyLabel(this.privacy) - }, - nsfw: this.nsfw, - description: this.getTruncatedDescription(), - isLocal: this.isOwned(), - duration: this.duration, - views: this.views, - likes: this.likes, - dislikes: this.dislikes, - thumbnailPath: this.getThumbnailStaticPath(), - previewPath: this.getPreviewStaticPath(), - embedPath: this.getEmbedStaticPath(), - createdAt: this.createdAt, - updatedAt: this.updatedAt, - publishedAt: this.publishedAt, - account: { - id: formattedAccount.id, - uuid: formattedAccount.uuid, - name: formattedAccount.name, - displayName: formattedAccount.displayName, - url: formattedAccount.url, - host: formattedAccount.host, - avatar: formattedAccount.avatar - }, - channel: { - id: formattedVideoChannel.id, - uuid: formattedVideoChannel.uuid, - name: formattedVideoChannel.name, - displayName: formattedVideoChannel.displayName, - url: formattedVideoChannel.url, - host: formattedVideoChannel.host, - avatar: formattedVideoChannel.avatar - } - } - - if (options) { - if (options.additionalAttributes.state === true) { - videoObject.state = { - id: this.state, - label: VideoModel.getStateLabel(this.state) - } - } - - if (options.additionalAttributes.waitTranscoding === true) { - videoObject.waitTranscoding = this.waitTranscoding - } - - if (options.additionalAttributes.scheduledUpdate === true && this.ScheduleVideoUpdate) { - videoObject.scheduledUpdate = { - updateAt: this.ScheduleVideoUpdate.updateAt, - privacy: this.ScheduleVideoUpdate.privacy || undefined - } - } - - if (options.additionalAttributes.blacklistInfo === true) { - videoObject.blacklisted = !!this.VideoBlacklist - videoObject.blacklistedReason = this.VideoBlacklist ? this.VideoBlacklist.reason : null - } - } - - return videoObject + toFormattedJSON (options?: VideoFormattingJSONOptions): Video { + return videoModelToFormattedJSON(this, options) } toFormattedDetailsJSON (): VideoDetails { - const formattedJson = this.toFormattedJSON({ - additionalAttributes: { - scheduledUpdate: true, - blacklistInfo: true - } - }) - - const detailsJson = { - support: this.support, - descriptionPath: this.getDescriptionPath(), - channel: this.VideoChannel.toFormattedJSON(), - account: this.VideoChannel.Account.toFormattedJSON(), - tags: map(this.Tags, 'name'), - commentsEnabled: this.commentsEnabled, - waitTranscoding: this.waitTranscoding, - state: { - id: this.state, - label: VideoModel.getStateLabel(this.state) - }, - files: [] - } - - // Format and sort video files - detailsJson.files = this.getFormattedVideoFilesJSON() - - return Object.assign(formattedJson, detailsJson) + return videoModelToFormattedDetailsJSON(this) } getFormattedVideoFilesJSON (): VideoFile[] { - const { baseUrlHttp, baseUrlWs } = this.getBaseUrls() - - return this.VideoFiles - .map(videoFile => { - let resolutionLabel = videoFile.resolution + 'p' - - return { - resolution: { - id: videoFile.resolution, - label: resolutionLabel - }, - magnetUri: this.generateMagnetUri(videoFile, baseUrlHttp, baseUrlWs), - size: videoFile.size, - fps: videoFile.fps, - torrentUrl: this.getTorrentUrl(videoFile, baseUrlHttp), - torrentDownloadUrl: this.getTorrentDownloadUrl(videoFile, baseUrlHttp), - fileUrl: this.getVideoFileUrl(videoFile, baseUrlHttp), - fileDownloadUrl: this.getVideoFileDownloadUrl(videoFile, baseUrlHttp) - } as VideoFile - }) - .sort((a, b) => { - if (a.resolution.id < b.resolution.id) return 1 - if (a.resolution.id === b.resolution.id) return 0 - return -1 - }) + return videoFilesModelToFormattedJSON(this, this.VideoFiles) } toActivityPubObject (): VideoTorrentObject { - const { baseUrlHttp, baseUrlWs } = this.getBaseUrls() - if (!this.Tags) this.Tags = [] - - const tag = this.Tags.map(t => ({ - type: 'Hashtag' as 'Hashtag', - name: t.name - })) - - let language - if (this.language) { - language = { - identifier: this.language, - name: VideoModel.getLanguageLabel(this.language) - } - } - - let category - if (this.category) { - category = { - identifier: this.category + '', - name: VideoModel.getCategoryLabel(this.category) - } - } - - let licence - if (this.licence) { - licence = { - identifier: this.licence + '', - name: VideoModel.getLicenceLabel(this.licence) - } - } - - const url: ActivityUrlObject[] = [] - for (const file of this.VideoFiles) { - url.push({ - type: 'Link', - mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, - href: this.getVideoFileUrl(file, baseUrlHttp), - height: file.resolution, - size: file.size, - fps: file.fps - }) - - url.push({ - type: 'Link', - mimeType: 'application/x-bittorrent' as 'application/x-bittorrent', - href: this.getTorrentUrl(file, baseUrlHttp), - height: file.resolution - }) - - url.push({ - type: 'Link', - mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', - href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs), - height: file.resolution - }) - } - - // Add video url too - url.push({ - type: 'Link', - mimeType: 'text/html', - href: CONFIG.WEBSERVER.URL + '/videos/watch/' + this.uuid - }) - - const subtitleLanguage = [] - for (const caption of this.VideoCaptions) { - subtitleLanguage.push({ - identifier: caption.language, - name: VideoCaptionModel.getLanguageLabel(caption.language) - }) - } - - return { - type: 'Video' as 'Video', - id: this.url, - name: this.name, - duration: this.getActivityStreamDuration(), - uuid: this.uuid, - tag, - category, - licence, - language, - views: this.views, - sensitive: this.nsfw, - waitTranscoding: this.waitTranscoding, - state: this.state, - commentsEnabled: this.commentsEnabled, - published: this.publishedAt.toISOString(), - updated: this.updatedAt.toISOString(), - mediaType: 'text/markdown', - content: this.getTruncatedDescription(), - support: this.support, - subtitleLanguage, - icon: { - type: 'Image', - url: this.getThumbnailUrl(baseUrlHttp), - mediaType: 'image/jpeg', - width: THUMBNAILS_SIZE.width, - height: THUMBNAILS_SIZE.height - }, - url, - likes: getVideoLikesActivityPubUrl(this), - dislikes: getVideoDislikesActivityPubUrl(this), - shares: getVideoSharesActivityPubUrl(this), - comments: getVideoCommentsActivityPubUrl(this), - attributedTo: [ - { - type: 'Person', - id: this.VideoChannel.Account.Actor.url - }, - { - type: 'Group', - id: this.VideoChannel.Actor.url - } - ] - } + return videoModelToActivityPubObject(this) } getTruncatedDescription () { @@ -1635,130 +1714,13 @@ export class VideoModel extends Model { return peertubeTruncate(this.description, maxLength) } - async optimizeOriginalVideofile () { - const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR - const newExtname = '.mp4' - const inputVideoFile = this.getOriginalFile() - const videoInputPath = join(videosDirectory, this.getVideoFilename(inputVideoFile)) - const videoTranscodedPath = join(videosDirectory, this.id + '-transcoded' + newExtname) - - const transcodeOptions = { - inputPath: videoInputPath, - outputPath: videoTranscodedPath - } - - // Could be very long! - await transcode(transcodeOptions) - - try { - await remove(videoInputPath) - - // Important to do this before getVideoFilename() to take in account the new file extension - inputVideoFile.set('extname', newExtname) - - const videoOutputPath = this.getVideoFilePath(inputVideoFile) - await rename(videoTranscodedPath, videoOutputPath) - const stats = await stat(videoOutputPath) - const fps = await getVideoFileFPS(videoOutputPath) - - inputVideoFile.set('size', stats.size) - inputVideoFile.set('fps', fps) - - await this.createTorrentAndSetInfoHash(inputVideoFile) - await inputVideoFile.save() - - } catch (err) { - // Auto destruction... - this.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) - - throw err - } - } - - async transcodeOriginalVideofile (resolution: VideoResolution, isPortraitMode: boolean) { - const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR - const extname = '.mp4' - - // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed - const videoInputPath = join(videosDirectory, this.getVideoFilename(this.getOriginalFile())) - - const newVideoFile = new VideoFileModel({ - resolution, - extname, - size: 0, - videoId: this.id - }) - const videoOutputPath = join(videosDirectory, this.getVideoFilename(newVideoFile)) - - const transcodeOptions = { - inputPath: videoInputPath, - outputPath: videoOutputPath, - resolution, - isPortraitMode - } - - await transcode(transcodeOptions) - - const stats = await stat(videoOutputPath) - const fps = await getVideoFileFPS(videoOutputPath) - - newVideoFile.set('size', stats.size) - newVideoFile.set('fps', fps) - - await this.createTorrentAndSetInfoHash(newVideoFile) - - await newVideoFile.save() - - this.VideoFiles.push(newVideoFile) - } - - async importVideoFile (inputFilePath: string) { - const { videoFileResolution } = await getVideoFileResolution(inputFilePath) - const { size } = await stat(inputFilePath) - const fps = await getVideoFileFPS(inputFilePath) - - let updatedVideoFile = new VideoFileModel({ - resolution: videoFileResolution, - extname: extname(inputFilePath), - size, - fps, - videoId: this.id - }) - - const currentVideoFile = this.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution) - - if (currentVideoFile) { - // Remove old file and old torrent - await this.removeFile(currentVideoFile) - await this.removeTorrent(currentVideoFile) - // Remove the old video file from the array - this.VideoFiles = this.VideoFiles.filter(f => f !== currentVideoFile) - - // Update the database - currentVideoFile.set('extname', updatedVideoFile.extname) - currentVideoFile.set('size', updatedVideoFile.size) - currentVideoFile.set('fps', updatedVideoFile.fps) - - updatedVideoFile = currentVideoFile - } - - const outputPath = this.getVideoFilePath(updatedVideoFile) - await copy(inputFilePath, outputPath) - - await this.createTorrentAndSetInfoHash(updatedVideoFile) - - await updatedVideoFile.save() - - this.VideoFiles.push(updatedVideoFile) - } - getOriginalFileResolution () { const originalFilePath = this.getVideoFilePath(this.getOriginalFile()) return getVideoFileResolution(originalFilePath) } - getDescriptionPath () { + getDescriptionAPIPath () { return `/api/${API_VERSION}/videos/${this.uuid}/description` } @@ -1774,8 +1736,10 @@ export class VideoModel extends Model { .catch(err => logger.warn('Cannot delete preview %s.', previewPath, { err })) } - removeFile (videoFile: VideoFileModel) { - const filePath = join(CONFIG.STORAGE.VIDEOS_DIR, this.getVideoFilename(videoFile)) + removeFile (videoFile: VideoFileModel, isRedundancy = false) { + const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR + + const filePath = join(baseDir, this.getVideoFilename(videoFile)) return remove(filePath) .catch(err => logger.warn('Cannot delete file %s.', filePath, { err })) } @@ -1786,9 +1750,12 @@ export class VideoModel extends Model { .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) } - getActivityStreamDuration () { - // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration - return 'PT' + this.duration + 'S' + removeStreamingPlaylist (isRedundancy = false) { + const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_STREAMING_PLAYLIST_DIRECTORY + + const filePath = join(baseDir, this.uuid) + return remove(filePath) + .catch(err => logger.warn('Cannot delete playlist directory %s.', filePath, { err })) } isOutdated () { @@ -1802,6 +1769,12 @@ export class VideoModel extends Model { (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL } + setAsRefreshed () { + this.changed('updatedAt', true) + + return this.save() + } + getBaseUrls () { let baseUrlHttp let baseUrlWs @@ -1819,7 +1792,7 @@ export class VideoModel extends Model { generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { const xs = this.getTorrentUrl(videoFile, baseUrlHttp) - const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] + const announce = this.getTrackerUrls(baseUrlHttp, baseUrlWs) let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] const redundancies = videoFile.RedundancyVideos @@ -1836,6 +1809,10 @@ export class VideoModel extends Model { return magnetUtil.encode(magnetHash) } + getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) { + return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] + } + getThumbnailUrl (baseUrlHttp: string) { return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() } @@ -1852,7 +1829,15 @@ export class VideoModel extends Model { return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) } + getVideoRedundancyUrl (videoFile: VideoFileModel, baseUrlHttp: string) { + return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getVideoFilename(videoFile) + } + getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) } + + getBandwidthBits (videoFile: VideoFileModel) { + return Math.ceil((videoFile.size * 8) / this.duration) + } }