From 9f1ddd249652c1e35b45db33885a00a005f9b059 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 8 Oct 2018 16:50:56 +0200 Subject: Change a little bit optimize-old-videos logic --- server/models/video/video.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'server/models/video') diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 46d823240..070ac7623 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -788,6 +788,16 @@ export class VideoModel extends Model { return VideoModel.scope(ScopeNames.WITH_FILES).findAll() } + static listLocal () { + const query = { + where: { + remote: false + } + } + + return VideoModel.scope(ScopeNames.WITH_FILES).findAll(query) + } + static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { function getRawQuery (select: string) { const queryVideo = 'SELECT ' + select + ' FROM "video" AS "Video" ' + -- cgit v1.2.3 From 1cd3facc3de899ac864e979cd6d6a704b712cce3 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 10 Oct 2018 11:46:50 +0200 Subject: Add ability to list all local videos Including private/unlisted for moderators/admins --- server/models/video/video.ts | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'server/models/video') diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 070ac7623..4f3f75613 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -235,7 +235,14 @@ type AvailableForListIDsOptions = { ) } ] - }, + } + }, + include: [] + } + + // Only list public/published videos + if (!options.filter || options.filter !== 'all-local') { + const privacyWhere = { // Always list public videos privacy: VideoPrivacy.PUBLIC, // Always list published videos, or videos that are being transcoded but on which we don't want to wait for transcoding @@ -250,8 +257,9 @@ type AvailableForListIDsOptions = { } } ] - }, - include: [] + } + + Object.assign(query.where, privacyWhere) } if (options.filter || options.accountId || options.videoChannelId) { @@ -969,6 +977,10 @@ export class VideoModel extends Model { trendingDays?: number, userId?: number }, countVideos = true) { + if (options.filter && options.filter === 'all-local' && !options.userId) { + throw new Error('Try to filter all-local but no userId is provided') + } + const query: IFindOptions = { offset: options.start, limit: options.count, @@ -1021,7 +1033,8 @@ export class VideoModel extends Model { tagsAllOf?: string[] durationMin?: number // seconds durationMax?: number // seconds - userId?: number + userId?: number, + filter?: VideoFilter }) { const whereAnd = [] @@ -1098,7 +1111,8 @@ export class VideoModel extends Model { languageOneOf: options.languageOneOf, tagsOneOf: options.tagsOneOf, tagsAllOf: options.tagsAllOf, - userId: options.userId + userId: options.userId, + filter: options.filter } return VideoModel.getAvailableForApi(query, queryOptions) @@ -1262,7 +1276,7 @@ export class VideoModel extends Model { } private static buildActorWhereWithFilter (filter?: VideoFilter) { - if (filter && filter === 'local') { + if (filter && (filter === 'local' || filter === 'all-local')) { return { serverId: null } -- cgit v1.2.3 From 7ad9b9846c44d198a736183fb186c2039f5236b5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 12 Oct 2018 15:26:04 +0200 Subject: Add ability for users to block an account/instance on server side --- server/models/video/video-comment.ts | 95 ++++++++++++++++++++++++++++-------- server/models/video/video.ts | 40 ++++++++++----- 2 files changed, 104 insertions(+), 31 deletions(-) (limited to 'server/models/video') diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts index f84c1880c..08c6b3ff0 100644 --- a/server/models/video/video-comment.ts +++ b/server/models/video/video-comment.ts @@ -1,6 +1,17 @@ import * as Sequelize from 'sequelize' import { - AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, IFindOptions, Is, Model, Scopes, Table, + AllowNull, + BeforeDestroy, + BelongsTo, + Column, + CreatedAt, + DataType, + ForeignKey, + IFindOptions, + Is, + Model, + Scopes, + Table, UpdatedAt } from 'sequelize-typescript' import { ActivityTagObject } from '../../../shared/models/activitypub/objects/common-objects' @@ -13,9 +24,11 @@ import { AccountModel } from '../account/account' import { ActorModel } from '../activitypub/actor' import { AvatarModel } from '../avatar/avatar' import { ServerModel } from '../server/server' -import { getSort, throwIfNotValid } from '../utils' +import { buildBlockedAccountSQL, getSort, throwIfNotValid } from '../utils' import { VideoModel } from './video' import { VideoChannelModel } from './video-channel' +import { getServerActor } from '../../helpers/utils' +import { UserModel } from '../account/user' enum ScopeNames { WITH_ACCOUNT = 'WITH_ACCOUNT', @@ -25,18 +38,29 @@ enum ScopeNames { } @Scopes({ - [ScopeNames.ATTRIBUTES_FOR_API]: { - attributes: { - include: [ - [ - Sequelize.literal( - '(SELECT COUNT("replies"."id") ' + - 'FROM "videoComment" AS "replies" ' + - 'WHERE "replies"."originCommentId" = "VideoCommentModel"."id")' - ), - 'totalReplies' + [ScopeNames.ATTRIBUTES_FOR_API]: (serverAccountId: number, userAccountId?: number) => { + return { + attributes: { + include: [ + [ + Sequelize.literal( + '(' + + 'WITH "blocklist" AS (' + buildBlockedAccountSQL(serverAccountId, userAccountId) + ')' + + 'SELECT COUNT("replies"."id") - (' + + 'SELECT COUNT("replies"."id") ' + + 'FROM "videoComment" AS "replies" ' + + 'WHERE "replies"."originCommentId" = "VideoCommentModel"."id" ' + + 'AND "accountId" IN (SELECT "id" FROM "blocklist")' + + ')' + + 'FROM "videoComment" AS "replies" ' + + 'WHERE "replies"."originCommentId" = "VideoCommentModel"."id" ' + + 'AND "accountId" NOT IN (SELECT "id" FROM "blocklist")' + + ')' + ), + 'totalReplies' + ] ] - ] + } } }, [ScopeNames.WITH_ACCOUNT]: { @@ -267,26 +291,47 @@ export class VideoCommentModel extends Model { return VideoCommentModel.scope([ ScopeNames.WITH_IN_REPLY_TO, ScopeNames.WITH_VIDEO ]).findOne(query) } - static listThreadsForApi (videoId: number, start: number, count: number, sort: string) { + static async listThreadsForApi (videoId: number, start: number, count: number, sort: string, user?: UserModel) { + const serverActor = await getServerActor() + const serverAccountId = serverActor.Account.id + const userAccountId = user.Account.id + const query = { offset: start, limit: count, order: getSort(sort), where: { videoId, - inReplyToCommentId: null + inReplyToCommentId: null, + accountId: { + [Sequelize.Op.notIn]: Sequelize.literal( + '(' + buildBlockedAccountSQL(serverAccountId, userAccountId) + ')' + ) + } } } + // FIXME: typings + const scopes: any[] = [ + ScopeNames.WITH_ACCOUNT, + { + method: [ ScopeNames.ATTRIBUTES_FOR_API, serverAccountId, userAccountId ] + } + ] + return VideoCommentModel - .scope([ ScopeNames.WITH_ACCOUNT, ScopeNames.ATTRIBUTES_FOR_API ]) + .scope(scopes) .findAndCountAll(query) .then(({ rows, count }) => { return { total: count, data: rows } }) } - static listThreadCommentsForApi (videoId: number, threadId: number) { + static async listThreadCommentsForApi (videoId: number, threadId: number, user?: UserModel) { + const serverActor = await getServerActor() + const serverAccountId = serverActor.Account.id + const userAccountId = user.Account.id + const query = { order: [ [ 'createdAt', 'ASC' ], [ 'updatedAt', 'ASC' ] ], where: { @@ -294,12 +339,24 @@ export class VideoCommentModel extends Model { [ Sequelize.Op.or ]: [ { id: threadId }, { originCommentId: threadId } - ] + ], + accountId: { + [Sequelize.Op.notIn]: Sequelize.literal( + '(' + buildBlockedAccountSQL(serverAccountId, userAccountId) + ')' + ) + } } } + const scopes: any[] = [ + ScopeNames.WITH_ACCOUNT, + { + method: [ ScopeNames.ATTRIBUTES_FOR_API, serverAccountId, userAccountId ] + } + ] + return VideoCommentModel - .scope([ ScopeNames.WITH_ACCOUNT, ScopeNames.ATTRIBUTES_FOR_API ]) + .scope(scopes) .findAndCountAll(query) .then(({ rows, count }) => { return { total: count, data: rows } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 4f3f75613..eab99cba7 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -27,7 +27,7 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { VideoPrivacy, VideoState } from '../../../shared' +import { UserRight, VideoPrivacy, VideoState } from '../../../shared' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' import { VideoFilter } from '../../../shared/models/videos/video-query.type' @@ -70,7 +70,7 @@ import { AccountVideoRateModel } from '../account/account-video-rate' import { ActorModel } from '../activitypub/actor' import { AvatarModel } from '../avatar/avatar' import { ServerModel } from '../server/server' -import { buildTrigramSearchIndex, createSimilarityAttribute, getVideoSort, throwIfNotValid } from '../utils' +import { buildBlockedAccountSQL, buildTrigramSearchIndex, createSimilarityAttribute, getVideoSort, throwIfNotValid } from '../utils' import { TagModel } from './tag' import { VideoAbuseModel } from './video-abuse' import { VideoChannelModel } from './video-channel' @@ -93,6 +93,7 @@ import { } from './video-format-utils' import * as validator from 'validator' import { UserVideoHistoryModel } from '../account/user-video-history' +import { UserModel } from '../account/user' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: Sequelize.DefineIndexesOptions[] = [ @@ -138,6 +139,7 @@ type ForAPIOptions = { } type AvailableForListIDsOptions = { + serverAccountId: number actorId: number includeLocalVideos: boolean filter?: VideoFilter @@ -151,6 +153,7 @@ type AvailableForListIDsOptions = { accountId?: number videoChannelId?: number trendingDays?: number + user?: UserModel } @Scopes({ @@ -235,6 +238,15 @@ type AvailableForListIDsOptions = { ) } ] + }, + channelId: { + [ Sequelize.Op.notIn ]: Sequelize.literal( + '(' + + 'SELECT id FROM "videoChannel" WHERE "accountId" IN (' + + buildBlockedAccountSQL(options.serverAccountId, options.user ? options.user.Account.id : undefined) + + ')' + + ')' + ) } }, include: [] @@ -975,10 +987,10 @@ export class VideoModel extends Model { videoChannelId?: number, actorId?: number trendingDays?: number, - userId?: number + user?: UserModel }, countVideos = true) { - if (options.filter && options.filter === 'all-local' && !options.userId) { - throw new Error('Try to filter all-local but no userId is provided') + if (options.filter && options.filter === 'all-local' && !options.user.hasRight(UserRight.SEE_ALL_VIDEOS)) { + throw new Error('Try to filter all-local but no user has not the see all videos right') } const query: IFindOptions = { @@ -994,11 +1006,14 @@ export class VideoModel extends Model { query.group = 'VideoModel.id' } + const serverActor = await getServerActor() + // actorId === null has a meaning, so just check undefined - const actorId = options.actorId !== undefined ? options.actorId : (await getServerActor()).id + const actorId = options.actorId !== undefined ? options.actorId : serverActor.id const queryOptions = { actorId, + serverAccountId: serverActor.Account.id, nsfw: options.nsfw, categoryOneOf: options.categoryOneOf, licenceOneOf: options.licenceOneOf, @@ -1010,7 +1025,7 @@ export class VideoModel extends Model { accountId: options.accountId, videoChannelId: options.videoChannelId, includeLocalVideos: options.includeLocalVideos, - userId: options.userId, + user: options.user, trendingDays } @@ -1033,7 +1048,7 @@ export class VideoModel extends Model { tagsAllOf?: string[] durationMin?: number // seconds durationMax?: number // seconds - userId?: number, + user?: UserModel, filter?: VideoFilter }) { const whereAnd = [] @@ -1104,6 +1119,7 @@ export class VideoModel extends Model { const serverActor = await getServerActor() const queryOptions = { actorId: serverActor.id, + serverAccountId: serverActor.Account.id, includeLocalVideos: options.includeLocalVideos, nsfw: options.nsfw, categoryOneOf: options.categoryOneOf, @@ -1111,7 +1127,7 @@ export class VideoModel extends Model { languageOneOf: options.languageOneOf, tagsOneOf: options.tagsOneOf, tagsAllOf: options.tagsAllOf, - userId: options.userId, + user: options.user, filter: options.filter } @@ -1287,7 +1303,7 @@ export class VideoModel extends Model { private static async getAvailableForApi ( query: IFindOptions, - options: AvailableForListIDsOptions & { userId?: number}, + options: AvailableForListIDsOptions, countVideos = true ) { const idsScope = { @@ -1320,8 +1336,8 @@ export class VideoModel extends Model { } ] - if (options.userId) { - apiScope.push({ method: [ ScopeNames.WITH_USER_HISTORY, options.userId ] }) + if (options.user) { + apiScope.push({ method: [ ScopeNames.WITH_USER_HISTORY, options.user.id ] }) } const secondQuery = { -- cgit v1.2.3 From 65b21c961c69c4a63c7c0c34be3d6d034a1176c7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 15 Oct 2018 16:43:14 +0200 Subject: Add ability to mute a user/instance by server in client --- server/models/video/video-comment.ts | 4 ++-- server/models/video/video.ts | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'server/models/video') diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts index 08c6b3ff0..dd6d08139 100644 --- a/server/models/video/video-comment.ts +++ b/server/models/video/video-comment.ts @@ -294,7 +294,7 @@ export class VideoCommentModel extends Model { static async listThreadsForApi (videoId: number, start: number, count: number, sort: string, user?: UserModel) { const serverActor = await getServerActor() const serverAccountId = serverActor.Account.id - const userAccountId = user.Account.id + const userAccountId = user ? user.Account.id : undefined const query = { offset: start, @@ -330,7 +330,7 @@ export class VideoCommentModel extends Model { static async listThreadCommentsForApi (videoId: number, threadId: number, user?: UserModel) { const serverActor = await getServerActor() const serverAccountId = serverActor.Account.id - const userAccountId = user.Account.id + const userAccountId = user ? user.Account.id : undefined const query = { order: [ [ 'createdAt', 'ASC' ], [ 'updatedAt', 'ASC' ] ], diff --git a/server/models/video/video.ts b/server/models/video/video.ts index eab99cba7..6c183933b 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1255,9 +1255,11 @@ export class VideoModel extends Model { // threshold corresponds to how many video the field should have to be returned static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { - const actorId = (await getServerActor()).id + const serverActor = await getServerActor() + const actorId = serverActor.id - const scopeOptions = { + const scopeOptions: AvailableForListIDsOptions = { + serverAccountId: serverActor.Account.id, actorId, includeLocalVideos: true } -- cgit v1.2.3 From e27ff5da6ed7bc1f56f50f862b80fb0c7d8a6d98 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 18 Oct 2018 08:48:24 +0200 Subject: AP mimeType -> mediaType --- server/models/video/video-format-utils.ts | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'server/models/video') diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index 905e84449..e3f8d525b 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -208,6 +208,7 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { url.push({ type: 'Link', mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, + mediaType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, href: video.getVideoFileUrl(file, baseUrlHttp), height: file.resolution, size: file.size, @@ -217,6 +218,7 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { url.push({ type: 'Link', mimeType: 'application/x-bittorrent' as 'application/x-bittorrent', + mediaType: 'application/x-bittorrent' as 'application/x-bittorrent', href: video.getTorrentUrl(file, baseUrlHttp), height: file.resolution }) @@ -224,6 +226,7 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { url.push({ type: 'Link', mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', + mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', href: video.generateMagnetUri(file, baseUrlHttp, baseUrlWs), height: file.resolution }) @@ -233,6 +236,7 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { url.push({ type: 'Link', mimeType: 'text/html', + mediaType: 'text/html', href: CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid }) -- cgit v1.2.3 From 5c6d985faeef1d6793d3f44ca6374f1a9b722806 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 14 Nov 2018 15:01:28 +0100 Subject: Check activities host --- server/models/video/video-share.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/models/video') diff --git a/server/models/video/video-share.ts b/server/models/video/video-share.ts index fa9a70d50..c87f71277 100644 --- a/server/models/video/video-share.ts +++ b/server/models/video/video-share.ts @@ -88,7 +88,7 @@ export class VideoShareModel extends Model { }) Video: VideoModel - static load (actorId: number, videoId: number, t: Sequelize.Transaction) { + static load (actorId: number, videoId: number, t?: Sequelize.Transaction) { return VideoShareModel.scope(ScopeNames.WITH_ACTOR).findOne({ where: { actorId, -- cgit v1.2.3 From 8d4273463fb19d503b1aa0a32dc289f292ed614e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 16 Nov 2018 15:02:48 +0100 Subject: Check follow constraints when getting a video --- server/models/video/video.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'server/models/video') diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 6c183933b..1e68b380c 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1253,6 +1253,23 @@ export class VideoModel extends Model { }) } + static checkVideoHasInstanceFollow (videoId: number, followerActorId: number) { + // Instances only share videos + const query = 'SELECT 1 FROM "videoShare" ' + + 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' + + 'WHERE "actorFollow"."actorId" = $followerActorId AND "videoShare"."videoId" = $videoId ' + + 'LIMIT 1' + + const options = { + type: Sequelize.QueryTypes.SELECT, + bind: { followerActorId, videoId }, + raw: true + } + + return VideoModel.sequelize.query(query, options) + .then(results => results.length === 1) + } + // threshold corresponds to how many video the field should have to be returned static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { const serverActor = await getServerActor() -- cgit v1.2.3 From 04b8c3fba614efc3827f583096c78b08cb668470 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 20 Nov 2018 10:05:51 +0100 Subject: Delete invalid or deleted remote videos --- server/models/video/video.ts | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'server/models/video') diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 1e68b380c..0f18d9f0c 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1561,6 +1561,12 @@ export class VideoModel extends Model { (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL } + setAsRefreshed () { + this.changed('updatedAt', true) + + return this.save() + } + getBaseUrls () { let baseUrlHttp let baseUrlWs -- cgit v1.2.3 From b9fffa297f49a84df8ffd0d7b842599bc88a8e3e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 4 Dec 2018 17:08:55 +0100 Subject: Create redundancy endpoint --- server/models/video/video.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'server/models/video') diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 0f18d9f0c..e8cb5aa88 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1538,8 +1538,10 @@ export class VideoModel extends Model { .catch(err => logger.warn('Cannot delete preview %s.', previewPath, { err })) } - removeFile (videoFile: VideoFileModel) { - const filePath = join(CONFIG.STORAGE.VIDEOS_DIR, this.getVideoFilename(videoFile)) + removeFile (videoFile: VideoFileModel, isRedundancy = false) { + const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR + + const filePath = join(baseDir, this.getVideoFilename(videoFile)) return remove(filePath) .catch(err => logger.warn('Cannot delete file %s.', filePath, { err })) } @@ -1617,6 +1619,10 @@ export class VideoModel extends Model { return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) } + getVideoRedundancyUrl (videoFile: VideoFileModel, baseUrlHttp: string) { + return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getVideoFilename(videoFile) + } + getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) } -- cgit v1.2.3 From 4e74e8032be8293ffe3cb3c30528d4ef7c11a798 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 5 Dec 2018 14:36:05 +0100 Subject: Remove inferred type --- server/models/video/video.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'server/models/video') diff --git a/server/models/video/video.ts b/server/models/video/video.ts index e8cb5aa88..adef37937 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -140,7 +140,7 @@ type ForAPIOptions = { type AvailableForListIDsOptions = { serverAccountId: number - actorId: number + followerActorId: number includeLocalVideos: boolean filter?: VideoFilter categoryOneOf?: number[] @@ -315,7 +315,7 @@ type AvailableForListIDsOptions = { query.include.push(videoChannelInclude) } - if (options.actorId) { + if (options.followerActorId) { let localVideosReq = '' if (options.includeLocalVideos === true) { localVideosReq = ' UNION ALL ' + @@ -327,7 +327,7 @@ type AvailableForListIDsOptions = { } // Force actorId to be a number to avoid SQL injections - const actorIdNumber = parseInt(options.actorId.toString(), 10) + const actorIdNumber = parseInt(options.followerActorId.toString(), 10) query.where[ 'id' ][ Sequelize.Op.and ].push({ [ Sequelize.Op.in ]: Sequelize.literal( '(' + @@ -985,7 +985,7 @@ export class VideoModel extends Model { filter?: VideoFilter, accountId?: number, videoChannelId?: number, - actorId?: number + followerActorId?: number trendingDays?: number, user?: UserModel }, countVideos = true) { @@ -1008,11 +1008,11 @@ export class VideoModel extends Model { const serverActor = await getServerActor() - // actorId === null has a meaning, so just check undefined - const actorId = options.actorId !== undefined ? options.actorId : serverActor.id + // followerActorId === null has a meaning, so just check undefined + const followerActorId = options.followerActorId !== undefined ? options.followerActorId : serverActor.id const queryOptions = { - actorId, + followerActorId, serverAccountId: serverActor.Account.id, nsfw: options.nsfw, categoryOneOf: options.categoryOneOf, @@ -1118,7 +1118,7 @@ export class VideoModel extends Model { const serverActor = await getServerActor() const queryOptions = { - actorId: serverActor.id, + followerActorId: serverActor.id, serverAccountId: serverActor.Account.id, includeLocalVideos: options.includeLocalVideos, nsfw: options.nsfw, @@ -1273,11 +1273,11 @@ export class VideoModel extends Model { // threshold corresponds to how many video the field should have to be returned static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { const serverActor = await getServerActor() - const actorId = serverActor.id + const followerActorId = serverActor.id const scopeOptions: AvailableForListIDsOptions = { serverAccountId: serverActor.Account.id, - actorId, + followerActorId, includeLocalVideos: true } -- cgit v1.2.3 From 2feebf3e6afaad9ab80976d1557d3a7bcf94de03 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 5 Dec 2018 17:27:24 +0100 Subject: Add sitemap --- server/models/video/video-channel.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'server/models/video') diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index f4586917e..86bf0461a 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts @@ -233,6 +233,27 @@ export class VideoChannelModel extends Model { }) } + static listLocalsForSitemap (sort: string) { + const query = { + attributes: [ ], + offset: 0, + order: getSort(sort), + include: [ + { + attributes: [ 'preferredUsername', 'serverId' ], + model: ActorModel.unscoped(), + where: { + serverId: null + } + } + ] + } + + return VideoChannelModel + .unscoped() + .findAll(query) + } + static searchForApi (options: { actorId: number search: string -- cgit v1.2.3 From 14e2014acc1362cfbb770c051a7254b156cd8efb Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 11 Dec 2018 14:52:50 +0100 Subject: Support additional video extensions --- server/models/video/video-file.ts | 4 +++- server/models/video/video-format-utils.ts | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'server/models/video') diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index adebdf0c7..3fd2d5a99 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -14,6 +14,7 @@ import { UpdatedAt } from 'sequelize-typescript' import { + isVideoFileExtnameValid, isVideoFileInfoHashValid, isVideoFileResolutionValid, isVideoFileSizeValid, @@ -58,7 +59,8 @@ export class VideoFileModel extends Model { size: number @AllowNull(false) - @Column(DataType.ENUM(values(CONSTRAINTS_FIELDS.VIDEOS.EXTNAME))) + @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname')) + @Column extname: string @AllowNull(false) diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index e3f8d525b..de0747f22 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -2,7 +2,7 @@ import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' import { VideoModel } from './video' import { VideoFileModel } from './video-file' import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects' -import { CONFIG, THUMBNAILS_SIZE, VIDEO_EXT_MIMETYPE } from '../../initializers' +import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers' import { VideoCaptionModel } from './video-caption' import { getVideoCommentsActivityPubUrl, @@ -207,8 +207,8 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { for (const file of video.VideoFiles) { url.push({ type: 'Link', - mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, - mediaType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, + mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ file.extname ] as any, + mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ file.extname ] as any, href: video.getVideoFileUrl(file, baseUrlHttp), height: file.resolution, size: file.size, -- cgit v1.2.3 From 8b9a525a180cc9f3a98c334cc052dcfc8f36dcd4 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 17 Dec 2018 15:52:38 +0100 Subject: Add history on server side Add ability to disable, clear and list user videos history --- server/models/video/video.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'server/models/video') diff --git a/server/models/video/video.ts b/server/models/video/video.ts index adef37937..199ea9ea4 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -153,7 +153,8 @@ type AvailableForListIDsOptions = { accountId?: number videoChannelId?: number trendingDays?: number - user?: UserModel + user?: UserModel, + historyOfUser?: UserModel } @Scopes({ @@ -416,6 +417,16 @@ type AvailableForListIDsOptions = { query.subQuery = false } + if (options.historyOfUser) { + query.include.push({ + model: UserVideoHistoryModel, + required: true, + where: { + userId: options.historyOfUser.id + } + }) + } + return query }, [ ScopeNames.WITH_ACCOUNT_DETAILS ]: { @@ -987,7 +998,8 @@ export class VideoModel extends Model { videoChannelId?: number, followerActorId?: number trendingDays?: number, - user?: UserModel + user?: UserModel, + historyOfUser?: UserModel }, countVideos = true) { if (options.filter && options.filter === 'all-local' && !options.user.hasRight(UserRight.SEE_ALL_VIDEOS)) { throw new Error('Try to filter all-local but no user has not the see all videos right') @@ -1026,6 +1038,7 @@ export class VideoModel extends Model { videoChannelId: options.videoChannelId, includeLocalVideos: options.includeLocalVideos, user: options.user, + historyOfUser: options.historyOfUser, trendingDays } @@ -1341,7 +1354,7 @@ export class VideoModel extends Model { } const [ count, rowsId ] = await Promise.all([ - countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined), + countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined), VideoModel.scope(idsScope).findAll(query) ]) const ids = rowsId.map(r => r.id) -- cgit v1.2.3 From 80bfd33c0bf910e2cfdd3270b14ba9eddd90e2e8 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 18 Dec 2018 09:31:09 +0100 Subject: Add history page on client --- server/models/video/video.ts | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'server/models/video') diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 199ea9ea4..3f282580c 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -425,6 +425,11 @@ type AvailableForListIDsOptions = { userId: options.historyOfUser.id } }) + + // Even if the relation is n:m, we know that a user only have 0..1 video history + // So we won't have multiple rows for the same video + // Without this, we would not be able to sort on "updatedAt" column of UserVideoHistoryModel + query.subQuery = false } return query -- cgit v1.2.3 From 439b1744f5f50b8530cded9398d51aa4bb5ed4ff Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 20 Dec 2018 15:25:24 +0100 Subject: Optimize index sizes --- server/models/video/video.ts | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) (limited to 'server/models/video') diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 3f282580c..bcf327f32 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -102,16 +102,44 @@ const indexes: Sequelize.DefineIndexesOptions[] = [ { fields: [ 'createdAt' ] }, { fields: [ 'publishedAt' ] }, { fields: [ 'duration' ] }, - { fields: [ 'category' ] }, - { fields: [ 'licence' ] }, - { fields: [ 'nsfw' ] }, - { fields: [ 'language' ] }, - { fields: [ 'waitTranscoding' ] }, - { fields: [ 'state' ] }, - { fields: [ 'remote' ] }, { fields: [ 'views' ] }, - { fields: [ 'likes' ] }, { fields: [ 'channelId' ] }, + { + fields: [ 'category' ], // We don't care videos with an unknown category + where: { + category: { + [Sequelize.Op.ne]: null + } + } + }, + { + fields: [ 'licence' ], // We don't care videos with an unknown licence + where: { + licence: { + [Sequelize.Op.ne]: null + } + } + }, + { + fields: [ 'language' ], // We don't care videos with an unknown language + where: { + language: { + [Sequelize.Op.ne]: null + } + } + }, + { + fields: [ 'nsfw' ], // Most of the videos are not NSFW + where: { + nsfw: true + } + }, + { + fields: [ 'remote' ], // Only index local videos + where: { + remote: false + } + }, { fields: [ 'uuid' ], unique: true -- cgit v1.2.3 From cef534ed53e4518fe0acf581bfe880788d42fc36 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 26 Dec 2018 10:36:24 +0100 Subject: Add user notification base code --- server/models/video/video-abuse.ts | 5 ----- server/models/video/video-blacklist.ts | 10 ---------- server/models/video/video-comment.ts | 4 ++++ server/models/video/video.ts | 4 ++++ 4 files changed, 8 insertions(+), 15 deletions(-) (limited to 'server/models/video') diff --git a/server/models/video/video-abuse.ts b/server/models/video/video-abuse.ts index dbb88ca45..4c9e2d05e 100644 --- a/server/models/video/video-abuse.ts +++ b/server/models/video/video-abuse.ts @@ -86,11 +86,6 @@ export class VideoAbuseModel extends Model { }) Video: VideoModel - @AfterCreate - static sendEmailNotification (instance: VideoAbuseModel) { - return Emailer.Instance.addVideoAbuseReportJob(instance.videoId) - } - static loadByIdAndVideoId (id: number, videoId: number) { const query = { where: { diff --git a/server/models/video/video-blacklist.ts b/server/models/video/video-blacklist.ts index 67f7cd487..23e992685 100644 --- a/server/models/video/video-blacklist.ts +++ b/server/models/video/video-blacklist.ts @@ -53,16 +53,6 @@ export class VideoBlacklistModel extends Model { }) Video: VideoModel - @AfterCreate - static sendBlacklistEmailNotification (instance: VideoBlacklistModel) { - return Emailer.Instance.addVideoBlacklistReportJob(instance.videoId, instance.reason) - } - - @AfterDestroy - static sendUnblacklistEmailNotification (instance: VideoBlacklistModel) { - return Emailer.Instance.addVideoUnblacklistReportJob(instance.videoId) - } - static listForApi (start: number, count: number, sort: SortType) { const query = { offset: start, diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts index dd6d08139..d8fc2a564 100644 --- a/server/models/video/video-comment.ts +++ b/server/models/video/video-comment.ts @@ -448,6 +448,10 @@ export class VideoCommentModel extends Model { } } + getCommentStaticPath () { + return this.Video.getWatchStaticPath() + ';threadId=' + this.getThreadId() + } + getThreadId (): number { return this.originCommentId || this.id } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index bcf327f32..fc200e5d1 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1527,6 +1527,10 @@ export class VideoModel extends Model { videoFile.infoHash = parsedTorrent.infoHash } + getWatchStaticPath () { + return '/videos/watch/' + this.uuid + } + getEmbedStaticPath () { return '/videos/embed/' + this.uuid } -- cgit v1.2.3 From dc13348070d808d0ba3feb56a435b835c2e7e791 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 2 Jan 2019 16:37:43 +0100 Subject: Add import finished and video published notifs --- server/models/video/video-file.ts | 2 -- server/models/video/video-import.ts | 4 ++++ server/models/video/video.ts | 10 ++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) (limited to 'server/models/video') diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 3fd2d5a99..0fd868cd6 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -1,4 +1,3 @@ -import { values } from 'lodash' import { AllowNull, BelongsTo, @@ -20,7 +19,6 @@ import { isVideoFileSizeValid, isVideoFPSResolutionValid } from '../../helpers/custom-validators/videos' -import { CONSTRAINTS_FIELDS } from '../../initializers' import { throwIfNotValid } from '../utils' import { VideoModel } from './video' import * as Sequelize from 'sequelize' diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts index 8d442b3f8..c723e57c0 100644 --- a/server/models/video/video-import.ts +++ b/server/models/video/video-import.ts @@ -144,6 +144,10 @@ export class VideoImportModel extends Model { }) } + getTargetIdentifier () { + return this.targetUrl || this.magnetUri || this.torrentName + } + toFormattedJSON (): VideoImport { const videoFormatOptions = { completeDescription: true, diff --git a/server/models/video/video.ts b/server/models/video/video.ts index fc200e5d1..80a6c7832 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -94,6 +94,7 @@ import { import * as validator from 'validator' import { UserVideoHistoryModel } from '../account/user-video-history' import { UserModel } from '../account/user' +import { VideoImportModel } from './video-import' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: Sequelize.DefineIndexesOptions[] = [ @@ -785,6 +786,15 @@ export class VideoModel extends Model { }) VideoBlacklist: VideoBlacklistModel + @HasOne(() => VideoImportModel, { + foreignKey: { + name: 'videoId', + allowNull: true + }, + onDelete: 'set null' + }) + VideoImport: VideoImportModel + @HasMany(() => VideoCaptionModel, { foreignKey: { name: 'videoId', -- cgit v1.2.3 From f7cc67b455a12ccae9b0ea16876d166720364357 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 4 Jan 2019 08:56:20 +0100 Subject: Add new follow, mention and user registered notifs --- server/models/video/video-comment.ts | 41 ++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) (limited to 'server/models/video') diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts index d8fc2a564..cf6278da7 100644 --- a/server/models/video/video-comment.ts +++ b/server/models/video/video-comment.ts @@ -18,7 +18,7 @@ import { ActivityTagObject } from '../../../shared/models/activitypub/objects/co import { VideoCommentObject } from '../../../shared/models/activitypub/objects/video-comment-object' import { VideoComment } from '../../../shared/models/videos/video-comment.model' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { CONSTRAINTS_FIELDS } from '../../initializers' +import { CONFIG, CONSTRAINTS_FIELDS } from '../../initializers' import { sendDeleteVideoComment } from '../../lib/activitypub/send' import { AccountModel } from '../account/account' import { ActorModel } from '../activitypub/actor' @@ -29,6 +29,9 @@ import { VideoModel } from './video' import { VideoChannelModel } from './video-channel' import { getServerActor } from '../../helpers/utils' import { UserModel } from '../account/user' +import { actorNameAlphabet } from '../../helpers/custom-validators/activitypub/actor' +import { regexpCapture } from '../../helpers/regexp' +import { uniq } from 'lodash' enum ScopeNames { WITH_ACCOUNT = 'WITH_ACCOUNT', @@ -370,9 +373,11 @@ export class VideoCommentModel extends Model { id: { [ Sequelize.Op.in ]: Sequelize.literal('(' + 'WITH RECURSIVE children (id, "inReplyToCommentId") AS ( ' + - 'SELECT id, "inReplyToCommentId" FROM "videoComment" WHERE id = ' + comment.id + ' UNION ' + - 'SELECT p.id, p."inReplyToCommentId" from "videoComment" p ' + - 'INNER JOIN children c ON c."inReplyToCommentId" = p.id) ' + + `SELECT id, "inReplyToCommentId" FROM "videoComment" WHERE id = ${comment.id} ` + + 'UNION ' + + 'SELECT "parent"."id", "parent"."inReplyToCommentId" FROM "videoComment" "parent" ' + + 'INNER JOIN "children" ON "children"."inReplyToCommentId" = "parent"."id"' + + ') ' + 'SELECT id FROM children' + ')'), [ Sequelize.Op.ne ]: comment.id @@ -460,6 +465,34 @@ export class VideoCommentModel extends Model { return this.Account.isOwned() } + extractMentions () { + if (!this.text) return [] + + const localMention = `@(${actorNameAlphabet}+)` + const remoteMention = `${localMention}@${CONFIG.WEBSERVER.HOST}` + + const remoteMentionsRegex = new RegExp(' ' + remoteMention + ' ', 'g') + const localMentionsRegex = new RegExp(' ' + localMention + ' ', 'g') + const firstMentionRegex = new RegExp('^(?:(?:' + remoteMention + ')|(?:' + localMention + ')) ', 'g') + const endMentionRegex = new RegExp(' (?:(?:' + remoteMention + ')|(?:' + localMention + '))$', 'g') + + return uniq( + [].concat( + regexpCapture(this.text, remoteMentionsRegex) + .map(([ , username ]) => username), + + regexpCapture(this.text, localMentionsRegex) + .map(([ , username ]) => username), + + regexpCapture(this.text, firstMentionRegex) + .map(([ , username1, username2 ]) => username1 || username2), + + regexpCapture(this.text, endMentionRegex) + .map(([ , username1, username2 ]) => username1 || username2) + ) + ) + } + toFormattedJSON () { return { id: this.id, -- cgit v1.2.3 From 5abb9fbbd12e7097e348d6a38622d364b1fa47ed Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 10 Jan 2019 15:39:51 +0100 Subject: Add ability to unfederate a local video (on blacklist) --- server/models/video/video-blacklist.ts | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) (limited to 'server/models/video') diff --git a/server/models/video/video-blacklist.ts b/server/models/video/video-blacklist.ts index 23e992685..3b567e488 100644 --- a/server/models/video/video-blacklist.ts +++ b/server/models/video/video-blacklist.ts @@ -1,21 +1,7 @@ -import { - AfterCreate, - AfterDestroy, - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - ForeignKey, - Is, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' import { getSortOnModel, SortType, throwIfNotValid } from '../utils' import { VideoModel } from './video' import { isVideoBlacklistReasonValid } from '../../helpers/custom-validators/video-blacklist' -import { Emailer } from '../../lib/emailer' import { VideoBlacklist } from '../../../shared/models/videos' import { CONSTRAINTS_FIELDS } from '../../initializers' @@ -35,6 +21,10 @@ export class VideoBlacklistModel extends Model { @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_BLACKLIST.REASON.max)) reason: string + @AllowNull(false) + @Column + unfederated: boolean + @CreatedAt createdAt: Date @@ -93,6 +83,7 @@ export class VideoBlacklistModel extends Model { createdAt: this.createdAt, updatedAt: this.updatedAt, reason: this.reason, + unfederated: this.unfederated, video: { id: video.id, -- cgit v1.2.3 From 744d0eca195bce7dafeb4a958d0eb3c0046be32d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 14 Jan 2019 11:30:15 +0100 Subject: Refresh remote actors on GET enpoints --- server/models/video/video-channel.ts | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'server/models/video') diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index 86bf0461a..5598d80f6 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts @@ -470,4 +470,8 @@ export class VideoChannelModel extends Model { getDisplayName () { return this.name } + + isOutdated () { + return this.Actor.isOutdated() + } } -- cgit v1.2.3 From 1506307f2f903ce0f80155072a33345c702b7c76 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 14 Jan 2019 16:48:38 +0100 Subject: Increase abuse length to 3000 And correctly handle new lines --- server/models/video/video-abuse.ts | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) (limited to 'server/models/video') diff --git a/server/models/video/video-abuse.ts b/server/models/video/video-abuse.ts index 4c9e2d05e..cc47644f2 100644 --- a/server/models/video/video-abuse.ts +++ b/server/models/video/video-abuse.ts @@ -1,17 +1,4 @@ -import { - AfterCreate, - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - ForeignKey, - Is, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' import { VideoAbuseObject } from '../../../shared/models/activitypub/objects' import { VideoAbuse } from '../../../shared/models/videos' import { @@ -19,7 +6,6 @@ import { isVideoAbuseReasonValid, isVideoAbuseStateValid } from '../../helpers/custom-validators/video-abuses' -import { Emailer } from '../../lib/emailer' import { AccountModel } from '../account/account' import { getSort, throwIfNotValid } from '../utils' import { VideoModel } from './video' @@ -40,8 +26,9 @@ import { CONSTRAINTS_FIELDS, VIDEO_ABUSE_STATES } from '../../initializers' export class VideoAbuseModel extends Model { @AllowNull(false) + @Default(null) @Is('VideoAbuseReason', value => throwIfNotValid(value, isVideoAbuseReasonValid, 'reason')) - @Column + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_ABUSES.REASON.max)) reason: string @AllowNull(false) -- cgit v1.2.3 From 44b9c0ba31c4a97e3d874f33226ad935c3a90dd5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 15 Jan 2019 09:45:54 +0100 Subject: Add totalLocalVideoFilesSize in stats --- server/models/video/video-file.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'server/models/video') diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 0fd868cd6..1f1b76c1e 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -120,6 +120,26 @@ export class VideoFileModel extends Model { return VideoFileModel.findById(id, options) } + static async getStats () { + let totalLocalVideoFilesSize = await VideoFileModel.sum('size', { + include: [ + { + attributes: [], + model: VideoModel.unscoped(), + where: { + remote: false + } + } + ] + } as any) + // Sequelize could return null... + if (!totalLocalVideoFilesSize) totalLocalVideoFilesSize = 0 + + return { + totalLocalVideoFilesSize + } + } + hasSameUniqueKeysThan (other: VideoFileModel) { return this.fps === other.fps && this.resolution === other.resolution && -- cgit v1.2.3 From 092092969633bbcf6d4891a083ea497a7d5c3154 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 29 Jan 2019 08:37:25 +0100 Subject: Add hls support on server --- server/models/video/video-file.ts | 6 +- server/models/video/video-format-utils.ts | 61 +++++++- server/models/video/video-streaming-playlist.ts | 154 ++++++++++++++++++++ server/models/video/video.ts | 179 ++++++++++++++++++++---- 4 files changed, 364 insertions(+), 36 deletions(-) create mode 100644 server/models/video/video-streaming-playlist.ts (limited to 'server/models/video') diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 1f1b76c1e..7d1e371b9 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -62,7 +62,7 @@ export class VideoFileModel extends Model { extname: string @AllowNull(false) - @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash')) + @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash')) @Column infoHash: string @@ -86,14 +86,14 @@ export class VideoFileModel extends Model { @HasMany(() => VideoRedundancyModel, { foreignKey: { - allowNull: false + allowNull: true }, onDelete: 'CASCADE', hooks: true }) RedundancyVideos: VideoRedundancyModel[] - static isInfohashExists (infoHash: string) { + static doesInfohashExist (infoHash: string) { const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' const options = { type: Sequelize.QueryTypes.SELECT, diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index de0747f22..e49dbee30 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -1,7 +1,12 @@ import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' import { VideoModel } from './video' import { VideoFileModel } from './video-file' -import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects' +import { + ActivityPlaylistInfohashesObject, + ActivityPlaylistSegmentHashesObject, + ActivityUrlObject, + VideoTorrentObject +} from '../../../shared/models/activitypub/objects' import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers' import { VideoCaptionModel } from './video-caption' import { @@ -11,6 +16,8 @@ import { getVideoSharesActivityPubUrl } from '../../lib/activitypub' import { isArray } from '../../helpers/custom-validators/misc' +import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' +import { VideoStreamingPlaylistModel } from './video-streaming-playlist' export type VideoFormattingJSONOptions = { completeDescription?: boolean @@ -120,7 +127,12 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { } }) + const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() + const tags = video.Tags ? video.Tags.map(t => t.name) : [] + + const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists) + const detailsJson = { support: video.support, descriptionPath: video.getDescriptionAPIPath(), @@ -133,7 +145,11 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { id: video.state, label: VideoModel.getStateLabel(video.state) }, - files: [] + + trackerUrls: video.getTrackerUrls(baseUrlHttp, baseUrlWs), + + files: [], + streamingPlaylists } // Format and sort video files @@ -142,6 +158,25 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { return Object.assign(formattedJson, detailsJson) } +function streamingPlaylistsModelToFormattedJSON (video: VideoModel, playlists: VideoStreamingPlaylistModel[]): VideoStreamingPlaylist[] { + if (isArray(playlists) === false) return [] + + return playlists + .map(playlist => { + const redundancies = isArray(playlist.RedundancyVideos) + ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl })) + : [] + + return { + id: playlist.id, + type: playlist.type, + playlistUrl: playlist.playlistUrl, + segmentsSha256Url: playlist.segmentsSha256Url, + redundancies + } as VideoStreamingPlaylist + }) +} + function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] { const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() @@ -232,6 +267,28 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { }) } + for (const playlist of (video.VideoStreamingPlaylists || [])) { + let tag: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[] + + tag = playlist.p2pMediaLoaderInfohashes + .map(i => ({ type: 'Infohash' as 'Infohash', name: i })) + tag.push({ + type: 'Link', + name: 'sha256', + mimeType: 'application/json' as 'application/json', + mediaType: 'application/json' as 'application/json', + href: playlist.segmentsSha256Url + }) + + url.push({ + type: 'Link', + mimeType: 'application/x-mpegURL' as 'application/x-mpegURL', + mediaType: 'application/x-mpegURL' as 'application/x-mpegURL', + href: playlist.playlistUrl, + tag + }) + } + // Add video url too url.push({ type: 'Link', diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts new file mode 100644 index 000000000..bce537781 --- /dev/null +++ b/server/models/video/video-streaming-playlist.ts @@ -0,0 +1,154 @@ +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' +import { throwIfNotValid } from '../utils' +import { VideoModel } from './video' +import * as Sequelize from 'sequelize' +import { VideoRedundancyModel } from '../redundancy/video-redundancy' +import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' +import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' +import { CONSTRAINTS_FIELDS, STATIC_PATHS } from '../../initializers' +import { VideoFileModel } from './video-file' +import { join } from 'path' +import { sha1 } from '../../helpers/core-utils' +import { isArrayOf } from '../../helpers/custom-validators/misc' + +@Table({ + tableName: 'videoStreamingPlaylist', + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'videoId', 'type' ], + unique: true + }, + { + fields: [ 'p2pMediaLoaderInfohashes' ], + using: 'gin' + } + ] +}) +export class VideoStreamingPlaylistModel extends Model { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Column + type: VideoStreamingPlaylistType + + @AllowNull(false) + @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max)) + playlistUrl: string + + @AllowNull(false) + @Is('VideoStreamingPlaylistInfoHashes', value => throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes')) + @Column(DataType.ARRAY(DataType.STRING)) + p2pMediaLoaderInfohashes: string[] + + @AllowNull(false) + @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url')) + @Column + segmentsSha256Url: string + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Video: VideoModel + + @HasMany(() => VideoRedundancyModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE', + hooks: true + }) + RedundancyVideos: VideoRedundancyModel[] + + static doesInfohashExist (infoHash: string) { + const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1' + const options = { + type: Sequelize.QueryTypes.SELECT, + bind: { infoHash }, + raw: true + } + + return VideoModel.sequelize.query(query, options) + .then(results => { + return results.length === 1 + }) + } + + static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: VideoFileModel[]) { + const hashes: string[] = [] + + // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L97 + for (let i = 0; i < videoFiles.length; i++) { + hashes.push(sha1(`1${playlistUrl}+V${i}`)) + } + + return hashes + } + + static loadWithVideo (id: number) { + const options = { + include: [ + { + model: VideoModel.unscoped(), + required: true + } + ] + } + + return VideoStreamingPlaylistModel.findById(id, options) + } + + static getHlsPlaylistFilename (resolution: number) { + return resolution + '.m3u8' + } + + static getMasterHlsPlaylistFilename () { + return 'master.m3u8' + } + + static getHlsSha256SegmentsFilename () { + return 'segments-sha256.json' + } + + static getHlsMasterPlaylistStaticPath (videoUUID: string) { + return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) + } + + static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) { + return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) + } + + static getHlsSha256SegmentsStaticPath (videoUUID: string) { + return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) + } + + getStringType () { + if (this.type === VideoStreamingPlaylistType.HLS) return 'hls' + + return 'unknown' + } + + getVideoRedundancyUrl (baseUrlHttp: string) { + return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getStringType() + '/' + this.Video.uuid + } + + hasSameUniqueKeysThan (other: VideoStreamingPlaylistModel) { + return this.type === other.type && + this.videoId === other.videoId + } +} diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 80a6c7832..702260772 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -52,7 +52,7 @@ import { ACTIVITY_PUB, API_VERSION, CONFIG, - CONSTRAINTS_FIELDS, + CONSTRAINTS_FIELDS, HLS_PLAYLIST_DIRECTORY, HLS_REDUNDANCY_DIRECTORY, PREVIEWS_SIZE, REMOTE_SCHEME, STATIC_DOWNLOAD_PATHS, @@ -95,6 +95,7 @@ import * as validator from 'validator' import { UserVideoHistoryModel } from '../account/user-video-history' import { UserModel } from '../account/user' import { VideoImportModel } from './video-import' +import { VideoStreamingPlaylistModel } from './video-streaming-playlist' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: Sequelize.DefineIndexesOptions[] = [ @@ -159,7 +160,9 @@ export enum ScopeNames { WITH_FILES = 'WITH_FILES', WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', WITH_BLACKLISTED = 'WITH_BLACKLISTED', - WITH_USER_HISTORY = 'WITH_USER_HISTORY' + WITH_USER_HISTORY = 'WITH_USER_HISTORY', + WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS', + WITH_USER_ID = 'WITH_USER_ID' } type ForAPIOptions = { @@ -463,6 +466,22 @@ type AvailableForListIDsOptions = { return query }, + [ ScopeNames.WITH_USER_ID ]: { + include: [ + { + attributes: [ 'accountId' ], + model: () => VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'userId' ], + model: () => AccountModel.unscoped(), + required: true + } + ] + } + ] + }, [ ScopeNames.WITH_ACCOUNT_DETAILS ]: { include: [ { @@ -527,22 +546,55 @@ type AvailableForListIDsOptions = { } ] }, - [ ScopeNames.WITH_FILES ]: { - include: [ - { - model: () => VideoFileModel.unscoped(), - // FIXME: typings - [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join - required: false, - include: [ - { - attributes: [ 'fileUrl' ], - model: () => VideoRedundancyModel.unscoped(), - required: false - } - ] - } - ] + [ ScopeNames.WITH_FILES ]: (withRedundancies = false) => { + let subInclude: any[] = [] + + if (withRedundancies === true) { + subInclude = [ + { + attributes: [ 'fileUrl' ], + model: VideoRedundancyModel.unscoped(), + required: false + } + ] + } + + return { + include: [ + { + model: VideoFileModel.unscoped(), + // FIXME: typings + [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join + required: false, + include: subInclude + } + ] + } + }, + [ ScopeNames.WITH_STREAMING_PLAYLISTS ]: (withRedundancies = false) => { + let subInclude: any[] = [] + + if (withRedundancies === true) { + subInclude = [ + { + attributes: [ 'fileUrl' ], + model: VideoRedundancyModel.unscoped(), + required: false + } + ] + } + + return { + include: [ + { + model: VideoStreamingPlaylistModel.unscoped(), + // FIXME: typings + [ 'separate' as any ]: true, // We may have multiple streaming playlists, having multiple redundancies so let's separate this join + required: false, + include: subInclude + } + ] + } }, [ ScopeNames.WITH_SCHEDULED_UPDATE ]: { include: [ @@ -722,6 +774,16 @@ export class VideoModel extends Model { }) VideoFiles: VideoFileModel[] + @HasMany(() => VideoStreamingPlaylistModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + hooks: true, + onDelete: 'cascade' + }) + VideoStreamingPlaylists: VideoStreamingPlaylistModel[] + @HasMany(() => VideoShareModel, { foreignKey: { name: 'videoId', @@ -847,6 +909,9 @@ export class VideoModel extends Model { tasks.push(instance.removeFile(file)) tasks.push(instance.removeTorrent(file)) }) + + // Remove playlists file + tasks.push(instance.removeStreamingPlaylist()) } // Do not wait video deletion because we could be in a transaction @@ -858,10 +923,6 @@ export class VideoModel extends Model { return undefined } - static list () { - return VideoModel.scope(ScopeNames.WITH_FILES).findAll() - } - static listLocal () { const query = { where: { @@ -869,7 +930,7 @@ export class VideoModel extends Model { } } - return VideoModel.scope(ScopeNames.WITH_FILES).findAll(query) + return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]).findAll(query) } static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { @@ -1200,6 +1261,16 @@ export class VideoModel extends Model { return VideoModel.findOne(options) } + static loadWithRights (id: number | string, t?: Sequelize.Transaction) { + const where = VideoModel.buildWhereIdOrUUID(id) + const options = { + where, + transaction: t + } + + return VideoModel.scope([ ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_USER_ID ]).findOne(options) + } + static loadOnlyId (id: number | string, t?: Sequelize.Transaction) { const where = VideoModel.buildWhereIdOrUUID(id) @@ -1212,8 +1283,8 @@ export class VideoModel extends Model { return VideoModel.findOne(options) } - static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) { - return VideoModel.scope(ScopeNames.WITH_FILES) + static loadWithFiles (id: number, t?: Sequelize.Transaction, logging?: boolean) { + return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]) .findById(id, { transaction: t, logging }) } @@ -1224,9 +1295,7 @@ export class VideoModel extends Model { } } - return VideoModel - .scope([ ScopeNames.WITH_FILES ]) - .findOne(options) + return VideoModel.findOne(options) } static loadByUrl (url: string, transaction?: Sequelize.Transaction) { @@ -1248,7 +1317,11 @@ export class VideoModel extends Model { transaction } - return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) + return VideoModel.scope([ + ScopeNames.WITH_ACCOUNT_DETAILS, + ScopeNames.WITH_FILES, + ScopeNames.WITH_STREAMING_PLAYLISTS + ]).findOne(query) } static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) { @@ -1263,9 +1336,37 @@ export class VideoModel extends Model { const scopes = [ ScopeNames.WITH_TAGS, ScopeNames.WITH_BLACKLISTED, + ScopeNames.WITH_ACCOUNT_DETAILS, + ScopeNames.WITH_SCHEDULED_UPDATE, ScopeNames.WITH_FILES, + ScopeNames.WITH_STREAMING_PLAYLISTS + ] + + if (userId) { + scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings + } + + return VideoModel + .scope(scopes) + .findOne(options) + } + + static loadForGetAPI (id: number | string, t?: Sequelize.Transaction, userId?: number) { + const where = VideoModel.buildWhereIdOrUUID(id) + + const options = { + order: [ [ 'Tags', 'name', 'ASC' ] ], + where, + transaction: t + } + + const scopes = [ + ScopeNames.WITH_TAGS, + ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_ACCOUNT_DETAILS, - ScopeNames.WITH_SCHEDULED_UPDATE + ScopeNames.WITH_SCHEDULED_UPDATE, + { method: [ ScopeNames.WITH_FILES, true ] } as any, // FIXME: typings + { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } as any // FIXME: typings ] if (userId) { @@ -1612,6 +1713,14 @@ export class VideoModel extends Model { .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) } + removeStreamingPlaylist (isRedundancy = false) { + const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_PLAYLIST_DIRECTORY + + const filePath = join(baseDir, this.uuid) + return remove(filePath) + .catch(err => logger.warn('Cannot delete playlist directory %s.', filePath, { err })) + } + isOutdated () { if (this.isOwned()) return false @@ -1646,7 +1755,7 @@ export class VideoModel extends Model { generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { const xs = this.getTorrentUrl(videoFile, baseUrlHttp) - const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] + const announce = this.getTrackerUrls(baseUrlHttp, baseUrlWs) let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] const redundancies = videoFile.RedundancyVideos @@ -1663,6 +1772,10 @@ export class VideoModel extends Model { return magnetUtil.encode(magnetHash) } + getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) { + return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] + } + getThumbnailUrl (baseUrlHttp: string) { return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() } @@ -1686,4 +1799,8 @@ export class VideoModel extends Model { getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) } + + getBandwidthBits (videoFile: VideoFileModel) { + return Math.ceil((videoFile.size * 8) / this.duration) + } } -- cgit v1.2.3 From 4c280004ce62bf11ddb091854c28f1e1d54a54d6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 7 Feb 2019 15:08:19 +0100 Subject: Use a single file instead of segments for HLS --- server/models/video/video-streaming-playlist.ts | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'server/models/video') diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index bce537781..bf6f7b0c4 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts @@ -125,6 +125,10 @@ export class VideoStreamingPlaylistModel extends Model