From 3caf77d3b11f2dbc12e52d665183d36604c1dab9 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Jun 2019 14:55:58 +0200 Subject: Add language filters in user preferences --- server/controllers/api/users/me.ts | 1 + server/helpers/custom-validators/users.ts | 7 +- server/initializers/constants.ts | 3 +- .../migrations/0395-user-video-languages.ts | 25 +++ server/middlewares/validators/users.ts | 5 +- server/models/account/user.ts | 8 + server/models/utils.ts | 12 +- server/models/video/video.ts | 209 ++++++++++++--------- server/tests/api/check-params/users.ts | 23 +++ server/tests/api/search/search-videos.ts | 53 +++++- 10 files changed, 245 insertions(+), 101 deletions(-) create mode 100644 server/initializers/migrations/0395-user-video-languages.ts (limited to 'server') diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index 1750a02e9..a078334fe 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -182,6 +182,7 @@ async function updateMe (req: express.Request, res: express.Response) { if (body.webTorrentEnabled !== undefined) user.webTorrentEnabled = body.webTorrentEnabled if (body.autoPlayVideo !== undefined) user.autoPlayVideo = body.autoPlayVideo if (body.videosHistoryEnabled !== undefined) user.videosHistoryEnabled = body.videosHistoryEnabled + if (body.videoLanguages !== undefined) user.videoLanguages = body.videoLanguages if (body.email !== undefined) { if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) { diff --git a/server/helpers/custom-validators/users.ts b/server/helpers/custom-validators/users.ts index 56bc10b16..738d5cbbf 100644 --- a/server/helpers/custom-validators/users.ts +++ b/server/helpers/custom-validators/users.ts @@ -2,7 +2,7 @@ import 'express-validator' import * as validator from 'validator' import { UserRole } from '../../../shared' import { CONSTRAINTS_FIELDS, NSFW_POLICY_TYPES } from '../../initializers/constants' -import { exists, isBooleanValid, isFileValid } from './misc' +import { exists, isArray, isBooleanValid, isFileValid } from './misc' import { values } from 'lodash' const USERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USERS @@ -54,6 +54,10 @@ function isUserAutoPlayVideoValid (value: any) { return isBooleanValid(value) } +function isUserVideoLanguages (value: any) { + return value === null || (isArray(value) && value.length < CONSTRAINTS_FIELDS.USERS.VIDEO_LANGUAGES.max) +} + function isUserAdminFlagsValid (value: any) { return exists(value) && validator.isInt('' + value) } @@ -84,6 +88,7 @@ export { isUserVideosHistoryEnabledValid, isUserBlockedValid, isUserPasswordValid, + isUserVideoLanguages, isUserBlockedReasonValid, isUserRoleValid, isUserVideoQuotaValid, diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index c2b8eff95..500f8770a 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 390 +const LAST_MIGRATION_VERSION = 395 // --------------------------------------------------------------------------- @@ -177,6 +177,7 @@ let CONSTRAINTS_FIELDS = { PASSWORD: { min: 6, max: 255 }, // Length VIDEO_QUOTA: { min: -1 }, VIDEO_QUOTA_DAILY: { min: -1 }, + VIDEO_LANGUAGES: { max: 500 }, // Array length BLOCKED_REASON: { min: 3, max: 250 } // Length }, VIDEO_ABUSES: { diff --git a/server/initializers/migrations/0395-user-video-languages.ts b/server/initializers/migrations/0395-user-video-languages.ts new file mode 100644 index 000000000..278698bf4 --- /dev/null +++ b/server/initializers/migrations/0395-user-video-languages.ts @@ -0,0 +1,25 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize, + db: any +}): Promise { + const data = { + type: Sequelize.ARRAY(Sequelize.STRING), + allowNull: true, + defaultValue: null + } + + await utils.queryInterface.addColumn('user', 'videoLanguages', data) +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts index ec70fa0fd..947ed36c3 100644 --- a/server/middlewares/validators/users.ts +++ b/server/middlewares/validators/users.ts @@ -13,7 +13,7 @@ import { isUserNSFWPolicyValid, isUserPasswordValid, isUserRoleValid, - isUserUsernameValid, + isUserUsernameValid, isUserVideoLanguages, isUserVideoQuotaDailyValid, isUserVideoQuotaValid, isUserVideosHistoryEnabledValid @@ -198,6 +198,9 @@ const usersUpdateMeValidator = [ body('autoPlayVideo') .optional() .custom(isUserAutoPlayVideoValid).withMessage('Should have a valid automatically plays video attribute'), + body('videoLanguages') + .optional() + .custom(isUserVideoLanguages).withMessage('Should have a valid video languages attribute'), body('videosHistoryEnabled') .optional() .custom(isUserVideosHistoryEnabledValid).withMessage('Should have a valid videos history enabled attribute'), diff --git a/server/models/account/user.ts b/server/models/account/user.ts index e75039521..aac691d66 100644 --- a/server/models/account/user.ts +++ b/server/models/account/user.ts @@ -31,6 +31,7 @@ import { isUserPasswordValid, isUserRoleValid, isUserUsernameValid, + isUserVideoLanguages, isUserVideoQuotaDailyValid, isUserVideoQuotaValid, isUserVideosHistoryEnabledValid, @@ -147,6 +148,12 @@ export class UserModel extends Model { @Column autoPlayVideo: boolean + @AllowNull(true) + @Default(null) + @Is('UserVideoLanguages', value => throwIfNotValid(value, isUserVideoLanguages, 'video languages')) + @Column(DataType.ARRAY(DataType.STRING)) + videoLanguages: string[] + @AllowNull(false) @Default(UserAdminFlag.NONE) @Is('UserAdminFlags', value => throwIfNotValid(value, isUserAdminFlagsValid, 'user admin flags')) @@ -551,6 +558,7 @@ export class UserModel extends Model { webTorrentEnabled: this.webTorrentEnabled, videosHistoryEnabled: this.videosHistoryEnabled, autoPlayVideo: this.autoPlayVideo, + videoLanguages: this.videoLanguages, role: this.role, roleLabel: USER_ROLE_LABELS[ this.role ], videoQuota: this.videoQuota, diff --git a/server/models/utils.ts b/server/models/utils.ts index 2b172f608..206e108c3 100644 --- a/server/models/utils.ts +++ b/server/models/utils.ts @@ -1,7 +1,7 @@ -import { Sequelize } from 'sequelize-typescript' +import { Model, Sequelize } from 'sequelize-typescript' import * as validator from 'validator' -import { OrderItem } from 'sequelize' import { Col } from 'sequelize/types/lib/utils' +import { OrderItem } from 'sequelize/types' type SortType = { sortModel: any, sortValue: string } @@ -127,6 +127,11 @@ function parseAggregateResult (result: any) { return total } +const createSafeIn = (model: typeof Model, stringArr: string[]) => { + return stringArr.map(t => model.sequelize.escape(t)) + .join(', ') +} + // --------------------------------------------------------------------------- export { @@ -141,7 +146,8 @@ export { buildTrigramSearchIndex, buildWhereIdOrUUID, isOutdated, - parseAggregateResult + parseAggregateResult, + createSafeIn } // --------------------------------------------------------------------------- diff --git a/server/models/video/video.ts b/server/models/video/video.ts index eccf0a4fa..92d07b5bc 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -83,6 +83,7 @@ import { buildBlockedAccountSQL, buildTrigramSearchIndex, buildWhereIdOrUUID, + createSafeIn, createSimilarityAttribute, getVideoSort, isOutdated, @@ -227,6 +228,8 @@ type AvailableForListIDsOptions = { trendingDays?: number user?: UserModel, historyOfUser?: UserModel + + baseWhere?: WhereOptions[] } @Scopes(() => ({ @@ -270,34 +273,34 @@ type AvailableForListIDsOptions = { return query }, [ ScopeNames.AVAILABLE_FOR_LIST_IDS ]: (options: AvailableForListIDsOptions) => { - const attributes = options.withoutId === true ? [] : [ 'id' ] + const whereAnd = options.baseWhere ? options.baseWhere : [] const query: FindOptions = { raw: true, - attributes, - where: { - id: { - [ Op.and ]: [ - { - [ Op.notIn ]: Sequelize.literal( - '(SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")' - ) - } - ] - }, - channelId: { - [ Op.notIn ]: Sequelize.literal( - '(' + - 'SELECT id FROM "videoChannel" WHERE "accountId" IN (' + - buildBlockedAccountSQL(options.serverAccountId, options.user ? options.user.Account.id : undefined) + - ')' + - ')' - ) - } - }, + attributes: options.withoutId === true ? [] : [ 'id' ], include: [] } + whereAnd.push({ + id: { + [ Op.notIn ]: Sequelize.literal( + '(SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")' + ) + } + }) + + whereAnd.push({ + channelId: { + [ Op.notIn ]: Sequelize.literal( + '(' + + 'SELECT id FROM "videoChannel" WHERE "accountId" IN (' + + buildBlockedAccountSQL(options.serverAccountId, options.user ? options.user.Account.id : undefined) + + ')' + + ')' + ) + } + }) + // Only list public/published videos if (!options.filter || options.filter !== 'all-local') { const privacyWhere = { @@ -317,7 +320,7 @@ type AvailableForListIDsOptions = { ] } - Object.assign(query.where, privacyWhere) + whereAnd.push(privacyWhere) } if (options.videoPlaylistId) { @@ -387,86 +390,114 @@ type AvailableForListIDsOptions = { // Force actorId to be a number to avoid SQL injections const actorIdNumber = parseInt(options.followerActorId.toString(), 10) - query.where[ 'id' ][ Op.and ].push({ - [ Op.in ]: Sequelize.literal( - '(' + - 'SELECT "videoShare"."videoId" AS "id" FROM "videoShare" ' + - 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' + - 'WHERE "actorFollow"."actorId" = ' + actorIdNumber + - ' UNION ALL ' + - 'SELECT "video"."id" AS "id" FROM "video" ' + - 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + - 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ' + - 'INNER JOIN "actor" ON "account"."actorId" = "actor"."id" ' + - 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "actor"."id" ' + - 'WHERE "actorFollow"."actorId" = ' + actorIdNumber + - localVideosReq + - ')' - ) + whereAnd.push({ + id: { + [ Op.in ]: Sequelize.literal( + '(' + + 'SELECT "videoShare"."videoId" AS "id" FROM "videoShare" ' + + 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' + + 'WHERE "actorFollow"."actorId" = ' + actorIdNumber + + ' UNION ALL ' + + 'SELECT "video"."id" AS "id" FROM "video" ' + + 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + + 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ' + + 'INNER JOIN "actor" ON "account"."actorId" = "actor"."id" ' + + 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "actor"."id" ' + + 'WHERE "actorFollow"."actorId" = ' + actorIdNumber + + localVideosReq + + ')' + ) + } }) } if (options.withFiles === true) { - query.where[ 'id' ][ Op.and ].push({ - [ Op.in ]: Sequelize.literal( - '(SELECT "videoId" FROM "videoFile")' - ) + whereAnd.push({ + id: { + [ Op.in ]: Sequelize.literal( + '(SELECT "videoId" FROM "videoFile")' + ) + } }) } // FIXME: issues with sequelize count when making a join on n:m relation, so we just make a IN() if (options.tagsAllOf || options.tagsOneOf) { - const createTagsIn = (tags: string[]) => { - return tags.map(t => VideoModel.sequelize.escape(t)) - .join(', ') - } - if (options.tagsOneOf) { - query.where[ 'id' ][ Op.and ].push({ - [ Op.in ]: Sequelize.literal( - '(' + - 'SELECT "videoId" FROM "videoTag" ' + - 'INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + - 'WHERE "tag"."name" IN (' + createTagsIn(options.tagsOneOf) + ')' + - ')' - ) + whereAnd.push({ + id: { + [ Op.in ]: Sequelize.literal( + '(' + + 'SELECT "videoId" FROM "videoTag" ' + + 'INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + + 'WHERE "tag"."name" IN (' + createSafeIn(VideoModel, options.tagsOneOf) + ')' + + ')' + ) + } }) } if (options.tagsAllOf) { - query.where[ 'id' ][ Op.and ].push({ - [ Op.in ]: Sequelize.literal( - '(' + - 'SELECT "videoId" FROM "videoTag" ' + - 'INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + - 'WHERE "tag"."name" IN (' + createTagsIn(options.tagsAllOf) + ')' + - 'GROUP BY "videoTag"."videoId" HAVING COUNT(*) = ' + options.tagsAllOf.length + - ')' - ) + whereAnd.push({ + id: { + [ Op.in ]: Sequelize.literal( + '(' + + 'SELECT "videoId" FROM "videoTag" ' + + 'INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + + 'WHERE "tag"."name" IN (' + createSafeIn(VideoModel, options.tagsAllOf) + ')' + + 'GROUP BY "videoTag"."videoId" HAVING COUNT(*) = ' + options.tagsAllOf.length + + ')' + ) + } }) } } if (options.nsfw === true || options.nsfw === false) { - query.where[ 'nsfw' ] = options.nsfw + whereAnd.push({ nsfw: options.nsfw }) } if (options.categoryOneOf) { - query.where[ 'category' ] = { - [ Op.or ]: options.categoryOneOf - } + whereAnd.push({ + category: { + [ Op.or ]: options.categoryOneOf + } + }) } if (options.licenceOneOf) { - query.where[ 'licence' ] = { - [ Op.or ]: options.licenceOneOf - } + whereAnd.push({ + licence: { + [ Op.or ]: options.licenceOneOf + } + }) } if (options.languageOneOf) { - query.where[ 'language' ] = { - [ Op.or ]: options.languageOneOf + let videoLanguages = options.languageOneOf + if (options.languageOneOf.find(l => l === '_unknown')) { + videoLanguages = videoLanguages.concat([ null ]) } + + whereAnd.push({ + [Op.or]: [ + { + language: { + [ Op.or ]: videoLanguages + } + }, + { + id: { + [ Op.in ]: Sequelize.literal( + '(' + + 'SELECT "videoId" FROM "videoCaption" ' + + 'WHERE "language" IN (' + createSafeIn(VideoModel, options.languageOneOf) + ') ' + + ')' + ) + } + } + ] + }) } if (options.trendingDays) { @@ -490,6 +521,10 @@ type AvailableForListIDsOptions = { query.subQuery = false } + query.where = { + [ Op.and ]: whereAnd + } + return query }, [ ScopeNames.WITH_THUMBNAILS ]: { @@ -1175,7 +1210,7 @@ export class VideoModel extends Model { throw new Error('Try to filter all-local but no user has not the see all videos right') } - const query: FindOptions = { + const query: FindOptions & { where?: null } = { offset: options.start, limit: options.count, order: getVideoSort(options.sort) @@ -1299,16 +1334,13 @@ export class VideoModel extends Model { ) } - const query: FindOptions = { + const query = { attributes: { include: attributesInclude }, offset: options.start, limit: options.count, - order: getVideoSort(options.sort), - where: { - [ Op.and ]: whereAnd - } + order: getVideoSort(options.sort) } const serverActor = await getServerActor() @@ -1323,7 +1355,8 @@ export class VideoModel extends Model { tagsOneOf: options.tagsOneOf, tagsAllOf: options.tagsAllOf, user: options.user, - filter: options.filter + filter: options.filter, + baseWhere: whereAnd } return VideoModel.getAvailableForApi(query, queryOptions) @@ -1590,7 +1623,7 @@ export class VideoModel extends Model { } private static async getAvailableForApi ( - query: FindOptions, + query: FindOptions & { where?: null }, // Forbid where field in query options: AvailableForListIDsOptions, countVideos = true ) { @@ -1609,11 +1642,15 @@ export class VideoModel extends Model { ] } - const [ count, rowsId ] = await Promise.all([ - countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined), - VideoModel.scope(idsScope).findAll(query) + const [ count, ids ] = await Promise.all([ + countVideos + ? VideoModel.scope(countScope).count(countQuery) + : Promise.resolve(undefined), + + VideoModel.scope(idsScope) + .findAll(query) + .then(rows => rows.map(r => r.id)) ]) - const ids = rowsId.map(r => r.id) if (ids.length === 0) return { data: [], total: count } diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts index 2316033a1..5d62fe2b3 100644 --- a/server/tests/api/check-params/users.ts +++ b/server/tests/api/check-params/users.ts @@ -364,6 +364,29 @@ describe('Test users API validators', function () { await makePutBodyRequest({ url: server.url, path: path + 'me', token: userAccessToken, fields }) }) + it('Should fail with an invalid videoLanguages attribute', async function () { + { + const fields = { + videoLanguages: 'toto' + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userAccessToken, fields }) + } + + { + const languages = [] + for (let i = 0; i < 1000; i++) { + languages.push('fr') + } + + const fields = { + videoLanguages: languages + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userAccessToken, fields }) + } + }) + it('Should succeed to change password with the correct params', async function () { const fields = { currentPassword: 'my super password', diff --git a/server/tests/api/search/search-videos.ts b/server/tests/api/search/search-videos.ts index 92cc0dc71..c06200ffe 100644 --- a/server/tests/api/search/search-videos.ts +++ b/server/tests/api/search/search-videos.ts @@ -13,6 +13,7 @@ import { uploadVideo, wait } from '../../../../shared/extra-utils' +import { createVideoCaption } from '../../../../shared/extra-utils/videos/video-captions' const expect = chai.expect @@ -41,8 +42,29 @@ describe('Test videos search', function () { const attributes2 = immutableAssign(attributes1, { name: attributes1.name + ' - 2', fixture: 'video_short.mp4' }) await uploadVideo(server.url, server.accessToken, attributes2) - const attributes3 = immutableAssign(attributes1, { name: attributes1.name + ' - 3', language: 'en' }) - await uploadVideo(server.url, server.accessToken, attributes3) + { + const attributes3 = immutableAssign(attributes1, { name: attributes1.name + ' - 3', language: undefined }) + const res = await uploadVideo(server.url, server.accessToken, attributes3) + const videoId = res.body.video.id + + await createVideoCaption({ + url: server.url, + accessToken: server.accessToken, + language: 'en', + videoId, + fixture: 'subtitle-good2.vtt', + mimeType: 'application/octet-stream' + }) + + await createVideoCaption({ + url: server.url, + accessToken: server.accessToken, + language: 'aa', + videoId, + fixture: 'subtitle-good2.vtt', + mimeType: 'application/octet-stream' + }) + } const attributes4 = immutableAssign(attributes1, { name: attributes1.name + ' - 4', language: 'pl', nsfw: true }) await uploadVideo(server.url, server.accessToken, attributes4) @@ -51,7 +73,7 @@ describe('Test videos search', function () { startDate = new Date().toISOString() - const attributes5 = immutableAssign(attributes1, { name: attributes1.name + ' - 5', licence: 2 }) + const attributes5 = immutableAssign(attributes1, { name: attributes1.name + ' - 5', licence: 2, language: undefined }) await uploadVideo(server.url, server.accessToken, attributes5) const attributes6 = immutableAssign(attributes1, { name: attributes1.name + ' - 6', tags: [ 't1', 't2 '] }) @@ -241,13 +263,26 @@ describe('Test videos search', function () { search: '1111 2222 3333', languageOneOf: [ 'pl', 'en' ] } - const res1 = await advancedVideosSearch(server.url, query) - expect(res1.body.total).to.equal(2) - expect(res1.body.data[0].name).to.equal('1111 2222 3333 - 3') - expect(res1.body.data[1].name).to.equal('1111 2222 3333 - 4') - const res2 = await advancedVideosSearch(server.url, immutableAssign(query, { languageOneOf: [ 'eo' ] })) - expect(res2.body.total).to.equal(0) + { + const res = await advancedVideosSearch(server.url, query) + expect(res.body.total).to.equal(2) + expect(res.body.data[ 0 ].name).to.equal('1111 2222 3333 - 3') + expect(res.body.data[ 1 ].name).to.equal('1111 2222 3333 - 4') + } + + { + const res = await advancedVideosSearch(server.url, immutableAssign(query, { languageOneOf: [ 'pl', 'en', '_unknown' ] })) + expect(res.body.total).to.equal(3) + expect(res.body.data[ 0 ].name).to.equal('1111 2222 3333 - 3') + expect(res.body.data[ 1 ].name).to.equal('1111 2222 3333 - 4') + expect(res.body.data[ 2 ].name).to.equal('1111 2222 3333 - 5') + } + + { + const res = await advancedVideosSearch(server.url, immutableAssign(query, { languageOneOf: [ 'eo' ] })) + expect(res.body.total).to.equal(0) + } }) it('Should search by start date', async function () { -- cgit v1.2.3