From fbd67e7f386504e50f2504cb6386700a58906f16 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 28 Jul 2021 16:40:21 +0200 Subject: [PATCH] Add ability to search by uuids/actor names --- .../api/search/search-video-channels.ts | 7 +- .../api/search/search-video-playlists.ts | 3 +- server/helpers/custom-validators/misc.ts | 10 +++ server/middlewares/validators/search.ts | 34 +++++++-- .../video/sql/videos-id-list-query-builder.ts | 10 +++ server/models/video/video-channel.ts | 76 ++++++++++++------- server/models/video/video-playlist.ts | 16 +++- server/models/video/video.ts | 3 + server/tests/api/check-params/search.ts | 15 ++++ server/tests/api/search/search-channels.ts | 24 +++++- server/tests/api/search/search-playlists.ts | 28 ++++++- server/tests/api/search/search-videos.ts | 28 ++++++- .../video-channels-search-query.model.ts | 3 +- .../video-playlists-search-query.model.ts | 3 +- .../search/videos-search-query.model.ts | 3 + 15 files changed, 215 insertions(+), 48 deletions(-) diff --git a/server/controllers/api/search/search-video-channels.ts b/server/controllers/api/search/search-video-channels.ts index be0b6b9a2..9fc2d53a5 100644 --- a/server/controllers/api/search/search-video-channels.ts +++ b/server/controllers/api/search/search-video-channels.ts @@ -46,7 +46,7 @@ export { searchChannelsRouter } function searchVideoChannels (req: express.Request, res: express.Response) { const query: VideoChannelsSearchQuery = req.query - const search = query.search + let search = query.search || '' const parts = search.split('@') @@ -57,7 +57,7 @@ function searchVideoChannels (req: express.Request, res: express.Response) { if (isURISearch(search) || isWebfingerSearch) return searchVideoChannelURI(search, isWebfingerSearch, res) // @username -> username to search in DB - if (query.search.startsWith('@')) query.search = query.search.replace(/^@/, '') + if (search.startsWith('@')) search = search.replace(/^@/, '') if (isSearchIndexSearch(query)) { return searchVideoChannelsIndex(query, res) @@ -99,7 +99,8 @@ async function searchVideoChannelsDB (query: VideoChannelsSearchQuery, res: expr start: query.start, count: query.count, sort: query.sort, - host: query.host + host: query.host, + names: query.names }, 'filter:api.search.video-channels.local.list.params') const resultList = await Hooks.wrapPromiseFun( diff --git a/server/controllers/api/search/search-video-playlists.ts b/server/controllers/api/search/search-video-playlists.ts index 60d1a44f7..bd6a2a564 100644 --- a/server/controllers/api/search/search-video-playlists.ts +++ b/server/controllers/api/search/search-video-playlists.ts @@ -89,7 +89,8 @@ async function searchVideoPlaylistsDB (query: VideoPlaylistsSearchQuery, res: ex start: query.start, count: query.count, sort: query.sort, - host: query.host + host: query.host, + uuids: query.uuids }, 'filter:api.search.video-playlists.local.list.params') const resultList = await Hooks.wrapPromiseFun( diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index 528bfcfb8..f8f168149 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts @@ -39,6 +39,10 @@ function isUUIDValid (value: string) { return exists(value) && validator.isUUID('' + value, 4) } +function areUUIDsValid (values: string[]) { + return isArray(values) && values.every(v => isUUIDValid(v)) +} + function isIdOrUUIDValid (value: string) { return isIdValid(value) || isUUIDValid(value) } @@ -132,6 +136,10 @@ function toCompleteUUID (value: string) { return value } +function toCompleteUUIDs (values: string[]) { + return values.map(v => toCompleteUUID(v)) +} + function toIntOrNull (value: string) { const v = toValueOrNull(value) @@ -180,6 +188,7 @@ export { isIdValid, isSafePath, isUUIDValid, + toCompleteUUIDs, toCompleteUUID, isIdOrUUIDValid, isDateValid, @@ -187,6 +196,7 @@ export { toBooleanOrNull, isBooleanValid, toIntOrNull, + areUUIDsValid, toArray, toIntArray, isFileFieldValid, diff --git a/server/middlewares/validators/search.ts b/server/middlewares/validators/search.ts index ea6a490b2..cde300968 100644 --- a/server/middlewares/validators/search.ts +++ b/server/middlewares/validators/search.ts @@ -2,7 +2,7 @@ import * as express from 'express' import { query } from 'express-validator' import { isSearchTargetValid } from '@server/helpers/custom-validators/search' import { isHostValid } from '@server/helpers/custom-validators/servers' -import { isDateValid } from '../../helpers/custom-validators/misc' +import { areUUIDsValid, isDateValid, toCompleteUUIDs } from '../../helpers/custom-validators/misc' import { logger } from '../../helpers/logger' import { areValidationErrors } from './shared' @@ -27,8 +27,18 @@ const videosSearchValidator = [ .optional() .custom(isDateValid).withMessage('Should have a published end date that conforms to ISO 8601'), - query('durationMin').optional().isInt().withMessage('Should have a valid min duration'), - query('durationMax').optional().isInt().withMessage('Should have a valid max duration'), + query('durationMin') + .optional() + .isInt().withMessage('Should have a valid min duration'), + query('durationMax') + .optional() + .isInt().withMessage('Should have a valid max duration'), + + query('uuids') + .optional() + .toArray() + .customSanitizer(toCompleteUUIDs) + .custom(areUUIDsValid).withMessage('Should have valid uuids'), query('searchTarget').optional().custom(isSearchTargetValid).withMessage('Should have a valid search target'), @@ -42,7 +52,9 @@ const videosSearchValidator = [ ] const videoChannelsListSearchValidator = [ - query('search').not().isEmpty().withMessage('Should have a valid search'), + query('search') + .optional() + .not().isEmpty().withMessage('Should have a valid search'), query('host') .optional() @@ -52,6 +64,10 @@ const videoChannelsListSearchValidator = [ .optional() .custom(isSearchTargetValid).withMessage('Should have a valid search target'), + query('names') + .optional() + .toArray(), + (req: express.Request, res: express.Response, next: express.NextFunction) => { logger.debug('Checking video channels search query', { parameters: req.query }) @@ -62,7 +78,9 @@ const videoChannelsListSearchValidator = [ ] const videoPlaylistsListSearchValidator = [ - query('search').not().isEmpty().withMessage('Should have a valid search'), + query('search') + .optional() + .not().isEmpty().withMessage('Should have a valid search'), query('host') .optional() @@ -72,6 +90,12 @@ const videoPlaylistsListSearchValidator = [ .optional() .custom(isSearchTargetValid).withMessage('Should have a valid search target'), + query('uuids') + .optional() + .toArray() + .customSanitizer(toCompleteUUIDs) + .custom(areUUIDsValid).withMessage('Should have valid uuids'), + (req: express.Request, res: express.Response, next: express.NextFunction) => { logger.debug('Checking video playlists search query', { parameters: req.query }) diff --git a/server/models/video/sql/videos-id-list-query-builder.ts b/server/models/video/sql/videos-id-list-query-builder.ts index d4260c69c..7625c003d 100644 --- a/server/models/video/sql/videos-id-list-query-builder.ts +++ b/server/models/video/sql/videos-id-list-query-builder.ts @@ -35,6 +35,8 @@ export type BuildVideosListQueryOptions = { tagsOneOf?: string[] tagsAllOf?: string[] + uuids?: string[] + withFiles?: boolean accountId?: number @@ -161,6 +163,10 @@ export class VideosIdListQueryBuilder extends AbstractVideosQueryBuilder { this.whereTagsAllOf(options.tagsAllOf) } + if (options.uuids) { + this.whereUUIDs(options.uuids) + } + if (options.nsfw === true) { this.whereNSFW() } else if (options.nsfw === false) { @@ -386,6 +392,10 @@ export class VideosIdListQueryBuilder extends AbstractVideosQueryBuilder { ) } + private whereUUIDs (uuids: string[]) { + this.and.push('"video"."uuid" IN (' + createSafeIn(this.sequelize, uuids) + ')') + } + private whereCategoryOneOf (categoryOneOf: number[]) { this.and.push('"video"."category" IN (:categoryOneOf)') this.replacements.categoryOneOf = categoryOneOf diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index 9aa271711..327f49304 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts @@ -59,6 +59,7 @@ type AvailableForListOptions = { actorId: number search?: string host?: string + names?: string[] } type AvailableWithStatsOptions = { @@ -84,18 +85,20 @@ export type SummaryOptions = { // Only list local channels OR channels that are on an instance followed by actorId const inQueryInstanceFollow = buildServerIdsFollowedBy(options.actorId) - const whereActor = { - [Op.or]: [ - { - serverId: null - }, - { - serverId: { - [Op.in]: Sequelize.literal(inQueryInstanceFollow) + const whereActorAnd: WhereOptions[] = [ + { + [Op.or]: [ + { + serverId: null + }, + { + serverId: { + [Op.in]: Sequelize.literal(inQueryInstanceFollow) + } } - } - ] - } + ] + } + ] let serverRequired = false let whereServer: WhereOptions @@ -106,8 +109,16 @@ export type SummaryOptions = { } if (options.host === WEBSERVER.HOST) { - Object.assign(whereActor, { - [Op.and]: [ { serverId: null } ] + whereActorAnd.push({ + serverId: null + }) + } + + if (options.names) { + whereActorAnd.push({ + preferredUsername: { + [Op.in]: options.names + } }) } @@ -118,7 +129,9 @@ export type SummaryOptions = { exclude: unusedActorAttributesForAPI }, model: ActorModel, - where: whereActor, + where: { + [Op.and]: whereActorAnd + }, include: [ { model: ServerModel, @@ -454,26 +467,23 @@ ON "Account->Actor"."serverId" = "Account->Actor->Server"."id"` static searchForApi (options: { actorId: number - search: string + search?: string start: number count: number sort: string host?: string + names?: string[] }) { - const attributesInclude = [] - const escapedSearch = VideoChannelModel.sequelize.escape(options.search) - const escapedLikeSearch = VideoChannelModel.sequelize.escape('%' + options.search + '%') - attributesInclude.push(createSimilarityAttribute('VideoChannelModel.name', options.search)) + let attributesInclude: any[] = [ literal('0 as similarity') ] + let where: WhereOptions - const query = { - attributes: { - include: attributesInclude - }, - offset: options.start, - limit: options.count, - order: getSort(options.sort), - where: { + if (options.search) { + const escapedSearch = VideoChannelModel.sequelize.escape(options.search) + const escapedLikeSearch = VideoChannelModel.sequelize.escape('%' + options.search + '%') + attributesInclude = [ createSimilarityAttribute('VideoChannelModel.name', options.search) ] + + where = { [Op.or]: [ Sequelize.literal( 'lower(immutable_unaccent("VideoChannelModel"."name")) % lower(immutable_unaccent(' + escapedSearch + '))' @@ -485,9 +495,19 @@ ON "Account->Actor"."serverId" = "Account->Actor->Server"."id"` } } + const query = { + attributes: { + include: attributesInclude + }, + offset: options.start, + limit: options.count, + order: getSort(options.sort), + where + } + return VideoChannelModel .scope({ - method: [ ScopeNames.FOR_API, { actorId: options.actorId, host: options.host } as AvailableForListOptions ] + method: [ ScopeNames.FOR_API, { actorId: options.actorId, host: options.host, names: options.names } as AvailableForListOptions ] }) .findAndCountAll(query) .then(({ rows, count }) => { diff --git a/server/models/video/video-playlist.ts b/server/models/video/video-playlist.ts index a2dc7075d..caa79952d 100644 --- a/server/models/video/video-playlist.ts +++ b/server/models/video/video-playlist.ts @@ -83,6 +83,7 @@ type AvailableForListOptions = { listMyPlaylists?: boolean search?: string host?: string + uuids?: string[] withVideos?: boolean } @@ -200,18 +201,26 @@ function getVideoLengthSelect () { }) } + if (options.uuids) { + whereAnd.push({ + uuid: { + [Op.in]: options.uuids + } + }) + } + if (options.withVideos === true) { whereAnd.push( literal(`(${getVideoLengthSelect()}) != 0`) ) } - const attributesInclude = [] + let attributesInclude: any[] = [ literal('0 as similarity') ] if (options.search) { const escapedSearch = VideoPlaylistModel.sequelize.escape(options.search) const escapedLikeSearch = VideoPlaylistModel.sequelize.escape('%' + options.search + '%') - attributesInclude.push(createSimilarityAttribute('VideoPlaylistModel.name', options.search)) + attributesInclude = [ createSimilarityAttribute('VideoPlaylistModel.name', options.search) ] whereAnd.push({ [Op.or]: [ @@ -359,6 +368,7 @@ export class VideoPlaylistModel extends Model>> { durationMax?: number // seconds user?: MUserAccountId filter?: VideoFilter + uuids?: string[] }) { const serverActor = await getServerActor() @@ -1167,6 +1168,8 @@ export class VideoModel extends Model>> { durationMin: options.durationMin, durationMax: options.durationMax, + uuids: options.uuids, + search: options.search } diff --git a/server/tests/api/check-params/search.ts b/server/tests/api/check-params/search.ts index 72ad6c842..789ea7754 100644 --- a/server/tests/api/check-params/search.ts +++ b/server/tests/api/check-params/search.ts @@ -146,6 +146,16 @@ describe('Test videos API validator', function () { const customQuery = { ...query, host: 'example.com' } await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) }) + + it('Should fail with invalid uuids', async function () { + const customQuery = { ...query, uuids: [ '6565', 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with valid uuids', async function () { + const customQuery = { ...query, uuids: [ 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) + }) }) describe('When searching video playlists', function () { @@ -172,6 +182,11 @@ describe('Test videos API validator', function () { await makeGetRequest({ url: server.url, path, query: { ...query, host: '6565' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) }) + it('Should fail with invalid uuids', async function () { + const customQuery = { ...query, uuids: [ '6565', 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + it('Should succeed with the correct parameters', async function () { await makeGetRequest({ url: server.url, path, query, expectedStatus: HttpStatusCode.OK_200 }) }) diff --git a/server/tests/api/search/search-channels.ts b/server/tests/api/search/search-channels.ts index aab03bfd1..ef78c0f67 100644 --- a/server/tests/api/search/search-channels.ts +++ b/server/tests/api/search/search-channels.ts @@ -22,8 +22,12 @@ describe('Test channels search', function () { before(async function () { this.timeout(120000) - server = await createSingleServer(1) - remoteServer = await createSingleServer(2, { transcoding: { enabled: false } }) + const servers = await Promise.all([ + createSingleServer(1), + createSingleServer(2, { transcoding: { enabled: false } }) + ]) + server = servers[0] + remoteServer = servers[1] await setAccessTokensToServers([ server, remoteServer ]) @@ -116,6 +120,22 @@ describe('Test channels search', function () { } }) + it('Should filter by names', async function () { + { + const body = await command.advancedChannelSearch({ search: { names: [ 'squall_channel', 'zell_channel' ] } }) + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) + expect(body.data[0].displayName).to.equal('Squall channel') + expect(body.data[1].displayName).to.equal('Zell channel') + } + + { + const body = await command.advancedChannelSearch({ search: { names: [ 'chocobozzz_channel' ] } }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + }) + after(async function () { await cleanupTests([ server ]) }) diff --git a/server/tests/api/search/search-playlists.ts b/server/tests/api/search/search-playlists.ts index e7e53ff41..85be1eb59 100644 --- a/server/tests/api/search/search-playlists.ts +++ b/server/tests/api/search/search-playlists.ts @@ -19,12 +19,18 @@ describe('Test playlists search', function () { let server: PeerTubeServer let remoteServer: PeerTubeServer let command: SearchCommand + let playlistUUID: string + let playlistShortUUID: string before(async function () { this.timeout(120000) - server = await createSingleServer(1) - remoteServer = await createSingleServer(2, { transcoding: { enabled: false } }) + const servers = await Promise.all([ + createSingleServer(1), + createSingleServer(2, { transcoding: { enabled: false } }) + ]) + server = servers[0] + remoteServer = servers[1] await setAccessTokensToServers([ remoteServer, server ]) await setDefaultVideoChannel([ remoteServer, server ]) @@ -38,6 +44,8 @@ describe('Test playlists search', function () { videoChannelId: server.store.channel.id } const created = await server.playlists.create({ attributes }) + playlistUUID = created.uuid + playlistShortUUID = created.shortUUID await server.playlists.addElement({ playlistId: created.id, attributes: { videoId } }) } @@ -136,6 +144,22 @@ describe('Test playlists search', function () { } }) + it('Should filter by UUIDs', async function () { + for (const uuid of [ playlistUUID, playlistShortUUID ]) { + const body = await command.advancedPlaylistSearch({ search: { uuids: [ uuid ] } }) + + expect(body.total).to.equal(1) + expect(body.data[0].displayName).to.equal('Dr. Kenzo Tenma hospital videos') + } + + { + const body = await command.advancedPlaylistSearch({ search: { uuids: [ 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + }) + it('Should not display playlists without videos', async function () { const search = { search: 'Lunge', diff --git a/server/tests/api/search/search-videos.ts b/server/tests/api/search/search-videos.ts index a56dc1d87..bd1e4d266 100644 --- a/server/tests/api/search/search-videos.ts +++ b/server/tests/api/search/search-videos.ts @@ -22,14 +22,19 @@ describe('Test videos search', function () { let remoteServer: PeerTubeServer let startDate: string let videoUUID: string + let videoShortUUID: string let command: SearchCommand before(async function () { this.timeout(120000) - server = await createSingleServer(1) - remoteServer = await createSingleServer(2) + const servers = await Promise.all([ + createSingleServer(1), + createSingleServer(2) + ]) + server = servers[0] + remoteServer = servers[1] await setAccessTokensToServers([ server, remoteServer ]) await setDefaultVideoChannel([ server, remoteServer ]) @@ -50,8 +55,9 @@ describe('Test videos search', function () { { const attributes3 = { ...attributes1, name: attributes1.name + ' - 3', language: undefined } - const { id, uuid } = await server.videos.upload({ attributes: attributes3 }) + const { id, uuid, shortUUID } = await server.videos.upload({ attributes: attributes3 }) videoUUID = uuid + videoShortUUID = shortUUID await server.captions.add({ language: 'en', @@ -479,6 +485,22 @@ describe('Test videos search', function () { expect(body.data[0].name).to.equal('1111 2222 3333 - 3') }) + it('Should filter by UUIDs', async function () { + for (const uuid of [ videoUUID, videoShortUUID ]) { + const body = await command.advancedVideoSearch({ search: { uuids: [ uuid ] } }) + + expect(body.total).to.equal(1) + expect(body.data[0].name).to.equal('1111 2222 3333 - 3') + } + + { + const body = await command.advancedVideoSearch({ search: { uuids: [ 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + }) + it('Should search by host', async function () { { const body = await command.advancedVideoSearch({ search: { search: '6666 7777 8888', host: server.host } }) diff --git a/shared/models/search/video-channels-search-query.model.ts b/shared/models/search/video-channels-search-query.model.ts index 2622dfbc6..50c59d41d 100644 --- a/shared/models/search/video-channels-search-query.model.ts +++ b/shared/models/search/video-channels-search-query.model.ts @@ -1,11 +1,12 @@ import { SearchTargetQuery } from './search-target-query.model' export interface VideoChannelsSearchQuery extends SearchTargetQuery { - search: string + search?: string start?: number count?: number sort?: string host?: string + names?: string[] } diff --git a/shared/models/search/video-playlists-search-query.model.ts b/shared/models/search/video-playlists-search-query.model.ts index dcf66e9e3..55393c92a 100644 --- a/shared/models/search/video-playlists-search-query.model.ts +++ b/shared/models/search/video-playlists-search-query.model.ts @@ -1,11 +1,12 @@ import { SearchTargetQuery } from './search-target-query.model' export interface VideoPlaylistsSearchQuery extends SearchTargetQuery { - search: string + search?: string start?: number count?: number sort?: string host?: string + uuids?: string[] } diff --git a/shared/models/search/videos-search-query.model.ts b/shared/models/search/videos-search-query.model.ts index a568c960e..736d89577 100644 --- a/shared/models/search/videos-search-query.model.ts +++ b/shared/models/search/videos-search-query.model.ts @@ -14,4 +14,7 @@ export interface VideosSearchQuery extends SearchTargetQuery, VideosCommonQuery durationMin?: number // seconds durationMax?: number // seconds + + // UUIDs or short + uuids?: string[] } -- 2.41.0