From 5fb2e2888ce032c638e4b75d07458642f0833e52 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 29 May 2020 16:16:24 +0200 Subject: First implem global search --- server/controllers/api/config.ts | 20 +++++++- server/controllers/api/search.ts | 103 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 115 insertions(+), 8 deletions(-) (limited to 'server/controllers/api') diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 41e5027b9..1d48b4b26 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -76,6 +76,12 @@ async function getConfig (req: express.Request, res: express.Response) { remoteUri: { users: CONFIG.SEARCH.REMOTE_URI.USERS, anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS + }, + searchIndex: { + enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED, + url: CONFIG.SEARCH.SEARCH_INDEX.URL, + disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH, + isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH } }, plugin: { @@ -445,7 +451,19 @@ function customConfig (): CustomConfig { message: CONFIG.BROADCAST_MESSAGE.MESSAGE, level: CONFIG.BROADCAST_MESSAGE.LEVEL, dismissable: CONFIG.BROADCAST_MESSAGE.DISMISSABLE - } + }, + search: { + remoteUri: { + users: CONFIG.SEARCH.REMOTE_URI.USERS, + anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS + }, + searchIndex: { + enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED, + url: CONFIG.SEARCH.SEARCH_INDEX.URL, + disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH, + isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH + } + }, } } diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index 35d94d747..e08e1d79f 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts @@ -1,7 +1,19 @@ import * as express from 'express' +import { sanitizeUrl } from '@server/helpers/core-utils' +import { doRequest } from '@server/helpers/requests' +import { CONFIG } from '@server/initializers/config' +import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos' +import { AccountBlocklistModel } from '@server/models/account/account-blocklist' +import { getServerActor } from '@server/models/application/application' +import { ServerBlocklistModel } from '@server/models/server/server-blocklist' +import { ResultList, Video, VideoChannel } from '@shared/models' +import { SearchTargetQuery } from '@shared/models/search/search-target-query.model' +import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search' import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' +import { logger } from '../../helpers/logger' import { getFormattedObjects } from '../../helpers/utils' -import { VideoModel } from '../../models/video/video' +import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger' +import { getOrCreateActorAndServerAndModel } from '../../lib/activitypub/actor' import { asyncMiddleware, commonVideosFiltersValidator, @@ -14,14 +26,9 @@ import { videosSearchSortValidator, videosSearchValidator } from '../../middlewares' -import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search' -import { getOrCreateActorAndServerAndModel } from '../../lib/activitypub/actor' -import { logger } from '../../helpers/logger' +import { VideoModel } from '../../models/video/video' import { VideoChannelModel } from '../../models/video/video-channel' -import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger' import { MChannelAccountDefault, MVideoAccountLightBlacklistAllFiles } from '../../typings/models' -import { getServerActor } from '@server/models/application/application' -import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos' const searchRouter = express.Router() @@ -68,9 +75,34 @@ function searchVideoChannels (req: express.Request, res: express.Response) { // @username -> username to search in DB if (query.search.startsWith('@')) query.search = query.search.replace(/^@/, '') + + if (isSearchIndexEnabled(query)) { + return searchVideoChannelsIndex(query, res) + } + return searchVideoChannelsDB(query, res) } +async function searchVideoChannelsIndex (query: VideoChannelsSearchQuery, res: express.Response) { + logger.debug('Doing channels search on search index.') + + const result = await buildMutedForSearchIndex(res) + + const body = Object.assign(query, result) + + const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/video-channels' + + try { + const searchIndexResult = await doRequest>({ uri: url, body, json: true }) + + return res.json(searchIndexResult.body) + } catch (err) { + logger.warn('Cannot use search index to make video channels search.', { err }) + + return res.sendStatus(500) + } +} + async function searchVideoChannelsDB (query: VideoChannelsSearchQuery, res: express.Response) { const serverActor = await getServerActor() @@ -120,13 +152,38 @@ async function searchVideoChannelURI (search: string, isWebfingerSearch: boolean function searchVideos (req: express.Request, res: express.Response) { const query: VideosSearchQuery = req.query const search = query.search + if (search && (search.startsWith('http://') || search.startsWith('https://'))) { return searchVideoURI(search, res) } + if (isSearchIndexEnabled(query)) { + return searchVideosIndex(query, res) + } + return searchVideosDB(query, res) } +async function searchVideosIndex (query: VideosSearchQuery, res: express.Response) { + logger.debug('Doing videos search on search index.') + + const result = await buildMutedForSearchIndex(res) + + const body = Object.assign(query, result) + + const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/videos' + + try { + const searchIndexResult = await doRequest>({ uri: url, body, json: true }) + + return res.json(searchIndexResult.body) + } catch (err) { + logger.warn('Cannot use search index to make video search.', { err }) + + return res.sendStatus(500) + } +} + async function searchVideosDB (query: VideosSearchQuery, res: express.Response) { const options = Object.assign(query, { includeLocalVideos: true, @@ -168,3 +225,35 @@ async function searchVideoURI (url: string, res: express.Response) { data: video ? [ video.toFormattedJSON() ] : [] }) } + +function isSearchIndexEnabled (query: SearchTargetQuery) { + if (query.searchTarget === 'search-index') return true + + const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX + + if (searchIndexConfig.ENABLED !== true) return false + + if (searchIndexConfig.DISABLE_LOCAL_SEARCH) return true + if (searchIndexConfig.IS_DEFAULT_SEARCH && !query.searchTarget) return true + + return false +} + +async function buildMutedForSearchIndex (res: express.Response) { + const serverActor = await getServerActor() + const accountIds = [ serverActor.Account.id ] + + if (res.locals.oauth) { + accountIds.push(res.locals.oauth.token.User.Account.id) + } + + const [ blockedHosts, blockedAccounts ] = await Promise.all([ + ServerBlocklistModel.listHostsBlockedBy(accountIds), + AccountBlocklistModel.listHandlesBlockedBy(accountIds) + ]) + + return { + blockedHosts, + blockedAccounts + } +} -- cgit v1.2.3