From 5fb2e2888ce032c638e4b75d07458642f0833e52 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 29 May 2020 16:16:24 +0200 Subject: First implem global search --- server/controllers/api/config.ts | 20 +++++- server/controllers/api/search.ts | 103 +++++++++++++++++++++++++++-- server/initializers/checker-after-init.ts | 7 ++ server/initializers/checker-before-init.ts | 4 +- server/initializers/config.ts | 18 +++-- server/initializers/constants.ts | 11 +++ server/lib/activitypub/videos.ts | 17 ++++- server/lib/plugins/plugin-index.ts | 3 +- server/middlewares/validators/config.ts | 9 ++- server/models/account/account-blocklist.ts | 38 +++++++++++ server/models/server/server-blocklist.ts | 21 ++++++ server/tests/api/check-params/config.ts | 12 ++++ server/tests/api/server/config.ts | 12 ++++ 13 files changed, 255 insertions(+), 20 deletions(-) (limited to 'server') diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 41e5027b9..1d48b4b26 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -76,6 +76,12 @@ async function getConfig (req: express.Request, res: express.Response) { remoteUri: { users: CONFIG.SEARCH.REMOTE_URI.USERS, anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS + }, + searchIndex: { + enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED, + url: CONFIG.SEARCH.SEARCH_INDEX.URL, + disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH, + isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH } }, plugin: { @@ -445,7 +451,19 @@ function customConfig (): CustomConfig { message: CONFIG.BROADCAST_MESSAGE.MESSAGE, level: CONFIG.BROADCAST_MESSAGE.LEVEL, dismissable: CONFIG.BROADCAST_MESSAGE.DISMISSABLE - } + }, + search: { + remoteUri: { + users: CONFIG.SEARCH.REMOTE_URI.USERS, + anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS + }, + searchIndex: { + enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED, + url: CONFIG.SEARCH.SEARCH_INDEX.URL, + disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH, + isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH + } + }, } } diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index 35d94d747..e08e1d79f 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts @@ -1,7 +1,19 @@ import * as express from 'express' +import { sanitizeUrl } from '@server/helpers/core-utils' +import { doRequest } from '@server/helpers/requests' +import { CONFIG } from '@server/initializers/config' +import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos' +import { AccountBlocklistModel } from '@server/models/account/account-blocklist' +import { getServerActor } from '@server/models/application/application' +import { ServerBlocklistModel } from '@server/models/server/server-blocklist' +import { ResultList, Video, VideoChannel } from '@shared/models' +import { SearchTargetQuery } from '@shared/models/search/search-target-query.model' +import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search' import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' +import { logger } from '../../helpers/logger' import { getFormattedObjects } from '../../helpers/utils' -import { VideoModel } from '../../models/video/video' +import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger' +import { getOrCreateActorAndServerAndModel } from '../../lib/activitypub/actor' import { asyncMiddleware, commonVideosFiltersValidator, @@ -14,14 +26,9 @@ import { videosSearchSortValidator, videosSearchValidator } from '../../middlewares' -import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search' -import { getOrCreateActorAndServerAndModel } from '../../lib/activitypub/actor' -import { logger } from '../../helpers/logger' +import { VideoModel } from '../../models/video/video' import { VideoChannelModel } from '../../models/video/video-channel' -import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger' import { MChannelAccountDefault, MVideoAccountLightBlacklistAllFiles } from '../../typings/models' -import { getServerActor } from '@server/models/application/application' -import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos' const searchRouter = express.Router() @@ -68,9 +75,34 @@ function searchVideoChannels (req: express.Request, res: express.Response) { // @username -> username to search in DB if (query.search.startsWith('@')) query.search = query.search.replace(/^@/, '') + + if (isSearchIndexEnabled(query)) { + return searchVideoChannelsIndex(query, res) + } + return searchVideoChannelsDB(query, res) } +async function searchVideoChannelsIndex (query: VideoChannelsSearchQuery, res: express.Response) { + logger.debug('Doing channels search on search index.') + + const result = await buildMutedForSearchIndex(res) + + const body = Object.assign(query, result) + + const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/video-channels' + + try { + const searchIndexResult = await doRequest>({ uri: url, body, json: true }) + + return res.json(searchIndexResult.body) + } catch (err) { + logger.warn('Cannot use search index to make video channels search.', { err }) + + return res.sendStatus(500) + } +} + async function searchVideoChannelsDB (query: VideoChannelsSearchQuery, res: express.Response) { const serverActor = await getServerActor() @@ -120,13 +152,38 @@ async function searchVideoChannelURI (search: string, isWebfingerSearch: boolean function searchVideos (req: express.Request, res: express.Response) { const query: VideosSearchQuery = req.query const search = query.search + if (search && (search.startsWith('http://') || search.startsWith('https://'))) { return searchVideoURI(search, res) } + if (isSearchIndexEnabled(query)) { + return searchVideosIndex(query, res) + } + return searchVideosDB(query, res) } +async function searchVideosIndex (query: VideosSearchQuery, res: express.Response) { + logger.debug('Doing videos search on search index.') + + const result = await buildMutedForSearchIndex(res) + + const body = Object.assign(query, result) + + const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/videos' + + try { + const searchIndexResult = await doRequest>({ uri: url, body, json: true }) + + return res.json(searchIndexResult.body) + } catch (err) { + logger.warn('Cannot use search index to make video search.', { err }) + + return res.sendStatus(500) + } +} + async function searchVideosDB (query: VideosSearchQuery, res: express.Response) { const options = Object.assign(query, { includeLocalVideos: true, @@ -168,3 +225,35 @@ async function searchVideoURI (url: string, res: express.Response) { data: video ? [ video.toFormattedJSON() ] : [] }) } + +function isSearchIndexEnabled (query: SearchTargetQuery) { + if (query.searchTarget === 'search-index') return true + + const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX + + if (searchIndexConfig.ENABLED !== true) return false + + if (searchIndexConfig.DISABLE_LOCAL_SEARCH) return true + if (searchIndexConfig.IS_DEFAULT_SEARCH && !query.searchTarget) return true + + return false +} + +async function buildMutedForSearchIndex (res: express.Response) { + const serverActor = await getServerActor() + const accountIds = [ serverActor.Account.id ] + + if (res.locals.oauth) { + accountIds.push(res.locals.oauth.token.User.Account.id) + } + + const [ blockedHosts, blockedAccounts ] = await Promise.all([ + ServerBlocklistModel.listHostsBlockedBy(accountIds), + AccountBlocklistModel.listHandlesBlockedBy(accountIds) + ]) + + return { + blockedHosts, + blockedAccounts + } +} diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index b5b854137..b49ab6bca 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts @@ -128,6 +128,13 @@ function checkConfig () { } } + // Search index + if (CONFIG.SEARCH.SEARCH_INDEX.ENABLED === true) { + if (CONFIG.SEARCH.REMOTE_URI.USERS === false) { + return 'You cannot enable search index without enabling remote URI search for users.' + } + } + return null } diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index bd8f02bc0..e0819c4aa 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -35,7 +35,9 @@ function checkMissedConfig () { 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', 'theme.default', 'remote_redundancy.videos.accept_from', - 'federation.videos.federate_unlisted' + 'federation.videos.federate_unlisted', + 'search.remote_uri.users', 'search.remote_uri.anonymous', 'search.search_index.enabled', 'search.search_index.url', + 'search.search_index.disable_local_search', 'search.search_index.is_default_search' ] const requiredAlternatives = [ [ // set diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 44fd9045b..5b402dd74 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -104,12 +104,6 @@ const CONFIG = { }, ANONYMIZE_IP: config.get('log.anonymizeIP') }, - SEARCH: { - REMOTE_URI: { - USERS: config.get('search.remote_uri.users'), - ANONYMOUS: config.get('search.remote_uri.anonymous') - } - }, TRENDING: { VIDEOS: { INTERVAL_DAYS: config.get('trending.videos.interval_days') @@ -297,6 +291,18 @@ const CONFIG = { get MESSAGE () { return config.get('broadcast_message.message') }, get LEVEL () { return config.get('broadcast_message.level') }, get DISMISSABLE () { return config.get('broadcast_message.dismissable') } + }, + SEARCH: { + REMOTE_URI: { + USERS: config.get('search.remote_uri.users'), + ANONYMOUS: config.get('search.remote_uri.anonymous') + }, + SEARCH_INDEX: { + get ENABLED () { return config.get('search.search_index.enabled') }, + get URL () { return config.get('search.search_index.url') }, + get DISABLE_LOCAL_SEARCH () { return config.get('search.search_index.disable_local_search') }, + get IS_DEFAULT_SEARCH () { return config.get('search.search_index.is_default_search') } + } } } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index d201df3d8..314f094b3 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -61,6 +61,7 @@ const SORTABLE_COLUMNS = { VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending' ], + // Don't forget to update peertube-search-index with the same values VIDEOS_SEARCH: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'match' ], VIDEO_CHANNELS_SEARCH: [ 'match', 'displayName', 'createdAt' ], @@ -649,6 +650,15 @@ const DEFAULT_USER_THEME_NAME = 'instance-default' // --------------------------------------------------------------------------- +const SEARCH_INDEX = { + ROUTES: { + VIDEOS: '/api/v1/search/videos', + VIDEO_CHANNELS: '/api/v1/search/video-channels' + } +} + +// --------------------------------------------------------------------------- + // Special constants for a test instance if (isTestInstance() === true) { PRIVATE_RSA_KEY_SIZE = 1024 @@ -704,6 +714,7 @@ export { API_VERSION, PEERTUBE_VERSION, LAZY_STATIC_PATHS, + SEARCH_INDEX, HLS_REDUNDANCY_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, AVATARS_SIZE, diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 7d16bd390..6d20e0e65 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -272,11 +272,22 @@ async function getOrCreateVideoAndAccountAndChannel ( const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) const videoChannel = actor.VideoChannel - const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail) - await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam) + try { + const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail) + + await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam) - return { video: videoCreated, created: true, autoBlacklisted } + return { video: videoCreated, created: true, autoBlacklisted } + } catch (err) { + // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video + if (err.name === 'SequelizeUniqueConstraintError') { + const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType) + if (fallbackVideo) return { video: fallbackVideo, created: false } + } + + throw err + } } async function updateVideoFromAP (options: { diff --git a/server/lib/plugins/plugin-index.ts b/server/lib/plugins/plugin-index.ts index 170f0c7e2..7bcb6ed4c 100644 --- a/server/lib/plugins/plugin-index.ts +++ b/server/lib/plugins/plugin-index.ts @@ -11,6 +11,7 @@ import { PluginModel } from '../../models/server/plugin' import { PluginManager } from './plugin-manager' import { logger } from '../../helpers/logger' import { PEERTUBE_VERSION } from '../../initializers/constants' +import { sanitizeUrl } from '@server/helpers/core-utils' async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) { const { start = 0, count = 20, search, sort = 'npmName', pluginType } = options @@ -55,7 +56,7 @@ async function getLatestPluginsVersion (npmNames: string[]): Promise({ uri, body: bodyRequest, json: true, method: 'POST' }) diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index 6905ac762..d3669f6be 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts @@ -58,7 +58,14 @@ const customConfigUpdateValidator = [ body('broadcastMessage.enabled').isBoolean().withMessage('Should have a valid broadcast message enabled boolean'), body('broadcastMessage.message').exists().withMessage('Should have a valid broadcast message'), body('broadcastMessage.level').exists().withMessage('Should have a valid broadcast level'), - body('broadcastMessage.dismissable').exists().withMessage('Should have a valid broadcast dismissable boolean'), + body('broadcastMessage.dismissable').isBoolean().withMessage('Should have a valid broadcast dismissable boolean'), + + body('search.remoteUri.users').isBoolean().withMessage('Should have a remote URI search for users boolean'), + body('search.remoteUri.anonymous').isBoolean().withMessage('Should have a valid remote URI search for anonymous boolean'), + body('search.searchIndex.enabled').isBoolean().withMessage('Should have a valid search index enabled boolean'), + body('search.searchIndex.url').exists().withMessage('Should have a valid search index URL'), + body('search.searchIndex.disableLocalSearch').isBoolean().withMessage('Should have a valid search index disable local search boolean'), + body('search.searchIndex.isDefaultSearch').isBoolean().withMessage('Should have a valid search index default enabled boolean'), (req: express.Request, res: express.Response, next: express.NextFunction) => { logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body }) diff --git a/server/models/account/account-blocklist.ts b/server/models/account/account-blocklist.ts index d8a7ce4b4..2c6b756d2 100644 --- a/server/models/account/account-blocklist.ts +++ b/server/models/account/account-blocklist.ts @@ -5,6 +5,8 @@ import { AccountBlock } from '../../../shared/models/blocklist' import { Op } from 'sequelize' import * as Bluebird from 'bluebird' import { MAccountBlocklist, MAccountBlocklistAccounts, MAccountBlocklistFormattable } from '@server/typings/models' +import { ActorModel } from '../activitypub/actor' +import { ServerModel } from '../server/server' enum ScopeNames { WITH_ACCOUNTS = 'WITH_ACCOUNTS' @@ -149,6 +151,42 @@ export class AccountBlocklistModel extends Model { }) } + static listHandlesBlockedBy (accountIds: number[]): Bluebird { + const query = { + attributes: [], + where: { + accountId: { + [Op.in]: accountIds + } + }, + include: [ + { + attributes: [ 'id' ], + model: AccountModel.unscoped(), + required: true, + as: 'BlockedAccount', + include: [ + { + attributes: [ 'preferredUsername' ], + model: ActorModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: true + } + ] + } + ] + } + ] + } + + return AccountBlocklistModel.findAll(query) + .then(entries => entries.map(e => `${e.BlockedAccount.Actor.preferredUsername}@${e.BlockedAccount.Actor.Server.host}`)) + } + toFormattedJSON (this: MAccountBlocklistFormattable): AccountBlock { return { byAccount: this.ByAccount.toFormattedJSON(), diff --git a/server/models/server/server-blocklist.ts b/server/models/server/server-blocklist.ts index 892024c04..ad8e3d1e8 100644 --- a/server/models/server/server-blocklist.ts +++ b/server/models/server/server-blocklist.ts @@ -120,6 +120,27 @@ export class ServerBlocklistModel extends Model { return ServerBlocklistModel.findOne(query) } + static listHostsBlockedBy (accountIds: number[]): Bluebird { + const query = { + attributes: [ ], + where: { + accountId: { + [Op.in]: accountIds + } + }, + include: [ + { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: true + } + ] + } + + return ServerBlocklistModel.findAll(query) + .then(entries => entries.map(e => e.BlockedServer.host)) + } + static listForApi (parameters: { start: number count: number diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 7c96fa762..3f2708f94 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -139,6 +139,18 @@ describe('Test config API validators', function () { dismissable: true, message: 'super message', level: 'warning' + }, + search: { + remoteUri: { + users: true, + anonymous: true + }, + searchIndex: { + enabled: true, + url: 'https://search.joinpeertube.org', + disableLocalSearch: true, + isDefaultSearch: true + } } } diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index d18a93082..597233588 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -340,6 +340,18 @@ describe('Test config', function () { level: 'error', message: 'super bad message', dismissable: true + }, + search: { + remoteUri: { + anonymous: true, + users: true + }, + searchIndex: { + enabled: true, + url: 'https://search.joinpeertube.org', + disableLocalSearch: true, + isDefaultSearch: true + } } } await updateCustomConfig(server.url, server.accessToken, newCustomConfig) -- cgit v1.2.3