First implem global search

13 files changed, 255 insertions, 20 deletions

diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 41e5027b9..1d48b4b26 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -76,6 +76,12 @@ async function getConfig (req: express.Request, res: express.Response) {
       remoteUri: {
         users: CONFIG.SEARCH.REMOTE_URI.USERS,
         anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS
+      },
+      searchIndex: {
+        enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED,
+        url: CONFIG.SEARCH.SEARCH_INDEX.URL,
+        disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH,
+        isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH
       }
     },
     plugin: {
@@ -445,7 +451,19 @@ function customConfig (): CustomConfig {
       message: CONFIG.BROADCAST_MESSAGE.MESSAGE,
       level: CONFIG.BROADCAST_MESSAGE.LEVEL,
       dismissable: CONFIG.BROADCAST_MESSAGE.DISMISSABLE
-    }
+    },
+    search: {
+      remoteUri: {
+        users: CONFIG.SEARCH.REMOTE_URI.USERS,
+        anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS
+      },
+      searchIndex: {
+        enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED,
+        url: CONFIG.SEARCH.SEARCH_INDEX.URL,
+        disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH,
+        isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH
+      }
+    },
   }
 }
 
diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts
index 35d94d747..e08e1d79f 100644
--- a/server/controllers/api/search.ts
+++ b/server/controllers/api/search.ts
@@ -1,7 +1,19 @@
 import * as express from 'express'
+import { sanitizeUrl } from '@server/helpers/core-utils'
+import { doRequest } from '@server/helpers/requests'
+import { CONFIG } from '@server/initializers/config'
+import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos'
+import { AccountBlocklistModel } from '@server/models/account/account-blocklist'
+import { getServerActor } from '@server/models/application/application'
+import { ServerBlocklistModel } from '@server/models/server/server-blocklist'
+import { ResultList, Video, VideoChannel } from '@shared/models'
+import { SearchTargetQuery } from '@shared/models/search/search-target-query.model'
+import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search'
 import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils'
+import { logger } from '../../helpers/logger'
 import { getFormattedObjects } from '../../helpers/utils'
-import { VideoModel } from '../../models/video/video'
+import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
+import { getOrCreateActorAndServerAndModel } from '../../lib/activitypub/actor'
 import {
   asyncMiddleware,
   commonVideosFiltersValidator,
@@ -14,14 +26,9 @@ import {
   videosSearchSortValidator,
   videosSearchValidator
 } from '../../middlewares'
-import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search'
-import { getOrCreateActorAndServerAndModel } from '../../lib/activitypub/actor'
-import { logger } from '../../helpers/logger'
+import { VideoModel } from '../../models/video/video'
 import { VideoChannelModel } from '../../models/video/video-channel'
-import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
 import { MChannelAccountDefault, MVideoAccountLightBlacklistAllFiles } from '../../typings/models'
-import { getServerActor } from '@server/models/application/application'
-import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos'
 
 const searchRouter = express.Router()
 
@@ -68,9 +75,34 @@ function searchVideoChannels (req: express.Request, res: express.Response) {
 
   // @username -> username to search in DB
   if (query.search.startsWith('@')) query.search = query.search.replace(/^@/, '')
+
+  if (isSearchIndexEnabled(query)) {
+    return searchVideoChannelsIndex(query, res)
+  }
+
   return searchVideoChannelsDB(query, res)
 }
 
+async function searchVideoChannelsIndex (query: VideoChannelsSearchQuery, res: express.Response) {
+  logger.debug('Doing channels search on search index.')
+
+  const result = await buildMutedForSearchIndex(res)
+
+  const body = Object.assign(query, result)
+
+  const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/video-channels'
+
+  try {
+    const searchIndexResult = await doRequest<ResultList<VideoChannel>>({ uri: url, body, json: true })
+
+    return res.json(searchIndexResult.body)
+  } catch (err) {
+    logger.warn('Cannot use search index to make video channels search.', { err })
+
+    return res.sendStatus(500)
+  }
+}
+
 async function searchVideoChannelsDB (query: VideoChannelsSearchQuery, res: express.Response) {
   const serverActor = await getServerActor()
 
@@ -120,13 +152,38 @@ async function searchVideoChannelURI (search: string, isWebfingerSearch: boolean
 function searchVideos (req: express.Request, res: express.Response) {
   const query: VideosSearchQuery = req.query
   const search = query.search
+
   if (search && (search.startsWith('http://') || search.startsWith('https://'))) {
     return searchVideoURI(search, res)
   }
 
+  if (isSearchIndexEnabled(query)) {
+    return searchVideosIndex(query, res)
+  }
+
   return searchVideosDB(query, res)
 }
 
+async function searchVideosIndex (query: VideosSearchQuery, res: express.Response) {
+  logger.debug('Doing videos search on search index.')
+
+  const result = await buildMutedForSearchIndex(res)
+
+  const body = Object.assign(query, result)
+
+  const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/videos'
+
+  try {
+    const searchIndexResult = await doRequest<ResultList<Video>>({ uri: url, body, json: true })
+
+    return res.json(searchIndexResult.body)
+  } catch (err) {
+    logger.warn('Cannot use search index to make video search.', { err })
+
+    return res.sendStatus(500)
+  }
+}
+
 async function searchVideosDB (query: VideosSearchQuery, res: express.Response) {
   const options = Object.assign(query, {
     includeLocalVideos: true,
@@ -168,3 +225,35 @@ async function searchVideoURI (url: string, res: express.Response) {
     data: video ? [ video.toFormattedJSON() ] : []
   })
 }
+
+function isSearchIndexEnabled (query: SearchTargetQuery) {
+  if (query.searchTarget === 'search-index') return true
+
+  const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX
+
+  if (searchIndexConfig.ENABLED !== true) return false
+
+  if (searchIndexConfig.DISABLE_LOCAL_SEARCH) return true
+  if (searchIndexConfig.IS_DEFAULT_SEARCH && !query.searchTarget) return true
+
+  return false
+}
+
+async function buildMutedForSearchIndex (res: express.Response) {
+  const serverActor = await getServerActor()
+  const accountIds = [ serverActor.Account.id ]
+
+  if (res.locals.oauth) {
+    accountIds.push(res.locals.oauth.token.User.Account.id)
+  }
+
+  const [ blockedHosts, blockedAccounts ] = await Promise.all([
+    ServerBlocklistModel.listHostsBlockedBy(accountIds),
+    AccountBlocklistModel.listHandlesBlockedBy(accountIds)
+  ])
+
+  return {
+    blockedHosts,
+    blockedAccounts
+  }
+}
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
index b5b854137..b49ab6bca 100644
--- a/server/initializers/checker-after-init.ts
+++ b/server/initializers/checker-after-init.ts
@@ -128,6 +128,13 @@ function checkConfig () {
     }
   }
 
+  // Search index
+  if (CONFIG.SEARCH.SEARCH_INDEX.ENABLED === true) {
+    if (CONFIG.SEARCH.REMOTE_URI.USERS === false) {
+      return 'You cannot enable search index without enabling remote URI search for users.'
+    }
+  }
+
   return null
 }
 
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index bd8f02bc0..e0819c4aa 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -35,7 +35,9 @@ function checkMissedConfig () {
     'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max',
     'theme.default',
     'remote_redundancy.videos.accept_from',
-    'federation.videos.federate_unlisted'
+    'federation.videos.federate_unlisted',
+    'search.remote_uri.users', 'search.remote_uri.anonymous', 'search.search_index.enabled', 'search.search_index.url',
+    'search.search_index.disable_local_search', 'search.search_index.is_default_search'
   ]
   const requiredAlternatives = [
     [ // set
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 44fd9045b..5b402dd74 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -104,12 +104,6 @@ const CONFIG = {
     },
     ANONYMIZE_IP: config.get<boolean>('log.anonymizeIP')
   },
-  SEARCH: {
-    REMOTE_URI: {
-      USERS: config.get<boolean>('search.remote_uri.users'),
-      ANONYMOUS: config.get<boolean>('search.remote_uri.anonymous')
-    }
-  },
   TRENDING: {
     VIDEOS: {
       INTERVAL_DAYS: config.get<number>('trending.videos.interval_days')
@@ -297,6 +291,18 @@ const CONFIG = {
     get MESSAGE () { return config.get<string>('broadcast_message.message') },
     get LEVEL () { return config.get<BroadcastMessageLevel>('broadcast_message.level') },
     get DISMISSABLE () { return config.get<boolean>('broadcast_message.dismissable') }
+  },
+  SEARCH: {
+    REMOTE_URI: {
+      USERS: config.get<boolean>('search.remote_uri.users'),
+      ANONYMOUS: config.get<boolean>('search.remote_uri.anonymous')
+    },
+    SEARCH_INDEX: {
+      get ENABLED () { return config.get<boolean>('search.search_index.enabled') },
+      get URL () { return config.get<string>('search.search_index.url') },
+      get DISABLE_LOCAL_SEARCH () { return config.get<boolean>('search.search_index.disable_local_search') },
+      get IS_DEFAULT_SEARCH () { return config.get<boolean>('search.search_index.is_default_search') }
+    }
   }
 }
 
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index d201df3d8..314f094b3 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -61,6 +61,7 @@ const SORTABLE_COLUMNS = {
 
   VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending' ],
 
+  // Don't forget to update peertube-search-index with the same values
   VIDEOS_SEARCH: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'match' ],
   VIDEO_CHANNELS_SEARCH: [ 'match', 'displayName', 'createdAt' ],
 
@@ -649,6 +650,15 @@ const DEFAULT_USER_THEME_NAME = 'instance-default'
 
 // ---------------------------------------------------------------------------
 
+const SEARCH_INDEX = {
+  ROUTES: {
+    VIDEOS: '/api/v1/search/videos',
+    VIDEO_CHANNELS: '/api/v1/search/video-channels'
+  }
+}
+
+// ---------------------------------------------------------------------------
+
 // Special constants for a test instance
 if (isTestInstance() === true) {
   PRIVATE_RSA_KEY_SIZE = 1024
@@ -704,6 +714,7 @@ export {
   API_VERSION,
   PEERTUBE_VERSION,
   LAZY_STATIC_PATHS,
+  SEARCH_INDEX,
   HLS_REDUNDANCY_DIRECTORY,
   P2P_MEDIA_LOADER_PEER_VERSION,
   AVATARS_SIZE,
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 7d16bd390..6d20e0e65 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -272,11 +272,22 @@ async function getOrCreateVideoAndAccountAndChannel (
 
   const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
   const videoChannel = actor.VideoChannel
-  const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail)
 
-  await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam)
+  try {
+    const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail)
+
+    await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam)
 
-  return { video: videoCreated, created: true, autoBlacklisted }
+    return { video: videoCreated, created: true, autoBlacklisted }
+  } catch (err) {
+    // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video
+    if (err.name === 'SequelizeUniqueConstraintError') {
+      const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType)
+      if (fallbackVideo) return { video: fallbackVideo, created: false }
+    }
+
+    throw err
+  }
 }
 
 async function updateVideoFromAP (options: {
diff --git a/server/lib/plugins/plugin-index.ts b/server/lib/plugins/plugin-index.ts
index 170f0c7e2..7bcb6ed4c 100644
--- a/server/lib/plugins/plugin-index.ts
+++ b/server/lib/plugins/plugin-index.ts
@@ -11,6 +11,7 @@ import { PluginModel } from '../../models/server/plugin'
 import { PluginManager } from './plugin-manager'
 import { logger } from '../../helpers/logger'
 import { PEERTUBE_VERSION } from '../../initializers/constants'
+import { sanitizeUrl } from '@server/helpers/core-utils'
 
 async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) {
   const { start = 0, count = 20, search, sort = 'npmName', pluginType } = options
@@ -55,7 +56,7 @@ async function getLatestPluginsVersion (npmNames: string[]): Promise<PeertubePlu
     currentPeerTubeEngine: PEERTUBE_VERSION
   }
 
-  const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins/latest-version'
+  const uri = sanitizeUrl(CONFIG.PLUGINS.INDEX.URL) + '/api/v1/plugins/latest-version'
 
   const { body } = await doRequest<any>({ uri, body: bodyRequest, json: true, method: 'POST' })
 
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts
index 6905ac762..d3669f6be 100644
--- a/server/middlewares/validators/config.ts
+++ b/server/middlewares/validators/config.ts
@@ -58,7 +58,14 @@ const customConfigUpdateValidator = [
   body('broadcastMessage.enabled').isBoolean().withMessage('Should have a valid broadcast message enabled boolean'),
   body('broadcastMessage.message').exists().withMessage('Should have a valid broadcast message'),
   body('broadcastMessage.level').exists().withMessage('Should have a valid broadcast level'),
-  body('broadcastMessage.dismissable').exists().withMessage('Should have a valid broadcast dismissable boolean'),
+  body('broadcastMessage.dismissable').isBoolean().withMessage('Should have a valid broadcast dismissable boolean'),
+
+  body('search.remoteUri.users').isBoolean().withMessage('Should have a remote URI search for users boolean'),
+  body('search.remoteUri.anonymous').isBoolean().withMessage('Should have a valid remote URI search for anonymous boolean'),
+  body('search.searchIndex.enabled').isBoolean().withMessage('Should have a valid search index enabled boolean'),
+  body('search.searchIndex.url').exists().withMessage('Should have a valid search index URL'),
+  body('search.searchIndex.disableLocalSearch').isBoolean().withMessage('Should have a valid search index disable local search boolean'),
+  body('search.searchIndex.isDefaultSearch').isBoolean().withMessage('Should have a valid search index default enabled boolean'),
 
   (req: express.Request, res: express.Response, next: express.NextFunction) => {
     logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body })
diff --git a/server/models/account/account-blocklist.ts b/server/models/account/account-blocklist.ts
index d8a7ce4b4..2c6b756d2 100644
--- a/server/models/account/account-blocklist.ts
+++ b/server/models/account/account-blocklist.ts
@@ -5,6 +5,8 @@ import { AccountBlock } from '../../../shared/models/blocklist'
 import { Op } from 'sequelize'
 import * as Bluebird from 'bluebird'
 import { MAccountBlocklist, MAccountBlocklistAccounts, MAccountBlocklistFormattable } from '@server/typings/models'
+import { ActorModel } from '../activitypub/actor'
+import { ServerModel } from '../server/server'
 
 enum ScopeNames {
   WITH_ACCOUNTS = 'WITH_ACCOUNTS'
@@ -149,6 +151,42 @@ export class AccountBlocklistModel extends Model<AccountBlocklistModel> {
       })
   }
 
+  static listHandlesBlockedBy (accountIds: number[]): Bluebird<string[]> {
+    const query = {
+      attributes: [],
+      where: {
+        accountId: {
+          [Op.in]: accountIds
+        }
+      },
+      include: [
+        {
+          attributes: [ 'id' ],
+          model: AccountModel.unscoped(),
+          required: true,
+          as: 'BlockedAccount',
+          include: [
+            {
+              attributes: [ 'preferredUsername' ],
+              model: ActorModel.unscoped(),
+              required: true,
+              include: [
+                {
+                  attributes: [ 'host' ],
+                  model: ServerModel.unscoped(),
+                  required: true
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+
+    return AccountBlocklistModel.findAll(query)
+      .then(entries => entries.map(e => `${e.BlockedAccount.Actor.preferredUsername}@${e.BlockedAccount.Actor.Server.host}`))
+  }
+
   toFormattedJSON (this: MAccountBlocklistFormattable): AccountBlock {
     return {
       byAccount: this.ByAccount.toFormattedJSON(),
diff --git a/server/models/server/server-blocklist.ts b/server/models/server/server-blocklist.ts
index 892024c04..ad8e3d1e8 100644
--- a/server/models/server/server-blocklist.ts
+++ b/server/models/server/server-blocklist.ts
@@ -120,6 +120,27 @@ export class ServerBlocklistModel extends Model<ServerBlocklistModel> {
     return ServerBlocklistModel.findOne(query)
   }
 
+  static listHostsBlockedBy (accountIds: number[]): Bluebird<string[]> {
+    const query = {
+      attributes: [ ],
+      where: {
+        accountId: {
+          [Op.in]: accountIds
+        }
+      },
+      include: [
+        {
+          attributes: [ 'host' ],
+          model: ServerModel.unscoped(),
+          required: true
+        }
+      ]
+    }
+
+    return ServerBlocklistModel.findAll(query)
+      .then(entries => entries.map(e => e.BlockedServer.host))
+  }
+
   static listForApi (parameters: {
     start: number
     count: number
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 7c96fa762..3f2708f94 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -139,6 +139,18 @@ describe('Test config API validators', function () {
       dismissable: true,
       message: 'super message',
       level: 'warning'
+    },
+    search: {
+      remoteUri: {
+        users: true,
+        anonymous: true
+      },
+      searchIndex: {
+        enabled: true,
+        url: 'https://search.joinpeertube.org',
+        disableLocalSearch: true,
+        isDefaultSearch: true
+      }
     }
   }
 
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index d18a93082..597233588 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -340,6 +340,18 @@ describe('Test config', function () {
         level: 'error',
         message: 'super bad message',
         dismissable: true
+      },
+      search: {
+        remoteUri: {
+          anonymous: true,
+          users: true
+        },
+        searchIndex: {
+          enabled: true,
+          url: 'https://search.joinpeertube.org',
+          disableLocalSearch: true,
+          isDefaultSearch: true
+        }
       }
     }
     await updateCustomConfig(server.url, server.accessToken, newCustomConfig)