server/controllers/api/search.ts

   1 import * as express from 'express'
   2 import { sanitizeUrl } from '@server/helpers/core-utils'
   3 import { doRequest } from '@server/helpers/requests'
   4 import { CONFIG } from '@server/initializers/config'
   5 import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos'
   6 import { AccountBlocklistModel } from '@server/models/account/account-blocklist'
   7 import { getServerActor } from '@server/models/application/application'
   8 import { ServerBlocklistModel } from '@server/models/server/server-blocklist'
   9 import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes'
  10 import { ResultList, Video, VideoChannel } from '@shared/models'
  11 import { SearchTargetQuery } from '@shared/models/search/search-target-query.model'
  12 import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search'
  13 import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils'
  14 import { logger } from '../../helpers/logger'
  15 import { getFormattedObjects } from '../../helpers/utils'
  16 import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
  17 import { getOrCreateActorAndServerAndModel } from '../../lib/activitypub/actor'
  18 import {
  19   asyncMiddleware,
  20   commonVideosFiltersValidator,
  21   optionalAuthenticate,
  22   paginationValidator,
  23   setDefaultPagination,
  24   setDefaultSearchSort,
  25   videoChannelsSearchSortValidator,
  26   videoChannelsListSearchValidator,
  27   videosSearchSortValidator,
  28   videosSearchValidator
  29 } from '../../middlewares'
  30 import { VideoModel } from '../../models/video/video'
  31 import { VideoChannelModel } from '../../models/video/video-channel'
  32 import { MChannelAccountDefault, MVideoAccountLightBlacklistAllFiles } from '../../types/models'
  33
  34 const searchRouter = express.Router()
  35
  36 searchRouter.get('/videos',
  37   paginationValidator,
  38   setDefaultPagination,
  39   videosSearchSortValidator,
  40   setDefaultSearchSort,
  41   optionalAuthenticate,
  42   commonVideosFiltersValidator,
  43   videosSearchValidator,
  44   asyncMiddleware(searchVideos)
  45 )
  46
  47 searchRouter.get('/video-channels',
  48   paginationValidator,
  49   setDefaultPagination,
  50   videoChannelsSearchSortValidator,
  51   setDefaultSearchSort,
  52   optionalAuthenticate,
  53   videoChannelsListSearchValidator,
  54   asyncMiddleware(searchVideoChannels)
  55 )
  56
  57 // ---------------------------------------------------------------------------
  58
  59 export { searchRouter }
  60
  61 // ---------------------------------------------------------------------------
  62
  63 function searchVideoChannels (req: express.Request, res: express.Response) {
  64   const query: VideoChannelsSearchQuery = req.query
  65   const search = query.search
  66
  67   const isURISearch = search.startsWith('http://') || search.startsWith('https://')
  68
  69   const parts = search.split('@')
  70
  71   // Handle strings like @toto@example.com
  72   if (parts.length === 3 && parts[0].length === 0) parts.shift()
  73   const isWebfingerSearch = parts.length === 2 && parts.every(p => p && !p.includes(' '))
  74
  75   if (isURISearch || isWebfingerSearch) return searchVideoChannelURI(search, isWebfingerSearch, res)
  76
  77   // @username -> username to search in DB
  78   if (query.search.startsWith('@')) query.search = query.search.replace(/^@/, '')
  79
  80   if (isSearchIndexSearch(query)) {
  81     return searchVideoChannelsIndex(query, res)
  82   }
  83
  84   return searchVideoChannelsDB(query, res)
  85 }
  86
  87 async function searchVideoChannelsIndex (query: VideoChannelsSearchQuery, res: express.Response) {
  88   const result = await buildMutedForSearchIndex(res)
  89
  90   const body = Object.assign(query, result)
  91
  92   const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/video-channels'
  93
  94   try {
  95     logger.debug('Doing video channels search index request on %s.', url, { body })
  96
  97     const searchIndexResult = await doRequest<ResultList<VideoChannel>>({ uri: url, body, json: true })
  98
  99     return res.json(searchIndexResult.body)
 100   } catch (err) {
 101     logger.warn('Cannot use search index to make video channels search.', { err })
 102
 103     return res.sendStatus(HttpStatusCode.INTERNAL_SERVER_ERROR_500)
 104   }
 105 }
 106
 107 async function searchVideoChannelsDB (query: VideoChannelsSearchQuery, res: express.Response) {
 108   const serverActor = await getServerActor()
 109
 110   const options = {
 111     actorId: serverActor.id,
 112     search: query.search,
 113     start: query.start,
 114     count: query.count,
 115     sort: query.sort
 116   }
 117   const resultList = await VideoChannelModel.searchForApi(options)
 118
 119   return res.json(getFormattedObjects(resultList.data, resultList.total))
 120 }
 121
 122 async function searchVideoChannelURI (search: string, isWebfingerSearch: boolean, res: express.Response) {
 123   let videoChannel: MChannelAccountDefault
 124   let uri = search
 125
 126   if (isWebfingerSearch) {
 127     try {
 128       uri = await loadActorUrlOrGetFromWebfinger(search)
 129     } catch (err) {
 130       logger.warn('Cannot load actor URL or get from webfinger.', { search, err })
 131
 132       return res.json({ total: 0, data: [] })
 133     }
 134   }
 135
 136   if (isUserAbleToSearchRemoteURI(res)) {
 137     try {
 138       const actor = await getOrCreateActorAndServerAndModel(uri, 'all', true, true)
 139       videoChannel = actor.VideoChannel
 140     } catch (err) {
 141       logger.info('Cannot search remote video channel %s.', uri, { err })
 142     }
 143   } else {
 144     videoChannel = await VideoChannelModel.loadByUrlAndPopulateAccount(uri)
 145   }
 146
 147   return res.json({
 148     total: videoChannel ? 1 : 0,
 149     data: videoChannel ? [ videoChannel.toFormattedJSON() ] : []
 150   })
 151 }
 152
 153 function searchVideos (req: express.Request, res: express.Response) {
 154   const query: VideosSearchQuery = req.query
 155   const search = query.search
 156
 157   if (search && (search.startsWith('http://') || search.startsWith('https://'))) {
 158     return searchVideoURI(search, res)
 159   }
 160
 161   if (isSearchIndexSearch(query)) {
 162     return searchVideosIndex(query, res)
 163   }
 164
 165   return searchVideosDB(query, res)
 166 }
 167
 168 async function searchVideosIndex (query: VideosSearchQuery, res: express.Response) {
 169   const result = await buildMutedForSearchIndex(res)
 170
 171   const body: VideosSearchQuery = Object.assign(query, result)
 172
 173   // Use the default instance NSFW policy if not specified
 174   if (!body.nsfw) {
 175     const nsfwPolicy = res.locals.oauth
 176       ? res.locals.oauth.token.User.nsfwPolicy
 177       : CONFIG.INSTANCE.DEFAULT_NSFW_POLICY
 178
 179     body.nsfw = nsfwPolicy === 'do_not_list'
 180       ? 'false'
 181       : 'both'
 182   }
 183
 184   const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/videos'
 185
 186   try {
 187     logger.debug('Doing videos search index request on %s.', url, { body })
 188
 189     const searchIndexResult = await doRequest<ResultList<Video>>({ uri: url, body, json: true })
 190
 191     return res.json(searchIndexResult.body)
 192   } catch (err) {
 193     logger.warn('Cannot use search index to make video search.', { err })
 194
 195     return res.sendStatus(HttpStatusCode.INTERNAL_SERVER_ERROR_500)
 196   }
 197 }
 198
 199 async function searchVideosDB (query: VideosSearchQuery, res: express.Response) {
 200   const options = Object.assign(query, {
 201     includeLocalVideos: true,
 202     nsfw: buildNSFWFilter(res, query.nsfw),
 203     filter: query.filter,
 204     user: res.locals.oauth ? res.locals.oauth.token.User : undefined
 205   })
 206   const resultList = await VideoModel.searchAndPopulateAccountAndServer(options)
 207
 208   return res.json(getFormattedObjects(resultList.data, resultList.total))
 209 }
 210
 211 async function searchVideoURI (url: string, res: express.Response) {
 212   let video: MVideoAccountLightBlacklistAllFiles
 213
 214   // Check if we can fetch a remote video with the URL
 215   if (isUserAbleToSearchRemoteURI(res)) {
 216     try {
 217       const syncParam = {
 218         likes: false,
 219         dislikes: false,
 220         shares: false,
 221         comments: false,
 222         thumbnail: true,
 223         refreshVideo: false
 224       }
 225
 226       const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam })
 227       video = result ? result.video : undefined
 228     } catch (err) {
 229       logger.info('Cannot search remote video %s.', url, { err })
 230     }
 231   } else {
 232     video = await VideoModel.loadByUrlAndPopulateAccount(url)
 233   }
 234
 235   return res.json({
 236     total: video ? 1 : 0,
 237     data: video ? [ video.toFormattedJSON() ] : []
 238   })
 239 }
 240
 241 function isSearchIndexSearch (query: SearchTargetQuery) {
 242   if (query.searchTarget === 'search-index') return true
 243
 244   const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX
 245
 246   if (searchIndexConfig.ENABLED !== true) return false
 247
 248   if (searchIndexConfig.DISABLE_LOCAL_SEARCH) return true
 249   if (searchIndexConfig.IS_DEFAULT_SEARCH && !query.searchTarget) return true
 250
 251   return false
 252 }
 253
 254 async function buildMutedForSearchIndex (res: express.Response) {
 255   const serverActor = await getServerActor()
 256   const accountIds = [ serverActor.Account.id ]
 257
 258   if (res.locals.oauth) {
 259     accountIds.push(res.locals.oauth.token.User.Account.id)
 260   }
 261
 262   const [ blockedHosts, blockedAccounts ] = await Promise.all([
 263     ServerBlocklistModel.listHostsBlockedBy(accountIds),
 264     AccountBlocklistModel.listHandlesBlockedBy(accountIds)
 265   ])
 266
 267   return {
 268     blockedHosts,
 269     blockedAccounts
 270   }
 271 }