Add ability to search playlists

11 files changed, 128 insertions, 32 deletions

diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts
index d7cf2b678..8681ea02a 100644
--- a/server/lib/activitypub/actors/get.ts
+++ b/server/lib/activitypub/actors/get.ts
@@ -116,7 +116,7 @@ async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, ref
 async function schedulePlaylistFetchIfNeeded (actor: MActorAccountId, created: boolean, accountPlaylistsUrl: string) {
   // We created a new account: fetch the playlists
   if (created === true && actor.Account && accountPlaylistsUrl) {
-    const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' }
+    const payload = { uri: accountPlaylistsUrl, type: 'account-playlists' as 'account-playlists' }
     await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })
   }
 }
diff --git a/server/lib/activitypub/playlists/create-update.ts b/server/lib/activitypub/playlists/create-update.ts
index 37d748de4..ea3e61ac5 100644
--- a/server/lib/activitypub/playlists/create-update.ts
+++ b/server/lib/activitypub/playlists/create-update.ts
@@ -1,3 +1,5 @@
+import * as Bluebird from 'bluebird'
+import { getAPId } from '@server/helpers/activitypub'
 import { isArray } from '@server/helpers/custom-validators/misc'
 import { logger, loggerTagsFactory } from '@server/helpers/logger'
 import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants'
@@ -6,7 +8,7 @@ import { updatePlaylistMiniatureFromUrl } from '@server/lib/thumbnail'
 import { VideoPlaylistModel } from '@server/models/video/video-playlist'
 import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
 import { FilteredModelAttributes } from '@server/types'
-import { MAccountDefault, MAccountId, MThumbnail, MVideoPlaylist, MVideoPlaylistFull } from '@server/types/models'
+import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models'
 import { AttributesOnly } from '@shared/core-utils'
 import { PlaylistObject } from '@shared/models'
 import { getOrCreateAPActor } from '../actors'
@@ -19,11 +21,9 @@ import {
   playlistObjectToDBAttributes
 } from './shared'
 
-import Bluebird = require('bluebird')
-
 const lTags = loggerTagsFactory('ap', 'video-playlist')
 
-async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
+async function createAccountPlaylists (playlistUrls: string[]) {
   await Bluebird.map(playlistUrls, async playlistUrl => {
     try {
       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
@@ -35,19 +35,19 @@ async function createAccountPlaylists (playlistUrls: string[], account: MAccount
         throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
       }
 
-      return createOrUpdateVideoPlaylist(playlistObject, account, playlistObject.to)
+      return createOrUpdateVideoPlaylist(playlistObject)
     } catch (err) {
       logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) })
     }
   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 }
 
-async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
-  const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
+async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) {
+  const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to)
 
-  await setVideoChannelIfNeeded(playlistObject, playlistAttributes)
+  await setVideoChannel(playlistObject, playlistAttributes)
 
-  const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
+  const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylistVideosLength>(playlistAttributes, { returning: true })
 
   const playlistElementUrls = await fetchElementUrls(playlistObject)
 
@@ -56,7 +56,10 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc
 
   await updatePlaylistThumbnail(playlistObject, playlist)
 
-  return rebuildVideoPlaylistElements(playlistElementUrls, playlist)
+  const elementsLength = await rebuildVideoPlaylistElements(playlistElementUrls, playlist)
+  playlist.setVideosLength(elementsLength)
+
+  return playlist
 }
 
 // ---------------------------------------------------------------------------
@@ -68,10 +71,12 @@ export {
 
 // ---------------------------------------------------------------------------
 
-async function setVideoChannelIfNeeded (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
-  if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) return
+async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
+  if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) {
+    throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject))
+  }
 
-  const actor = await getOrCreateAPActor(playlistObject.attributedTo[0])
+  const actor = await getOrCreateAPActor(playlistObject.attributedTo[0], 'all')
 
   if (!actor.VideoChannel) {
     logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
@@ -79,6 +84,7 @@ async function setVideoChannelIfNeeded (playlistObject: PlaylistObject, playlist
   }
 
   playlistAttributes.videoChannelId = actor.VideoChannel.id
+  playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id
 }
 
 async function fetchElementUrls (playlistObject: PlaylistObject) {
@@ -128,7 +134,7 @@ async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MV
 
   logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url))
 
-  return undefined
+  return elementsToCreate.length
 }
 
 async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) {
diff --git a/server/lib/activitypub/playlists/get.ts b/server/lib/activitypub/playlists/get.ts
new file mode 100644
index 000000000..2c19c503a
--- /dev/null
+++ b/server/lib/activitypub/playlists/get.ts
@@ -0,0 +1,35 @@
+import { getAPId } from '@server/helpers/activitypub'
+import { VideoPlaylistModel } from '@server/models/video/video-playlist'
+import { MVideoPlaylistFullSummary } from '@server/types/models'
+import { APObject } from '@shared/models'
+import { createOrUpdateVideoPlaylist } from './create-update'
+import { scheduleRefreshIfNeeded } from './refresh'
+import { fetchRemoteVideoPlaylist } from './shared'
+
+async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObject): Promise<MVideoPlaylistFullSummary> {
+  const playlistUrl = getAPId(playlistObjectArg)
+
+  const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl)
+
+  if (playlistFromDatabase) {
+    scheduleRefreshIfNeeded(playlistFromDatabase)
+
+    return playlistFromDatabase
+  }
+
+  const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
+  if (!playlistObject) throw new Error('Cannot fetch remote playlist with url: ' + playlistUrl)
+
+  // playlistUrl is just an alias/rediraction, so process object id instead
+  if (playlistObject.id !== playlistUrl) return getOrCreateAPVideoPlaylist(playlistObject)
+
+  const playlistCreated = await createOrUpdateVideoPlaylist(playlistObject)
+
+  return playlistCreated
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getOrCreateAPVideoPlaylist
+}
diff --git a/server/lib/activitypub/playlists/index.ts b/server/lib/activitypub/playlists/index.ts
index 2885830b4..e2470a674 100644
--- a/server/lib/activitypub/playlists/index.ts
+++ b/server/lib/activitypub/playlists/index.ts
@@ -1,2 +1,3 @@
+export * from './get'
 export * from './create-update'
 export * from './refresh'
diff --git a/server/lib/activitypub/playlists/refresh.ts b/server/lib/activitypub/playlists/refresh.ts
index 6f3a6be37..ef3cb3fe4 100644
--- a/server/lib/activitypub/playlists/refresh.ts
+++ b/server/lib/activitypub/playlists/refresh.ts
@@ -1,10 +1,17 @@
 import { logger, loggerTagsFactory } from '@server/helpers/logger'
 import { PeerTubeRequestError } from '@server/helpers/requests'
-import { MVideoPlaylistOwner } from '@server/types/models'
+import { JobQueue } from '@server/lib/job-queue'
+import { MVideoPlaylist, MVideoPlaylistOwner } from '@server/types/models'
 import { HttpStatusCode } from '@shared/core-utils'
 import { createOrUpdateVideoPlaylist } from './create-update'
 import { fetchRemoteVideoPlaylist } from './shared'
 
+function scheduleRefreshIfNeeded (playlist: MVideoPlaylist) {
+  if (!playlist.isOutdated()) return
+
+  JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: playlist.url } })
+}
+
 async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
   if (!videoPlaylist.isOutdated()) return videoPlaylist
 
@@ -22,8 +29,7 @@ async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner)
       return videoPlaylist
     }
 
-    const byAccount = videoPlaylist.OwnerAccount
-    await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
+    await createOrUpdateVideoPlaylist(playlistObject)
 
     return videoPlaylist
   } catch (err) {
@@ -42,5 +48,6 @@ async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner)
 }
 
 export {
+  scheduleRefreshIfNeeded,
   refreshVideoPlaylistIfNeeded
 }
diff --git a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts
index 6ec44485e..70fd335bc 100644
--- a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts
+++ b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts
@@ -1,11 +1,11 @@
 import { ACTIVITY_PUB } from '@server/initializers/constants'
 import { VideoPlaylistModel } from '@server/models/video/video-playlist'
 import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
-import { MAccountId, MVideoId, MVideoPlaylistId } from '@server/types/models'
+import { MVideoId, MVideoPlaylistId } from '@server/types/models'
 import { AttributesOnly } from '@shared/core-utils'
 import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models'
 
-function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
+function playlistObjectToDBAttributes (playlistObject: PlaylistObject, to: string[]) {
   const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
     ? VideoPlaylistPrivacy.PUBLIC
     : VideoPlaylistPrivacy.UNLISTED
@@ -16,7 +16,7 @@ function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount
     privacy,
     url: playlistObject.id,
     uuid: playlistObject.uuid,
-    ownerAccountId: byAccount.id,
+    ownerAccountId: null,
     videoChannelId: null,
     createdAt: new Date(playlistObject.published),
     updatedAt: new Date(playlistObject.updated)
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 6b7f5aae8..70e048d6e 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -128,5 +128,5 @@ async function processCreatePlaylist (activity: ActivityCreate, byActor: MActorS
 
   if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url)
 
-  await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to)
+  await createOrUpdateVideoPlaylist(playlistObject, activity.to)
 }
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index aa80d5d09..f40008a6b 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -111,5 +111,5 @@ async function processUpdatePlaylist (byActor: MActorSignature, activity: Activi
 
   if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url)
 
-  await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to)
+  await createOrUpdateVideoPlaylist(playlistObject, activity.to)
 }
diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts
index 7bb14adc4..f3e2f0625 100644
--- a/server/lib/activitypub/videos/get.ts
+++ b/server/lib/activitypub/videos/get.ts
@@ -3,6 +3,7 @@ import { retryTransactionWrapper } from '@server/helpers/database-utils'
 import { JobQueue } from '@server/lib/job-queue'
 import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders'
 import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models'
+import { APObject } from '@shared/models'
 import { refreshVideoIfNeeded } from './refresh'
 import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared'
 
@@ -13,21 +14,21 @@ type GetVideoResult <T> = Promise<{
 }>
 
 type GetVideoParamAll = {
-  videoObject: { id: string } | string
+  videoObject: APObject
   syncParam?: SyncParam
   fetchType?: 'all'
   allowRefresh?: boolean
 }
 
 type GetVideoParamImmutable = {
-  videoObject: { id: string } | string
+  videoObject: APObject
   syncParam?: SyncParam
   fetchType: 'only-immutable-attributes'
   allowRefresh: false
 }
 
 type GetVideoParamOther = {
-  videoObject: { id: string } | string
+  videoObject: APObject
   syncParam?: SyncParam
   fetchType?: 'all' | 'only-video'
   allowRefresh?: boolean
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index 04b25f955..ab9675cae 100644
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -1,12 +1,11 @@
 import * as Bull from 'bull'
 import { ActivitypubHttpFetcherPayload, FetchType } from '@shared/models'
 import { logger } from '../../../helpers/logger'
-import { AccountModel } from '../../../models/account/account'
 import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
 import { VideoModel } from '../../../models/video/video'
 import { VideoCommentModel } from '../../../models/video/video-comment'
 import { VideoShareModel } from '../../../models/video/video-share'
-import { MAccountDefault, MVideoFullLight } from '../../../types/models'
+import { MVideoFullLight } from '../../../types/models'
 import { crawlCollectionPage } from '../../activitypub/crawl'
 import { createAccountPlaylists } from '../../activitypub/playlists'
 import { processActivities } from '../../activitypub/process'
@@ -22,16 +21,13 @@ async function processActivityPubHttpFetcher (job: Bull.Job) {
   let video: MVideoFullLight
   if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId)
 
-  let account: MAccountDefault
-  if (payload.accountId) account = await AccountModel.load(payload.accountId)
-
   const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = {
     'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }),
     'video-likes': items => createRates(items, video, 'like'),
     'video-dislikes': items => createRates(items, video, 'dislike'),
     'video-shares': items => addVideoShares(items, video),
     'video-comments': items => addVideoComments(items),
-    'account-playlists': items => createAccountPlaylists(items, account)
+    'account-playlists': items => createAccountPlaylists(items)
   }
 
   const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Promise<any> } = {
diff --git a/server/lib/search.ts b/server/lib/search.ts
new file mode 100644
index 000000000..b643a4055
--- /dev/null
+++ b/server/lib/search.ts
@@ -0,0 +1,50 @@
+import * as express from 'express'
+import { CONFIG } from '@server/initializers/config'
+import { AccountBlocklistModel } from '@server/models/account/account-blocklist'
+import { getServerActor } from '@server/models/application/application'
+import { ServerBlocklistModel } from '@server/models/server/server-blocklist'
+import { SearchTargetQuery } from '@shared/models'
+
+function isSearchIndexSearch (query: SearchTargetQuery) {
+  if (query.searchTarget === 'search-index') return true
+
+  const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX
+
+  if (searchIndexConfig.ENABLED !== true) return false
+
+  if (searchIndexConfig.DISABLE_LOCAL_SEARCH) return true
+  if (searchIndexConfig.IS_DEFAULT_SEARCH && !query.searchTarget) return true
+
+  return false
+}
+
+async function buildMutedForSearchIndex (res: express.Response) {
+  const serverActor = await getServerActor()
+  const accountIds = [ serverActor.Account.id ]
+
+  if (res.locals.oauth) {
+    accountIds.push(res.locals.oauth.token.User.Account.id)
+  }
+
+  const [ blockedHosts, blockedAccounts ] = await Promise.all([
+    ServerBlocklistModel.listHostsBlockedBy(accountIds),
+    AccountBlocklistModel.listHandlesBlockedBy(accountIds)
+  ])
+
+  return {
+    blockedHosts,
+    blockedAccounts
+  }
+}
+
+function isURISearch (search: string) {
+  if (!search) return false
+
+  return search.startsWith('http://') || search.startsWith('https://')
+}
+
+export {
+  isSearchIndexSearch,
+  buildMutedForSearchIndex,
+  isURISearch
+}