From 37a44fc915eef2140e22ceb96aba6b6eb2509007 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 17 Jun 2021 16:02:38 +0200 Subject: Add ability to search playlists --- server/lib/activitypub/actors/get.ts | 2 +- server/lib/activitypub/playlists/create-update.ts | 34 ++++++++++++--------- server/lib/activitypub/playlists/get.ts | 35 ++++++++++++++++++++++ server/lib/activitypub/playlists/index.ts | 1 + server/lib/activitypub/playlists/refresh.ts | 13 ++++++-- .../playlists/shared/object-to-model-attributes.ts | 6 ++-- server/lib/activitypub/process/process-create.ts | 2 +- server/lib/activitypub/process/process-update.ts | 2 +- server/lib/activitypub/videos/get.ts | 7 +++-- 9 files changed, 76 insertions(+), 26 deletions(-) create mode 100644 server/lib/activitypub/playlists/get.ts (limited to 'server/lib/activitypub') diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts index d7cf2b678..8681ea02a 100644 --- a/server/lib/activitypub/actors/get.ts +++ b/server/lib/activitypub/actors/get.ts @@ -116,7 +116,7 @@ async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, ref async function schedulePlaylistFetchIfNeeded (actor: MActorAccountId, created: boolean, accountPlaylistsUrl: string) { // We created a new account: fetch the playlists if (created === true && actor.Account && accountPlaylistsUrl) { - const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' } + const payload = { uri: accountPlaylistsUrl, type: 'account-playlists' as 'account-playlists' } await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }) } } diff --git a/server/lib/activitypub/playlists/create-update.ts b/server/lib/activitypub/playlists/create-update.ts index 37d748de4..ea3e61ac5 100644 --- a/server/lib/activitypub/playlists/create-update.ts +++ b/server/lib/activitypub/playlists/create-update.ts @@ -1,3 +1,5 @@ +import * as Bluebird from 'bluebird' +import { getAPId } from '@server/helpers/activitypub' import { isArray } from '@server/helpers/custom-validators/misc' import { logger, loggerTagsFactory } from '@server/helpers/logger' import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants' @@ -6,7 +8,7 @@ import { updatePlaylistMiniatureFromUrl } from '@server/lib/thumbnail' import { VideoPlaylistModel } from '@server/models/video/video-playlist' import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' import { FilteredModelAttributes } from '@server/types' -import { MAccountDefault, MAccountId, MThumbnail, MVideoPlaylist, MVideoPlaylistFull } from '@server/types/models' +import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models' import { AttributesOnly } from '@shared/core-utils' import { PlaylistObject } from '@shared/models' import { getOrCreateAPActor } from '../actors' @@ -19,11 +21,9 @@ import { playlistObjectToDBAttributes } from './shared' -import Bluebird = require('bluebird') - const lTags = loggerTagsFactory('ap', 'video-playlist') -async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) { +async function createAccountPlaylists (playlistUrls: string[]) { await Bluebird.map(playlistUrls, async playlistUrl => { try { const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl) @@ -35,19 +35,19 @@ async function createAccountPlaylists (playlistUrls: string[], account: MAccount throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`) } - return createOrUpdateVideoPlaylist(playlistObject, account, playlistObject.to) + return createOrUpdateVideoPlaylist(playlistObject) } catch (err) { logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) }) } }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) } -async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { - const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to) +async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) { + const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to) - await setVideoChannelIfNeeded(playlistObject, playlistAttributes) + await setVideoChannel(playlistObject, playlistAttributes) - const [ upsertPlaylist ] = await VideoPlaylistModel.upsert(playlistAttributes, { returning: true }) + const [ upsertPlaylist ] = await VideoPlaylistModel.upsert(playlistAttributes, { returning: true }) const playlistElementUrls = await fetchElementUrls(playlistObject) @@ -56,7 +56,10 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc await updatePlaylistThumbnail(playlistObject, playlist) - return rebuildVideoPlaylistElements(playlistElementUrls, playlist) + const elementsLength = await rebuildVideoPlaylistElements(playlistElementUrls, playlist) + playlist.setVideosLength(elementsLength) + + return playlist } // --------------------------------------------------------------------------- @@ -68,10 +71,12 @@ export { // --------------------------------------------------------------------------- -async function setVideoChannelIfNeeded (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly) { - if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) return +async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly) { + if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) { + throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject)) + } - const actor = await getOrCreateAPActor(playlistObject.attributedTo[0]) + const actor = await getOrCreateAPActor(playlistObject.attributedTo[0], 'all') if (!actor.VideoChannel) { logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) }) @@ -79,6 +84,7 @@ async function setVideoChannelIfNeeded (playlistObject: PlaylistObject, playlist } playlistAttributes.videoChannelId = actor.VideoChannel.id + playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id } async function fetchElementUrls (playlistObject: PlaylistObject) { @@ -128,7 +134,7 @@ async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MV logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url)) - return undefined + return elementsToCreate.length } async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) { diff --git a/server/lib/activitypub/playlists/get.ts b/server/lib/activitypub/playlists/get.ts new file mode 100644 index 000000000..2c19c503a --- /dev/null +++ b/server/lib/activitypub/playlists/get.ts @@ -0,0 +1,35 @@ +import { getAPId } from '@server/helpers/activitypub' +import { VideoPlaylistModel } from '@server/models/video/video-playlist' +import { MVideoPlaylistFullSummary } from '@server/types/models' +import { APObject } from '@shared/models' +import { createOrUpdateVideoPlaylist } from './create-update' +import { scheduleRefreshIfNeeded } from './refresh' +import { fetchRemoteVideoPlaylist } from './shared' + +async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObject): Promise { + const playlistUrl = getAPId(playlistObjectArg) + + const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl) + + if (playlistFromDatabase) { + scheduleRefreshIfNeeded(playlistFromDatabase) + + return playlistFromDatabase + } + + const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl) + if (!playlistObject) throw new Error('Cannot fetch remote playlist with url: ' + playlistUrl) + + // playlistUrl is just an alias/rediraction, so process object id instead + if (playlistObject.id !== playlistUrl) return getOrCreateAPVideoPlaylist(playlistObject) + + const playlistCreated = await createOrUpdateVideoPlaylist(playlistObject) + + return playlistCreated +} + +// --------------------------------------------------------------------------- + +export { + getOrCreateAPVideoPlaylist +} diff --git a/server/lib/activitypub/playlists/index.ts b/server/lib/activitypub/playlists/index.ts index 2885830b4..e2470a674 100644 --- a/server/lib/activitypub/playlists/index.ts +++ b/server/lib/activitypub/playlists/index.ts @@ -1,2 +1,3 @@ +export * from './get' export * from './create-update' export * from './refresh' diff --git a/server/lib/activitypub/playlists/refresh.ts b/server/lib/activitypub/playlists/refresh.ts index 6f3a6be37..ef3cb3fe4 100644 --- a/server/lib/activitypub/playlists/refresh.ts +++ b/server/lib/activitypub/playlists/refresh.ts @@ -1,10 +1,17 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger' import { PeerTubeRequestError } from '@server/helpers/requests' -import { MVideoPlaylistOwner } from '@server/types/models' +import { JobQueue } from '@server/lib/job-queue' +import { MVideoPlaylist, MVideoPlaylistOwner } from '@server/types/models' import { HttpStatusCode } from '@shared/core-utils' import { createOrUpdateVideoPlaylist } from './create-update' import { fetchRemoteVideoPlaylist } from './shared' +function scheduleRefreshIfNeeded (playlist: MVideoPlaylist) { + if (!playlist.isOutdated()) return + + JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: playlist.url } }) +} + async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise { if (!videoPlaylist.isOutdated()) return videoPlaylist @@ -22,8 +29,7 @@ async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner) return videoPlaylist } - const byAccount = videoPlaylist.OwnerAccount - await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to) + await createOrUpdateVideoPlaylist(playlistObject) return videoPlaylist } catch (err) { @@ -42,5 +48,6 @@ async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner) } export { + scheduleRefreshIfNeeded, refreshVideoPlaylistIfNeeded } diff --git a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts index 6ec44485e..70fd335bc 100644 --- a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts +++ b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts @@ -1,11 +1,11 @@ import { ACTIVITY_PUB } from '@server/initializers/constants' import { VideoPlaylistModel } from '@server/models/video/video-playlist' import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' -import { MAccountId, MVideoId, MVideoPlaylistId } from '@server/types/models' +import { MVideoId, MVideoPlaylistId } from '@server/types/models' import { AttributesOnly } from '@shared/core-utils' import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models' -function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { +function playlistObjectToDBAttributes (playlistObject: PlaylistObject, to: string[]) { const privacy = to.includes(ACTIVITY_PUB.PUBLIC) ? VideoPlaylistPrivacy.PUBLIC : VideoPlaylistPrivacy.UNLISTED @@ -16,7 +16,7 @@ function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount privacy, url: playlistObject.id, uuid: playlistObject.uuid, - ownerAccountId: byAccount.id, + ownerAccountId: null, videoChannelId: null, createdAt: new Date(playlistObject.published), updatedAt: new Date(playlistObject.updated) diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 6b7f5aae8..70e048d6e 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -128,5 +128,5 @@ async function processCreatePlaylist (activity: ActivityCreate, byActor: MActorS if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url) - await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to) + await createOrUpdateVideoPlaylist(playlistObject, activity.to) } diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index aa80d5d09..f40008a6b 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -111,5 +111,5 @@ async function processUpdatePlaylist (byActor: MActorSignature, activity: Activi if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url) - await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to) + await createOrUpdateVideoPlaylist(playlistObject, activity.to) } diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts index 7bb14adc4..f3e2f0625 100644 --- a/server/lib/activitypub/videos/get.ts +++ b/server/lib/activitypub/videos/get.ts @@ -3,6 +3,7 @@ import { retryTransactionWrapper } from '@server/helpers/database-utils' import { JobQueue } from '@server/lib/job-queue' import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders' import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' +import { APObject } from '@shared/models' import { refreshVideoIfNeeded } from './refresh' import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' @@ -13,21 +14,21 @@ type GetVideoResult = Promise<{ }> type GetVideoParamAll = { - videoObject: { id: string } | string + videoObject: APObject syncParam?: SyncParam fetchType?: 'all' allowRefresh?: boolean } type GetVideoParamImmutable = { - videoObject: { id: string } | string + videoObject: APObject syncParam?: SyncParam fetchType: 'only-immutable-attributes' allowRefresh: false } type GetVideoParamOther = { - videoObject: { id: string } | string + videoObject: APObject syncParam?: SyncParam fetchType?: 'all' | 'only-video' allowRefresh?: boolean -- cgit v1.2.3