From d4defe07d26013a75577b30608841fe3f8334308 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 11:41:21 +0200 Subject: [PATCH] Optimize video view AP processing --- .../lib/activitypub/process/process-create.ts | 10 +- .../lib/activitypub/process/process-update.ts | 10 +- server/lib/activitypub/videos.ts | 115 +++++++++++------- server/models/activitypub/actor.ts | 12 ++ 4 files changed, 96 insertions(+), 51 deletions(-) diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 99841da14..559a0c23c 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -86,10 +86,14 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { const view = activity.object as ViewObject - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: view.object }) + const options = { + videoObject: view.object, + fetchType: 'only-video' as 'only-video' + } + const { video } = await getOrCreateVideoAndAccountAndChannel(options) - const actor = await ActorModel.loadByUrl(view.actor) - if (!actor) throw new Error('Unknown actor ' + view.actor) + const actorExists = await ActorModel.isActorUrlExist(view.actor) + if (actorExists === false) throw new Error('Unknown actor ' + view.actor) await Redis.Instance.addVideoView(video.id) diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 935da5a54..0bceb370e 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -51,7 +51,15 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id }) const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) - return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to) + const updateOptions = { + video, + videoObject, + account: actor.Account, + channel: channelActor.VideoChannel, + updateViews: true, + overrideTo: activity.to + } + return updateVideoFromAP(updateOptions) } async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) { diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 5aabd3e0d..de22e3584 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -157,18 +157,26 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid async function getOrCreateVideoAndAccountAndChannel (options: { videoObject: VideoTorrentObject | string, syncParam?: SyncParam, - fetchType?: VideoFetchByUrlType + fetchType?: VideoFetchByUrlType, + refreshViews?: boolean }) { // Default params const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } const fetchType = options.fetchType || 'all' + const refreshViews = options.refreshViews || false // Get video url const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) if (videoFromDatabase) { - const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase, fetchType, syncParam) + const refreshOptions = { + video: videoFromDatabase, + fetchedType: fetchType, + syncParam, + refreshViews + } + const p = retryTransactionWrapper(refreshVideoIfNeeded, refreshOptions) if (syncParam.refreshVideo === true) videoFromDatabase = await p return { video: videoFromDatabase } @@ -185,14 +193,15 @@ async function getOrCreateVideoAndAccountAndChannel (options: { return { video } } -async function updateVideoFromAP ( +async function updateVideoFromAP (options: { video: VideoModel, videoObject: VideoTorrentObject, account: AccountModel, channel: VideoChannelModel, + updateViews: boolean, overrideTo?: string[] -) { - logger.debug('Updating remote video "%s".', videoObject.uuid) +}) { + logger.debug('Updating remote video "%s".', options.videoObject.uuid) let videoFieldsSave: any try { @@ -201,72 +210,72 @@ async function updateVideoFromAP ( transaction: t } - videoFieldsSave = video.toJSON() + videoFieldsSave = options.video.toJSON() // Check actor has the right to update the video - const videoChannel = video.VideoChannel - if (videoChannel.Account.id !== account.id) { - throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url) + const videoChannel = options.video.VideoChannel + if (videoChannel.Account.id !== options.account.id) { + throw new Error('Account ' + options.account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url) } - const to = overrideTo ? overrideTo : videoObject.to - const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to) - video.set('name', videoData.name) - video.set('uuid', videoData.uuid) - video.set('url', videoData.url) - video.set('category', videoData.category) - video.set('licence', videoData.licence) - video.set('language', videoData.language) - video.set('description', videoData.description) - video.set('support', videoData.support) - video.set('nsfw', videoData.nsfw) - video.set('commentsEnabled', videoData.commentsEnabled) - video.set('waitTranscoding', videoData.waitTranscoding) - video.set('state', videoData.state) - video.set('duration', videoData.duration) - video.set('createdAt', videoData.createdAt) - video.set('publishedAt', videoData.publishedAt) - video.set('views', videoData.views) - video.set('privacy', videoData.privacy) - video.set('channelId', videoData.channelId) - - await video.save(sequelizeOptions) + const to = options.overrideTo ? options.overrideTo : options.videoObject.to + const videoData = await videoActivityObjectToDBAttributes(options.channel, options.videoObject, to) + options.video.set('name', videoData.name) + options.video.set('uuid', videoData.uuid) + options.video.set('url', videoData.url) + options.video.set('category', videoData.category) + options.video.set('licence', videoData.licence) + options.video.set('language', videoData.language) + options.video.set('description', videoData.description) + options.video.set('support', videoData.support) + options.video.set('nsfw', videoData.nsfw) + options.video.set('commentsEnabled', videoData.commentsEnabled) + options.video.set('waitTranscoding', videoData.waitTranscoding) + options.video.set('state', videoData.state) + options.video.set('duration', videoData.duration) + options.video.set('createdAt', videoData.createdAt) + options.video.set('publishedAt', videoData.publishedAt) + options.video.set('privacy', videoData.privacy) + options.video.set('channelId', videoData.channelId) + + if (options.updateViews === true) options.video.set('views', videoData.views) + await options.video.save(sequelizeOptions) // Don't block on request - generateThumbnailFromUrl(video, videoObject.icon) - .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) + generateThumbnailFromUrl(options.video, options.videoObject.icon) + .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })) // Remove old video files const videoFileDestroyTasks: Bluebird[] = [] - for (const videoFile of video.VideoFiles) { + for (const videoFile of options.video.VideoFiles) { videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) } await Promise.all(videoFileDestroyTasks) - const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject) + const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) await Promise.all(tasks) // Update Tags - const tags = videoObject.tag.map(tag => tag.name) + const tags = options.videoObject.tag.map(tag => tag.name) const tagInstances = await TagModel.findOrCreateTags(tags, t) - await video.$set('Tags', tagInstances, sequelizeOptions) + await options.video.$set('Tags', tagInstances, sequelizeOptions) // Update captions - await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t) + await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t) - const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { - return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t) + const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => { + return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t) }) await Promise.all(videoCaptionsPromises) }) - logger.info('Remote video with uuid %s updated', videoObject.uuid) + logger.info('Remote video with uuid %s updated', options.videoObject.uuid) return updatedVideo } catch (err) { - if (video !== undefined && videoFieldsSave !== undefined) { - resetSequelizeInstance(video, videoFieldsSave) + if (options.video !== undefined && videoFieldsSave !== undefined) { + resetSequelizeInstance(options.video, videoFieldsSave) } // This is just a debug because we will retry the insert @@ -339,9 +348,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor return videoCreated } -async function refreshVideoIfNeeded (videoArg: VideoModel, fetchedType: VideoFetchByUrlType, syncParam: SyncParam): Promise { +async function refreshVideoIfNeeded (options: { + video: VideoModel, + fetchedType: VideoFetchByUrlType, + syncParam: SyncParam, + refreshViews: boolean +}): Promise { // We need more attributes if the argument video was fetched with not enough joints - const video = fetchedType === 'all' ? videoArg : await VideoModel.loadByUrlAndPopulateAccount(videoArg.url) + const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) if (!video.isOutdated()) return video @@ -361,8 +375,15 @@ async function refreshVideoIfNeeded (videoArg: VideoModel, fetchedType: VideoFet const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) const account = await AccountModel.load(channelActor.VideoChannel.accountId) - await updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) - await syncVideoExternalAttributes(video, videoObject, syncParam) + const updateOptions = { + video, + videoObject, + account, + channel: channelActor.VideoChannel, + updateViews: options.refreshViews + } + await updateVideoFromAP(updateOptions) + await syncVideoExternalAttributes(video, videoObject, options.syncParam) } catch (err) { logger.warn('Cannot refresh video.', { err }) return video diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index ef8dd9f7c..69c2eca57 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts @@ -266,6 +266,18 @@ export class ActorModel extends Model { return ActorModel.unscoped().findById(id) } + static isActorUrlExist (url: string) { + const query = { + raw: true, + where: { + url + } + } + + return ActorModel.unscoped().findOne(query) + .then(a => !!a) + } + static listByFollowersUrls (followersUrls: string[], transaction?: Sequelize.Transaction) { const query = { where: { -- 2.41.0