From 1297eb5db651a230474670c5da1517862fb9cc3e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 22 Aug 2018 16:15:35 +0200 Subject: Add refresh video on search --- server/lib/activitypub/process/process-announce.ts | 4 +- server/lib/activitypub/process/process-create.ts | 10 +- server/lib/activitypub/process/process-like.ts | 4 +- server/lib/activitypub/process/process-undo.ts | 6 +- server/lib/activitypub/process/process-update.ts | 99 +-------- server/lib/activitypub/share.ts | 38 ++++ server/lib/activitypub/video-comments.ts | 4 +- server/lib/activitypub/video-rates.ts | 40 ++++ server/lib/activitypub/videos.ts | 241 +++++++++++++-------- 9 files changed, 244 insertions(+), 202 deletions(-) (limited to 'server/lib/activitypub') diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index b08156aa1..814556817 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts @@ -6,7 +6,7 @@ import { VideoModel } from '../../../models/video/video' import { VideoShareModel } from '../../../models/video/video-share' import { getOrCreateActorAndServerAndModel } from '../actor' import { forwardVideoRelatedActivity } from '../send/utils' -import { getOrCreateAccountAndVideoAndChannel } from '../videos' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' async function processAnnounceActivity (activity: ActivityAnnounce) { const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor) @@ -25,7 +25,7 @@ export { async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id - const { video } = await getOrCreateAccountAndVideoAndChannel(objectUri) + const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri) return sequelizeTypescript.transaction(async t => { // Add share entry diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 9655d015f..e8f5ade06 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -10,7 +10,7 @@ import { VideoAbuseModel } from '../../../models/video/video-abuse' import { VideoCommentModel } from '../../../models/video/video-comment' import { getOrCreateActorAndServerAndModel } from '../actor' import { resolveThread } from '../video-comments' -import { getOrCreateAccountAndVideoAndChannel } from '../videos' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' async function processCreateActivity (activity: ActivityCreate) { @@ -45,7 +45,7 @@ export { async function processCreateVideo (activity: ActivityCreate) { const videoToCreateData = activity.object as VideoTorrentObject - const { video } = await getOrCreateAccountAndVideoAndChannel(videoToCreateData) + const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData) return video } @@ -56,7 +56,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) - const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object) + const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) return sequelizeTypescript.transaction(async t => { const rate = { @@ -83,7 +83,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { const view = activity.object as ViewObject - const { video } = await getOrCreateAccountAndVideoAndChannel(view.object) + const { video } = await getOrCreateVideoAndAccountAndChannel(view.object) const actor = await ActorModel.loadByUrl(view.actor) if (!actor) throw new Error('Unknown actor ' + view.actor) @@ -103,7 +103,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat const account = actor.Account if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) - const { video } = await getOrCreateAccountAndVideoAndChannel(videoAbuseToCreateData.object) + const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object) return sequelizeTypescript.transaction(async t => { const videoAbuseData = { diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index d0865b78c..9e1664fd8 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts @@ -5,7 +5,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat import { ActorModel } from '../../../models/activitypub/actor' import { getOrCreateActorAndServerAndModel } from '../actor' import { forwardVideoRelatedActivity } from '../send/utils' -import { getOrCreateAccountAndVideoAndChannel } from '../videos' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' async function processLikeActivity (activity: ActivityLike) { const actor = await getOrCreateActorAndServerAndModel(activity.actor) @@ -27,7 +27,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { const byAccount = byActor.Account if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) - const { video } = await getOrCreateAccountAndVideoAndChannel(videoUrl) + const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl) return sequelizeTypescript.transaction(async t => { const rate = { diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index b6de107ad..eab9e3d61 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -9,7 +9,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { forwardVideoRelatedActivity } from '../send/utils' -import { getOrCreateAccountAndVideoAndChannel } from '../videos' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { VideoShareModel } from '../../../models/video/video-share' async function processUndoActivity (activity: ActivityUndo) { @@ -43,7 +43,7 @@ export { async function processUndoLike (actorUrl: string, activity: ActivityUndo) { const likeActivity = activity.object as ActivityLike - const { video } = await getOrCreateAccountAndVideoAndChannel(likeActivity.object) + const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object) return sequelizeTypescript.transaction(async t => { const byAccount = await AccountModel.loadByUrl(actorUrl, t) @@ -67,7 +67,7 @@ async function processUndoLike (actorUrl: string, activity: ActivityUndo) { async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { const dislike = activity.object.object as DislikeObject - const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object) + const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) return sequelizeTypescript.transaction(async t => { const byAccount = await AccountModel.loadByUrl(actorUrl, t) diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 11226e275..07a5ff92f 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -1,4 +1,3 @@ -import * as Bluebird from 'bluebird' import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub' import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' @@ -6,19 +5,10 @@ import { logger } from '../../../helpers/logger' import { sequelizeTypescript } from '../../../initializers' import { AccountModel } from '../../../models/account/account' import { ActorModel } from '../../../models/activitypub/actor' -import { TagModel } from '../../../models/video/tag' import { VideoChannelModel } from '../../../models/video/video-channel' -import { VideoFileModel } from '../../../models/video/video-file' import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' -import { - generateThumbnailFromUrl, - getOrCreateAccountAndVideoAndChannel, - getOrCreateVideoChannel, - videoActivityObjectToDBAttributes, - videoFileActivityUrlToDBAttributes -} from '../videos' +import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannel, updateVideoFromAP } from '../videos' import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' -import { VideoCaptionModel } from '../../../models/video/video-caption' async function processUpdateActivity (activity: ActivityUpdate) { const actor = await getOrCreateActorAndServerAndModel(activity.actor) @@ -49,91 +39,10 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) return undefined } - const res = await getOrCreateAccountAndVideoAndChannel(videoObject.id) + const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) + const channelActor = await getOrCreateVideoChannel(videoObject) - // Fetch video channel outside the transaction - const newVideoChannelActor = await getOrCreateVideoChannel(videoObject) - const newVideoChannel = newVideoChannelActor.VideoChannel - - logger.debug('Updating remote video "%s".', videoObject.uuid) - let videoInstance = res.video - let videoFieldsSave: any - - try { - await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { - transaction: t - } - - videoFieldsSave = videoInstance.toJSON() - - // Check actor has the right to update the video - const videoChannel = videoInstance.VideoChannel - if (videoChannel.Account.Actor.id !== actor.id) { - throw new Error('Account ' + actor.url + ' does not own video channel ' + videoChannel.Actor.url) - } - - const videoData = await videoActivityObjectToDBAttributes(newVideoChannel, videoObject, activity.to) - videoInstance.set('name', videoData.name) - videoInstance.set('uuid', videoData.uuid) - videoInstance.set('url', videoData.url) - videoInstance.set('category', videoData.category) - videoInstance.set('licence', videoData.licence) - videoInstance.set('language', videoData.language) - videoInstance.set('description', videoData.description) - videoInstance.set('support', videoData.support) - videoInstance.set('nsfw', videoData.nsfw) - videoInstance.set('commentsEnabled', videoData.commentsEnabled) - videoInstance.set('waitTranscoding', videoData.waitTranscoding) - videoInstance.set('state', videoData.state) - videoInstance.set('duration', videoData.duration) - videoInstance.set('createdAt', videoData.createdAt) - videoInstance.set('updatedAt', videoData.updatedAt) - videoInstance.set('views', videoData.views) - videoInstance.set('privacy', videoData.privacy) - videoInstance.set('channelId', videoData.channelId) - - await videoInstance.save(sequelizeOptions) - - // Don't block on request - generateThumbnailFromUrl(videoInstance, videoObject.icon) - .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) - - // Remove old video files - const videoFileDestroyTasks: Bluebird[] = [] - for (const videoFile of videoInstance.VideoFiles) { - videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) - } - await Promise.all(videoFileDestroyTasks) - - const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoInstance, videoObject) - const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) - await Promise.all(tasks) - - // Update Tags - const tags = videoObject.tag.map(tag => tag.name) - const tagInstances = await TagModel.findOrCreateTags(tags, t) - await videoInstance.$set('Tags', tagInstances, sequelizeOptions) - - // Update captions - await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoInstance.id, t) - - const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { - return VideoCaptionModel.insertOrReplaceLanguage(videoInstance.id, c.identifier, t) - }) - await Promise.all(videoCaptionsPromises) - }) - - logger.info('Remote video with uuid %s updated', videoObject.uuid) - } catch (err) { - if (videoInstance !== undefined && videoFieldsSave !== undefined) { - resetSequelizeInstance(videoInstance, videoFieldsSave) - } - - // This is just a debug because we will retry the insert - logger.debug('Cannot update the remote video.', { err }) - throw err - } + return updateVideoFromAP(video, videoObject, actor, channelActor, activity.to) } async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 698414867..fe3d73e9b 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -6,6 +6,11 @@ import { VideoShareModel } from '../../models/video/video-share' import { sendUndoAnnounce, sendVideoAnnounce } from './send' import { getAnnounceActivityPubUrl } from './url' import { VideoChannelModel } from '../../models/video/video-channel' +import * as Bluebird from 'bluebird' +import { doRequest } from '../../helpers/requests' +import { getOrCreateActorAndServerAndModel } from './actor' +import { logger } from '../../helpers/logger' +import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { if (video.privacy === VideoPrivacy.PRIVATE) return undefined @@ -22,8 +27,41 @@ async function changeVideoChannelShare (video: VideoModel, oldVideoChannel: Vide await shareByVideoChannel(video, t) } +async function addVideoShares (shareUrls: string[], instance: VideoModel) { + await Bluebird.map(shareUrls, async shareUrl => { + try { + // Fetch url + const { body } = await doRequest({ + uri: shareUrl, + json: true, + activityPub: true + }) + if (!body || !body.actor) throw new Error('Body of body actor is invalid') + + const actorUrl = body.actor + const actor = await getOrCreateActorAndServerAndModel(actorUrl) + + const entry = { + actorId: actor.id, + videoId: instance.id, + url: shareUrl + } + + await VideoShareModel.findOrCreate({ + where: { + url: shareUrl + }, + defaults: entry + }) + } catch (err) { + logger.warn('Cannot add share %s.', shareUrl, { err }) + } + }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) +} + export { changeVideoChannelShare, + addVideoShares, shareVideoByServerAndChannel } diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index 14c7fde69..beff557bc 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -7,7 +7,7 @@ import { ActorModel } from '../../models/activitypub/actor' import { VideoModel } from '../../models/video/video' import { VideoCommentModel } from '../../models/video/video-comment' import { getOrCreateActorAndServerAndModel } from './actor' -import { getOrCreateAccountAndVideoAndChannel } from './videos' +import { getOrCreateVideoAndAccountAndChannel } from './videos' import * as Bluebird from 'bluebird' async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { @@ -91,7 +91,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) { try { // Maybe it's a reply to a video? - const { video } = await getOrCreateAccountAndVideoAndChannel(url) + const { video } = await getOrCreateVideoAndAccountAndChannel(url) if (comments.length !== 0) { const firstReply = comments[ comments.length - 1 ] diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 19011b4ab..1619251c3 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -2,6 +2,45 @@ import { Transaction } from 'sequelize' import { AccountModel } from '../../models/account/account' import { VideoModel } from '../../models/video/video' import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send' +import { VideoRateType } from '../../../shared/models/videos' +import * as Bluebird from 'bluebird' +import { getOrCreateActorAndServerAndModel } from './actor' +import { AccountVideoRateModel } from '../../models/account/account-video-rate' +import { logger } from '../../helpers/logger' +import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' + +async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { + let rateCounts = 0 + + await Bluebird.map(actorUrls, async actorUrl => { + try { + const actor = await getOrCreateActorAndServerAndModel(actorUrl) + const [ , created ] = await AccountVideoRateModel + .findOrCreate({ + where: { + videoId: video.id, + accountId: actor.Account.id + }, + defaults: { + videoId: video.id, + accountId: actor.Account.id, + type: rate + } + }) + + if (created) rateCounts += 1 + } catch (err) { + logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err }) + } + }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) + + logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid) + + // This is "likes" and "dislikes" + if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts }) + + return +} async function sendVideoRateChange (account: AccountModel, video: VideoModel, @@ -24,5 +63,6 @@ async function sendVideoRateChange (account: AccountModel, } export { + createRates, sendVideoRateChange } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index fac1d3fc7..388c31fe5 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -5,29 +5,30 @@ import { join } from 'path' import * as request from 'request' import { ActivityIconObject, VideoState } from '../../../shared/index' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' -import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos' +import { VideoPrivacy } from '../../../shared/models/videos' import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' -import { retryTransactionWrapper } from '../../helpers/database-utils' +import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' -import { ACTIVITY_PUB, CONFIG, CRAWL_REQUEST_CONCURRENCY, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' -import { AccountVideoRateModel } from '../../models/account/account-video-rate' +import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' import { ActorModel } from '../../models/activitypub/actor' import { TagModel } from '../../models/video/tag' import { VideoModel } from '../../models/video/video' import { VideoChannelModel } from '../../models/video/video-channel' import { VideoFileModel } from '../../models/video/video-file' -import { VideoShareModel } from '../../models/video/video-share' -import { getOrCreateActorAndServerAndModel } from './actor' +import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor' import { addVideoComments } from './video-comments' import { crawlCollectionPage } from './crawl' import { sendCreateVideo, sendUpdateVideo } from './send' -import { shareVideoByServerAndChannel } from './index' import { isArray } from '../../helpers/custom-validators/misc' import { VideoCaptionModel } from '../../models/video/video-caption' import { JobQueue } from '../job-queue' import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher' +import { getUrlFromWebfinger } from '../../helpers/webfinger' +import { createRates } from './video-rates' +import { addVideoShares, shareVideoByServerAndChannel } from './share' +import { AccountModel } from '../../models/account/account' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { // If the video is not private and published, we federate it @@ -180,15 +181,11 @@ function getOrCreateVideoChannel (videoObject: VideoTorrentObject) { return getOrCreateActorAndServerAndModel(channel.id) } -async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { +async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { logger.debug('Adding remote video %s.', videoObject.id) const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { - transaction: t - } - const videoFromDatabase = await VideoModel.loadByUUIDOrURLAndPopulateAccount(videoObject.uuid, videoObject.id, t) - if (videoFromDatabase) return videoFromDatabase + const sequelizeOptions = { transaction: t } const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) const video = VideoModel.build(videoData) @@ -230,26 +227,32 @@ async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: } type SyncParam = { - likes: boolean, - dislikes: boolean, - shares: boolean, - comments: boolean, + likes: boolean + dislikes: boolean + shares: boolean + comments: boolean thumbnail: boolean + refreshVideo: boolean } -async function getOrCreateAccountAndVideoAndChannel ( +async function getOrCreateVideoAndAccountAndChannel ( videoObject: VideoTorrentObject | string, - syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } + syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } ) { const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id - const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) - if (videoFromDatabase) return { video: videoFromDatabase } + let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) + if (videoFromDatabase) { + const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase) + if (syncParam.refreshVideo === true) videoFromDatabase = await p + + return { video: videoFromDatabase } + } - const fetchedVideo = await fetchRemoteVideo(videoUrl) + const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) const channelActor = await getOrCreateVideoChannel(fetchedVideo) - const video = await retryTransactionWrapper(getOrCreateVideo, fetchedVideo, channelActor, syncParam.thumbnail) + const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) // Process outside the transaction because we could fetch remote data @@ -290,101 +293,153 @@ async function getOrCreateAccountAndVideoAndChannel ( return { video } } -async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { - let rateCounts = 0 - - await Bluebird.map(actorUrls, async actorUrl => { - try { - const actor = await getOrCreateActorAndServerAndModel(actorUrl) - const [ , created ] = await AccountVideoRateModel - .findOrCreate({ - where: { - videoId: video.id, - accountId: actor.Account.id - }, - defaults: { - videoId: video.id, - accountId: actor.Account.id, - type: rate - } - }) - - if (created) rateCounts += 1 - } catch (err) { - logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err }) +async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { + const options = { + uri: videoUrl, + method: 'GET', + json: true, + activityPub: true + } + + logger.info('Fetching remote video %s.', videoUrl) + + const { response, body } = await doRequest(options) + + if (sanitizeAndCheckVideoTorrentObject(body) === false) { + logger.debug('Remote video JSON is not valid.', { body }) + return { response, videoObject: undefined } + } + + return { response, videoObject: body } +} + +async function refreshVideoIfNeeded (video: VideoModel): Promise { + if (!video.isOutdated()) return video + + try { + const { response, videoObject } = await fetchRemoteVideo(video.url) + if (response.statusCode === 404) { + // Video does not exist anymore + await video.destroy() + return undefined } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) - logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid) + if (videoObject === undefined) { + logger.warn('Cannot refresh remote video: invalid body.') + return video + } - // This is "likes" and "dislikes" - if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts }) + const channelActor = await getOrCreateVideoChannel(videoObject) + const account = await AccountModel.load(channelActor.VideoChannel.accountId) + return updateVideoFromAP(video, videoObject, account.Actor, channelActor) - return + } catch (err) { + logger.warn('Cannot refresh video.', { err }) + return video + } } -async function addVideoShares (shareUrls: string[], instance: VideoModel) { - await Bluebird.map(shareUrls, async shareUrl => { - try { - // Fetch url - const { body } = await doRequest({ - uri: shareUrl, - json: true, - activityPub: true - }) - if (!body || !body.actor) throw new Error('Body of body actor is invalid') +async function updateVideoFromAP ( + video: VideoModel, + videoObject: VideoTorrentObject, + accountActor: ActorModel, + channelActor: ActorModel, + overrideTo?: string[] +) { + logger.debug('Updating remote video "%s".', videoObject.uuid) + let videoFieldsSave: any + + try { + const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { + transaction: t + } - const actorUrl = body.actor - const actor = await getOrCreateActorAndServerAndModel(actorUrl) + videoFieldsSave = video.toJSON() - const entry = { - actorId: actor.id, - videoId: instance.id, - url: shareUrl + // Check actor has the right to update the video + const videoChannel = video.VideoChannel + if (videoChannel.Account.Actor.id !== accountActor.id) { + throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url) } - await VideoShareModel.findOrCreate({ - where: { - url: shareUrl - }, - defaults: entry - }) - } catch (err) { - logger.warn('Cannot add share %s.', shareUrl, { err }) - } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) -} + const to = overrideTo ? overrideTo : videoObject.to + const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to) + video.set('name', videoData.name) + video.set('uuid', videoData.uuid) + video.set('url', videoData.url) + video.set('category', videoData.category) + video.set('licence', videoData.licence) + video.set('language', videoData.language) + video.set('description', videoData.description) + video.set('support', videoData.support) + video.set('nsfw', videoData.nsfw) + video.set('commentsEnabled', videoData.commentsEnabled) + video.set('waitTranscoding', videoData.waitTranscoding) + video.set('state', videoData.state) + video.set('duration', videoData.duration) + video.set('createdAt', videoData.createdAt) + video.set('publishedAt', videoData.publishedAt) + video.set('views', videoData.views) + video.set('privacy', videoData.privacy) + video.set('channelId', videoData.channelId) + + await video.save(sequelizeOptions) + + // Don't block on request + generateThumbnailFromUrl(video, videoObject.icon) + .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) + + // Remove old video files + const videoFileDestroyTasks: Bluebird[] = [] + for (const videoFile of video.VideoFiles) { + videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) + } + await Promise.all(videoFileDestroyTasks) -async function fetchRemoteVideo (videoUrl: string): Promise { - const options = { - uri: videoUrl, - method: 'GET', - json: true, - activityPub: true - } + const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject) + const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) + await Promise.all(tasks) - logger.info('Fetching remote video %s.', videoUrl) + // Update Tags + const tags = videoObject.tag.map(tag => tag.name) + const tagInstances = await TagModel.findOrCreateTags(tags, t) + await video.$set('Tags', tagInstances, sequelizeOptions) - const { body } = await doRequest(options) + // Update captions + await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t) - if (sanitizeAndCheckVideoTorrentObject(body) === false) { - logger.debug('Remote video JSON is not valid.', { body }) - return undefined - } + const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { + return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t) + }) + await Promise.all(videoCaptionsPromises) + }) + + logger.info('Remote video with uuid %s updated', videoObject.uuid) - return body + return updatedVideo + } catch (err) { + if (video !== undefined && videoFieldsSave !== undefined) { + resetSequelizeInstance(video, videoFieldsSave) + } + + // This is just a debug because we will retry the insert + logger.debug('Cannot update the remote video.', { err }) + throw err + } } export { + updateVideoFromAP, federateVideoIfNeeded, fetchRemoteVideo, - getOrCreateAccountAndVideoAndChannel, + getOrCreateVideoAndAccountAndChannel, fetchRemoteVideoStaticFile, fetchRemoteVideoDescription, generateThumbnailFromUrl, videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes, - getOrCreateVideo, + createVideo, getOrCreateVideoChannel, addVideoShares, createRates -- cgit v1.2.3