From 69290ab37b8aead01477b9b98fdfad0e69b08582 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 2 Jun 2021 09:35:01 +0200 Subject: Refactor AP video update --- server/lib/activitypub/videos/fetch.ts | 202 +++++++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 server/lib/activitypub/videos/fetch.ts (limited to 'server/lib/activitypub/videos/fetch.ts') diff --git a/server/lib/activitypub/videos/fetch.ts b/server/lib/activitypub/videos/fetch.ts new file mode 100644 index 000000000..fdcf4ee5c --- /dev/null +++ b/server/lib/activitypub/videos/fetch.ts @@ -0,0 +1,202 @@ +import { checkUrlsSameHost, getAPId } from "@server/helpers/activitypub" +import { sanitizeAndCheckVideoTorrentObject } from "@server/helpers/custom-validators/activitypub/videos" +import { retryTransactionWrapper } from "@server/helpers/database-utils" +import { logger } from "@server/helpers/logger" +import { doJSONRequest, PeerTubeRequestError } from "@server/helpers/requests" +import { fetchVideoByUrl, VideoFetchByUrlType } from "@server/helpers/video" +import { REMOTE_SCHEME } from "@server/initializers/constants" +import { ActorFollowScoreCache } from "@server/lib/files-cache" +import { JobQueue } from "@server/lib/job-queue" +import { VideoModel } from "@server/models/video/video" +import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from "@server/types/models" +import { HttpStatusCode } from "@shared/core-utils" +import { VideoObject } from "@shared/models" +import { getOrCreateActorAndServerAndModel } from "../actor" +import { SyncParam, syncVideoExternalAttributes } from "./shared" +import { createVideo } from "./shared/video-create" +import { APVideoUpdater } from "./update" + +async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { + logger.info('Fetching remote video %s.', videoUrl) + + const { statusCode, body } = await doJSONRequest(videoUrl, { activityPub: true }) + + if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { + logger.debug('Remote video JSON is not valid.', { body }) + return { statusCode, videoObject: undefined } + } + + return { statusCode, videoObject: body } +} + +async function fetchRemoteVideoDescription (video: MVideoAccountLight) { + const host = video.VideoChannel.Account.Actor.Server.host + const path = video.getDescriptionAPIPath() + const url = REMOTE_SCHEME.HTTP + '://' + host + path + + const { body } = await doJSONRequest(url) + return body.description || '' +} + +function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) { + const channel = videoObject.attributedTo.find(a => a.type === 'Group') + if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) + + if (checkUrlsSameHost(channel.id, videoObject.id) !== true) { + throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`) + } + + return getOrCreateActorAndServerAndModel(channel.id, 'all') +} + +type GetVideoResult = Promise<{ + video: T + created: boolean + autoBlacklisted?: boolean +}> + +type GetVideoParamAll = { + videoObject: { id: string } | string + syncParam?: SyncParam + fetchType?: 'all' + allowRefresh?: boolean +} + +type GetVideoParamImmutable = { + videoObject: { id: string } | string + syncParam?: SyncParam + fetchType: 'only-immutable-attributes' + allowRefresh: false +} + +type GetVideoParamOther = { + videoObject: { id: string } | string + syncParam?: SyncParam + fetchType?: 'all' | 'only-video' + allowRefresh?: boolean +} + +function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult +function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult +function getOrCreateVideoAndAccountAndChannel ( + options: GetVideoParamOther +): GetVideoResult +async function getOrCreateVideoAndAccountAndChannel ( + options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther +): GetVideoResult { + // Default params + const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } + const fetchType = options.fetchType || 'all' + const allowRefresh = options.allowRefresh !== false + + // Get video url + const videoUrl = getAPId(options.videoObject) + let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) + + if (videoFromDatabase) { + // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type + if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) { + const refreshOptions = { + video: videoFromDatabase as MVideoThumbnail, + fetchedType: fetchType, + syncParam + } + + if (syncParam.refreshVideo === true) { + videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) + } else { + await JobQueue.Instance.createJobWithPromise({ + type: 'activitypub-refresher', + payload: { type: 'video', url: videoFromDatabase.url } + }) + } + } + + return { video: videoFromDatabase, created: false } + } + + const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) + if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) + + const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) + const videoChannel = actor.VideoChannel + + try { + const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail) + + await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam) + + return { video: videoCreated, created: true, autoBlacklisted } + } catch (err) { + // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video + if (err.name === 'SequelizeUniqueConstraintError') { + const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType) + if (fallbackVideo) return { video: fallbackVideo, created: false } + } + + throw err + } +} + +async function refreshVideoIfNeeded (options: { + video: MVideoThumbnail + fetchedType: VideoFetchByUrlType + syncParam: SyncParam +}): Promise { + if (!options.video.isOutdated()) return options.video + + // We need more attributes if the argument video was fetched with not enough joints + const video = options.fetchedType === 'all' + ? options.video as MVideoAccountLightBlacklistAllFiles + : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) + + try { + const { videoObject } = await fetchRemoteVideo(video.url) + + if (videoObject === undefined) { + logger.warn('Cannot refresh remote video %s: invalid body.', video.url) + + await video.setAsRefreshed() + return video + } + + const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) + + const videoUpdater = new APVideoUpdater({ + video, + videoObject, + channel: channelActor.VideoChannel + }) + await videoUpdater.update() + + await syncVideoExternalAttributes(video, videoObject, options.syncParam) + + ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) + + return video + } catch (err) { + if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { + logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) + + // Video does not exist anymore + await video.destroy() + return undefined + } + + logger.warn('Cannot refresh video %s.', options.video.url, { err }) + + ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) + + // Don't refresh in loop + await video.setAsRefreshed() + return video + } +} + +export { + fetchRemoteVideo, + fetchRemoteVideoDescription, + refreshVideoIfNeeded, + getOrCreateVideoChannelFromVideoObject, + getOrCreateVideoAndAccountAndChannel +} -- cgit v1.2.3