From 304a84d59c3a800b7f7aef48cf55f307534c0926 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 2 Jun 2021 15:47:05 +0200 Subject: Refactor getOrCreateAPVideo --- server/lib/activitypub/videos/fetch.ts | 180 --------------------- server/lib/activitypub/videos/get.ts | 109 +++++++++++++ server/lib/activitypub/videos/index.ts | 3 +- server/lib/activitypub/videos/refresh.ts | 64 ++++++++ server/lib/activitypub/videos/shared/index.ts | 1 + .../lib/activitypub/videos/shared/url-to-object.ts | 22 +++ 6 files changed, 198 insertions(+), 181 deletions(-) delete mode 100644 server/lib/activitypub/videos/fetch.ts create mode 100644 server/lib/activitypub/videos/get.ts create mode 100644 server/lib/activitypub/videos/refresh.ts create mode 100644 server/lib/activitypub/videos/shared/url-to-object.ts (limited to 'server/lib/activitypub/videos') diff --git a/server/lib/activitypub/videos/fetch.ts b/server/lib/activitypub/videos/fetch.ts deleted file mode 100644 index 5113c9d7e..000000000 --- a/server/lib/activitypub/videos/fetch.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { checkUrlsSameHost, getAPId } from '@server/helpers/activitypub' -import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests' -import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video' -import { REMOTE_SCHEME } from '@server/initializers/constants' -import { ActorFollowScoreCache } from '@server/lib/files-cache' -import { JobQueue } from '@server/lib/job-queue' -import { VideoModel } from '@server/models/video/video' -import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' -import { HttpStatusCode } from '@shared/core-utils' -import { VideoObject } from '@shared/models' -import { APVideoCreator, SyncParam, syncVideoExternalAttributes } from './shared' -import { APVideoUpdater } from './updater' - -async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { - logger.info('Fetching remote video %s.', videoUrl) - - const { statusCode, body } = await doJSONRequest(videoUrl, { activityPub: true }) - - if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { - logger.debug('Remote video JSON is not valid.', { body }) - return { statusCode, videoObject: undefined } - } - - return { statusCode, videoObject: body } -} - -async function fetchRemoteVideoDescription (video: MVideoAccountLight) { - const host = video.VideoChannel.Account.Actor.Server.host - const path = video.getDescriptionAPIPath() - const url = REMOTE_SCHEME.HTTP + '://' + host + path - - const { body } = await doJSONRequest(url) - return body.description || '' -} - -type GetVideoResult = Promise<{ - video: T - created: boolean - autoBlacklisted?: boolean -}> - -type GetVideoParamAll = { - videoObject: { id: string } | string - syncParam?: SyncParam - fetchType?: 'all' - allowRefresh?: boolean -} - -type GetVideoParamImmutable = { - videoObject: { id: string } | string - syncParam?: SyncParam - fetchType: 'only-immutable-attributes' - allowRefresh: false -} - -type GetVideoParamOther = { - videoObject: { id: string } | string - syncParam?: SyncParam - fetchType?: 'all' | 'only-video' - allowRefresh?: boolean -} - -function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult -function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult -function getOrCreateVideoAndAccountAndChannel ( - options: GetVideoParamOther -): GetVideoResult -async function getOrCreateVideoAndAccountAndChannel ( - options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther -): GetVideoResult { - // Default params - const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } - const fetchType = options.fetchType || 'all' - const allowRefresh = options.allowRefresh !== false - - // Get video url - const videoUrl = getAPId(options.videoObject) - let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) - - if (videoFromDatabase) { - // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type - if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) { - const refreshOptions = { - video: videoFromDatabase as MVideoThumbnail, - fetchedType: fetchType, - syncParam - } - - if (syncParam.refreshVideo === true) { - videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) - } else { - await JobQueue.Instance.createJobWithPromise({ - type: 'activitypub-refresher', - payload: { type: 'video', url: videoFromDatabase.url } - }) - } - } - - return { video: videoFromDatabase, created: false } - } - - const { videoObject } = await fetchRemoteVideo(videoUrl) - if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) - - try { - const creator = new APVideoCreator(videoObject) - const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail) - - await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) - - return { video: videoCreated, created: true, autoBlacklisted } - } catch (err) { - // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video - if (err.name === 'SequelizeUniqueConstraintError') { - const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType) - if (fallbackVideo) return { video: fallbackVideo, created: false } - } - - throw err - } -} - -async function refreshVideoIfNeeded (options: { - video: MVideoThumbnail - fetchedType: VideoFetchByUrlType - syncParam: SyncParam -}): Promise { - if (!options.video.isOutdated()) return options.video - - // We need more attributes if the argument video was fetched with not enough joints - const video = options.fetchedType === 'all' - ? options.video as MVideoAccountLightBlacklistAllFiles - : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) - - try { - const { videoObject } = await fetchRemoteVideo(video.url) - - if (videoObject === undefined) { - logger.warn('Cannot refresh remote video %s: invalid body.', video.url) - - await video.setAsRefreshed() - return video - } - - const videoUpdater = new APVideoUpdater(videoObject, video) - await videoUpdater.update() - - await syncVideoExternalAttributes(video, videoObject, options.syncParam) - - ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) - - return video - } catch (err) { - if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { - logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) - - // Video does not exist anymore - await video.destroy() - return undefined - } - - logger.warn('Cannot refresh video %s.', options.video.url, { err }) - - ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) - - // Don't refresh in loop - await video.setAsRefreshed() - return video - } -} - -export { - fetchRemoteVideo, - fetchRemoteVideoDescription, - refreshVideoIfNeeded, - getOrCreateVideoAndAccountAndChannel -} diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts new file mode 100644 index 000000000..a8c41e178 --- /dev/null +++ b/server/lib/activitypub/videos/get.ts @@ -0,0 +1,109 @@ +import { getAPId } from '@server/helpers/activitypub' +import { retryTransactionWrapper } from '@server/helpers/database-utils' +import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video' +import { JobQueue } from '@server/lib/job-queue' +import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' +import { refreshVideoIfNeeded } from './refresh' +import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' + +type GetVideoResult = Promise<{ + video: T + created: boolean + autoBlacklisted?: boolean +}> + +type GetVideoParamAll = { + videoObject: { id: string } | string + syncParam?: SyncParam + fetchType?: 'all' + allowRefresh?: boolean +} + +type GetVideoParamImmutable = { + videoObject: { id: string } | string + syncParam?: SyncParam + fetchType: 'only-immutable-attributes' + allowRefresh: false +} + +type GetVideoParamOther = { + videoObject: { id: string } | string + syncParam?: SyncParam + fetchType?: 'all' | 'only-video' + allowRefresh?: boolean +} + +function getOrCreateAPVideo (options: GetVideoParamAll): GetVideoResult +function getOrCreateAPVideo (options: GetVideoParamImmutable): GetVideoResult +function getOrCreateAPVideo (options: GetVideoParamOther): GetVideoResult + +async function getOrCreateAPVideo ( + options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther +): GetVideoResult { + // Default params + const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } + const fetchType = options.fetchType || 'all' + const allowRefresh = options.allowRefresh !== false + + // Get video url + const videoUrl = getAPId(options.videoObject) + let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) + + if (videoFromDatabase) { + if (allowRefresh === true) { + // Typings ensure allowRefresh === false in only-immutable-attributes fetch type + videoFromDatabase = await scheduleRefresh(videoFromDatabase as MVideoThumbnail, fetchType, syncParam) + } + + return { video: videoFromDatabase, created: false } + } + + const { videoObject } = await fetchRemoteVideo(videoUrl) + if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) + + try { + const creator = new APVideoCreator(videoObject) + const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail) + + await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) + + return { video: videoCreated, created: true, autoBlacklisted } + } catch (err) { + // Maybe a concurrent getOrCreateAPVideo call created this video + if (err.name === 'SequelizeUniqueConstraintError') { + const alreadyCreatedVideo = await fetchVideoByUrl(videoUrl, fetchType) + if (alreadyCreatedVideo) return { video: alreadyCreatedVideo, created: false } + } + + throw err + } +} + +// --------------------------------------------------------------------------- + +export { + getOrCreateAPVideo +} + +// --------------------------------------------------------------------------- + +async function scheduleRefresh (video: MVideoThumbnail, fetchType: VideoFetchByUrlType, syncParam: SyncParam) { + if (!video.isOutdated()) return video + + const refreshOptions = { + video, + fetchedType: fetchType, + syncParam + } + + if (syncParam.refreshVideo === true) { + return refreshVideoIfNeeded(refreshOptions) + } + + await JobQueue.Instance.createJobWithPromise({ + type: 'activitypub-refresher', + payload: { type: 'video', url: video.url } + }) + + return video +} diff --git a/server/lib/activitypub/videos/index.ts b/server/lib/activitypub/videos/index.ts index b560acb76..b22062598 100644 --- a/server/lib/activitypub/videos/index.ts +++ b/server/lib/activitypub/videos/index.ts @@ -1,3 +1,4 @@ export * from './federate' -export * from './fetch' +export * from './get' +export * from './refresh' export * from './updater' diff --git a/server/lib/activitypub/videos/refresh.ts b/server/lib/activitypub/videos/refresh.ts new file mode 100644 index 000000000..205a3ccb1 --- /dev/null +++ b/server/lib/activitypub/videos/refresh.ts @@ -0,0 +1,64 @@ +import { logger } from '@server/helpers/logger' +import { PeerTubeRequestError } from '@server/helpers/requests' +import { VideoFetchByUrlType } from '@server/helpers/video' +import { ActorFollowScoreCache } from '@server/lib/files-cache' +import { VideoModel } from '@server/models/video/video' +import { MVideoAccountLightBlacklistAllFiles, MVideoThumbnail } from '@server/types/models' +import { HttpStatusCode } from '@shared/core-utils' +import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' +import { APVideoUpdater } from './updater' + +async function refreshVideoIfNeeded (options: { + video: MVideoThumbnail + fetchedType: VideoFetchByUrlType + syncParam: SyncParam +}): Promise { + if (!options.video.isOutdated()) return options.video + + // We need more attributes if the argument video was fetched with not enough joints + const video = options.fetchedType === 'all' + ? options.video as MVideoAccountLightBlacklistAllFiles + : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) + + try { + const { videoObject } = await fetchRemoteVideo(video.url) + + if (videoObject === undefined) { + logger.warn('Cannot refresh remote video %s: invalid body.', video.url) + + await video.setAsRefreshed() + return video + } + + const videoUpdater = new APVideoUpdater(videoObject, video) + await videoUpdater.update() + + await syncVideoExternalAttributes(video, videoObject, options.syncParam) + + ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) + + return video + } catch (err) { + if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { + logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) + + // Video does not exist anymore + await video.destroy() + return undefined + } + + logger.warn('Cannot refresh video %s.', options.video.url, { err }) + + ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) + + // Don't refresh in loop + await video.setAsRefreshed() + return video + } +} + +// --------------------------------------------------------------------------- + +export { + refreshVideoIfNeeded +} diff --git a/server/lib/activitypub/videos/shared/index.ts b/server/lib/activitypub/videos/shared/index.ts index 208a43705..951403493 100644 --- a/server/lib/activitypub/videos/shared/index.ts +++ b/server/lib/activitypub/videos/shared/index.ts @@ -2,4 +2,5 @@ export * from './abstract-builder' export * from './creator' export * from './object-to-model-attributes' export * from './trackers' +export * from './url-to-object' export * from './video-sync-attributes' diff --git a/server/lib/activitypub/videos/shared/url-to-object.ts b/server/lib/activitypub/videos/shared/url-to-object.ts new file mode 100644 index 000000000..b1ecac8ca --- /dev/null +++ b/server/lib/activitypub/videos/shared/url-to-object.ts @@ -0,0 +1,22 @@ +import { checkUrlsSameHost } from '@server/helpers/activitypub' +import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' +import { logger } from '@server/helpers/logger' +import { doJSONRequest } from '@server/helpers/requests' +import { VideoObject } from '@shared/models' + +async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { + logger.info('Fetching remote video %s.', videoUrl) + + const { statusCode, body } = await doJSONRequest(videoUrl, { activityPub: true }) + + if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { + logger.debug('Remote video JSON is not valid.', { body }) + return { statusCode, videoObject: undefined } + } + + return { statusCode, videoObject: body } +} + +export { + fetchRemoteVideo +} -- cgit v1.2.3