From 304a84d59c3a800b7f7aef48cf55f307534c0926 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 2 Jun 2021 15:47:05 +0200 Subject: [PATCH] Refactor getOrCreateAPVideo --- server/controllers/api/search.ts | 4 +- server/controllers/api/videos/index.ts | 17 +- server/lib/activitypub/playlist.ts | 4 +- .../activitypub/process/process-announce.ts | 4 +- .../lib/activitypub/process/process-create.ts | 6 +- .../activitypub/process/process-dislike.ts | 4 +- .../lib/activitypub/process/process-like.ts | 4 +- .../lib/activitypub/process/process-undo.ts | 8 +- .../lib/activitypub/process/process-update.ts | 6 +- .../lib/activitypub/process/process-view.ts | 4 +- server/lib/activitypub/video-comments.ts | 4 +- server/lib/activitypub/videos/fetch.ts | 180 ------------------ server/lib/activitypub/videos/get.ts | 109 +++++++++++ server/lib/activitypub/videos/index.ts | 3 +- server/lib/activitypub/videos/refresh.ts | 64 +++++++ server/lib/activitypub/videos/shared/index.ts | 1 + .../videos/shared/url-to-object.ts | 22 +++ .../schedulers/videos-redundancy-scheduler.ts | 4 +- server/models/utils.ts | 4 + 19 files changed, 243 insertions(+), 209 deletions(-) delete mode 100644 server/lib/activitypub/videos/fetch.ts create mode 100644 server/lib/activitypub/videos/get.ts create mode 100644 server/lib/activitypub/videos/refresh.ts create mode 100644 server/lib/activitypub/videos/shared/url-to-object.ts diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index 77e3a024d..0cb5674c2 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts @@ -2,7 +2,7 @@ import * as express from 'express' import { sanitizeUrl } from '@server/helpers/core-utils' import { doJSONRequest } from '@server/helpers/requests' import { CONFIG } from '@server/initializers/config' -import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos' +import { getOrCreateAPVideo } from '@server/lib/activitypub/videos' import { Hooks } from '@server/lib/plugins/hooks' import { AccountBlocklistModel } from '@server/models/account/account-blocklist' import { getServerActor } from '@server/models/application/application' @@ -244,7 +244,7 @@ async function searchVideoURI (url: string, res: express.Response) { refreshVideo: false } - const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam }) + const result = await getOrCreateAPVideo({ videoObject: url, syncParam }) video = result ? result.video : undefined } catch (err) { logger.info('Cannot search remote video %s.', url, { err }) diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 47ab098ef..db23e5630 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -1,17 +1,18 @@ import * as express from 'express' import toInt from 'validator/lib/toInt' +import { doJSONRequest } from '@server/helpers/requests' import { LiveManager } from '@server/lib/live-manager' import { getServerActor } from '@server/models/application/application' +import { MVideoAccountLight } from '@server/types/models' import { VideosCommonQuery } from '../../../../shared' import { HttpStatusCode } from '../../../../shared/core-utils/miscs' import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' import { buildNSFWFilter, getCountVideos } from '../../../helpers/express-utils' import { logger } from '../../../helpers/logger' import { getFormattedObjects } from '../../../helpers/utils' -import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers/constants' +import { REMOTE_SCHEME, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers/constants' import { sequelizeTypescript } from '../../../initializers/database' import { sendView } from '../../../lib/activitypub/send/send-view' -import { fetchRemoteVideoDescription } from '../../../lib/activitypub/videos' import { JobQueue } from '../../../lib/job-queue' import { Hooks } from '../../../lib/plugins/hooks' import { Redis } from '../../../lib/redis' @@ -245,3 +246,15 @@ async function removeVideo (_req: express.Request, res: express.Response) { .status(HttpStatusCode.NO_CONTENT_204) .end() } + +// --------------------------------------------------------------------------- + +// FIXME: Should not exist, we rely on specific API +async function fetchRemoteVideoDescription (video: MVideoAccountLight) { + const host = video.VideoChannel.Account.Actor.Server.host + const path = video.getDescriptionAPIPath() + const url = REMOTE_SCHEME.HTTP + '://' + host + path + + const { body } = await doJSONRequest(url) + return body.description || '' +} diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts index 7166c68a6..8fe6e79f2 100644 --- a/server/lib/activitypub/playlist.ts +++ b/server/lib/activitypub/playlist.ts @@ -18,7 +18,7 @@ import { FilteredModelAttributes } from '../../types/sequelize' import { createPlaylistMiniatureFromUrl } from '../thumbnail' import { getOrCreateActorAndServerAndModel } from './actor' import { crawlCollectionPage } from './crawl' -import { getOrCreateVideoAndAccountAndChannel } from './videos' +import { getOrCreateAPVideo } from './videos' function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { const privacy = to.includes(ACTIVITY_PUB.PUBLIC) @@ -169,7 +169,7 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) } - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' }) + const { video } = await getOrCreateAPVideo({ videoObject: { id: body.url }, fetchType: 'only-video' }) elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video)) } catch (err) { diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index 63082466e..ec23c705e 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts @@ -3,7 +3,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' import { sequelizeTypescript } from '../../../initializers/database' import { VideoShareModel } from '../../../models/video/video-share' import { forwardVideoRelatedActivity } from '../send/utils' -import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { getOrCreateAPVideo } from '../videos' import { Notifier } from '../../notifier' import { logger } from '../../../helpers/logger' import { APProcessorOptions } from '../../../types/activitypub-processor.model' @@ -32,7 +32,7 @@ async function processVideoShare (actorAnnouncer: MActorSignature, activity: Act let videoCreated: boolean try { - const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) + const result = await getOrCreateAPVideo({ videoObject: objectUri }) video = result.video videoCreated = result.created } catch (err) { diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 9cded4dec..ef5a3100e 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -12,7 +12,7 @@ import { createOrUpdateCacheFile } from '../cache-file' import { createOrUpdateVideoPlaylist } from '../playlist' import { forwardVideoRelatedActivity } from '../send/utils' import { resolveThread } from '../video-comments' -import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { getOrCreateAPVideo } from '../videos' import { isBlockedByServerOrAccount } from '@server/lib/blocklist' async function processCreateActivity (options: APProcessorOptions) { @@ -55,7 +55,7 @@ async function processCreateVideo (activity: ActivityCreate, notify: boolean) { const videoToCreateData = activity.object as VideoObject const syncParam = { likes: false, dislikes: false, shares: false, comments: false, thumbnail: true, refreshVideo: false } - const { video, created } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData, syncParam }) + const { video, created } = await getOrCreateAPVideo({ videoObject: videoToCreateData, syncParam }) if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) @@ -67,7 +67,7 @@ async function processCreateCacheFile (activity: ActivityCreate, byActor: MActor const cacheFile = activity.object as CacheFileObject - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) + const { video } = await getOrCreateAPVideo({ videoObject: cacheFile.object }) await sequelizeTypescript.transaction(async t => { return createOrUpdateCacheFile(cacheFile, video, byActor, t) diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts index 089c7b881..ecc57cd10 100644 --- a/server/lib/activitypub/process/process-dislike.ts +++ b/server/lib/activitypub/process/process-dislike.ts @@ -6,7 +6,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat import { APProcessorOptions } from '../../../types/activitypub-processor.model' import { MActorSignature } from '../../../types/models' import { forwardVideoRelatedActivity } from '../send/utils' -import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { getOrCreateAPVideo } from '../videos' async function processDislikeActivity (options: APProcessorOptions) { const { activity, byActor } = options @@ -30,7 +30,7 @@ async function processDislike (activity: ActivityCreate | ActivityDislike, byAct if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislikeObject }) + const { video } = await getOrCreateAPVideo({ videoObject: dislikeObject }) return sequelizeTypescript.transaction(async t => { const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index 8688b3b47..cd4e86cbb 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts @@ -6,7 +6,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat import { APProcessorOptions } from '../../../types/activitypub-processor.model' import { MActorSignature } from '../../../types/models' import { forwardVideoRelatedActivity } from '../send/utils' -import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { getOrCreateAPVideo } from '../videos' async function processLikeActivity (options: APProcessorOptions) { const { activity, byActor } = options @@ -27,7 +27,7 @@ async function processLikeVideo (byActor: MActorSignature, activity: ActivityLik const byAccount = byActor.Account if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl }) + const { video } = await getOrCreateAPVideo({ videoObject: videoUrl }) return sequelizeTypescript.transaction(async t => { const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 9f031b528..fdb8dac24 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -11,7 +11,7 @@ import { VideoShareModel } from '../../../models/video/video-share' import { APProcessorOptions } from '../../../types/activitypub-processor.model' import { MActorSignature } from '../../../types/models' import { forwardVideoRelatedActivity } from '../send/utils' -import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { getOrCreateAPVideo } from '../videos' async function processUndoActivity (options: APProcessorOptions) { const { activity, byActor } = options @@ -55,7 +55,7 @@ export { async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) { const likeActivity = activity.object as ActivityLike - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object }) + const { video } = await getOrCreateAPVideo({ videoObject: likeActivity.object }) return sequelizeTypescript.transaction(async t => { if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) @@ -80,7 +80,7 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU ? activity.object : activity.object.object as DislikeObject - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) + const { video } = await getOrCreateAPVideo({ videoObject: dislike.object }) return sequelizeTypescript.transaction(async t => { if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) @@ -103,7 +103,7 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU async function processUndoCacheFile (byActor: MActorSignature, activity: ActivityUndo) { const cacheFileObject = activity.object.object as CacheFileObject - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) + const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) return sequelizeTypescript.transaction(async t => { const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 516bd8d70..be3f6acac 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -17,7 +17,7 @@ import { getImageInfoIfExists, updateActorImageInstance, updateActorInstance } f import { createOrUpdateCacheFile } from '../cache-file' import { createOrUpdateVideoPlaylist } from '../playlist' import { forwardVideoRelatedActivity } from '../send/utils' -import { APVideoUpdater, getOrCreateVideoAndAccountAndChannel } from '../videos' +import { APVideoUpdater, getOrCreateAPVideo } from '../videos' async function processUpdateActivity (options: APProcessorOptions) { const { activity, byActor } = options @@ -63,7 +63,7 @@ async function processUpdateVideo (activity: ActivityUpdate) { return undefined } - const { video, created } = await getOrCreateVideoAndAccountAndChannel({ + const { video, created } = await getOrCreateAPVideo({ videoObject: videoObject.id, allowRefresh: false, fetchType: 'all' @@ -85,7 +85,7 @@ async function processUpdateCacheFile (byActor: MActorSignature, activity: Activ return undefined } - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) + const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) await sequelizeTypescript.transaction(async t => { await createOrUpdateCacheFile(cacheFileObject, video, byActor, t) diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts index 84697673b..c2d41dd28 100644 --- a/server/lib/activitypub/process/process-view.ts +++ b/server/lib/activitypub/process/process-view.ts @@ -1,4 +1,4 @@ -import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { getOrCreateAPVideo } from '../videos' import { forwardVideoRelatedActivity } from '../send/utils' import { Redis } from '../../redis' import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' @@ -29,7 +29,7 @@ async function processCreateView (activity: ActivityView | ActivityCreate, byAct fetchType: 'only-video' as 'only-video', allowRefresh: false as false } - const { video } = await getOrCreateVideoAndAccountAndChannel(options) + const { video } = await getOrCreateAPVideo(options) if (!video.isLive) { await Redis.Instance.addVideoView(video.id) diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index e23e0c0e7..722147b69 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -7,7 +7,7 @@ import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/cons import { VideoCommentModel } from '../../models/video/video-comment' import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' import { getOrCreateActorAndServerAndModel } from './actor' -import { getOrCreateVideoAndAccountAndChannel } from './videos' +import { getOrCreateAPVideo } from './videos' type ResolveThreadParams = { url: string @@ -89,7 +89,7 @@ async function tryResolveThreadFromVideo (params: ResolveThreadParams) { // Maybe it's a reply to a video? // If yes, it's done: we resolved all the thread const syncParam = { likes: true, dislikes: true, shares: true, comments: false, thumbnail: true, refreshVideo: false } - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam }) + const { video } = await getOrCreateAPVideo({ videoObject: url, syncParam }) if (video.isOwned() && !video.hasPrivacyForFederation()) { throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation') diff --git a/server/lib/activitypub/videos/fetch.ts b/server/lib/activitypub/videos/fetch.ts deleted file mode 100644 index 5113c9d7e..000000000 --- a/server/lib/activitypub/videos/fetch.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { checkUrlsSameHost, getAPId } from '@server/helpers/activitypub' -import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests' -import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video' -import { REMOTE_SCHEME } from '@server/initializers/constants' -import { ActorFollowScoreCache } from '@server/lib/files-cache' -import { JobQueue } from '@server/lib/job-queue' -import { VideoModel } from '@server/models/video/video' -import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' -import { HttpStatusCode } from '@shared/core-utils' -import { VideoObject } from '@shared/models' -import { APVideoCreator, SyncParam, syncVideoExternalAttributes } from './shared' -import { APVideoUpdater } from './updater' - -async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { - logger.info('Fetching remote video %s.', videoUrl) - - const { statusCode, body } = await doJSONRequest(videoUrl, { activityPub: true }) - - if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { - logger.debug('Remote video JSON is not valid.', { body }) - return { statusCode, videoObject: undefined } - } - - return { statusCode, videoObject: body } -} - -async function fetchRemoteVideoDescription (video: MVideoAccountLight) { - const host = video.VideoChannel.Account.Actor.Server.host - const path = video.getDescriptionAPIPath() - const url = REMOTE_SCHEME.HTTP + '://' + host + path - - const { body } = await doJSONRequest(url) - return body.description || '' -} - -type GetVideoResult = Promise<{ - video: T - created: boolean - autoBlacklisted?: boolean -}> - -type GetVideoParamAll = { - videoObject: { id: string } | string - syncParam?: SyncParam - fetchType?: 'all' - allowRefresh?: boolean -} - -type GetVideoParamImmutable = { - videoObject: { id: string } | string - syncParam?: SyncParam - fetchType: 'only-immutable-attributes' - allowRefresh: false -} - -type GetVideoParamOther = { - videoObject: { id: string } | string - syncParam?: SyncParam - fetchType?: 'all' | 'only-video' - allowRefresh?: boolean -} - -function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult -function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult -function getOrCreateVideoAndAccountAndChannel ( - options: GetVideoParamOther -): GetVideoResult -async function getOrCreateVideoAndAccountAndChannel ( - options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther -): GetVideoResult { - // Default params - const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } - const fetchType = options.fetchType || 'all' - const allowRefresh = options.allowRefresh !== false - - // Get video url - const videoUrl = getAPId(options.videoObject) - let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) - - if (videoFromDatabase) { - // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type - if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) { - const refreshOptions = { - video: videoFromDatabase as MVideoThumbnail, - fetchedType: fetchType, - syncParam - } - - if (syncParam.refreshVideo === true) { - videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) - } else { - await JobQueue.Instance.createJobWithPromise({ - type: 'activitypub-refresher', - payload: { type: 'video', url: videoFromDatabase.url } - }) - } - } - - return { video: videoFromDatabase, created: false } - } - - const { videoObject } = await fetchRemoteVideo(videoUrl) - if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) - - try { - const creator = new APVideoCreator(videoObject) - const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail) - - await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) - - return { video: videoCreated, created: true, autoBlacklisted } - } catch (err) { - // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video - if (err.name === 'SequelizeUniqueConstraintError') { - const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType) - if (fallbackVideo) return { video: fallbackVideo, created: false } - } - - throw err - } -} - -async function refreshVideoIfNeeded (options: { - video: MVideoThumbnail - fetchedType: VideoFetchByUrlType - syncParam: SyncParam -}): Promise { - if (!options.video.isOutdated()) return options.video - - // We need more attributes if the argument video was fetched with not enough joints - const video = options.fetchedType === 'all' - ? options.video as MVideoAccountLightBlacklistAllFiles - : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) - - try { - const { videoObject } = await fetchRemoteVideo(video.url) - - if (videoObject === undefined) { - logger.warn('Cannot refresh remote video %s: invalid body.', video.url) - - await video.setAsRefreshed() - return video - } - - const videoUpdater = new APVideoUpdater(videoObject, video) - await videoUpdater.update() - - await syncVideoExternalAttributes(video, videoObject, options.syncParam) - - ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) - - return video - } catch (err) { - if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { - logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) - - // Video does not exist anymore - await video.destroy() - return undefined - } - - logger.warn('Cannot refresh video %s.', options.video.url, { err }) - - ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) - - // Don't refresh in loop - await video.setAsRefreshed() - return video - } -} - -export { - fetchRemoteVideo, - fetchRemoteVideoDescription, - refreshVideoIfNeeded, - getOrCreateVideoAndAccountAndChannel -} diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts new file mode 100644 index 000000000..a8c41e178 --- /dev/null +++ b/server/lib/activitypub/videos/get.ts @@ -0,0 +1,109 @@ +import { getAPId } from '@server/helpers/activitypub' +import { retryTransactionWrapper } from '@server/helpers/database-utils' +import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video' +import { JobQueue } from '@server/lib/job-queue' +import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' +import { refreshVideoIfNeeded } from './refresh' +import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' + +type GetVideoResult = Promise<{ + video: T + created: boolean + autoBlacklisted?: boolean +}> + +type GetVideoParamAll = { + videoObject: { id: string } | string + syncParam?: SyncParam + fetchType?: 'all' + allowRefresh?: boolean +} + +type GetVideoParamImmutable = { + videoObject: { id: string } | string + syncParam?: SyncParam + fetchType: 'only-immutable-attributes' + allowRefresh: false +} + +type GetVideoParamOther = { + videoObject: { id: string } | string + syncParam?: SyncParam + fetchType?: 'all' | 'only-video' + allowRefresh?: boolean +} + +function getOrCreateAPVideo (options: GetVideoParamAll): GetVideoResult +function getOrCreateAPVideo (options: GetVideoParamImmutable): GetVideoResult +function getOrCreateAPVideo (options: GetVideoParamOther): GetVideoResult + +async function getOrCreateAPVideo ( + options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther +): GetVideoResult { + // Default params + const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } + const fetchType = options.fetchType || 'all' + const allowRefresh = options.allowRefresh !== false + + // Get video url + const videoUrl = getAPId(options.videoObject) + let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) + + if (videoFromDatabase) { + if (allowRefresh === true) { + // Typings ensure allowRefresh === false in only-immutable-attributes fetch type + videoFromDatabase = await scheduleRefresh(videoFromDatabase as MVideoThumbnail, fetchType, syncParam) + } + + return { video: videoFromDatabase, created: false } + } + + const { videoObject } = await fetchRemoteVideo(videoUrl) + if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) + + try { + const creator = new APVideoCreator(videoObject) + const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail) + + await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) + + return { video: videoCreated, created: true, autoBlacklisted } + } catch (err) { + // Maybe a concurrent getOrCreateAPVideo call created this video + if (err.name === 'SequelizeUniqueConstraintError') { + const alreadyCreatedVideo = await fetchVideoByUrl(videoUrl, fetchType) + if (alreadyCreatedVideo) return { video: alreadyCreatedVideo, created: false } + } + + throw err + } +} + +// --------------------------------------------------------------------------- + +export { + getOrCreateAPVideo +} + +// --------------------------------------------------------------------------- + +async function scheduleRefresh (video: MVideoThumbnail, fetchType: VideoFetchByUrlType, syncParam: SyncParam) { + if (!video.isOutdated()) return video + + const refreshOptions = { + video, + fetchedType: fetchType, + syncParam + } + + if (syncParam.refreshVideo === true) { + return refreshVideoIfNeeded(refreshOptions) + } + + await JobQueue.Instance.createJobWithPromise({ + type: 'activitypub-refresher', + payload: { type: 'video', url: video.url } + }) + + return video +} diff --git a/server/lib/activitypub/videos/index.ts b/server/lib/activitypub/videos/index.ts index b560acb76..b22062598 100644 --- a/server/lib/activitypub/videos/index.ts +++ b/server/lib/activitypub/videos/index.ts @@ -1,3 +1,4 @@ export * from './federate' -export * from './fetch' +export * from './get' +export * from './refresh' export * from './updater' diff --git a/server/lib/activitypub/videos/refresh.ts b/server/lib/activitypub/videos/refresh.ts new file mode 100644 index 000000000..205a3ccb1 --- /dev/null +++ b/server/lib/activitypub/videos/refresh.ts @@ -0,0 +1,64 @@ +import { logger } from '@server/helpers/logger' +import { PeerTubeRequestError } from '@server/helpers/requests' +import { VideoFetchByUrlType } from '@server/helpers/video' +import { ActorFollowScoreCache } from '@server/lib/files-cache' +import { VideoModel } from '@server/models/video/video' +import { MVideoAccountLightBlacklistAllFiles, MVideoThumbnail } from '@server/types/models' +import { HttpStatusCode } from '@shared/core-utils' +import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' +import { APVideoUpdater } from './updater' + +async function refreshVideoIfNeeded (options: { + video: MVideoThumbnail + fetchedType: VideoFetchByUrlType + syncParam: SyncParam +}): Promise { + if (!options.video.isOutdated()) return options.video + + // We need more attributes if the argument video was fetched with not enough joints + const video = options.fetchedType === 'all' + ? options.video as MVideoAccountLightBlacklistAllFiles + : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) + + try { + const { videoObject } = await fetchRemoteVideo(video.url) + + if (videoObject === undefined) { + logger.warn('Cannot refresh remote video %s: invalid body.', video.url) + + await video.setAsRefreshed() + return video + } + + const videoUpdater = new APVideoUpdater(videoObject, video) + await videoUpdater.update() + + await syncVideoExternalAttributes(video, videoObject, options.syncParam) + + ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) + + return video + } catch (err) { + if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { + logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) + + // Video does not exist anymore + await video.destroy() + return undefined + } + + logger.warn('Cannot refresh video %s.', options.video.url, { err }) + + ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) + + // Don't refresh in loop + await video.setAsRefreshed() + return video + } +} + +// --------------------------------------------------------------------------- + +export { + refreshVideoIfNeeded +} diff --git a/server/lib/activitypub/videos/shared/index.ts b/server/lib/activitypub/videos/shared/index.ts index 208a43705..951403493 100644 --- a/server/lib/activitypub/videos/shared/index.ts +++ b/server/lib/activitypub/videos/shared/index.ts @@ -2,4 +2,5 @@ export * from './abstract-builder' export * from './creator' export * from './object-to-model-attributes' export * from './trackers' +export * from './url-to-object' export * from './video-sync-attributes' diff --git a/server/lib/activitypub/videos/shared/url-to-object.ts b/server/lib/activitypub/videos/shared/url-to-object.ts new file mode 100644 index 000000000..b1ecac8ca --- /dev/null +++ b/server/lib/activitypub/videos/shared/url-to-object.ts @@ -0,0 +1,22 @@ +import { checkUrlsSameHost } from '@server/helpers/activitypub' +import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' +import { logger } from '@server/helpers/logger' +import { doJSONRequest } from '@server/helpers/requests' +import { VideoObject } from '@shared/models' + +async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { + logger.info('Fetching remote video %s.', videoUrl) + + const { statusCode, body } = await doJSONRequest(videoUrl, { activityPub: true }) + + if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { + logger.debug('Remote video JSON is not valid.', { body }) + return { statusCode, videoObject: undefined } + } + + return { statusCode, videoObject: body } +} + +export { + fetchRemoteVideo +} diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 59b55cccc..b5a5eb697 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -23,7 +23,7 @@ import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../. import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' -import { getOrCreateVideoAndAccountAndChannel } from '../activitypub/videos' +import { getOrCreateAPVideo } from '../activitypub/videos' import { downloadPlaylistSegments } from '../hls' import { removeVideoRedundancy } from '../redundancy' import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' @@ -351,7 +351,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { syncParam: { likes: false, dislikes: false, shares: false, comments: false, thumbnail: false, refreshVideo: true }, fetchType: 'all' as 'all' } - const { video } = await getOrCreateVideoAndAccountAndChannel(getVideoOptions) + const { video } = await getOrCreateAPVideo(getVideoOptions) return video } diff --git a/server/models/utils.ts b/server/models/utils.ts index e27625bc8..83b2b8f03 100644 --- a/server/models/utils.ts +++ b/server/models/utils.ts @@ -102,6 +102,10 @@ function getFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): } function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) { + if (!model.createdAt || !model.updatedAt) { + throw new Error('Miss createdAt & updatedAt attribuets to model') + } + const now = Date.now() const createdAtTime = model.createdAt.getTime() const updatedAtTime = model.updatedAt.getTime() -- 2.41.0