diff options
Diffstat (limited to 'server/lib')
-rw-r--r-- | server/lib/activitypub/playlist.ts | 4 | ||||
-rw-r--r-- | server/lib/activitypub/process/process-announce.ts | 4 | ||||
-rw-r--r-- | server/lib/activitypub/process/process-create.ts | 6 | ||||
-rw-r--r-- | server/lib/activitypub/process/process-dislike.ts | 4 | ||||
-rw-r--r-- | server/lib/activitypub/process/process-like.ts | 4 | ||||
-rw-r--r-- | server/lib/activitypub/process/process-undo.ts | 8 | ||||
-rw-r--r-- | server/lib/activitypub/process/process-update.ts | 6 | ||||
-rw-r--r-- | server/lib/activitypub/process/process-view.ts | 4 | ||||
-rw-r--r-- | server/lib/activitypub/video-comments.ts | 4 | ||||
-rw-r--r-- | server/lib/activitypub/videos/fetch.ts | 180 | ||||
-rw-r--r-- | server/lib/activitypub/videos/get.ts | 109 | ||||
-rw-r--r-- | server/lib/activitypub/videos/index.ts | 3 | ||||
-rw-r--r-- | server/lib/activitypub/videos/refresh.ts | 64 | ||||
-rw-r--r-- | server/lib/activitypub/videos/shared/index.ts | 1 | ||||
-rw-r--r-- | server/lib/activitypub/videos/shared/url-to-object.ts | 22 | ||||
-rw-r--r-- | server/lib/schedulers/videos-redundancy-scheduler.ts | 4 |
16 files changed, 222 insertions, 205 deletions
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts index 7166c68a6..8fe6e79f2 100644 --- a/server/lib/activitypub/playlist.ts +++ b/server/lib/activitypub/playlist.ts | |||
@@ -18,7 +18,7 @@ import { FilteredModelAttributes } from '../../types/sequelize' | |||
18 | import { createPlaylistMiniatureFromUrl } from '../thumbnail' | 18 | import { createPlaylistMiniatureFromUrl } from '../thumbnail' |
19 | import { getOrCreateActorAndServerAndModel } from './actor' | 19 | import { getOrCreateActorAndServerAndModel } from './actor' |
20 | import { crawlCollectionPage } from './crawl' | 20 | import { crawlCollectionPage } from './crawl' |
21 | import { getOrCreateVideoAndAccountAndChannel } from './videos' | 21 | import { getOrCreateAPVideo } from './videos' |
22 | 22 | ||
23 | function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { | 23 | function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { |
24 | const privacy = to.includes(ACTIVITY_PUB.PUBLIC) | 24 | const privacy = to.includes(ACTIVITY_PUB.PUBLIC) |
@@ -169,7 +169,7 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid | |||
169 | throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) | 169 | throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) |
170 | } | 170 | } |
171 | 171 | ||
172 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' }) | 172 | const { video } = await getOrCreateAPVideo({ videoObject: { id: body.url }, fetchType: 'only-video' }) |
173 | 173 | ||
174 | elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video)) | 174 | elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video)) |
175 | } catch (err) { | 175 | } catch (err) { |
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index 63082466e..ec23c705e 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts | |||
@@ -3,7 +3,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' | |||
3 | import { sequelizeTypescript } from '../../../initializers/database' | 3 | import { sequelizeTypescript } from '../../../initializers/database' |
4 | import { VideoShareModel } from '../../../models/video/video-share' | 4 | import { VideoShareModel } from '../../../models/video/video-share' |
5 | import { forwardVideoRelatedActivity } from '../send/utils' | 5 | import { forwardVideoRelatedActivity } from '../send/utils' |
6 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 6 | import { getOrCreateAPVideo } from '../videos' |
7 | import { Notifier } from '../../notifier' | 7 | import { Notifier } from '../../notifier' |
8 | import { logger } from '../../../helpers/logger' | 8 | import { logger } from '../../../helpers/logger' |
9 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 9 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
@@ -32,7 +32,7 @@ async function processVideoShare (actorAnnouncer: MActorSignature, activity: Act | |||
32 | let videoCreated: boolean | 32 | let videoCreated: boolean |
33 | 33 | ||
34 | try { | 34 | try { |
35 | const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) | 35 | const result = await getOrCreateAPVideo({ videoObject: objectUri }) |
36 | video = result.video | 36 | video = result.video |
37 | videoCreated = result.created | 37 | videoCreated = result.created |
38 | } catch (err) { | 38 | } catch (err) { |
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 9cded4dec..ef5a3100e 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -12,7 +12,7 @@ import { createOrUpdateCacheFile } from '../cache-file' | |||
12 | import { createOrUpdateVideoPlaylist } from '../playlist' | 12 | import { createOrUpdateVideoPlaylist } from '../playlist' |
13 | import { forwardVideoRelatedActivity } from '../send/utils' | 13 | import { forwardVideoRelatedActivity } from '../send/utils' |
14 | import { resolveThread } from '../video-comments' | 14 | import { resolveThread } from '../video-comments' |
15 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 15 | import { getOrCreateAPVideo } from '../videos' |
16 | import { isBlockedByServerOrAccount } from '@server/lib/blocklist' | 16 | import { isBlockedByServerOrAccount } from '@server/lib/blocklist' |
17 | 17 | ||
18 | async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) { | 18 | async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) { |
@@ -55,7 +55,7 @@ async function processCreateVideo (activity: ActivityCreate, notify: boolean) { | |||
55 | const videoToCreateData = activity.object as VideoObject | 55 | const videoToCreateData = activity.object as VideoObject |
56 | 56 | ||
57 | const syncParam = { likes: false, dislikes: false, shares: false, comments: false, thumbnail: true, refreshVideo: false } | 57 | const syncParam = { likes: false, dislikes: false, shares: false, comments: false, thumbnail: true, refreshVideo: false } |
58 | const { video, created } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData, syncParam }) | 58 | const { video, created } = await getOrCreateAPVideo({ videoObject: videoToCreateData, syncParam }) |
59 | 59 | ||
60 | if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) | 60 | if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) |
61 | 61 | ||
@@ -67,7 +67,7 @@ async function processCreateCacheFile (activity: ActivityCreate, byActor: MActor | |||
67 | 67 | ||
68 | const cacheFile = activity.object as CacheFileObject | 68 | const cacheFile = activity.object as CacheFileObject |
69 | 69 | ||
70 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) | 70 | const { video } = await getOrCreateAPVideo({ videoObject: cacheFile.object }) |
71 | 71 | ||
72 | await sequelizeTypescript.transaction(async t => { | 72 | await sequelizeTypescript.transaction(async t => { |
73 | return createOrUpdateCacheFile(cacheFile, video, byActor, t) | 73 | return createOrUpdateCacheFile(cacheFile, video, byActor, t) |
diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts index 089c7b881..ecc57cd10 100644 --- a/server/lib/activitypub/process/process-dislike.ts +++ b/server/lib/activitypub/process/process-dislike.ts | |||
@@ -6,7 +6,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat | |||
6 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 6 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
7 | import { MActorSignature } from '../../../types/models' | 7 | import { MActorSignature } from '../../../types/models' |
8 | import { forwardVideoRelatedActivity } from '../send/utils' | 8 | import { forwardVideoRelatedActivity } from '../send/utils' |
9 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 9 | import { getOrCreateAPVideo } from '../videos' |
10 | 10 | ||
11 | async function processDislikeActivity (options: APProcessorOptions<ActivityCreate | ActivityDislike>) { | 11 | async function processDislikeActivity (options: APProcessorOptions<ActivityCreate | ActivityDislike>) { |
12 | const { activity, byActor } = options | 12 | const { activity, byActor } = options |
@@ -30,7 +30,7 @@ async function processDislike (activity: ActivityCreate | ActivityDislike, byAct | |||
30 | 30 | ||
31 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | 31 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) |
32 | 32 | ||
33 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislikeObject }) | 33 | const { video } = await getOrCreateAPVideo({ videoObject: dislikeObject }) |
34 | 34 | ||
35 | return sequelizeTypescript.transaction(async t => { | 35 | return sequelizeTypescript.transaction(async t => { |
36 | const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) | 36 | const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) |
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index 8688b3b47..cd4e86cbb 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts | |||
@@ -6,7 +6,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat | |||
6 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 6 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
7 | import { MActorSignature } from '../../../types/models' | 7 | import { MActorSignature } from '../../../types/models' |
8 | import { forwardVideoRelatedActivity } from '../send/utils' | 8 | import { forwardVideoRelatedActivity } from '../send/utils' |
9 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 9 | import { getOrCreateAPVideo } from '../videos' |
10 | 10 | ||
11 | async function processLikeActivity (options: APProcessorOptions<ActivityLike>) { | 11 | async function processLikeActivity (options: APProcessorOptions<ActivityLike>) { |
12 | const { activity, byActor } = options | 12 | const { activity, byActor } = options |
@@ -27,7 +27,7 @@ async function processLikeVideo (byActor: MActorSignature, activity: ActivityLik | |||
27 | const byAccount = byActor.Account | 27 | const byAccount = byActor.Account |
28 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) | 28 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) |
29 | 29 | ||
30 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl }) | 30 | const { video } = await getOrCreateAPVideo({ videoObject: videoUrl }) |
31 | 31 | ||
32 | return sequelizeTypescript.transaction(async t => { | 32 | return sequelizeTypescript.transaction(async t => { |
33 | const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) | 33 | const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) |
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 9f031b528..fdb8dac24 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts | |||
@@ -11,7 +11,7 @@ import { VideoShareModel } from '../../../models/video/video-share' | |||
11 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 11 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
12 | import { MActorSignature } from '../../../types/models' | 12 | import { MActorSignature } from '../../../types/models' |
13 | import { forwardVideoRelatedActivity } from '../send/utils' | 13 | import { forwardVideoRelatedActivity } from '../send/utils' |
14 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 14 | import { getOrCreateAPVideo } from '../videos' |
15 | 15 | ||
16 | async function processUndoActivity (options: APProcessorOptions<ActivityUndo>) { | 16 | async function processUndoActivity (options: APProcessorOptions<ActivityUndo>) { |
17 | const { activity, byActor } = options | 17 | const { activity, byActor } = options |
@@ -55,7 +55,7 @@ export { | |||
55 | async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) { | 55 | async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) { |
56 | const likeActivity = activity.object as ActivityLike | 56 | const likeActivity = activity.object as ActivityLike |
57 | 57 | ||
58 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object }) | 58 | const { video } = await getOrCreateAPVideo({ videoObject: likeActivity.object }) |
59 | 59 | ||
60 | return sequelizeTypescript.transaction(async t => { | 60 | return sequelizeTypescript.transaction(async t => { |
61 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) | 61 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) |
@@ -80,7 +80,7 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU | |||
80 | ? activity.object | 80 | ? activity.object |
81 | : activity.object.object as DislikeObject | 81 | : activity.object.object as DislikeObject |
82 | 82 | ||
83 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) | 83 | const { video } = await getOrCreateAPVideo({ videoObject: dislike.object }) |
84 | 84 | ||
85 | return sequelizeTypescript.transaction(async t => { | 85 | return sequelizeTypescript.transaction(async t => { |
86 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) | 86 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) |
@@ -103,7 +103,7 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU | |||
103 | async function processUndoCacheFile (byActor: MActorSignature, activity: ActivityUndo) { | 103 | async function processUndoCacheFile (byActor: MActorSignature, activity: ActivityUndo) { |
104 | const cacheFileObject = activity.object.object as CacheFileObject | 104 | const cacheFileObject = activity.object.object as CacheFileObject |
105 | 105 | ||
106 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) | 106 | const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) |
107 | 107 | ||
108 | return sequelizeTypescript.transaction(async t => { | 108 | return sequelizeTypescript.transaction(async t => { |
109 | const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) | 109 | const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) |
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 516bd8d70..be3f6acac 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts | |||
@@ -17,7 +17,7 @@ import { getImageInfoIfExists, updateActorImageInstance, updateActorInstance } f | |||
17 | import { createOrUpdateCacheFile } from '../cache-file' | 17 | import { createOrUpdateCacheFile } from '../cache-file' |
18 | import { createOrUpdateVideoPlaylist } from '../playlist' | 18 | import { createOrUpdateVideoPlaylist } from '../playlist' |
19 | import { forwardVideoRelatedActivity } from '../send/utils' | 19 | import { forwardVideoRelatedActivity } from '../send/utils' |
20 | import { APVideoUpdater, getOrCreateVideoAndAccountAndChannel } from '../videos' | 20 | import { APVideoUpdater, getOrCreateAPVideo } from '../videos' |
21 | 21 | ||
22 | async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) { | 22 | async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) { |
23 | const { activity, byActor } = options | 23 | const { activity, byActor } = options |
@@ -63,7 +63,7 @@ async function processUpdateVideo (activity: ActivityUpdate) { | |||
63 | return undefined | 63 | return undefined |
64 | } | 64 | } |
65 | 65 | ||
66 | const { video, created } = await getOrCreateVideoAndAccountAndChannel({ | 66 | const { video, created } = await getOrCreateAPVideo({ |
67 | videoObject: videoObject.id, | 67 | videoObject: videoObject.id, |
68 | allowRefresh: false, | 68 | allowRefresh: false, |
69 | fetchType: 'all' | 69 | fetchType: 'all' |
@@ -85,7 +85,7 @@ async function processUpdateCacheFile (byActor: MActorSignature, activity: Activ | |||
85 | return undefined | 85 | return undefined |
86 | } | 86 | } |
87 | 87 | ||
88 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) | 88 | const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) |
89 | 89 | ||
90 | await sequelizeTypescript.transaction(async t => { | 90 | await sequelizeTypescript.transaction(async t => { |
91 | await createOrUpdateCacheFile(cacheFileObject, video, byActor, t) | 91 | await createOrUpdateCacheFile(cacheFileObject, video, byActor, t) |
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts index 84697673b..c2d41dd28 100644 --- a/server/lib/activitypub/process/process-view.ts +++ b/server/lib/activitypub/process/process-view.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 1 | import { getOrCreateAPVideo } from '../videos' |
2 | import { forwardVideoRelatedActivity } from '../send/utils' | 2 | import { forwardVideoRelatedActivity } from '../send/utils' |
3 | import { Redis } from '../../redis' | 3 | import { Redis } from '../../redis' |
4 | import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' | 4 | import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' |
@@ -29,7 +29,7 @@ async function processCreateView (activity: ActivityView | ActivityCreate, byAct | |||
29 | fetchType: 'only-video' as 'only-video', | 29 | fetchType: 'only-video' as 'only-video', |
30 | allowRefresh: false as false | 30 | allowRefresh: false as false |
31 | } | 31 | } |
32 | const { video } = await getOrCreateVideoAndAccountAndChannel(options) | 32 | const { video } = await getOrCreateAPVideo(options) |
33 | 33 | ||
34 | if (!video.isLive) { | 34 | if (!video.isLive) { |
35 | await Redis.Instance.addVideoView(video.id) | 35 | await Redis.Instance.addVideoView(video.id) |
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index e23e0c0e7..722147b69 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts | |||
@@ -7,7 +7,7 @@ import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/cons | |||
7 | import { VideoCommentModel } from '../../models/video/video-comment' | 7 | import { VideoCommentModel } from '../../models/video/video-comment' |
8 | import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' | 8 | import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' |
9 | import { getOrCreateActorAndServerAndModel } from './actor' | 9 | import { getOrCreateActorAndServerAndModel } from './actor' |
10 | import { getOrCreateVideoAndAccountAndChannel } from './videos' | 10 | import { getOrCreateAPVideo } from './videos' |
11 | 11 | ||
12 | type ResolveThreadParams = { | 12 | type ResolveThreadParams = { |
13 | url: string | 13 | url: string |
@@ -89,7 +89,7 @@ async function tryResolveThreadFromVideo (params: ResolveThreadParams) { | |||
89 | // Maybe it's a reply to a video? | 89 | // Maybe it's a reply to a video? |
90 | // If yes, it's done: we resolved all the thread | 90 | // If yes, it's done: we resolved all the thread |
91 | const syncParam = { likes: true, dislikes: true, shares: true, comments: false, thumbnail: true, refreshVideo: false } | 91 | const syncParam = { likes: true, dislikes: true, shares: true, comments: false, thumbnail: true, refreshVideo: false } |
92 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam }) | 92 | const { video } = await getOrCreateAPVideo({ videoObject: url, syncParam }) |
93 | 93 | ||
94 | if (video.isOwned() && !video.hasPrivacyForFederation()) { | 94 | if (video.isOwned() && !video.hasPrivacyForFederation()) { |
95 | throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation') | 95 | throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation') |
diff --git a/server/lib/activitypub/videos/fetch.ts b/server/lib/activitypub/videos/fetch.ts deleted file mode 100644 index 5113c9d7e..000000000 --- a/server/lib/activitypub/videos/fetch.ts +++ /dev/null | |||
@@ -1,180 +0,0 @@ | |||
1 | import { checkUrlsSameHost, getAPId } from '@server/helpers/activitypub' | ||
2 | import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' | ||
3 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
4 | import { logger } from '@server/helpers/logger' | ||
5 | import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests' | ||
6 | import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video' | ||
7 | import { REMOTE_SCHEME } from '@server/initializers/constants' | ||
8 | import { ActorFollowScoreCache } from '@server/lib/files-cache' | ||
9 | import { JobQueue } from '@server/lib/job-queue' | ||
10 | import { VideoModel } from '@server/models/video/video' | ||
11 | import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' | ||
12 | import { HttpStatusCode } from '@shared/core-utils' | ||
13 | import { VideoObject } from '@shared/models' | ||
14 | import { APVideoCreator, SyncParam, syncVideoExternalAttributes } from './shared' | ||
15 | import { APVideoUpdater } from './updater' | ||
16 | |||
17 | async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { | ||
18 | logger.info('Fetching remote video %s.', videoUrl) | ||
19 | |||
20 | const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true }) | ||
21 | |||
22 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { | ||
23 | logger.debug('Remote video JSON is not valid.', { body }) | ||
24 | return { statusCode, videoObject: undefined } | ||
25 | } | ||
26 | |||
27 | return { statusCode, videoObject: body } | ||
28 | } | ||
29 | |||
30 | async function fetchRemoteVideoDescription (video: MVideoAccountLight) { | ||
31 | const host = video.VideoChannel.Account.Actor.Server.host | ||
32 | const path = video.getDescriptionAPIPath() | ||
33 | const url = REMOTE_SCHEME.HTTP + '://' + host + path | ||
34 | |||
35 | const { body } = await doJSONRequest<any>(url) | ||
36 | return body.description || '' | ||
37 | } | ||
38 | |||
39 | type GetVideoResult <T> = Promise<{ | ||
40 | video: T | ||
41 | created: boolean | ||
42 | autoBlacklisted?: boolean | ||
43 | }> | ||
44 | |||
45 | type GetVideoParamAll = { | ||
46 | videoObject: { id: string } | string | ||
47 | syncParam?: SyncParam | ||
48 | fetchType?: 'all' | ||
49 | allowRefresh?: boolean | ||
50 | } | ||
51 | |||
52 | type GetVideoParamImmutable = { | ||
53 | videoObject: { id: string } | string | ||
54 | syncParam?: SyncParam | ||
55 | fetchType: 'only-immutable-attributes' | ||
56 | allowRefresh: false | ||
57 | } | ||
58 | |||
59 | type GetVideoParamOther = { | ||
60 | videoObject: { id: string } | string | ||
61 | syncParam?: SyncParam | ||
62 | fetchType?: 'all' | 'only-video' | ||
63 | allowRefresh?: boolean | ||
64 | } | ||
65 | |||
66 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles> | ||
67 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable> | ||
68 | function getOrCreateVideoAndAccountAndChannel ( | ||
69 | options: GetVideoParamOther | ||
70 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> | ||
71 | async function getOrCreateVideoAndAccountAndChannel ( | ||
72 | options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther | ||
73 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> { | ||
74 | // Default params | ||
75 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | ||
76 | const fetchType = options.fetchType || 'all' | ||
77 | const allowRefresh = options.allowRefresh !== false | ||
78 | |||
79 | // Get video url | ||
80 | const videoUrl = getAPId(options.videoObject) | ||
81 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | ||
82 | |||
83 | if (videoFromDatabase) { | ||
84 | // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type | ||
85 | if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) { | ||
86 | const refreshOptions = { | ||
87 | video: videoFromDatabase as MVideoThumbnail, | ||
88 | fetchedType: fetchType, | ||
89 | syncParam | ||
90 | } | ||
91 | |||
92 | if (syncParam.refreshVideo === true) { | ||
93 | videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) | ||
94 | } else { | ||
95 | await JobQueue.Instance.createJobWithPromise({ | ||
96 | type: 'activitypub-refresher', | ||
97 | payload: { type: 'video', url: videoFromDatabase.url } | ||
98 | }) | ||
99 | } | ||
100 | } | ||
101 | |||
102 | return { video: videoFromDatabase, created: false } | ||
103 | } | ||
104 | |||
105 | const { videoObject } = await fetchRemoteVideo(videoUrl) | ||
106 | if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | ||
107 | |||
108 | try { | ||
109 | const creator = new APVideoCreator(videoObject) | ||
110 | const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail) | ||
111 | |||
112 | await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) | ||
113 | |||
114 | return { video: videoCreated, created: true, autoBlacklisted } | ||
115 | } catch (err) { | ||
116 | // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video | ||
117 | if (err.name === 'SequelizeUniqueConstraintError') { | ||
118 | const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType) | ||
119 | if (fallbackVideo) return { video: fallbackVideo, created: false } | ||
120 | } | ||
121 | |||
122 | throw err | ||
123 | } | ||
124 | } | ||
125 | |||
126 | async function refreshVideoIfNeeded (options: { | ||
127 | video: MVideoThumbnail | ||
128 | fetchedType: VideoFetchByUrlType | ||
129 | syncParam: SyncParam | ||
130 | }): Promise<MVideoThumbnail> { | ||
131 | if (!options.video.isOutdated()) return options.video | ||
132 | |||
133 | // We need more attributes if the argument video was fetched with not enough joints | ||
134 | const video = options.fetchedType === 'all' | ||
135 | ? options.video as MVideoAccountLightBlacklistAllFiles | ||
136 | : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | ||
137 | |||
138 | try { | ||
139 | const { videoObject } = await fetchRemoteVideo(video.url) | ||
140 | |||
141 | if (videoObject === undefined) { | ||
142 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url) | ||
143 | |||
144 | await video.setAsRefreshed() | ||
145 | return video | ||
146 | } | ||
147 | |||
148 | const videoUpdater = new APVideoUpdater(videoObject, video) | ||
149 | await videoUpdater.update() | ||
150 | |||
151 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | ||
152 | |||
153 | ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) | ||
154 | |||
155 | return video | ||
156 | } catch (err) { | ||
157 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
158 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) | ||
159 | |||
160 | // Video does not exist anymore | ||
161 | await video.destroy() | ||
162 | return undefined | ||
163 | } | ||
164 | |||
165 | logger.warn('Cannot refresh video %s.', options.video.url, { err }) | ||
166 | |||
167 | ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) | ||
168 | |||
169 | // Don't refresh in loop | ||
170 | await video.setAsRefreshed() | ||
171 | return video | ||
172 | } | ||
173 | } | ||
174 | |||
175 | export { | ||
176 | fetchRemoteVideo, | ||
177 | fetchRemoteVideoDescription, | ||
178 | refreshVideoIfNeeded, | ||
179 | getOrCreateVideoAndAccountAndChannel | ||
180 | } | ||
diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts new file mode 100644 index 000000000..a8c41e178 --- /dev/null +++ b/server/lib/activitypub/videos/get.ts | |||
@@ -0,0 +1,109 @@ | |||
1 | import { getAPId } from '@server/helpers/activitypub' | ||
2 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
3 | import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video' | ||
4 | import { JobQueue } from '@server/lib/job-queue' | ||
5 | import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' | ||
6 | import { refreshVideoIfNeeded } from './refresh' | ||
7 | import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' | ||
8 | |||
9 | type GetVideoResult <T> = Promise<{ | ||
10 | video: T | ||
11 | created: boolean | ||
12 | autoBlacklisted?: boolean | ||
13 | }> | ||
14 | |||
15 | type GetVideoParamAll = { | ||
16 | videoObject: { id: string } | string | ||
17 | syncParam?: SyncParam | ||
18 | fetchType?: 'all' | ||
19 | allowRefresh?: boolean | ||
20 | } | ||
21 | |||
22 | type GetVideoParamImmutable = { | ||
23 | videoObject: { id: string } | string | ||
24 | syncParam?: SyncParam | ||
25 | fetchType: 'only-immutable-attributes' | ||
26 | allowRefresh: false | ||
27 | } | ||
28 | |||
29 | type GetVideoParamOther = { | ||
30 | videoObject: { id: string } | string | ||
31 | syncParam?: SyncParam | ||
32 | fetchType?: 'all' | 'only-video' | ||
33 | allowRefresh?: boolean | ||
34 | } | ||
35 | |||
36 | function getOrCreateAPVideo (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles> | ||
37 | function getOrCreateAPVideo (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable> | ||
38 | function getOrCreateAPVideo (options: GetVideoParamOther): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> | ||
39 | |||
40 | async function getOrCreateAPVideo ( | ||
41 | options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther | ||
42 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> { | ||
43 | // Default params | ||
44 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | ||
45 | const fetchType = options.fetchType || 'all' | ||
46 | const allowRefresh = options.allowRefresh !== false | ||
47 | |||
48 | // Get video url | ||
49 | const videoUrl = getAPId(options.videoObject) | ||
50 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | ||
51 | |||
52 | if (videoFromDatabase) { | ||
53 | if (allowRefresh === true) { | ||
54 | // Typings ensure allowRefresh === false in only-immutable-attributes fetch type | ||
55 | videoFromDatabase = await scheduleRefresh(videoFromDatabase as MVideoThumbnail, fetchType, syncParam) | ||
56 | } | ||
57 | |||
58 | return { video: videoFromDatabase, created: false } | ||
59 | } | ||
60 | |||
61 | const { videoObject } = await fetchRemoteVideo(videoUrl) | ||
62 | if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | ||
63 | |||
64 | try { | ||
65 | const creator = new APVideoCreator(videoObject) | ||
66 | const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail) | ||
67 | |||
68 | await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) | ||
69 | |||
70 | return { video: videoCreated, created: true, autoBlacklisted } | ||
71 | } catch (err) { | ||
72 | // Maybe a concurrent getOrCreateAPVideo call created this video | ||
73 | if (err.name === 'SequelizeUniqueConstraintError') { | ||
74 | const alreadyCreatedVideo = await fetchVideoByUrl(videoUrl, fetchType) | ||
75 | if (alreadyCreatedVideo) return { video: alreadyCreatedVideo, created: false } | ||
76 | } | ||
77 | |||
78 | throw err | ||
79 | } | ||
80 | } | ||
81 | |||
82 | // --------------------------------------------------------------------------- | ||
83 | |||
84 | export { | ||
85 | getOrCreateAPVideo | ||
86 | } | ||
87 | |||
88 | // --------------------------------------------------------------------------- | ||
89 | |||
90 | async function scheduleRefresh (video: MVideoThumbnail, fetchType: VideoFetchByUrlType, syncParam: SyncParam) { | ||
91 | if (!video.isOutdated()) return video | ||
92 | |||
93 | const refreshOptions = { | ||
94 | video, | ||
95 | fetchedType: fetchType, | ||
96 | syncParam | ||
97 | } | ||
98 | |||
99 | if (syncParam.refreshVideo === true) { | ||
100 | return refreshVideoIfNeeded(refreshOptions) | ||
101 | } | ||
102 | |||
103 | await JobQueue.Instance.createJobWithPromise({ | ||
104 | type: 'activitypub-refresher', | ||
105 | payload: { type: 'video', url: video.url } | ||
106 | }) | ||
107 | |||
108 | return video | ||
109 | } | ||
diff --git a/server/lib/activitypub/videos/index.ts b/server/lib/activitypub/videos/index.ts index b560acb76..b22062598 100644 --- a/server/lib/activitypub/videos/index.ts +++ b/server/lib/activitypub/videos/index.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | export * from './federate' | 1 | export * from './federate' |
2 | export * from './fetch' | 2 | export * from './get' |
3 | export * from './refresh' | ||
3 | export * from './updater' | 4 | export * from './updater' |
diff --git a/server/lib/activitypub/videos/refresh.ts b/server/lib/activitypub/videos/refresh.ts new file mode 100644 index 000000000..205a3ccb1 --- /dev/null +++ b/server/lib/activitypub/videos/refresh.ts | |||
@@ -0,0 +1,64 @@ | |||
1 | import { logger } from '@server/helpers/logger' | ||
2 | import { PeerTubeRequestError } from '@server/helpers/requests' | ||
3 | import { VideoFetchByUrlType } from '@server/helpers/video' | ||
4 | import { ActorFollowScoreCache } from '@server/lib/files-cache' | ||
5 | import { VideoModel } from '@server/models/video/video' | ||
6 | import { MVideoAccountLightBlacklistAllFiles, MVideoThumbnail } from '@server/types/models' | ||
7 | import { HttpStatusCode } from '@shared/core-utils' | ||
8 | import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' | ||
9 | import { APVideoUpdater } from './updater' | ||
10 | |||
11 | async function refreshVideoIfNeeded (options: { | ||
12 | video: MVideoThumbnail | ||
13 | fetchedType: VideoFetchByUrlType | ||
14 | syncParam: SyncParam | ||
15 | }): Promise<MVideoThumbnail> { | ||
16 | if (!options.video.isOutdated()) return options.video | ||
17 | |||
18 | // We need more attributes if the argument video was fetched with not enough joints | ||
19 | const video = options.fetchedType === 'all' | ||
20 | ? options.video as MVideoAccountLightBlacklistAllFiles | ||
21 | : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | ||
22 | |||
23 | try { | ||
24 | const { videoObject } = await fetchRemoteVideo(video.url) | ||
25 | |||
26 | if (videoObject === undefined) { | ||
27 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url) | ||
28 | |||
29 | await video.setAsRefreshed() | ||
30 | return video | ||
31 | } | ||
32 | |||
33 | const videoUpdater = new APVideoUpdater(videoObject, video) | ||
34 | await videoUpdater.update() | ||
35 | |||
36 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | ||
37 | |||
38 | ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) | ||
39 | |||
40 | return video | ||
41 | } catch (err) { | ||
42 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
43 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) | ||
44 | |||
45 | // Video does not exist anymore | ||
46 | await video.destroy() | ||
47 | return undefined | ||
48 | } | ||
49 | |||
50 | logger.warn('Cannot refresh video %s.', options.video.url, { err }) | ||
51 | |||
52 | ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) | ||
53 | |||
54 | // Don't refresh in loop | ||
55 | await video.setAsRefreshed() | ||
56 | return video | ||
57 | } | ||
58 | } | ||
59 | |||
60 | // --------------------------------------------------------------------------- | ||
61 | |||
62 | export { | ||
63 | refreshVideoIfNeeded | ||
64 | } | ||
diff --git a/server/lib/activitypub/videos/shared/index.ts b/server/lib/activitypub/videos/shared/index.ts index 208a43705..951403493 100644 --- a/server/lib/activitypub/videos/shared/index.ts +++ b/server/lib/activitypub/videos/shared/index.ts | |||
@@ -2,4 +2,5 @@ export * from './abstract-builder' | |||
2 | export * from './creator' | 2 | export * from './creator' |
3 | export * from './object-to-model-attributes' | 3 | export * from './object-to-model-attributes' |
4 | export * from './trackers' | 4 | export * from './trackers' |
5 | export * from './url-to-object' | ||
5 | export * from './video-sync-attributes' | 6 | export * from './video-sync-attributes' |
diff --git a/server/lib/activitypub/videos/shared/url-to-object.ts b/server/lib/activitypub/videos/shared/url-to-object.ts new file mode 100644 index 000000000..b1ecac8ca --- /dev/null +++ b/server/lib/activitypub/videos/shared/url-to-object.ts | |||
@@ -0,0 +1,22 @@ | |||
1 | import { checkUrlsSameHost } from '@server/helpers/activitypub' | ||
2 | import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' | ||
3 | import { logger } from '@server/helpers/logger' | ||
4 | import { doJSONRequest } from '@server/helpers/requests' | ||
5 | import { VideoObject } from '@shared/models' | ||
6 | |||
7 | async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { | ||
8 | logger.info('Fetching remote video %s.', videoUrl) | ||
9 | |||
10 | const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true }) | ||
11 | |||
12 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { | ||
13 | logger.debug('Remote video JSON is not valid.', { body }) | ||
14 | return { statusCode, videoObject: undefined } | ||
15 | } | ||
16 | |||
17 | return { statusCode, videoObject: body } | ||
18 | } | ||
19 | |||
20 | export { | ||
21 | fetchRemoteVideo | ||
22 | } | ||
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 59b55cccc..b5a5eb697 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -23,7 +23,7 @@ import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../. | |||
23 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 23 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
24 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | 24 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' |
25 | import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' | 25 | import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' |
26 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub/videos' | 26 | import { getOrCreateAPVideo } from '../activitypub/videos' |
27 | import { downloadPlaylistSegments } from '../hls' | 27 | import { downloadPlaylistSegments } from '../hls' |
28 | import { removeVideoRedundancy } from '../redundancy' | 28 | import { removeVideoRedundancy } from '../redundancy' |
29 | import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' | 29 | import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' |
@@ -351,7 +351,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
351 | syncParam: { likes: false, dislikes: false, shares: false, comments: false, thumbnail: false, refreshVideo: true }, | 351 | syncParam: { likes: false, dislikes: false, shares: false, comments: false, thumbnail: false, refreshVideo: true }, |
352 | fetchType: 'all' as 'all' | 352 | fetchType: 'all' as 'all' |
353 | } | 353 | } |
354 | const { video } = await getOrCreateVideoAndAccountAndChannel(getVideoOptions) | 354 | const { video } = await getOrCreateAPVideo(getVideoOptions) |
355 | 355 | ||
356 | return video | 356 | return video |
357 | } | 357 | } |