diff options
author | Chocobozzz <me@florianbigard.com> | 2021-06-02 09:35:01 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2021-06-02 16:57:53 +0200 |
commit | 69290ab37b8aead01477b9b98fdfad0e69b08582 (patch) | |
tree | 4b93d349dfce5014925e7f060b3b158ac9b3bbc2 /server/lib/activitypub/videos/fetch.ts | |
parent | 81628e5069e0168b11857f276fe8e03b93102dde (diff) | |
download | PeerTube-69290ab37b8aead01477b9b98fdfad0e69b08582.tar.gz PeerTube-69290ab37b8aead01477b9b98fdfad0e69b08582.tar.zst PeerTube-69290ab37b8aead01477b9b98fdfad0e69b08582.zip |
Refactor AP video update
Diffstat (limited to 'server/lib/activitypub/videos/fetch.ts')
-rw-r--r-- | server/lib/activitypub/videos/fetch.ts | 202 |
1 files changed, 202 insertions, 0 deletions
diff --git a/server/lib/activitypub/videos/fetch.ts b/server/lib/activitypub/videos/fetch.ts new file mode 100644 index 000000000..fdcf4ee5c --- /dev/null +++ b/server/lib/activitypub/videos/fetch.ts | |||
@@ -0,0 +1,202 @@ | |||
1 | import { checkUrlsSameHost, getAPId } from "@server/helpers/activitypub" | ||
2 | import { sanitizeAndCheckVideoTorrentObject } from "@server/helpers/custom-validators/activitypub/videos" | ||
3 | import { retryTransactionWrapper } from "@server/helpers/database-utils" | ||
4 | import { logger } from "@server/helpers/logger" | ||
5 | import { doJSONRequest, PeerTubeRequestError } from "@server/helpers/requests" | ||
6 | import { fetchVideoByUrl, VideoFetchByUrlType } from "@server/helpers/video" | ||
7 | import { REMOTE_SCHEME } from "@server/initializers/constants" | ||
8 | import { ActorFollowScoreCache } from "@server/lib/files-cache" | ||
9 | import { JobQueue } from "@server/lib/job-queue" | ||
10 | import { VideoModel } from "@server/models/video/video" | ||
11 | import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from "@server/types/models" | ||
12 | import { HttpStatusCode } from "@shared/core-utils" | ||
13 | import { VideoObject } from "@shared/models" | ||
14 | import { getOrCreateActorAndServerAndModel } from "../actor" | ||
15 | import { SyncParam, syncVideoExternalAttributes } from "./shared" | ||
16 | import { createVideo } from "./shared/video-create" | ||
17 | import { APVideoUpdater } from "./update" | ||
18 | |||
19 | async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { | ||
20 | logger.info('Fetching remote video %s.', videoUrl) | ||
21 | |||
22 | const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true }) | ||
23 | |||
24 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { | ||
25 | logger.debug('Remote video JSON is not valid.', { body }) | ||
26 | return { statusCode, videoObject: undefined } | ||
27 | } | ||
28 | |||
29 | return { statusCode, videoObject: body } | ||
30 | } | ||
31 | |||
32 | async function fetchRemoteVideoDescription (video: MVideoAccountLight) { | ||
33 | const host = video.VideoChannel.Account.Actor.Server.host | ||
34 | const path = video.getDescriptionAPIPath() | ||
35 | const url = REMOTE_SCHEME.HTTP + '://' + host + path | ||
36 | |||
37 | const { body } = await doJSONRequest<any>(url) | ||
38 | return body.description || '' | ||
39 | } | ||
40 | |||
41 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) { | ||
42 | const channel = videoObject.attributedTo.find(a => a.type === 'Group') | ||
43 | if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) | ||
44 | |||
45 | if (checkUrlsSameHost(channel.id, videoObject.id) !== true) { | ||
46 | throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`) | ||
47 | } | ||
48 | |||
49 | return getOrCreateActorAndServerAndModel(channel.id, 'all') | ||
50 | } | ||
51 | |||
52 | type GetVideoResult <T> = Promise<{ | ||
53 | video: T | ||
54 | created: boolean | ||
55 | autoBlacklisted?: boolean | ||
56 | }> | ||
57 | |||
58 | type GetVideoParamAll = { | ||
59 | videoObject: { id: string } | string | ||
60 | syncParam?: SyncParam | ||
61 | fetchType?: 'all' | ||
62 | allowRefresh?: boolean | ||
63 | } | ||
64 | |||
65 | type GetVideoParamImmutable = { | ||
66 | videoObject: { id: string } | string | ||
67 | syncParam?: SyncParam | ||
68 | fetchType: 'only-immutable-attributes' | ||
69 | allowRefresh: false | ||
70 | } | ||
71 | |||
72 | type GetVideoParamOther = { | ||
73 | videoObject: { id: string } | string | ||
74 | syncParam?: SyncParam | ||
75 | fetchType?: 'all' | 'only-video' | ||
76 | allowRefresh?: boolean | ||
77 | } | ||
78 | |||
79 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles> | ||
80 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable> | ||
81 | function getOrCreateVideoAndAccountAndChannel ( | ||
82 | options: GetVideoParamOther | ||
83 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> | ||
84 | async function getOrCreateVideoAndAccountAndChannel ( | ||
85 | options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther | ||
86 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> { | ||
87 | // Default params | ||
88 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | ||
89 | const fetchType = options.fetchType || 'all' | ||
90 | const allowRefresh = options.allowRefresh !== false | ||
91 | |||
92 | // Get video url | ||
93 | const videoUrl = getAPId(options.videoObject) | ||
94 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | ||
95 | |||
96 | if (videoFromDatabase) { | ||
97 | // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type | ||
98 | if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) { | ||
99 | const refreshOptions = { | ||
100 | video: videoFromDatabase as MVideoThumbnail, | ||
101 | fetchedType: fetchType, | ||
102 | syncParam | ||
103 | } | ||
104 | |||
105 | if (syncParam.refreshVideo === true) { | ||
106 | videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) | ||
107 | } else { | ||
108 | await JobQueue.Instance.createJobWithPromise({ | ||
109 | type: 'activitypub-refresher', | ||
110 | payload: { type: 'video', url: videoFromDatabase.url } | ||
111 | }) | ||
112 | } | ||
113 | } | ||
114 | |||
115 | return { video: videoFromDatabase, created: false } | ||
116 | } | ||
117 | |||
118 | const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) | ||
119 | if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | ||
120 | |||
121 | const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) | ||
122 | const videoChannel = actor.VideoChannel | ||
123 | |||
124 | try { | ||
125 | const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail) | ||
126 | |||
127 | await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam) | ||
128 | |||
129 | return { video: videoCreated, created: true, autoBlacklisted } | ||
130 | } catch (err) { | ||
131 | // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video | ||
132 | if (err.name === 'SequelizeUniqueConstraintError') { | ||
133 | const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType) | ||
134 | if (fallbackVideo) return { video: fallbackVideo, created: false } | ||
135 | } | ||
136 | |||
137 | throw err | ||
138 | } | ||
139 | } | ||
140 | |||
141 | async function refreshVideoIfNeeded (options: { | ||
142 | video: MVideoThumbnail | ||
143 | fetchedType: VideoFetchByUrlType | ||
144 | syncParam: SyncParam | ||
145 | }): Promise<MVideoThumbnail> { | ||
146 | if (!options.video.isOutdated()) return options.video | ||
147 | |||
148 | // We need more attributes if the argument video was fetched with not enough joints | ||
149 | const video = options.fetchedType === 'all' | ||
150 | ? options.video as MVideoAccountLightBlacklistAllFiles | ||
151 | : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | ||
152 | |||
153 | try { | ||
154 | const { videoObject } = await fetchRemoteVideo(video.url) | ||
155 | |||
156 | if (videoObject === undefined) { | ||
157 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url) | ||
158 | |||
159 | await video.setAsRefreshed() | ||
160 | return video | ||
161 | } | ||
162 | |||
163 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | ||
164 | |||
165 | const videoUpdater = new APVideoUpdater({ | ||
166 | video, | ||
167 | videoObject, | ||
168 | channel: channelActor.VideoChannel | ||
169 | }) | ||
170 | await videoUpdater.update() | ||
171 | |||
172 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | ||
173 | |||
174 | ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) | ||
175 | |||
176 | return video | ||
177 | } catch (err) { | ||
178 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
179 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) | ||
180 | |||
181 | // Video does not exist anymore | ||
182 | await video.destroy() | ||
183 | return undefined | ||
184 | } | ||
185 | |||
186 | logger.warn('Cannot refresh video %s.', options.video.url, { err }) | ||
187 | |||
188 | ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) | ||
189 | |||
190 | // Don't refresh in loop | ||
191 | await video.setAsRefreshed() | ||
192 | return video | ||
193 | } | ||
194 | } | ||
195 | |||
196 | export { | ||
197 | fetchRemoteVideo, | ||
198 | fetchRemoteVideoDescription, | ||
199 | refreshVideoIfNeeded, | ||
200 | getOrCreateVideoChannelFromVideoObject, | ||
201 | getOrCreateVideoAndAccountAndChannel | ||
202 | } | ||