]>
Commit | Line | Data |
---|---|---|
69290ab3 C |
1 | import { checkUrlsSameHost, getAPId } from "@server/helpers/activitypub" |
2 | import { sanitizeAndCheckVideoTorrentObject } from "@server/helpers/custom-validators/activitypub/videos" | |
3 | import { retryTransactionWrapper } from "@server/helpers/database-utils" | |
4 | import { logger } from "@server/helpers/logger" | |
5 | import { doJSONRequest, PeerTubeRequestError } from "@server/helpers/requests" | |
6 | import { fetchVideoByUrl, VideoFetchByUrlType } from "@server/helpers/video" | |
7 | import { REMOTE_SCHEME } from "@server/initializers/constants" | |
8 | import { ActorFollowScoreCache } from "@server/lib/files-cache" | |
9 | import { JobQueue } from "@server/lib/job-queue" | |
10 | import { VideoModel } from "@server/models/video/video" | |
11 | import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from "@server/types/models" | |
12 | import { HttpStatusCode } from "@shared/core-utils" | |
13 | import { VideoObject } from "@shared/models" | |
14 | import { getOrCreateActorAndServerAndModel } from "../actor" | |
15 | import { SyncParam, syncVideoExternalAttributes } from "./shared" | |
16 | import { createVideo } from "./shared/video-create" | |
17 | import { APVideoUpdater } from "./update" | |
18 | ||
19 | async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { | |
20 | logger.info('Fetching remote video %s.', videoUrl) | |
21 | ||
22 | const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true }) | |
23 | ||
24 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { | |
25 | logger.debug('Remote video JSON is not valid.', { body }) | |
26 | return { statusCode, videoObject: undefined } | |
27 | } | |
28 | ||
29 | return { statusCode, videoObject: body } | |
30 | } | |
31 | ||
32 | async function fetchRemoteVideoDescription (video: MVideoAccountLight) { | |
33 | const host = video.VideoChannel.Account.Actor.Server.host | |
34 | const path = video.getDescriptionAPIPath() | |
35 | const url = REMOTE_SCHEME.HTTP + '://' + host + path | |
36 | ||
37 | const { body } = await doJSONRequest<any>(url) | |
38 | return body.description || '' | |
39 | } | |
40 | ||
41 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) { | |
42 | const channel = videoObject.attributedTo.find(a => a.type === 'Group') | |
43 | if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) | |
44 | ||
45 | if (checkUrlsSameHost(channel.id, videoObject.id) !== true) { | |
46 | throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`) | |
47 | } | |
48 | ||
49 | return getOrCreateActorAndServerAndModel(channel.id, 'all') | |
50 | } | |
51 | ||
52 | type GetVideoResult <T> = Promise<{ | |
53 | video: T | |
54 | created: boolean | |
55 | autoBlacklisted?: boolean | |
56 | }> | |
57 | ||
58 | type GetVideoParamAll = { | |
59 | videoObject: { id: string } | string | |
60 | syncParam?: SyncParam | |
61 | fetchType?: 'all' | |
62 | allowRefresh?: boolean | |
63 | } | |
64 | ||
65 | type GetVideoParamImmutable = { | |
66 | videoObject: { id: string } | string | |
67 | syncParam?: SyncParam | |
68 | fetchType: 'only-immutable-attributes' | |
69 | allowRefresh: false | |
70 | } | |
71 | ||
72 | type GetVideoParamOther = { | |
73 | videoObject: { id: string } | string | |
74 | syncParam?: SyncParam | |
75 | fetchType?: 'all' | 'only-video' | |
76 | allowRefresh?: boolean | |
77 | } | |
78 | ||
79 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles> | |
80 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable> | |
81 | function getOrCreateVideoAndAccountAndChannel ( | |
82 | options: GetVideoParamOther | |
83 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> | |
84 | async function getOrCreateVideoAndAccountAndChannel ( | |
85 | options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther | |
86 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> { | |
87 | // Default params | |
88 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | |
89 | const fetchType = options.fetchType || 'all' | |
90 | const allowRefresh = options.allowRefresh !== false | |
91 | ||
92 | // Get video url | |
93 | const videoUrl = getAPId(options.videoObject) | |
94 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | |
95 | ||
96 | if (videoFromDatabase) { | |
97 | // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type | |
98 | if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) { | |
99 | const refreshOptions = { | |
100 | video: videoFromDatabase as MVideoThumbnail, | |
101 | fetchedType: fetchType, | |
102 | syncParam | |
103 | } | |
104 | ||
105 | if (syncParam.refreshVideo === true) { | |
106 | videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) | |
107 | } else { | |
108 | await JobQueue.Instance.createJobWithPromise({ | |
109 | type: 'activitypub-refresher', | |
110 | payload: { type: 'video', url: videoFromDatabase.url } | |
111 | }) | |
112 | } | |
113 | } | |
114 | ||
115 | return { video: videoFromDatabase, created: false } | |
116 | } | |
117 | ||
118 | const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) | |
119 | if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | |
120 | ||
121 | const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) | |
122 | const videoChannel = actor.VideoChannel | |
123 | ||
124 | try { | |
125 | const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail) | |
126 | ||
127 | await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam) | |
128 | ||
129 | return { video: videoCreated, created: true, autoBlacklisted } | |
130 | } catch (err) { | |
131 | // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video | |
132 | if (err.name === 'SequelizeUniqueConstraintError') { | |
133 | const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType) | |
134 | if (fallbackVideo) return { video: fallbackVideo, created: false } | |
135 | } | |
136 | ||
137 | throw err | |
138 | } | |
139 | } | |
140 | ||
141 | async function refreshVideoIfNeeded (options: { | |
142 | video: MVideoThumbnail | |
143 | fetchedType: VideoFetchByUrlType | |
144 | syncParam: SyncParam | |
145 | }): Promise<MVideoThumbnail> { | |
146 | if (!options.video.isOutdated()) return options.video | |
147 | ||
148 | // We need more attributes if the argument video was fetched with not enough joints | |
149 | const video = options.fetchedType === 'all' | |
150 | ? options.video as MVideoAccountLightBlacklistAllFiles | |
151 | : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | |
152 | ||
153 | try { | |
154 | const { videoObject } = await fetchRemoteVideo(video.url) | |
155 | ||
156 | if (videoObject === undefined) { | |
157 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url) | |
158 | ||
159 | await video.setAsRefreshed() | |
160 | return video | |
161 | } | |
162 | ||
163 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | |
164 | ||
165 | const videoUpdater = new APVideoUpdater({ | |
166 | video, | |
167 | videoObject, | |
168 | channel: channelActor.VideoChannel | |
169 | }) | |
170 | await videoUpdater.update() | |
171 | ||
172 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | |
173 | ||
174 | ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) | |
175 | ||
176 | return video | |
177 | } catch (err) { | |
178 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | |
179 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) | |
180 | ||
181 | // Video does not exist anymore | |
182 | await video.destroy() | |
183 | return undefined | |
184 | } | |
185 | ||
186 | logger.warn('Cannot refresh video %s.', options.video.url, { err }) | |
187 | ||
188 | ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) | |
189 | ||
190 | // Don't refresh in loop | |
191 | await video.setAsRefreshed() | |
192 | return video | |
193 | } | |
194 | } | |
195 | ||
196 | export { | |
197 | fetchRemoteVideo, | |
198 | fetchRemoteVideoDescription, | |
199 | refreshVideoIfNeeded, | |
200 | getOrCreateVideoChannelFromVideoObject, | |
201 | getOrCreateVideoAndAccountAndChannel | |
202 | } |