]>
Commit | Line | Data |
---|---|---|
1 | import { checkUrlsSameHost, getAPId } from '@server/helpers/activitypub' | |
2 | import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' | |
3 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | |
4 | import { logger } from '@server/helpers/logger' | |
5 | import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests' | |
6 | import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video' | |
7 | import { REMOTE_SCHEME } from '@server/initializers/constants' | |
8 | import { ActorFollowScoreCache } from '@server/lib/files-cache' | |
9 | import { JobQueue } from '@server/lib/job-queue' | |
10 | import { VideoModel } from '@server/models/video/video' | |
11 | import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' | |
12 | import { HttpStatusCode } from '@shared/core-utils' | |
13 | import { VideoObject } from '@shared/models' | |
14 | import { getOrCreateActorAndServerAndModel } from '../actor' | |
15 | import { APVideoCreator, SyncParam, syncVideoExternalAttributes } from './shared' | |
16 | import { APVideoUpdater } from './updater' | |
17 | ||
18 | async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { | |
19 | logger.info('Fetching remote video %s.', videoUrl) | |
20 | ||
21 | const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true }) | |
22 | ||
23 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { | |
24 | logger.debug('Remote video JSON is not valid.', { body }) | |
25 | return { statusCode, videoObject: undefined } | |
26 | } | |
27 | ||
28 | return { statusCode, videoObject: body } | |
29 | } | |
30 | ||
31 | async function fetchRemoteVideoDescription (video: MVideoAccountLight) { | |
32 | const host = video.VideoChannel.Account.Actor.Server.host | |
33 | const path = video.getDescriptionAPIPath() | |
34 | const url = REMOTE_SCHEME.HTTP + '://' + host + path | |
35 | ||
36 | const { body } = await doJSONRequest<any>(url) | |
37 | return body.description || '' | |
38 | } | |
39 | ||
40 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) { | |
41 | const channel = videoObject.attributedTo.find(a => a.type === 'Group') | |
42 | if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) | |
43 | ||
44 | if (checkUrlsSameHost(channel.id, videoObject.id) !== true) { | |
45 | throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`) | |
46 | } | |
47 | ||
48 | return getOrCreateActorAndServerAndModel(channel.id, 'all') | |
49 | } | |
50 | ||
51 | type GetVideoResult <T> = Promise<{ | |
52 | video: T | |
53 | created: boolean | |
54 | autoBlacklisted?: boolean | |
55 | }> | |
56 | ||
57 | type GetVideoParamAll = { | |
58 | videoObject: { id: string } | string | |
59 | syncParam?: SyncParam | |
60 | fetchType?: 'all' | |
61 | allowRefresh?: boolean | |
62 | } | |
63 | ||
64 | type GetVideoParamImmutable = { | |
65 | videoObject: { id: string } | string | |
66 | syncParam?: SyncParam | |
67 | fetchType: 'only-immutable-attributes' | |
68 | allowRefresh: false | |
69 | } | |
70 | ||
71 | type GetVideoParamOther = { | |
72 | videoObject: { id: string } | string | |
73 | syncParam?: SyncParam | |
74 | fetchType?: 'all' | 'only-video' | |
75 | allowRefresh?: boolean | |
76 | } | |
77 | ||
78 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles> | |
79 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable> | |
80 | function getOrCreateVideoAndAccountAndChannel ( | |
81 | options: GetVideoParamOther | |
82 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> | |
83 | async function getOrCreateVideoAndAccountAndChannel ( | |
84 | options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther | |
85 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> { | |
86 | // Default params | |
87 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | |
88 | const fetchType = options.fetchType || 'all' | |
89 | const allowRefresh = options.allowRefresh !== false | |
90 | ||
91 | // Get video url | |
92 | const videoUrl = getAPId(options.videoObject) | |
93 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | |
94 | ||
95 | if (videoFromDatabase) { | |
96 | // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type | |
97 | if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) { | |
98 | const refreshOptions = { | |
99 | video: videoFromDatabase as MVideoThumbnail, | |
100 | fetchedType: fetchType, | |
101 | syncParam | |
102 | } | |
103 | ||
104 | if (syncParam.refreshVideo === true) { | |
105 | videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) | |
106 | } else { | |
107 | await JobQueue.Instance.createJobWithPromise({ | |
108 | type: 'activitypub-refresher', | |
109 | payload: { type: 'video', url: videoFromDatabase.url } | |
110 | }) | |
111 | } | |
112 | } | |
113 | ||
114 | return { video: videoFromDatabase, created: false } | |
115 | } | |
116 | ||
117 | const { videoObject } = await fetchRemoteVideo(videoUrl) | |
118 | if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | |
119 | ||
120 | const actor = await getOrCreateVideoChannelFromVideoObject(videoObject) | |
121 | const videoChannel = actor.VideoChannel | |
122 | ||
123 | try { | |
124 | const creator = new APVideoCreator({ videoObject, channel: videoChannel }) | |
125 | const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail) | |
126 | ||
127 | await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) | |
128 | ||
129 | return { video: videoCreated, created: true, autoBlacklisted } | |
130 | } catch (err) { | |
131 | // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video | |
132 | if (err.name === 'SequelizeUniqueConstraintError') { | |
133 | const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType) | |
134 | if (fallbackVideo) return { video: fallbackVideo, created: false } | |
135 | } | |
136 | ||
137 | throw err | |
138 | } | |
139 | } | |
140 | ||
141 | async function refreshVideoIfNeeded (options: { | |
142 | video: MVideoThumbnail | |
143 | fetchedType: VideoFetchByUrlType | |
144 | syncParam: SyncParam | |
145 | }): Promise<MVideoThumbnail> { | |
146 | if (!options.video.isOutdated()) return options.video | |
147 | ||
148 | // We need more attributes if the argument video was fetched with not enough joints | |
149 | const video = options.fetchedType === 'all' | |
150 | ? options.video as MVideoAccountLightBlacklistAllFiles | |
151 | : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | |
152 | ||
153 | try { | |
154 | const { videoObject } = await fetchRemoteVideo(video.url) | |
155 | ||
156 | if (videoObject === undefined) { | |
157 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url) | |
158 | ||
159 | await video.setAsRefreshed() | |
160 | return video | |
161 | } | |
162 | ||
163 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | |
164 | ||
165 | const videoUpdater = new APVideoUpdater({ | |
166 | video, | |
167 | videoObject, | |
168 | channel: channelActor.VideoChannel | |
169 | }) | |
170 | await videoUpdater.update() | |
171 | ||
172 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | |
173 | ||
174 | ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) | |
175 | ||
176 | return video | |
177 | } catch (err) { | |
178 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | |
179 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) | |
180 | ||
181 | // Video does not exist anymore | |
182 | await video.destroy() | |
183 | return undefined | |
184 | } | |
185 | ||
186 | logger.warn('Cannot refresh video %s.', options.video.url, { err }) | |
187 | ||
188 | ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) | |
189 | ||
190 | // Don't refresh in loop | |
191 | await video.setAsRefreshed() | |
192 | return video | |
193 | } | |
194 | } | |
195 | ||
196 | export { | |
197 | fetchRemoteVideo, | |
198 | fetchRemoteVideoDescription, | |
199 | refreshVideoIfNeeded, | |
200 | getOrCreateVideoChannelFromVideoObject, | |
201 | getOrCreateVideoAndAccountAndChannel | |
202 | } |