]>
Commit | Line | Data |
---|---|---|
1 | import { checkUrlsSameHost, getAPId } from '@server/helpers/activitypub' | |
2 | import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' | |
3 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | |
4 | import { logger } from '@server/helpers/logger' | |
5 | import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests' | |
6 | import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video' | |
7 | import { REMOTE_SCHEME } from '@server/initializers/constants' | |
8 | import { ActorFollowScoreCache } from '@server/lib/files-cache' | |
9 | import { JobQueue } from '@server/lib/job-queue' | |
10 | import { VideoModel } from '@server/models/video/video' | |
11 | import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' | |
12 | import { HttpStatusCode } from '@shared/core-utils' | |
13 | import { VideoObject } from '@shared/models' | |
14 | import { APVideoCreator, SyncParam, syncVideoExternalAttributes } from './shared' | |
15 | import { APVideoUpdater } from './updater' | |
16 | ||
17 | async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { | |
18 | logger.info('Fetching remote video %s.', videoUrl) | |
19 | ||
20 | const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true }) | |
21 | ||
22 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { | |
23 | logger.debug('Remote video JSON is not valid.', { body }) | |
24 | return { statusCode, videoObject: undefined } | |
25 | } | |
26 | ||
27 | return { statusCode, videoObject: body } | |
28 | } | |
29 | ||
30 | async function fetchRemoteVideoDescription (video: MVideoAccountLight) { | |
31 | const host = video.VideoChannel.Account.Actor.Server.host | |
32 | const path = video.getDescriptionAPIPath() | |
33 | const url = REMOTE_SCHEME.HTTP + '://' + host + path | |
34 | ||
35 | const { body } = await doJSONRequest<any>(url) | |
36 | return body.description || '' | |
37 | } | |
38 | ||
39 | type GetVideoResult <T> = Promise<{ | |
40 | video: T | |
41 | created: boolean | |
42 | autoBlacklisted?: boolean | |
43 | }> | |
44 | ||
45 | type GetVideoParamAll = { | |
46 | videoObject: { id: string } | string | |
47 | syncParam?: SyncParam | |
48 | fetchType?: 'all' | |
49 | allowRefresh?: boolean | |
50 | } | |
51 | ||
52 | type GetVideoParamImmutable = { | |
53 | videoObject: { id: string } | string | |
54 | syncParam?: SyncParam | |
55 | fetchType: 'only-immutable-attributes' | |
56 | allowRefresh: false | |
57 | } | |
58 | ||
59 | type GetVideoParamOther = { | |
60 | videoObject: { id: string } | string | |
61 | syncParam?: SyncParam | |
62 | fetchType?: 'all' | 'only-video' | |
63 | allowRefresh?: boolean | |
64 | } | |
65 | ||
66 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles> | |
67 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable> | |
68 | function getOrCreateVideoAndAccountAndChannel ( | |
69 | options: GetVideoParamOther | |
70 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> | |
71 | async function getOrCreateVideoAndAccountAndChannel ( | |
72 | options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther | |
73 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> { | |
74 | // Default params | |
75 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | |
76 | const fetchType = options.fetchType || 'all' | |
77 | const allowRefresh = options.allowRefresh !== false | |
78 | ||
79 | // Get video url | |
80 | const videoUrl = getAPId(options.videoObject) | |
81 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | |
82 | ||
83 | if (videoFromDatabase) { | |
84 | // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type | |
85 | if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) { | |
86 | const refreshOptions = { | |
87 | video: videoFromDatabase as MVideoThumbnail, | |
88 | fetchedType: fetchType, | |
89 | syncParam | |
90 | } | |
91 | ||
92 | if (syncParam.refreshVideo === true) { | |
93 | videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) | |
94 | } else { | |
95 | await JobQueue.Instance.createJobWithPromise({ | |
96 | type: 'activitypub-refresher', | |
97 | payload: { type: 'video', url: videoFromDatabase.url } | |
98 | }) | |
99 | } | |
100 | } | |
101 | ||
102 | return { video: videoFromDatabase, created: false } | |
103 | } | |
104 | ||
105 | const { videoObject } = await fetchRemoteVideo(videoUrl) | |
106 | if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | |
107 | ||
108 | try { | |
109 | const creator = new APVideoCreator(videoObject) | |
110 | const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail) | |
111 | ||
112 | await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) | |
113 | ||
114 | return { video: videoCreated, created: true, autoBlacklisted } | |
115 | } catch (err) { | |
116 | // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video | |
117 | if (err.name === 'SequelizeUniqueConstraintError') { | |
118 | const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType) | |
119 | if (fallbackVideo) return { video: fallbackVideo, created: false } | |
120 | } | |
121 | ||
122 | throw err | |
123 | } | |
124 | } | |
125 | ||
126 | async function refreshVideoIfNeeded (options: { | |
127 | video: MVideoThumbnail | |
128 | fetchedType: VideoFetchByUrlType | |
129 | syncParam: SyncParam | |
130 | }): Promise<MVideoThumbnail> { | |
131 | if (!options.video.isOutdated()) return options.video | |
132 | ||
133 | // We need more attributes if the argument video was fetched with not enough joints | |
134 | const video = options.fetchedType === 'all' | |
135 | ? options.video as MVideoAccountLightBlacklistAllFiles | |
136 | : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | |
137 | ||
138 | try { | |
139 | const { videoObject } = await fetchRemoteVideo(video.url) | |
140 | ||
141 | if (videoObject === undefined) { | |
142 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url) | |
143 | ||
144 | await video.setAsRefreshed() | |
145 | return video | |
146 | } | |
147 | ||
148 | const videoUpdater = new APVideoUpdater(videoObject, video) | |
149 | await videoUpdater.update() | |
150 | ||
151 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | |
152 | ||
153 | ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) | |
154 | ||
155 | return video | |
156 | } catch (err) { | |
157 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | |
158 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) | |
159 | ||
160 | // Video does not exist anymore | |
161 | await video.destroy() | |
162 | return undefined | |
163 | } | |
164 | ||
165 | logger.warn('Cannot refresh video %s.', options.video.url, { err }) | |
166 | ||
167 | ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) | |
168 | ||
169 | // Don't refresh in loop | |
170 | await video.setAsRefreshed() | |
171 | return video | |
172 | } | |
173 | } | |
174 | ||
175 | export { | |
176 | fetchRemoteVideo, | |
177 | fetchRemoteVideoDescription, | |
178 | refreshVideoIfNeeded, | |
179 | getOrCreateVideoAndAccountAndChannel | |
180 | } |