diff options
Diffstat (limited to 'server/lib/activitypub/videos/fetch.ts')
-rw-r--r-- | server/lib/activitypub/videos/fetch.ts | 180 |
1 files changed, 0 insertions, 180 deletions
diff --git a/server/lib/activitypub/videos/fetch.ts b/server/lib/activitypub/videos/fetch.ts deleted file mode 100644 index 5113c9d7e..000000000 --- a/server/lib/activitypub/videos/fetch.ts +++ /dev/null | |||
@@ -1,180 +0,0 @@ | |||
1 | import { checkUrlsSameHost, getAPId } from '@server/helpers/activitypub' | ||
2 | import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' | ||
3 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
4 | import { logger } from '@server/helpers/logger' | ||
5 | import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests' | ||
6 | import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video' | ||
7 | import { REMOTE_SCHEME } from '@server/initializers/constants' | ||
8 | import { ActorFollowScoreCache } from '@server/lib/files-cache' | ||
9 | import { JobQueue } from '@server/lib/job-queue' | ||
10 | import { VideoModel } from '@server/models/video/video' | ||
11 | import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' | ||
12 | import { HttpStatusCode } from '@shared/core-utils' | ||
13 | import { VideoObject } from '@shared/models' | ||
14 | import { APVideoCreator, SyncParam, syncVideoExternalAttributes } from './shared' | ||
15 | import { APVideoUpdater } from './updater' | ||
16 | |||
17 | async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { | ||
18 | logger.info('Fetching remote video %s.', videoUrl) | ||
19 | |||
20 | const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true }) | ||
21 | |||
22 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { | ||
23 | logger.debug('Remote video JSON is not valid.', { body }) | ||
24 | return { statusCode, videoObject: undefined } | ||
25 | } | ||
26 | |||
27 | return { statusCode, videoObject: body } | ||
28 | } | ||
29 | |||
30 | async function fetchRemoteVideoDescription (video: MVideoAccountLight) { | ||
31 | const host = video.VideoChannel.Account.Actor.Server.host | ||
32 | const path = video.getDescriptionAPIPath() | ||
33 | const url = REMOTE_SCHEME.HTTP + '://' + host + path | ||
34 | |||
35 | const { body } = await doJSONRequest<any>(url) | ||
36 | return body.description || '' | ||
37 | } | ||
38 | |||
39 | type GetVideoResult <T> = Promise<{ | ||
40 | video: T | ||
41 | created: boolean | ||
42 | autoBlacklisted?: boolean | ||
43 | }> | ||
44 | |||
45 | type GetVideoParamAll = { | ||
46 | videoObject: { id: string } | string | ||
47 | syncParam?: SyncParam | ||
48 | fetchType?: 'all' | ||
49 | allowRefresh?: boolean | ||
50 | } | ||
51 | |||
52 | type GetVideoParamImmutable = { | ||
53 | videoObject: { id: string } | string | ||
54 | syncParam?: SyncParam | ||
55 | fetchType: 'only-immutable-attributes' | ||
56 | allowRefresh: false | ||
57 | } | ||
58 | |||
59 | type GetVideoParamOther = { | ||
60 | videoObject: { id: string } | string | ||
61 | syncParam?: SyncParam | ||
62 | fetchType?: 'all' | 'only-video' | ||
63 | allowRefresh?: boolean | ||
64 | } | ||
65 | |||
66 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles> | ||
67 | function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable> | ||
68 | function getOrCreateVideoAndAccountAndChannel ( | ||
69 | options: GetVideoParamOther | ||
70 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> | ||
71 | async function getOrCreateVideoAndAccountAndChannel ( | ||
72 | options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther | ||
73 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> { | ||
74 | // Default params | ||
75 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | ||
76 | const fetchType = options.fetchType || 'all' | ||
77 | const allowRefresh = options.allowRefresh !== false | ||
78 | |||
79 | // Get video url | ||
80 | const videoUrl = getAPId(options.videoObject) | ||
81 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | ||
82 | |||
83 | if (videoFromDatabase) { | ||
84 | // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type | ||
85 | if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) { | ||
86 | const refreshOptions = { | ||
87 | video: videoFromDatabase as MVideoThumbnail, | ||
88 | fetchedType: fetchType, | ||
89 | syncParam | ||
90 | } | ||
91 | |||
92 | if (syncParam.refreshVideo === true) { | ||
93 | videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) | ||
94 | } else { | ||
95 | await JobQueue.Instance.createJobWithPromise({ | ||
96 | type: 'activitypub-refresher', | ||
97 | payload: { type: 'video', url: videoFromDatabase.url } | ||
98 | }) | ||
99 | } | ||
100 | } | ||
101 | |||
102 | return { video: videoFromDatabase, created: false } | ||
103 | } | ||
104 | |||
105 | const { videoObject } = await fetchRemoteVideo(videoUrl) | ||
106 | if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | ||
107 | |||
108 | try { | ||
109 | const creator = new APVideoCreator(videoObject) | ||
110 | const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail) | ||
111 | |||
112 | await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) | ||
113 | |||
114 | return { video: videoCreated, created: true, autoBlacklisted } | ||
115 | } catch (err) { | ||
116 | // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video | ||
117 | if (err.name === 'SequelizeUniqueConstraintError') { | ||
118 | const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType) | ||
119 | if (fallbackVideo) return { video: fallbackVideo, created: false } | ||
120 | } | ||
121 | |||
122 | throw err | ||
123 | } | ||
124 | } | ||
125 | |||
126 | async function refreshVideoIfNeeded (options: { | ||
127 | video: MVideoThumbnail | ||
128 | fetchedType: VideoFetchByUrlType | ||
129 | syncParam: SyncParam | ||
130 | }): Promise<MVideoThumbnail> { | ||
131 | if (!options.video.isOutdated()) return options.video | ||
132 | |||
133 | // We need more attributes if the argument video was fetched with not enough joints | ||
134 | const video = options.fetchedType === 'all' | ||
135 | ? options.video as MVideoAccountLightBlacklistAllFiles | ||
136 | : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | ||
137 | |||
138 | try { | ||
139 | const { videoObject } = await fetchRemoteVideo(video.url) | ||
140 | |||
141 | if (videoObject === undefined) { | ||
142 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url) | ||
143 | |||
144 | await video.setAsRefreshed() | ||
145 | return video | ||
146 | } | ||
147 | |||
148 | const videoUpdater = new APVideoUpdater(videoObject, video) | ||
149 | await videoUpdater.update() | ||
150 | |||
151 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | ||
152 | |||
153 | ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) | ||
154 | |||
155 | return video | ||
156 | } catch (err) { | ||
157 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
158 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) | ||
159 | |||
160 | // Video does not exist anymore | ||
161 | await video.destroy() | ||
162 | return undefined | ||
163 | } | ||
164 | |||
165 | logger.warn('Cannot refresh video %s.', options.video.url, { err }) | ||
166 | |||
167 | ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) | ||
168 | |||
169 | // Don't refresh in loop | ||
170 | await video.setAsRefreshed() | ||
171 | return video | ||
172 | } | ||
173 | } | ||
174 | |||
175 | export { | ||
176 | fetchRemoteVideo, | ||
177 | fetchRemoteVideoDescription, | ||
178 | refreshVideoIfNeeded, | ||
179 | getOrCreateVideoAndAccountAndChannel | ||
180 | } | ||