]>
Commit | Line | Data |
---|---|---|
2a491182 F |
1 | import { remove } from 'fs-extra' |
2 | import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils' | |
3 | import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' | |
4 | import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' | |
5 | import { isResolvingToUnicastOnly } from '@server/helpers/dns' | |
6 | import { logger } from '@server/helpers/logger' | |
7 | import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl' | |
8 | import { CONFIG } from '@server/initializers/config' | |
9 | import { sequelizeTypescript } from '@server/initializers/database' | |
10 | import { Hooks } from '@server/lib/plugins/hooks' | |
11 | import { ServerConfigManager } from '@server/lib/server-config-manager' | |
12 | import { setVideoTags } from '@server/lib/video' | |
13 | import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' | |
14 | import { VideoModel } from '@server/models/video/video' | |
15 | import { VideoCaptionModel } from '@server/models/video/video-caption' | |
16 | import { VideoImportModel } from '@server/models/video/video-import' | |
17 | import { FilteredModelAttributes } from '@server/types' | |
18 | import { | |
19 | MChannelAccountDefault, | |
20 | MChannelSync, | |
21 | MThumbnail, | |
22 | MUser, | |
23 | MVideoAccountDefault, | |
24 | MVideoCaption, | |
25 | MVideoImportFormattable, | |
26 | MVideoTag, | |
27 | MVideoThumbnail, | |
28 | MVideoWithBlacklistLight | |
29 | } from '@server/types/models' | |
30 | import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models' | |
31 | import { getLocalVideoActivityPubUrl } from './activitypub/url' | |
32 | import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail' | |
33 | ||
34 | class YoutubeDlImportError extends Error { | |
35 | code: YoutubeDlImportError.CODE | |
36 | cause?: Error // Property to remove once ES2022 is used | |
37 | constructor ({ message, code }) { | |
38 | super(message) | |
39 | this.code = code | |
40 | } | |
41 | ||
42 | static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) { | |
43 | const ytDlErr = new this({ message: message ?? err.message, code }) | |
44 | ytDlErr.cause = err | |
45 | ytDlErr.stack = err.stack // Useless once ES2022 is used | |
46 | return ytDlErr | |
47 | } | |
48 | } | |
49 | ||
50 | namespace YoutubeDlImportError { | |
51 | export enum CODE { | |
52 | FETCH_ERROR, | |
53 | NOT_ONLY_UNICAST_URL | |
54 | } | |
55 | } | |
56 | ||
57 | // --------------------------------------------------------------------------- | |
58 | ||
59 | async function insertFromImportIntoDB (parameters: { | |
60 | video: MVideoThumbnail | |
61 | thumbnailModel: MThumbnail | |
62 | previewModel: MThumbnail | |
63 | videoChannel: MChannelAccountDefault | |
64 | tags: string[] | |
65 | videoImportAttributes: FilteredModelAttributes<VideoImportModel> | |
66 | user: MUser | |
67 | }): Promise<MVideoImportFormattable> { | |
68 | const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters | |
69 | ||
70 | const videoImport = await sequelizeTypescript.transaction(async t => { | |
71 | const sequelizeOptions = { transaction: t } | |
72 | ||
73 | // Save video object in database | |
74 | const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag) | |
75 | videoCreated.VideoChannel = videoChannel | |
76 | ||
77 | if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) | |
78 | if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) | |
79 | ||
80 | await autoBlacklistVideoIfNeeded({ | |
81 | video: videoCreated, | |
82 | user, | |
83 | notify: false, | |
84 | isRemote: false, | |
85 | isNew: true, | |
86 | transaction: t | |
87 | }) | |
88 | ||
89 | await setVideoTags({ video: videoCreated, tags, transaction: t }) | |
90 | ||
91 | // Create video import object in database | |
92 | const videoImport = await VideoImportModel.create( | |
93 | Object.assign({ videoId: videoCreated.id }, videoImportAttributes), | |
94 | sequelizeOptions | |
95 | ) as MVideoImportFormattable | |
96 | videoImport.Video = videoCreated | |
97 | ||
98 | return videoImport | |
99 | }) | |
100 | ||
101 | return videoImport | |
102 | } | |
103 | ||
104 | async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: { | |
105 | channelId: number | |
106 | importData: YoutubeDLInfo | |
107 | importDataOverride?: Partial<VideoImportCreate> | |
108 | importType: 'url' | 'torrent' | |
109 | }): Promise<MVideoThumbnail> { | |
110 | let videoData = { | |
111 | name: importDataOverride?.name || importData.name || 'Unknown name', | |
112 | remote: false, | |
113 | category: importDataOverride?.category || importData.category, | |
114 | licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE, | |
115 | language: importDataOverride?.language || importData.language, | |
116 | commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, | |
117 | downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, | |
3c4754a3 | 118 | waitTranscoding: importDataOverride?.waitTranscoding ?? true, |
2a491182 F |
119 | state: VideoState.TO_IMPORT, |
120 | nsfw: importDataOverride?.nsfw || importData.nsfw || false, | |
121 | description: importDataOverride?.description || importData.description, | |
122 | support: importDataOverride?.support || null, | |
123 | privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE, | |
124 | duration: 0, // duration will be set by the import job | |
125 | channelId, | |
126 | originallyPublishedAt: importDataOverride?.originallyPublishedAt | |
127 | ? new Date(importDataOverride?.originallyPublishedAt) | |
3204f4d1 | 128 | : importData.originallyPublishedAtWithoutTime |
2a491182 F |
129 | } |
130 | ||
131 | videoData = await Hooks.wrapObject( | |
132 | videoData, | |
133 | importType === 'url' | |
134 | ? 'filter:api.video.import-url.video-attribute.result' | |
135 | : 'filter:api.video.import-torrent.video-attribute.result' | |
136 | ) | |
137 | ||
138 | const video = new VideoModel(videoData) | |
139 | video.url = getLocalVideoActivityPubUrl(video) | |
140 | ||
141 | return video | |
142 | } | |
143 | ||
144 | async function buildYoutubeDLImport (options: { | |
145 | targetUrl: string | |
146 | channel: MChannelAccountDefault | |
147 | user: MUser | |
148 | channelSync?: MChannelSync | |
149 | importDataOverride?: Partial<VideoImportCreate> | |
150 | thumbnailFilePath?: string | |
151 | previewFilePath?: string | |
152 | }) { | |
153 | const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options | |
154 | ||
155 | const youtubeDL = new YoutubeDLWrapper( | |
156 | targetUrl, | |
157 | ServerConfigManager.Instance.getEnabledResolutions('vod'), | |
158 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | |
159 | ) | |
160 | ||
161 | // Get video infos | |
162 | let youtubeDLInfo: YoutubeDLInfo | |
163 | try { | |
164 | youtubeDLInfo = await youtubeDL.getInfoForDownload() | |
165 | } catch (err) { | |
166 | throw YoutubeDlImportError.fromError( | |
167 | err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}` | |
168 | ) | |
169 | } | |
170 | ||
171 | if (!await hasUnicastURLsOnly(youtubeDLInfo)) { | |
172 | throw new YoutubeDlImportError({ | |
173 | message: 'Cannot use non unicast IP as targetUrl.', | |
174 | code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL | |
175 | }) | |
176 | } | |
177 | ||
178 | const video = await buildVideoFromImport({ | |
179 | channelId: channel.id, | |
180 | importData: youtubeDLInfo, | |
181 | importDataOverride, | |
182 | importType: 'url' | |
183 | }) | |
184 | ||
185 | const thumbnailModel = await forgeThumbnail({ | |
186 | inputPath: thumbnailFilePath, | |
187 | downloadUrl: youtubeDLInfo.thumbnailUrl, | |
188 | video, | |
189 | type: ThumbnailType.MINIATURE | |
190 | }) | |
191 | ||
192 | const previewModel = await forgeThumbnail({ | |
193 | inputPath: previewFilePath, | |
194 | downloadUrl: youtubeDLInfo.thumbnailUrl, | |
195 | video, | |
196 | type: ThumbnailType.PREVIEW | |
197 | }) | |
198 | ||
199 | const videoImport = await insertFromImportIntoDB({ | |
200 | video, | |
201 | thumbnailModel, | |
202 | previewModel, | |
203 | videoChannel: channel, | |
204 | tags: importDataOverride?.tags || youtubeDLInfo.tags, | |
205 | user, | |
206 | videoImportAttributes: { | |
207 | targetUrl, | |
208 | state: VideoImportState.PENDING, | |
a3b472a1 C |
209 | userId: user.id, |
210 | videoChannelSyncId: channelSync?.id | |
2a491182 F |
211 | } |
212 | }) | |
213 | ||
214 | // Get video subtitles | |
215 | await processYoutubeSubtitles(youtubeDL, targetUrl, video.id) | |
216 | ||
217 | let fileExt = `.${youtubeDLInfo.ext}` | |
218 | if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4' | |
219 | ||
220 | const payload: VideoImportPayload = { | |
221 | type: 'youtube-dl' as 'youtube-dl', | |
222 | videoImportId: videoImport.id, | |
223 | fileExt, | |
224 | // If part of a sync process, there is a parent job that will aggregate children results | |
225 | preventException: !!channelSync | |
226 | } | |
227 | ||
228 | return { | |
229 | videoImport, | |
230 | job: { type: 'video-import' as 'video-import', payload } | |
231 | } | |
232 | } | |
233 | ||
234 | // --------------------------------------------------------------------------- | |
235 | ||
236 | export { | |
237 | buildYoutubeDLImport, | |
238 | YoutubeDlImportError, | |
239 | insertFromImportIntoDB, | |
240 | buildVideoFromImport | |
241 | } | |
242 | ||
243 | // --------------------------------------------------------------------------- | |
244 | ||
245 | async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: { | |
246 | inputPath?: string | |
247 | downloadUrl?: string | |
248 | video: MVideoThumbnail | |
249 | type: ThumbnailType | |
250 | }): Promise<MThumbnail> { | |
251 | if (inputPath) { | |
252 | return updateVideoMiniatureFromExisting({ | |
253 | inputPath, | |
254 | video, | |
255 | type, | |
256 | automaticallyGenerated: false | |
257 | }) | |
258 | } else if (downloadUrl) { | |
259 | try { | |
260 | return await updateVideoMiniatureFromUrl({ downloadUrl, video, type }) | |
261 | } catch (err) { | |
262 | logger.warn('Cannot process thumbnail %s from youtubedl.', downloadUrl, { err }) | |
263 | } | |
264 | } | |
265 | return null | |
266 | } | |
267 | ||
268 | async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) { | |
269 | try { | |
270 | const subtitles = await youtubeDL.getSubtitles() | |
271 | ||
272 | logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl) | |
273 | ||
274 | for (const subtitle of subtitles) { | |
275 | if (!await isVTTFileValid(subtitle.path)) { | |
276 | await remove(subtitle.path) | |
277 | continue | |
278 | } | |
279 | ||
280 | const videoCaption = new VideoCaptionModel({ | |
281 | videoId, | |
282 | language: subtitle.language, | |
283 | filename: VideoCaptionModel.generateCaptionName(subtitle.language) | |
284 | }) as MVideoCaption | |
285 | ||
286 | // Move physical file | |
287 | await moveAndProcessCaptionFile(subtitle, videoCaption) | |
288 | ||
289 | await sequelizeTypescript.transaction(async t => { | |
290 | await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) | |
291 | }) | |
292 | } | |
293 | } catch (err) { | |
294 | logger.warn('Cannot get video subtitles.', { err }) | |
295 | } | |
296 | } | |
297 | ||
298 | async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) { | |
299 | const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname) | |
300 | const uniqHosts = new Set(hosts) | |
301 | ||
302 | for (const h of uniqHosts) { | |
303 | if (await isResolvingToUnicastOnly(h) !== true) { | |
304 | return false | |
305 | } | |
306 | } | |
307 | ||
308 | return true | |
309 | } |