diff options
Diffstat (limited to 'server')
-rw-r--r-- | server/controllers/api/videos/import.ts | 29 | ||||
-rw-r--r-- | server/helpers/custom-validators/video-captions.ts | 13 | ||||
-rw-r--r-- | server/helpers/youtube-dl/youtube-dl-info-builder.ts | 49 |
3 files changed, 90 insertions, 1 deletions
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 8cbfd3286..b54fa822c 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts | |||
@@ -1,9 +1,11 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { move, readFile } from 'fs-extra' | 2 | import { move, readFile, remove } from 'fs-extra' |
3 | import { decode } from 'magnet-uri' | 3 | import { decode } from 'magnet-uri' |
4 | import parseTorrent, { Instance } from 'parse-torrent' | 4 | import parseTorrent, { Instance } from 'parse-torrent' |
5 | import { join } from 'path' | 5 | import { join } from 'path' |
6 | import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' | ||
6 | import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' | 7 | import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' |
8 | import { isResolvingToUnicastOnly } from '@server/helpers/dns' | ||
7 | import { Hooks } from '@server/lib/plugins/hooks' | 9 | import { Hooks } from '@server/lib/plugins/hooks' |
8 | import { ServerConfigManager } from '@server/lib/server-config-manager' | 10 | import { ServerConfigManager } from '@server/lib/server-config-manager' |
9 | import { setVideoTags } from '@server/lib/video' | 11 | import { setVideoTags } from '@server/lib/video' |
@@ -195,6 +197,13 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) | |||
195 | }) | 197 | }) |
196 | } | 198 | } |
197 | 199 | ||
200 | if (!await hasUnicastURLsOnly(youtubeDLInfo)) { | ||
201 | return res.fail({ | ||
202 | status: HttpStatusCode.FORBIDDEN_403, | ||
203 | message: 'Cannot use non unicast IP as targetUrl.' | ||
204 | }) | ||
205 | } | ||
206 | |||
198 | const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo) | 207 | const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo) |
199 | 208 | ||
200 | // Process video thumbnail from request.files | 209 | // Process video thumbnail from request.files |
@@ -432,6 +441,11 @@ async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: | |||
432 | logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl) | 441 | logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl) |
433 | 442 | ||
434 | for (const subtitle of subtitles) { | 443 | for (const subtitle of subtitles) { |
444 | if (!await isVTTFileValid(subtitle.path)) { | ||
445 | await remove(subtitle.path) | ||
446 | continue | ||
447 | } | ||
448 | |||
435 | const videoCaption = new VideoCaptionModel({ | 449 | const videoCaption = new VideoCaptionModel({ |
436 | videoId, | 450 | videoId, |
437 | language: subtitle.language, | 451 | language: subtitle.language, |
@@ -449,3 +463,16 @@ async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: | |||
449 | logger.warn('Cannot get video subtitles.', { err }) | 463 | logger.warn('Cannot get video subtitles.', { err }) |
450 | } | 464 | } |
451 | } | 465 | } |
466 | |||
467 | async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) { | ||
468 | const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname) | ||
469 | const uniqHosts = new Set(hosts) | ||
470 | |||
471 | for (const h of uniqHosts) { | ||
472 | if (await isResolvingToUnicastOnly(h) !== true) { | ||
473 | return false | ||
474 | } | ||
475 | } | ||
476 | |||
477 | return true | ||
478 | } | ||
diff --git a/server/helpers/custom-validators/video-captions.ts b/server/helpers/custom-validators/video-captions.ts index 528edf60c..4cc7dcaf4 100644 --- a/server/helpers/custom-validators/video-captions.ts +++ b/server/helpers/custom-validators/video-captions.ts | |||
@@ -1,3 +1,5 @@ | |||
1 | import { getFileSize } from '@shared/extra-utils' | ||
2 | import { readFile } from 'fs-extra' | ||
1 | import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initializers/constants' | 3 | import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initializers/constants' |
2 | import { exists, isFileValid } from './misc' | 4 | import { exists, isFileValid } from './misc' |
3 | 5 | ||
@@ -13,9 +15,20 @@ function isVideoCaptionFile (files: { [ fieldname: string ]: Express.Multer.File | |||
13 | return isFileValid(files, videoCaptionTypesRegex, field, CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) | 15 | return isFileValid(files, videoCaptionTypesRegex, field, CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) |
14 | } | 16 | } |
15 | 17 | ||
18 | async function isVTTFileValid (filePath: string) { | ||
19 | const size = await getFileSize(filePath) | ||
20 | |||
21 | if (size > CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) return false | ||
22 | |||
23 | const content = await readFile(filePath, 'utf8') | ||
24 | |||
25 | return content?.startsWith('WEBVTT\n') | ||
26 | } | ||
27 | |||
16 | // --------------------------------------------------------------------------- | 28 | // --------------------------------------------------------------------------- |
17 | 29 | ||
18 | export { | 30 | export { |
19 | isVideoCaptionFile, | 31 | isVideoCaptionFile, |
32 | isVTTFileValid, | ||
20 | isVideoCaptionLanguageValid | 33 | isVideoCaptionLanguageValid |
21 | } | 34 | } |
diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts index 9746a7067..71572f292 100644 --- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts +++ b/server/helpers/youtube-dl/youtube-dl-info-builder.ts | |||
@@ -1,5 +1,6 @@ | |||
1 | import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../../initializers/constants' | 1 | import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../../initializers/constants' |
2 | import { peertubeTruncate } from '../core-utils' | 2 | import { peertubeTruncate } from '../core-utils' |
3 | import { isUrlValid } from '../custom-validators/activitypub/misc' | ||
3 | 4 | ||
4 | type YoutubeDLInfo = { | 5 | type YoutubeDLInfo = { |
5 | name?: string | 6 | name?: string |
@@ -12,6 +13,8 @@ type YoutubeDLInfo = { | |||
12 | thumbnailUrl?: string | 13 | thumbnailUrl?: string |
13 | ext?: string | 14 | ext?: string |
14 | originallyPublishedAt?: Date | 15 | originallyPublishedAt?: Date |
16 | |||
17 | urls?: string[] | ||
15 | } | 18 | } |
16 | 19 | ||
17 | class YoutubeDLInfoBuilder { | 20 | class YoutubeDLInfoBuilder { |
@@ -76,11 +79,57 @@ class YoutubeDLInfoBuilder { | |||
76 | nsfw: this.isNSFW(obj), | 79 | nsfw: this.isNSFW(obj), |
77 | tags: this.getTags(obj.tags), | 80 | tags: this.getTags(obj.tags), |
78 | thumbnailUrl: obj.thumbnail || undefined, | 81 | thumbnailUrl: obj.thumbnail || undefined, |
82 | urls: this.buildAvailableUrl(obj), | ||
79 | originallyPublishedAt: this.buildOriginallyPublishedAt(obj), | 83 | originallyPublishedAt: this.buildOriginallyPublishedAt(obj), |
80 | ext: obj.ext | 84 | ext: obj.ext |
81 | } | 85 | } |
82 | } | 86 | } |
83 | 87 | ||
88 | private buildAvailableUrl (obj: any) { | ||
89 | const urls: string[] = [] | ||
90 | |||
91 | if (obj.url) urls.push(obj.url) | ||
92 | if (obj.urls) { | ||
93 | if (Array.isArray(obj.urls)) urls.push(...obj.urls) | ||
94 | else urls.push(obj.urls) | ||
95 | } | ||
96 | |||
97 | const formats = Array.isArray(obj.formats) | ||
98 | ? obj.formats | ||
99 | : [] | ||
100 | |||
101 | for (const format of formats) { | ||
102 | if (!format.url) continue | ||
103 | |||
104 | urls.push(format.url) | ||
105 | } | ||
106 | |||
107 | const thumbnails = Array.isArray(obj.thumbnails) | ||
108 | ? obj.thumbnails | ||
109 | : [] | ||
110 | |||
111 | for (const thumbnail of thumbnails) { | ||
112 | if (!thumbnail.url) continue | ||
113 | |||
114 | urls.push(thumbnail.url) | ||
115 | } | ||
116 | |||
117 | if (obj.thumbnail) urls.push(obj.thumbnail) | ||
118 | |||
119 | for (const subtitleKey of Object.keys(obj.subtitles || {})) { | ||
120 | const subtitles = obj.subtitles[subtitleKey] | ||
121 | if (!Array.isArray(subtitles)) continue | ||
122 | |||
123 | for (const subtitle of subtitles) { | ||
124 | if (!subtitle.url) continue | ||
125 | |||
126 | urls.push(subtitle.url) | ||
127 | } | ||
128 | } | ||
129 | |||
130 | return urls.filter(u => u && isUrlValid(u)) | ||
131 | } | ||
132 | |||
84 | private titleTruncation (title: string) { | 133 | private titleTruncation (title: string) { |
85 | return peertubeTruncate(title, { | 134 | return peertubeTruncate(title, { |
86 | length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max, | 135 | length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max, |