aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--CHANGELOG.md2
-rw-r--r--client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html12
-rw-r--r--client/src/app/+admin/follows/follows.routes.ts7
-rw-r--r--config/default.yaml10
-rw-r--r--config/production.yaml.example10
-rw-r--r--server/controllers/api/videos/import.ts29
-rw-r--r--server/helpers/custom-validators/video-captions.ts13
-rw-r--r--server/helpers/youtube-dl/youtube-dl-info-builder.ts49
8 files changed, 123 insertions, 9 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f8a44e9a7..16a74f315 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,6 @@
1# Changelog 1# Changelog
2 2
3## v4.1.0-rc.1 3## v4.1.0-rc.1 (unreleased)
4 4
5### IMPORTANT NOTES 5### IMPORTANT NOTES
6 6
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html
index c9533208a..37989cb59 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html
+++ b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html
@@ -267,14 +267,22 @@
267 <my-peertube-checkbox 267 <my-peertube-checkbox
268 inputName="importVideosHttpEnabled" formControlName="enabled" 268 inputName="importVideosHttpEnabled" formControlName="enabled"
269 i18n-labelText labelText="Allow import with HTTP URL (e.g. YouTube)" 269 i18n-labelText labelText="Allow import with HTTP URL (e.g. YouTube)"
270 ></my-peertube-checkbox> 270 >
271 <ng-container ngProjectAs="description">
272 <span i18n>⚠️ If enabled, we recommend to use <a href="https://docs.joinpeertube.org/maintain-configuration?id=security">a HTTP proxy</a> to prevent private URL access from your PeerTube server</span>
273 </ng-container>
274 </my-peertube-checkbox>
271 </div> 275 </div>
272 276
273 <div class="form-group" formGroupName="torrent"> 277 <div class="form-group" formGroupName="torrent">
274 <my-peertube-checkbox 278 <my-peertube-checkbox
275 inputName="importVideosTorrentEnabled" formControlName="enabled" 279 inputName="importVideosTorrentEnabled" formControlName="enabled"
276 i18n-labelText labelText="Allow import with a torrent file or a magnet URI" 280 i18n-labelText labelText="Allow import with a torrent file or a magnet URI"
277 ></my-peertube-checkbox> 281 >
282 <ng-container ngProjectAs="description">
283 <span i18n>⚠️ We don't recommend to enable this feature if you don't trust your users</span>
284 </ng-container>
285 </my-peertube-checkbox>
278 </div> 286 </div>
279 287
280 </ng-container> 288 </ng-container>
diff --git a/client/src/app/+admin/follows/follows.routes.ts b/client/src/app/+admin/follows/follows.routes.ts
index 1825ce278..718493dc7 100644
--- a/client/src/app/+admin/follows/follows.routes.ts
+++ b/client/src/app/+admin/follows/follows.routes.ts
@@ -42,7 +42,12 @@ export const FollowsRoutes: Routes = [
42 }, 42 },
43 { 43 {
44 path: 'video-redundancies-list', 44 path: 'video-redundancies-list',
45 component: VideoRedundanciesListComponent 45 component: VideoRedundanciesListComponent,
46 data: {
47 meta: {
48 title: $localize`Redundancy`
49 }
50 }
46 } 51 }
47 ] 52 ]
48 } 53 }
diff --git a/config/default.yaml b/config/default.yaml
index 2b0419535..23be08f85 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -431,7 +431,10 @@ import:
431 # Amount of import jobs to execute in parallel 431 # Amount of import jobs to execute in parallel
432 concurrency: 1 432 concurrency: 1
433 433
434 http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html 434 # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
435 http:
436 # We recommend to use a HTTP proxy if you enable HTTP import to prevent private URL access from this server
437 # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
435 enabled: false 438 enabled: false
436 439
437 youtube_dl_release: 440 youtube_dl_release:
@@ -452,7 +455,10 @@ import:
452 # IPv6 is very strongly rate-limited on most sites supported by youtube-dl 455 # IPv6 is very strongly rate-limited on most sites supported by youtube-dl
453 force_ipv4: false 456 force_ipv4: false
454 457
455 torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) 458 # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
459 torrent:
460 # We recommend to only enable magnet URI/torrent import if you trust your users
461 # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
456 enabled: false 462 enabled: false
457 463
458auto_blacklist: 464auto_blacklist:
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 893ccc33e..675801caa 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -439,7 +439,10 @@ import:
439 # Amount of import jobs to execute in parallel 439 # Amount of import jobs to execute in parallel
440 concurrency: 1 440 concurrency: 1
441 441
442 http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html 442 # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
443 http:
444 # We recommend to use a HTTP proxy if you enable HTTP import to prevent private URL access from this server
445 # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
443 enabled: false 446 enabled: false
444 447
445 youtube_dl_release: 448 youtube_dl_release:
@@ -460,7 +463,10 @@ import:
460 # IPv6 is very strongly rate-limited on most sites supported by youtube-dl 463 # IPv6 is very strongly rate-limited on most sites supported by youtube-dl
461 force_ipv4: false 464 force_ipv4: false
462 465
463 torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) 466 # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
467 torrent:
468 # We recommend to only enable magnet URI/torrent import if you trust your users
469 # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
464 enabled: false 470 enabled: false
465 471
466auto_blacklist: 472auto_blacklist:
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 8cbfd3286..b54fa822c 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -1,9 +1,11 @@
1import express from 'express' 1import express from 'express'
2import { move, readFile } from 'fs-extra' 2import { move, readFile, remove } from 'fs-extra'
3import { decode } from 'magnet-uri' 3import { decode } from 'magnet-uri'
4import parseTorrent, { Instance } from 'parse-torrent' 4import parseTorrent, { Instance } from 'parse-torrent'
5import { join } from 'path' 5import { join } from 'path'
6import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
6import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' 7import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
8import { isResolvingToUnicastOnly } from '@server/helpers/dns'
7import { Hooks } from '@server/lib/plugins/hooks' 9import { Hooks } from '@server/lib/plugins/hooks'
8import { ServerConfigManager } from '@server/lib/server-config-manager' 10import { ServerConfigManager } from '@server/lib/server-config-manager'
9import { setVideoTags } from '@server/lib/video' 11import { setVideoTags } from '@server/lib/video'
@@ -195,6 +197,13 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
195 }) 197 })
196 } 198 }
197 199
200 if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
201 return res.fail({
202 status: HttpStatusCode.FORBIDDEN_403,
203 message: 'Cannot use non unicast IP as targetUrl.'
204 })
205 }
206
198 const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo) 207 const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
199 208
200 // Process video thumbnail from request.files 209 // Process video thumbnail from request.files
@@ -432,6 +441,11 @@ async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl:
432 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl) 441 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
433 442
434 for (const subtitle of subtitles) { 443 for (const subtitle of subtitles) {
444 if (!await isVTTFileValid(subtitle.path)) {
445 await remove(subtitle.path)
446 continue
447 }
448
435 const videoCaption = new VideoCaptionModel({ 449 const videoCaption = new VideoCaptionModel({
436 videoId, 450 videoId,
437 language: subtitle.language, 451 language: subtitle.language,
@@ -449,3 +463,16 @@ async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl:
449 logger.warn('Cannot get video subtitles.', { err }) 463 logger.warn('Cannot get video subtitles.', { err })
450 } 464 }
451} 465}
466
467async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
468 const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
469 const uniqHosts = new Set(hosts)
470
471 for (const h of uniqHosts) {
472 if (await isResolvingToUnicastOnly(h) !== true) {
473 return false
474 }
475 }
476
477 return true
478}
diff --git a/server/helpers/custom-validators/video-captions.ts b/server/helpers/custom-validators/video-captions.ts
index 528edf60c..4cc7dcaf4 100644
--- a/server/helpers/custom-validators/video-captions.ts
+++ b/server/helpers/custom-validators/video-captions.ts
@@ -1,3 +1,5 @@
1import { getFileSize } from '@shared/extra-utils'
2import { readFile } from 'fs-extra'
1import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initializers/constants' 3import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initializers/constants'
2import { exists, isFileValid } from './misc' 4import { exists, isFileValid } from './misc'
3 5
@@ -13,9 +15,20 @@ function isVideoCaptionFile (files: { [ fieldname: string ]: Express.Multer.File
13 return isFileValid(files, videoCaptionTypesRegex, field, CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) 15 return isFileValid(files, videoCaptionTypesRegex, field, CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max)
14} 16}
15 17
18async function isVTTFileValid (filePath: string) {
19 const size = await getFileSize(filePath)
20
21 if (size > CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) return false
22
23 const content = await readFile(filePath, 'utf8')
24
25 return content?.startsWith('WEBVTT\n')
26}
27
16// --------------------------------------------------------------------------- 28// ---------------------------------------------------------------------------
17 29
18export { 30export {
19 isVideoCaptionFile, 31 isVideoCaptionFile,
32 isVTTFileValid,
20 isVideoCaptionLanguageValid 33 isVideoCaptionLanguageValid
21} 34}
diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
index 9746a7067..71572f292 100644
--- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts
+++ b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
@@ -1,5 +1,6 @@
1import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../../initializers/constants' 1import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../../initializers/constants'
2import { peertubeTruncate } from '../core-utils' 2import { peertubeTruncate } from '../core-utils'
3import { isUrlValid } from '../custom-validators/activitypub/misc'
3 4
4type YoutubeDLInfo = { 5type YoutubeDLInfo = {
5 name?: string 6 name?: string
@@ -12,6 +13,8 @@ type YoutubeDLInfo = {
12 thumbnailUrl?: string 13 thumbnailUrl?: string
13 ext?: string 14 ext?: string
14 originallyPublishedAt?: Date 15 originallyPublishedAt?: Date
16
17 urls?: string[]
15} 18}
16 19
17class YoutubeDLInfoBuilder { 20class YoutubeDLInfoBuilder {
@@ -76,11 +79,57 @@ class YoutubeDLInfoBuilder {
76 nsfw: this.isNSFW(obj), 79 nsfw: this.isNSFW(obj),
77 tags: this.getTags(obj.tags), 80 tags: this.getTags(obj.tags),
78 thumbnailUrl: obj.thumbnail || undefined, 81 thumbnailUrl: obj.thumbnail || undefined,
82 urls: this.buildAvailableUrl(obj),
79 originallyPublishedAt: this.buildOriginallyPublishedAt(obj), 83 originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
80 ext: obj.ext 84 ext: obj.ext
81 } 85 }
82 } 86 }
83 87
88 private buildAvailableUrl (obj: any) {
89 const urls: string[] = []
90
91 if (obj.url) urls.push(obj.url)
92 if (obj.urls) {
93 if (Array.isArray(obj.urls)) urls.push(...obj.urls)
94 else urls.push(obj.urls)
95 }
96
97 const formats = Array.isArray(obj.formats)
98 ? obj.formats
99 : []
100
101 for (const format of formats) {
102 if (!format.url) continue
103
104 urls.push(format.url)
105 }
106
107 const thumbnails = Array.isArray(obj.thumbnails)
108 ? obj.thumbnails
109 : []
110
111 for (const thumbnail of thumbnails) {
112 if (!thumbnail.url) continue
113
114 urls.push(thumbnail.url)
115 }
116
117 if (obj.thumbnail) urls.push(obj.thumbnail)
118
119 for (const subtitleKey of Object.keys(obj.subtitles || {})) {
120 const subtitles = obj.subtitles[subtitleKey]
121 if (!Array.isArray(subtitles)) continue
122
123 for (const subtitle of subtitles) {
124 if (!subtitle.url) continue
125
126 urls.push(subtitle.url)
127 }
128 }
129
130 return urls.filter(u => u && isUrlValid(u))
131 }
132
84 private titleTruncation (title: string) { 133 private titleTruncation (title: string) {
85 return peertubeTruncate(title, { 134 return peertubeTruncate(title, {
86 length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max, 135 length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max,