From 474542d7ac60f7860daf9ea34d1c31968f43ab29 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 9 Feb 2022 08:58:40 +0100 Subject: Add additional checks when importing a video --- CHANGELOG.md | 2 +- .../edit-basic-configuration.component.html | 12 +++++- client/src/app/+admin/follows/follows.routes.ts | 7 +++- config/default.yaml | 10 ++++- config/production.yaml.example | 10 ++++- server/controllers/api/videos/import.ts | 29 ++++++++++++- server/helpers/custom-validators/video-captions.ts | 13 ++++++ .../helpers/youtube-dl/youtube-dl-info-builder.ts | 49 ++++++++++++++++++++++ 8 files changed, 123 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8a44e9a7..16a74f315 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## v4.1.0-rc.1 +## v4.1.0-rc.1 (unreleased) ### IMPORTANT NOTES diff --git a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html index c9533208a..37989cb59 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html +++ b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html @@ -267,14 +267,22 @@ + > + + ⚠️ If enabled, we recommend to use a HTTP proxy to prevent private URL access from your PeerTube server + +
+ > + + ⚠️ We don't recommend to enable this feature if you don't trust your users + +
diff --git a/client/src/app/+admin/follows/follows.routes.ts b/client/src/app/+admin/follows/follows.routes.ts index 1825ce278..718493dc7 100644 --- a/client/src/app/+admin/follows/follows.routes.ts +++ b/client/src/app/+admin/follows/follows.routes.ts @@ -42,7 +42,12 @@ export const FollowsRoutes: Routes = [ }, { path: 'video-redundancies-list', - component: VideoRedundanciesListComponent + component: VideoRedundanciesListComponent, + data: { + meta: { + title: $localize`Redundancy` + } + } } ] } diff --git a/config/default.yaml b/config/default.yaml index 2b0419535..23be08f85 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -431,7 +431,10 @@ import: # Amount of import jobs to execute in parallel concurrency: 1 - http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html + # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html + http: + # We recommend to use a HTTP proxy if you enable HTTP import to prevent private URL access from this server + # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information enabled: false youtube_dl_release: @@ -452,7 +455,10 @@ import: # IPv6 is very strongly rate-limited on most sites supported by youtube-dl force_ipv4: false - torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) + # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) + torrent: + # We recommend to only enable magnet URI/torrent import if you trust your users + # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information enabled: false auto_blacklist: diff --git a/config/production.yaml.example b/config/production.yaml.example index 893ccc33e..675801caa 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -439,7 +439,10 @@ import: # Amount of import jobs to execute in parallel concurrency: 1 - http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html + # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html + http: + # We recommend to use a HTTP proxy if you enable HTTP import to prevent private URL access from this server + # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information enabled: false youtube_dl_release: @@ -460,7 +463,10 @@ import: # IPv6 is very strongly rate-limited on most sites supported by youtube-dl force_ipv4: false - torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) + # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) + torrent: + # We recommend to only enable magnet URI/torrent import if you trust your users + # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information enabled: false auto_blacklist: diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 8cbfd3286..b54fa822c 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -1,9 +1,11 @@ import express from 'express' -import { move, readFile } from 'fs-extra' +import { move, readFile, remove } from 'fs-extra' import { decode } from 'magnet-uri' import parseTorrent, { Instance } from 'parse-torrent' import { join } from 'path' +import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' +import { isResolvingToUnicastOnly } from '@server/helpers/dns' import { Hooks } from '@server/lib/plugins/hooks' import { ServerConfigManager } from '@server/lib/server-config-manager' import { setVideoTags } from '@server/lib/video' @@ -195,6 +197,13 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) }) } + if (!await hasUnicastURLsOnly(youtubeDLInfo)) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot use non unicast IP as targetUrl.' + }) + } + const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo) // Process video thumbnail from request.files @@ -432,6 +441,11 @@ async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl) for (const subtitle of subtitles) { + if (!await isVTTFileValid(subtitle.path)) { + await remove(subtitle.path) + continue + } + const videoCaption = new VideoCaptionModel({ videoId, language: subtitle.language, @@ -449,3 +463,16 @@ async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: logger.warn('Cannot get video subtitles.', { err }) } } + +async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) { + const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname) + const uniqHosts = new Set(hosts) + + for (const h of uniqHosts) { + if (await isResolvingToUnicastOnly(h) !== true) { + return false + } + } + + return true +} diff --git a/server/helpers/custom-validators/video-captions.ts b/server/helpers/custom-validators/video-captions.ts index 528edf60c..4cc7dcaf4 100644 --- a/server/helpers/custom-validators/video-captions.ts +++ b/server/helpers/custom-validators/video-captions.ts @@ -1,3 +1,5 @@ +import { getFileSize } from '@shared/extra-utils' +import { readFile } from 'fs-extra' import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initializers/constants' import { exists, isFileValid } from './misc' @@ -13,9 +15,20 @@ function isVideoCaptionFile (files: { [ fieldname: string ]: Express.Multer.File return isFileValid(files, videoCaptionTypesRegex, field, CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) } +async function isVTTFileValid (filePath: string) { + const size = await getFileSize(filePath) + + if (size > CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) return false + + const content = await readFile(filePath, 'utf8') + + return content?.startsWith('WEBVTT\n') +} + // --------------------------------------------------------------------------- export { isVideoCaptionFile, + isVTTFileValid, isVideoCaptionLanguageValid } diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts index 9746a7067..71572f292 100644 --- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts +++ b/server/helpers/youtube-dl/youtube-dl-info-builder.ts @@ -1,5 +1,6 @@ import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../../initializers/constants' import { peertubeTruncate } from '../core-utils' +import { isUrlValid } from '../custom-validators/activitypub/misc' type YoutubeDLInfo = { name?: string @@ -12,6 +13,8 @@ type YoutubeDLInfo = { thumbnailUrl?: string ext?: string originallyPublishedAt?: Date + + urls?: string[] } class YoutubeDLInfoBuilder { @@ -76,11 +79,57 @@ class YoutubeDLInfoBuilder { nsfw: this.isNSFW(obj), tags: this.getTags(obj.tags), thumbnailUrl: obj.thumbnail || undefined, + urls: this.buildAvailableUrl(obj), originallyPublishedAt: this.buildOriginallyPublishedAt(obj), ext: obj.ext } } + private buildAvailableUrl (obj: any) { + const urls: string[] = [] + + if (obj.url) urls.push(obj.url) + if (obj.urls) { + if (Array.isArray(obj.urls)) urls.push(...obj.urls) + else urls.push(obj.urls) + } + + const formats = Array.isArray(obj.formats) + ? obj.formats + : [] + + for (const format of formats) { + if (!format.url) continue + + urls.push(format.url) + } + + const thumbnails = Array.isArray(obj.thumbnails) + ? obj.thumbnails + : [] + + for (const thumbnail of thumbnails) { + if (!thumbnail.url) continue + + urls.push(thumbnail.url) + } + + if (obj.thumbnail) urls.push(obj.thumbnail) + + for (const subtitleKey of Object.keys(obj.subtitles || {})) { + const subtitles = obj.subtitles[subtitleKey] + if (!Array.isArray(subtitles)) continue + + for (const subtitle of subtitles) { + if (!subtitle.url) continue + + urls.push(subtitle.url) + } + } + + return urls.filter(u => u && isUrlValid(u)) + } + private titleTruncation (title: string) { return peertubeTruncate(title, { length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max, -- cgit v1.2.3