Import torrents with webtorrent

9 files changed, 116 insertions, 31 deletions

diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index c16a254d2..df151e79d 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -1,8 +1,16 @@
-import * as magnetUtil from 'magnet-uri'
 import * as express from 'express'
+import * as magnetUtil from 'magnet-uri'
+import 'multer'
 import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger'
 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
-import { CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE } from '../../../initializers'
+import {
+  CONFIG,
+  IMAGE_MIMETYPE_EXT,
+  PREVIEWS_SIZE,
+  sequelizeTypescript,
+  THUMBNAILS_SIZE,
+  TORRENT_MIMETYPE_EXT
+} from '../../../initializers'
 import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl'
 import { createReqFiles } from '../../../helpers/express-utils'
 import { logger } from '../../../helpers/logger'
@@ -18,16 +26,20 @@ import { isArray } from '../../../helpers/custom-validators/misc'
 import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
 import { VideoChannelModel } from '../../../models/video/video-channel'
 import * as Bluebird from 'bluebird'
+import * as parseTorrent from 'parse-torrent'
+import { readFileBufferPromise, renamePromise } from '../../../helpers/core-utils'
+import { getSecureTorrentName } from '../../../helpers/utils'
 
 const auditLogger = auditLoggerFactory('video-imports')
 const videoImportsRouter = express.Router()
 
 const reqVideoFileImport = createReqFiles(
-  [ 'thumbnailfile', 'previewfile' ],
-  IMAGE_MIMETYPE_EXT,
+  [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
+  Object.assign({}, TORRENT_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT),
   {
     thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
-    previewfile: CONFIG.STORAGE.PREVIEWS_DIR
+    previewfile: CONFIG.STORAGE.PREVIEWS_DIR,
+    torrentfile: CONFIG.STORAGE.TORRENTS_DIR
   }
 )
 
@@ -49,17 +61,37 @@ export {
 function addVideoImport (req: express.Request, res: express.Response) {
   if (req.body.targetUrl) return addYoutubeDLImport(req, res)
 
-  if (req.body.magnetUri) return addTorrentImport(req, res)
+  const file = req.files['torrentfile'][0]
+  if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
 }
 
-async function addTorrentImport (req: express.Request, res: express.Response) {
+async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
   const body: VideoImportCreate = req.body
-  const magnetUri = body.magnetUri
 
-  const parsed = magnetUtil.decode(magnetUri)
-  const magnetName = isArray(parsed.name) ? parsed.name[0] : parsed.name as string
+  let videoName: string
+  let torrentName: string
+  let magnetUri: string
+
+  if (torrentfile) {
+    torrentName = torrentfile.originalname
+
+    // Rename the torrent to a secured name
+    const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
+    await renamePromise(torrentfile.path, newTorrentPath)
+    torrentfile.path = newTorrentPath
+
+    const buf = await readFileBufferPromise(torrentfile.path)
+    const parsedTorrent = parseTorrent(buf)
+
+    videoName = isArray(parsedTorrent.name) ? parsedTorrent.name[ 0 ] : parsedTorrent.name as string
+  } else {
+    magnetUri = body.magnetUri
+
+    const parsed = magnetUtil.decode(magnetUri)
+    videoName = isArray(parsed.name) ? parsed.name[ 0 ] : parsed.name as string
+  }
 
-  const video = buildVideo(res.locals.videoChannel.id, body, { name: magnetName })
+  const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName })
 
   await processThumbnail(req, video)
   await processPreview(req, video)
@@ -67,13 +99,14 @@ async function addTorrentImport (req: express.Request, res: express.Response) {
   const tags = null
   const videoImportAttributes = {
     magnetUri,
+    torrentName,
     state: VideoImportState.PENDING
   }
   const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
 
   // Create job to import the video
   const payload = {
-    type: 'magnet-uri' as 'magnet-uri',
+    type: torrentfile ? 'torrent-file' as 'torrent-file' : 'magnet-uri' as 'magnet-uri',
     videoImportId: videoImport.id,
     magnetUri
   }
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 884206aad..25eb6454a 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -13,6 +13,7 @@ import * as pem from 'pem'
 import * as rimraf from 'rimraf'
 import { URL } from 'url'
 import { truncate } from 'lodash'
+import * as crypto from 'crypto'
 
 function sanitizeUrl (url: string) {
   const urlObject = new URL(url)
@@ -95,6 +96,10 @@ function peertubeTruncate (str: string, maxLength: number) {
   return truncate(str, options)
 }
 
+function sha256 (str: string) {
+  return crypto.createHash('sha256').update(str).digest('hex')
+}
+
 function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> {
   return function promisified (): Promise<A> {
     return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
@@ -165,6 +170,7 @@ export {
   sanitizeHost,
   buildPath,
   peertubeTruncate,
+  sha256,
 
   promisify0,
   promisify1,
diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts
index d8b9bfaff..4d6ab1fa4 100644
--- a/server/helpers/custom-validators/video-imports.ts
+++ b/server/helpers/custom-validators/video-imports.ts
@@ -1,10 +1,9 @@
 import 'express-validator'
 import 'multer'
 import * as validator from 'validator'
-import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers'
-import { exists } from './misc'
+import { CONSTRAINTS_FIELDS, TORRENT_MIMETYPE_EXT, VIDEO_IMPORT_STATES } from '../../initializers'
+import { exists, isFileValid } from './misc'
 import * as express from 'express'
-import { VideoChannelModel } from '../../models/video/video-channel'
 import { VideoImportModel } from '../../models/video/video-import'
 
 function isVideoImportTargetUrlValid (url: string) {
@@ -25,6 +24,12 @@ function isVideoImportStateValid (value: any) {
   return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined
 }
 
+const videoTorrentImportTypes = Object.keys(TORRENT_MIMETYPE_EXT).map(m => `(${m})`)
+const videoTorrentImportRegex = videoTorrentImportTypes.join('|')
+function isVideoImportTorrentFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) {
+  return isFileValid(files, videoTorrentImportRegex, 'torrentfile', CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max, true)
+}
+
 async function isVideoImportExist (id: number, res: express.Response) {
   const videoImport = await VideoImportModel.loadAndPopulateVideo(id)
 
@@ -45,5 +50,6 @@ async function isVideoImportExist (id: number, res: express.Response) {
 export {
   isVideoImportStateValid,
   isVideoImportTargetUrlValid,
-  isVideoImportExist
+  isVideoImportExist,
+  isVideoImportTorrentFile
 }
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index f4cc5547d..2ad87951e 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -6,11 +6,12 @@ import { CONFIG } from '../initializers'
 import { UserModel } from '../models/account/user'
 import { ActorModel } from '../models/activitypub/actor'
 import { ApplicationModel } from '../models/application/application'
-import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils'
+import { pseudoRandomBytesPromise, sha256, unlinkPromise } from './core-utils'
 import { logger } from './logger'
 import { isArray } from './custom-validators/misc'
 import * as crypto from "crypto"
 import { join } from "path"
+import { Instance as ParseTorrent } from 'parse-torrent'
 
 const isCidr = require('is-cidr')
 
@@ -183,13 +184,18 @@ async function getServerActor () {
   return Promise.resolve(serverActor)
 }
 
-function generateVideoTmpPath (id: string) {
-  const hash = crypto.createHash('sha256').update(id).digest('hex')
+function generateVideoTmpPath (target: string | ParseTorrent) {
+  const id = typeof target === 'string' ? target : target.infoHash
+
+  const hash = sha256(id)
   return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
 }
 
-type SortType = { sortModel: any, sortValue: string }
+function getSecureTorrentName (originalName: string) {
+  return sha256(originalName) + '.torrent'
+}
 
+type SortType = { sortModel: any, sortValue: string }
 
 // ---------------------------------------------------------------------------
 
@@ -199,6 +205,7 @@ export {
   generateRandomString,
   getFormattedObjects,
   isSignupAllowed,
+  getSecureTorrentName,
   isSignupAllowedForCurrentIP,
   computeResolutionsToTranscode,
   resetSequelizeInstance,
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index fce88a1f6..04b3ac71b 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -2,17 +2,22 @@ import { logger } from './logger'
 import { generateVideoTmpPath } from './utils'
 import * as WebTorrent from 'webtorrent'
 import { createWriteStream } from 'fs'
+import { Instance as ParseTorrent } from 'parse-torrent'
+import { CONFIG } from '../initializers'
+import { join } from 'path'
 
-function downloadWebTorrentVideo (target: string) {
-  const path = generateVideoTmpPath(target)
+function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: string }) {
+  const id = target.magnetUri || target.torrentName
 
-  logger.info('Importing torrent video %s', target)
+  const path = generateVideoTmpPath(id)
+  logger.info('Importing torrent video %s', id)
 
   return new Promise<string>((res, rej) => {
     const webtorrent = new WebTorrent()
 
-    const torrent = webtorrent.add(target, torrent => {
-      if (torrent.files.length !== 1) throw new Error('The number of files is not equal to 1 for ' + target)
+    const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName)
+    const torrent = webtorrent.add(torrentId, torrent => {
+      if (torrent.files.length !== 1) return rej(new Error('The number of files is not equal to 1 for ' + torrentId))
 
       const file = torrent.files[ 0 ]
       file.createReadStream().pipe(createWriteStream(path))
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 243d544ea..cf7cd3d74 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -273,6 +273,12 @@ const CONSTRAINTS_FIELDS = {
   VIDEO_IMPORTS: {
     URL: { min: 3, max: 2000 }, // Length
     TORRENT_NAME: { min: 3, max: 255 }, // Length
+    TORRENT_FILE: {
+      EXTNAME: [ '.torrent' ],
+      FILE_SIZE: {
+        max: 1024 * 200 // 200 KB
+      }
+    }
   },
   VIDEOS: {
     NAME: { min: 3, max: 120 }, // Length
@@ -417,6 +423,10 @@ const VIDEO_CAPTIONS_MIMETYPE_EXT = {
   'application/x-subrip': '.srt'
 }
 
+const TORRENT_MIMETYPE_EXT = {
+  'application/x-bittorrent': '.torrent'
+}
+
 // ---------------------------------------------------------------------------
 
 const SERVER_ACTOR_NAME = 'peertube'
@@ -596,6 +606,7 @@ export {
   FEEDS,
   JOB_TTL,
   NSFW_POLICY_TYPES,
+  TORRENT_MIMETYPE_EXT,
   STATIC_MAX_AGE,
   STATIC_PATHS,
   ACTIVITY_PUB,
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index c457b71fc..fd61aecad 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -14,6 +14,7 @@ import { JobQueue } from '../index'
 import { federateVideoIfNeeded } from '../../activitypub'
 import { VideoModel } from '../../../models/video/video'
 import { downloadWebTorrentVideo } from '../../../helpers/webtorrent'
+import { getSecureTorrentName } from '../../../helpers/utils'
 
 type VideoImportYoutubeDLPayload = {
   type: 'youtube-dl'
@@ -25,7 +26,7 @@ type VideoImportYoutubeDLPayload = {
 }
 
 type VideoImportTorrentPayload = {
-  type: 'magnet-uri'
+  type: 'magnet-uri' | 'torrent-file'
   videoImportId: number
 }
 
@@ -35,7 +36,7 @@ async function processVideoImport (job: Bull.Job) {
   const payload = job.data as VideoImportPayload
 
   if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload)
-  if (payload.type === 'magnet-uri') return processTorrentImport(job, payload)
+  if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, payload)
 }
 
 // ---------------------------------------------------------------------------
@@ -50,6 +51,7 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP
   logger.info('Processing torrent video import in job %d.', job.id)
 
   const videoImport = await getVideoImportOrDie(payload.videoImportId)
+
   const options = {
     videoImportId: payload.videoImportId,
 
@@ -59,7 +61,11 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP
     generateThumbnail: true,
     generatePreview: true
   }
-  return processFile(() => downloadWebTorrentVideo(videoImport.magnetUri), videoImport, options)
+  const target = {
+    torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined,
+    magnetUri: videoImport.magnetUri
+  }
+  return processFile(() => downloadWebTorrentVideo(target), videoImport, options)
 }
 
 async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) {
diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts
index 8ec9373fb..c03cf2e4d 100644
--- a/server/middlewares/validators/video-imports.ts
+++ b/server/middlewares/validators/video-imports.ts
@@ -4,10 +4,11 @@ import { isIdValid } from '../../helpers/custom-validators/misc'
 import { logger } from '../../helpers/logger'
 import { areValidationErrors } from './utils'
 import { getCommonVideoAttributes } from './videos'
-import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
+import { isVideoImportTargetUrlValid, isVideoImportTorrentFile } from '../../helpers/custom-validators/video-imports'
 import { cleanUpReqFiles } from '../../helpers/utils'
 import { isVideoChannelOfAccountExist, isVideoMagnetUriValid, isVideoNameValid } from '../../helpers/custom-validators/videos'
 import { CONFIG } from '../../initializers/constants'
+import { CONSTRAINTS_FIELDS } from '../../initializers'
 
 const videoImportAddValidator = getCommonVideoAttributes().concat([
   body('channelId')
@@ -19,6 +20,11 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([
   body('magnetUri')
     .optional()
     .custom(isVideoMagnetUriValid).withMessage('Should have a valid video magnet URI'),
+  body('torrentfile')
+    .custom((value, { req }) => isVideoImportTorrentFile(req.files)).withMessage(
+    'This torrent file is not supported or too large. Please, make sure it is of the following type: '
+    + CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.EXTNAME.join(', ')
+  ),
   body('name')
     .optional()
     .custom(isVideoNameValid).withMessage('Should have a valid name'),
@@ -40,11 +46,12 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([
     if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
 
     // Check we have at least 1 required param
-    if (!req.body.targetUrl && !req.body.magnetUri) {
+    const file = req.files['torrentfile'][0]
+    if (!req.body.targetUrl && !req.body.magnetUri && !file) {
       cleanUpReqFiles(req)
 
       return res.status(400)
-        .json({ error: 'Should have a magnetUri or a targetUrl.' })
+        .json({ error: 'Should have a magnetUri or a targetUrl or a torrent file.' })
         .end()
     }
 
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts
index 55fca28b8..d6c02e5ac 100644
--- a/server/models/video/video-import.ts
+++ b/server/models/video/video-import.ts
@@ -171,7 +171,11 @@ export class VideoImportModel extends Model<VideoImportModel> {
 
     return {
       id: this.id,
+
       targetUrl: this.targetUrl,
+      magnetUri: this.magnetUri,
+      torrentName: this.torrentName,
+
       state: {
         id: this.state,
         label: VideoImportModel.getStateLabel(this.state)