diff options
author | Chocobozzz <me@florianbigard.com> | 2018-08-07 09:54:36 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2018-08-08 09:30:31 +0200 |
commit | 990b6a0b0c4fbebc165e5cf7cec8fbc1cbaa6c66 (patch) | |
tree | 8aaa0638798bfa14813f4d6ed5247242313b9ce6 | |
parent | ce33919c24e7402d92d81f3cd8e545df52d98240 (diff) | |
download | PeerTube-990b6a0b0c4fbebc165e5cf7cec8fbc1cbaa6c66.tar.gz PeerTube-990b6a0b0c4fbebc165e5cf7cec8fbc1cbaa6c66.tar.zst PeerTube-990b6a0b0c4fbebc165e5cf7cec8fbc1cbaa6c66.zip |
Import torrents with webtorrent
16 files changed, 169 insertions, 41 deletions
diff --git a/client/src/app/+my-account/my-account-video-imports/my-account-video-imports.component.html b/client/src/app/+my-account/my-account-video-imports/my-account-video-imports.component.html index 00b2d7cb0..b2b6c3d60 100644 --- a/client/src/app/+my-account/my-account-video-imports/my-account-video-imports.component.html +++ b/client/src/app/+my-account/my-account-video-imports/my-account-video-imports.component.html | |||
@@ -5,7 +5,7 @@ | |||
5 | <ng-template pTemplate="header"> | 5 | <ng-template pTemplate="header"> |
6 | <tr> | 6 | <tr> |
7 | <th style="width: 40px;"></th> | 7 | <th style="width: 40px;"></th> |
8 | <th i18n>URL</th> | 8 | <th i18n>Target</th> |
9 | <th i18n>Video</th> | 9 | <th i18n>Video</th> |
10 | <th i18n style="width: 150px">State</th> | 10 | <th i18n style="width: 150px">State</th> |
11 | <th i18n pSortableColumn="createdAt">Created <p-sortIcon field="createdAt"></p-sortIcon></th> | 11 | <th i18n pSortableColumn="createdAt">Created <p-sortIcon field="createdAt"></p-sortIcon></th> |
@@ -22,7 +22,10 @@ | |||
22 | </td> | 22 | </td> |
23 | 23 | ||
24 | <td> | 24 | <td> |
25 | <a [href]="videoImport.targetUrl" target="_blank" rel="noopener noreferrer">{{ videoImport.targetUrl }}</a> | 25 | <a *ngIf="videoImport.targetUrl; else torrent" [href]="videoImport.targetUrl" target="_blank" rel="noopener noreferrer">{{ videoImport.targetUrl }}</a> |
26 | <ng-template #torrent> | ||
27 | <span [title]="videoImport.torrentName || videoImport.magnetUri">{{ videoImport.torrentName || videoImport.magnetUri }}</span> | ||
28 | </ng-template> | ||
26 | </td> | 29 | </td> |
27 | 30 | ||
28 | <td *ngIf="isVideoImportPending(videoImport)"> | 31 | <td *ngIf="isVideoImportPending(videoImport)"> |
diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html index 409e4de5e..2f0c9abb5 100644 --- a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html +++ b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html | |||
@@ -2,8 +2,16 @@ | |||
2 | <div class="import-video-torrent"> | 2 | <div class="import-video-torrent"> |
3 | <div class="icon icon-upload"></div> | 3 | <div class="icon icon-upload"></div> |
4 | 4 | ||
5 | <div class="form-group"> | 5 | <div class="button-file"> |
6 | <label i18n for="magnetUri">Magnet URI</label> | 6 | <span i18n>Select the torrent to import</span> |
7 | <input #torrentfileInput type="file" name="torrentfile" id="torrentfile" accept=".torrent" (change)="fileChange()" /> | ||
8 | </div> | ||
9 | <span class="button-file-extension">(.torrent)</span> | ||
10 | |||
11 | <div class="torrent-or-magnet">Or</div> | ||
12 | |||
13 | <div class="form-group form-group-magnet-uri"> | ||
14 | <label i18n for="magnetUri">Paste magnet URI</label> | ||
7 | <my-help | 15 | <my-help |
8 | helpType="custom" i18n-customHtml | 16 | helpType="custom" i18n-customHtml |
9 | customHtml="You can import any torrent file that points to a mp4 file. You should make sure you have diffusion rights over the content it points to, otherwise it could cause legal trouble to yourself and your instance." | 17 | customHtml="You can import any torrent file that points to a mp4 file. You should make sure you have diffusion rights over the content it points to, otherwise it could cause legal trouble to yourself and your instance." |
diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.scss b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.scss index 1ef5adc25..262b0b68e 100644 --- a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.scss +++ b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.scss | |||
@@ -20,6 +20,26 @@ $width-size: 190px; | |||
20 | background-image: url('../../../../assets/images/video/upload.svg'); | 20 | background-image: url('../../../../assets/images/video/upload.svg'); |
21 | } | 21 | } |
22 | 22 | ||
23 | .button-file { | ||
24 | @include peertube-button-file(auto); | ||
25 | |||
26 | min-width: 190px; | ||
27 | } | ||
28 | |||
29 | .button-file-extension { | ||
30 | display: block; | ||
31 | font-size: 12px; | ||
32 | margin-top: 5px; | ||
33 | } | ||
34 | |||
35 | .torrent-or-magnet { | ||
36 | margin: 10px 0; | ||
37 | } | ||
38 | |||
39 | .form-group-magnet-uri { | ||
40 | margin-bottom: 40px; | ||
41 | } | ||
42 | |||
23 | input[type=text] { | 43 | input[type=text] { |
24 | @include peertube-input-text($width-size); | 44 | @include peertube-input-text($width-size); |
25 | display: block; | 45 | display: block; |
diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.ts b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.ts index 330c37718..9623c2bf4 100644 --- a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.ts +++ b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { Component, EventEmitter, OnInit, Output } from '@angular/core' | 1 | import { Component, EventEmitter, OnInit, Output, ViewChild } from '@angular/core' |
2 | import { Router } from '@angular/router' | 2 | import { Router } from '@angular/router' |
3 | import { NotificationsService } from 'angular2-notifications' | 3 | import { NotificationsService } from 'angular2-notifications' |
4 | import { VideoPrivacy, VideoUpdate } from '../../../../../../shared/models/videos' | 4 | import { VideoPrivacy, VideoUpdate } from '../../../../../../shared/models/videos' |
@@ -23,6 +23,7 @@ import { VideoImportService } from '@app/shared/video-import' | |||
23 | }) | 23 | }) |
24 | export class VideoImportTorrentComponent extends VideoSend implements OnInit, CanComponentDeactivate { | 24 | export class VideoImportTorrentComponent extends VideoSend implements OnInit, CanComponentDeactivate { |
25 | @Output() firstStepDone = new EventEmitter<string>() | 25 | @Output() firstStepDone = new EventEmitter<string>() |
26 | @ViewChild('torrentfileInput') torrentfileInput | ||
26 | 27 | ||
27 | videoFileName: string | 28 | videoFileName: string |
28 | magnetUri = '' | 29 | magnetUri = '' |
@@ -33,7 +34,7 @@ export class VideoImportTorrentComponent extends VideoSend implements OnInit, Ca | |||
33 | 34 | ||
34 | video: VideoEdit | 35 | video: VideoEdit |
35 | 36 | ||
36 | protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PRIVATE | 37 | protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PUBLIC |
37 | 38 | ||
38 | constructor ( | 39 | constructor ( |
39 | protected formValidatorService: FormValidatorService, | 40 | protected formValidatorService: FormValidatorService, |
@@ -62,7 +63,14 @@ export class VideoImportTorrentComponent extends VideoSend implements OnInit, Ca | |||
62 | return !!this.magnetUri | 63 | return !!this.magnetUri |
63 | } | 64 | } |
64 | 65 | ||
65 | importVideo () { | 66 | fileChange () { |
67 | const torrentfile = this.torrentfileInput.nativeElement.files[0] as File | ||
68 | if (!torrentfile) return | ||
69 | |||
70 | this.importVideo(torrentfile) | ||
71 | } | ||
72 | |||
73 | importVideo (torrentfile?: Blob) { | ||
66 | this.isImportingVideo = true | 74 | this.isImportingVideo = true |
67 | 75 | ||
68 | const videoUpdate: VideoUpdate = { | 76 | const videoUpdate: VideoUpdate = { |
@@ -74,7 +82,7 @@ export class VideoImportTorrentComponent extends VideoSend implements OnInit, Ca | |||
74 | 82 | ||
75 | this.loadingBar.start() | 83 | this.loadingBar.start() |
76 | 84 | ||
77 | this.videoImportService.importVideoTorrent(this.magnetUri, videoUpdate).subscribe( | 85 | this.videoImportService.importVideoTorrent(torrentfile || this.magnetUri, videoUpdate).subscribe( |
78 | res => { | 86 | res => { |
79 | this.loadingBar.complete() | 87 | this.loadingBar.complete() |
80 | this.firstStepDone.emit(res.video.name) | 88 | this.firstStepDone.emit(res.video.name) |
diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts index 842ede732..97b402bfe 100644 --- a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts +++ b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts | |||
@@ -33,7 +33,7 @@ export class VideoImportUrlComponent extends VideoSend implements OnInit, CanCom | |||
33 | 33 | ||
34 | video: VideoEdit | 34 | video: VideoEdit |
35 | 35 | ||
36 | protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PRIVATE | 36 | protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PUBLIC |
37 | 37 | ||
38 | constructor ( | 38 | constructor ( |
39 | protected formValidatorService: FormValidatorService, | 39 | protected formValidatorService: FormValidatorService, |
diff --git a/client/src/app/videos/+video-edit/video-add.component.scss b/client/src/app/videos/+video-edit/video-add.component.scss index 02ee295f9..443361f50 100644 --- a/client/src/app/videos/+video-edit/video-add.component.scss +++ b/client/src/app/videos/+video-edit/video-add.component.scss | |||
@@ -49,7 +49,7 @@ $background-color: #F7F7F7; | |||
49 | background-color: $background-color; | 49 | background-color: $background-color; |
50 | border-radius: 3px; | 50 | border-radius: 3px; |
51 | width: 100%; | 51 | width: 100%; |
52 | height: 440px; | 52 | min-height: 440px; |
53 | display: flex; | 53 | display: flex; |
54 | justify-content: center; | 54 | justify-content: center; |
55 | align-items: center; | 55 | align-items: center; |
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index c16a254d2..df151e79d 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts | |||
@@ -1,8 +1,16 @@ | |||
1 | import * as magnetUtil from 'magnet-uri' | ||
2 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import * as magnetUtil from 'magnet-uri' | ||
3 | import 'multer' | ||
3 | import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger' | 4 | import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger' |
4 | import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' | 5 | import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' |
5 | import { CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE } from '../../../initializers' | 6 | import { |
7 | CONFIG, | ||
8 | IMAGE_MIMETYPE_EXT, | ||
9 | PREVIEWS_SIZE, | ||
10 | sequelizeTypescript, | ||
11 | THUMBNAILS_SIZE, | ||
12 | TORRENT_MIMETYPE_EXT | ||
13 | } from '../../../initializers' | ||
6 | import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl' | 14 | import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl' |
7 | import { createReqFiles } from '../../../helpers/express-utils' | 15 | import { createReqFiles } from '../../../helpers/express-utils' |
8 | import { logger } from '../../../helpers/logger' | 16 | import { logger } from '../../../helpers/logger' |
@@ -18,16 +26,20 @@ import { isArray } from '../../../helpers/custom-validators/misc' | |||
18 | import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' | 26 | import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' |
19 | import { VideoChannelModel } from '../../../models/video/video-channel' | 27 | import { VideoChannelModel } from '../../../models/video/video-channel' |
20 | import * as Bluebird from 'bluebird' | 28 | import * as Bluebird from 'bluebird' |
29 | import * as parseTorrent from 'parse-torrent' | ||
30 | import { readFileBufferPromise, renamePromise } from '../../../helpers/core-utils' | ||
31 | import { getSecureTorrentName } from '../../../helpers/utils' | ||
21 | 32 | ||
22 | const auditLogger = auditLoggerFactory('video-imports') | 33 | const auditLogger = auditLoggerFactory('video-imports') |
23 | const videoImportsRouter = express.Router() | 34 | const videoImportsRouter = express.Router() |
24 | 35 | ||
25 | const reqVideoFileImport = createReqFiles( | 36 | const reqVideoFileImport = createReqFiles( |
26 | [ 'thumbnailfile', 'previewfile' ], | 37 | [ 'thumbnailfile', 'previewfile', 'torrentfile' ], |
27 | IMAGE_MIMETYPE_EXT, | 38 | Object.assign({}, TORRENT_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT), |
28 | { | 39 | { |
29 | thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR, | 40 | thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR, |
30 | previewfile: CONFIG.STORAGE.PREVIEWS_DIR | 41 | previewfile: CONFIG.STORAGE.PREVIEWS_DIR, |
42 | torrentfile: CONFIG.STORAGE.TORRENTS_DIR | ||
31 | } | 43 | } |
32 | ) | 44 | ) |
33 | 45 | ||
@@ -49,17 +61,37 @@ export { | |||
49 | function addVideoImport (req: express.Request, res: express.Response) { | 61 | function addVideoImport (req: express.Request, res: express.Response) { |
50 | if (req.body.targetUrl) return addYoutubeDLImport(req, res) | 62 | if (req.body.targetUrl) return addYoutubeDLImport(req, res) |
51 | 63 | ||
52 | if (req.body.magnetUri) return addTorrentImport(req, res) | 64 | const file = req.files['torrentfile'][0] |
65 | if (req.body.magnetUri || file) return addTorrentImport(req, res, file) | ||
53 | } | 66 | } |
54 | 67 | ||
55 | async function addTorrentImport (req: express.Request, res: express.Response) { | 68 | async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { |
56 | const body: VideoImportCreate = req.body | 69 | const body: VideoImportCreate = req.body |
57 | const magnetUri = body.magnetUri | ||
58 | 70 | ||
59 | const parsed = magnetUtil.decode(magnetUri) | 71 | let videoName: string |
60 | const magnetName = isArray(parsed.name) ? parsed.name[0] : parsed.name as string | 72 | let torrentName: string |
73 | let magnetUri: string | ||
74 | |||
75 | if (torrentfile) { | ||
76 | torrentName = torrentfile.originalname | ||
77 | |||
78 | // Rename the torrent to a secured name | ||
79 | const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName)) | ||
80 | await renamePromise(torrentfile.path, newTorrentPath) | ||
81 | torrentfile.path = newTorrentPath | ||
82 | |||
83 | const buf = await readFileBufferPromise(torrentfile.path) | ||
84 | const parsedTorrent = parseTorrent(buf) | ||
85 | |||
86 | videoName = isArray(parsedTorrent.name) ? parsedTorrent.name[ 0 ] : parsedTorrent.name as string | ||
87 | } else { | ||
88 | magnetUri = body.magnetUri | ||
89 | |||
90 | const parsed = magnetUtil.decode(magnetUri) | ||
91 | videoName = isArray(parsed.name) ? parsed.name[ 0 ] : parsed.name as string | ||
92 | } | ||
61 | 93 | ||
62 | const video = buildVideo(res.locals.videoChannel.id, body, { name: magnetName }) | 94 | const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName }) |
63 | 95 | ||
64 | await processThumbnail(req, video) | 96 | await processThumbnail(req, video) |
65 | await processPreview(req, video) | 97 | await processPreview(req, video) |
@@ -67,13 +99,14 @@ async function addTorrentImport (req: express.Request, res: express.Response) { | |||
67 | const tags = null | 99 | const tags = null |
68 | const videoImportAttributes = { | 100 | const videoImportAttributes = { |
69 | magnetUri, | 101 | magnetUri, |
102 | torrentName, | ||
70 | state: VideoImportState.PENDING | 103 | state: VideoImportState.PENDING |
71 | } | 104 | } |
72 | const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes) | 105 | const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes) |
73 | 106 | ||
74 | // Create job to import the video | 107 | // Create job to import the video |
75 | const payload = { | 108 | const payload = { |
76 | type: 'magnet-uri' as 'magnet-uri', | 109 | type: torrentfile ? 'torrent-file' as 'torrent-file' : 'magnet-uri' as 'magnet-uri', |
77 | videoImportId: videoImport.id, | 110 | videoImportId: videoImport.id, |
78 | magnetUri | 111 | magnetUri |
79 | } | 112 | } |
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 884206aad..25eb6454a 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts | |||
@@ -13,6 +13,7 @@ import * as pem from 'pem' | |||
13 | import * as rimraf from 'rimraf' | 13 | import * as rimraf from 'rimraf' |
14 | import { URL } from 'url' | 14 | import { URL } from 'url' |
15 | import { truncate } from 'lodash' | 15 | import { truncate } from 'lodash' |
16 | import * as crypto from 'crypto' | ||
16 | 17 | ||
17 | function sanitizeUrl (url: string) { | 18 | function sanitizeUrl (url: string) { |
18 | const urlObject = new URL(url) | 19 | const urlObject = new URL(url) |
@@ -95,6 +96,10 @@ function peertubeTruncate (str: string, maxLength: number) { | |||
95 | return truncate(str, options) | 96 | return truncate(str, options) |
96 | } | 97 | } |
97 | 98 | ||
99 | function sha256 (str: string) { | ||
100 | return crypto.createHash('sha256').update(str).digest('hex') | ||
101 | } | ||
102 | |||
98 | function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> { | 103 | function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> { |
99 | return function promisified (): Promise<A> { | 104 | return function promisified (): Promise<A> { |
100 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { | 105 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { |
@@ -165,6 +170,7 @@ export { | |||
165 | sanitizeHost, | 170 | sanitizeHost, |
166 | buildPath, | 171 | buildPath, |
167 | peertubeTruncate, | 172 | peertubeTruncate, |
173 | sha256, | ||
168 | 174 | ||
169 | promisify0, | 175 | promisify0, |
170 | promisify1, | 176 | promisify1, |
diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts index d8b9bfaff..4d6ab1fa4 100644 --- a/server/helpers/custom-validators/video-imports.ts +++ b/server/helpers/custom-validators/video-imports.ts | |||
@@ -1,10 +1,9 @@ | |||
1 | import 'express-validator' | 1 | import 'express-validator' |
2 | import 'multer' | 2 | import 'multer' |
3 | import * as validator from 'validator' | 3 | import * as validator from 'validator' |
4 | import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers' | 4 | import { CONSTRAINTS_FIELDS, TORRENT_MIMETYPE_EXT, VIDEO_IMPORT_STATES } from '../../initializers' |
5 | import { exists } from './misc' | 5 | import { exists, isFileValid } from './misc' |
6 | import * as express from 'express' | 6 | import * as express from 'express' |
7 | import { VideoChannelModel } from '../../models/video/video-channel' | ||
8 | import { VideoImportModel } from '../../models/video/video-import' | 7 | import { VideoImportModel } from '../../models/video/video-import' |
9 | 8 | ||
10 | function isVideoImportTargetUrlValid (url: string) { | 9 | function isVideoImportTargetUrlValid (url: string) { |
@@ -25,6 +24,12 @@ function isVideoImportStateValid (value: any) { | |||
25 | return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined | 24 | return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined |
26 | } | 25 | } |
27 | 26 | ||
27 | const videoTorrentImportTypes = Object.keys(TORRENT_MIMETYPE_EXT).map(m => `(${m})`) | ||
28 | const videoTorrentImportRegex = videoTorrentImportTypes.join('|') | ||
29 | function isVideoImportTorrentFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { | ||
30 | return isFileValid(files, videoTorrentImportRegex, 'torrentfile', CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max, true) | ||
31 | } | ||
32 | |||
28 | async function isVideoImportExist (id: number, res: express.Response) { | 33 | async function isVideoImportExist (id: number, res: express.Response) { |
29 | const videoImport = await VideoImportModel.loadAndPopulateVideo(id) | 34 | const videoImport = await VideoImportModel.loadAndPopulateVideo(id) |
30 | 35 | ||
@@ -45,5 +50,6 @@ async function isVideoImportExist (id: number, res: express.Response) { | |||
45 | export { | 50 | export { |
46 | isVideoImportStateValid, | 51 | isVideoImportStateValid, |
47 | isVideoImportTargetUrlValid, | 52 | isVideoImportTargetUrlValid, |
48 | isVideoImportExist | 53 | isVideoImportExist, |
54 | isVideoImportTorrentFile | ||
49 | } | 55 | } |
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index f4cc5547d..2ad87951e 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts | |||
@@ -6,11 +6,12 @@ import { CONFIG } from '../initializers' | |||
6 | import { UserModel } from '../models/account/user' | 6 | import { UserModel } from '../models/account/user' |
7 | import { ActorModel } from '../models/activitypub/actor' | 7 | import { ActorModel } from '../models/activitypub/actor' |
8 | import { ApplicationModel } from '../models/application/application' | 8 | import { ApplicationModel } from '../models/application/application' |
9 | import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils' | 9 | import { pseudoRandomBytesPromise, sha256, unlinkPromise } from './core-utils' |
10 | import { logger } from './logger' | 10 | import { logger } from './logger' |
11 | import { isArray } from './custom-validators/misc' | 11 | import { isArray } from './custom-validators/misc' |
12 | import * as crypto from "crypto" | 12 | import * as crypto from "crypto" |
13 | import { join } from "path" | 13 | import { join } from "path" |
14 | import { Instance as ParseTorrent } from 'parse-torrent' | ||
14 | 15 | ||
15 | const isCidr = require('is-cidr') | 16 | const isCidr = require('is-cidr') |
16 | 17 | ||
@@ -183,13 +184,18 @@ async function getServerActor () { | |||
183 | return Promise.resolve(serverActor) | 184 | return Promise.resolve(serverActor) |
184 | } | 185 | } |
185 | 186 | ||
186 | function generateVideoTmpPath (id: string) { | 187 | function generateVideoTmpPath (target: string | ParseTorrent) { |
187 | const hash = crypto.createHash('sha256').update(id).digest('hex') | 188 | const id = typeof target === 'string' ? target : target.infoHash |
189 | |||
190 | const hash = sha256(id) | ||
188 | return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4') | 191 | return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4') |
189 | } | 192 | } |
190 | 193 | ||
191 | type SortType = { sortModel: any, sortValue: string } | 194 | function getSecureTorrentName (originalName: string) { |
195 | return sha256(originalName) + '.torrent' | ||
196 | } | ||
192 | 197 | ||
198 | type SortType = { sortModel: any, sortValue: string } | ||
193 | 199 | ||
194 | // --------------------------------------------------------------------------- | 200 | // --------------------------------------------------------------------------- |
195 | 201 | ||
@@ -199,6 +205,7 @@ export { | |||
199 | generateRandomString, | 205 | generateRandomString, |
200 | getFormattedObjects, | 206 | getFormattedObjects, |
201 | isSignupAllowed, | 207 | isSignupAllowed, |
208 | getSecureTorrentName, | ||
202 | isSignupAllowedForCurrentIP, | 209 | isSignupAllowedForCurrentIP, |
203 | computeResolutionsToTranscode, | 210 | computeResolutionsToTranscode, |
204 | resetSequelizeInstance, | 211 | resetSequelizeInstance, |
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index fce88a1f6..04b3ac71b 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts | |||
@@ -2,17 +2,22 @@ import { logger } from './logger' | |||
2 | import { generateVideoTmpPath } from './utils' | 2 | import { generateVideoTmpPath } from './utils' |
3 | import * as WebTorrent from 'webtorrent' | 3 | import * as WebTorrent from 'webtorrent' |
4 | import { createWriteStream } from 'fs' | 4 | import { createWriteStream } from 'fs' |
5 | import { Instance as ParseTorrent } from 'parse-torrent' | ||
6 | import { CONFIG } from '../initializers' | ||
7 | import { join } from 'path' | ||
5 | 8 | ||
6 | function downloadWebTorrentVideo (target: string) { | 9 | function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: string }) { |
7 | const path = generateVideoTmpPath(target) | 10 | const id = target.magnetUri || target.torrentName |
8 | 11 | ||
9 | logger.info('Importing torrent video %s', target) | 12 | const path = generateVideoTmpPath(id) |
13 | logger.info('Importing torrent video %s', id) | ||
10 | 14 | ||
11 | return new Promise<string>((res, rej) => { | 15 | return new Promise<string>((res, rej) => { |
12 | const webtorrent = new WebTorrent() | 16 | const webtorrent = new WebTorrent() |
13 | 17 | ||
14 | const torrent = webtorrent.add(target, torrent => { | 18 | const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) |
15 | if (torrent.files.length !== 1) throw new Error('The number of files is not equal to 1 for ' + target) | 19 | const torrent = webtorrent.add(torrentId, torrent => { |
20 | if (torrent.files.length !== 1) return rej(new Error('The number of files is not equal to 1 for ' + torrentId)) | ||
16 | 21 | ||
17 | const file = torrent.files[ 0 ] | 22 | const file = torrent.files[ 0 ] |
18 | file.createReadStream().pipe(createWriteStream(path)) | 23 | file.createReadStream().pipe(createWriteStream(path)) |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 243d544ea..cf7cd3d74 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -273,6 +273,12 @@ const CONSTRAINTS_FIELDS = { | |||
273 | VIDEO_IMPORTS: { | 273 | VIDEO_IMPORTS: { |
274 | URL: { min: 3, max: 2000 }, // Length | 274 | URL: { min: 3, max: 2000 }, // Length |
275 | TORRENT_NAME: { min: 3, max: 255 }, // Length | 275 | TORRENT_NAME: { min: 3, max: 255 }, // Length |
276 | TORRENT_FILE: { | ||
277 | EXTNAME: [ '.torrent' ], | ||
278 | FILE_SIZE: { | ||
279 | max: 1024 * 200 // 200 KB | ||
280 | } | ||
281 | } | ||
276 | }, | 282 | }, |
277 | VIDEOS: { | 283 | VIDEOS: { |
278 | NAME: { min: 3, max: 120 }, // Length | 284 | NAME: { min: 3, max: 120 }, // Length |
@@ -417,6 +423,10 @@ const VIDEO_CAPTIONS_MIMETYPE_EXT = { | |||
417 | 'application/x-subrip': '.srt' | 423 | 'application/x-subrip': '.srt' |
418 | } | 424 | } |
419 | 425 | ||
426 | const TORRENT_MIMETYPE_EXT = { | ||
427 | 'application/x-bittorrent': '.torrent' | ||
428 | } | ||
429 | |||
420 | // --------------------------------------------------------------------------- | 430 | // --------------------------------------------------------------------------- |
421 | 431 | ||
422 | const SERVER_ACTOR_NAME = 'peertube' | 432 | const SERVER_ACTOR_NAME = 'peertube' |
@@ -596,6 +606,7 @@ export { | |||
596 | FEEDS, | 606 | FEEDS, |
597 | JOB_TTL, | 607 | JOB_TTL, |
598 | NSFW_POLICY_TYPES, | 608 | NSFW_POLICY_TYPES, |
609 | TORRENT_MIMETYPE_EXT, | ||
599 | STATIC_MAX_AGE, | 610 | STATIC_MAX_AGE, |
600 | STATIC_PATHS, | 611 | STATIC_PATHS, |
601 | ACTIVITY_PUB, | 612 | ACTIVITY_PUB, |
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index c457b71fc..fd61aecad 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -14,6 +14,7 @@ import { JobQueue } from '../index' | |||
14 | import { federateVideoIfNeeded } from '../../activitypub' | 14 | import { federateVideoIfNeeded } from '../../activitypub' |
15 | import { VideoModel } from '../../../models/video/video' | 15 | import { VideoModel } from '../../../models/video/video' |
16 | import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' | 16 | import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' |
17 | import { getSecureTorrentName } from '../../../helpers/utils' | ||
17 | 18 | ||
18 | type VideoImportYoutubeDLPayload = { | 19 | type VideoImportYoutubeDLPayload = { |
19 | type: 'youtube-dl' | 20 | type: 'youtube-dl' |
@@ -25,7 +26,7 @@ type VideoImportYoutubeDLPayload = { | |||
25 | } | 26 | } |
26 | 27 | ||
27 | type VideoImportTorrentPayload = { | 28 | type VideoImportTorrentPayload = { |
28 | type: 'magnet-uri' | 29 | type: 'magnet-uri' | 'torrent-file' |
29 | videoImportId: number | 30 | videoImportId: number |
30 | } | 31 | } |
31 | 32 | ||
@@ -35,7 +36,7 @@ async function processVideoImport (job: Bull.Job) { | |||
35 | const payload = job.data as VideoImportPayload | 36 | const payload = job.data as VideoImportPayload |
36 | 37 | ||
37 | if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload) | 38 | if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload) |
38 | if (payload.type === 'magnet-uri') return processTorrentImport(job, payload) | 39 | if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, payload) |
39 | } | 40 | } |
40 | 41 | ||
41 | // --------------------------------------------------------------------------- | 42 | // --------------------------------------------------------------------------- |
@@ -50,6 +51,7 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP | |||
50 | logger.info('Processing torrent video import in job %d.', job.id) | 51 | logger.info('Processing torrent video import in job %d.', job.id) |
51 | 52 | ||
52 | const videoImport = await getVideoImportOrDie(payload.videoImportId) | 53 | const videoImport = await getVideoImportOrDie(payload.videoImportId) |
54 | |||
53 | const options = { | 55 | const options = { |
54 | videoImportId: payload.videoImportId, | 56 | videoImportId: payload.videoImportId, |
55 | 57 | ||
@@ -59,7 +61,11 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP | |||
59 | generateThumbnail: true, | 61 | generateThumbnail: true, |
60 | generatePreview: true | 62 | generatePreview: true |
61 | } | 63 | } |
62 | return processFile(() => downloadWebTorrentVideo(videoImport.magnetUri), videoImport, options) | 64 | const target = { |
65 | torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, | ||
66 | magnetUri: videoImport.magnetUri | ||
67 | } | ||
68 | return processFile(() => downloadWebTorrentVideo(target), videoImport, options) | ||
63 | } | 69 | } |
64 | 70 | ||
65 | async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) { | 71 | async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) { |
diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts index 8ec9373fb..c03cf2e4d 100644 --- a/server/middlewares/validators/video-imports.ts +++ b/server/middlewares/validators/video-imports.ts | |||
@@ -4,10 +4,11 @@ import { isIdValid } from '../../helpers/custom-validators/misc' | |||
4 | import { logger } from '../../helpers/logger' | 4 | import { logger } from '../../helpers/logger' |
5 | import { areValidationErrors } from './utils' | 5 | import { areValidationErrors } from './utils' |
6 | import { getCommonVideoAttributes } from './videos' | 6 | import { getCommonVideoAttributes } from './videos' |
7 | import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' | 7 | import { isVideoImportTargetUrlValid, isVideoImportTorrentFile } from '../../helpers/custom-validators/video-imports' |
8 | import { cleanUpReqFiles } from '../../helpers/utils' | 8 | import { cleanUpReqFiles } from '../../helpers/utils' |
9 | import { isVideoChannelOfAccountExist, isVideoMagnetUriValid, isVideoNameValid } from '../../helpers/custom-validators/videos' | 9 | import { isVideoChannelOfAccountExist, isVideoMagnetUriValid, isVideoNameValid } from '../../helpers/custom-validators/videos' |
10 | import { CONFIG } from '../../initializers/constants' | 10 | import { CONFIG } from '../../initializers/constants' |
11 | import { CONSTRAINTS_FIELDS } from '../../initializers' | ||
11 | 12 | ||
12 | const videoImportAddValidator = getCommonVideoAttributes().concat([ | 13 | const videoImportAddValidator = getCommonVideoAttributes().concat([ |
13 | body('channelId') | 14 | body('channelId') |
@@ -19,6 +20,11 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([ | |||
19 | body('magnetUri') | 20 | body('magnetUri') |
20 | .optional() | 21 | .optional() |
21 | .custom(isVideoMagnetUriValid).withMessage('Should have a valid video magnet URI'), | 22 | .custom(isVideoMagnetUriValid).withMessage('Should have a valid video magnet URI'), |
23 | body('torrentfile') | ||
24 | .custom((value, { req }) => isVideoImportTorrentFile(req.files)).withMessage( | ||
25 | 'This torrent file is not supported or too large. Please, make sure it is of the following type: ' | ||
26 | + CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.EXTNAME.join(', ') | ||
27 | ), | ||
22 | body('name') | 28 | body('name') |
23 | .optional() | 29 | .optional() |
24 | .custom(isVideoNameValid).withMessage('Should have a valid name'), | 30 | .custom(isVideoNameValid).withMessage('Should have a valid name'), |
@@ -40,11 +46,12 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([ | |||
40 | if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) | 46 | if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) |
41 | 47 | ||
42 | // Check we have at least 1 required param | 48 | // Check we have at least 1 required param |
43 | if (!req.body.targetUrl && !req.body.magnetUri) { | 49 | const file = req.files['torrentfile'][0] |
50 | if (!req.body.targetUrl && !req.body.magnetUri && !file) { | ||
44 | cleanUpReqFiles(req) | 51 | cleanUpReqFiles(req) |
45 | 52 | ||
46 | return res.status(400) | 53 | return res.status(400) |
47 | .json({ error: 'Should have a magnetUri or a targetUrl.' }) | 54 | .json({ error: 'Should have a magnetUri or a targetUrl or a torrent file.' }) |
48 | .end() | 55 | .end() |
49 | } | 56 | } |
50 | 57 | ||
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts index 55fca28b8..d6c02e5ac 100644 --- a/server/models/video/video-import.ts +++ b/server/models/video/video-import.ts | |||
@@ -171,7 +171,11 @@ export class VideoImportModel extends Model<VideoImportModel> { | |||
171 | 171 | ||
172 | return { | 172 | return { |
173 | id: this.id, | 173 | id: this.id, |
174 | |||
174 | targetUrl: this.targetUrl, | 175 | targetUrl: this.targetUrl, |
176 | magnetUri: this.magnetUri, | ||
177 | torrentName: this.torrentName, | ||
178 | |||
175 | state: { | 179 | state: { |
176 | id: this.state, | 180 | id: this.state, |
177 | label: VideoImportModel.getStateLabel(this.state) | 181 | label: VideoImportModel.getStateLabel(this.state) |
diff --git a/shared/models/videos/video-import.model.ts b/shared/models/videos/video-import.model.ts index a5c582c67..293854006 100644 --- a/shared/models/videos/video-import.model.ts +++ b/shared/models/videos/video-import.model.ts | |||
@@ -4,7 +4,11 @@ import { VideoImportState } from './video-import-state.enum' | |||
4 | 4 | ||
5 | export interface VideoImport { | 5 | export interface VideoImport { |
6 | id: number | 6 | id: number |
7 | |||
7 | targetUrl: string | 8 | targetUrl: string |
9 | magnetUri: string | ||
10 | torrentName: string | ||
11 | |||
8 | createdAt: string | 12 | createdAt: string |
9 | updatedAt: string | 13 | updatedAt: string |
10 | state: VideoConstant<VideoImportState> | 14 | state: VideoConstant<VideoImportState> |