From: Chocobozzz Date: Mon, 6 Aug 2018 15:13:39 +0000 (+0200) Subject: Import magnets with webtorrent X-Git-Tag: delete~46 X-Git-Url: https://git.immae.eu/?a=commitdiff_plain;h=ce33919c24e7402d92d81f3cd8e545df52d98240;hp=788487140c500abeb69ca44daf3a9e26efa8d36f;p=github%2FChocobozzz%2FPeerTube.git Import magnets with webtorrent --- diff --git a/client/src/app/shared/video-import/video-import.service.ts b/client/src/app/shared/video-import/video-import.service.ts index 59b58ab38..002412bd7 100644 --- a/client/src/app/shared/video-import/video-import.service.ts +++ b/client/src/app/shared/video-import/video-import.service.ts @@ -26,8 +26,43 @@ export class VideoImportService { private serverService: ServerService ) {} - importVideo (targetUrl: string, video: VideoUpdate): Observable { + importVideoUrl (targetUrl: string, video: VideoUpdate): Observable { const url = VideoImportService.BASE_VIDEO_IMPORT_URL + + const body = this.buildImportVideoObject(video) + body.targetUrl = targetUrl + + const data = objectToFormData(body) + return this.authHttp.post(url, data) + .pipe(catchError(res => this.restExtractor.handleError(res))) + } + + importVideoTorrent (target: string | Blob, video: VideoUpdate): Observable { + const url = VideoImportService.BASE_VIDEO_IMPORT_URL + const body: VideoImportCreate = this.buildImportVideoObject(video) + + if (typeof target === 'string') body.magnetUri = target + else body.torrentfile = target + + const data = objectToFormData(body) + return this.authHttp.post(url, data) + .pipe(catchError(res => this.restExtractor.handleError(res))) + } + + getMyVideoImports (pagination: RestPagination, sort: SortMeta): Observable> { + let params = new HttpParams() + params = this.restService.addRestGetParams(params, pagination, sort) + + return this.authHttp + .get>(UserService.BASE_USERS_URL + '/me/videos/imports', { params }) + .pipe( + switchMap(res => this.extractVideoImports(res)), + map(res => this.restExtractor.convertResultListDateToHuman(res)), + catchError(err => this.restExtractor.handleError(err)) + ) + } + + private buildImportVideoObject (video: VideoUpdate): VideoImportCreate { const language = video.language || null const licence = video.licence || null const category = video.category || null @@ -35,9 +70,7 @@ export class VideoImportService { const support = video.support || null const scheduleUpdate = video.scheduleUpdate || null - const body: VideoImportCreate = { - targetUrl, - + return { name: video.name, category, licence, @@ -54,23 +87,6 @@ export class VideoImportService { previewfile: video.previewfile, scheduleUpdate } - - const data = objectToFormData(body) - return this.authHttp.post(url, data) - .pipe(catchError(res => this.restExtractor.handleError(res))) - } - - getMyVideoImports (pagination: RestPagination, sort: SortMeta): Observable> { - let params = new HttpParams() - params = this.restService.addRestGetParams(params, pagination, sort) - - return this.authHttp - .get>(UserService.BASE_USERS_URL + '/me/videos/imports', { params }) - .pipe( - switchMap(res => this.extractVideoImports(res)), - map(res => this.restExtractor.convertResultListDateToHuman(res)), - catchError(err => this.restExtractor.handleError(err)) - ) } private extractVideoImports (result: ResultList): Observable> { diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html new file mode 100644 index 000000000..409e4de5e --- /dev/null +++ b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html @@ -0,0 +1,60 @@ +
+
+
+ +
+ + + + +
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ + +
+
+ +
+ Congratulations, the video will be imported with BitTorrent! You can already add information about this video. +
+ + +
+ + +
+
+ + +
+
+
diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.scss b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.scss new file mode 100644 index 000000000..1ef5adc25 --- /dev/null +++ b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.scss @@ -0,0 +1,37 @@ +@import 'variables'; +@import 'mixins'; + +$width-size: 190px; + +.peertube-select-container { + @include peertube-select-container($width-size); +} + +.import-video-torrent { + display: flex; + flex-direction: column; + align-items: center; + + .icon.icon-upload { + @include icon(90px); + margin-bottom: 25px; + cursor: default; + + background-image: url('../../../../assets/images/video/upload.svg'); + } + + input[type=text] { + @include peertube-input-text($width-size); + display: block; + } + + input[type=button] { + @include peertube-button; + @include orange-button; + + width: $width-size; + margin-top: 30px; + } +} + + diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.ts b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.ts new file mode 100644 index 000000000..330c37718 --- /dev/null +++ b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.ts @@ -0,0 +1,132 @@ +import { Component, EventEmitter, OnInit, Output } from '@angular/core' +import { Router } from '@angular/router' +import { NotificationsService } from 'angular2-notifications' +import { VideoPrivacy, VideoUpdate } from '../../../../../../shared/models/videos' +import { AuthService, ServerService } from '../../../core' +import { VideoService } from '../../../shared/video/video.service' +import { I18n } from '@ngx-translate/i18n-polyfill' +import { LoadingBarService } from '@ngx-loading-bar/core' +import { VideoSend } from '@app/videos/+video-edit/video-add-components/video-send' +import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.service' +import { VideoEdit } from '@app/shared/video/video-edit.model' +import { FormValidatorService } from '@app/shared' +import { VideoCaptionService } from '@app/shared/video-caption' +import { VideoImportService } from '@app/shared/video-import' + +@Component({ + selector: 'my-video-import-torrent', + templateUrl: './video-import-torrent.component.html', + styleUrls: [ + '../shared/video-edit.component.scss', + './video-import-torrent.component.scss' + ] +}) +export class VideoImportTorrentComponent extends VideoSend implements OnInit, CanComponentDeactivate { + @Output() firstStepDone = new EventEmitter() + + videoFileName: string + magnetUri = '' + + isImportingVideo = false + hasImportedVideo = false + isUpdatingVideo = false + + video: VideoEdit + + protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PRIVATE + + constructor ( + protected formValidatorService: FormValidatorService, + protected loadingBar: LoadingBarService, + protected notificationsService: NotificationsService, + protected authService: AuthService, + protected serverService: ServerService, + protected videoService: VideoService, + protected videoCaptionService: VideoCaptionService, + private router: Router, + private videoImportService: VideoImportService, + private i18n: I18n + ) { + super() + } + + ngOnInit () { + super.ngOnInit() + } + + canDeactivate () { + return { canDeactivate: true } + } + + isMagnetUrlValid () { + return !!this.magnetUri + } + + importVideo () { + this.isImportingVideo = true + + const videoUpdate: VideoUpdate = { + privacy: this.firstStepPrivacyId, + waitTranscoding: false, + commentsEnabled: true, + channelId: this.firstStepChannelId + } + + this.loadingBar.start() + + this.videoImportService.importVideoTorrent(this.magnetUri, videoUpdate).subscribe( + res => { + this.loadingBar.complete() + this.firstStepDone.emit(res.video.name) + this.isImportingVideo = false + this.hasImportedVideo = true + + this.video = new VideoEdit(Object.assign(res.video, { + commentsEnabled: videoUpdate.commentsEnabled, + support: null, + thumbnailUrl: null, + previewUrl: null + })) + this.hydrateFormFromVideo() + }, + + err => { + this.loadingBar.complete() + this.isImportingVideo = false + this.notificationsService.error(this.i18n('Error'), err.message) + } + ) + } + + updateSecondStep () { + if (this.checkForm() === false) { + return + } + + this.video.patch(this.form.value) + + this.isUpdatingVideo = true + + // Update the video + this.updateVideoAndCaptions(this.video) + .subscribe( + () => { + this.isUpdatingVideo = false + this.notificationsService.success(this.i18n('Success'), this.i18n('Video to import updated.')) + + this.router.navigate([ '/my-account', 'video-imports' ]) + }, + + err => { + this.isUpdatingVideo = false + this.notificationsService.error(this.i18n('Error'), err.message) + console.error(err) + } + ) + + } + + private hydrateFormFromVideo () { + this.form.patchValue(this.video.toFormPatch()) + } +} diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.html b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.html index 6b431f6f6..9f5fc6d22 100644 --- a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.html +++ b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.html @@ -1,5 +1,5 @@
-
+
diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.scss b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.scss index 5e713ab97..7c6deda1d 100644 --- a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.scss +++ b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.scss @@ -7,7 +7,7 @@ $width-size: 190px; @include peertube-select-container($width-size); } -.import-video { +.import-video-url { display: flex; flex-direction: column; align-items: center; diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts index dbe69409f..842ede732 100644 --- a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts +++ b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts @@ -74,7 +74,7 @@ export class VideoImportUrlComponent extends VideoSend implements OnInit, CanCom this.loadingBar.start() - this.videoImportService.importVideo(this.targetUrl, videoUpdate).subscribe( + this.videoImportService.importVideoUrl(this.targetUrl, videoUpdate).subscribe( res => { this.loadingBar.complete() this.firstStepDone.emit(res.video.name) diff --git a/client/src/app/videos/+video-edit/video-add.component.html b/client/src/app/videos/+video-edit/video-add.component.html index 7a50372e9..340820180 100644 --- a/client/src/app/videos/+video-edit/video-add.component.html +++ b/client/src/app/videos/+video-edit/video-add.component.html @@ -10,8 +10,12 @@ - + + + + +
diff --git a/client/src/app/videos/+video-edit/video-add.component.ts b/client/src/app/videos/+video-edit/video-add.component.ts index e74fa1f15..7d360598d 100644 --- a/client/src/app/videos/+video-edit/video-add.component.ts +++ b/client/src/app/videos/+video-edit/video-add.component.ts @@ -3,6 +3,7 @@ import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard. import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-components/video-import-url.component' import { VideoUploadComponent } from '@app/videos/+video-edit/video-add-components/video-upload.component' import { ServerService } from '@app/core' +import { VideoImportTorrentComponent } from '@app/videos/+video-edit/video-add-components/video-import-torrent.component' @Component({ selector: 'my-videos-add', @@ -12,15 +13,16 @@ import { ServerService } from '@app/core' export class VideoAddComponent implements CanComponentDeactivate { @ViewChild('videoUpload') videoUpload: VideoUploadComponent @ViewChild('videoImportUrl') videoImportUrl: VideoImportUrlComponent + @ViewChild('videoImportTorrent') videoImportTorrent: VideoImportTorrentComponent - secondStepType: 'upload' | 'import-url' + secondStepType: 'upload' | 'import-url' | 'import-torrent' videoName: string constructor ( private serverService: ServerService ) {} - onFirstStepDone (type: 'upload' | 'import-url', videoName: string) { + onFirstStepDone (type: 'upload' | 'import-url' | 'import-torrent', videoName: string) { this.secondStepType = type this.videoName = videoName } @@ -28,11 +30,16 @@ export class VideoAddComponent implements CanComponentDeactivate { canDeactivate () { if (this.secondStepType === 'upload') return this.videoUpload.canDeactivate() if (this.secondStepType === 'import-url') return this.videoImportUrl.canDeactivate() + if (this.secondStepType === 'import-torrent') return this.videoImportTorrent.canDeactivate() return { canDeactivate: true } } - isVideoImportEnabled () { + isVideoImportHttpEnabled () { + return this.serverService.getConfig().import.videos.http.enabled + } + + isVideoImportTorrentEnabled () { return this.serverService.getConfig().import.videos.http.enabled } } diff --git a/client/src/app/videos/+video-edit/video-add.module.ts b/client/src/app/videos/+video-edit/video-add.module.ts index a1324b397..3ecf96459 100644 --- a/client/src/app/videos/+video-edit/video-add.module.ts +++ b/client/src/app/videos/+video-edit/video-add.module.ts @@ -7,6 +7,7 @@ import { VideoAddComponent } from './video-add.component' import { CanDeactivateGuard } from '../../shared/guards/can-deactivate-guard.service' import { VideoUploadComponent } from '@app/videos/+video-edit/video-add-components/video-upload.component' import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-components/video-import-url.component' +import { VideoImportTorrentComponent } from '@app/videos/+video-edit/video-add-components/video-import-torrent.component' @NgModule({ imports: [ @@ -18,7 +19,8 @@ import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-compo declarations: [ VideoAddComponent, VideoUploadComponent, - VideoImportUrlComponent + VideoImportUrlComponent, + VideoImportTorrentComponent ], exports: [ VideoAddComponent diff --git a/package.json b/package.json index b5d695344..6348bbb6a 100644 --- a/package.json +++ b/package.json @@ -134,6 +134,7 @@ "uuid": "^3.1.0", "validator": "^10.2.0", "webfinger.js": "^2.6.6", + "webtorrent": "^0.100.0", "winston": "3.0.0", "ws": "^5.0.0", "youtube-dl": "^1.12.2" @@ -187,7 +188,6 @@ "tslint": "^5.7.0", "tslint-config-standard": "^7.0.0", "typescript": "^2.5.2", - "webtorrent": "^0.100.0", "xliff": "^3.0.1" }, "scripty": { diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 30a7d816c..c16a254d2 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -1,3 +1,4 @@ +import * as magnetUtil from 'magnet-uri' import * as express from 'express' import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger' import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' @@ -13,6 +14,10 @@ import { VideoImportModel } from '../../../models/video/video-import' import { JobQueue } from '../../../lib/job-queue/job-queue' import { processImage } from '../../../helpers/image-utils' import { join } from 'path' +import { isArray } from '../../../helpers/custom-validators/misc' +import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' +import { VideoChannelModel } from '../../../models/video/video-channel' +import * as Bluebird from 'bluebird' const auditLogger = auditLoggerFactory('video-imports') const videoImportsRouter = express.Router() @@ -41,7 +46,45 @@ export { // --------------------------------------------------------------------------- -async function addVideoImport (req: express.Request, res: express.Response) { +function addVideoImport (req: express.Request, res: express.Response) { + if (req.body.targetUrl) return addYoutubeDLImport(req, res) + + if (req.body.magnetUri) return addTorrentImport(req, res) +} + +async function addTorrentImport (req: express.Request, res: express.Response) { + const body: VideoImportCreate = req.body + const magnetUri = body.magnetUri + + const parsed = magnetUtil.decode(magnetUri) + const magnetName = isArray(parsed.name) ? parsed.name[0] : parsed.name as string + + const video = buildVideo(res.locals.videoChannel.id, body, { name: magnetName }) + + await processThumbnail(req, video) + await processPreview(req, video) + + const tags = null + const videoImportAttributes = { + magnetUri, + state: VideoImportState.PENDING + } + const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes) + + // Create job to import the video + const payload = { + type: 'magnet-uri' as 'magnet-uri', + videoImportId: videoImport.id, + magnetUri + } + await JobQueue.Instance.createJob({ type: 'video-import', payload }) + + auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON())) + + return res.json(videoImport.toFormattedJSON()).end() +} + +async function addYoutubeDLImport (req: express.Request, res: express.Response) { const body: VideoImportCreate = req.body const targetUrl = body.targetUrl @@ -56,53 +99,94 @@ async function addVideoImport (req: express.Request, res: express.Response) { }).end() } - // Create video DB object + const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo) + + const downloadThumbnail = !await processThumbnail(req, video) + const downloadPreview = !await processPreview(req, video) + + const tags = body.tags || youtubeDLInfo.tags + const videoImportAttributes = { + targetUrl, + state: VideoImportState.PENDING + } + const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes) + + // Create job to import the video + const payload = { + type: 'youtube-dl' as 'youtube-dl', + videoImportId: videoImport.id, + thumbnailUrl: youtubeDLInfo.thumbnailUrl, + downloadThumbnail, + downloadPreview + } + await JobQueue.Instance.createJob({ type: 'video-import', payload }) + + auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON())) + + return res.json(videoImport.toFormattedJSON()).end() +} + +function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo) { const videoData = { - name: body.name || youtubeDLInfo.name, + name: body.name || importData.name || 'Unknown name', remote: false, - category: body.category || youtubeDLInfo.category, - licence: body.licence || youtubeDLInfo.licence, + category: body.category || importData.category, + licence: body.licence || importData.licence, language: body.language || undefined, commentsEnabled: body.commentsEnabled || true, waitTranscoding: body.waitTranscoding || false, state: VideoState.TO_IMPORT, - nsfw: body.nsfw || youtubeDLInfo.nsfw || false, - description: body.description || youtubeDLInfo.description, + nsfw: body.nsfw || importData.nsfw || false, + description: body.description || importData.description, support: body.support || null, privacy: body.privacy || VideoPrivacy.PRIVATE, duration: 0, // duration will be set by the import job - channelId: res.locals.videoChannel.id + channelId: channelId } const video = new VideoModel(videoData) video.url = getVideoActivityPubUrl(video) - // Process thumbnail file? + return video +} + +async function processThumbnail (req: express.Request, video: VideoModel) { const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined - let downloadThumbnail = true if (thumbnailField) { const thumbnailPhysicalFile = thumbnailField[ 0 ] await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE) - downloadThumbnail = false + + return true } - // Process preview file? + return false +} + +async function processPreview (req: express.Request, video: VideoModel) { const previewField = req.files ? req.files['previewfile'] : undefined - let downloadPreview = true if (previewField) { const previewPhysicalFile = previewField[0] await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE) - downloadPreview = false + + return true } - const videoImport: VideoImportModel = await sequelizeTypescript.transaction(async t => { + return false +} + +function insertIntoDB ( + video: VideoModel, + videoChannel: VideoChannelModel, + tags: string[], + videoImportAttributes: FilteredModelAttributes +): Bluebird { + return sequelizeTypescript.transaction(async t => { const sequelizeOptions = { transaction: t } // Save video object in database const videoCreated = await video.save(sequelizeOptions) - videoCreated.VideoChannel = res.locals.videoChannel + videoCreated.VideoChannel = videoChannel // Set tags to the video - const tags = body.tags ? body.tags : youtubeDLInfo.tags if (tags !== undefined) { const tagInstances = await TagModel.findOrCreateTags(tags, t) @@ -111,28 +195,12 @@ async function addVideoImport (req: express.Request, res: express.Response) { } // Create video import object in database - const videoImport = await VideoImportModel.create({ - targetUrl, - state: VideoImportState.PENDING, - videoId: videoCreated.id - }, sequelizeOptions) - + const videoImport = await VideoImportModel.create( + Object.assign({ videoId: videoCreated.id }, videoImportAttributes), + sequelizeOptions + ) videoImport.Video = videoCreated return videoImport }) - - // Create job to import the video - const payload = { - type: 'youtube-dl' as 'youtube-dl', - videoImportId: videoImport.id, - thumbnailUrl: youtubeDLInfo.thumbnailUrl, - downloadThumbnail, - downloadPreview - } - await JobQueue.Instance.createJob({ type: 'video-import', payload }) - - auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON())) - - return res.json(videoImport.toFormattedJSON()).end() } diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index 338c96582..f4c1c8b07 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -17,6 +17,7 @@ import { VideoModel } from '../../models/video/video' import { exists, isArray, isFileValid } from './misc' import { VideoChannelModel } from '../../models/video/video-channel' import { UserModel } from '../../models/account/user' +import * as magnetUtil from 'magnet-uri' const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES @@ -126,6 +127,13 @@ function isVideoFileSizeValid (value: string) { return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE) } +function isVideoMagnetUriValid (value: string) { + if (!exists(value)) return false + + const parsed = magnetUtil.decode(value) + return parsed && isVideoFileInfoHashValid(parsed.infoHash) +} + function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: UserRight, res: Response) { // Retrieve the user who did the request if (video.isOwned() === false) { @@ -214,6 +222,7 @@ export { isScheduleVideoUpdatePrivacyValid, isVideoAbuseReasonValid, isVideoFile, + isVideoMagnetUriValid, isVideoStateValid, isVideoViewsValid, isVideoRatingTypeValid, diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 7abcec5d7..f4cc5547d 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -9,6 +9,8 @@ import { ApplicationModel } from '../models/application/application' import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils' import { logger } from './logger' import { isArray } from './custom-validators/misc' +import * as crypto from "crypto" +import { join } from "path" const isCidr = require('is-cidr') @@ -181,8 +183,14 @@ async function getServerActor () { return Promise.resolve(serverActor) } +function generateVideoTmpPath (id: string) { + const hash = crypto.createHash('sha256').update(id).digest('hex') + return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4') +} + type SortType = { sortModel: any, sortValue: string } + // --------------------------------------------------------------------------- export { @@ -195,5 +203,6 @@ export { computeResolutionsToTranscode, resetSequelizeInstance, getServerActor, - SortType + SortType, + generateVideoTmpPath } diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts new file mode 100644 index 000000000..fce88a1f6 --- /dev/null +++ b/server/helpers/webtorrent.ts @@ -0,0 +1,31 @@ +import { logger } from './logger' +import { generateVideoTmpPath } from './utils' +import * as WebTorrent from 'webtorrent' +import { createWriteStream } from 'fs' + +function downloadWebTorrentVideo (target: string) { + const path = generateVideoTmpPath(target) + + logger.info('Importing torrent video %s', target) + + return new Promise((res, rej) => { + const webtorrent = new WebTorrent() + + const torrent = webtorrent.add(target, torrent => { + if (torrent.files.length !== 1) throw new Error('The number of files is not equal to 1 for ' + target) + + const file = torrent.files[ 0 ] + file.createReadStream().pipe(createWriteStream(path)) + }) + + torrent.on('done', () => res(path)) + + torrent.on('error', err => rej(err)) + }) +} + +// --------------------------------------------------------------------------- + +export { + downloadWebTorrentVideo +} diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index c59ab9de0..77986f407 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts @@ -1,18 +1,17 @@ import * as youtubeDL from 'youtube-dl' import { truncate } from 'lodash' -import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' -import { join } from 'path' -import * as crypto from 'crypto' +import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' import { logger } from './logger' +import { generateVideoTmpPath } from './utils' export type YoutubeDLInfo = { - name: string - description: string - category: number - licence: number - nsfw: boolean - tags: string[] - thumbnailUrl: string + name?: string + description?: string + category?: number + licence?: number + nsfw?: boolean + tags?: string[] + thumbnailUrl?: string } function getYoutubeDLInfo (url: string): Promise { @@ -30,10 +29,9 @@ function getYoutubeDLInfo (url: string): Promise { } function downloadYoutubeDLVideo (url: string) { - const hash = crypto.createHash('sha256').update(url).digest('hex') - const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4') + const path = generateVideoTmpPath(url) - logger.info('Importing video %s', url) + logger.info('Importing youtubeDL video %s', url) const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 74fe7965d..243d544ea 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -15,7 +15,7 @@ let config: IConfig = require('config') // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 240 +const LAST_MIGRATION_VERSION = 245 // --------------------------------------------------------------------------- @@ -271,7 +271,8 @@ const CONSTRAINTS_FIELDS = { } }, VIDEO_IMPORTS: { - URL: { min: 3, max: 2000 } // Length + URL: { min: 3, max: 2000 }, // Length + TORRENT_NAME: { min: 3, max: 255 }, // Length }, VIDEOS: { NAME: { min: 3, max: 120 }, // Length diff --git a/server/initializers/migrations/0245-import-magnet.ts b/server/initializers/migrations/0245-import-magnet.ts new file mode 100644 index 000000000..87603b006 --- /dev/null +++ b/server/initializers/migrations/0245-import-magnet.ts @@ -0,0 +1,42 @@ +import * as Sequelize from 'sequelize' +import { Migration } from '../../models/migrations' +import { CONSTRAINTS_FIELDS } from '../index' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + { + const data = { + type: Sequelize.STRING, + allowNull: true, + defaultValue: null + } as Migration.String + await utils.queryInterface.changeColumn('videoImport', 'targetUrl', data) + } + + { + const data = { + type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max), + allowNull: true, + defaultValue: null + } + await utils.queryInterface.addColumn('videoImport', 'magnetUri', data) + } + + { + const data = { + type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max), + allowNull: true, + defaultValue: null + } + await utils.queryInterface.addColumn('videoImport', 'torrentName', data) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { up, down } diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index cdfe412cc..c457b71fc 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -13,30 +13,99 @@ import { VideoState } from '../../../../shared' import { JobQueue } from '../index' import { federateVideoIfNeeded } from '../../activitypub' import { VideoModel } from '../../../models/video/video' +import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' -export type VideoImportPayload = { +type VideoImportYoutubeDLPayload = { type: 'youtube-dl' videoImportId: number + thumbnailUrl: string downloadThumbnail: boolean downloadPreview: boolean } +type VideoImportTorrentPayload = { + type: 'magnet-uri' + videoImportId: number +} + +export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload + async function processVideoImport (job: Bull.Job) { const payload = job.data as VideoImportPayload - logger.info('Processing video import in job %d.', job.id) - const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId) + if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload) + if (payload.type === 'magnet-uri') return processTorrentImport(job, payload) +} + +// --------------------------------------------------------------------------- + +export { + processVideoImport +} + +// --------------------------------------------------------------------------- + +async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentPayload) { + logger.info('Processing torrent video import in job %d.', job.id) + + const videoImport = await getVideoImportOrDie(payload.videoImportId) + const options = { + videoImportId: payload.videoImportId, + + downloadThumbnail: false, + downloadPreview: false, + + generateThumbnail: true, + generatePreview: true + } + return processFile(() => downloadWebTorrentVideo(videoImport.magnetUri), videoImport, options) +} + +async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) { + logger.info('Processing youtubeDL video import in job %d.', job.id) + + const videoImport = await getVideoImportOrDie(payload.videoImportId) + const options = { + videoImportId: videoImport.id, + + downloadThumbnail: payload.downloadThumbnail, + downloadPreview: payload.downloadPreview, + thumbnailUrl: payload.thumbnailUrl, + + generateThumbnail: false, + generatePreview: false + } + + return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl), videoImport, options) +} + +async function getVideoImportOrDie (videoImportId: number) { + const videoImport = await VideoImportModel.loadAndPopulateVideo(videoImportId) if (!videoImport || !videoImport.Video) { throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.') } + return videoImport +} + +type ProcessFileOptions = { + videoImportId: number + + downloadThumbnail: boolean + downloadPreview: boolean + thumbnailUrl?: string + + generateThumbnail: boolean + generatePreview: boolean +} +async function processFile (downloader: () => Promise, videoImport: VideoImportModel, options: ProcessFileOptions) { let tempVideoPath: string let videoDestFile: string let videoFile: VideoFileModel try { // Download video from youtubeDL - tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl) + tempVideoPath = await downloader() // Get information about this video const { videoFileResolution } = await getVideoFileResolution(tempVideoPath) @@ -62,23 +131,27 @@ async function processVideoImport (job: Bull.Job) { tempVideoPath = null // This path is not used anymore // Process thumbnail - if (payload.downloadThumbnail) { - if (payload.thumbnailUrl) { + if (options.downloadThumbnail) { + if (options.thumbnailUrl) { const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) - await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath) + await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destThumbnailPath) } else { await videoImport.Video.createThumbnail(videoFile) } + } else if (options.generateThumbnail) { + await videoImport.Video.createThumbnail(videoFile) } // Process preview - if (payload.downloadPreview) { - if (payload.thumbnailUrl) { + if (options.downloadPreview) { + if (options.thumbnailUrl) { const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) - await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath) + await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destPreviewPath) } else { await videoImport.Video.createPreview(videoFile) } + } else if (options.generatePreview) { + await videoImport.Video.createPreview(videoFile) } // Create torrent @@ -137,9 +210,3 @@ async function processVideoImport (job: Bull.Job) { throw err } } - -// --------------------------------------------------------------------------- - -export { - processVideoImport -} diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 8a24604e1..ddb357db5 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -32,13 +32,6 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise} = { 'video-import': processVideoImport } -const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = { - 'activitypub-http-broadcast': true, - 'activitypub-http-unicast': true, - 'activitypub-http-fetcher': true, - 'activitypub-follow': true -} - const jobTypes: JobType[] = [ 'activitypub-follow', 'activitypub-http-broadcast', diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts index d806edfa3..8ec9373fb 100644 --- a/server/middlewares/validators/video-imports.ts +++ b/server/middlewares/validators/video-imports.ts @@ -6,14 +6,19 @@ import { areValidationErrors } from './utils' import { getCommonVideoAttributes } from './videos' import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' import { cleanUpReqFiles } from '../../helpers/utils' -import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos' +import { isVideoChannelOfAccountExist, isVideoMagnetUriValid, isVideoNameValid } from '../../helpers/custom-validators/videos' import { CONFIG } from '../../initializers/constants' const videoImportAddValidator = getCommonVideoAttributes().concat([ - body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'), body('channelId') .toInt() .custom(isIdValid).withMessage('Should have correct video channel id'), + body('targetUrl') + .optional() + .custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'), + body('magnetUri') + .optional() + .custom(isVideoMagnetUriValid).withMessage('Should have a valid video magnet URI'), body('name') .optional() .custom(isVideoNameValid).withMessage('Should have a valid name'), @@ -34,6 +39,15 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([ if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) + // Check we have at least 1 required param + if (!req.body.targetUrl && !req.body.magnetUri) { + cleanUpReqFiles(req) + + return res.status(400) + .json({ error: 'Should have a magnetUri or a targetUrl.' }) + .end() + } + return next() } ]) diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts index eca87163d..55fca28b8 100644 --- a/server/models/video/video-import.ts +++ b/server/models/video/video-import.ts @@ -21,6 +21,7 @@ import { VideoImport, VideoImportState } from '../../../shared' import { VideoChannelModel } from './video-channel' import { AccountModel } from '../account/account' import { TagModel } from './tag' +import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos' @DefaultScope({ include: [ @@ -62,11 +63,23 @@ export class VideoImportModel extends Model { @UpdatedAt updatedAt: Date - @AllowNull(false) + @AllowNull(true) + @Default(null) @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl')) @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) targetUrl: string + @AllowNull(true) + @Default(null) + @Is('VideoImportMagnetUri', value => throwIfNotValid(value, isVideoMagnetUriValid, 'magnetUri')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) // Use the same constraints than URLs + magnetUri: string + + @AllowNull(true) + @Default(null) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max)) + torrentName: string + @AllowNull(false) @Default(null) @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state')) diff --git a/shared/models/videos/video-import-create.model.ts b/shared/models/videos/video-import-create.model.ts index 65d142c2b..e76084e06 100644 --- a/shared/models/videos/video-import-create.model.ts +++ b/shared/models/videos/video-import-create.model.ts @@ -1,6 +1,9 @@ import { VideoUpdate } from './video-update.model' export interface VideoImportCreate extends VideoUpdate { - targetUrl: string + targetUrl?: string + magnetUri?: string + torrentfile?: Blob + channelId: number // Required }