aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-08-06 17:13:39 +0200
committerChocobozzz <me@florianbigard.com>2018-08-08 09:30:31 +0200
commitce33919c24e7402d92d81f3cd8e545df52d98240 (patch)
tree7e131a2f8df649899d0a71294665cf386ffb50d4
parent788487140c500abeb69ca44daf3a9e26efa8d36f (diff)
downloadPeerTube-ce33919c24e7402d92d81f3cd8e545df52d98240.tar.gz
PeerTube-ce33919c24e7402d92d81f3cd8e545df52d98240.tar.zst
PeerTube-ce33919c24e7402d92d81f3cd8e545df52d98240.zip
Import magnets with webtorrent
-rw-r--r--client/src/app/shared/video-import/video-import.service.ts58
-rw-r--r--client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html60
-rw-r--r--client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.scss37
-rw-r--r--client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.ts132
-rw-r--r--client/src/app/videos/+video-edit/video-add-components/video-import-url.component.html2
-rw-r--r--client/src/app/videos/+video-edit/video-add-components/video-import-url.component.scss2
-rw-r--r--client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts2
-rw-r--r--client/src/app/videos/+video-edit/video-add.component.html6
-rw-r--r--client/src/app/videos/+video-edit/video-add.component.ts13
-rw-r--r--client/src/app/videos/+video-edit/video-add.module.ts4
-rw-r--r--package.json2
-rw-r--r--server/controllers/api/videos/import.ts142
-rw-r--r--server/helpers/custom-validators/videos.ts9
-rw-r--r--server/helpers/utils.ts11
-rw-r--r--server/helpers/webtorrent.ts31
-rw-r--r--server/helpers/youtube-dl.ts24
-rw-r--r--server/initializers/constants.ts5
-rw-r--r--server/initializers/migrations/0245-import-magnet.ts42
-rw-r--r--server/lib/job-queue/handlers/video-import.ts99
-rw-r--r--server/lib/job-queue/job-queue.ts7
-rw-r--r--server/middlewares/validators/video-imports.ts18
-rw-r--r--server/models/video/video-import.ts15
-rw-r--r--shared/models/videos/video-import-create.model.ts5
23 files changed, 616 insertions, 110 deletions
diff --git a/client/src/app/shared/video-import/video-import.service.ts b/client/src/app/shared/video-import/video-import.service.ts
index 59b58ab38..002412bd7 100644
--- a/client/src/app/shared/video-import/video-import.service.ts
+++ b/client/src/app/shared/video-import/video-import.service.ts
@@ -26,8 +26,43 @@ export class VideoImportService {
26 private serverService: ServerService 26 private serverService: ServerService
27 ) {} 27 ) {}
28 28
29 importVideo (targetUrl: string, video: VideoUpdate): Observable<VideoImport> { 29 importVideoUrl (targetUrl: string, video: VideoUpdate): Observable<VideoImport> {
30 const url = VideoImportService.BASE_VIDEO_IMPORT_URL 30 const url = VideoImportService.BASE_VIDEO_IMPORT_URL
31
32 const body = this.buildImportVideoObject(video)
33 body.targetUrl = targetUrl
34
35 const data = objectToFormData(body)
36 return this.authHttp.post<VideoImport>(url, data)
37 .pipe(catchError(res => this.restExtractor.handleError(res)))
38 }
39
40 importVideoTorrent (target: string | Blob, video: VideoUpdate): Observable<VideoImport> {
41 const url = VideoImportService.BASE_VIDEO_IMPORT_URL
42 const body: VideoImportCreate = this.buildImportVideoObject(video)
43
44 if (typeof target === 'string') body.magnetUri = target
45 else body.torrentfile = target
46
47 const data = objectToFormData(body)
48 return this.authHttp.post<VideoImport>(url, data)
49 .pipe(catchError(res => this.restExtractor.handleError(res)))
50 }
51
52 getMyVideoImports (pagination: RestPagination, sort: SortMeta): Observable<ResultList<VideoImport>> {
53 let params = new HttpParams()
54 params = this.restService.addRestGetParams(params, pagination, sort)
55
56 return this.authHttp
57 .get<ResultList<VideoImport>>(UserService.BASE_USERS_URL + '/me/videos/imports', { params })
58 .pipe(
59 switchMap(res => this.extractVideoImports(res)),
60 map(res => this.restExtractor.convertResultListDateToHuman(res)),
61 catchError(err => this.restExtractor.handleError(err))
62 )
63 }
64
65 private buildImportVideoObject (video: VideoUpdate): VideoImportCreate {
31 const language = video.language || null 66 const language = video.language || null
32 const licence = video.licence || null 67 const licence = video.licence || null
33 const category = video.category || null 68 const category = video.category || null
@@ -35,9 +70,7 @@ export class VideoImportService {
35 const support = video.support || null 70 const support = video.support || null
36 const scheduleUpdate = video.scheduleUpdate || null 71 const scheduleUpdate = video.scheduleUpdate || null
37 72
38 const body: VideoImportCreate = { 73 return {
39 targetUrl,
40
41 name: video.name, 74 name: video.name,
42 category, 75 category,
43 licence, 76 licence,
@@ -54,23 +87,6 @@ export class VideoImportService {
54 previewfile: video.previewfile, 87 previewfile: video.previewfile,
55 scheduleUpdate 88 scheduleUpdate
56 } 89 }
57
58 const data = objectToFormData(body)
59 return this.authHttp.post<VideoImport>(url, data)
60 .pipe(catchError(res => this.restExtractor.handleError(res)))
61 }
62
63 getMyVideoImports (pagination: RestPagination, sort: SortMeta): Observable<ResultList<VideoImport>> {
64 let params = new HttpParams()
65 params = this.restService.addRestGetParams(params, pagination, sort)
66
67 return this.authHttp
68 .get<ResultList<VideoImport>>(UserService.BASE_USERS_URL + '/me/videos/imports', { params })
69 .pipe(
70 switchMap(res => this.extractVideoImports(res)),
71 map(res => this.restExtractor.convertResultListDateToHuman(res)),
72 catchError(err => this.restExtractor.handleError(err))
73 )
74 } 90 }
75 91
76 private extractVideoImports (result: ResultList<VideoImport>): Observable<ResultList<VideoImport>> { 92 private extractVideoImports (result: ResultList<VideoImport>): Observable<ResultList<VideoImport>> {
diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html
new file mode 100644
index 000000000..409e4de5e
--- /dev/null
+++ b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html
@@ -0,0 +1,60 @@
1<div *ngIf="!hasImportedVideo" class="upload-video-container">
2 <div class="import-video-torrent">
3 <div class="icon icon-upload"></div>
4
5 <div class="form-group">
6 <label i18n for="magnetUri">Magnet URI</label>
7 <my-help
8 helpType="custom" i18n-customHtml
9 customHtml="You can import any torrent file that points to a mp4 file. You should make sure you have diffusion rights over the content it points to, otherwise it could cause legal trouble to yourself and your instance."
10 ></my-help>
11
12 <input type="text" id="magnetUri" [(ngModel)]="magnetUri" />
13 </div>
14
15 <div class="form-group">
16 <label i18n for="first-step-channel">Channel</label>
17 <div class="peertube-select-container">
18 <select id="first-step-channel" [(ngModel)]="firstStepChannelId">
19 <option *ngFor="let channel of userVideoChannels" [value]="channel.id">{{ channel.label }}</option>
20 </select>
21 </div>
22 </div>
23
24 <div class="form-group">
25 <label i18n for="first-step-privacy">Privacy</label>
26 <div class="peertube-select-container">
27 <select id="first-step-privacy" [(ngModel)]="firstStepPrivacyId">
28 <option *ngFor="let privacy of videoPrivacies" [value]="privacy.id">{{ privacy.label }}</option>
29 </select>
30 </div>
31 </div>
32
33 <input
34 type="button" i18n-value value="Import"
35 [disabled]="!isMagnetUrlValid() || isImportingVideo" (click)="importVideo()"
36 />
37 </div>
38</div>
39
40<div *ngIf="hasImportedVideo" class="alert alert-info" i18n>
41 Congratulations, the video will be imported with BitTorrent! You can already add information about this video.
42</div>
43
44<!-- Hidden because we want to load the component -->
45<form [hidden]="!hasImportedVideo" novalidate [formGroup]="form">
46 <my-video-edit
47 [form]="form" [formErrors]="formErrors" [videoCaptions]="videoCaptions" [schedulePublicationPossible]="false"
48 [validationMessages]="validationMessages" [videoPrivacies]="videoPrivacies" [userVideoChannels]="userVideoChannels"
49 ></my-video-edit>
50
51 <div class="submit-container">
52 <div class="submit-button"
53 (click)="updateSecondStep()"
54 [ngClass]="{ disabled: !form.valid || isUpdatingVideo === true }"
55 >
56 <span class="icon icon-validate"></span>
57 <input type="button" i18n-value value="Update" />
58 </div>
59 </div>
60</form>
diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.scss b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.scss
new file mode 100644
index 000000000..1ef5adc25
--- /dev/null
+++ b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.scss
@@ -0,0 +1,37 @@
1@import 'variables';
2@import 'mixins';
3
4$width-size: 190px;
5
6.peertube-select-container {
7 @include peertube-select-container($width-size);
8}
9
10.import-video-torrent {
11 display: flex;
12 flex-direction: column;
13 align-items: center;
14
15 .icon.icon-upload {
16 @include icon(90px);
17 margin-bottom: 25px;
18 cursor: default;
19
20 background-image: url('../../../../assets/images/video/upload.svg');
21 }
22
23 input[type=text] {
24 @include peertube-input-text($width-size);
25 display: block;
26 }
27
28 input[type=button] {
29 @include peertube-button;
30 @include orange-button;
31
32 width: $width-size;
33 margin-top: 30px;
34 }
35}
36
37
diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.ts b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.ts
new file mode 100644
index 000000000..330c37718
--- /dev/null
+++ b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.ts
@@ -0,0 +1,132 @@
1import { Component, EventEmitter, OnInit, Output } from '@angular/core'
2import { Router } from '@angular/router'
3import { NotificationsService } from 'angular2-notifications'
4import { VideoPrivacy, VideoUpdate } from '../../../../../../shared/models/videos'
5import { AuthService, ServerService } from '../../../core'
6import { VideoService } from '../../../shared/video/video.service'
7import { I18n } from '@ngx-translate/i18n-polyfill'
8import { LoadingBarService } from '@ngx-loading-bar/core'
9import { VideoSend } from '@app/videos/+video-edit/video-add-components/video-send'
10import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.service'
11import { VideoEdit } from '@app/shared/video/video-edit.model'
12import { FormValidatorService } from '@app/shared'
13import { VideoCaptionService } from '@app/shared/video-caption'
14import { VideoImportService } from '@app/shared/video-import'
15
16@Component({
17 selector: 'my-video-import-torrent',
18 templateUrl: './video-import-torrent.component.html',
19 styleUrls: [
20 '../shared/video-edit.component.scss',
21 './video-import-torrent.component.scss'
22 ]
23})
24export class VideoImportTorrentComponent extends VideoSend implements OnInit, CanComponentDeactivate {
25 @Output() firstStepDone = new EventEmitter<string>()
26
27 videoFileName: string
28 magnetUri = ''
29
30 isImportingVideo = false
31 hasImportedVideo = false
32 isUpdatingVideo = false
33
34 video: VideoEdit
35
36 protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PRIVATE
37
38 constructor (
39 protected formValidatorService: FormValidatorService,
40 protected loadingBar: LoadingBarService,
41 protected notificationsService: NotificationsService,
42 protected authService: AuthService,
43 protected serverService: ServerService,
44 protected videoService: VideoService,
45 protected videoCaptionService: VideoCaptionService,
46 private router: Router,
47 private videoImportService: VideoImportService,
48 private i18n: I18n
49 ) {
50 super()
51 }
52
53 ngOnInit () {
54 super.ngOnInit()
55 }
56
57 canDeactivate () {
58 return { canDeactivate: true }
59 }
60
61 isMagnetUrlValid () {
62 return !!this.magnetUri
63 }
64
65 importVideo () {
66 this.isImportingVideo = true
67
68 const videoUpdate: VideoUpdate = {
69 privacy: this.firstStepPrivacyId,
70 waitTranscoding: false,
71 commentsEnabled: true,
72 channelId: this.firstStepChannelId
73 }
74
75 this.loadingBar.start()
76
77 this.videoImportService.importVideoTorrent(this.magnetUri, videoUpdate).subscribe(
78 res => {
79 this.loadingBar.complete()
80 this.firstStepDone.emit(res.video.name)
81 this.isImportingVideo = false
82 this.hasImportedVideo = true
83
84 this.video = new VideoEdit(Object.assign(res.video, {
85 commentsEnabled: videoUpdate.commentsEnabled,
86 support: null,
87 thumbnailUrl: null,
88 previewUrl: null
89 }))
90 this.hydrateFormFromVideo()
91 },
92
93 err => {
94 this.loadingBar.complete()
95 this.isImportingVideo = false
96 this.notificationsService.error(this.i18n('Error'), err.message)
97 }
98 )
99 }
100
101 updateSecondStep () {
102 if (this.checkForm() === false) {
103 return
104 }
105
106 this.video.patch(this.form.value)
107
108 this.isUpdatingVideo = true
109
110 // Update the video
111 this.updateVideoAndCaptions(this.video)
112 .subscribe(
113 () => {
114 this.isUpdatingVideo = false
115 this.notificationsService.success(this.i18n('Success'), this.i18n('Video to import updated.'))
116
117 this.router.navigate([ '/my-account', 'video-imports' ])
118 },
119
120 err => {
121 this.isUpdatingVideo = false
122 this.notificationsService.error(this.i18n('Error'), err.message)
123 console.error(err)
124 }
125 )
126
127 }
128
129 private hydrateFormFromVideo () {
130 this.form.patchValue(this.video.toFormPatch())
131 }
132}
diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.html b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.html
index 6b431f6f6..9f5fc6d22 100644
--- a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.html
+++ b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.html
@@ -1,5 +1,5 @@
1<div *ngIf="!hasImportedVideo" class="upload-video-container"> 1<div *ngIf="!hasImportedVideo" class="upload-video-container">
2 <div class="import-video"> 2 <div class="import-video-url">
3 <div class="icon icon-upload"></div> 3 <div class="icon icon-upload"></div>
4 4
5 <div class="form-group"> 5 <div class="form-group">
diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.scss b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.scss
index 5e713ab97..7c6deda1d 100644
--- a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.scss
+++ b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.scss
@@ -7,7 +7,7 @@ $width-size: 190px;
7 @include peertube-select-container($width-size); 7 @include peertube-select-container($width-size);
8} 8}
9 9
10.import-video { 10.import-video-url {
11 display: flex; 11 display: flex;
12 flex-direction: column; 12 flex-direction: column;
13 align-items: center; 13 align-items: center;
diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts
index dbe69409f..842ede732 100644
--- a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts
+++ b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts
@@ -74,7 +74,7 @@ export class VideoImportUrlComponent extends VideoSend implements OnInit, CanCom
74 74
75 this.loadingBar.start() 75 this.loadingBar.start()
76 76
77 this.videoImportService.importVideo(this.targetUrl, videoUpdate).subscribe( 77 this.videoImportService.importVideoUrl(this.targetUrl, videoUpdate).subscribe(
78 res => { 78 res => {
79 this.loadingBar.complete() 79 this.loadingBar.complete()
80 this.firstStepDone.emit(res.video.name) 80 this.firstStepDone.emit(res.video.name)
diff --git a/client/src/app/videos/+video-edit/video-add.component.html b/client/src/app/videos/+video-edit/video-add.component.html
index 7a50372e9..340820180 100644
--- a/client/src/app/videos/+video-edit/video-add.component.html
+++ b/client/src/app/videos/+video-edit/video-add.component.html
@@ -10,8 +10,12 @@
10 <my-video-upload #videoUpload (firstStepDone)="onFirstStepDone('upload', $event)"></my-video-upload> 10 <my-video-upload #videoUpload (firstStepDone)="onFirstStepDone('upload', $event)"></my-video-upload>
11 </tab> 11 </tab>
12 12
13 <tab *ngIf="isVideoImportEnabled()" i18n-heading heading="Import with URL"> 13 <tab *ngIf="isVideoImportHttpEnabled()" i18n-heading heading="Import with URL">
14 <my-video-import-url #videoImportUrl (firstStepDone)="onFirstStepDone('import-url', $event)"></my-video-import-url> 14 <my-video-import-url #videoImportUrl (firstStepDone)="onFirstStepDone('import-url', $event)"></my-video-import-url>
15 </tab> 15 </tab>
16
17 <tab *ngIf="isVideoImportTorrentEnabled()" i18n-heading heading="Import with torrent">
18 <my-video-import-torrent #videoImportTorrent (firstStepDone)="onFirstStepDone('import-torrent', $event)"></my-video-import-torrent>
19 </tab>
16 </tabset> 20 </tabset>
17</div> 21</div>
diff --git a/client/src/app/videos/+video-edit/video-add.component.ts b/client/src/app/videos/+video-edit/video-add.component.ts
index e74fa1f15..7d360598d 100644
--- a/client/src/app/videos/+video-edit/video-add.component.ts
+++ b/client/src/app/videos/+video-edit/video-add.component.ts
@@ -3,6 +3,7 @@ import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.
3import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-components/video-import-url.component' 3import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-components/video-import-url.component'
4import { VideoUploadComponent } from '@app/videos/+video-edit/video-add-components/video-upload.component' 4import { VideoUploadComponent } from '@app/videos/+video-edit/video-add-components/video-upload.component'
5import { ServerService } from '@app/core' 5import { ServerService } from '@app/core'
6import { VideoImportTorrentComponent } from '@app/videos/+video-edit/video-add-components/video-import-torrent.component'
6 7
7@Component({ 8@Component({
8 selector: 'my-videos-add', 9 selector: 'my-videos-add',
@@ -12,15 +13,16 @@ import { ServerService } from '@app/core'
12export class VideoAddComponent implements CanComponentDeactivate { 13export class VideoAddComponent implements CanComponentDeactivate {
13 @ViewChild('videoUpload') videoUpload: VideoUploadComponent 14 @ViewChild('videoUpload') videoUpload: VideoUploadComponent
14 @ViewChild('videoImportUrl') videoImportUrl: VideoImportUrlComponent 15 @ViewChild('videoImportUrl') videoImportUrl: VideoImportUrlComponent
16 @ViewChild('videoImportTorrent') videoImportTorrent: VideoImportTorrentComponent
15 17
16 secondStepType: 'upload' | 'import-url' 18 secondStepType: 'upload' | 'import-url' | 'import-torrent'
17 videoName: string 19 videoName: string
18 20
19 constructor ( 21 constructor (
20 private serverService: ServerService 22 private serverService: ServerService
21 ) {} 23 ) {}
22 24
23 onFirstStepDone (type: 'upload' | 'import-url', videoName: string) { 25 onFirstStepDone (type: 'upload' | 'import-url' | 'import-torrent', videoName: string) {
24 this.secondStepType = type 26 this.secondStepType = type
25 this.videoName = videoName 27 this.videoName = videoName
26 } 28 }
@@ -28,11 +30,16 @@ export class VideoAddComponent implements CanComponentDeactivate {
28 canDeactivate () { 30 canDeactivate () {
29 if (this.secondStepType === 'upload') return this.videoUpload.canDeactivate() 31 if (this.secondStepType === 'upload') return this.videoUpload.canDeactivate()
30 if (this.secondStepType === 'import-url') return this.videoImportUrl.canDeactivate() 32 if (this.secondStepType === 'import-url') return this.videoImportUrl.canDeactivate()
33 if (this.secondStepType === 'import-torrent') return this.videoImportTorrent.canDeactivate()
31 34
32 return { canDeactivate: true } 35 return { canDeactivate: true }
33 } 36 }
34 37
35 isVideoImportEnabled () { 38 isVideoImportHttpEnabled () {
39 return this.serverService.getConfig().import.videos.http.enabled
40 }
41
42 isVideoImportTorrentEnabled () {
36 return this.serverService.getConfig().import.videos.http.enabled 43 return this.serverService.getConfig().import.videos.http.enabled
37 } 44 }
38} 45}
diff --git a/client/src/app/videos/+video-edit/video-add.module.ts b/client/src/app/videos/+video-edit/video-add.module.ts
index a1324b397..3ecf96459 100644
--- a/client/src/app/videos/+video-edit/video-add.module.ts
+++ b/client/src/app/videos/+video-edit/video-add.module.ts
@@ -7,6 +7,7 @@ import { VideoAddComponent } from './video-add.component'
7import { CanDeactivateGuard } from '../../shared/guards/can-deactivate-guard.service' 7import { CanDeactivateGuard } from '../../shared/guards/can-deactivate-guard.service'
8import { VideoUploadComponent } from '@app/videos/+video-edit/video-add-components/video-upload.component' 8import { VideoUploadComponent } from '@app/videos/+video-edit/video-add-components/video-upload.component'
9import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-components/video-import-url.component' 9import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-components/video-import-url.component'
10import { VideoImportTorrentComponent } from '@app/videos/+video-edit/video-add-components/video-import-torrent.component'
10 11
11@NgModule({ 12@NgModule({
12 imports: [ 13 imports: [
@@ -18,7 +19,8 @@ import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-compo
18 declarations: [ 19 declarations: [
19 VideoAddComponent, 20 VideoAddComponent,
20 VideoUploadComponent, 21 VideoUploadComponent,
21 VideoImportUrlComponent 22 VideoImportUrlComponent,
23 VideoImportTorrentComponent
22 ], 24 ],
23 exports: [ 25 exports: [
24 VideoAddComponent 26 VideoAddComponent
diff --git a/package.json b/package.json
index b5d695344..6348bbb6a 100644
--- a/package.json
+++ b/package.json
@@ -134,6 +134,7 @@
134 "uuid": "^3.1.0", 134 "uuid": "^3.1.0",
135 "validator": "^10.2.0", 135 "validator": "^10.2.0",
136 "webfinger.js": "^2.6.6", 136 "webfinger.js": "^2.6.6",
137 "webtorrent": "^0.100.0",
137 "winston": "3.0.0", 138 "winston": "3.0.0",
138 "ws": "^5.0.0", 139 "ws": "^5.0.0",
139 "youtube-dl": "^1.12.2" 140 "youtube-dl": "^1.12.2"
@@ -187,7 +188,6 @@
187 "tslint": "^5.7.0", 188 "tslint": "^5.7.0",
188 "tslint-config-standard": "^7.0.0", 189 "tslint-config-standard": "^7.0.0",
189 "typescript": "^2.5.2", 190 "typescript": "^2.5.2",
190 "webtorrent": "^0.100.0",
191 "xliff": "^3.0.1" 191 "xliff": "^3.0.1"
192 }, 192 },
193 "scripty": { 193 "scripty": {
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 30a7d816c..c16a254d2 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -1,3 +1,4 @@
1import * as magnetUtil from 'magnet-uri'
1import * as express from 'express' 2import * as express from 'express'
2import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger' 3import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger'
3import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' 4import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
@@ -13,6 +14,10 @@ import { VideoImportModel } from '../../../models/video/video-import'
13import { JobQueue } from '../../../lib/job-queue/job-queue' 14import { JobQueue } from '../../../lib/job-queue/job-queue'
14import { processImage } from '../../../helpers/image-utils' 15import { processImage } from '../../../helpers/image-utils'
15import { join } from 'path' 16import { join } from 'path'
17import { isArray } from '../../../helpers/custom-validators/misc'
18import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
19import { VideoChannelModel } from '../../../models/video/video-channel'
20import * as Bluebird from 'bluebird'
16 21
17const auditLogger = auditLoggerFactory('video-imports') 22const auditLogger = auditLoggerFactory('video-imports')
18const videoImportsRouter = express.Router() 23const videoImportsRouter = express.Router()
@@ -41,7 +46,45 @@ export {
41 46
42// --------------------------------------------------------------------------- 47// ---------------------------------------------------------------------------
43 48
44async function addVideoImport (req: express.Request, res: express.Response) { 49function addVideoImport (req: express.Request, res: express.Response) {
50 if (req.body.targetUrl) return addYoutubeDLImport(req, res)
51
52 if (req.body.magnetUri) return addTorrentImport(req, res)
53}
54
55async function addTorrentImport (req: express.Request, res: express.Response) {
56 const body: VideoImportCreate = req.body
57 const magnetUri = body.magnetUri
58
59 const parsed = magnetUtil.decode(magnetUri)
60 const magnetName = isArray(parsed.name) ? parsed.name[0] : parsed.name as string
61
62 const video = buildVideo(res.locals.videoChannel.id, body, { name: magnetName })
63
64 await processThumbnail(req, video)
65 await processPreview(req, video)
66
67 const tags = null
68 const videoImportAttributes = {
69 magnetUri,
70 state: VideoImportState.PENDING
71 }
72 const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
73
74 // Create job to import the video
75 const payload = {
76 type: 'magnet-uri' as 'magnet-uri',
77 videoImportId: videoImport.id,
78 magnetUri
79 }
80 await JobQueue.Instance.createJob({ type: 'video-import', payload })
81
82 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON()))
83
84 return res.json(videoImport.toFormattedJSON()).end()
85}
86
87async function addYoutubeDLImport (req: express.Request, res: express.Response) {
45 const body: VideoImportCreate = req.body 88 const body: VideoImportCreate = req.body
46 const targetUrl = body.targetUrl 89 const targetUrl = body.targetUrl
47 90
@@ -56,53 +99,94 @@ async function addVideoImport (req: express.Request, res: express.Response) {
56 }).end() 99 }).end()
57 } 100 }
58 101
59 // Create video DB object 102 const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
103
104 const downloadThumbnail = !await processThumbnail(req, video)
105 const downloadPreview = !await processPreview(req, video)
106
107 const tags = body.tags || youtubeDLInfo.tags
108 const videoImportAttributes = {
109 targetUrl,
110 state: VideoImportState.PENDING
111 }
112 const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
113
114 // Create job to import the video
115 const payload = {
116 type: 'youtube-dl' as 'youtube-dl',
117 videoImportId: videoImport.id,
118 thumbnailUrl: youtubeDLInfo.thumbnailUrl,
119 downloadThumbnail,
120 downloadPreview
121 }
122 await JobQueue.Instance.createJob({ type: 'video-import', payload })
123
124 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON()))
125
126 return res.json(videoImport.toFormattedJSON()).end()
127}
128
129function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo) {
60 const videoData = { 130 const videoData = {
61 name: body.name || youtubeDLInfo.name, 131 name: body.name || importData.name || 'Unknown name',
62 remote: false, 132 remote: false,
63 category: body.category || youtubeDLInfo.category, 133 category: body.category || importData.category,
64 licence: body.licence || youtubeDLInfo.licence, 134 licence: body.licence || importData.licence,
65 language: body.language || undefined, 135 language: body.language || undefined,
66 commentsEnabled: body.commentsEnabled || true, 136 commentsEnabled: body.commentsEnabled || true,
67 waitTranscoding: body.waitTranscoding || false, 137 waitTranscoding: body.waitTranscoding || false,
68 state: VideoState.TO_IMPORT, 138 state: VideoState.TO_IMPORT,
69 nsfw: body.nsfw || youtubeDLInfo.nsfw || false, 139 nsfw: body.nsfw || importData.nsfw || false,
70 description: body.description || youtubeDLInfo.description, 140 description: body.description || importData.description,
71 support: body.support || null, 141 support: body.support || null,
72 privacy: body.privacy || VideoPrivacy.PRIVATE, 142 privacy: body.privacy || VideoPrivacy.PRIVATE,
73 duration: 0, // duration will be set by the import job 143 duration: 0, // duration will be set by the import job
74 channelId: res.locals.videoChannel.id 144 channelId: channelId
75 } 145 }
76 const video = new VideoModel(videoData) 146 const video = new VideoModel(videoData)
77 video.url = getVideoActivityPubUrl(video) 147 video.url = getVideoActivityPubUrl(video)
78 148
79 // Process thumbnail file? 149 return video
150}
151
152async function processThumbnail (req: express.Request, video: VideoModel) {
80 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined 153 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
81 let downloadThumbnail = true
82 if (thumbnailField) { 154 if (thumbnailField) {
83 const thumbnailPhysicalFile = thumbnailField[ 0 ] 155 const thumbnailPhysicalFile = thumbnailField[ 0 ]
84 await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE) 156 await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
85 downloadThumbnail = false 157
158 return true
86 } 159 }
87 160
88 // Process preview file? 161 return false
162}
163
164async function processPreview (req: express.Request, video: VideoModel) {
89 const previewField = req.files ? req.files['previewfile'] : undefined 165 const previewField = req.files ? req.files['previewfile'] : undefined
90 let downloadPreview = true
91 if (previewField) { 166 if (previewField) {
92 const previewPhysicalFile = previewField[0] 167 const previewPhysicalFile = previewField[0]
93 await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE) 168 await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
94 downloadPreview = false 169
170 return true
95 } 171 }
96 172
97 const videoImport: VideoImportModel = await sequelizeTypescript.transaction(async t => { 173 return false
174}
175
176function insertIntoDB (
177 video: VideoModel,
178 videoChannel: VideoChannelModel,
179 tags: string[],
180 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
181): Bluebird<VideoImportModel> {
182 return sequelizeTypescript.transaction(async t => {
98 const sequelizeOptions = { transaction: t } 183 const sequelizeOptions = { transaction: t }
99 184
100 // Save video object in database 185 // Save video object in database
101 const videoCreated = await video.save(sequelizeOptions) 186 const videoCreated = await video.save(sequelizeOptions)
102 videoCreated.VideoChannel = res.locals.videoChannel 187 videoCreated.VideoChannel = videoChannel
103 188
104 // Set tags to the video 189 // Set tags to the video
105 const tags = body.tags ? body.tags : youtubeDLInfo.tags
106 if (tags !== undefined) { 190 if (tags !== undefined) {
107 const tagInstances = await TagModel.findOrCreateTags(tags, t) 191 const tagInstances = await TagModel.findOrCreateTags(tags, t)
108 192
@@ -111,28 +195,12 @@ async function addVideoImport (req: express.Request, res: express.Response) {
111 } 195 }
112 196
113 // Create video import object in database 197 // Create video import object in database
114 const videoImport = await VideoImportModel.create({ 198 const videoImport = await VideoImportModel.create(
115 targetUrl, 199 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
116 state: VideoImportState.PENDING, 200 sequelizeOptions
117 videoId: videoCreated.id 201 )
118 }, sequelizeOptions)
119
120 videoImport.Video = videoCreated 202 videoImport.Video = videoCreated
121 203
122 return videoImport 204 return videoImport
123 }) 205 })
124
125 // Create job to import the video
126 const payload = {
127 type: 'youtube-dl' as 'youtube-dl',
128 videoImportId: videoImport.id,
129 thumbnailUrl: youtubeDLInfo.thumbnailUrl,
130 downloadThumbnail,
131 downloadPreview
132 }
133 await JobQueue.Instance.createJob({ type: 'video-import', payload })
134
135 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON()))
136
137 return res.json(videoImport.toFormattedJSON()).end()
138} 206}
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index 338c96582..f4c1c8b07 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -17,6 +17,7 @@ import { VideoModel } from '../../models/video/video'
17import { exists, isArray, isFileValid } from './misc' 17import { exists, isArray, isFileValid } from './misc'
18import { VideoChannelModel } from '../../models/video/video-channel' 18import { VideoChannelModel } from '../../models/video/video-channel'
19import { UserModel } from '../../models/account/user' 19import { UserModel } from '../../models/account/user'
20import * as magnetUtil from 'magnet-uri'
20 21
21const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS 22const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
22const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES 23const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES
@@ -126,6 +127,13 @@ function isVideoFileSizeValid (value: string) {
126 return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE) 127 return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE)
127} 128}
128 129
130function isVideoMagnetUriValid (value: string) {
131 if (!exists(value)) return false
132
133 const parsed = magnetUtil.decode(value)
134 return parsed && isVideoFileInfoHashValid(parsed.infoHash)
135}
136
129function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: UserRight, res: Response) { 137function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: UserRight, res: Response) {
130 // Retrieve the user who did the request 138 // Retrieve the user who did the request
131 if (video.isOwned() === false) { 139 if (video.isOwned() === false) {
@@ -214,6 +222,7 @@ export {
214 isScheduleVideoUpdatePrivacyValid, 222 isScheduleVideoUpdatePrivacyValid,
215 isVideoAbuseReasonValid, 223 isVideoAbuseReasonValid,
216 isVideoFile, 224 isVideoFile,
225 isVideoMagnetUriValid,
217 isVideoStateValid, 226 isVideoStateValid,
218 isVideoViewsValid, 227 isVideoViewsValid,
219 isVideoRatingTypeValid, 228 isVideoRatingTypeValid,
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index 7abcec5d7..f4cc5547d 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -9,6 +9,8 @@ import { ApplicationModel } from '../models/application/application'
9import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils' 9import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils'
10import { logger } from './logger' 10import { logger } from './logger'
11import { isArray } from './custom-validators/misc' 11import { isArray } from './custom-validators/misc'
12import * as crypto from "crypto"
13import { join } from "path"
12 14
13const isCidr = require('is-cidr') 15const isCidr = require('is-cidr')
14 16
@@ -181,8 +183,14 @@ async function getServerActor () {
181 return Promise.resolve(serverActor) 183 return Promise.resolve(serverActor)
182} 184}
183 185
186function generateVideoTmpPath (id: string) {
187 const hash = crypto.createHash('sha256').update(id).digest('hex')
188 return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
189}
190
184type SortType = { sortModel: any, sortValue: string } 191type SortType = { sortModel: any, sortValue: string }
185 192
193
186// --------------------------------------------------------------------------- 194// ---------------------------------------------------------------------------
187 195
188export { 196export {
@@ -195,5 +203,6 @@ export {
195 computeResolutionsToTranscode, 203 computeResolutionsToTranscode,
196 resetSequelizeInstance, 204 resetSequelizeInstance,
197 getServerActor, 205 getServerActor,
198 SortType 206 SortType,
207 generateVideoTmpPath
199} 208}
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
new file mode 100644
index 000000000..fce88a1f6
--- /dev/null
+++ b/server/helpers/webtorrent.ts
@@ -0,0 +1,31 @@
1import { logger } from './logger'
2import { generateVideoTmpPath } from './utils'
3import * as WebTorrent from 'webtorrent'
4import { createWriteStream } from 'fs'
5
6function downloadWebTorrentVideo (target: string) {
7 const path = generateVideoTmpPath(target)
8
9 logger.info('Importing torrent video %s', target)
10
11 return new Promise<string>((res, rej) => {
12 const webtorrent = new WebTorrent()
13
14 const torrent = webtorrent.add(target, torrent => {
15 if (torrent.files.length !== 1) throw new Error('The number of files is not equal to 1 for ' + target)
16
17 const file = torrent.files[ 0 ]
18 file.createReadStream().pipe(createWriteStream(path))
19 })
20
21 torrent.on('done', () => res(path))
22
23 torrent.on('error', err => rej(err))
24 })
25}
26
27// ---------------------------------------------------------------------------
28
29export {
30 downloadWebTorrentVideo
31}
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index c59ab9de0..77986f407 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -1,18 +1,17 @@
1import * as youtubeDL from 'youtube-dl' 1import * as youtubeDL from 'youtube-dl'
2import { truncate } from 'lodash' 2import { truncate } from 'lodash'
3import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' 3import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
4import { join } from 'path'
5import * as crypto from 'crypto'
6import { logger } from './logger' 4import { logger } from './logger'
5import { generateVideoTmpPath } from './utils'
7 6
8export type YoutubeDLInfo = { 7export type YoutubeDLInfo = {
9 name: string 8 name?: string
10 description: string 9 description?: string
11 category: number 10 category?: number
12 licence: number 11 licence?: number
13 nsfw: boolean 12 nsfw?: boolean
14 tags: string[] 13 tags?: string[]
15 thumbnailUrl: string 14 thumbnailUrl?: string
16} 15}
17 16
18function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> { 17function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> {
@@ -30,10 +29,9 @@ function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> {
30} 29}
31 30
32function downloadYoutubeDLVideo (url: string) { 31function downloadYoutubeDLVideo (url: string) {
33 const hash = crypto.createHash('sha256').update(url).digest('hex') 32 const path = generateVideoTmpPath(url)
34 const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
35 33
36 logger.info('Importing video %s', url) 34 logger.info('Importing youtubeDL video %s', url)
37 35
38 const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] 36 const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
39 37
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 74fe7965d..243d544ea 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -15,7 +15,7 @@ let config: IConfig = require('config')
15 15
16// --------------------------------------------------------------------------- 16// ---------------------------------------------------------------------------
17 17
18const LAST_MIGRATION_VERSION = 240 18const LAST_MIGRATION_VERSION = 245
19 19
20// --------------------------------------------------------------------------- 20// ---------------------------------------------------------------------------
21 21
@@ -271,7 +271,8 @@ const CONSTRAINTS_FIELDS = {
271 } 271 }
272 }, 272 },
273 VIDEO_IMPORTS: { 273 VIDEO_IMPORTS: {
274 URL: { min: 3, max: 2000 } // Length 274 URL: { min: 3, max: 2000 }, // Length
275 TORRENT_NAME: { min: 3, max: 255 }, // Length
275 }, 276 },
276 VIDEOS: { 277 VIDEOS: {
277 NAME: { min: 3, max: 120 }, // Length 278 NAME: { min: 3, max: 120 }, // Length
diff --git a/server/initializers/migrations/0245-import-magnet.ts b/server/initializers/migrations/0245-import-magnet.ts
new file mode 100644
index 000000000..87603b006
--- /dev/null
+++ b/server/initializers/migrations/0245-import-magnet.ts
@@ -0,0 +1,42 @@
1import * as Sequelize from 'sequelize'
2import { Migration } from '../../models/migrations'
3import { CONSTRAINTS_FIELDS } from '../index'
4
5async function up (utils: {
6 transaction: Sequelize.Transaction
7 queryInterface: Sequelize.QueryInterface
8 sequelize: Sequelize.Sequelize
9}): Promise<any> {
10 {
11 const data = {
12 type: Sequelize.STRING,
13 allowNull: true,
14 defaultValue: null
15 } as Migration.String
16 await utils.queryInterface.changeColumn('videoImport', 'targetUrl', data)
17 }
18
19 {
20 const data = {
21 type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max),
22 allowNull: true,
23 defaultValue: null
24 }
25 await utils.queryInterface.addColumn('videoImport', 'magnetUri', data)
26 }
27
28 {
29 const data = {
30 type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max),
31 allowNull: true,
32 defaultValue: null
33 }
34 await utils.queryInterface.addColumn('videoImport', 'torrentName', data)
35 }
36}
37
38function down (options) {
39 throw new Error('Not implemented.')
40}
41
42export { up, down }
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index cdfe412cc..c457b71fc 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -13,30 +13,99 @@ import { VideoState } from '../../../../shared'
13import { JobQueue } from '../index' 13import { JobQueue } from '../index'
14import { federateVideoIfNeeded } from '../../activitypub' 14import { federateVideoIfNeeded } from '../../activitypub'
15import { VideoModel } from '../../../models/video/video' 15import { VideoModel } from '../../../models/video/video'
16import { downloadWebTorrentVideo } from '../../../helpers/webtorrent'
16 17
17export type VideoImportPayload = { 18type VideoImportYoutubeDLPayload = {
18 type: 'youtube-dl' 19 type: 'youtube-dl'
19 videoImportId: number 20 videoImportId: number
21
20 thumbnailUrl: string 22 thumbnailUrl: string
21 downloadThumbnail: boolean 23 downloadThumbnail: boolean
22 downloadPreview: boolean 24 downloadPreview: boolean
23} 25}
24 26
27type VideoImportTorrentPayload = {
28 type: 'magnet-uri'
29 videoImportId: number
30}
31
32export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
33
25async function processVideoImport (job: Bull.Job) { 34async function processVideoImport (job: Bull.Job) {
26 const payload = job.data as VideoImportPayload 35 const payload = job.data as VideoImportPayload
27 logger.info('Processing video import in job %d.', job.id)
28 36
29 const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId) 37 if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload)
38 if (payload.type === 'magnet-uri') return processTorrentImport(job, payload)
39}
40
41// ---------------------------------------------------------------------------
42
43export {
44 processVideoImport
45}
46
47// ---------------------------------------------------------------------------
48
49async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentPayload) {
50 logger.info('Processing torrent video import in job %d.', job.id)
51
52 const videoImport = await getVideoImportOrDie(payload.videoImportId)
53 const options = {
54 videoImportId: payload.videoImportId,
55
56 downloadThumbnail: false,
57 downloadPreview: false,
58
59 generateThumbnail: true,
60 generatePreview: true
61 }
62 return processFile(() => downloadWebTorrentVideo(videoImport.magnetUri), videoImport, options)
63}
64
65async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) {
66 logger.info('Processing youtubeDL video import in job %d.', job.id)
67
68 const videoImport = await getVideoImportOrDie(payload.videoImportId)
69 const options = {
70 videoImportId: videoImport.id,
71
72 downloadThumbnail: payload.downloadThumbnail,
73 downloadPreview: payload.downloadPreview,
74 thumbnailUrl: payload.thumbnailUrl,
75
76 generateThumbnail: false,
77 generatePreview: false
78 }
79
80 return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl), videoImport, options)
81}
82
83async function getVideoImportOrDie (videoImportId: number) {
84 const videoImport = await VideoImportModel.loadAndPopulateVideo(videoImportId)
30 if (!videoImport || !videoImport.Video) { 85 if (!videoImport || !videoImport.Video) {
31 throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.') 86 throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.')
32 } 87 }
33 88
89 return videoImport
90}
91
92type ProcessFileOptions = {
93 videoImportId: number
94
95 downloadThumbnail: boolean
96 downloadPreview: boolean
97 thumbnailUrl?: string
98
99 generateThumbnail: boolean
100 generatePreview: boolean
101}
102async function processFile (downloader: () => Promise<string>, videoImport: VideoImportModel, options: ProcessFileOptions) {
34 let tempVideoPath: string 103 let tempVideoPath: string
35 let videoDestFile: string 104 let videoDestFile: string
36 let videoFile: VideoFileModel 105 let videoFile: VideoFileModel
37 try { 106 try {
38 // Download video from youtubeDL 107 // Download video from youtubeDL
39 tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl) 108 tempVideoPath = await downloader()
40 109
41 // Get information about this video 110 // Get information about this video
42 const { videoFileResolution } = await getVideoFileResolution(tempVideoPath) 111 const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
@@ -62,23 +131,27 @@ async function processVideoImport (job: Bull.Job) {
62 tempVideoPath = null // This path is not used anymore 131 tempVideoPath = null // This path is not used anymore
63 132
64 // Process thumbnail 133 // Process thumbnail
65 if (payload.downloadThumbnail) { 134 if (options.downloadThumbnail) {
66 if (payload.thumbnailUrl) { 135 if (options.thumbnailUrl) {
67 const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) 136 const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
68 await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath) 137 await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destThumbnailPath)
69 } else { 138 } else {
70 await videoImport.Video.createThumbnail(videoFile) 139 await videoImport.Video.createThumbnail(videoFile)
71 } 140 }
141 } else if (options.generateThumbnail) {
142 await videoImport.Video.createThumbnail(videoFile)
72 } 143 }
73 144
74 // Process preview 145 // Process preview
75 if (payload.downloadPreview) { 146 if (options.downloadPreview) {
76 if (payload.thumbnailUrl) { 147 if (options.thumbnailUrl) {
77 const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) 148 const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
78 await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath) 149 await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destPreviewPath)
79 } else { 150 } else {
80 await videoImport.Video.createPreview(videoFile) 151 await videoImport.Video.createPreview(videoFile)
81 } 152 }
153 } else if (options.generatePreview) {
154 await videoImport.Video.createPreview(videoFile)
82 } 155 }
83 156
84 // Create torrent 157 // Create torrent
@@ -137,9 +210,3 @@ async function processVideoImport (job: Bull.Job) {
137 throw err 210 throw err
138 } 211 }
139} 212}
140
141// ---------------------------------------------------------------------------
142
143export {
144 processVideoImport
145}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 8a24604e1..ddb357db5 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -32,13 +32,6 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
32 'video-import': processVideoImport 32 'video-import': processVideoImport
33} 33}
34 34
35const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = {
36 'activitypub-http-broadcast': true,
37 'activitypub-http-unicast': true,
38 'activitypub-http-fetcher': true,
39 'activitypub-follow': true
40}
41
42const jobTypes: JobType[] = [ 35const jobTypes: JobType[] = [
43 'activitypub-follow', 36 'activitypub-follow',
44 'activitypub-http-broadcast', 37 'activitypub-http-broadcast',
diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts
index d806edfa3..8ec9373fb 100644
--- a/server/middlewares/validators/video-imports.ts
+++ b/server/middlewares/validators/video-imports.ts
@@ -6,14 +6,19 @@ import { areValidationErrors } from './utils'
6import { getCommonVideoAttributes } from './videos' 6import { getCommonVideoAttributes } from './videos'
7import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' 7import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
8import { cleanUpReqFiles } from '../../helpers/utils' 8import { cleanUpReqFiles } from '../../helpers/utils'
9import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos' 9import { isVideoChannelOfAccountExist, isVideoMagnetUriValid, isVideoNameValid } from '../../helpers/custom-validators/videos'
10import { CONFIG } from '../../initializers/constants' 10import { CONFIG } from '../../initializers/constants'
11 11
12const videoImportAddValidator = getCommonVideoAttributes().concat([ 12const videoImportAddValidator = getCommonVideoAttributes().concat([
13 body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'),
14 body('channelId') 13 body('channelId')
15 .toInt() 14 .toInt()
16 .custom(isIdValid).withMessage('Should have correct video channel id'), 15 .custom(isIdValid).withMessage('Should have correct video channel id'),
16 body('targetUrl')
17 .optional()
18 .custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'),
19 body('magnetUri')
20 .optional()
21 .custom(isVideoMagnetUriValid).withMessage('Should have a valid video magnet URI'),
17 body('name') 22 body('name')
18 .optional() 23 .optional()
19 .custom(isVideoNameValid).withMessage('Should have a valid name'), 24 .custom(isVideoNameValid).withMessage('Should have a valid name'),
@@ -34,6 +39,15 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([
34 39
35 if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) 40 if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
36 41
42 // Check we have at least 1 required param
43 if (!req.body.targetUrl && !req.body.magnetUri) {
44 cleanUpReqFiles(req)
45
46 return res.status(400)
47 .json({ error: 'Should have a magnetUri or a targetUrl.' })
48 .end()
49 }
50
37 return next() 51 return next()
38 } 52 }
39]) 53])
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts
index eca87163d..55fca28b8 100644
--- a/server/models/video/video-import.ts
+++ b/server/models/video/video-import.ts
@@ -21,6 +21,7 @@ import { VideoImport, VideoImportState } from '../../../shared'
21import { VideoChannelModel } from './video-channel' 21import { VideoChannelModel } from './video-channel'
22import { AccountModel } from '../account/account' 22import { AccountModel } from '../account/account'
23import { TagModel } from './tag' 23import { TagModel } from './tag'
24import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos'
24 25
25@DefaultScope({ 26@DefaultScope({
26 include: [ 27 include: [
@@ -62,11 +63,23 @@ export class VideoImportModel extends Model<VideoImportModel> {
62 @UpdatedAt 63 @UpdatedAt
63 updatedAt: Date 64 updatedAt: Date
64 65
65 @AllowNull(false) 66 @AllowNull(true)
67 @Default(null)
66 @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl')) 68 @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl'))
67 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) 69 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max))
68 targetUrl: string 70 targetUrl: string
69 71
72 @AllowNull(true)
73 @Default(null)
74 @Is('VideoImportMagnetUri', value => throwIfNotValid(value, isVideoMagnetUriValid, 'magnetUri'))
75 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) // Use the same constraints than URLs
76 magnetUri: string
77
78 @AllowNull(true)
79 @Default(null)
80 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max))
81 torrentName: string
82
70 @AllowNull(false) 83 @AllowNull(false)
71 @Default(null) 84 @Default(null)
72 @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state')) 85 @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state'))
diff --git a/shared/models/videos/video-import-create.model.ts b/shared/models/videos/video-import-create.model.ts
index 65d142c2b..e76084e06 100644
--- a/shared/models/videos/video-import-create.model.ts
+++ b/shared/models/videos/video-import-create.model.ts
@@ -1,6 +1,9 @@
1import { VideoUpdate } from './video-update.model' 1import { VideoUpdate } from './video-update.model'
2 2
3export interface VideoImportCreate extends VideoUpdate { 3export interface VideoImportCreate extends VideoUpdate {
4 targetUrl: string 4 targetUrl?: string
5 magnetUri?: string
6 torrentfile?: Blob
7
5 channelId: number // Required 8 channelId: number // Required
6} 9}