diff options
author | Chocobozzz <me@florianbigard.com> | 2020-04-15 14:16:40 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2020-04-15 14:16:40 +0200 |
commit | 62068f4153cb1e67fe30a7f92947c3f2ec058c73 (patch) | |
tree | 9253408bc4e96f3dac4afe230eafecc1356c49d8 | |
parent | f757be65b8dc2d3b286b5d8b22c64637d7bc2fb8 (diff) | |
parent | 652c64165b3d8d1c5d5fc646c29e5cd1c82a3330 (diff) | |
download | PeerTube-62068f4153cb1e67fe30a7f92947c3f2ec058c73.tar.gz PeerTube-62068f4153cb1e67fe30a7f92947c3f2ec058c73.tar.zst PeerTube-62068f4153cb1e67fe30a7f92947c3f2ec058c73.zip |
Merge branch 'pr/2629' into develop
-rw-r--r-- | client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts | 64 | ||||
-rw-r--r-- | server/controllers/api/videos/import.ts | 30 | ||||
-rw-r--r-- | server/helpers/youtube-dl.ts | 41 | ||||
-rw-r--r-- | server/lib/activitypub/actor.ts | 2 | ||||
-rw-r--r-- | server/models/video/video-import.ts | 1 | ||||
-rw-r--r-- | server/tests/api/videos/video-imports.ts | 50 | ||||
-rw-r--r-- | shared/extra-utils/index.ts | 1 |
7 files changed, 160 insertions, 29 deletions
diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts index a5578bebd..a17d73683 100644 --- a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts +++ b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts | |||
@@ -12,6 +12,7 @@ import { FormValidatorService } from '@app/shared' | |||
12 | import { VideoCaptionService } from '@app/shared/video-caption' | 12 | import { VideoCaptionService } from '@app/shared/video-caption' |
13 | import { VideoImportService } from '@app/shared/video-import' | 13 | import { VideoImportService } from '@app/shared/video-import' |
14 | import { scrollToTop } from '@app/shared/misc/utils' | 14 | import { scrollToTop } from '@app/shared/misc/utils' |
15 | import { switchMap, map } from 'rxjs/operators' | ||
15 | 16 | ||
16 | @Component({ | 17 | @Component({ |
17 | selector: 'my-video-import-url', | 18 | selector: 'my-video-import-url', |
@@ -76,31 +77,44 @@ export class VideoImportUrlComponent extends VideoSend implements OnInit, CanCom | |||
76 | 77 | ||
77 | this.loadingBar.start() | 78 | this.loadingBar.start() |
78 | 79 | ||
79 | this.videoImportService.importVideoUrl(this.targetUrl, videoUpdate).subscribe( | 80 | this.videoImportService |
80 | res => { | 81 | .importVideoUrl(this.targetUrl, videoUpdate) |
81 | this.loadingBar.complete() | 82 | .pipe( |
82 | this.firstStepDone.emit(res.video.name) | 83 | switchMap(res => { |
83 | this.isImportingVideo = false | 84 | return this.videoCaptionService |
84 | this.hasImportedVideo = true | 85 | .listCaptions(res.video.id) |
85 | 86 | .pipe( | |
86 | this.video = new VideoEdit(Object.assign(res.video, { | 87 | map(result => ({ video: res.video, videoCaptions: result.data })) |
87 | commentsEnabled: videoUpdate.commentsEnabled, | 88 | ) |
88 | downloadEnabled: videoUpdate.downloadEnabled, | 89 | }) |
89 | support: null, | 90 | ) |
90 | thumbnailUrl: null, | 91 | .subscribe( |
91 | previewUrl: null | 92 | ({ video, videoCaptions }) => { |
92 | })) | 93 | this.loadingBar.complete() |
93 | 94 | this.firstStepDone.emit(video.name) | |
94 | this.hydrateFormFromVideo() | 95 | this.isImportingVideo = false |
95 | }, | 96 | this.hasImportedVideo = true |
96 | 97 | ||
97 | err => { | 98 | this.video = new VideoEdit(Object.assign(video, { |
98 | this.loadingBar.complete() | 99 | commentsEnabled: videoUpdate.commentsEnabled, |
99 | this.isImportingVideo = false | 100 | downloadEnabled: videoUpdate.downloadEnabled, |
100 | this.firstStepError.emit() | 101 | support: null, |
101 | this.notifier.error(err.message) | 102 | thumbnailUrl: null, |
102 | } | 103 | previewUrl: null |
103 | ) | 104 | })) |
105 | |||
106 | this.videoCaptions = videoCaptions | ||
107 | |||
108 | this.hydrateFormFromVideo() | ||
109 | }, | ||
110 | |||
111 | err => { | ||
112 | this.loadingBar.complete() | ||
113 | this.isImportingVideo = false | ||
114 | this.firstStepError.emit() | ||
115 | this.notifier.error(err.message) | ||
116 | } | ||
117 | ) | ||
104 | } | 118 | } |
105 | 119 | ||
106 | updateSecondStep () { | 120 | updateSecondStep () { |
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index da0832258..fb9d73140 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts | |||
@@ -3,11 +3,13 @@ import * as magnetUtil from 'magnet-uri' | |||
3 | import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' | 3 | import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' |
4 | import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' | 4 | import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' |
5 | import { MIMETYPES } from '../../../initializers/constants' | 5 | import { MIMETYPES } from '../../../initializers/constants' |
6 | import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl' | 6 | import { getYoutubeDLInfo, YoutubeDLInfo, getYoutubeDLSubs } from '../../../helpers/youtube-dl' |
7 | import { createReqFiles } from '../../../helpers/express-utils' | 7 | import { createReqFiles } from '../../../helpers/express-utils' |
8 | import { logger } from '../../../helpers/logger' | 8 | import { logger } from '../../../helpers/logger' |
9 | import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared' | 9 | import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared' |
10 | import { VideoModel } from '../../../models/video/video' | 10 | import { VideoModel } from '../../../models/video/video' |
11 | import { VideoCaptionModel } from '../../../models/video/video-caption' | ||
12 | import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils' | ||
11 | import { getVideoActivityPubUrl } from '../../../lib/activitypub' | 13 | import { getVideoActivityPubUrl } from '../../../lib/activitypub' |
12 | import { TagModel } from '../../../models/video/tag' | 14 | import { TagModel } from '../../../models/video/tag' |
13 | import { VideoImportModel } from '../../../models/video/video-import' | 15 | import { VideoImportModel } from '../../../models/video/video-import' |
@@ -28,6 +30,7 @@ import { | |||
28 | MThumbnail, | 30 | MThumbnail, |
29 | MUser, | 31 | MUser, |
30 | MVideoAccountDefault, | 32 | MVideoAccountDefault, |
33 | MVideoCaptionVideo, | ||
31 | MVideoTag, | 34 | MVideoTag, |
32 | MVideoThumbnailAccountDefault, | 35 | MVideoThumbnailAccountDefault, |
33 | MVideoWithBlacklistLight | 36 | MVideoWithBlacklistLight |
@@ -136,6 +139,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) | |||
136 | const targetUrl = body.targetUrl | 139 | const targetUrl = body.targetUrl |
137 | const user = res.locals.oauth.token.User | 140 | const user = res.locals.oauth.token.User |
138 | 141 | ||
142 | // Get video infos | ||
139 | let youtubeDLInfo: YoutubeDLInfo | 143 | let youtubeDLInfo: YoutubeDLInfo |
140 | try { | 144 | try { |
141 | youtubeDLInfo = await getYoutubeDLInfo(targetUrl) | 145 | youtubeDLInfo = await getYoutubeDLInfo(targetUrl) |
@@ -168,6 +172,30 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) | |||
168 | user | 172 | user |
169 | }) | 173 | }) |
170 | 174 | ||
175 | // Get video subtitles | ||
176 | try { | ||
177 | const subtitles = await getYoutubeDLSubs(targetUrl) | ||
178 | |||
179 | logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl) | ||
180 | |||
181 | for (const subtitle of subtitles) { | ||
182 | const videoCaption = new VideoCaptionModel({ | ||
183 | videoId: video.id, | ||
184 | language: subtitle.language | ||
185 | }) as MVideoCaptionVideo | ||
186 | videoCaption.Video = video | ||
187 | |||
188 | // Move physical file | ||
189 | await moveAndProcessCaptionFile(subtitle, videoCaption) | ||
190 | |||
191 | await sequelizeTypescript.transaction(async t => { | ||
192 | await VideoCaptionModel.insertOrReplaceLanguage(video.id, subtitle.language, null, t) | ||
193 | }) | ||
194 | } | ||
195 | } catch (err) { | ||
196 | logger.warn('Cannot get video subtitles.', { err }) | ||
197 | } | ||
198 | |||
171 | // Create job to import the video | 199 | // Create job to import the video |
172 | const payload = { | 200 | const payload = { |
173 | type: 'youtube-dl' as 'youtube-dl', | 201 | type: 'youtube-dl' as 'youtube-dl', |
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index 07c85797a..6d2e6f6d1 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts | |||
@@ -20,6 +20,12 @@ export type YoutubeDLInfo = { | |||
20 | originallyPublishedAt?: Date | 20 | originallyPublishedAt?: Date |
21 | } | 21 | } |
22 | 22 | ||
23 | export type YoutubeDLSubs = { | ||
24 | language: string | ||
25 | filename: string | ||
26 | path: string | ||
27 | }[] | ||
28 | |||
23 | const processOptions = { | 29 | const processOptions = { |
24 | maxBuffer: 1024 * 1024 * 10 // 10MB | 30 | maxBuffer: 1024 * 1024 * 10 // 10MB |
25 | } | 31 | } |
@@ -45,6 +51,40 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> | |||
45 | }) | 51 | }) |
46 | } | 52 | } |
47 | 53 | ||
54 | function getYoutubeDLSubs (url: string, opts?: object): Promise<YoutubeDLSubs> { | ||
55 | return new Promise<YoutubeDLSubs>((res, rej) => { | ||
56 | const cwd = CONFIG.STORAGE.TMP_DIR | ||
57 | const options = opts || { all: true, format: 'vtt', cwd } | ||
58 | |||
59 | safeGetYoutubeDL() | ||
60 | .then(youtubeDL => { | ||
61 | youtubeDL.getSubs(url, options, (err, files) => { | ||
62 | if (err) return rej(err) | ||
63 | |||
64 | logger.debug('Get subtitles from youtube dl.', { url, files }) | ||
65 | |||
66 | const subtitles = files.reduce((acc, filename) => { | ||
67 | const matched = filename.match(/\.([a-z]{2})\.(vtt|ttml)/i) | ||
68 | |||
69 | if (matched[1]) { | ||
70 | return [ | ||
71 | ...acc, | ||
72 | { | ||
73 | language: matched[1], | ||
74 | path: join(cwd, filename), | ||
75 | filename | ||
76 | } | ||
77 | ] | ||
78 | } | ||
79 | }, []) | ||
80 | |||
81 | return res(subtitles) | ||
82 | }) | ||
83 | }) | ||
84 | .catch(err => rej(err)) | ||
85 | }) | ||
86 | } | ||
87 | |||
48 | function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) { | 88 | function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) { |
49 | const path = generateVideoImportTmpPath(url, extension) | 89 | const path = generateVideoImportTmpPath(url, extension) |
50 | let timer | 90 | let timer |
@@ -185,6 +225,7 @@ function buildOriginallyPublishedAt (obj: any) { | |||
185 | export { | 225 | export { |
186 | updateYoutubeDLBinary, | 226 | updateYoutubeDLBinary, |
187 | downloadYoutubeDLVideo, | 227 | downloadYoutubeDLVideo, |
228 | getYoutubeDLSubs, | ||
188 | getYoutubeDLInfo, | 229 | getYoutubeDLInfo, |
189 | safeGetYoutubeDL, | 230 | safeGetYoutubeDL, |
190 | buildOriginallyPublishedAt | 231 | buildOriginallyPublishedAt |
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index c3598b75b..8132ac135 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts | |||
@@ -117,7 +117,7 @@ async function getOrCreateActorAndServerAndModel ( | |||
117 | if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor | 117 | if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor |
118 | 118 | ||
119 | const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType) | 119 | const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType) |
120 | if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.') | 120 | if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.') |
121 | 121 | ||
122 | if ((created === true || refreshed === true) && updateCollections === true) { | 122 | if ((created === true || refreshed === true) && updateCollections === true) { |
123 | const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } | 123 | const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } |
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts index af5314ce9..fbe0ee0a7 100644 --- a/server/models/video/video-import.ts +++ b/server/models/video/video-import.ts | |||
@@ -129,6 +129,7 @@ export class VideoImportModel extends Model<VideoImportModel> { | |||
129 | distinct: true, | 129 | distinct: true, |
130 | include: [ | 130 | include: [ |
131 | { | 131 | { |
132 | attributes: [ 'id' ], | ||
132 | model: UserModel.unscoped(), // FIXME: Without this, sequelize try to COUNT(DISTINCT(*)) which is an invalid SQL query | 133 | model: UserModel.unscoped(), // FIXME: Without this, sequelize try to COUNT(DISTINCT(*)) which is an invalid SQL query |
133 | required: true | 134 | required: true |
134 | } | 135 | } |
diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts index a67e528c6..8e179b825 100644 --- a/server/tests/api/videos/video-imports.ts +++ b/server/tests/api/videos/video-imports.ts | |||
@@ -2,7 +2,7 @@ | |||
2 | 2 | ||
3 | import * as chai from 'chai' | 3 | import * as chai from 'chai' |
4 | import 'mocha' | 4 | import 'mocha' |
5 | import { VideoDetails, VideoImport, VideoPrivacy } from '../../../../shared/models/videos' | 5 | import { VideoDetails, VideoImport, VideoPrivacy, VideoCaption } from '../../../../shared/models/videos' |
6 | import { | 6 | import { |
7 | cleanupTests, | 7 | cleanupTests, |
8 | doubleFollow, | 8 | doubleFollow, |
@@ -11,6 +11,8 @@ import { | |||
11 | getMyVideos, | 11 | getMyVideos, |
12 | getVideo, | 12 | getVideo, |
13 | getVideosList, | 13 | getVideosList, |
14 | listVideoCaptions, | ||
15 | testCaptionFile, | ||
14 | immutableAssign, | 16 | immutableAssign, |
15 | ServerInfo, | 17 | ServerInfo, |
16 | setAccessTokensToServers | 18 | setAccessTokensToServers |
@@ -60,11 +62,14 @@ describe('Test video imports', function () { | |||
60 | 62 | ||
61 | expect(videoTorrent.name).to.contain('你好 世界 720p.mp4') | 63 | expect(videoTorrent.name).to.contain('你好 世界 720p.mp4') |
62 | expect(videoMagnet.name).to.contain('super peertube2 video') | 64 | expect(videoMagnet.name).to.contain('super peertube2 video') |
65 | |||
66 | const resCaptions = await listVideoCaptions(url, idHttp) | ||
67 | expect(resCaptions.body.total).to.equal(2) | ||
63 | } | 68 | } |
64 | 69 | ||
65 | async function checkVideoServer2 (url: string, id: number | string) { | 70 | async function checkVideoServer2 (url: string, id: number | string) { |
66 | const res = await getVideo(url, id) | 71 | const res = await getVideo(url, id) |
67 | const video = res.body | 72 | const video: VideoDetails = res.body |
68 | 73 | ||
69 | expect(video.name).to.equal('my super name') | 74 | expect(video.name).to.equal('my super name') |
70 | expect(video.category.label).to.equal('Entertainment') | 75 | expect(video.category.label).to.equal('Entertainment') |
@@ -75,6 +80,9 @@ describe('Test video imports', function () { | |||
75 | expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ]) | 80 | expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ]) |
76 | 81 | ||
77 | expect(video.files).to.have.lengthOf(1) | 82 | expect(video.files).to.have.lengthOf(1) |
83 | |||
84 | const resCaptions = await listVideoCaptions(url, id) | ||
85 | expect(resCaptions.body.total).to.equal(2) | ||
78 | } | 86 | } |
79 | 87 | ||
80 | before(async function () { | 88 | before(async function () { |
@@ -110,6 +118,44 @@ describe('Test video imports', function () { | |||
110 | const attributes = immutableAssign(baseAttributes, { targetUrl: getYoutubeVideoUrl() }) | 118 | const attributes = immutableAssign(baseAttributes, { targetUrl: getYoutubeVideoUrl() }) |
111 | const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) | 119 | const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) |
112 | expect(res.body.video.name).to.equal('small video - youtube') | 120 | expect(res.body.video.name).to.equal('small video - youtube') |
121 | |||
122 | const resCaptions = await listVideoCaptions(servers[0].url, res.body.video.id) | ||
123 | const videoCaptions: VideoCaption[] = resCaptions.body.data | ||
124 | expect(videoCaptions).to.have.lengthOf(2) | ||
125 | |||
126 | const enCaption = videoCaptions.find(caption => caption.language.id === 'en') | ||
127 | expect(enCaption).to.exist | ||
128 | expect(enCaption.language.label).to.equal('English') | ||
129 | expect(enCaption.captionPath).to.equal(`/static/video-captions/${res.body.video.uuid}-en.vtt`) | ||
130 | await testCaptionFile(servers[0].url, enCaption.captionPath, `WEBVTT | ||
131 | Kind: captions | ||
132 | Language: en | ||
133 | |||
134 | 00:00:01.600 --> 00:00:04.200 | ||
135 | English (US) | ||
136 | |||
137 | 00:00:05.900 --> 00:00:07.999 | ||
138 | This is a subtitle in American English | ||
139 | |||
140 | 00:00:10.000 --> 00:00:14.000 | ||
141 | Adding subtitles is very easy to do`) | ||
142 | |||
143 | const frCaption = videoCaptions.find(caption => caption.language.id === 'fr') | ||
144 | expect(frCaption).to.exist | ||
145 | expect(frCaption.language.label).to.equal('French') | ||
146 | expect(frCaption.captionPath).to.equal(`/static/video-captions/${res.body.video.uuid}-fr.vtt`) | ||
147 | await testCaptionFile(servers[0].url, frCaption.captionPath, `WEBVTT | ||
148 | Kind: captions | ||
149 | Language: fr | ||
150 | |||
151 | 00:00:01.600 --> 00:00:04.200 | ||
152 | Français (FR) | ||
153 | |||
154 | 00:00:05.900 --> 00:00:07.999 | ||
155 | C'est un sous-titre français | ||
156 | |||
157 | 00:00:10.000 --> 00:00:14.000 | ||
158 | Ajouter un sous-titre est vraiment facile`) | ||
113 | } | 159 | } |
114 | 160 | ||
115 | { | 161 | { |
diff --git a/shared/extra-utils/index.ts b/shared/extra-utils/index.ts index 78acf72aa..fd8fef5dc 100644 --- a/shared/extra-utils/index.ts +++ b/shared/extra-utils/index.ts | |||
@@ -18,6 +18,7 @@ export * from './users/users' | |||
18 | export * from './users/accounts' | 18 | export * from './users/accounts' |
19 | export * from './videos/video-abuses' | 19 | export * from './videos/video-abuses' |
20 | export * from './videos/video-blacklist' | 20 | export * from './videos/video-blacklist' |
21 | export * from './videos/video-captions' | ||
21 | export * from './videos/video-channels' | 22 | export * from './videos/video-channels' |
22 | export * from './videos/video-comments' | 23 | export * from './videos/video-comments' |
23 | export * from './videos/video-streaming-playlists' | 24 | export * from './videos/video-streaming-playlists' |