aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2020-04-15 14:16:40 +0200
committerChocobozzz <me@florianbigard.com>2020-04-15 14:16:40 +0200
commit62068f4153cb1e67fe30a7f92947c3f2ec058c73 (patch)
tree9253408bc4e96f3dac4afe230eafecc1356c49d8 /server
parentf757be65b8dc2d3b286b5d8b22c64637d7bc2fb8 (diff)
parent652c64165b3d8d1c5d5fc646c29e5cd1c82a3330 (diff)
downloadPeerTube-62068f4153cb1e67fe30a7f92947c3f2ec058c73.tar.gz
PeerTube-62068f4153cb1e67fe30a7f92947c3f2ec058c73.tar.zst
PeerTube-62068f4153cb1e67fe30a7f92947c3f2ec058c73.zip
Merge branch 'pr/2629' into develop
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/videos/import.ts30
-rw-r--r--server/helpers/youtube-dl.ts41
-rw-r--r--server/lib/activitypub/actor.ts2
-rw-r--r--server/models/video/video-import.ts1
-rw-r--r--server/tests/api/videos/video-imports.ts50
5 files changed, 120 insertions, 4 deletions
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index da0832258..fb9d73140 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -3,11 +3,13 @@ import * as magnetUtil from 'magnet-uri'
3import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' 3import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
4import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' 4import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
5import { MIMETYPES } from '../../../initializers/constants' 5import { MIMETYPES } from '../../../initializers/constants'
6import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl' 6import { getYoutubeDLInfo, YoutubeDLInfo, getYoutubeDLSubs } from '../../../helpers/youtube-dl'
7import { createReqFiles } from '../../../helpers/express-utils' 7import { createReqFiles } from '../../../helpers/express-utils'
8import { logger } from '../../../helpers/logger' 8import { logger } from '../../../helpers/logger'
9import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared' 9import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
10import { VideoModel } from '../../../models/video/video' 10import { VideoModel } from '../../../models/video/video'
11import { VideoCaptionModel } from '../../../models/video/video-caption'
12import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
11import { getVideoActivityPubUrl } from '../../../lib/activitypub' 13import { getVideoActivityPubUrl } from '../../../lib/activitypub'
12import { TagModel } from '../../../models/video/tag' 14import { TagModel } from '../../../models/video/tag'
13import { VideoImportModel } from '../../../models/video/video-import' 15import { VideoImportModel } from '../../../models/video/video-import'
@@ -28,6 +30,7 @@ import {
28 MThumbnail, 30 MThumbnail,
29 MUser, 31 MUser,
30 MVideoAccountDefault, 32 MVideoAccountDefault,
33 MVideoCaptionVideo,
31 MVideoTag, 34 MVideoTag,
32 MVideoThumbnailAccountDefault, 35 MVideoThumbnailAccountDefault,
33 MVideoWithBlacklistLight 36 MVideoWithBlacklistLight
@@ -136,6 +139,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
136 const targetUrl = body.targetUrl 139 const targetUrl = body.targetUrl
137 const user = res.locals.oauth.token.User 140 const user = res.locals.oauth.token.User
138 141
142 // Get video infos
139 let youtubeDLInfo: YoutubeDLInfo 143 let youtubeDLInfo: YoutubeDLInfo
140 try { 144 try {
141 youtubeDLInfo = await getYoutubeDLInfo(targetUrl) 145 youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
@@ -168,6 +172,30 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
168 user 172 user
169 }) 173 })
170 174
175 // Get video subtitles
176 try {
177 const subtitles = await getYoutubeDLSubs(targetUrl)
178
179 logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
180
181 for (const subtitle of subtitles) {
182 const videoCaption = new VideoCaptionModel({
183 videoId: video.id,
184 language: subtitle.language
185 }) as MVideoCaptionVideo
186 videoCaption.Video = video
187
188 // Move physical file
189 await moveAndProcessCaptionFile(subtitle, videoCaption)
190
191 await sequelizeTypescript.transaction(async t => {
192 await VideoCaptionModel.insertOrReplaceLanguage(video.id, subtitle.language, null, t)
193 })
194 }
195 } catch (err) {
196 logger.warn('Cannot get video subtitles.', { err })
197 }
198
171 // Create job to import the video 199 // Create job to import the video
172 const payload = { 200 const payload = {
173 type: 'youtube-dl' as 'youtube-dl', 201 type: 'youtube-dl' as 'youtube-dl',
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 07c85797a..6d2e6f6d1 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -20,6 +20,12 @@ export type YoutubeDLInfo = {
20 originallyPublishedAt?: Date 20 originallyPublishedAt?: Date
21} 21}
22 22
23export type YoutubeDLSubs = {
24 language: string
25 filename: string
26 path: string
27}[]
28
23const processOptions = { 29const processOptions = {
24 maxBuffer: 1024 * 1024 * 10 // 10MB 30 maxBuffer: 1024 * 1024 * 10 // 10MB
25} 31}
@@ -45,6 +51,40 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
45 }) 51 })
46} 52}
47 53
54function getYoutubeDLSubs (url: string, opts?: object): Promise<YoutubeDLSubs> {
55 return new Promise<YoutubeDLSubs>((res, rej) => {
56 const cwd = CONFIG.STORAGE.TMP_DIR
57 const options = opts || { all: true, format: 'vtt', cwd }
58
59 safeGetYoutubeDL()
60 .then(youtubeDL => {
61 youtubeDL.getSubs(url, options, (err, files) => {
62 if (err) return rej(err)
63
64 logger.debug('Get subtitles from youtube dl.', { url, files })
65
66 const subtitles = files.reduce((acc, filename) => {
67 const matched = filename.match(/\.([a-z]{2})\.(vtt|ttml)/i)
68
69 if (matched[1]) {
70 return [
71 ...acc,
72 {
73 language: matched[1],
74 path: join(cwd, filename),
75 filename
76 }
77 ]
78 }
79 }, [])
80
81 return res(subtitles)
82 })
83 })
84 .catch(err => rej(err))
85 })
86}
87
48function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) { 88function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) {
49 const path = generateVideoImportTmpPath(url, extension) 89 const path = generateVideoImportTmpPath(url, extension)
50 let timer 90 let timer
@@ -185,6 +225,7 @@ function buildOriginallyPublishedAt (obj: any) {
185export { 225export {
186 updateYoutubeDLBinary, 226 updateYoutubeDLBinary,
187 downloadYoutubeDLVideo, 227 downloadYoutubeDLVideo,
228 getYoutubeDLSubs,
188 getYoutubeDLInfo, 229 getYoutubeDLInfo,
189 safeGetYoutubeDL, 230 safeGetYoutubeDL,
190 buildOriginallyPublishedAt 231 buildOriginallyPublishedAt
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index c3598b75b..8132ac135 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -117,7 +117,7 @@ async function getOrCreateActorAndServerAndModel (
117 if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor 117 if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor
118 118
119 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType) 119 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
120 if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.') 120 if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.')
121 121
122 if ((created === true || refreshed === true) && updateCollections === true) { 122 if ((created === true || refreshed === true) && updateCollections === true) {
123 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } 123 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts
index af5314ce9..fbe0ee0a7 100644
--- a/server/models/video/video-import.ts
+++ b/server/models/video/video-import.ts
@@ -129,6 +129,7 @@ export class VideoImportModel extends Model<VideoImportModel> {
129 distinct: true, 129 distinct: true,
130 include: [ 130 include: [
131 { 131 {
132 attributes: [ 'id' ],
132 model: UserModel.unscoped(), // FIXME: Without this, sequelize try to COUNT(DISTINCT(*)) which is an invalid SQL query 133 model: UserModel.unscoped(), // FIXME: Without this, sequelize try to COUNT(DISTINCT(*)) which is an invalid SQL query
133 required: true 134 required: true
134 } 135 }
diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts
index a67e528c6..8e179b825 100644
--- a/server/tests/api/videos/video-imports.ts
+++ b/server/tests/api/videos/video-imports.ts
@@ -2,7 +2,7 @@
2 2
3import * as chai from 'chai' 3import * as chai from 'chai'
4import 'mocha' 4import 'mocha'
5import { VideoDetails, VideoImport, VideoPrivacy } from '../../../../shared/models/videos' 5import { VideoDetails, VideoImport, VideoPrivacy, VideoCaption } from '../../../../shared/models/videos'
6import { 6import {
7 cleanupTests, 7 cleanupTests,
8 doubleFollow, 8 doubleFollow,
@@ -11,6 +11,8 @@ import {
11 getMyVideos, 11 getMyVideos,
12 getVideo, 12 getVideo,
13 getVideosList, 13 getVideosList,
14 listVideoCaptions,
15 testCaptionFile,
14 immutableAssign, 16 immutableAssign,
15 ServerInfo, 17 ServerInfo,
16 setAccessTokensToServers 18 setAccessTokensToServers
@@ -60,11 +62,14 @@ describe('Test video imports', function () {
60 62
61 expect(videoTorrent.name).to.contain('你好 世界 720p.mp4') 63 expect(videoTorrent.name).to.contain('你好 世界 720p.mp4')
62 expect(videoMagnet.name).to.contain('super peertube2 video') 64 expect(videoMagnet.name).to.contain('super peertube2 video')
65
66 const resCaptions = await listVideoCaptions(url, idHttp)
67 expect(resCaptions.body.total).to.equal(2)
63 } 68 }
64 69
65 async function checkVideoServer2 (url: string, id: number | string) { 70 async function checkVideoServer2 (url: string, id: number | string) {
66 const res = await getVideo(url, id) 71 const res = await getVideo(url, id)
67 const video = res.body 72 const video: VideoDetails = res.body
68 73
69 expect(video.name).to.equal('my super name') 74 expect(video.name).to.equal('my super name')
70 expect(video.category.label).to.equal('Entertainment') 75 expect(video.category.label).to.equal('Entertainment')
@@ -75,6 +80,9 @@ describe('Test video imports', function () {
75 expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ]) 80 expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ])
76 81
77 expect(video.files).to.have.lengthOf(1) 82 expect(video.files).to.have.lengthOf(1)
83
84 const resCaptions = await listVideoCaptions(url, id)
85 expect(resCaptions.body.total).to.equal(2)
78 } 86 }
79 87
80 before(async function () { 88 before(async function () {
@@ -110,6 +118,44 @@ describe('Test video imports', function () {
110 const attributes = immutableAssign(baseAttributes, { targetUrl: getYoutubeVideoUrl() }) 118 const attributes = immutableAssign(baseAttributes, { targetUrl: getYoutubeVideoUrl() })
111 const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) 119 const res = await importVideo(servers[0].url, servers[0].accessToken, attributes)
112 expect(res.body.video.name).to.equal('small video - youtube') 120 expect(res.body.video.name).to.equal('small video - youtube')
121
122 const resCaptions = await listVideoCaptions(servers[0].url, res.body.video.id)
123 const videoCaptions: VideoCaption[] = resCaptions.body.data
124 expect(videoCaptions).to.have.lengthOf(2)
125
126 const enCaption = videoCaptions.find(caption => caption.language.id === 'en')
127 expect(enCaption).to.exist
128 expect(enCaption.language.label).to.equal('English')
129 expect(enCaption.captionPath).to.equal(`/static/video-captions/${res.body.video.uuid}-en.vtt`)
130 await testCaptionFile(servers[0].url, enCaption.captionPath, `WEBVTT
131Kind: captions
132Language: en
133
13400:00:01.600 --> 00:00:04.200
135English (US)
136
13700:00:05.900 --> 00:00:07.999
138This is a subtitle in American English
139
14000:00:10.000 --> 00:00:14.000
141Adding subtitles is very easy to do`)
142
143 const frCaption = videoCaptions.find(caption => caption.language.id === 'fr')
144 expect(frCaption).to.exist
145 expect(frCaption.language.label).to.equal('French')
146 expect(frCaption.captionPath).to.equal(`/static/video-captions/${res.body.video.uuid}-fr.vtt`)
147 await testCaptionFile(servers[0].url, frCaption.captionPath, `WEBVTT
148Kind: captions
149Language: fr
150
15100:00:01.600 --> 00:00:04.200
152Français (FR)
153
15400:00:05.900 --> 00:00:07.999
155C'est un sous-titre français
156
15700:00:10.000 --> 00:00:14.000
158Ajouter un sous-titre est vraiment facile`)
113 } 159 }
114 160
115 { 161 {