aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorkimsible <kimsible@users.noreply.github.com>2020-04-11 04:24:42 +0200
committerkimsible <kimsible@users.noreply.github.com>2020-04-14 14:39:30 +0200
commit50ad0a1c1699fb1799c9ba2a99bf888894f88df4 (patch)
tree52279eaa419c94b96b1bf9169d396270e6146aff /server
parent5def04e17f1df09a708622afe634ccac256eab86 (diff)
downloadPeerTube-50ad0a1c1699fb1799c9ba2a99bf888894f88df4.tar.gz
PeerTube-50ad0a1c1699fb1799c9ba2a99bf888894f88df4.tar.zst
PeerTube-50ad0a1c1699fb1799c9ba2a99bf888894f88df4.zip
Add getSubs to YoutubeDL video import
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/videos/import.ts29
-rw-r--r--server/helpers/youtube-dl.ts36
2 files changed, 64 insertions, 1 deletions
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index da0832258..e9b9d68d7 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -3,11 +3,13 @@ import * as magnetUtil from 'magnet-uri'
3import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' 3import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
4import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' 4import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
5import { MIMETYPES } from '../../../initializers/constants' 5import { MIMETYPES } from '../../../initializers/constants'
6import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl' 6import { getYoutubeDLInfo, YoutubeDLInfo, getYoutubeDLSubs } from '../../../helpers/youtube-dl'
7import { createReqFiles } from '../../../helpers/express-utils' 7import { createReqFiles } from '../../../helpers/express-utils'
8import { logger } from '../../../helpers/logger' 8import { logger } from '../../../helpers/logger'
9import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared' 9import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
10import { VideoModel } from '../../../models/video/video' 10import { VideoModel } from '../../../models/video/video'
11import { VideoCaptionModel } from '../../../models/video/video-caption'
12import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
11import { getVideoActivityPubUrl } from '../../../lib/activitypub' 13import { getVideoActivityPubUrl } from '../../../lib/activitypub'
12import { TagModel } from '../../../models/video/tag' 14import { TagModel } from '../../../models/video/tag'
13import { VideoImportModel } from '../../../models/video/video-import' 15import { VideoImportModel } from '../../../models/video/video-import'
@@ -28,6 +30,7 @@ import {
28 MThumbnail, 30 MThumbnail,
29 MUser, 31 MUser,
30 MVideoAccountDefault, 32 MVideoAccountDefault,
33 MVideoCaptionVideo,
31 MVideoTag, 34 MVideoTag,
32 MVideoThumbnailAccountDefault, 35 MVideoThumbnailAccountDefault,
33 MVideoWithBlacklistLight 36 MVideoWithBlacklistLight
@@ -136,6 +139,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
136 const targetUrl = body.targetUrl 139 const targetUrl = body.targetUrl
137 const user = res.locals.oauth.token.User 140 const user = res.locals.oauth.token.User
138 141
142 // Get video infos
139 let youtubeDLInfo: YoutubeDLInfo 143 let youtubeDLInfo: YoutubeDLInfo
140 try { 144 try {
141 youtubeDLInfo = await getYoutubeDLInfo(targetUrl) 145 youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
@@ -168,6 +172,29 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
168 user 172 user
169 }) 173 })
170 174
175
176 // Get video subtitles
177 try {
178 const subtitles = await getYoutubeDLSubs(targetUrl)
179
180 for (const subtitle of subtitles) {
181 const videoCaption = new VideoCaptionModel({
182 videoId: video.id,
183 language: subtitle.language
184 }) as MVideoCaptionVideo
185 videoCaption.Video = video
186
187 // Move physical file
188 await moveAndProcessCaptionFile(subtitle, videoCaption)
189
190 await sequelizeTypescript.transaction(async t => {
191 await VideoCaptionModel.insertOrReplaceLanguage(video.id, subtitle.language, null, t)
192 })
193 }
194 } catch (err) {
195 logger.warn('Cannot get video subtitles.', { err })
196 }
197
171 // Create job to import the video 198 // Create job to import the video
172 const payload = { 199 const payload = {
173 type: 'youtube-dl' as 'youtube-dl', 200 type: 'youtube-dl' as 'youtube-dl',
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 07c85797a..277422645 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -20,6 +20,12 @@ export type YoutubeDLInfo = {
20 originallyPublishedAt?: Date 20 originallyPublishedAt?: Date
21} 21}
22 22
23export type YoutubeDLSubs = {
24 language: string,
25 filename: string,
26 path: string
27}[]
28
23const processOptions = { 29const processOptions = {
24 maxBuffer: 1024 * 1024 * 10 // 10MB 30 maxBuffer: 1024 * 1024 * 10 // 10MB
25} 31}
@@ -45,6 +51,35 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
45 }) 51 })
46} 52}
47 53
54function getYoutubeDLSubs (url: string, opts?: object): Promise<YoutubeDLSubs> {
55 return new Promise<YoutubeDLSubs>((res, rej) => {
56 const cwd = CONFIG.STORAGE.TMP_DIR
57 const options = opts || { all: true, format: 'vtt', cwd }
58
59 safeGetYoutubeDL()
60 .then(youtubeDL => {
61 youtubeDL.getSubs(url, options, (err, files) => {
62 if (err) return rej(err)
63
64 const subtitles = files.reduce((acc, filename) => {
65 const matched = filename.match(/\.([a-z]{2})\.(vtt|ttml)/i)
66
67 if (matched[1]) {
68 return [...acc, {
69 language: matched[1],
70 path: join(cwd, filename),
71 filename
72 }]
73 }
74 }, [])
75
76 return res(subtitles)
77 })
78 })
79 .catch(err => rej(err))
80 })
81}
82
48function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) { 83function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) {
49 const path = generateVideoImportTmpPath(url, extension) 84 const path = generateVideoImportTmpPath(url, extension)
50 let timer 85 let timer
@@ -185,6 +220,7 @@ function buildOriginallyPublishedAt (obj: any) {
185export { 220export {
186 updateYoutubeDLBinary, 221 updateYoutubeDLBinary,
187 downloadYoutubeDLVideo, 222 downloadYoutubeDLVideo,
223 getYoutubeDLSubs,
188 getYoutubeDLInfo, 224 getYoutubeDLInfo,
189 safeGetYoutubeDL, 225 safeGetYoutubeDL,
190 buildOriginallyPublishedAt 226 buildOriginallyPublishedAt