aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2020-04-03 15:41:39 +0200
committerChocobozzz <me@florianbigard.com>2020-04-03 15:41:39 +0200
commitd57d1d83c6a4d98a735b21f4e8e749a5c1e1a479 (patch)
treed625ca7c2c626fcf186bb55c6852c1e5a7b51c9e
parent1fe654e0963da8c2801561be10de3222055a2497 (diff)
downloadPeerTube-d57d1d83c6a4d98a735b21f4e8e749a5c1e1a479.tar.gz
PeerTube-d57d1d83c6a4d98a735b21f4e8e749a5c1e1a479.tar.zst
PeerTube-d57d1d83c6a4d98a735b21f4e8e749a5c1e1a479.zip
Support audio files import
-rw-r--r--server/controllers/api/videos/import.ts5
-rw-r--r--server/controllers/api/videos/index.ts28
-rw-r--r--server/helpers/utils.ts14
-rw-r--r--server/helpers/youtube-dl.ts10
-rw-r--r--server/lib/job-queue/handlers/video-import.ts35
-rw-r--r--server/lib/videos.ts27
6 files changed, 73 insertions, 46 deletions
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index ed223cbc9..da0832258 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -174,7 +174,10 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
174 videoImportId: videoImport.id, 174 videoImportId: videoImport.id,
175 thumbnailUrl: youtubeDLInfo.thumbnailUrl, 175 thumbnailUrl: youtubeDLInfo.thumbnailUrl,
176 downloadThumbnail: !thumbnailModel, 176 downloadThumbnail: !thumbnailModel,
177 downloadPreview: !previewModel 177 downloadPreview: !previewModel,
178 fileExt: youtubeDLInfo.fileExt
179 ? `.${youtubeDLInfo.fileExt}`
180 : '.mp4'
178 } 181 }
179 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload }) 182 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
180 183
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 9b19c394d..04d775cbf 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -1,7 +1,7 @@
1import * as express from 'express' 1import * as express from 'express'
2import { extname } from 'path' 2import { extname } from 'path'
3import { VideoCreate, VideoPrivacy, VideoState, VideoUpdate } from '../../../../shared' 3import { VideoCreate, VideoPrivacy, VideoState, VideoUpdate } from '../../../../shared'
4import { getVideoFileFPS, getVideoFileResolution, getMetadataFromFile } from '../../../helpers/ffmpeg-utils' 4import { getMetadataFromFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
5import { logger } from '../../../helpers/logger' 5import { logger } from '../../../helpers/logger'
6import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' 6import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
7import { getFormattedObjects, getServerActor } from '../../../helpers/utils' 7import { getFormattedObjects, getServerActor } from '../../../helpers/utils'
@@ -32,13 +32,13 @@ import {
32 paginationValidator, 32 paginationValidator,
33 setDefaultPagination, 33 setDefaultPagination,
34 setDefaultSort, 34 setDefaultSort,
35 videoFileMetadataGetValidator,
35 videosAddValidator, 36 videosAddValidator,
36 videosCustomGetValidator, 37 videosCustomGetValidator,
37 videosGetValidator, 38 videosGetValidator,
38 videosRemoveValidator, 39 videosRemoveValidator,
39 videosSortValidator, 40 videosSortValidator,
40 videosUpdateValidator, 41 videosUpdateValidator
41 videoFileMetadataGetValidator
42} from '../../../middlewares' 42} from '../../../middlewares'
43import { TagModel } from '../../../models/video/tag' 43import { TagModel } from '../../../models/video/tag'
44import { VideoModel } from '../../../models/video/video' 44import { VideoModel } from '../../../models/video/video'
@@ -62,12 +62,12 @@ import { CONFIG } from '../../../initializers/config'
62import { sequelizeTypescript } from '../../../initializers/database' 62import { sequelizeTypescript } from '../../../initializers/database'
63import { createVideoMiniatureFromExisting, generateVideoMiniature } from '../../../lib/thumbnail' 63import { createVideoMiniatureFromExisting, generateVideoMiniature } from '../../../lib/thumbnail'
64import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' 64import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
65import { VideoTranscodingPayload } from '../../../lib/job-queue/handlers/video-transcoding'
66import { Hooks } from '../../../lib/plugins/hooks' 65import { Hooks } from '../../../lib/plugins/hooks'
67import { MVideoDetails, MVideoFullLight } from '@server/typings/models' 66import { MVideoDetails, MVideoFullLight } from '@server/typings/models'
68import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 67import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
69import { getVideoFilePath } from '@server/lib/video-paths' 68import { getVideoFilePath } from '@server/lib/video-paths'
70import toInt from 'validator/lib/toInt' 69import toInt from 'validator/lib/toInt'
70import { addOptimizeOrMergeAudioJob } from '@server/lib/videos'
71 71
72const auditLogger = auditLoggerFactory('videos') 72const auditLogger = auditLoggerFactory('videos')
73const videosRouter = express.Router() 73const videosRouter = express.Router()
@@ -296,25 +296,7 @@ async function addVideo (req: express.Request, res: express.Response) {
296 Notifier.Instance.notifyOnNewVideoIfNeeded(videoCreated) 296 Notifier.Instance.notifyOnNewVideoIfNeeded(videoCreated)
297 297
298 if (video.state === VideoState.TO_TRANSCODE) { 298 if (video.state === VideoState.TO_TRANSCODE) {
299 // Put uuid because we don't have id auto incremented for now 299 await addOptimizeOrMergeAudioJob(videoCreated, videoFile)
300 let dataInput: VideoTranscodingPayload
301
302 if (videoFile.isAudio()) {
303 dataInput = {
304 type: 'merge-audio' as 'merge-audio',
305 resolution: DEFAULT_AUDIO_RESOLUTION,
306 videoUUID: videoCreated.uuid,
307 isNewVideo: true
308 }
309 } else {
310 dataInput = {
311 type: 'optimize' as 'optimize',
312 videoUUID: videoCreated.uuid,
313 isNewVideo: true
314 }
315 }
316
317 await JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput })
318 } 300 }
319 301
320 Hooks.runAction('action:api.video.uploaded', { video: videoCreated }) 302 Hooks.runAction('action:api.video.uploaded', { video: videoCreated })
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index 7a4c781cc..11c118292 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -7,6 +7,7 @@ import { Instance as ParseTorrent } from 'parse-torrent'
7import { remove } from 'fs-extra' 7import { remove } from 'fs-extra'
8import * as memoizee from 'memoizee' 8import * as memoizee from 'memoizee'
9import { CONFIG } from '../initializers/config' 9import { CONFIG } from '../initializers/config'
10import { isVideoFileExtnameValid } from './custom-validators/videos'
10 11
11function deleteFileAsync (path: string) { 12function deleteFileAsync (path: string) {
12 remove(path) 13 remove(path)
@@ -42,11 +43,18 @@ const getServerActor = memoizee(async function () {
42 return actor 43 return actor
43}, { promise: true }) 44}, { promise: true })
44 45
45function generateVideoImportTmpPath (target: string | ParseTorrent) { 46function generateVideoImportTmpPath (target: string | ParseTorrent, extensionArg?: string) {
46 const id = typeof target === 'string' ? target : target.infoHash 47 const id = typeof target === 'string'
48 ? target
49 : target.infoHash
50
51 let extension = '.mp4'
52 if (extensionArg && isVideoFileExtnameValid(extensionArg)) {
53 extension = extensionArg
54 }
47 55
48 const hash = sha256(id) 56 const hash = sha256(id)
49 return join(CONFIG.STORAGE.TMP_DIR, hash + '-import.mp4') 57 return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`)
50} 58}
51 59
52function getSecureTorrentName (originalName: string) { 60function getSecureTorrentName (originalName: string) {
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 26dbe6543..07c85797a 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -16,6 +16,7 @@ export type YoutubeDLInfo = {
16 nsfw?: boolean 16 nsfw?: boolean
17 tags?: string[] 17 tags?: string[]
18 thumbnailUrl?: string 18 thumbnailUrl?: string
19 fileExt?: string
19 originallyPublishedAt?: Date 20 originallyPublishedAt?: Date
20} 21}
21 22
@@ -44,11 +45,11 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
44 }) 45 })
45} 46}
46 47
47function downloadYoutubeDLVideo (url: string, timeout: number) { 48function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) {
48 const path = generateVideoImportTmpPath(url) 49 const path = generateVideoImportTmpPath(url, extension)
49 let timer 50 let timer
50 51
51 logger.info('Importing youtubeDL video %s', url) 52 logger.info('Importing youtubeDL video %s to %s', url, path)
52 53
53 let options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] 54 let options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
54 options = wrapWithProxyOptions(options) 55 options = wrapWithProxyOptions(options)
@@ -219,7 +220,8 @@ function buildVideoInfo (obj: any) {
219 nsfw: isNSFW(obj), 220 nsfw: isNSFW(obj),
220 tags: getTags(obj.tags), 221 tags: getTags(obj.tags),
221 thumbnailUrl: obj.thumbnail || undefined, 222 thumbnailUrl: obj.thumbnail || undefined,
222 originallyPublishedAt: buildOriginallyPublishedAt(obj) 223 originallyPublishedAt: buildOriginallyPublishedAt(obj),
224 fileExt: obj.ext
223 } 225 }
224} 226}
225 227
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 09f225cec..d8052da72 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -8,7 +8,6 @@ import { extname } from 'path'
8import { VideoFileModel } from '../../../models/video/video-file' 8import { VideoFileModel } from '../../../models/video/video-file'
9import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' 9import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
10import { VideoState } from '../../../../shared' 10import { VideoState } from '../../../../shared'
11import { JobQueue } from '../index'
12import { federateVideoIfNeeded } from '../../activitypub' 11import { federateVideoIfNeeded } from '../../activitypub'
13import { VideoModel } from '../../../models/video/video' 12import { VideoModel } from '../../../models/video/video'
14import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' 13import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
@@ -22,6 +21,7 @@ import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
22import { MThumbnail } from '../../../typings/models/video/thumbnail' 21import { MThumbnail } from '../../../typings/models/video/thumbnail'
23import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import' 22import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import'
24import { getVideoFilePath } from '@server/lib/video-paths' 23import { getVideoFilePath } from '@server/lib/video-paths'
24import { addOptimizeOrMergeAudioJob } from '@server/lib/videos'
25 25
26type VideoImportYoutubeDLPayload = { 26type VideoImportYoutubeDLPayload = {
27 type: 'youtube-dl' 27 type: 'youtube-dl'
@@ -30,6 +30,8 @@ type VideoImportYoutubeDLPayload = {
30 thumbnailUrl: string 30 thumbnailUrl: string
31 downloadThumbnail: boolean 31 downloadThumbnail: boolean
32 downloadPreview: boolean 32 downloadPreview: boolean
33
34 fileExt?: string
33} 35}
34 36
35type VideoImportTorrentPayload = { 37type VideoImportTorrentPayload = {
@@ -90,7 +92,7 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
90 generatePreview: false 92 generatePreview: false
91 } 93 }
92 94
93 return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl, VIDEO_IMPORT_TIMEOUT), videoImport, options) 95 return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT), videoImport, options)
94} 96}
95 97
96async function getVideoImportOrDie (videoImportId: number) { 98async function getVideoImportOrDie (videoImportId: number) {
@@ -154,16 +156,28 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
154 // Process thumbnail 156 // Process thumbnail
155 let thumbnailModel: MThumbnail 157 let thumbnailModel: MThumbnail
156 if (options.downloadThumbnail && options.thumbnailUrl) { 158 if (options.downloadThumbnail && options.thumbnailUrl) {
157 thumbnailModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImportWithFiles.Video, ThumbnailType.MINIATURE) 159 try {
158 } else if (options.generateThumbnail || options.downloadThumbnail) { 160 thumbnailModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImportWithFiles.Video, ThumbnailType.MINIATURE)
161 } catch (err) {
162 logger.warn('Cannot generate video thumbnail %s for %s.', options.thumbnailUrl, videoImportWithFiles.Video.url, { err })
163 }
164 }
165
166 if (!thumbnailModel && (options.generateThumbnail || options.downloadThumbnail)) {
159 thumbnailModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.MINIATURE) 167 thumbnailModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.MINIATURE)
160 } 168 }
161 169
162 // Process preview 170 // Process preview
163 let previewModel: MThumbnail 171 let previewModel: MThumbnail
164 if (options.downloadPreview && options.thumbnailUrl) { 172 if (options.downloadPreview && options.thumbnailUrl) {
165 previewModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImportWithFiles.Video, ThumbnailType.PREVIEW) 173 try {
166 } else if (options.generatePreview || options.downloadPreview) { 174 previewModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImportWithFiles.Video, ThumbnailType.PREVIEW)
175 } catch (err) {
176 logger.warn('Cannot generate video preview %s for %s.', options.thumbnailUrl, videoImportWithFiles.Video.url, { err })
177 }
178 }
179
180 if (!previewModel && (options.generatePreview || options.downloadPreview)) {
167 previewModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.PREVIEW) 181 previewModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.PREVIEW)
168 } 182 }
169 183
@@ -214,14 +228,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
214 228
215 // Create transcoding jobs? 229 // Create transcoding jobs?
216 if (video.state === VideoState.TO_TRANSCODE) { 230 if (video.state === VideoState.TO_TRANSCODE) {
217 // Put uuid because we don't have id auto incremented for now 231 await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile)
218 const dataInput = {
219 type: 'optimize' as 'optimize',
220 videoUUID: videoImportUpdated.Video.uuid,
221 isNewVideo: true
222 }
223
224 await JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput })
225 } 232 }
226 233
227 } catch (err) { 234 } catch (err) {
diff --git a/server/lib/videos.ts b/server/lib/videos.ts
index 22e9afbf9..96bdd42e9 100644
--- a/server/lib/videos.ts
+++ b/server/lib/videos.ts
@@ -1,4 +1,7 @@
1import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/typings/models' 1import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/typings/models'
2import { VideoTranscodingPayload } from '@server/lib/job-queue/handlers/video-transcoding'
3import { DEFAULT_AUDIO_RESOLUTION } from '@server/initializers/constants'
4import { JobQueue } from '@server/lib/job-queue'
2 5
3function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) { 6function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
4 return isStreamingPlaylist(videoOrPlaylist) 7 return isStreamingPlaylist(videoOrPlaylist)
@@ -6,6 +9,28 @@ function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
6 : videoOrPlaylist 9 : videoOrPlaylist
7} 10}
8 11
12function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile) {
13 let dataInput: VideoTranscodingPayload
14
15 if (videoFile.isAudio()) {
16 dataInput = {
17 type: 'merge-audio' as 'merge-audio',
18 resolution: DEFAULT_AUDIO_RESOLUTION,
19 videoUUID: video.uuid,
20 isNewVideo: true
21 }
22 } else {
23 dataInput = {
24 type: 'optimize' as 'optimize',
25 videoUUID: video.uuid,
26 isNewVideo: true
27 }
28 }
29
30 return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput })
31}
32
9export { 33export {
34 addOptimizeOrMergeAudioJob,
10 extractVideo 35 extractVideo
11} 36}