aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/helpers/youtube-dl
diff options
context:
space:
mode:
Diffstat (limited to 'server/helpers/youtube-dl')
-rw-r--r--server/helpers/youtube-dl/index.ts3
-rw-r--r--server/helpers/youtube-dl/youtube-dl-cli.ts259
-rw-r--r--server/helpers/youtube-dl/youtube-dl-info-builder.ts205
-rw-r--r--server/helpers/youtube-dl/youtube-dl-wrapper.ts154
4 files changed, 0 insertions, 621 deletions
diff --git a/server/helpers/youtube-dl/index.ts b/server/helpers/youtube-dl/index.ts
deleted file mode 100644
index 6afc77dcf..000000000
--- a/server/helpers/youtube-dl/index.ts
+++ /dev/null
@@ -1,3 +0,0 @@
1export * from './youtube-dl-cli'
2export * from './youtube-dl-info-builder'
3export * from './youtube-dl-wrapper'
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
deleted file mode 100644
index 765038cea..000000000
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ /dev/null
@@ -1,259 +0,0 @@
1import execa from 'execa'
2import { ensureDir, pathExists, writeFile } from 'fs-extra'
3import { dirname, join } from 'path'
4import { CONFIG } from '@server/initializers/config'
5import { VideoResolution } from '@shared/models'
6import { logger, loggerTagsFactory } from '../logger'
7import { getProxy, isProxyEnabled } from '../proxy'
8import { isBinaryResponse, peertubeGot } from '../requests'
9import { OptionsOfBufferResponseBody } from 'got/dist/source'
10
11const lTags = loggerTagsFactory('youtube-dl')
12
13const youtubeDLBinaryPath = join(CONFIG.STORAGE.BIN_DIR, CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME)
14
15export class YoutubeDLCLI {
16
17 static async safeGet () {
18 if (!await pathExists(youtubeDLBinaryPath)) {
19 await ensureDir(dirname(youtubeDLBinaryPath))
20
21 await this.updateYoutubeDLBinary()
22 }
23
24 return new YoutubeDLCLI()
25 }
26
27 static async updateYoutubeDLBinary () {
28 const url = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.URL
29
30 logger.info('Updating youtubeDL binary from %s.', url, lTags())
31
32 const gotOptions: OptionsOfBufferResponseBody = {
33 context: { bodyKBLimit: 20_000 },
34 responseType: 'buffer' as 'buffer'
35 }
36
37 if (process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN) {
38 gotOptions.headers = {
39 authorization: 'Bearer ' + process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN
40 }
41 }
42
43 try {
44 let gotResult = await peertubeGot(url, gotOptions)
45
46 if (!isBinaryResponse(gotResult)) {
47 const json = JSON.parse(gotResult.body.toString())
48 const latest = json.filter(release => release.prerelease === false)[0]
49 if (!latest) throw new Error('Cannot find latest release')
50
51 const releaseName = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME
52 const releaseAsset = latest.assets.find(a => a.name === releaseName)
53 if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`)
54
55 gotResult = await peertubeGot(releaseAsset.browser_download_url, gotOptions)
56 }
57
58 if (!isBinaryResponse(gotResult)) {
59 throw new Error('Not a binary response')
60 }
61
62 await writeFile(youtubeDLBinaryPath, gotResult.body)
63
64 logger.info('youtube-dl updated %s.', youtubeDLBinaryPath, lTags())
65 } catch (err) {
66 logger.error('Cannot update youtube-dl from %s.', url, { err, ...lTags() })
67 }
68 }
69
70 static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[], useBestFormat: boolean) {
71 /**
72 * list of format selectors in order or preference
73 * see https://github.com/ytdl-org/youtube-dl#format-selection
74 *
75 * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
76 * of being able to do a "quick-transcode"
77 * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
78 * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
79 *
80 * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
81 **/
82
83 let result: string[] = []
84
85 if (!useBestFormat) {
86 const resolution = enabledResolutions.length === 0
87 ? VideoResolution.H_720P
88 : Math.max(...enabledResolutions)
89
90 result = [
91 `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
92 `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
93 `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]` // case #
94 ]
95 }
96
97 return result.concat([
98 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
99 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
100 'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
101 'best' // Ultimate fallback
102 ]).join('/')
103 }
104
105 private constructor () {
106
107 }
108
109 download (options: {
110 url: string
111 format: string
112 output: string
113 processOptions: execa.NodeOptions
114 timeout?: number
115 additionalYoutubeDLArgs?: string[]
116 }) {
117 let args = options.additionalYoutubeDLArgs || []
118 args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
119
120 return this.run({
121 url: options.url,
122 processOptions: options.processOptions,
123 timeout: options.timeout,
124 args
125 })
126 }
127
128 async getInfo (options: {
129 url: string
130 format: string
131 processOptions: execa.NodeOptions
132 additionalYoutubeDLArgs?: string[]
133 }) {
134 const { url, format, additionalYoutubeDLArgs = [], processOptions } = options
135
136 const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ])
137
138 const data = await this.run({ url, args: completeArgs, processOptions })
139 if (!data) return undefined
140
141 const info = data.map(d => JSON.parse(d))
142
143 return info.length === 1
144 ? info[0]
145 : info
146 }
147
148 async getListInfo (options: {
149 url: string
150 latestVideosCount?: number
151 processOptions: execa.NodeOptions
152 }): Promise<{ upload_date: string, webpage_url: string }[]> {
153 const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
154
155 if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') {
156 // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel
157 additionalYoutubeDLArgs.push('--flat-playlist')
158 }
159
160 if (options.latestVideosCount !== undefined) {
161 additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
162 }
163
164 const result = await this.getInfo({
165 url: options.url,
166 format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
167 processOptions: options.processOptions,
168 additionalYoutubeDLArgs
169 })
170
171 if (!result) return result
172 if (!Array.isArray(result)) return [ result ]
173
174 return result
175 }
176
177 async getSubs (options: {
178 url: string
179 format: 'vtt'
180 processOptions: execa.NodeOptions
181 }) {
182 const { url, format, processOptions } = options
183
184 const args = [ '--skip-download', '--all-subs', `--sub-format=${format}` ]
185
186 const data = await this.run({ url, args, processOptions })
187 const files: string[] = []
188
189 const skipString = '[info] Writing video subtitles to: '
190
191 for (let i = 0, len = data.length; i < len; i++) {
192 const line = data[i]
193
194 if (line.indexOf(skipString) === 0) {
195 files.push(line.slice(skipString.length))
196 }
197 }
198
199 return files
200 }
201
202 private async run (options: {
203 url: string
204 args: string[]
205 timeout?: number
206 processOptions: execa.NodeOptions
207 }) {
208 const { url, args, timeout, processOptions } = options
209
210 let completeArgs = this.wrapWithProxyOptions(args)
211 completeArgs = this.wrapWithIPOptions(completeArgs)
212 completeArgs = this.wrapWithFFmpegOptions(completeArgs)
213
214 const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE
215 const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions)
216
217 if (timeout) {
218 setTimeout(() => subProcess.cancel(), timeout)
219 }
220
221 const output = await subProcess
222
223 logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
224
225 return output.stdout
226 ? output.stdout.trim().split(/\r?\n/)
227 : undefined
228 }
229
230 private wrapWithProxyOptions (args: string[]) {
231 if (isProxyEnabled()) {
232 logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags())
233
234 return [ '--proxy', getProxy() ].concat(args)
235 }
236
237 return args
238 }
239
240 private wrapWithIPOptions (args: string[]) {
241 if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
242 logger.debug('Force ipv4 for YoutubeDL')
243
244 return [ '--force-ipv4' ].concat(args)
245 }
246
247 return args
248 }
249
250 private wrapWithFFmpegOptions (args: string[]) {
251 if (process.env.FFMPEG_PATH) {
252 logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags())
253
254 return [ '--ffmpeg-location', process.env.FFMPEG_PATH ].concat(args)
255 }
256
257 return args
258 }
259}
diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
deleted file mode 100644
index a74904e43..000000000
--- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts
+++ /dev/null
@@ -1,205 +0,0 @@
1import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../../initializers/constants'
2import { peertubeTruncate } from '../core-utils'
3import { isUrlValid } from '../custom-validators/activitypub/misc'
4
5type YoutubeDLInfo = {
6 name?: string
7 description?: string
8 category?: number
9 language?: string
10 licence?: number
11 nsfw?: boolean
12 tags?: string[]
13 thumbnailUrl?: string
14 ext?: string
15 originallyPublishedAtWithoutTime?: Date
16 webpageUrl?: string
17
18 urls?: string[]
19}
20
21class YoutubeDLInfoBuilder {
22 private readonly info: any
23
24 constructor (info: any) {
25 this.info = { ...info }
26 }
27
28 getInfo () {
29 const obj = this.buildVideoInfo(this.normalizeObject(this.info))
30 if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
31
32 return obj
33 }
34
35 private normalizeObject (obj: any) {
36 const newObj: any = {}
37
38 for (const key of Object.keys(obj)) {
39 // Deprecated key
40 if (key === 'resolution') continue
41
42 const value = obj[key]
43
44 if (typeof value === 'string') {
45 newObj[key] = value.normalize()
46 } else {
47 newObj[key] = value
48 }
49 }
50
51 return newObj
52 }
53
54 private buildOriginallyPublishedAt (obj: any) {
55 let originallyPublishedAt: Date = null
56
57 const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date)
58 if (uploadDateMatcher) {
59 originallyPublishedAt = new Date()
60 originallyPublishedAt.setHours(0, 0, 0, 0)
61
62 const year = parseInt(uploadDateMatcher[1], 10)
63 // Month starts from 0
64 const month = parseInt(uploadDateMatcher[2], 10) - 1
65 const day = parseInt(uploadDateMatcher[3], 10)
66
67 originallyPublishedAt.setFullYear(year, month, day)
68 }
69
70 return originallyPublishedAt
71 }
72
73 private buildVideoInfo (obj: any): YoutubeDLInfo {
74 return {
75 name: this.titleTruncation(obj.title),
76 description: this.descriptionTruncation(obj.description),
77 category: this.getCategory(obj.categories),
78 licence: this.getLicence(obj.license),
79 language: this.getLanguage(obj.language),
80 nsfw: this.isNSFW(obj),
81 tags: this.getTags(obj.tags),
82 thumbnailUrl: obj.thumbnail || undefined,
83 urls: this.buildAvailableUrl(obj),
84 originallyPublishedAtWithoutTime: this.buildOriginallyPublishedAt(obj),
85 ext: obj.ext,
86 webpageUrl: obj.webpage_url
87 }
88 }
89
90 private buildAvailableUrl (obj: any) {
91 const urls: string[] = []
92
93 if (obj.url) urls.push(obj.url)
94 if (obj.urls) {
95 if (Array.isArray(obj.urls)) urls.push(...obj.urls)
96 else urls.push(obj.urls)
97 }
98
99 const formats = Array.isArray(obj.formats)
100 ? obj.formats
101 : []
102
103 for (const format of formats) {
104 if (!format.url) continue
105
106 urls.push(format.url)
107 }
108
109 const thumbnails = Array.isArray(obj.thumbnails)
110 ? obj.thumbnails
111 : []
112
113 for (const thumbnail of thumbnails) {
114 if (!thumbnail.url) continue
115
116 urls.push(thumbnail.url)
117 }
118
119 if (obj.thumbnail) urls.push(obj.thumbnail)
120
121 for (const subtitleKey of Object.keys(obj.subtitles || {})) {
122 const subtitles = obj.subtitles[subtitleKey]
123 if (!Array.isArray(subtitles)) continue
124
125 for (const subtitle of subtitles) {
126 if (!subtitle.url) continue
127
128 urls.push(subtitle.url)
129 }
130 }
131
132 return urls.filter(u => u && isUrlValid(u))
133 }
134
135 private titleTruncation (title: string) {
136 return peertubeTruncate(title, {
137 length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
138 separator: /,? +/,
139 omission: ' […]'
140 })
141 }
142
143 private descriptionTruncation (description: string) {
144 if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined
145
146 return peertubeTruncate(description, {
147 length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
148 separator: /,? +/,
149 omission: ' […]'
150 })
151 }
152
153 private isNSFW (info: any) {
154 return info?.age_limit >= 16
155 }
156
157 private getTags (tags: string[]) {
158 if (Array.isArray(tags) === false) return []
159
160 return tags
161 .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
162 .map(t => t.normalize())
163 .slice(0, 5)
164 }
165
166 private getLicence (licence: string) {
167 if (!licence) return undefined
168
169 if (licence.includes('Creative Commons Attribution')) return 1
170
171 for (const key of Object.keys(VIDEO_LICENCES)) {
172 const peertubeLicence = VIDEO_LICENCES[key]
173 if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10)
174 }
175
176 return undefined
177 }
178
179 private getCategory (categories: string[]) {
180 if (!categories) return undefined
181
182 const categoryString = categories[0]
183 if (!categoryString || typeof categoryString !== 'string') return undefined
184
185 if (categoryString === 'News & Politics') return 11
186
187 for (const key of Object.keys(VIDEO_CATEGORIES)) {
188 const category = VIDEO_CATEGORIES[key]
189 if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
190 }
191
192 return undefined
193 }
194
195 private getLanguage (language: string) {
196 return VIDEO_LANGUAGES[language] ? language : undefined
197 }
198}
199
200// ---------------------------------------------------------------------------
201
202export {
203 YoutubeDLInfo,
204 YoutubeDLInfoBuilder
205}
diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
deleted file mode 100644
index ac3cd190e..000000000
--- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts
+++ /dev/null
@@ -1,154 +0,0 @@
1import { move, pathExists, readdir, remove } from 'fs-extra'
2import { dirname, join } from 'path'
3import { inspect } from 'util'
4import { CONFIG } from '@server/initializers/config'
5import { isVideoFileExtnameValid } from '../custom-validators/videos'
6import { logger, loggerTagsFactory } from '../logger'
7import { generateVideoImportTmpPath } from '../utils'
8import { YoutubeDLCLI } from './youtube-dl-cli'
9import { YoutubeDLInfo, YoutubeDLInfoBuilder } from './youtube-dl-info-builder'
10
11const lTags = loggerTagsFactory('youtube-dl')
12
13export type YoutubeDLSubs = {
14 language: string
15 filename: string
16 path: string
17}[]
18
19const processOptions = {
20 maxBuffer: 1024 * 1024 * 30 // 30MB
21}
22
23class YoutubeDLWrapper {
24
25 constructor (
26 private readonly url: string,
27 private readonly enabledResolutions: number[],
28 private readonly useBestFormat: boolean
29 ) {
30
31 }
32
33 async getInfoForDownload (youtubeDLArgs: string[] = []): Promise<YoutubeDLInfo> {
34 const youtubeDL = await YoutubeDLCLI.safeGet()
35
36 const info = await youtubeDL.getInfo({
37 url: this.url,
38 format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
39 additionalYoutubeDLArgs: youtubeDLArgs,
40 processOptions
41 })
42
43 if (!info) throw new Error(`YoutubeDL could not get info from ${this.url}`)
44
45 if (info.is_live === true) throw new Error('Cannot download a live streaming.')
46
47 const infoBuilder = new YoutubeDLInfoBuilder(info)
48
49 return infoBuilder.getInfo()
50 }
51
52 async getInfoForListImport (options: {
53 latestVideosCount?: number
54 }) {
55 const youtubeDL = await YoutubeDLCLI.safeGet()
56
57 const list = await youtubeDL.getListInfo({
58 url: this.url,
59 latestVideosCount: options.latestVideosCount,
60 processOptions
61 })
62
63 if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`)
64
65 return list.map(info => info.webpage_url)
66 }
67
68 async getSubtitles (): Promise<YoutubeDLSubs> {
69 const cwd = CONFIG.STORAGE.TMP_DIR
70
71 const youtubeDL = await YoutubeDLCLI.safeGet()
72
73 const files = await youtubeDL.getSubs({ url: this.url, format: 'vtt', processOptions: { cwd } })
74 if (!files) return []
75
76 logger.debug('Get subtitles from youtube dl.', { url: this.url, files, ...lTags() })
77
78 const subtitles = files.reduce((acc, filename) => {
79 const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i)
80 if (!matched?.[1]) return acc
81
82 return [
83 ...acc,
84 {
85 language: matched[1],
86 path: join(cwd, filename),
87 filename
88 }
89 ]
90 }, [])
91
92 return subtitles
93 }
94
95 async downloadVideo (fileExt: string, timeout: number): Promise<string> {
96 // Leave empty the extension, youtube-dl will add it
97 const pathWithoutExtension = generateVideoImportTmpPath(this.url, '')
98
99 logger.info('Importing youtubeDL video %s to %s', this.url, pathWithoutExtension, lTags())
100
101 const youtubeDL = await YoutubeDLCLI.safeGet()
102
103 try {
104 await youtubeDL.download({
105 url: this.url,
106 format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
107 output: pathWithoutExtension,
108 timeout,
109 processOptions
110 })
111
112 // If youtube-dl did not guess an extension for our file, just use .mp4 as default
113 if (await pathExists(pathWithoutExtension)) {
114 await move(pathWithoutExtension, pathWithoutExtension + '.mp4')
115 }
116
117 return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
118 } catch (err) {
119 this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
120 .then(path => {
121 logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() })
122
123 return remove(path)
124 })
125 .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
126
127 throw err
128 }
129 }
130
131 private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) {
132 if (!isVideoFileExtnameValid(sourceExt)) {
133 throw new Error('Invalid video extension ' + sourceExt)
134 }
135
136 const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ]
137
138 for (const extension of extensions) {
139 const path = tmpPath + extension
140
141 if (await pathExists(path)) return path
142 }
143
144 const directoryContent = await readdir(dirname(tmpPath))
145
146 throw new Error(`Cannot guess path of ${tmpPath}. Directory content: ${directoryContent.join(', ')}`)
147 }
148}
149
150// ---------------------------------------------------------------------------
151
152export {
153 YoutubeDLWrapper
154}
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-24 18:39:50 +0000