aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/helpers/youtube-dl/youtube-dl-cli.ts
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-10-21 16:28:39 +0200
committerChocobozzz <me@florianbigard.com>2021-10-22 10:25:24 +0200
commit62549e6c9818f422698f030e0b242609115493ed (patch)
tree12a969f694239fe5f926f779698df9523605ee80 /server/helpers/youtube-dl/youtube-dl-cli.ts
parenta71d4140a5b7831dbe2eb7a0dfaa6a755cb2e906 (diff)
downloadPeerTube-62549e6c9818f422698f030e0b242609115493ed.tar.gz
PeerTube-62549e6c9818f422698f030e0b242609115493ed.tar.zst
PeerTube-62549e6c9818f422698f030e0b242609115493ed.zip
Rewrite youtube-dl import
Use python3 binary Allows to use a custom youtube-dl release URL Allows to use yt-dlp (youtube-dl fork) Remove proxy config from configuration to use HTTP_PROXY and HTTPS_PROXY env variables
Diffstat (limited to 'server/helpers/youtube-dl/youtube-dl-cli.ts')
-rw-r--r--server/helpers/youtube-dl/youtube-dl-cli.ts198
1 files changed, 198 insertions, 0 deletions
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
new file mode 100644
index 000000000..440869205
--- /dev/null
+++ b/server/helpers/youtube-dl/youtube-dl-cli.ts
@@ -0,0 +1,198 @@
1import execa from 'execa'
2import { pathExists, writeFile } from 'fs-extra'
3import { join } from 'path'
4import { CONFIG } from '@server/initializers/config'
5import { VideoResolution } from '@shared/models'
6import { logger, loggerTagsFactory } from '../logger'
7import { getProxy, isProxyEnabled } from '../proxy'
8import { isBinaryResponse, peertubeGot } from '../requests'
9
10const lTags = loggerTagsFactory('youtube-dl')
11
12const youtubeDLBinaryPath = join(CONFIG.STORAGE.BIN_DIR, CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME)
13
14export class YoutubeDLCLI {
15
16 static async safeGet () {
17 if (!await pathExists(youtubeDLBinaryPath)) {
18 await this.updateYoutubeDLBinary()
19 }
20
21 return new YoutubeDLCLI()
22 }
23
24 static async updateYoutubeDLBinary () {
25 const url = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.URL
26
27 logger.info('Updating youtubeDL binary from %s.', url, lTags())
28
29 const gotOptions = { context: { bodyKBLimit: 20_000 }, responseType: 'buffer' as 'buffer' }
30
31 try {
32 let gotResult = await peertubeGot(url, gotOptions)
33
34 if (!isBinaryResponse(gotResult)) {
35 const json = JSON.parse(gotResult.body.toString())
36 const latest = json.filter(release => release.prerelease === false)[0]
37 if (!latest) throw new Error('Cannot find latest release')
38
39 const releaseName = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME
40 const releaseAsset = latest.assets.find(a => a.name === releaseName)
41 if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`)
42
43 gotResult = await peertubeGot(releaseAsset.browser_download_url, gotOptions)
44 }
45
46 if (!isBinaryResponse(gotResult)) {
47 throw new Error('Not a binary response')
48 }
49
50 await writeFile(youtubeDLBinaryPath, gotResult.body)
51
52 logger.info('youtube-dl updated %s.', youtubeDLBinaryPath, lTags())
53 } catch (err) {
54 logger.error('Cannot update youtube-dl from %s.', url, { err, ...lTags() })
55 }
56 }
57
58 static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[]) {
59 /**
60 * list of format selectors in order or preference
61 * see https://github.com/ytdl-org/youtube-dl#format-selection
62 *
63 * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
64 * of being able to do a "quick-transcode"
65 * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
66 * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
67 *
68 * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
69 **/
70 const resolution = enabledResolutions.length === 0
71 ? VideoResolution.H_720P
72 : Math.max(...enabledResolutions)
73
74 return [
75 `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
76 `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
77 `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3
78 `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`,
79 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
80 'best' // Ultimate fallback
81 ].join('/')
82 }
83
84 private constructor () {
85
86 }
87
88 download (options: {
89 url: string
90 format: string
91 output: string
92 processOptions: execa.NodeOptions
93 additionalYoutubeDLArgs?: string[]
94 }) {
95 return this.run({
96 url: options.url,
97 processOptions: options.processOptions,
98 args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ])
99 })
100 }
101
102 async getInfo (options: {
103 url: string
104 format: string
105 processOptions: execa.NodeOptions
106 additionalYoutubeDLArgs?: string[]
107 }) {
108 const { url, format, additionalYoutubeDLArgs = [], processOptions } = options
109
110 const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ])
111
112 const data = await this.run({ url, args: completeArgs, processOptions })
113 const info = data.map(this.parseInfo)
114
115 return info.length === 1
116 ? info[0]
117 : info
118 }
119
120 async getSubs (options: {
121 url: string
122 format: 'vtt'
123 processOptions: execa.NodeOptions
124 }) {
125 const { url, format, processOptions } = options
126
127 const args = [ '--skip-download', '--all-subs', `--sub-format=${format}` ]
128
129 const data = await this.run({ url, args, processOptions })
130 const files: string[] = []
131
132 const skipString = '[info] Writing video subtitles to: '
133
134 for (let i = 0, len = data.length; i < len; i++) {
135 const line = data[i]
136
137 if (line.indexOf(skipString) === 0) {
138 files.push(line.slice(skipString.length))
139 }
140 }
141
142 return files
143 }
144
145 private async run (options: {
146 url: string
147 args: string[]
148 processOptions: execa.NodeOptions
149 }) {
150 const { url, args, processOptions } = options
151
152 let completeArgs = this.wrapWithProxyOptions(args)
153 completeArgs = this.wrapWithIPOptions(completeArgs)
154 completeArgs = this.wrapWithFFmpegOptions(completeArgs)
155
156 const output = await execa('python', [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions)
157
158 logger.debug('Runned youtube-dl command.', { command: output.command, stdout: output.stdout, ...lTags() })
159
160 return output.stdout
161 ? output.stdout.trim().split(/\r?\n/)
162 : undefined
163 }
164
165 private wrapWithProxyOptions (args: string[]) {
166 if (isProxyEnabled()) {
167 logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags())
168
169 return [ '--proxy', getProxy() ].concat(args)
170 }
171
172 return args
173 }
174
175 private wrapWithIPOptions (args: string[]) {
176 if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
177 logger.debug('Force ipv4 for YoutubeDL')
178
179 return [ '--force-ipv4' ].concat(args)
180 }
181
182 return args
183 }
184
185 private wrapWithFFmpegOptions (args: string[]) {
186 if (process.env.FFMPEG_PATH) {
187 logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags())
188
189 return [ '--ffmpeg-location', process.env.FFMPEG_PATH ].concat(args)
190 }
191
192 return args
193 }
194
195 private parseInfo (data: string) {
196 return JSON.parse(data)
197 }
198}