]>
Commit | Line | Data |
---|---|---|
1 | import execa from 'execa' | |
2 | import { ensureDir, pathExists, writeFile } from 'fs-extra' | |
3 | import { dirname, join } from 'path' | |
4 | import { CONFIG } from '@server/initializers/config' | |
5 | import { VideoResolution } from '@shared/models' | |
6 | import { logger, loggerTagsFactory } from '../logger' | |
7 | import { getProxy, isProxyEnabled } from '../proxy' | |
8 | import { isBinaryResponse, peertubeGot } from '../requests' | |
9 | import { OptionsOfBufferResponseBody } from 'got/dist/source' | |
10 | ||
11 | const lTags = loggerTagsFactory('youtube-dl') | |
12 | ||
13 | const youtubeDLBinaryPath = join(CONFIG.STORAGE.BIN_DIR, CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME) | |
14 | ||
15 | export class YoutubeDLCLI { | |
16 | ||
17 | static async safeGet () { | |
18 | if (!await pathExists(youtubeDLBinaryPath)) { | |
19 | await ensureDir(dirname(youtubeDLBinaryPath)) | |
20 | ||
21 | await this.updateYoutubeDLBinary() | |
22 | } | |
23 | ||
24 | return new YoutubeDLCLI() | |
25 | } | |
26 | ||
27 | static async updateYoutubeDLBinary () { | |
28 | const url = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.URL | |
29 | ||
30 | logger.info('Updating youtubeDL binary from %s.', url, lTags()) | |
31 | ||
32 | const gotOptions: OptionsOfBufferResponseBody = { | |
33 | context: { bodyKBLimit: 20_000 }, | |
34 | responseType: 'buffer' as 'buffer' | |
35 | } | |
36 | ||
37 | if (process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN) { | |
38 | gotOptions.headers = { | |
39 | authorization: 'Bearer ' + process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN | |
40 | } | |
41 | } | |
42 | ||
43 | try { | |
44 | let gotResult = await peertubeGot(url, gotOptions) | |
45 | ||
46 | if (!isBinaryResponse(gotResult)) { | |
47 | const json = JSON.parse(gotResult.body.toString()) | |
48 | const latest = json.filter(release => release.prerelease === false)[0] | |
49 | if (!latest) throw new Error('Cannot find latest release') | |
50 | ||
51 | const releaseName = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME | |
52 | const releaseAsset = latest.assets.find(a => a.name === releaseName) | |
53 | if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`) | |
54 | ||
55 | gotResult = await peertubeGot(releaseAsset.browser_download_url, gotOptions) | |
56 | } | |
57 | ||
58 | if (!isBinaryResponse(gotResult)) { | |
59 | throw new Error('Not a binary response') | |
60 | } | |
61 | ||
62 | await writeFile(youtubeDLBinaryPath, gotResult.body) | |
63 | ||
64 | logger.info('youtube-dl updated %s.', youtubeDLBinaryPath, lTags()) | |
65 | } catch (err) { | |
66 | logger.error('Cannot update youtube-dl from %s.', url, { err, ...lTags() }) | |
67 | } | |
68 | } | |
69 | ||
70 | static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[], useBestFormat: boolean) { | |
71 | /** | |
72 | * list of format selectors in order or preference | |
73 | * see https://github.com/ytdl-org/youtube-dl#format-selection | |
74 | * | |
75 | * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope | |
76 | * of being able to do a "quick-transcode" | |
77 | * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9) | |
78 | * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback | |
79 | * | |
80 | * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499 | |
81 | **/ | |
82 | ||
83 | let result: string[] = [] | |
84 | ||
85 | if (!useBestFormat) { | |
86 | const resolution = enabledResolutions.length === 0 | |
87 | ? VideoResolution.H_720P | |
88 | : Math.max(...enabledResolutions) | |
89 | ||
90 | result = [ | |
91 | `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1 | |
92 | `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2 | |
93 | `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]` // case # | |
94 | ] | |
95 | } | |
96 | ||
97 | return result.concat([ | |
98 | 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio', | |
99 | 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats | |
100 | 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', | |
101 | 'best' // Ultimate fallback | |
102 | ]).join('/') | |
103 | } | |
104 | ||
105 | private constructor () { | |
106 | ||
107 | } | |
108 | ||
109 | download (options: { | |
110 | url: string | |
111 | format: string | |
112 | output: string | |
113 | processOptions: execa.NodeOptions | |
114 | timeout?: number | |
115 | additionalYoutubeDLArgs?: string[] | |
116 | }) { | |
117 | let args = options.additionalYoutubeDLArgs || [] | |
118 | args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ]) | |
119 | ||
120 | return this.run({ | |
121 | url: options.url, | |
122 | processOptions: options.processOptions, | |
123 | timeout: options.timeout, | |
124 | args | |
125 | }) | |
126 | } | |
127 | ||
128 | async getInfo (options: { | |
129 | url: string | |
130 | format: string | |
131 | processOptions: execa.NodeOptions | |
132 | additionalYoutubeDLArgs?: string[] | |
133 | }) { | |
134 | const { url, format, additionalYoutubeDLArgs = [], processOptions } = options | |
135 | ||
136 | const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ]) | |
137 | ||
138 | const data = await this.run({ url, args: completeArgs, processOptions }) | |
139 | if (!data) return undefined | |
140 | ||
141 | const info = data.map(d => JSON.parse(d)) | |
142 | ||
143 | return info.length === 1 | |
144 | ? info[0] | |
145 | : info | |
146 | } | |
147 | ||
148 | async getListInfo (options: { | |
149 | url: string | |
150 | latestVideosCount?: number | |
151 | processOptions: execa.NodeOptions | |
152 | }): Promise<{ upload_date: string, webpage_url: string }[]> { | |
153 | const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ] | |
154 | ||
155 | if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') { | |
156 | // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel | |
157 | additionalYoutubeDLArgs.push('--flat-playlist') | |
158 | } | |
159 | ||
160 | if (options.latestVideosCount !== undefined) { | |
161 | additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString()) | |
162 | } | |
163 | ||
164 | const result = await this.getInfo({ | |
165 | url: options.url, | |
166 | format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false), | |
167 | processOptions: options.processOptions, | |
168 | additionalYoutubeDLArgs | |
169 | }) | |
170 | ||
171 | if (!result) return result | |
172 | if (!Array.isArray(result)) return [ result ] | |
173 | ||
174 | return result | |
175 | } | |
176 | ||
177 | async getSubs (options: { | |
178 | url: string | |
179 | format: 'vtt' | |
180 | processOptions: execa.NodeOptions | |
181 | }) { | |
182 | const { url, format, processOptions } = options | |
183 | ||
184 | const args = [ '--skip-download', '--all-subs', `--sub-format=${format}` ] | |
185 | ||
186 | const data = await this.run({ url, args, processOptions }) | |
187 | const files: string[] = [] | |
188 | ||
189 | const skipString = '[info] Writing video subtitles to: ' | |
190 | ||
191 | for (let i = 0, len = data.length; i < len; i++) { | |
192 | const line = data[i] | |
193 | ||
194 | if (line.indexOf(skipString) === 0) { | |
195 | files.push(line.slice(skipString.length)) | |
196 | } | |
197 | } | |
198 | ||
199 | return files | |
200 | } | |
201 | ||
202 | private async run (options: { | |
203 | url: string | |
204 | args: string[] | |
205 | timeout?: number | |
206 | processOptions: execa.NodeOptions | |
207 | }) { | |
208 | const { url, args, timeout, processOptions } = options | |
209 | ||
210 | let completeArgs = this.wrapWithProxyOptions(args) | |
211 | completeArgs = this.wrapWithIPOptions(completeArgs) | |
212 | completeArgs = this.wrapWithFFmpegOptions(completeArgs) | |
213 | ||
214 | const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE | |
215 | const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions) | |
216 | ||
217 | if (timeout) { | |
218 | setTimeout(() => subProcess.cancel(), timeout) | |
219 | } | |
220 | ||
221 | const output = await subProcess | |
222 | ||
223 | logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() }) | |
224 | ||
225 | return output.stdout | |
226 | ? output.stdout.trim().split(/\r?\n/) | |
227 | : undefined | |
228 | } | |
229 | ||
230 | private wrapWithProxyOptions (args: string[]) { | |
231 | if (isProxyEnabled()) { | |
232 | logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags()) | |
233 | ||
234 | return [ '--proxy', getProxy() ].concat(args) | |
235 | } | |
236 | ||
237 | return args | |
238 | } | |
239 | ||
240 | private wrapWithIPOptions (args: string[]) { | |
241 | if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) { | |
242 | logger.debug('Force ipv4 for YoutubeDL') | |
243 | ||
244 | return [ '--force-ipv4' ].concat(args) | |
245 | } | |
246 | ||
247 | return args | |
248 | } | |
249 | ||
250 | private wrapWithFFmpegOptions (args: string[]) { | |
251 | if (process.env.FFMPEG_PATH) { | |
252 | logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags()) | |
253 | ||
254 | return [ '--ffmpeg-location', process.env.FFMPEG_PATH ].concat(args) | |
255 | } | |
256 | ||
257 | return args | |
258 | } | |
259 | } |