aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/tools/peertube-import-videos.ts
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-10-21 16:28:39 +0200
committerChocobozzz <me@florianbigard.com>2021-10-22 10:25:24 +0200
commit62549e6c9818f422698f030e0b242609115493ed (patch)
tree12a969f694239fe5f926f779698df9523605ee80 /server/tools/peertube-import-videos.ts
parenta71d4140a5b7831dbe2eb7a0dfaa6a755cb2e906 (diff)
downloadPeerTube-62549e6c9818f422698f030e0b242609115493ed.tar.gz
PeerTube-62549e6c9818f422698f030e0b242609115493ed.tar.zst
PeerTube-62549e6c9818f422698f030e0b242609115493ed.zip
Rewrite youtube-dl import
Use python3 binary Allows to use a custom youtube-dl release URL Allows to use yt-dlp (youtube-dl fork) Remove proxy config from configuration to use HTTP_PROXY and HTTPS_PROXY env variables
Diffstat (limited to 'server/tools/peertube-import-videos.ts')
-rw-r--r--server/tools/peertube-import-videos.ts179
1 files changed, 52 insertions, 127 deletions
diff --git a/server/tools/peertube-import-videos.ts b/server/tools/peertube-import-videos.ts
index 758b561e1..54ac910e6 100644
--- a/server/tools/peertube-import-videos.ts
+++ b/server/tools/peertube-import-videos.ts
@@ -4,13 +4,9 @@ registerTSPaths()
4import { program } from 'commander' 4import { program } from 'commander'
5import { accessSync, constants } from 'fs' 5import { accessSync, constants } from 'fs'
6import { remove } from 'fs-extra' 6import { remove } from 'fs-extra'
7import { truncate } from 'lodash'
8import { join } from 'path' 7import { join } from 'path'
9import { promisify } from 'util'
10import { YoutubeDL } from '@server/helpers/youtube-dl'
11import { sha256 } from '../helpers/core-utils' 8import { sha256 } from '../helpers/core-utils'
12import { doRequestAndSaveToFile } from '../helpers/requests' 9import { doRequestAndSaveToFile } from '../helpers/requests'
13import { CONSTRAINTS_FIELDS } from '../initializers/constants'
14import { 10import {
15 assignToken, 11 assignToken,
16 buildCommonVideoOptions, 12 buildCommonVideoOptions,
@@ -19,8 +15,8 @@ import {
19 getLogger, 15 getLogger,
20 getServerCredentials 16 getServerCredentials
21} from './cli' 17} from './cli'
22import { PeerTubeServer } from '@shared/extra-utils' 18import { wait } from '@shared/extra-utils'
23 19import { YoutubeDLCLI, YoutubeDLInfo, YoutubeDLInfoBuilder } from '@server/helpers/youtube-dl'
24import prompt = require('prompt') 20import prompt = require('prompt')
25 21
26const processOptions = { 22const processOptions = {
@@ -73,7 +69,7 @@ getServerCredentials(command)
73async function run (url: string, username: string, password: string) { 69async function run (url: string, username: string, password: string) {
74 if (!password) password = await promptPassword() 70 if (!password) password = await promptPassword()
75 71
76 const youtubeDLBinary = await YoutubeDL.safeGetYoutubeDL() 72 const youtubeDLBinary = await YoutubeDLCLI.safeGet()
77 73
78 let info = await getYoutubeDLInfo(youtubeDLBinary, options.targetUrl, command.args) 74 let info = await getYoutubeDLInfo(youtubeDLBinary, options.targetUrl, command.args)
79 75
@@ -96,8 +92,6 @@ async function run (url: string, username: string, password: string) {
96 } else if (options.last) { 92 } else if (options.last) {
97 infoArray = infoArray.slice(-options.last) 93 infoArray = infoArray.slice(-options.last)
98 } 94 }
99 // Normalize utf8 fields
100 infoArray = infoArray.map(i => normalizeObject(i))
101 95
102 log.info('Will download and upload %d videos.\n', infoArray.length) 96 log.info('Will download and upload %d videos.\n', infoArray.length)
103 97
@@ -105,8 +99,9 @@ async function run (url: string, username: string, password: string) {
105 try { 99 try {
106 if (index > 0 && options.waitInterval) { 100 if (index > 0 && options.waitInterval) {
107 log.info("Wait for %d seconds before continuing.", options.waitInterval / 1000) 101 log.info("Wait for %d seconds before continuing.", options.waitInterval / 1000)
108 await new Promise(res => setTimeout(res, options.waitInterval)) 102 await wait(options.waitInterval)
109 } 103 }
104
110 await processVideo({ 105 await processVideo({
111 cwd: options.tmpdir, 106 cwd: options.tmpdir,
112 url, 107 url,
@@ -131,29 +126,26 @@ async function processVideo (parameters: {
131 youtubeInfo: any 126 youtubeInfo: any
132}) { 127}) {
133 const { youtubeInfo, cwd, url, username, password } = parameters 128 const { youtubeInfo, cwd, url, username, password } = parameters
134 const youtubeDL = new YoutubeDL('', [])
135 129
136 log.debug('Fetching object.', youtubeInfo) 130 log.debug('Fetching object.', youtubeInfo)
137 131
138 const videoInfo = await fetchObject(youtubeInfo) 132 const videoInfo = await fetchObject(youtubeInfo)
139 log.debug('Fetched object.', videoInfo) 133 log.debug('Fetched object.', videoInfo)
140 134
141 const originallyPublishedAt = youtubeDL.buildOriginallyPublishedAt(videoInfo) 135 if (options.since && videoInfo.originallyPublishedAt && videoInfo.originallyPublishedAt.getTime() < options.since.getTime()) {
142 136 log.info('Video "%s" has been published before "%s", don\'t upload it.\n', videoInfo.name, formatDate(options.since))
143 if (options.since && originallyPublishedAt && originallyPublishedAt.getTime() < options.since.getTime()) {
144 log.info('Video "%s" has been published before "%s", don\'t upload it.\n', videoInfo.title, formatDate(options.since))
145 return 137 return
146 } 138 }
147 139
148 if (options.until && originallyPublishedAt && originallyPublishedAt.getTime() > options.until.getTime()) { 140 if (options.until && videoInfo.originallyPublishedAt && videoInfo.originallyPublishedAt.getTime() > options.until.getTime()) {
149 log.info('Video "%s" has been published after "%s", don\'t upload it.\n', videoInfo.title, formatDate(options.until)) 141 log.info('Video "%s" has been published after "%s", don\'t upload it.\n', videoInfo.name, formatDate(options.until))
150 return 142 return
151 } 143 }
152 144
153 const server = buildServer(url) 145 const server = buildServer(url)
154 const { data } = await server.search.advancedVideoSearch({ 146 const { data } = await server.search.advancedVideoSearch({
155 search: { 147 search: {
156 search: videoInfo.title, 148 search: videoInfo.name,
157 sort: '-match', 149 sort: '-match',
158 searchTarget: 'local' 150 searchTarget: 'local'
159 } 151 }
@@ -161,28 +153,32 @@ async function processVideo (parameters: {
161 153
162 log.info('############################################################\n') 154 log.info('############################################################\n')
163 155
164 if (data.find(v => v.name === videoInfo.title)) { 156 if (data.find(v => v.name === videoInfo.name)) {
165 log.info('Video "%s" already exists, don\'t reupload it.\n', videoInfo.title) 157 log.info('Video "%s" already exists, don\'t reupload it.\n', videoInfo.name)
166 return 158 return
167 } 159 }
168 160
169 const path = join(cwd, sha256(videoInfo.url) + '.mp4') 161 const path = join(cwd, sha256(videoInfo.url) + '.mp4')
170 162
171 log.info('Downloading video "%s"...', videoInfo.title) 163 log.info('Downloading video "%s"...', videoInfo.name)
172 164
173 const youtubeDLOptions = [ '-f', youtubeDL.getYoutubeDLVideoFormat(), ...command.args, '-o', path ]
174 try { 165 try {
175 const youtubeDLBinary = await YoutubeDL.safeGetYoutubeDL() 166 const youtubeDLBinary = await YoutubeDLCLI.safeGet()
176 const youtubeDLExec = promisify(youtubeDLBinary.exec).bind(youtubeDLBinary) 167 const output = await youtubeDLBinary.download({
177 const output = await youtubeDLExec(videoInfo.url, youtubeDLOptions, processOptions) 168 url: videoInfo.url,
169 format: YoutubeDLCLI.getYoutubeDLVideoFormat([]),
170 output: path,
171 additionalYoutubeDLArgs: command.args,
172 processOptions
173 })
174
178 log.info(output.join('\n')) 175 log.info(output.join('\n'))
179 await uploadVideoOnPeerTube({ 176 await uploadVideoOnPeerTube({
180 youtubeDL,
181 cwd, 177 cwd,
182 url, 178 url,
183 username, 179 username,
184 password, 180 password,
185 videoInfo: normalizeObject(videoInfo), 181 videoInfo,
186 videoPath: path 182 videoPath: path
187 }) 183 })
188 } catch (err) { 184 } catch (err) {
@@ -191,57 +187,34 @@ async function processVideo (parameters: {
191} 187}
192 188
193async function uploadVideoOnPeerTube (parameters: { 189async function uploadVideoOnPeerTube (parameters: {
194 youtubeDL: YoutubeDL 190 videoInfo: YoutubeDLInfo
195 videoInfo: any
196 videoPath: string 191 videoPath: string
197 cwd: string 192 cwd: string
198 url: string 193 url: string
199 username: string 194 username: string
200 password: string 195 password: string
201}) { 196}) {
202 const { youtubeDL, videoInfo, videoPath, cwd, url, username, password } = parameters 197 const { videoInfo, videoPath, cwd, url, username, password } = parameters
203 198
204 const server = buildServer(url) 199 const server = buildServer(url)
205 await assignToken(server, username, password) 200 await assignToken(server, username, password)
206 201
207 const category = await getCategory(server, videoInfo.categories) 202 let thumbnailfile: string
208 const licence = getLicence(videoInfo.license) 203 if (videoInfo.thumbnailUrl) {
209 let tags = [] 204 thumbnailfile = join(cwd, sha256(videoInfo.thumbnailUrl) + '.jpg')
210 if (Array.isArray(videoInfo.tags)) {
211 tags = videoInfo.tags
212 .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
213 .map(t => t.normalize())
214 .slice(0, 5)
215 }
216
217 let thumbnailfile
218 if (videoInfo.thumbnail) {
219 thumbnailfile = join(cwd, sha256(videoInfo.thumbnail) + '.jpg')
220 205
221 await doRequestAndSaveToFile(videoInfo.thumbnail, thumbnailfile) 206 await doRequestAndSaveToFile(videoInfo.thumbnailUrl, thumbnailfile)
222 } 207 }
223 208
224 const originallyPublishedAt = youtubeDL.buildOriginallyPublishedAt(videoInfo) 209 const baseAttributes = await buildVideoAttributesFromCommander(server, program, videoInfo)
225
226 const defaultAttributes = {
227 name: truncate(videoInfo.title, {
228 length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
229 separator: /,? +/,
230 omission: ' […]'
231 }),
232 category,
233 licence,
234 nsfw: isNSFW(videoInfo),
235 description: videoInfo.description,
236 tags
237 }
238
239 const baseAttributes = await buildVideoAttributesFromCommander(server, program, defaultAttributes)
240 210
241 const attributes = { 211 const attributes = {
242 ...baseAttributes, 212 ...baseAttributes,
243 213
244 originallyPublishedAt: originallyPublishedAt ? originallyPublishedAt.toISOString() : null, 214 originallyPublishedAt: videoInfo.originallyPublishedAt
215 ? videoInfo.originallyPublishedAt.toISOString()
216 : null,
217
245 thumbnailfile, 218 thumbnailfile,
246 previewfile: thumbnailfile, 219 previewfile: thumbnailfile,
247 fixture: videoPath 220 fixture: videoPath
@@ -266,67 +239,26 @@ async function uploadVideoOnPeerTube (parameters: {
266 await remove(videoPath) 239 await remove(videoPath)
267 if (thumbnailfile) await remove(thumbnailfile) 240 if (thumbnailfile) await remove(thumbnailfile)
268 241
269 log.warn('Uploaded video "%s"!\n', attributes.name) 242 log.info('Uploaded video "%s"!\n', attributes.name)
270} 243}
271 244
272/* ---------------------------------------------------------- */ 245/* ---------------------------------------------------------- */
273 246
274async function getCategory (server: PeerTubeServer, categories: string[]) { 247async function fetchObject (info: any) {
275 if (!categories) return undefined 248 const url = buildUrl(info)
276
277 const categoryString = categories[0]
278
279 if (categoryString === 'News & Politics') return 11
280
281 const categoriesServer = await server.videos.getCategories()
282
283 for (const key of Object.keys(categoriesServer)) {
284 const categoryServer = categoriesServer[key]
285 if (categoryString.toLowerCase() === categoryServer.toLowerCase()) return parseInt(key, 10)
286 }
287
288 return undefined
289}
290
291function getLicence (licence: string) {
292 if (!licence) return undefined
293
294 if (licence.includes('Creative Commons Attribution licence')) return 1
295
296 return undefined
297}
298
299function normalizeObject (obj: any) {
300 const newObj: any = {}
301
302 for (const key of Object.keys(obj)) {
303 // Deprecated key
304 if (key === 'resolution') continue
305
306 const value = obj[key]
307
308 if (typeof value === 'string') {
309 newObj[key] = value.normalize()
310 } else {
311 newObj[key] = value
312 }
313 }
314 249
315 return newObj 250 const youtubeDLCLI = await YoutubeDLCLI.safeGet()
316} 251 const result = await youtubeDLCLI.getInfo({
252 url,
253 format: YoutubeDLCLI.getYoutubeDLVideoFormat([]),
254 processOptions
255 })
317 256
318function fetchObject (info: any) { 257 const builder = new YoutubeDLInfoBuilder(result)
319 const url = buildUrl(info)
320 258
321 return new Promise<any>(async (res, rej) => { 259 const videoInfo = builder.getInfo()
322 const youtubeDL = await YoutubeDL.safeGetYoutubeDL()
323 youtubeDL.getInfo(url, undefined, processOptions, (err, videoInfo) => {
324 if (err) return rej(err)
325 260
326 const videoInfoWithUrl = Object.assign(videoInfo, { url }) 261 return { ...videoInfo, url }
327 return res(normalizeObject(videoInfoWithUrl))
328 })
329 })
330} 262}
331 263
332function buildUrl (info: any) { 264function buildUrl (info: any) {
@@ -340,10 +272,6 @@ function buildUrl (info: any) {
340 return 'https://www.youtube.com/watch?v=' + info.id 272 return 'https://www.youtube.com/watch?v=' + info.id
341} 273}
342 274
343function isNSFW (info: any) {
344 return info.age_limit && info.age_limit >= 16
345}
346
347function normalizeTargetUrl (url: string) { 275function normalizeTargetUrl (url: string) {
348 let normalizedUrl = url.replace(/\/+$/, '') 276 let normalizedUrl = url.replace(/\/+$/, '')
349 277
@@ -404,14 +332,11 @@ function exitError (message: string, ...meta: any[]) {
404 process.exit(-1) 332 process.exit(-1)
405} 333}
406 334
407function getYoutubeDLInfo (youtubeDL: any, url: string, args: string[]) { 335function getYoutubeDLInfo (youtubeDLCLI: YoutubeDLCLI, url: string, args: string[]) {
408 return new Promise<any>((res, rej) => { 336 return youtubeDLCLI.getInfo({
409 const options = [ '-j', '--flat-playlist', '--playlist-reverse', ...args ] 337 url,
410 338 format: YoutubeDLCLI.getYoutubeDLVideoFormat([]),
411 youtubeDL.getInfo(url, options, processOptions, (err, info) => { 339 additionalYoutubeDLArgs: [ '-j', '--flat-playlist', '--playlist-reverse', ...args ],
412 if (err) return rej(err) 340 processOptions
413
414 return res(info)
415 })
416 }) 341 })
417} 342}