aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorFlorent <florent.git-projects@zeteo.me>2021-01-25 15:42:44 +0100
committerGitHub <noreply@github.com>2021-01-25 15:42:44 +0100
commit82f5527fc3945f433ad8f1af66d13b46efb877da (patch)
treefaeec00f67b98db951d99a92e4c0f0b00a0cb36c
parent2a6cf69cffb83d0fbd73c4a0aabbb94df47b06c8 (diff)
downloadPeerTube-82f5527fc3945f433ad8f1af66d13b46efb877da.tar.gz
PeerTube-82f5527fc3945f433ad8f1af66d13b46efb877da.tar.zst
PeerTube-82f5527fc3945f433ad8f1af66d13b46efb877da.zip
add option for video-import CLI to wait between two video imports (#3310)
* peertube-import-videos: Add --wait-interval option * peertube-import-videos: various code style improvements Co-authored-by: Florent F <florent.fayolle69@gmail.com> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
-rw-r--r--server/tools/peertube-import-videos.ts121
1 files changed, 63 insertions, 58 deletions
diff --git a/server/tools/peertube-import-videos.ts b/server/tools/peertube-import-videos.ts
index 3a82b3832..65798adb4 100644
--- a/server/tools/peertube-import-videos.ts
+++ b/server/tools/peertube-import-videos.ts
@@ -1,15 +1,18 @@
1import { registerTSPaths } from '../helpers/register-ts-paths' 1import { registerTSPaths } from '../helpers/register-ts-paths'
2registerTSPaths() 2registerTSPaths()
3 3
4import * as program from 'commander' 4import { accessSync, constants } from 'fs'
5import { join } from 'path' 5import { join } from 'path'
6import { promisify } from 'util'
7
8import * as program from 'commander'
9import { remove } from 'fs-extra'
10import { truncate } from 'lodash'
11import * as prompt from 'prompt'
12
6import { doRequestAndSaveToFile } from '../helpers/requests' 13import { doRequestAndSaveToFile } from '../helpers/requests'
7import { CONSTRAINTS_FIELDS } from '../initializers/constants' 14import { CONSTRAINTS_FIELDS } from '../initializers/constants'
8import { getClient, getVideoCategories, login, searchVideoWithSort, uploadVideo } from '../../shared/extra-utils/index' 15import { getClient, getVideoCategories, login, searchVideoWithSort, uploadVideo } from '../../shared/extra-utils/index'
9import { truncate } from 'lodash'
10import * as prompt from 'prompt'
11import { accessSync, constants } from 'fs'
12import { remove } from 'fs-extra'
13import { sha256 } from '../helpers/core-utils' 16import { sha256 } from '../helpers/core-utils'
14import { buildOriginallyPublishedAt, getYoutubeDLVideoFormat, safeGetYoutubeDL } from '../helpers/youtube-dl' 17import { buildOriginallyPublishedAt, getYoutubeDLVideoFormat, safeGetYoutubeDL } from '../helpers/youtube-dl'
15import { buildCommonVideoOptions, buildVideoAttributesFromCommander, getLogger, getServerCredentials } from './cli' 18import { buildCommonVideoOptions, buildVideoAttributesFromCommander, getLogger, getServerCredentials } from './cli'
@@ -37,6 +40,7 @@ command
37 .option('--until <until>', 'Publication date (inclusive) until which the videos can be imported (YYYY-MM-DD)', parseDate) 40 .option('--until <until>', 'Publication date (inclusive) until which the videos can be imported (YYYY-MM-DD)', parseDate)
38 .option('--first <first>', 'Process first n elements of returned playlist') 41 .option('--first <first>', 'Process first n elements of returned playlist')
39 .option('--last <last>', 'Process last n elements of returned playlist') 42 .option('--last <last>', 'Process last n elements of returned playlist')
43 .option('--wait-interval <waitInterval>', 'Duration between two video imports (in seconds)', convertIntoMs)
40 .option('-T, --tmpdir <tmpdir>', 'Working directory', __dirname) 44 .option('-T, --tmpdir <tmpdir>', 'Working directory', __dirname)
41 .usage("[global options] [ -- youtube-dl options]") 45 .usage("[global options] [ -- youtube-dl options]")
42 .parse(process.argv) 46 .parse(process.argv)
@@ -87,19 +91,23 @@ async function run (url: string, user: UserInfo) {
87 91
88 let infoArray: any[] 92 let infoArray: any[]
89 93
90 // Normalize utf8 fields
91 infoArray = [].concat(info) 94 infoArray = [].concat(info)
92 if (program['first']) { 95 if (program['first']) {
93 infoArray = infoArray.slice(0, program['first']) 96 infoArray = infoArray.slice(0, program['first'])
94 } else if (program['last']) { 97 } else if (program['last']) {
95 infoArray = infoArray.slice(-program['last']) 98 infoArray = infoArray.slice(-program['last'])
96 } 99 }
100 // Normalize utf8 fields
97 infoArray = infoArray.map(i => normalizeObject(i)) 101 infoArray = infoArray.map(i => normalizeObject(i))
98 102
99 log.info('Will download and upload %d videos.\n', infoArray.length) 103 log.info('Will download and upload %d videos.\n', infoArray.length)
100 104
101 for (const info of infoArray) { 105 for (const [ index, info ] of infoArray.entries()) {
102 try { 106 try {
107 if (index > 0 && program['waitInterval']) {
108 log.info("Wait for %d seconds before continuing.", program['waitInterval'] / 1000)
109 await new Promise(res => setTimeout(res, program['waitInterval']))
110 }
103 await processVideo({ 111 await processVideo({
104 cwd: program['tmpdir'], 112 cwd: program['tmpdir'],
105 url, 113 url,
@@ -107,7 +115,7 @@ async function run (url: string, user: UserInfo) {
107 youtubeInfo: info 115 youtubeInfo: info
108 }) 116 })
109 } catch (err) { 117 } catch (err) {
110 console.error('Cannot process video.', { info, url }) 118 console.error('Cannot process video.', { info, url, err })
111 } 119 }
112 } 120 }
113 121
@@ -115,7 +123,7 @@ async function run (url: string, user: UserInfo) {
115 process.exit(0) 123 process.exit(0)
116} 124}
117 125
118function processVideo (parameters: { 126async function processVideo (parameters: {
119 cwd: string 127 cwd: string
120 url: string 128 url: string
121 user: { username: string, password: string } 129 user: { username: string, password: string }
@@ -123,63 +131,52 @@ function processVideo (parameters: {
123}) { 131}) {
124 const { youtubeInfo, cwd, url, user } = parameters 132 const { youtubeInfo, cwd, url, user } = parameters
125 133
126 return new Promise(async res => { 134 log.debug('Fetching object.', youtubeInfo)
127 log.debug('Fetching object.', youtubeInfo)
128 135
129 const videoInfo = await fetchObject(youtubeInfo) 136 const videoInfo = await fetchObject(youtubeInfo)
130 log.debug('Fetched object.', videoInfo) 137 log.debug('Fetched object.', videoInfo)
131
132 const originallyPublishedAt = buildOriginallyPublishedAt(videoInfo)
133
134 if (program['since'] && originallyPublishedAt && originallyPublishedAt.getTime() < program['since'].getTime()) {
135 log.info('Video "%s" has been published before "%s", don\'t upload it.\n',
136 videoInfo.title, formatDate(program['since']))
137 return res()
138 }
139
140 if (program['until'] && originallyPublishedAt && originallyPublishedAt.getTime() > program['until'].getTime()) {
141 log.info('Video "%s" has been published after "%s", don\'t upload it.\n',
142 videoInfo.title, formatDate(program['until']))
143 return res()
144 }
145 138
146 const result = await searchVideoWithSort(url, videoInfo.title, '-match') 139 const originallyPublishedAt = buildOriginallyPublishedAt(videoInfo)
140 if (program['since'] && originallyPublishedAt && originallyPublishedAt.getTime() < program['since'].getTime()) {
141 log.info('Video "%s" has been published before "%s", don\'t upload it.\n',
142 videoInfo.title, formatDate(program['since']))
143 return
144 }
145 if (program['until'] && originallyPublishedAt && originallyPublishedAt.getTime() > program['until'].getTime()) {
146 log.info('Video "%s" has been published after "%s", don\'t upload it.\n',
147 videoInfo.title, formatDate(program['until']))
148 return
149 }
147 150
148 log.info('############################################################\n') 151 const result = await searchVideoWithSort(url, videoInfo.title, '-match')
149 152
150 if (result.body.data.find(v => v.name === videoInfo.title)) { 153 log.info('############################################################\n')
151 log.info('Video "%s" already exists, don\'t reupload it.\n', videoInfo.title)
152 return res()
153 }
154 154
155 const path = join(cwd, sha256(videoInfo.url) + '.mp4') 155 if (result.body.data.find(v => v.name === videoInfo.title)) {
156 log.info('Video "%s" already exists, don\'t reupload it.\n', videoInfo.title)
157 return
158 }
156 159
157 log.info('Downloading video "%s"...', videoInfo.title) 160 const path = join(cwd, sha256(videoInfo.url) + '.mp4')
158 161
159 const options = [ '-f', getYoutubeDLVideoFormat(), ...command.args, '-o', path ] 162 log.info('Downloading video "%s"...', videoInfo.title)
160 try {
161 const youtubeDL = await safeGetYoutubeDL()
162 youtubeDL.exec(videoInfo.url, options, processOptions, async (err, output) => {
163 if (err) {
164 log.error(err)
165 return res()
166 }
167 163
168 log.info(output.join('\n')) 164 const options = [ '-f', getYoutubeDLVideoFormat(), ...command.args, '-o', path ]
169 await uploadVideoOnPeerTube({ 165 try {
170 cwd, 166 const youtubeDL = await safeGetYoutubeDL()
171 url, 167 const youtubeDLExec = promisify(youtubeDL.exec).bind(youtubeDL)
172 user, 168 const output = await youtubeDLExec(videoInfo.url, options, processOptions)
173 videoInfo: normalizeObject(videoInfo), 169 log.info(output.join('\n'))
174 videoPath: path 170 await uploadVideoOnPeerTube({
175 }) 171 cwd,
176 return res() 172 url,
177 }) 173 user,
178 } catch (err) { 174 videoInfo: normalizeObject(videoInfo),
179 log.error(err.message) 175 videoPath: path
180 return res() 176 })
181 } 177 } catch (err) {
182 }) 178 log.error(err.message)
179 }
183} 180}
184 181
185async function uploadVideoOnPeerTube (parameters: { 182async function uploadVideoOnPeerTube (parameters: {
@@ -396,6 +393,14 @@ function formatDate (date: Date): string {
396 return date.toISOString().split('T')[0] 393 return date.toISOString().split('T')[0]
397} 394}
398 395
396function convertIntoMs (secondsAsStr: string): number {
397 const seconds = parseInt(secondsAsStr, 10)
398 if (seconds <= 0) {
399 exitError(`Invalid duration passed: ${seconds}. Expected duration to be strictly positive and in seconds`)
400 }
401 return Math.round(seconds * 1000)
402}
403
399function exitError (message: string, ...meta: any[]) { 404function exitError (message: string, ...meta: any[]) {
400 // use console.error instead of log.error here 405 // use console.error instead of log.error here
401 console.error(message, ...meta) 406 console.error(message, ...meta)