aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/helpers
diff options
context:
space:
mode:
authorFlorent <florent.git@zeteo.me>2022-08-10 09:53:39 +0200
committerGitHub <noreply@github.com>2022-08-10 09:53:39 +0200
commit2a491182e483b97afb1b65c908b23cb48d591807 (patch)
treeec13503216ad72a3ea8f1ce3b659899f8167fb47 /server/helpers
parent06ac128958c489efe1008eeca1df683819bd2f18 (diff)
downloadPeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.gz
PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.zst
PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.zip
Channel sync (#5135)
* Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'server/helpers')
-rw-r--r--server/helpers/audit-logger.ts17
-rw-r--r--server/helpers/custom-validators/video-channel-syncs.ts6
-rw-r--r--server/helpers/youtube-dl/youtube-dl-cli.ts27
-rw-r--r--server/helpers/youtube-dl/youtube-dl-info-builder.ts4
-rw-r--r--server/helpers/youtube-dl/youtube-dl-wrapper.ts20
5 files changed, 68 insertions, 6 deletions
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts
index 076b7f11d..7e8a03e8f 100644
--- a/server/helpers/audit-logger.ts
+++ b/server/helpers/audit-logger.ts
@@ -5,7 +5,7 @@ import { chain } from 'lodash'
5import { join } from 'path' 5import { join } from 'path'
6import { addColors, config, createLogger, format, transports } from 'winston' 6import { addColors, config, createLogger, format, transports } from 'winston'
7import { AUDIT_LOG_FILENAME } from '@server/initializers/constants' 7import { AUDIT_LOG_FILENAME } from '@server/initializers/constants'
8import { AdminAbuse, CustomConfig, User, VideoChannel, VideoComment, VideoDetails, VideoImport } from '@shared/models' 8import { AdminAbuse, CustomConfig, User, VideoChannel, VideoChannelSync, VideoComment, VideoDetails, VideoImport } from '@shared/models'
9import { CONFIG } from '../initializers/config' 9import { CONFIG } from '../initializers/config'
10import { jsonLoggerFormat, labelFormatter } from './logger' 10import { jsonLoggerFormat, labelFormatter } from './logger'
11 11
@@ -260,6 +260,18 @@ class CustomConfigAuditView extends EntityAuditView {
260 } 260 }
261} 261}
262 262
263const channelSyncKeysToKeep = [
264 'id',
265 'externalChannelUrl',
266 'channel-id',
267 'channel-name'
268]
269class VideoChannelSyncAuditView extends EntityAuditView {
270 constructor (channelSync: VideoChannelSync) {
271 super(channelSyncKeysToKeep, 'channelSync', channelSync)
272 }
273}
274
263export { 275export {
264 getAuditIdFromRes, 276 getAuditIdFromRes,
265 277
@@ -270,5 +282,6 @@ export {
270 UserAuditView, 282 UserAuditView,
271 VideoAuditView, 283 VideoAuditView,
272 AbuseAuditView, 284 AbuseAuditView,
273 CustomConfigAuditView 285 CustomConfigAuditView,
286 VideoChannelSyncAuditView
274} 287}
diff --git a/server/helpers/custom-validators/video-channel-syncs.ts b/server/helpers/custom-validators/video-channel-syncs.ts
new file mode 100644
index 000000000..c5a9afa96
--- /dev/null
+++ b/server/helpers/custom-validators/video-channel-syncs.ts
@@ -0,0 +1,6 @@
1import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
2import { exists } from './misc'
3
4export function isVideoChannelSyncStateValid (value: any) {
5 return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined
6}
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
index 13c990a1e..5a87b99b4 100644
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/helpers/youtube-dl/youtube-dl-cli.ts
@@ -87,6 +87,7 @@ export class YoutubeDLCLI {
87 return result.concat([ 87 return result.concat([
88 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio', 88 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
89 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats 89 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
90 'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
90 'best' // Ultimate fallback 91 'best' // Ultimate fallback
91 ]).join('/') 92 ]).join('/')
92 } 93 }
@@ -103,11 +104,14 @@ export class YoutubeDLCLI {
103 timeout?: number 104 timeout?: number
104 additionalYoutubeDLArgs?: string[] 105 additionalYoutubeDLArgs?: string[]
105 }) { 106 }) {
107 let args = options.additionalYoutubeDLArgs || []
108 args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
109
106 return this.run({ 110 return this.run({
107 url: options.url, 111 url: options.url,
108 processOptions: options.processOptions, 112 processOptions: options.processOptions,
109 timeout: options.timeout, 113 timeout: options.timeout,
110 args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ]) 114 args
111 }) 115 })
112 } 116 }
113 117
@@ -129,6 +133,25 @@ export class YoutubeDLCLI {
129 : info 133 : info
130 } 134 }
131 135
136 getListInfo (options: {
137 url: string
138 latestVideosCount?: number
139 processOptions: execa.NodeOptions
140 }): Promise<{ upload_date: string, webpage_url: string }[]> {
141 const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
142
143 if (options.latestVideosCount !== undefined) {
144 additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
145 }
146
147 return this.getInfo({
148 url: options.url,
149 format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
150 processOptions: options.processOptions,
151 additionalYoutubeDLArgs
152 })
153 }
154
132 async getSubs (options: { 155 async getSubs (options: {
133 url: string 156 url: string
134 format: 'vtt' 157 format: 'vtt'
@@ -175,7 +198,7 @@ export class YoutubeDLCLI {
175 198
176 const output = await subProcess 199 const output = await subProcess
177 200
178 logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() }) 201 logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
179 202
180 return output.stdout 203 return output.stdout
181 ? output.stdout.trim().split(/\r?\n/) 204 ? output.stdout.trim().split(/\r?\n/)
diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
index 71572f292..303e4051f 100644
--- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts
+++ b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
@@ -13,6 +13,7 @@ type YoutubeDLInfo = {
13 thumbnailUrl?: string 13 thumbnailUrl?: string
14 ext?: string 14 ext?: string
15 originallyPublishedAt?: Date 15 originallyPublishedAt?: Date
16 webpageUrl?: string
16 17
17 urls?: string[] 18 urls?: string[]
18} 19}
@@ -81,7 +82,8 @@ class YoutubeDLInfoBuilder {
81 thumbnailUrl: obj.thumbnail || undefined, 82 thumbnailUrl: obj.thumbnail || undefined,
82 urls: this.buildAvailableUrl(obj), 83 urls: this.buildAvailableUrl(obj),
83 originallyPublishedAt: this.buildOriginallyPublishedAt(obj), 84 originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
84 ext: obj.ext 85 ext: obj.ext,
86 webpageUrl: obj.webpage_url
85 } 87 }
86 } 88 }
87 89
diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
index 176cf3b69..7cd5e3310 100644
--- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts
+++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
@@ -46,6 +46,24 @@ class YoutubeDLWrapper {
46 return infoBuilder.getInfo() 46 return infoBuilder.getInfo()
47 } 47 }
48 48
49 async getInfoForListImport (options: {
50 latestVideosCount?: number
51 }) {
52 const youtubeDL = await YoutubeDLCLI.safeGet()
53
54 const list = await youtubeDL.getListInfo({
55 url: this.url,
56 latestVideosCount: options.latestVideosCount,
57 processOptions
58 })
59
60 return list.map(info => {
61 const infoBuilder = new YoutubeDLInfoBuilder(info)
62
63 return infoBuilder.getInfo()
64 })
65 }
66
49 async getSubtitles (): Promise<YoutubeDLSubs> { 67 async getSubtitles (): Promise<YoutubeDLSubs> {
50 const cwd = CONFIG.STORAGE.TMP_DIR 68 const cwd = CONFIG.STORAGE.TMP_DIR
51 69
@@ -103,7 +121,7 @@ class YoutubeDLWrapper {
103 121
104 return remove(path) 122 return remove(path)
105 }) 123 })
106 .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() })) 124 .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
107 125
108 throw err 126 throw err
109 } 127 }