aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/job-queue
diff options
context:
space:
mode:
authorFlorent <florent.git@zeteo.me>2022-08-10 09:53:39 +0200
committerGitHub <noreply@github.com>2022-08-10 09:53:39 +0200
commit2a491182e483b97afb1b65c908b23cb48d591807 (patch)
treeec13503216ad72a3ea8f1ce3b659899f8167fb47 /server/lib/job-queue
parent06ac128958c489efe1008eeca1df683819bd2f18 (diff)
downloadPeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.gz
PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.zst
PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.zip
Channel sync (#5135)
* Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'server/lib/job-queue')
-rw-r--r--server/lib/job-queue/handlers/after-video-channel-import.ts37
-rw-r--r--server/lib/job-queue/handlers/video-channel-import.ts36
-rw-r--r--server/lib/job-queue/handlers/video-import.ts20
-rw-r--r--server/lib/job-queue/job-queue.ts19
4 files changed, 103 insertions, 9 deletions
diff --git a/server/lib/job-queue/handlers/after-video-channel-import.ts b/server/lib/job-queue/handlers/after-video-channel-import.ts
new file mode 100644
index 000000000..ffdd8c5b5
--- /dev/null
+++ b/server/lib/job-queue/handlers/after-video-channel-import.ts
@@ -0,0 +1,37 @@
1import { Job } from 'bullmq'
2import { logger } from '@server/helpers/logger'
3import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
4import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@shared/models'
5
6export async function processAfterVideoChannelImport (job: Job) {
7 const payload = job.data as AfterVideoChannelImportPayload
8 if (!payload.channelSyncId) return
9
10 logger.info('Processing after video channel import in job %s.', job.id)
11
12 const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId)
13 if (!sync) {
14 logger.error('Unknown sync id %d.', payload.channelSyncId)
15 return
16 }
17
18 const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()
19
20 let errors = 0
21 let successes = 0
22
23 for (const value of Object.values(childrenValues)) {
24 if (value.resultType === 'success') successes++
25 else if (value.resultType === 'error') errors++
26 }
27
28 if (errors > 0) {
29 sync.state = VideoChannelSyncState.FAILED
30 logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes })
31 } else {
32 sync.state = VideoChannelSyncState.SYNCED
33 logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes })
34 }
35
36 await sync.save()
37}
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts
new file mode 100644
index 000000000..9bdb2d269
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-channel-import.ts
@@ -0,0 +1,36 @@
1import { Job } from 'bullmq'
2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config'
4import { synchronizeChannel } from '@server/lib/sync-channel'
5import { VideoChannelModel } from '@server/models/video/video-channel'
6import { VideoChannelImportPayload } from '@shared/models'
7
8export async function processVideoChannelImport (job: Job) {
9 const payload = job.data as VideoChannelImportPayload
10
11 logger.info('Processing video channel import in job %s.', job.id)
12
13 // Channel import requires only http upload to be allowed
14 if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
15 logger.error('Cannot import channel as the HTTP upload is disabled')
16 return
17 }
18
19 if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
20 logger.error('Cannot import channel as the synchronization is disabled')
21 return
22 }
23
24 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
25
26 try {
27 logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
28
29 await synchronizeChannel({
30 channel: videoChannel,
31 externalChannelUrl: payload.externalChannelUrl
32 })
33 } catch (err) {
34 logger.error(`Failed to import channel ${videoChannel.name}`, { err })
35 }
36}
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index f4629159c..9901b878c 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -8,7 +8,7 @@ import { generateWebTorrentVideoFilename } from '@server/lib/paths'
8import { Hooks } from '@server/lib/plugins/hooks' 8import { Hooks } from '@server/lib/plugins/hooks'
9import { ServerConfigManager } from '@server/lib/server-config-manager' 9import { ServerConfigManager } from '@server/lib/server-config-manager'
10import { isAbleToUploadVideo } from '@server/lib/user' 10import { isAbleToUploadVideo } from '@server/lib/user'
11import { buildOptimizeOrMergeAudioJob, buildMoveToObjectStorageJob } from '@server/lib/video' 11import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video'
12import { VideoPathManager } from '@server/lib/video-path-manager' 12import { VideoPathManager } from '@server/lib/video-path-manager'
13import { buildNextVideoState } from '@server/lib/video-state' 13import { buildNextVideoState } from '@server/lib/video-state'
14import { ThumbnailModel } from '@server/models/video/thumbnail' 14import { ThumbnailModel } from '@server/models/video/thumbnail'
@@ -18,6 +18,7 @@ import { isAudioFile } from '@shared/extra-utils'
18import { 18import {
19 ThumbnailType, 19 ThumbnailType,
20 VideoImportPayload, 20 VideoImportPayload,
21 VideoImportPreventExceptionResult,
21 VideoImportState, 22 VideoImportState,
22 VideoImportTorrentPayload, 23 VideoImportTorrentPayload,
23 VideoImportTorrentPayloadType, 24 VideoImportTorrentPayloadType,
@@ -41,20 +42,29 @@ import { Notifier } from '../../notifier'
41import { generateVideoMiniature } from '../../thumbnail' 42import { generateVideoMiniature } from '../../thumbnail'
42import { JobQueue } from '../job-queue' 43import { JobQueue } from '../job-queue'
43 44
44async function processVideoImport (job: Job) { 45async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
45 const payload = job.data as VideoImportPayload 46 const payload = job.data as VideoImportPayload
46 47
47 const videoImport = await getVideoImportOrDie(payload) 48 const videoImport = await getVideoImportOrDie(payload)
48 if (videoImport.state === VideoImportState.CANCELLED) { 49 if (videoImport.state === VideoImportState.CANCELLED) {
49 logger.info('Do not process import since it has been cancelled', { payload }) 50 logger.info('Do not process import since it has been cancelled', { payload })
50 return 51 return { resultType: 'success' }
51 } 52 }
52 53
53 videoImport.state = VideoImportState.PROCESSING 54 videoImport.state = VideoImportState.PROCESSING
54 await videoImport.save() 55 await videoImport.save()
55 56
56 if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, videoImport, payload) 57 try {
57 if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, videoImport, payload) 58 if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload)
59 if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload)
60
61 return { resultType: 'success' }
62 } catch (err) {
63 if (!payload.preventException) throw err
64
65 logger.warn('Catch error in video import to send value to parent job.', { payload, err })
66 return { resultType: 'error' }
67 }
58} 68}
59 69
60// --------------------------------------------------------------------------- 70// ---------------------------------------------------------------------------
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 281e2e51a..3970d48b7 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -22,6 +22,7 @@ import {
22 ActivitypubHttpFetcherPayload, 22 ActivitypubHttpFetcherPayload,
23 ActivitypubHttpUnicastPayload, 23 ActivitypubHttpUnicastPayload,
24 ActorKeysPayload, 24 ActorKeysPayload,
25 AfterVideoChannelImportPayload,
25 DeleteResumableUploadMetaFilePayload, 26 DeleteResumableUploadMetaFilePayload,
26 EmailPayload, 27 EmailPayload,
27 FederateVideoPayload, 28 FederateVideoPayload,
@@ -31,6 +32,7 @@ import {
31 MoveObjectStoragePayload, 32 MoveObjectStoragePayload,
32 NotifyPayload, 33 NotifyPayload,
33 RefreshPayload, 34 RefreshPayload,
35 VideoChannelImportPayload,
34 VideoFileImportPayload, 36 VideoFileImportPayload,
35 VideoImportPayload, 37 VideoImportPayload,
36 VideoLiveEndingPayload, 38 VideoLiveEndingPayload,
@@ -53,12 +55,14 @@ import { processFederateVideo } from './handlers/federate-video'
53import { processManageVideoTorrent } from './handlers/manage-video-torrent' 55import { processManageVideoTorrent } from './handlers/manage-video-torrent'
54import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' 56import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage'
55import { processNotify } from './handlers/notify' 57import { processNotify } from './handlers/notify'
58import { processVideoChannelImport } from './handlers/video-channel-import'
56import { processVideoFileImport } from './handlers/video-file-import' 59import { processVideoFileImport } from './handlers/video-file-import'
57import { processVideoImport } from './handlers/video-import' 60import { processVideoImport } from './handlers/video-import'
58import { processVideoLiveEnding } from './handlers/video-live-ending' 61import { processVideoLiveEnding } from './handlers/video-live-ending'
59import { processVideoStudioEdition } from './handlers/video-studio-edition' 62import { processVideoStudioEdition } from './handlers/video-studio-edition'
60import { processVideoTranscoding } from './handlers/video-transcoding' 63import { processVideoTranscoding } from './handlers/video-transcoding'
61import { processVideosViewsStats } from './handlers/video-views-stats' 64import { processVideosViewsStats } from './handlers/video-views-stats'
65import { processAfterVideoChannelImport } from './handlers/after-video-channel-import'
62 66
63export type CreateJobArgument = 67export type CreateJobArgument =
64 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 68 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -79,6 +83,9 @@ export type CreateJobArgument =
79 { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } | 83 { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } |
80 { type: 'video-studio-edition', payload: VideoStudioEditionPayload } | 84 { type: 'video-studio-edition', payload: VideoStudioEditionPayload } |
81 { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } | 85 { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } |
86 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
87 { type: 'video-channel-import', payload: VideoChannelImportPayload } |
88 { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
82 { type: 'notify', payload: NotifyPayload } | 89 { type: 'notify', payload: NotifyPayload } |
83 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | 90 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
84 { type: 'federate-video', payload: FederateVideoPayload } 91 { type: 'federate-video', payload: FederateVideoPayload }
@@ -106,8 +113,10 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
106 'video-redundancy': processVideoRedundancy, 113 'video-redundancy': processVideoRedundancy,
107 'move-to-object-storage': processMoveToObjectStorage, 114 'move-to-object-storage': processMoveToObjectStorage,
108 'manage-video-torrent': processManageVideoTorrent, 115 'manage-video-torrent': processManageVideoTorrent,
109 'notify': processNotify,
110 'video-studio-edition': processVideoStudioEdition, 116 'video-studio-edition': processVideoStudioEdition,
117 'video-channel-import': processVideoChannelImport,
118 'after-video-channel-import': processAfterVideoChannelImport,
119 'notify': processNotify,
111 'federate-video': processFederateVideo 120 'federate-video': processFederateVideo
112} 121}
113 122
@@ -134,6 +143,8 @@ const jobTypes: JobType[] = [
134 'move-to-object-storage', 143 'move-to-object-storage',
135 'manage-video-torrent', 144 'manage-video-torrent',
136 'video-studio-edition', 145 'video-studio-edition',
146 'video-channel-import',
147 'after-video-channel-import',
137 'notify', 148 'notify',
138 'federate-video' 149 'federate-video'
139] 150]
@@ -306,7 +317,7 @@ class JobQueue {
306 .catch(err => logger.error('Cannot create job.', { err, options })) 317 .catch(err => logger.error('Cannot create job.', { err, options }))
307 } 318 }
308 319
309 async createJob (options: CreateJobArgument & CreateJobOptions) { 320 createJob (options: CreateJobArgument & CreateJobOptions) {
310 const queue: Queue = this.queues[options.type] 321 const queue: Queue = this.queues[options.type]
311 if (queue === undefined) { 322 if (queue === undefined) {
312 logger.error('Unknown queue %s: cannot create job.', options.type) 323 logger.error('Unknown queue %s: cannot create job.', options.type)
@@ -318,7 +329,7 @@ class JobQueue {
318 return queue.add('job', options.payload, jobOptions) 329 return queue.add('job', options.payload, jobOptions)
319 } 330 }
320 331
321 async createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) { 332 createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
322 let lastJob: FlowJob 333 let lastJob: FlowJob
323 334
324 for (const job of jobs) { 335 for (const job of jobs) {
@@ -336,7 +347,7 @@ class JobQueue {
336 return this.flowProducer.add(lastJob) 347 return this.flowProducer.add(lastJob)
337 } 348 }
338 349
339 async createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) { 350 createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
340 return this.flowProducer.add({ 351 return this.flowProducer.add({
341 ...this.buildJobFlowOption(parent), 352 ...this.buildJobFlowOption(parent),
342 353