From 8dc8a34ee8428e7657414115d1c137592efa174d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 23 Apr 2020 09:32:53 +0200 Subject: Avoir some circular dependencies --- .../lib/job-queue/handlers/activitypub-follow.ts | 9 +---- .../handlers/activitypub-http-broadcast.ts | 9 +---- .../job-queue/handlers/activitypub-http-fetcher.ts | 13 ++----- .../job-queue/handlers/activitypub-http-unicast.ts | 9 +---- .../job-queue/handlers/activitypub-refresher.ts | 10 +++--- server/lib/job-queue/handlers/email.ts | 5 ++- .../handlers/utils/activitypub-http-utils.ts | 11 ++---- server/lib/job-queue/handlers/video-file-import.ts | 6 +--- server/lib/job-queue/handlers/video-import.ts | 23 ++---------- server/lib/job-queue/handlers/video-redundancy.ts | 5 +-- server/lib/job-queue/handlers/video-transcoding.ts | 42 ++++------------------ server/lib/job-queue/handlers/video-views.ts | 2 +- server/lib/job-queue/job-queue.ts | 28 +++++++++------ 13 files changed, 44 insertions(+), 128 deletions(-) (limited to 'server/lib/job-queue') diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts index e467c5b1b..7034c10d0 100644 --- a/server/lib/job-queue/handlers/activitypub-follow.ts +++ b/server/lib/job-queue/handlers/activitypub-follow.ts @@ -11,14 +11,7 @@ import { ActorModel } from '../../../models/activitypub/actor' import { Notifier } from '../../notifier' import { sequelizeTypescript } from '../../../initializers/database' import { MActor, MActorFollowActors, MActorFull } from '../../../typings/models' - -export type ActivitypubFollowPayload = { - followerActorId: number - name: string - host: string - isAutoFollow?: boolean - assertIsChannel?: boolean -} +import { ActivitypubFollowPayload } from '@shared/models' async function processActivityPubFollow (job: Bull.Job) { const payload = job.data as ActivitypubFollowPayload diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts index 7d9dd61e9..e4d3dbbff 100644 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts @@ -5,14 +5,7 @@ import { doRequest } from '../../../helpers/requests' import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers/constants' import { ActorFollowScoreCache } from '../../files-cache' -import { ContextType } from '@server/helpers/activitypub' - -export type ActivitypubHttpBroadcastPayload = { - uris: string[] - signatureActorId?: number - body: any - contextType?: ContextType -} +import { ActivitypubHttpBroadcastPayload } from '@shared/models' async function processActivityPubHttpBroadcast (job: Bull.Job) { logger.info('Processing ActivityPub broadcast in job %d.', job.id) diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts index 0182c5169..524aadc27 100644 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts @@ -5,22 +5,15 @@ import { processActivities } from '../../activitypub/process' import { addVideoComments } from '../../activitypub/video-comments' import { crawlCollectionPage } from '../../activitypub/crawl' import { VideoModel } from '../../../models/video/video' -import { addVideoShares, createRates } from '../../activitypub' +import { addVideoShares } from '../../activitypub/share' +import { createRates } from '../../activitypub/video-rates' import { createAccountPlaylists } from '../../activitypub/playlist' import { AccountModel } from '../../../models/account/account' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { VideoShareModel } from '../../../models/video/video-share' import { VideoCommentModel } from '../../../models/video/video-comment' import { MAccountDefault, MVideoFullLight } from '../../../typings/models' - -type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' | 'account-playlists' - -export type ActivitypubHttpFetcherPayload = { - uri: string - type: FetchType - videoId?: number - accountId?: number -} +import { ActivitypubHttpFetcherPayload, FetchType } from '@shared/models' async function processActivityPubHttpFetcher (job: Bull.Job) { logger.info('Processing ActivityPub fetcher in job %d.', job.id) diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts index 6b71e2891..b65eeb677 100644 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts @@ -4,14 +4,7 @@ import { doRequest } from '../../../helpers/requests' import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' import { JOB_REQUEST_TIMEOUT } from '../../../initializers/constants' import { ActorFollowScoreCache } from '../../files-cache' -import { ContextType } from '@server/helpers/activitypub' - -export type ActivitypubHttpUnicastPayload = { - uri: string - signatureActorId?: number - body: any - contextType?: ContextType -} +import { ActivitypubHttpUnicastPayload } from '@shared/models' async function processActivityPubHttpUnicast (job: Bull.Job) { logger.info('Processing ActivityPub unicast in job %d.', job.id) diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts index 4d6c38cfa..666e56868 100644 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts @@ -1,14 +1,12 @@ import * as Bull from 'bull' import { logger } from '../../../helpers/logger' import { fetchVideoByUrl } from '../../../helpers/video' -import { refreshActorIfNeeded, refreshVideoIfNeeded, refreshVideoPlaylistIfNeeded } from '../../activitypub' +import { refreshActorIfNeeded } from '../../activitypub/actor' +import { refreshVideoIfNeeded } from '../../activitypub/videos' import { ActorModel } from '../../../models/activitypub/actor' import { VideoPlaylistModel } from '../../../models/video/video-playlist' - -export type RefreshPayload = { - type: 'video' | 'video-playlist' | 'actor' - url: string -} +import { RefreshPayload } from '@shared/models' +import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlist' async function refreshAPObject (job: Bull.Job) { const payload = job.data as RefreshPayload diff --git a/server/lib/job-queue/handlers/email.ts b/server/lib/job-queue/handlers/email.ts index 62701222c..3157731e2 100644 --- a/server/lib/job-queue/handlers/email.ts +++ b/server/lib/job-queue/handlers/email.ts @@ -1,8 +1,7 @@ import * as Bull from 'bull' import { logger } from '../../../helpers/logger' -import { Emailer, SendEmailOptions } from '../../emailer' - -export type EmailPayload = SendEmailOptions +import { Emailer } from '../../emailer' +import { EmailPayload } from '@shared/models' async function processEmail (job: Bull.Job) { const payload = job.data as EmailPayload diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts index 258ffabee..437ea06fc 100644 --- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts +++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts @@ -1,9 +1,9 @@ import { buildSignedActivity, ContextType } from '../../../../helpers/activitypub' -import { getServerActor } from '../../../../helpers/utils' import { ActorModel } from '../../../../models/activitypub/actor' -import { sha256 } from '../../../../helpers/core-utils' import { ACTIVITY_PUB, HTTP_SIGNATURE } from '../../../../initializers/constants' import { MActor } from '../../../../typings/models' +import { getServerActor } from '@server/models/application/application' +import { buildDigest } from '@server/helpers/peertube-crypto' type Payload = { body: any, contextType?: ContextType, signatureActorId?: number } @@ -48,14 +48,7 @@ function buildGlobalHeaders (body: any) { } } -function buildDigest (body: any) { - const rawBody = typeof body === 'string' ? body : JSON.stringify(body) - - return 'SHA-256=' + sha256(rawBody, 'base64') -} - export { - buildDigest, buildGlobalHeaders, computeBody, buildSignedRequestOptions diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index be9e7d181..ae11f1de3 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts @@ -9,11 +9,7 @@ import { extname } from 'path' import { MVideoFile, MVideoWithFile } from '@server/typings/models' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { getVideoFilePath } from '@server/lib/video-paths' - -export type VideoFileImportPayload = { - videoUUID: string - filePath: string -} +import { VideoFileImportPayload } from '@shared/models' async function processVideoFileImport (job: Bull.Job) { const payload = job.data as VideoFileImportPayload diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 6cdae5b03..ad549c6fc 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -7,8 +7,8 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro import { extname } from 'path' import { VideoFileModel } from '../../../models/video/video-file' import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' -import { VideoState } from '../../../../shared' -import { federateVideoIfNeeded } from '../../activitypub' +import { VideoImportPayload, VideoImportTorrentPayload, VideoImportYoutubeDLPayload, VideoState } from '../../../../shared' +import { federateVideoIfNeeded } from '../../activitypub/videos' import { VideoModel } from '../../../models/video/video' import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' import { getSecureTorrentName } from '../../../helpers/utils' @@ -21,24 +21,7 @@ import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' import { MThumbnail } from '../../../typings/models/video/thumbnail' import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import' import { getVideoFilePath } from '@server/lib/video-paths' -import { addOptimizeOrMergeAudioJob } from '@server/lib/videos' - -type VideoImportYoutubeDLPayload = { - type: 'youtube-dl' - videoImportId: number - - generateThumbnail: boolean - generatePreview: boolean - - fileExt?: string -} - -type VideoImportTorrentPayload = { - type: 'magnet-uri' | 'torrent-file' - videoImportId: number -} - -export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload +import { addOptimizeOrMergeAudioJob } from '@server/helpers/video' async function processVideoImport (job: Bull.Job) { const payload = job.data as VideoImportPayload diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts index 319d7090e..6296dab05 100644 --- a/server/lib/job-queue/handlers/video-redundancy.ts +++ b/server/lib/job-queue/handlers/video-redundancy.ts @@ -1,10 +1,7 @@ import * as Bull from 'bull' import { logger } from '../../../helpers/logger' import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler' - -export type VideoRedundancyPayload = { - videoId: number -} +import { VideoRedundancyPayload } from '@shared/models' async function processVideoRedundancy (job: Bull.Job) { const payload = job.data as VideoRedundancyPayload diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index c020057c9..46add57d4 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -1,9 +1,14 @@ import * as Bull from 'bull' -import { VideoResolution } from '../../../../shared' +import { + MergeAudioTranscodingPayload, + NewResolutionTranscodingPayload, + OptimizeTranscodingPayload, + VideoTranscodingPayload +} from '../../../../shared' import { logger } from '../../../helpers/logger' import { VideoModel } from '../../../models/video/video' import { JobQueue } from '../job-queue' -import { federateVideoIfNeeded } from '../../activitypub' +import { federateVideoIfNeeded } from '../../activitypub/videos' import { retryTransactionWrapper } from '../../../helpers/database-utils' import { sequelizeTypescript } from '../../../initializers' import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' @@ -12,39 +17,6 @@ import { Notifier } from '../../notifier' import { CONFIG } from '../../../initializers/config' import { MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/typings/models' -interface BaseTranscodingPayload { - videoUUID: string - isNewVideo?: boolean -} - -interface HLSTranscodingPayload extends BaseTranscodingPayload { - type: 'hls' - isPortraitMode?: boolean - resolution: VideoResolution - copyCodecs: boolean -} - -interface NewResolutionTranscodingPayload extends BaseTranscodingPayload { - type: 'new-resolution' - isPortraitMode?: boolean - resolution: VideoResolution -} - -interface MergeAudioTranscodingPayload extends BaseTranscodingPayload { - type: 'merge-audio' - resolution: VideoResolution -} - -interface OptimizeTranscodingPayload extends BaseTranscodingPayload { - type: 'optimize' -} - -export type VideoTranscodingPayload = - HLSTranscodingPayload - | NewResolutionTranscodingPayload - | OptimizeTranscodingPayload - | MergeAudioTranscodingPayload - async function processVideoTranscoding (job: Bull.Job) { const payload = job.data as VideoTranscodingPayload logger.info('Processing video file in job %d.', job.id) diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts index 2258cd029..7211df237 100644 --- a/server/lib/job-queue/handlers/video-views.ts +++ b/server/lib/job-queue/handlers/video-views.ts @@ -3,7 +3,7 @@ import { logger } from '../../../helpers/logger' import { VideoModel } from '../../../models/video/video' import { VideoViewModel } from '../../../models/video/video-views' import { isTestInstance } from '../../../helpers/core-utils' -import { federateVideoIfNeeded } from '../../activitypub' +import { federateVideoIfNeeded } from '../../activitypub/videos' async function processVideosViews () { const lastHour = new Date() diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 14acace7d..d8d64caaf 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -1,19 +1,25 @@ import * as Bull from 'bull' -import { JobState, JobType } from '../../../shared/models' +import { + ActivitypubFollowPayload, + ActivitypubHttpBroadcastPayload, + ActivitypubHttpFetcherPayload, ActivitypubHttpUnicastPayload, EmailPayload, + JobState, + JobType, RefreshPayload, VideoFileImportPayload, VideoImportPayload, VideoRedundancyPayload, VideoTranscodingPayload +} from '../../../shared/models' import { logger } from '../../helpers/logger' import { Redis } from '../redis' import { JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants' -import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' -import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' -import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' -import { EmailPayload, processEmail } from './handlers/email' -import { processVideoTranscoding, VideoTranscodingPayload } from './handlers/video-transcoding' -import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' -import { processVideoImport, VideoImportPayload } from './handlers/video-import' +import { processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' +import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' +import { processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' +import { processEmail } from './handlers/email' +import { processVideoTranscoding} from './handlers/video-transcoding' +import { processActivityPubFollow } from './handlers/activitypub-follow' +import { processVideoImport} from './handlers/video-import' import { processVideosViews } from './handlers/video-views' -import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' -import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import' -import { processVideoRedundancy, VideoRedundancyPayload } from '@server/lib/job-queue/handlers/video-redundancy' +import { refreshAPObject} from './handlers/activitypub-refresher' +import { processVideoFileImport} from './handlers/video-file-import' +import { processVideoRedundancy} from '@server/lib/job-queue/handlers/video-redundancy' type CreateJobArgument = { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | -- cgit v1.2.3