aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/process/process-create.ts2
-rw-r--r--server/lib/activitypub/video-comments.ts35
-rw-r--r--server/lib/job-queue/handlers/video-channel-import.ts22
-rw-r--r--server/lib/moderation.ts42
-rw-r--r--server/lib/schedulers/video-channel-sync-latest-scheduler.ts35
-rw-r--r--server/lib/sync-channel.ts90
6 files changed, 131 insertions, 95 deletions
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 76ed37aae..1e6e8956c 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -109,8 +109,10 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: MAc
109 let video: MVideoAccountLightBlacklistAllFiles 109 let video: MVideoAccountLightBlacklistAllFiles
110 let created: boolean 110 let created: boolean
111 let comment: MCommentOwnerVideo 111 let comment: MCommentOwnerVideo
112
112 try { 113 try {
113 const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false }) 114 const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false })
115 if (!resolveThreadResult) return // Comment not accepted
114 116
115 video = resolveThreadResult.video 117 video = resolveThreadResult.video
116 created = resolveThreadResult.commentCreated 118 created = resolveThreadResult.commentCreated
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index 911c7cd30..b65baf0e9 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -4,7 +4,9 @@ import { logger } from '../../helpers/logger'
4import { doJSONRequest } from '../../helpers/requests' 4import { doJSONRequest } from '../../helpers/requests'
5import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' 5import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
6import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
7import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' 7import { MComment, MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video'
8import { isRemoteVideoCommentAccepted } from '../moderation'
9import { Hooks } from '../plugins/hooks'
8import { getOrCreateAPActor } from './actors' 10import { getOrCreateAPActor } from './actors'
9import { checkUrlsSameHost } from './url' 11import { checkUrlsSameHost } from './url'
10import { getOrCreateAPVideo } from './videos' 12import { getOrCreateAPVideo } from './videos'
@@ -103,6 +105,10 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
103 firstReply.changed('updatedAt', true) 105 firstReply.changed('updatedAt', true)
104 firstReply.Video = video 106 firstReply.Video = video
105 107
108 if (await isRemoteCommentAccepted(firstReply) !== true) {
109 return undefined
110 }
111
106 comments[comments.length - 1] = await firstReply.save() 112 comments[comments.length - 1] = await firstReply.save()
107 113
108 for (let i = comments.length - 2; i >= 0; i--) { 114 for (let i = comments.length - 2; i >= 0; i--) {
@@ -113,6 +119,10 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
113 comment.changed('updatedAt', true) 119 comment.changed('updatedAt', true)
114 comment.Video = video 120 comment.Video = video
115 121
122 if (await isRemoteCommentAccepted(comment) !== true) {
123 return undefined
124 }
125
116 comments[i] = await comment.save() 126 comments[i] = await comment.save()
117 } 127 }
118 128
@@ -169,3 +179,26 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) {
169 commentCreated: true 179 commentCreated: true
170 }) 180 })
171} 181}
182
183async function isRemoteCommentAccepted (comment: MComment) {
184 // Already created
185 if (comment.id) return true
186
187 const acceptParameters = {
188 comment
189 }
190
191 const acceptedResult = await Hooks.wrapFun(
192 isRemoteVideoCommentAccepted,
193 acceptParameters,
194 'filter:activity-pub.remote-video-comment.create.accept.result'
195 )
196
197 if (!acceptedResult || acceptedResult.accepted !== true) {
198 logger.info('Refused to create a remote comment.', { acceptedResult, acceptParameters })
199
200 return false
201 }
202
203 return true
204}
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts
index 600292844..c3dd8a688 100644
--- a/server/lib/job-queue/handlers/video-channel-import.ts
+++ b/server/lib/job-queue/handlers/video-channel-import.ts
@@ -5,7 +5,7 @@ import { synchronizeChannel } from '@server/lib/sync-channel'
5import { VideoChannelModel } from '@server/models/video/video-channel' 5import { VideoChannelModel } from '@server/models/video/video-channel'
6import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' 6import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
7import { MChannelSync } from '@server/types/models' 7import { MChannelSync } from '@server/types/models'
8import { VideoChannelImportPayload, VideoChannelSyncState } from '@shared/models' 8import { VideoChannelImportPayload } from '@shared/models'
9 9
10export async function processVideoChannelImport (job: Job) { 10export async function processVideoChannelImport (job: Job) {
11 const payload = job.data as VideoChannelImportPayload 11 const payload = job.data as VideoChannelImportPayload
@@ -32,17 +32,11 @@ export async function processVideoChannelImport (job: Job) {
32 32
33 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) 33 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
34 34
35 try { 35 logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
36 logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `) 36
37 37 await synchronizeChannel({
38 await synchronizeChannel({ 38 channel: videoChannel,
39 channel: videoChannel, 39 externalChannelUrl: payload.externalChannelUrl,
40 externalChannelUrl: payload.externalChannelUrl, 40 channelSync
41 channelSync 41 })
42 })
43 } catch (err) {
44 logger.error(`Failed to import channel ${videoChannel.name}`, { err })
45 channelSync.state = VideoChannelSyncState.FAILED
46 await channelSync.save()
47 }
48} 42}
diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts
index c23f5b6a6..3cc92ca30 100644
--- a/server/lib/moderation.ts
+++ b/server/lib/moderation.ts
@@ -1,4 +1,4 @@
1import { VideoUploadFile } from 'express' 1import express, { VideoUploadFile } from 'express'
2import { PathLike } from 'fs-extra' 2import { PathLike } from 'fs-extra'
3import { Transaction } from 'sequelize/types' 3import { Transaction } from 'sequelize/types'
4import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger' 4import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger'
@@ -13,18 +13,15 @@ import {
13 MAbuseFull, 13 MAbuseFull,
14 MAccountDefault, 14 MAccountDefault,
15 MAccountLight, 15 MAccountLight,
16 MComment,
16 MCommentAbuseAccountVideo, 17 MCommentAbuseAccountVideo,
17 MCommentOwnerVideo, 18 MCommentOwnerVideo,
18 MUser, 19 MUser,
19 MVideoAbuseVideoFull, 20 MVideoAbuseVideoFull,
20 MVideoAccountLightBlacklistAllFiles 21 MVideoAccountLightBlacklistAllFiles
21} from '@server/types/models' 22} from '@server/types/models'
22import { ActivityCreate } from '../../shared/models/activitypub'
23import { VideoObject } from '../../shared/models/activitypub/objects'
24import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object'
25import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos' 23import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos'
26import { VideoCommentCreate } from '../../shared/models/videos/comment' 24import { VideoCommentCreate } from '../../shared/models/videos/comment'
27import { ActorModel } from '../models/actor/actor'
28import { UserModel } from '../models/user/user' 25import { UserModel } from '../models/user/user'
29import { VideoModel } from '../models/video/video' 26import { VideoModel } from '../models/video/video'
30import { VideoCommentModel } from '../models/video/video-comment' 27import { VideoCommentModel } from '../models/video/video-comment'
@@ -36,7 +33,9 @@ export type AcceptResult = {
36 errorMessage?: string 33 errorMessage?: string
37} 34}
38 35
39// Can be filtered by plugins 36// ---------------------------------------------------------------------------
37
38// Stub function that can be filtered by plugins
40function isLocalVideoAccepted (object: { 39function isLocalVideoAccepted (object: {
41 videoBody: VideoCreate 40 videoBody: VideoCreate
42 videoFile: VideoUploadFile 41 videoFile: VideoUploadFile
@@ -45,6 +44,9 @@ function isLocalVideoAccepted (object: {
45 return { accepted: true } 44 return { accepted: true }
46} 45}
47 46
47// ---------------------------------------------------------------------------
48
49// Stub function that can be filtered by plugins
48function isLocalLiveVideoAccepted (object: { 50function isLocalLiveVideoAccepted (object: {
49 liveVideoBody: LiveVideoCreate 51 liveVideoBody: LiveVideoCreate
50 user: UserModel 52 user: UserModel
@@ -52,7 +54,11 @@ function isLocalLiveVideoAccepted (object: {
52 return { accepted: true } 54 return { accepted: true }
53} 55}
54 56
57// ---------------------------------------------------------------------------
58
59// Stub function that can be filtered by plugins
55function isLocalVideoThreadAccepted (_object: { 60function isLocalVideoThreadAccepted (_object: {
61 req: express.Request
56 commentBody: VideoCommentCreate 62 commentBody: VideoCommentCreate
57 video: VideoModel 63 video: VideoModel
58 user: UserModel 64 user: UserModel
@@ -60,7 +66,9 @@ function isLocalVideoThreadAccepted (_object: {
60 return { accepted: true } 66 return { accepted: true }
61} 67}
62 68
69// Stub function that can be filtered by plugins
63function isLocalVideoCommentReplyAccepted (_object: { 70function isLocalVideoCommentReplyAccepted (_object: {
71 req: express.Request
64 commentBody: VideoCommentCreate 72 commentBody: VideoCommentCreate
65 parentComment: VideoCommentModel 73 parentComment: VideoCommentModel
66 video: VideoModel 74 video: VideoModel
@@ -69,22 +77,18 @@ function isLocalVideoCommentReplyAccepted (_object: {
69 return { accepted: true } 77 return { accepted: true }
70} 78}
71 79
72function isRemoteVideoAccepted (_object: { 80// ---------------------------------------------------------------------------
73 activity: ActivityCreate
74 videoAP: VideoObject
75 byActor: ActorModel
76}): AcceptResult {
77 return { accepted: true }
78}
79 81
82// Stub function that can be filtered by plugins
80function isRemoteVideoCommentAccepted (_object: { 83function isRemoteVideoCommentAccepted (_object: {
81 activity: ActivityCreate 84 comment: MComment
82 commentAP: VideoCommentObject
83 byActor: ActorModel
84}): AcceptResult { 85}): AcceptResult {
85 return { accepted: true } 86 return { accepted: true }
86} 87}
87 88
89// ---------------------------------------------------------------------------
90
91// Stub function that can be filtered by plugins
88function isPreImportVideoAccepted (object: { 92function isPreImportVideoAccepted (object: {
89 videoImportBody: VideoImportCreate 93 videoImportBody: VideoImportCreate
90 user: MUser 94 user: MUser
@@ -92,6 +96,7 @@ function isPreImportVideoAccepted (object: {
92 return { accepted: true } 96 return { accepted: true }
93} 97}
94 98
99// Stub function that can be filtered by plugins
95function isPostImportVideoAccepted (object: { 100function isPostImportVideoAccepted (object: {
96 videoFilePath: PathLike 101 videoFilePath: PathLike
97 videoFile: VideoFileModel 102 videoFile: VideoFileModel
@@ -100,6 +105,8 @@ function isPostImportVideoAccepted (object: {
100 return { accepted: true } 105 return { accepted: true }
101} 106}
102 107
108// ---------------------------------------------------------------------------
109
103async function createVideoAbuse (options: { 110async function createVideoAbuse (options: {
104 baseAbuse: FilteredModelAttributes<AbuseModel> 111 baseAbuse: FilteredModelAttributes<AbuseModel>
105 videoInstance: MVideoAccountLightBlacklistAllFiles 112 videoInstance: MVideoAccountLightBlacklistAllFiles
@@ -189,12 +196,13 @@ function createAccountAbuse (options: {
189 }) 196 })
190} 197}
191 198
199// ---------------------------------------------------------------------------
200
192export { 201export {
193 isLocalLiveVideoAccepted, 202 isLocalLiveVideoAccepted,
194 203
195 isLocalVideoAccepted, 204 isLocalVideoAccepted,
196 isLocalVideoThreadAccepted, 205 isLocalVideoThreadAccepted,
197 isRemoteVideoAccepted,
198 isRemoteVideoCommentAccepted, 206 isRemoteVideoCommentAccepted,
199 isLocalVideoCommentReplyAccepted, 207 isLocalVideoCommentReplyAccepted,
200 isPreImportVideoAccepted, 208 isPreImportVideoAccepted,
diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
index a527f68b5..efb957fac 100644
--- a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
+++ b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
@@ -2,7 +2,6 @@ import { logger } from '@server/helpers/logger'
2import { CONFIG } from '@server/initializers/config' 2import { CONFIG } from '@server/initializers/config'
3import { VideoChannelModel } from '@server/models/video/video-channel' 3import { VideoChannelModel } from '@server/models/video/video-channel'
4import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' 4import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
5import { VideoChannelSyncState } from '@shared/models'
6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 5import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
7import { synchronizeChannel } from '../sync-channel' 6import { synchronizeChannel } from '../sync-channel'
8import { AbstractScheduler } from './abstract-scheduler' 7import { AbstractScheduler } from './abstract-scheduler'
@@ -28,26 +27,20 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
28 for (const sync of channelSyncs) { 27 for (const sync of channelSyncs) {
29 const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) 28 const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
30 29
31 try { 30 logger.info(
32 logger.info( 31 'Creating video import jobs for "%s" sync with external channel "%s"',
33 'Creating video import jobs for "%s" sync with external channel "%s"', 32 channel.Actor.preferredUsername, sync.externalChannelUrl
34 channel.Actor.preferredUsername, sync.externalChannelUrl 33 )
35 ) 34
36 35 const onlyAfter = sync.lastSyncAt || sync.createdAt
37 const onlyAfter = sync.lastSyncAt || sync.createdAt 36
38 37 await synchronizeChannel({
39 await synchronizeChannel({ 38 channel,
40 channel, 39 externalChannelUrl: sync.externalChannelUrl,
41 externalChannelUrl: sync.externalChannelUrl, 40 videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
42 videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, 41 channelSync: sync,
43 channelSync: sync, 42 onlyAfter
44 onlyAfter 43 })
45 })
46 } catch (err) {
47 logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err })
48 sync.state = VideoChannelSyncState.FAILED
49 await sync.save()
50 }
51 } 44 }
52 } 45 }
53 46
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts
index f91599c14..35af91429 100644
--- a/server/lib/sync-channel.ts
+++ b/server/lib/sync-channel.ts
@@ -24,56 +24,62 @@ export async function synchronizeChannel (options: {
24 await channelSync.save() 24 await channelSync.save()
25 } 25 }
26 26
27 const user = await UserModel.loadByChannelActorId(channel.actorId) 27 try {
28 const youtubeDL = new YoutubeDLWrapper( 28 const user = await UserModel.loadByChannelActorId(channel.actorId)
29 externalChannelUrl, 29 const youtubeDL = new YoutubeDLWrapper(
30 ServerConfigManager.Instance.getEnabledResolutions('vod'), 30 externalChannelUrl,
31 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION 31 ServerConfigManager.Instance.getEnabledResolutions('vod'),
32 ) 32 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
33 33 )
34 const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
35
36 logger.info(
37 'Fetched %d candidate URLs for sync channel %s.',
38 targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
39 )
40
41 if (targetUrls.length === 0) {
42 if (channelSync) {
43 channelSync.state = VideoChannelSyncState.SYNCED
44 await channelSync.save()
45 }
46
47 return
48 }
49 34
50 const children: CreateJobArgument[] = [] 35 const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
51 36
52 for (const targetUrl of targetUrls) { 37 logger.info(
53 if (await skipImport(channel, targetUrl, onlyAfter)) continue 38 'Fetched %d candidate URLs for sync channel %s.',
39 targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
40 )
54 41
55 const { job } = await buildYoutubeDLImport({ 42 if (targetUrls.length === 0) {
56 user, 43 if (channelSync) {
57 channel, 44 channelSync.state = VideoChannelSyncState.SYNCED
58 targetUrl, 45 await channelSync.save()
59 channelSync,
60 importDataOverride: {
61 privacy: VideoPrivacy.PUBLIC
62 } 46 }
63 })
64 47
65 children.push(job) 48 return
66 } 49 }
50
51 const children: CreateJobArgument[] = []
52
53 for (const targetUrl of targetUrls) {
54 if (await skipImport(channel, targetUrl, onlyAfter)) continue
67 55
68 // Will update the channel sync status 56 const { job } = await buildYoutubeDLImport({
69 const parent: CreateJobArgument = { 57 user,
70 type: 'after-video-channel-import', 58 channel,
71 payload: { 59 targetUrl,
72 channelSyncId: channelSync?.id 60 channelSync,
61 importDataOverride: {
62 privacy: VideoPrivacy.PUBLIC
63 }
64 })
65
66 children.push(job)
73 } 67 }
74 }
75 68
76 await JobQueue.Instance.createJobWithChildren(parent, children) 69 // Will update the channel sync status
70 const parent: CreateJobArgument = {
71 type: 'after-video-channel-import',
72 payload: {
73 channelSyncId: channelSync?.id
74 }
75 }
76
77 await JobQueue.Instance.createJobWithChildren(parent, children)
78 } catch (err) {
79 logger.error(`Failed to import channel ${channel.name}`, { err })
80 channelSync.state = VideoChannelSyncState.FAILED
81 await channelSync.save()
82 }
77} 83}
78 84
79// --------------------------------------------------------------------------- 85// ---------------------------------------------------------------------------