diff options
author | Chocobozzz <me@florianbigard.com> | 2018-09-11 16:27:07 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2018-09-13 14:05:49 +0200 |
commit | c48e82b5e0478434de30626d14594a97f2402e7c (patch) | |
tree | a78e5272bd0fe4f5b41831e571e02d05f1515b82 /server | |
parent | a651038487faa838bda3ce04695b08bc65baff70 (diff) | |
download | PeerTube-c48e82b5e0478434de30626d14594a97f2402e7c.tar.gz PeerTube-c48e82b5e0478434de30626d14594a97f2402e7c.tar.zst PeerTube-c48e82b5e0478434de30626d14594a97f2402e7c.zip |
Basic video redundancy implementation
Diffstat (limited to 'server')
50 files changed, 1471 insertions, 259 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index 54cf44419..2e168ea78 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts | |||
@@ -3,9 +3,9 @@ import * as express from 'express' | |||
3 | import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos' | 3 | import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos' |
4 | import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub' | 4 | import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub' |
5 | import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers' | 5 | import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers' |
6 | import { buildVideoAnnounce } from '../../lib/activitypub/send' | 6 | import { buildAnnounceWithVideoAudience } from '../../lib/activitypub/send' |
7 | import { audiencify, getAudience } from '../../lib/activitypub/audience' | 7 | import { audiencify, getAudience } from '../../lib/activitypub/audience' |
8 | import { createActivityData } from '../../lib/activitypub/send/send-create' | 8 | import { buildCreateActivity } from '../../lib/activitypub/send/send-create' |
9 | import { asyncMiddleware, executeIfActivityPub, localAccountValidator, localVideoChannelValidator } from '../../middlewares' | 9 | import { asyncMiddleware, executeIfActivityPub, localAccountValidator, localVideoChannelValidator } from '../../middlewares' |
10 | import { videosGetValidator, videosShareValidator } from '../../middlewares/validators' | 10 | import { videosGetValidator, videosShareValidator } from '../../middlewares/validators' |
11 | import { videoCommentGetValidator } from '../../middlewares/validators/video-comments' | 11 | import { videoCommentGetValidator } from '../../middlewares/validators/video-comments' |
@@ -26,6 +26,8 @@ import { | |||
26 | getVideoSharesActivityPubUrl | 26 | getVideoSharesActivityPubUrl |
27 | } from '../../lib/activitypub' | 27 | } from '../../lib/activitypub' |
28 | import { VideoCaptionModel } from '../../models/video/video-caption' | 28 | import { VideoCaptionModel } from '../../models/video/video-caption' |
29 | import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy' | ||
30 | import { getServerActor } from '../../helpers/utils' | ||
29 | 31 | ||
30 | const activityPubClientRouter = express.Router() | 32 | const activityPubClientRouter = express.Router() |
31 | 33 | ||
@@ -93,6 +95,11 @@ activityPubClientRouter.get('/video-channels/:name/following', | |||
93 | executeIfActivityPub(asyncMiddleware(videoChannelFollowingController)) | 95 | executeIfActivityPub(asyncMiddleware(videoChannelFollowingController)) |
94 | ) | 96 | ) |
95 | 97 | ||
98 | activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?', | ||
99 | executeIfActivityPub(asyncMiddleware(videoRedundancyGetValidator)), | ||
100 | executeIfActivityPub(asyncMiddleware(videoRedundancyController)) | ||
101 | ) | ||
102 | |||
96 | // --------------------------------------------------------------------------- | 103 | // --------------------------------------------------------------------------- |
97 | 104 | ||
98 | export { | 105 | export { |
@@ -131,7 +138,7 @@ async function videoController (req: express.Request, res: express.Response, nex | |||
131 | const videoObject = audiencify(video.toActivityPubObject(), audience) | 138 | const videoObject = audiencify(video.toActivityPubObject(), audience) |
132 | 139 | ||
133 | if (req.path.endsWith('/activity')) { | 140 | if (req.path.endsWith('/activity')) { |
134 | const data = createActivityData(video.url, video.VideoChannel.Account.Actor, videoObject, audience) | 141 | const data = buildCreateActivity(video.url, video.VideoChannel.Account.Actor, videoObject, audience) |
135 | return activityPubResponse(activityPubContextify(data), res) | 142 | return activityPubResponse(activityPubContextify(data), res) |
136 | } | 143 | } |
137 | 144 | ||
@@ -140,9 +147,9 @@ async function videoController (req: express.Request, res: express.Response, nex | |||
140 | 147 | ||
141 | async function videoAnnounceController (req: express.Request, res: express.Response, next: express.NextFunction) { | 148 | async function videoAnnounceController (req: express.Request, res: express.Response, next: express.NextFunction) { |
142 | const share = res.locals.videoShare as VideoShareModel | 149 | const share = res.locals.videoShare as VideoShareModel |
143 | const object = await buildVideoAnnounce(share.Actor, share, res.locals.video, undefined) | 150 | const { activity } = await buildAnnounceWithVideoAudience(share.Actor, share, res.locals.video, undefined) |
144 | 151 | ||
145 | return activityPubResponse(activityPubContextify(object), res) | 152 | return activityPubResponse(activityPubContextify(activity), res) |
146 | } | 153 | } |
147 | 154 | ||
148 | async function videoAnnouncesController (req: express.Request, res: express.Response, next: express.NextFunction) { | 155 | async function videoAnnouncesController (req: express.Request, res: express.Response, next: express.NextFunction) { |
@@ -219,13 +226,28 @@ async function videoCommentController (req: express.Request, res: express.Respon | |||
219 | const videoCommentObject = audiencify(videoComment.toActivityPubObject(threadParentComments), audience) | 226 | const videoCommentObject = audiencify(videoComment.toActivityPubObject(threadParentComments), audience) |
220 | 227 | ||
221 | if (req.path.endsWith('/activity')) { | 228 | if (req.path.endsWith('/activity')) { |
222 | const data = createActivityData(videoComment.url, videoComment.Account.Actor, videoCommentObject, audience) | 229 | const data = buildCreateActivity(videoComment.url, videoComment.Account.Actor, videoCommentObject, audience) |
223 | return activityPubResponse(activityPubContextify(data), res) | 230 | return activityPubResponse(activityPubContextify(data), res) |
224 | } | 231 | } |
225 | 232 | ||
226 | return activityPubResponse(activityPubContextify(videoCommentObject), res) | 233 | return activityPubResponse(activityPubContextify(videoCommentObject), res) |
227 | } | 234 | } |
228 | 235 | ||
236 | async function videoRedundancyController (req: express.Request, res: express.Response) { | ||
237 | const videoRedundancy = res.locals.videoRedundancy | ||
238 | const serverActor = await getServerActor() | ||
239 | |||
240 | const audience = getAudience(serverActor) | ||
241 | const object = audiencify(videoRedundancy.toActivityPubObject(), audience) | ||
242 | |||
243 | if (req.path.endsWith('/activity')) { | ||
244 | const data = buildCreateActivity(videoRedundancy.url, serverActor, object, audience) | ||
245 | return activityPubResponse(activityPubContextify(data), res) | ||
246 | } | ||
247 | |||
248 | return activityPubResponse(activityPubContextify(object), res) | ||
249 | } | ||
250 | |||
229 | // --------------------------------------------------------------------------- | 251 | // --------------------------------------------------------------------------- |
230 | 252 | ||
231 | async function actorFollowing (req: express.Request, actor: ActorModel) { | 253 | async function actorFollowing (req: express.Request, actor: ActorModel) { |
diff --git a/server/controllers/activitypub/outbox.ts b/server/controllers/activitypub/outbox.ts index db69ae54b..bd0e4fe9d 100644 --- a/server/controllers/activitypub/outbox.ts +++ b/server/controllers/activitypub/outbox.ts | |||
@@ -3,7 +3,7 @@ import { Activity } from '../../../shared/models/activitypub/activity' | |||
3 | import { VideoPrivacy } from '../../../shared/models/videos' | 3 | import { VideoPrivacy } from '../../../shared/models/videos' |
4 | import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub' | 4 | import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub' |
5 | import { logger } from '../../helpers/logger' | 5 | import { logger } from '../../helpers/logger' |
6 | import { announceActivityData, createActivityData } from '../../lib/activitypub/send' | 6 | import { buildAnnounceActivity, buildCreateActivity } from '../../lib/activitypub/send' |
7 | import { buildAudience } from '../../lib/activitypub/audience' | 7 | import { buildAudience } from '../../lib/activitypub/audience' |
8 | import { asyncMiddleware, localAccountValidator, localVideoChannelValidator } from '../../middlewares' | 8 | import { asyncMiddleware, localAccountValidator, localVideoChannelValidator } from '../../middlewares' |
9 | import { AccountModel } from '../../models/account/account' | 9 | import { AccountModel } from '../../models/account/account' |
@@ -60,12 +60,12 @@ async function buildActivities (actor: ActorModel, start: number, count: number) | |||
60 | // This is a shared video | 60 | // This is a shared video |
61 | if (video.VideoShares !== undefined && video.VideoShares.length !== 0) { | 61 | if (video.VideoShares !== undefined && video.VideoShares.length !== 0) { |
62 | const videoShare = video.VideoShares[0] | 62 | const videoShare = video.VideoShares[0] |
63 | const announceActivity = announceActivityData(videoShare.url, actor, video.url, createActivityAudience) | 63 | const announceActivity = buildAnnounceActivity(videoShare.url, actor, video.url, createActivityAudience) |
64 | 64 | ||
65 | activities.push(announceActivity) | 65 | activities.push(announceActivity) |
66 | } else { | 66 | } else { |
67 | const videoObject = video.toActivityPubObject() | 67 | const videoObject = video.toActivityPubObject() |
68 | const createActivity = createActivityData(video.url, byActor, videoObject, createActivityAudience) | 68 | const createActivity = buildCreateActivity(video.url, byActor, videoObject, createActivityAudience) |
69 | 69 | ||
70 | activities.push(createActivity) | 70 | activities.push(createActivity) |
71 | } | 71 | } |
diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index bb7174891..28a7a04ca 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts | |||
@@ -17,8 +17,6 @@ import { | |||
17 | import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search' | 17 | import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search' |
18 | import { getOrCreateActorAndServerAndModel, getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub' | 18 | import { getOrCreateActorAndServerAndModel, getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub' |
19 | import { logger } from '../../helpers/logger' | 19 | import { logger } from '../../helpers/logger' |
20 | import { User } from '../../../shared/models/users' | ||
21 | import { CONFIG } from '../../initializers/constants' | ||
22 | import { VideoChannelModel } from '../../models/video/video-channel' | 20 | import { VideoChannelModel } from '../../models/video/video-channel' |
23 | import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger' | 21 | import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger' |
24 | 22 | ||
diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts index 23308445f..a4eae6b45 100644 --- a/server/controllers/api/server/follows.ts +++ b/server/controllers/api/server/follows.ts | |||
@@ -96,6 +96,11 @@ async function removeFollow (req: express.Request, res: express.Response, next: | |||
96 | await sequelizeTypescript.transaction(async t => { | 96 | await sequelizeTypescript.transaction(async t => { |
97 | if (follow.state === 'accepted') await sendUndoFollow(follow, t) | 97 | if (follow.state === 'accepted') await sendUndoFollow(follow, t) |
98 | 98 | ||
99 | // Disable redundancy on unfollowed instances | ||
100 | const server = follow.ActorFollowing.Server | ||
101 | server.redundancyAllowed = false | ||
102 | await server.save({ transaction: t }) | ||
103 | |||
99 | await follow.destroy({ transaction: t }) | 104 | await follow.destroy({ transaction: t }) |
100 | }) | 105 | }) |
101 | 106 | ||
diff --git a/server/controllers/api/server/index.ts b/server/controllers/api/server/index.ts index 850a52cdb..43bca2c10 100644 --- a/server/controllers/api/server/index.ts +++ b/server/controllers/api/server/index.ts | |||
@@ -1,10 +1,12 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { serverFollowsRouter } from './follows' | 2 | import { serverFollowsRouter } from './follows' |
3 | import { statsRouter } from './stats' | 3 | import { statsRouter } from './stats' |
4 | import { serverRedundancyRouter } from './redundancy' | ||
4 | 5 | ||
5 | const serverRouter = express.Router() | 6 | const serverRouter = express.Router() |
6 | 7 | ||
7 | serverRouter.use('/', serverFollowsRouter) | 8 | serverRouter.use('/', serverFollowsRouter) |
9 | serverRouter.use('/', serverRedundancyRouter) | ||
8 | serverRouter.use('/', statsRouter) | 10 | serverRouter.use('/', statsRouter) |
9 | 11 | ||
10 | // --------------------------------------------------------------------------- | 12 | // --------------------------------------------------------------------------- |
diff --git a/server/controllers/api/server/redundancy.ts b/server/controllers/api/server/redundancy.ts new file mode 100644 index 000000000..4216b9e35 --- /dev/null +++ b/server/controllers/api/server/redundancy.ts | |||
@@ -0,0 +1,32 @@ | |||
1 | import * as express from 'express' | ||
2 | import { UserRight } from '../../../../shared/models/users' | ||
3 | import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' | ||
4 | import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy' | ||
5 | import { ServerModel } from '../../../models/server/server' | ||
6 | |||
7 | const serverRedundancyRouter = express.Router() | ||
8 | |||
9 | serverRedundancyRouter.put('/redundancy/:host', | ||
10 | authenticate, | ||
11 | ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), | ||
12 | asyncMiddleware(updateServerRedundancyValidator), | ||
13 | asyncMiddleware(updateRedundancy) | ||
14 | ) | ||
15 | |||
16 | // --------------------------------------------------------------------------- | ||
17 | |||
18 | export { | ||
19 | serverRedundancyRouter | ||
20 | } | ||
21 | |||
22 | // --------------------------------------------------------------------------- | ||
23 | |||
24 | async function updateRedundancy (req: express.Request, res: express.Response, next: express.NextFunction) { | ||
25 | const server = res.locals.server as ServerModel | ||
26 | |||
27 | server.redundancyAllowed = req.body.redundancyAllowed | ||
28 | |||
29 | await server.save() | ||
30 | |||
31 | return res.sendStatus(204) | ||
32 | } | ||
diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts index 59bdf6257..08e11b00b 100644 --- a/server/controllers/api/videos/abuse.ts +++ b/server/controllers/api/videos/abuse.ts | |||
@@ -112,7 +112,7 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) { | |||
112 | 112 | ||
113 | // We send the video abuse to the origin server | 113 | // We send the video abuse to the origin server |
114 | if (videoInstance.isOwned() === false) { | 114 | if (videoInstance.isOwned() === false) { |
115 | await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t) | 115 | await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance) |
116 | } | 116 | } |
117 | 117 | ||
118 | auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON())) | 118 | auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON())) |
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index a9de11fb0..1304c7559 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts | |||
@@ -14,20 +14,24 @@ function activityPubContextify <T> (data: T) { | |||
14 | 'https://w3id.org/security/v1', | 14 | 'https://w3id.org/security/v1', |
15 | { | 15 | { |
16 | RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', | 16 | RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', |
17 | pt: 'https://joinpeertube.org/ns', | ||
18 | schema: 'http://schema.org#', | ||
17 | Hashtag: 'as:Hashtag', | 19 | Hashtag: 'as:Hashtag', |
18 | uuid: 'http://schema.org/identifier', | 20 | uuid: 'schema:identifier', |
19 | category: 'http://schema.org/category', | 21 | category: 'schema:category', |
20 | licence: 'http://schema.org/license', | 22 | licence: 'schema:license', |
21 | subtitleLanguage: 'http://schema.org/subtitleLanguage', | 23 | subtitleLanguage: 'schema:subtitleLanguage', |
22 | sensitive: 'as:sensitive', | 24 | sensitive: 'as:sensitive', |
23 | language: 'http://schema.org/inLanguage', | 25 | language: 'schema:inLanguage', |
24 | views: 'http://schema.org/Number', | 26 | views: 'schema:Number', |
25 | stats: 'http://schema.org/Number', | 27 | stats: 'schema:Number', |
26 | size: 'http://schema.org/Number', | 28 | size: 'schema:Number', |
27 | fps: 'http://schema.org/Number', | 29 | fps: 'schema:Number', |
28 | commentsEnabled: 'http://schema.org/Boolean', | 30 | commentsEnabled: 'schema:Boolean', |
29 | waitTranscoding: 'http://schema.org/Boolean', | 31 | waitTranscoding: 'schema:Boolean', |
30 | support: 'http://schema.org/Text' | 32 | expires: 'schema:expires', |
33 | support: 'schema:Text', | ||
34 | CacheFile: 'pt:CacheFile' | ||
31 | }, | 35 | }, |
32 | { | 36 | { |
33 | likes: { | 37 | likes: { |
diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts index 381a29e66..2562ead9b 100644 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ b/server/helpers/custom-validators/activitypub/activity.ts | |||
@@ -1,7 +1,10 @@ | |||
1 | import * as validator from 'validator' | 1 | import * as validator from 'validator' |
2 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' | 2 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' |
3 | import { | 3 | import { |
4 | isActorAcceptActivityValid, isActorDeleteActivityValid, isActorFollowActivityValid, isActorRejectActivityValid, | 4 | isActorAcceptActivityValid, |
5 | isActorDeleteActivityValid, | ||
6 | isActorFollowActivityValid, | ||
7 | isActorRejectActivityValid, | ||
5 | isActorUpdateActivityValid | 8 | isActorUpdateActivityValid |
6 | } from './actor' | 9 | } from './actor' |
7 | import { isAnnounceActivityValid } from './announce' | 10 | import { isAnnounceActivityValid } from './announce' |
@@ -11,12 +14,13 @@ import { isUndoActivityValid } from './undo' | |||
11 | import { isVideoCommentCreateActivityValid, isVideoCommentDeleteActivityValid } from './video-comments' | 14 | import { isVideoCommentCreateActivityValid, isVideoCommentDeleteActivityValid } from './video-comments' |
12 | import { | 15 | import { |
13 | isVideoFlagValid, | 16 | isVideoFlagValid, |
14 | sanitizeAndCheckVideoTorrentCreateActivity, | ||
15 | isVideoTorrentDeleteActivityValid, | 17 | isVideoTorrentDeleteActivityValid, |
18 | sanitizeAndCheckVideoTorrentCreateActivity, | ||
16 | sanitizeAndCheckVideoTorrentUpdateActivity | 19 | sanitizeAndCheckVideoTorrentUpdateActivity |
17 | } from './videos' | 20 | } from './videos' |
18 | import { isViewActivityValid } from './view' | 21 | import { isViewActivityValid } from './view' |
19 | import { exists } from '../misc' | 22 | import { exists } from '../misc' |
23 | import { isCacheFileCreateActivityValid, isCacheFileUpdateActivityValid } from './cache-file' | ||
20 | 24 | ||
21 | function isRootActivityValid (activity: any) { | 25 | function isRootActivityValid (activity: any) { |
22 | return Array.isArray(activity['@context']) && ( | 26 | return Array.isArray(activity['@context']) && ( |
@@ -67,11 +71,13 @@ function checkCreateActivity (activity: any) { | |||
67 | isDislikeActivityValid(activity) || | 71 | isDislikeActivityValid(activity) || |
68 | sanitizeAndCheckVideoTorrentCreateActivity(activity) || | 72 | sanitizeAndCheckVideoTorrentCreateActivity(activity) || |
69 | isVideoFlagValid(activity) || | 73 | isVideoFlagValid(activity) || |
70 | isVideoCommentCreateActivityValid(activity) | 74 | isVideoCommentCreateActivityValid(activity) || |
75 | isCacheFileCreateActivityValid(activity) | ||
71 | } | 76 | } |
72 | 77 | ||
73 | function checkUpdateActivity (activity: any) { | 78 | function checkUpdateActivity (activity: any) { |
74 | return sanitizeAndCheckVideoTorrentUpdateActivity(activity) || | 79 | return isCacheFileUpdateActivityValid(activity) || |
80 | sanitizeAndCheckVideoTorrentUpdateActivity(activity) || | ||
75 | isActorUpdateActivityValid(activity) | 81 | isActorUpdateActivityValid(activity) |
76 | } | 82 | } |
77 | 83 | ||
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts new file mode 100644 index 000000000..bd70934c8 --- /dev/null +++ b/server/helpers/custom-validators/activitypub/cache-file.ts | |||
@@ -0,0 +1,28 @@ | |||
1 | import { isActivityPubUrlValid, isBaseActivityValid } from './misc' | ||
2 | import { isRemoteVideoUrlValid } from './videos' | ||
3 | import { isDateValid, exists } from '../misc' | ||
4 | import { CacheFileObject } from '../../../../shared/models/activitypub/objects' | ||
5 | |||
6 | function isCacheFileCreateActivityValid (activity: any) { | ||
7 | return isBaseActivityValid(activity, 'Create') && | ||
8 | isCacheFileObjectValid(activity.object) | ||
9 | } | ||
10 | |||
11 | function isCacheFileUpdateActivityValid (activity: any) { | ||
12 | return isBaseActivityValid(activity, 'Update') && | ||
13 | isCacheFileObjectValid(activity.object) | ||
14 | } | ||
15 | |||
16 | function isCacheFileObjectValid (object: CacheFileObject) { | ||
17 | return exists(object) && | ||
18 | object.type === 'CacheFile' && | ||
19 | isDateValid(object.expires) && | ||
20 | isActivityPubUrlValid(object.object) && | ||
21 | isRemoteVideoUrlValid(object.url) | ||
22 | } | ||
23 | |||
24 | export { | ||
25 | isCacheFileUpdateActivityValid, | ||
26 | isCacheFileCreateActivityValid, | ||
27 | isCacheFileObjectValid | ||
28 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts index 6c5c7abca..4e2c57f04 100644 --- a/server/helpers/custom-validators/activitypub/misc.ts +++ b/server/helpers/custom-validators/activitypub/misc.ts | |||
@@ -3,7 +3,7 @@ import { CONSTRAINTS_FIELDS } from '../../../initializers' | |||
3 | import { isTestInstance } from '../../core-utils' | 3 | import { isTestInstance } from '../../core-utils' |
4 | import { exists } from '../misc' | 4 | import { exists } from '../misc' |
5 | 5 | ||
6 | function isActivityPubUrlValid (url: string) { | 6 | function isUrlValid (url: string) { |
7 | const isURLOptions = { | 7 | const isURLOptions = { |
8 | require_host: true, | 8 | require_host: true, |
9 | require_tld: true, | 9 | require_tld: true, |
@@ -17,13 +17,18 @@ function isActivityPubUrlValid (url: string) { | |||
17 | isURLOptions.require_tld = false | 17 | isURLOptions.require_tld = false |
18 | } | 18 | } |
19 | 19 | ||
20 | return exists(url) && validator.isURL('' + url, isURLOptions) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL) | 20 | return exists(url) && validator.isURL('' + url, isURLOptions) |
21 | } | ||
22 | |||
23 | function isActivityPubUrlValid (url: string) { | ||
24 | return isUrlValid(url) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL) | ||
21 | } | 25 | } |
22 | 26 | ||
23 | function isBaseActivityValid (activity: any, type: string) { | 27 | function isBaseActivityValid (activity: any, type: string) { |
24 | return (activity['@context'] === undefined || Array.isArray(activity['@context'])) && | 28 | return (activity['@context'] === undefined || Array.isArray(activity['@context'])) && |
25 | activity.type === type && | 29 | activity.type === type && |
26 | isActivityPubUrlValid(activity.id) && | 30 | isActivityPubUrlValid(activity.id) && |
31 | exists(activity.actor) && | ||
27 | (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) && | 32 | (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) && |
28 | ( | 33 | ( |
29 | activity.to === undefined || | 34 | activity.to === undefined || |
@@ -49,6 +54,7 @@ function setValidAttributedTo (obj: any) { | |||
49 | } | 54 | } |
50 | 55 | ||
51 | export { | 56 | export { |
57 | isUrlValid, | ||
52 | isActivityPubUrlValid, | 58 | isActivityPubUrlValid, |
53 | isBaseActivityValid, | 59 | isBaseActivityValid, |
54 | setValidAttributedTo | 60 | setValidAttributedTo |
diff --git a/server/helpers/custom-validators/activitypub/undo.ts b/server/helpers/custom-validators/activitypub/undo.ts index f50f224fa..578035893 100644 --- a/server/helpers/custom-validators/activitypub/undo.ts +++ b/server/helpers/custom-validators/activitypub/undo.ts | |||
@@ -2,6 +2,7 @@ import { isActorFollowActivityValid } from './actor' | |||
2 | import { isBaseActivityValid } from './misc' | 2 | import { isBaseActivityValid } from './misc' |
3 | import { isDislikeActivityValid, isLikeActivityValid } from './rate' | 3 | import { isDislikeActivityValid, isLikeActivityValid } from './rate' |
4 | import { isAnnounceActivityValid } from './announce' | 4 | import { isAnnounceActivityValid } from './announce' |
5 | import { isCacheFileCreateActivityValid } from './cache-file' | ||
5 | 6 | ||
6 | function isUndoActivityValid (activity: any) { | 7 | function isUndoActivityValid (activity: any) { |
7 | return isBaseActivityValid(activity, 'Undo') && | 8 | return isBaseActivityValid(activity, 'Undo') && |
@@ -9,7 +10,8 @@ function isUndoActivityValid (activity: any) { | |||
9 | isActorFollowActivityValid(activity.object) || | 10 | isActorFollowActivityValid(activity.object) || |
10 | isLikeActivityValid(activity.object) || | 11 | isLikeActivityValid(activity.object) || |
11 | isDislikeActivityValid(activity.object) || | 12 | isDislikeActivityValid(activity.object) || |
12 | isAnnounceActivityValid(activity.object) | 13 | isAnnounceActivityValid(activity.object) || |
14 | isCacheFileCreateActivityValid(activity.object) | ||
13 | ) | 15 | ) |
14 | } | 16 | } |
15 | 17 | ||
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 0362f43ab..f76eba474 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts | |||
@@ -75,6 +75,30 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { | |||
75 | video.attributedTo.length !== 0 | 75 | video.attributedTo.length !== 0 |
76 | } | 76 | } |
77 | 77 | ||
78 | function isRemoteVideoUrlValid (url: any) { | ||
79 | // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few release (currently beta.11) | ||
80 | if (url.width && !url.height) url.height = url.width | ||
81 | |||
82 | return url.type === 'Link' && | ||
83 | ( | ||
84 | ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 && | ||
85 | isActivityPubUrlValid(url.href) && | ||
86 | validator.isInt(url.height + '', { min: 0 }) && | ||
87 | validator.isInt(url.size + '', { min: 0 }) && | ||
88 | (!url.fps || validator.isInt(url.fps + '', { min: 0 })) | ||
89 | ) || | ||
90 | ( | ||
91 | ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 && | ||
92 | isActivityPubUrlValid(url.href) && | ||
93 | validator.isInt(url.height + '', { min: 0 }) | ||
94 | ) || | ||
95 | ( | ||
96 | ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 && | ||
97 | validator.isLength(url.href, { min: 5 }) && | ||
98 | validator.isInt(url.height + '', { min: 0 }) | ||
99 | ) | ||
100 | } | ||
101 | |||
78 | // --------------------------------------------------------------------------- | 102 | // --------------------------------------------------------------------------- |
79 | 103 | ||
80 | export { | 104 | export { |
@@ -83,7 +107,8 @@ export { | |||
83 | isVideoTorrentDeleteActivityValid, | 107 | isVideoTorrentDeleteActivityValid, |
84 | isRemoteStringIdentifierValid, | 108 | isRemoteStringIdentifierValid, |
85 | isVideoFlagValid, | 109 | isVideoFlagValid, |
86 | sanitizeAndCheckVideoTorrentObject | 110 | sanitizeAndCheckVideoTorrentObject, |
111 | isRemoteVideoUrlValid | ||
87 | } | 112 | } |
88 | 113 | ||
89 | // --------------------------------------------------------------------------- | 114 | // --------------------------------------------------------------------------- |
@@ -147,26 +172,4 @@ function setRemoteVideoTruncatedContent (video: any) { | |||
147 | return true | 172 | return true |
148 | } | 173 | } |
149 | 174 | ||
150 | function isRemoteVideoUrlValid (url: any) { | ||
151 | // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few realease (currently beta.11) | ||
152 | if (url.width && !url.height) url.height = url.width | ||
153 | 175 | ||
154 | return url.type === 'Link' && | ||
155 | ( | ||
156 | ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 && | ||
157 | isActivityPubUrlValid(url.href) && | ||
158 | validator.isInt(url.height + '', { min: 0 }) && | ||
159 | validator.isInt(url.size + '', { min: 0 }) && | ||
160 | (!url.fps || validator.isInt(url.fps + '', { min: 0 })) | ||
161 | ) || | ||
162 | ( | ||
163 | ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 && | ||
164 | isActivityPubUrlValid(url.href) && | ||
165 | validator.isInt(url.height + '', { min: 0 }) | ||
166 | ) || | ||
167 | ( | ||
168 | ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 && | ||
169 | validator.isLength(url.href, { min: 5 }) && | ||
170 | validator.isInt(url.height + '', { min: 0 }) | ||
171 | ) | ||
172 | } | ||
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 1c0cc7058..2fdfd1876 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts | |||
@@ -5,44 +5,49 @@ import { createWriteStream, remove } from 'fs-extra' | |||
5 | import { CONFIG } from '../initializers' | 5 | import { CONFIG } from '../initializers' |
6 | import { join } from 'path' | 6 | import { join } from 'path' |
7 | 7 | ||
8 | function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: string }) { | 8 | function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout?: number) { |
9 | const id = target.magnetUri || target.torrentName | 9 | const id = target.magnetUri || target.torrentName |
10 | let timer | ||
10 | 11 | ||
11 | const path = generateVideoTmpPath(id) | 12 | const path = generateVideoTmpPath(id) |
12 | logger.info('Importing torrent video %s', id) | 13 | logger.info('Importing torrent video %s', id) |
13 | 14 | ||
14 | return new Promise<string>((res, rej) => { | 15 | return new Promise<string>((res, rej) => { |
15 | const webtorrent = new WebTorrent() | 16 | const webtorrent = new WebTorrent() |
17 | let file: WebTorrent.TorrentFile | ||
16 | 18 | ||
17 | const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) | 19 | const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) |
18 | 20 | ||
19 | const options = { path: CONFIG.STORAGE.VIDEOS_DIR } | 21 | const options = { path: CONFIG.STORAGE.VIDEOS_DIR } |
20 | const torrent = webtorrent.add(torrentId, options, torrent => { | 22 | const torrent = webtorrent.add(torrentId, options, torrent => { |
21 | if (torrent.files.length !== 1) return rej(new Error('The number of files is not equal to 1 for ' + torrentId)) | 23 | if (torrent.files.length !== 1) { |
24 | if (timer) clearTimeout(timer) | ||
22 | 25 | ||
23 | const file = torrent.files[ 0 ] | 26 | return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName) |
27 | .then(() => rej(new Error('The number of files is not equal to 1 for ' + torrentId))) | ||
28 | } | ||
29 | |||
30 | file = torrent.files[ 0 ] | ||
24 | 31 | ||
25 | const writeStream = createWriteStream(path) | 32 | const writeStream = createWriteStream(path) |
26 | writeStream.on('finish', () => { | 33 | writeStream.on('finish', () => { |
27 | webtorrent.destroy(async err => { | 34 | if (timer) clearTimeout(timer) |
28 | if (err) return rej(err) | ||
29 | |||
30 | if (target.torrentName) { | ||
31 | remove(torrentId) | ||
32 | .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err })) | ||
33 | } | ||
34 | 35 | ||
35 | remove(join(CONFIG.STORAGE.VIDEOS_DIR, file.name)) | 36 | return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName) |
36 | .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', file.name, { err })) | 37 | .then(() => res(path)) |
37 | |||
38 | res(path) | ||
39 | }) | ||
40 | }) | 38 | }) |
41 | 39 | ||
42 | file.createReadStream().pipe(writeStream) | 40 | file.createReadStream().pipe(writeStream) |
43 | }) | 41 | }) |
44 | 42 | ||
45 | torrent.on('error', err => rej(err)) | 43 | torrent.on('error', err => rej(err)) |
44 | |||
45 | if (timeout) { | ||
46 | timer = setTimeout(async () => { | ||
47 | return safeWebtorrentDestroy(webtorrent, torrentId, file ? file.name : undefined, target.torrentName) | ||
48 | .then(() => rej(new Error('Webtorrent download timeout.'))) | ||
49 | }, timeout) | ||
50 | } | ||
46 | }) | 51 | }) |
47 | } | 52 | } |
48 | 53 | ||
@@ -51,3 +56,29 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: stri | |||
51 | export { | 56 | export { |
52 | downloadWebTorrentVideo | 57 | downloadWebTorrentVideo |
53 | } | 58 | } |
59 | |||
60 | // --------------------------------------------------------------------------- | ||
61 | |||
62 | function safeWebtorrentDestroy (webtorrent: WebTorrent.Instance, torrentId: string, filename?: string, torrentName?: string) { | ||
63 | return new Promise(res => { | ||
64 | webtorrent.destroy(err => { | ||
65 | // Delete torrent file | ||
66 | if (torrentName) { | ||
67 | remove(torrentId) | ||
68 | .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err })) | ||
69 | } | ||
70 | |||
71 | // Delete downloaded file | ||
72 | if (filename) { | ||
73 | remove(join(CONFIG.STORAGE.VIDEOS_DIR, filename)) | ||
74 | .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', filename, { err })) | ||
75 | } | ||
76 | |||
77 | if (err) { | ||
78 | logger.warn('Cannot destroy webtorrent in timeout.', { err }) | ||
79 | } | ||
80 | |||
81 | return res() | ||
82 | }) | ||
83 | }) | ||
84 | } | ||
diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 9dd104035..6a2badd35 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts | |||
@@ -7,6 +7,9 @@ import { parse } from 'url' | |||
7 | import { CONFIG } from './constants' | 7 | import { CONFIG } from './constants' |
8 | import { logger } from '../helpers/logger' | 8 | import { logger } from '../helpers/logger' |
9 | import { getServerActor } from '../helpers/utils' | 9 | import { getServerActor } from '../helpers/utils' |
10 | import { VideosRedundancy } from '../../shared/models/redundancy' | ||
11 | import { isArray } from '../helpers/custom-validators/misc' | ||
12 | import { uniq } from 'lodash' | ||
10 | 13 | ||
11 | async function checkActivityPubUrls () { | 14 | async function checkActivityPubUrls () { |
12 | const actor = await getServerActor() | 15 | const actor = await getServerActor() |
@@ -35,6 +38,20 @@ function checkConfig () { | |||
35 | return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy | 38 | return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy |
36 | } | 39 | } |
37 | 40 | ||
41 | const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos') | ||
42 | if (isArray(redundancyVideos)) { | ||
43 | for (const r of redundancyVideos) { | ||
44 | if ([ 'most-views' ].indexOf(r.strategy) === -1) { | ||
45 | return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy | ||
46 | } | ||
47 | } | ||
48 | |||
49 | const filtered = uniq(redundancyVideos.map(r => r.strategy)) | ||
50 | if (filtered.length !== redundancyVideos.length) { | ||
51 | return 'Redundancy video entries should have uniq strategies' | ||
52 | } | ||
53 | } | ||
54 | |||
38 | return null | 55 | return null |
39 | } | 56 | } |
40 | 57 | ||
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 5b7ea5d6c..6b4afbfd8 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { IConfig } from 'config' | 1 | import { IConfig } from 'config' |
2 | import { dirname, join } from 'path' | 2 | import { dirname, join } from 'path' |
3 | import { JobType, VideoRateType, VideoState } from '../../shared/models' | 3 | import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models' |
4 | import { ActivityPubActorType } from '../../shared/models/activitypub' | 4 | import { ActivityPubActorType } from '../../shared/models/activitypub' |
5 | import { FollowState } from '../../shared/models/actors' | 5 | import { FollowState } from '../../shared/models/actors' |
6 | import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' | 6 | import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' |
@@ -9,13 +9,14 @@ import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../h | |||
9 | import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' | 9 | import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' |
10 | import { invert } from 'lodash' | 10 | import { invert } from 'lodash' |
11 | import { CronRepeatOptions, EveryRepeatOptions } from 'bull' | 11 | import { CronRepeatOptions, EveryRepeatOptions } from 'bull' |
12 | import * as bytes from 'bytes' | ||
12 | 13 | ||
13 | // Use a variable to reload the configuration if we need | 14 | // Use a variable to reload the configuration if we need |
14 | let config: IConfig = require('config') | 15 | let config: IConfig = require('config') |
15 | 16 | ||
16 | // --------------------------------------------------------------------------- | 17 | // --------------------------------------------------------------------------- |
17 | 18 | ||
18 | const LAST_MIGRATION_VERSION = 265 | 19 | const LAST_MIGRATION_VERSION = 270 |
19 | 20 | ||
20 | // --------------------------------------------------------------------------- | 21 | // --------------------------------------------------------------------------- |
21 | 22 | ||
@@ -137,7 +138,8 @@ let SCHEDULER_INTERVALS_MS = { | |||
137 | badActorFollow: 60000 * 60, // 1 hour | 138 | badActorFollow: 60000 * 60, // 1 hour |
138 | removeOldJobs: 60000 * 60, // 1 hour | 139 | removeOldJobs: 60000 * 60, // 1 hour |
139 | updateVideos: 60000, // 1 minute | 140 | updateVideos: 60000, // 1 minute |
140 | youtubeDLUpdate: 60000 * 60 * 24 // 1 day | 141 | youtubeDLUpdate: 60000 * 60 * 24, // 1 day |
142 | videosRedundancy: 60000 * 2 // 2 hours | ||
141 | } | 143 | } |
142 | 144 | ||
143 | // --------------------------------------------------------------------------- | 145 | // --------------------------------------------------------------------------- |
@@ -208,6 +210,9 @@ const CONFIG = { | |||
208 | INTERVAL_DAYS: config.get<number>('trending.videos.interval_days') | 210 | INTERVAL_DAYS: config.get<number>('trending.videos.interval_days') |
209 | } | 211 | } |
210 | }, | 212 | }, |
213 | REDUNDANCY: { | ||
214 | VIDEOS: buildVideosRedundancy(config.get<any[]>('redundancy.videos')) | ||
215 | }, | ||
211 | ADMIN: { | 216 | ADMIN: { |
212 | get EMAIL () { return config.get<string>('admin.email') } | 217 | get EMAIL () { return config.get<string>('admin.email') } |
213 | }, | 218 | }, |
@@ -321,6 +326,9 @@ const CONSTRAINTS_FIELDS = { | |||
321 | } | 326 | } |
322 | } | 327 | } |
323 | }, | 328 | }, |
329 | VIDEOS_REDUNDANCY: { | ||
330 | URL: { min: 3, max: 2000 } // Length | ||
331 | }, | ||
324 | VIDEOS: { | 332 | VIDEOS: { |
325 | NAME: { min: 3, max: 120 }, // Length | 333 | NAME: { min: 3, max: 120 }, // Length |
326 | LANGUAGE: { min: 1, max: 10 }, // Length | 334 | LANGUAGE: { min: 1, max: 10 }, // Length |
@@ -584,6 +592,13 @@ const CACHE = { | |||
584 | } | 592 | } |
585 | } | 593 | } |
586 | 594 | ||
595 | const REDUNDANCY = { | ||
596 | VIDEOS: { | ||
597 | EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days | ||
598 | RANDOMIZED_FACTOR: 5 | ||
599 | } | ||
600 | } | ||
601 | |||
587 | const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS) | 602 | const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS) |
588 | 603 | ||
589 | // --------------------------------------------------------------------------- | 604 | // --------------------------------------------------------------------------- |
@@ -629,8 +644,11 @@ if (isTestInstance() === true) { | |||
629 | SCHEDULER_INTERVALS_MS.badActorFollow = 10000 | 644 | SCHEDULER_INTERVALS_MS.badActorFollow = 10000 |
630 | SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 | 645 | SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 |
631 | SCHEDULER_INTERVALS_MS.updateVideos = 5000 | 646 | SCHEDULER_INTERVALS_MS.updateVideos = 5000 |
647 | SCHEDULER_INTERVALS_MS.videosRedundancy = 5000 | ||
632 | REPEAT_JOBS['videos-views'] = { every: 5000 } | 648 | REPEAT_JOBS['videos-views'] = { every: 5000 } |
633 | 649 | ||
650 | REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 | ||
651 | |||
634 | VIDEO_VIEW_LIFETIME = 1000 // 1 second | 652 | VIDEO_VIEW_LIFETIME = 1000 // 1 second |
635 | 653 | ||
636 | JOB_ATTEMPTS['email'] = 1 | 654 | JOB_ATTEMPTS['email'] = 1 |
@@ -653,6 +671,7 @@ export { | |||
653 | CONFIG, | 671 | CONFIG, |
654 | CONSTRAINTS_FIELDS, | 672 | CONSTRAINTS_FIELDS, |
655 | EMBED_SIZE, | 673 | EMBED_SIZE, |
674 | REDUNDANCY, | ||
656 | JOB_CONCURRENCY, | 675 | JOB_CONCURRENCY, |
657 | JOB_ATTEMPTS, | 676 | JOB_ATTEMPTS, |
658 | LAST_MIGRATION_VERSION, | 677 | LAST_MIGRATION_VERSION, |
@@ -722,6 +741,17 @@ function updateWebserverConfig () { | |||
722 | CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) | 741 | CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) |
723 | } | 742 | } |
724 | 743 | ||
744 | function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] { | ||
745 | if (!objs) return [] | ||
746 | |||
747 | return objs.map(obj => { | ||
748 | return { | ||
749 | strategy: obj.strategy, | ||
750 | size: bytes.parse(obj.size) | ||
751 | } | ||
752 | }) | ||
753 | } | ||
754 | |||
725 | function buildLanguages () { | 755 | function buildLanguages () { |
726 | const iso639 = require('iso-639-3') | 756 | const iso639 = require('iso-639-3') |
727 | 757 | ||
diff --git a/server/initializers/database.ts b/server/initializers/database.ts index b68e1a882..4d57bf8aa 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts | |||
@@ -27,6 +27,7 @@ import { VideoCaptionModel } from '../models/video/video-caption' | |||
27 | import { VideoImportModel } from '../models/video/video-import' | 27 | import { VideoImportModel } from '../models/video/video-import' |
28 | import { VideoViewModel } from '../models/video/video-views' | 28 | import { VideoViewModel } from '../models/video/video-views' |
29 | import { VideoChangeOwnershipModel } from '../models/video/video-change-ownership' | 29 | import { VideoChangeOwnershipModel } from '../models/video/video-change-ownership' |
30 | import { VideoRedundancyModel } from '../models/redundancy/video-redundancy' | ||
30 | 31 | ||
31 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string | 32 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string |
32 | 33 | ||
@@ -87,7 +88,8 @@ async function initDatabaseModels (silent: boolean) { | |||
87 | VideoCommentModel, | 88 | VideoCommentModel, |
88 | ScheduleVideoUpdateModel, | 89 | ScheduleVideoUpdateModel, |
89 | VideoImportModel, | 90 | VideoImportModel, |
90 | VideoViewModel | 91 | VideoViewModel, |
92 | VideoRedundancyModel | ||
91 | ]) | 93 | ]) |
92 | 94 | ||
93 | // Check extensions exist in the database | 95 | // Check extensions exist in the database |
diff --git a/server/initializers/migrations/0270-server-redundancy.ts b/server/initializers/migrations/0270-server-redundancy.ts new file mode 100644 index 000000000..903ba8a85 --- /dev/null +++ b/server/initializers/migrations/0270-server-redundancy.ts | |||
@@ -0,0 +1,24 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | |||
3 | async function up (utils: { | ||
4 | transaction: Sequelize.Transaction | ||
5 | queryInterface: Sequelize.QueryInterface | ||
6 | sequelize: Sequelize.Sequelize | ||
7 | }): Promise<any> { | ||
8 | { | ||
9 | const data = { | ||
10 | type: Sequelize.BOOLEAN, | ||
11 | allowNull: false, | ||
12 | defaultValue: false | ||
13 | } | ||
14 | |||
15 | await utils.queryInterface.addColumn('server', 'redundancyAllowed', data) | ||
16 | } | ||
17 | |||
18 | } | ||
19 | |||
20 | function down (options) { | ||
21 | throw new Error('Not implemented.') | ||
22 | } | ||
23 | |||
24 | export { up, down } | ||
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 1657262d7..3464add03 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts | |||
@@ -400,17 +400,15 @@ async function refreshActorIfNeeded (actor: ActorModel): Promise<{ actor: ActorM | |||
400 | await actor.save({ transaction: t }) | 400 | await actor.save({ transaction: t }) |
401 | 401 | ||
402 | if (actor.Account) { | 402 | if (actor.Account) { |
403 | await actor.save({ transaction: t }) | ||
404 | |||
405 | actor.Account.set('name', result.name) | 403 | actor.Account.set('name', result.name) |
406 | actor.Account.set('description', result.summary) | 404 | actor.Account.set('description', result.summary) |
405 | |||
407 | await actor.Account.save({ transaction: t }) | 406 | await actor.Account.save({ transaction: t }) |
408 | } else if (actor.VideoChannel) { | 407 | } else if (actor.VideoChannel) { |
409 | await actor.save({ transaction: t }) | ||
410 | |||
411 | actor.VideoChannel.set('name', result.name) | 408 | actor.VideoChannel.set('name', result.name) |
412 | actor.VideoChannel.set('description', result.summary) | 409 | actor.VideoChannel.set('description', result.summary) |
413 | actor.VideoChannel.set('support', result.support) | 410 | actor.VideoChannel.set('support', result.support) |
411 | |||
414 | await actor.VideoChannel.save({ transaction: t }) | 412 | await actor.VideoChannel.save({ transaction: t }) |
415 | } | 413 | } |
416 | 414 | ||
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts new file mode 100644 index 000000000..7325ddcb6 --- /dev/null +++ b/server/lib/activitypub/cache-file.ts | |||
@@ -0,0 +1,47 @@ | |||
1 | import { CacheFileObject } from '../../../shared/index' | ||
2 | import { VideoModel } from '../../models/video/video' | ||
3 | import { ActorModel } from '../../models/activitypub/actor' | ||
4 | import { sequelizeTypescript } from '../../initializers' | ||
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | ||
6 | |||
7 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) { | ||
8 | const url = cacheFileObject.url | ||
9 | |||
10 | const videoFile = video.VideoFiles.find(f => { | ||
11 | return f.resolution === url.height && f.fps === url.fps | ||
12 | }) | ||
13 | |||
14 | if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) | ||
15 | |||
16 | return { | ||
17 | expiresOn: new Date(cacheFileObject.expires), | ||
18 | url: cacheFileObject.id, | ||
19 | fileUrl: cacheFileObject.url.href, | ||
20 | strategy: null, | ||
21 | videoFileId: videoFile.id, | ||
22 | actorId: byActor.id | ||
23 | } | ||
24 | } | ||
25 | |||
26 | function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) { | ||
27 | return sequelizeTypescript.transaction(async t => { | ||
28 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) | ||
29 | |||
30 | return VideoRedundancyModel.create(attributes, { transaction: t }) | ||
31 | }) | ||
32 | } | ||
33 | |||
34 | function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: ActorModel) { | ||
35 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor) | ||
36 | |||
37 | redundancyModel.set('expires', attributes.expiresOn) | ||
38 | redundancyModel.set('fileUrl', attributes.fileUrl) | ||
39 | |||
40 | return redundancyModel.save() | ||
41 | } | ||
42 | |||
43 | export { | ||
44 | createCacheFile, | ||
45 | updateCacheFile, | ||
46 | cacheFileActivityObjectToDBAttributes | ||
47 | } | ||
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 16f426e23..32e555acf 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { ActivityCreate, VideoAbuseState, VideoTorrentObject } from '../../../../shared' | 1 | import { ActivityCreate, CacheFileObject, VideoAbuseState, VideoTorrentObject } from '../../../../shared' |
2 | import { DislikeObject, VideoAbuseObject, ViewObject } from '../../../../shared/models/activitypub/objects' | 2 | import { DislikeObject, VideoAbuseObject, ViewObject } from '../../../../shared/models/activitypub/objects' |
3 | import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object' | 3 | import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object' |
4 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 4 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
@@ -12,6 +12,7 @@ import { addVideoComment, resolveThread } from '../video-comments' | |||
12 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 12 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
13 | import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' | 13 | import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' |
14 | import { Redis } from '../../redis' | 14 | import { Redis } from '../../redis' |
15 | import { createCacheFile } from '../cache-file' | ||
15 | 16 | ||
16 | async function processCreateActivity (activity: ActivityCreate) { | 17 | async function processCreateActivity (activity: ActivityCreate) { |
17 | const activityObject = activity.object | 18 | const activityObject = activity.object |
@@ -28,6 +29,8 @@ async function processCreateActivity (activity: ActivityCreate) { | |||
28 | return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject) | 29 | return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject) |
29 | } else if (activityType === 'Note') { | 30 | } else if (activityType === 'Note') { |
30 | return retryTransactionWrapper(processCreateVideoComment, actor, activity) | 31 | return retryTransactionWrapper(processCreateVideoComment, actor, activity) |
32 | } else if (activityType === 'CacheFile') { | ||
33 | return retryTransactionWrapper(processCacheFile, actor, activity) | ||
31 | } | 34 | } |
32 | 35 | ||
33 | logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) | 36 | logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) |
@@ -97,6 +100,20 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate) | |||
97 | } | 100 | } |
98 | } | 101 | } |
99 | 102 | ||
103 | async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { | ||
104 | const cacheFile = activity.object as CacheFileObject | ||
105 | |||
106 | const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object) | ||
107 | |||
108 | await createCacheFile(cacheFile, video, byActor) | ||
109 | |||
110 | if (video.isOwned()) { | ||
111 | // Don't resend the activity to the sender | ||
112 | const exceptions = [ byActor ] | ||
113 | await forwardActivity(activity, undefined, exceptions) | ||
114 | } | ||
115 | } | ||
116 | |||
100 | async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { | 117 | async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { |
101 | logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) | 118 | logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) |
102 | 119 | ||
@@ -113,7 +130,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat | |||
113 | state: VideoAbuseState.PENDING | 130 | state: VideoAbuseState.PENDING |
114 | } | 131 | } |
115 | 132 | ||
116 | await VideoAbuseModel.create(videoAbuseData) | 133 | await VideoAbuseModel.create(videoAbuseData, { transaction: t }) |
117 | 134 | ||
118 | logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object) | 135 | logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object) |
119 | }) | 136 | }) |
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 1c1de8827..0eb5fa392 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo } from '../../../../shared/models/activitypub' | 1 | import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub' |
2 | import { DislikeObject } from '../../../../shared/models/activitypub/objects' | 2 | import { DislikeObject } from '../../../../shared/models/activitypub/objects' |
3 | import { getActorUrl } from '../../../helpers/activitypub' | 3 | import { getActorUrl } from '../../../helpers/activitypub' |
4 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 4 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
@@ -11,6 +11,7 @@ import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | |||
11 | import { forwardVideoRelatedActivity } from '../send/utils' | 11 | import { forwardVideoRelatedActivity } from '../send/utils' |
12 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 12 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
13 | import { VideoShareModel } from '../../../models/video/video-share' | 13 | import { VideoShareModel } from '../../../models/video/video-share' |
14 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | ||
14 | 15 | ||
15 | async function processUndoActivity (activity: ActivityUndo) { | 16 | async function processUndoActivity (activity: ActivityUndo) { |
16 | const activityToUndo = activity.object | 17 | const activityToUndo = activity.object |
@@ -19,11 +20,21 @@ async function processUndoActivity (activity: ActivityUndo) { | |||
19 | 20 | ||
20 | if (activityToUndo.type === 'Like') { | 21 | if (activityToUndo.type === 'Like') { |
21 | return retryTransactionWrapper(processUndoLike, actorUrl, activity) | 22 | return retryTransactionWrapper(processUndoLike, actorUrl, activity) |
22 | } else if (activityToUndo.type === 'Create' && activityToUndo.object.type === 'Dislike') { | 23 | } |
23 | return retryTransactionWrapper(processUndoDislike, actorUrl, activity) | 24 | |
24 | } else if (activityToUndo.type === 'Follow') { | 25 | if (activityToUndo.type === 'Create') { |
26 | if (activityToUndo.object.type === 'Dislike') { | ||
27 | return retryTransactionWrapper(processUndoDislike, actorUrl, activity) | ||
28 | } else if (activityToUndo.object.type === 'CacheFile') { | ||
29 | return retryTransactionWrapper(processUndoCacheFile, actorUrl, activity) | ||
30 | } | ||
31 | } | ||
32 | |||
33 | if (activityToUndo.type === 'Follow') { | ||
25 | return retryTransactionWrapper(processUndoFollow, actorUrl, activityToUndo) | 34 | return retryTransactionWrapper(processUndoFollow, actorUrl, activityToUndo) |
26 | } else if (activityToUndo.type === 'Announce') { | 35 | } |
36 | |||
37 | if (activityToUndo.type === 'Announce') { | ||
27 | return retryTransactionWrapper(processUndoAnnounce, actorUrl, activityToUndo) | 38 | return retryTransactionWrapper(processUndoAnnounce, actorUrl, activityToUndo) |
28 | } | 39 | } |
29 | 40 | ||
@@ -88,6 +99,29 @@ async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { | |||
88 | }) | 99 | }) |
89 | } | 100 | } |
90 | 101 | ||
102 | async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) { | ||
103 | const cacheFileObject = activity.object.object as CacheFileObject | ||
104 | |||
105 | const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object) | ||
106 | |||
107 | return sequelizeTypescript.transaction(async t => { | ||
108 | const byActor = await ActorModel.loadByUrl(actorUrl) | ||
109 | if (!byActor) throw new Error('Unknown actor ' + actorUrl) | ||
110 | |||
111 | const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) | ||
112 | if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url) | ||
113 | |||
114 | await cacheFile.destroy() | ||
115 | |||
116 | if (video.isOwned()) { | ||
117 | // Don't resend the activity to the sender | ||
118 | const exceptions = [ byActor ] | ||
119 | |||
120 | await forwardVideoRelatedActivity(activity, t, exceptions, video) | ||
121 | } | ||
122 | }) | ||
123 | } | ||
124 | |||
91 | function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) { | 125 | function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) { |
92 | return sequelizeTypescript.transaction(async t => { | 126 | return sequelizeTypescript.transaction(async t => { |
93 | const follower = await ActorModel.loadByUrl(actorUrl, t) | 127 | const follower = await ActorModel.loadByUrl(actorUrl, t) |
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index d2ad738a2..d3af1a181 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub' | 1 | import { ActivityUpdate, CacheFileObject, VideoTorrentObject } from '../../../../shared/models/activitypub' |
2 | import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' | 2 | import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' |
3 | import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' | 3 | import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' |
4 | import { logger } from '../../../helpers/logger' | 4 | import { logger } from '../../../helpers/logger' |
@@ -7,8 +7,11 @@ import { AccountModel } from '../../../models/account/account' | |||
7 | import { ActorModel } from '../../../models/activitypub/actor' | 7 | import { ActorModel } from '../../../models/activitypub/actor' |
8 | import { VideoChannelModel } from '../../../models/video/video-channel' | 8 | import { VideoChannelModel } from '../../../models/video/video-channel' |
9 | import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' | 9 | import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' |
10 | import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos' | 10 | import { getOrCreateVideoAndAccountAndChannel, updateVideoFromAP, getOrCreateVideoChannelFromVideoObject } from '../videos' |
11 | import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' | 11 | import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' |
12 | import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' | ||
13 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | ||
14 | import { createCacheFile, updateCacheFile } from '../cache-file' | ||
12 | 15 | ||
13 | async function processUpdateActivity (activity: ActivityUpdate) { | 16 | async function processUpdateActivity (activity: ActivityUpdate) { |
14 | const actor = await getOrCreateActorAndServerAndModel(activity.actor) | 17 | const actor = await getOrCreateActorAndServerAndModel(activity.actor) |
@@ -16,10 +19,16 @@ async function processUpdateActivity (activity: ActivityUpdate) { | |||
16 | 19 | ||
17 | if (objectType === 'Video') { | 20 | if (objectType === 'Video') { |
18 | return retryTransactionWrapper(processUpdateVideo, actor, activity) | 21 | return retryTransactionWrapper(processUpdateVideo, actor, activity) |
19 | } else if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { | 22 | } |
23 | |||
24 | if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { | ||
20 | return retryTransactionWrapper(processUpdateActor, actor, activity) | 25 | return retryTransactionWrapper(processUpdateActor, actor, activity) |
21 | } | 26 | } |
22 | 27 | ||
28 | if (objectType === 'CacheFile') { | ||
29 | return retryTransactionWrapper(processUpdateCacheFile, actor, activity) | ||
30 | } | ||
31 | |||
23 | return undefined | 32 | return undefined |
24 | } | 33 | } |
25 | 34 | ||
@@ -42,7 +51,24 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) | |||
42 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) | 51 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) |
43 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | 52 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) |
44 | 53 | ||
45 | return updateVideoFromAP(video, videoObject, actor, channelActor, activity.to) | 54 | return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to) |
55 | } | ||
56 | |||
57 | async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) { | ||
58 | const cacheFileObject = activity.object as CacheFileObject | ||
59 | |||
60 | if (!isCacheFileObjectValid(cacheFileObject) === false) { | ||
61 | logger.debug('Cahe file object sent by update is not valid.', { cacheFileObject }) | ||
62 | return undefined | ||
63 | } | ||
64 | |||
65 | const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) | ||
66 | if (!redundancyModel) { | ||
67 | const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id) | ||
68 | return createCacheFile(cacheFileObject, video, byActor) | ||
69 | } | ||
70 | |||
71 | return updateCacheFile(cacheFileObject, redundancyModel, byActor) | ||
46 | } | 72 | } |
47 | 73 | ||
48 | async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { | 74 | async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { |
diff --git a/server/lib/activitypub/send/send-accept.ts b/server/lib/activitypub/send/send-accept.ts index ef679707b..b6abde13d 100644 --- a/server/lib/activitypub/send/send-accept.ts +++ b/server/lib/activitypub/send/send-accept.ts | |||
@@ -3,7 +3,7 @@ import { ActorModel } from '../../../models/activitypub/actor' | |||
3 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 3 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
4 | import { getActorFollowAcceptActivityPubUrl, getActorFollowActivityPubUrl } from '../url' | 4 | import { getActorFollowAcceptActivityPubUrl, getActorFollowActivityPubUrl } from '../url' |
5 | import { unicastTo } from './utils' | 5 | import { unicastTo } from './utils' |
6 | import { followActivityData } from './send-follow' | 6 | import { buildFollowActivity } from './send-follow' |
7 | import { logger } from '../../../helpers/logger' | 7 | import { logger } from '../../../helpers/logger' |
8 | 8 | ||
9 | async function sendAccept (actorFollow: ActorFollowModel) { | 9 | async function sendAccept (actorFollow: ActorFollowModel) { |
@@ -18,10 +18,10 @@ async function sendAccept (actorFollow: ActorFollowModel) { | |||
18 | logger.info('Creating job to accept follower %s.', follower.url) | 18 | logger.info('Creating job to accept follower %s.', follower.url) |
19 | 19 | ||
20 | const followUrl = getActorFollowActivityPubUrl(actorFollow) | 20 | const followUrl = getActorFollowActivityPubUrl(actorFollow) |
21 | const followData = followActivityData(followUrl, follower, me) | 21 | const followData = buildFollowActivity(followUrl, follower, me) |
22 | 22 | ||
23 | const url = getActorFollowAcceptActivityPubUrl(actorFollow) | 23 | const url = getActorFollowAcceptActivityPubUrl(actorFollow) |
24 | const data = acceptActivityData(url, me, followData) | 24 | const data = buildAcceptActivity(url, me, followData) |
25 | 25 | ||
26 | return unicastTo(data, me, follower.inboxUrl) | 26 | return unicastTo(data, me, follower.inboxUrl) |
27 | } | 27 | } |
@@ -34,7 +34,7 @@ export { | |||
34 | 34 | ||
35 | // --------------------------------------------------------------------------- | 35 | // --------------------------------------------------------------------------- |
36 | 36 | ||
37 | function acceptActivityData (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityAccept { | 37 | function buildAcceptActivity (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityAccept { |
38 | return { | 38 | return { |
39 | type: 'Accept', | 39 | type: 'Accept', |
40 | id: url, | 40 | id: url, |
diff --git a/server/lib/activitypub/send/send-announce.ts b/server/lib/activitypub/send/send-announce.ts index 352813d73..f137217f8 100644 --- a/server/lib/activitypub/send/send-announce.ts +++ b/server/lib/activitypub/send/send-announce.ts | |||
@@ -4,45 +4,44 @@ import { ActorModel } from '../../../models/activitypub/actor' | |||
4 | import { VideoModel } from '../../../models/video/video' | 4 | import { VideoModel } from '../../../models/video/video' |
5 | import { VideoShareModel } from '../../../models/video/video-share' | 5 | import { VideoShareModel } from '../../../models/video/video-share' |
6 | import { broadcastToFollowers } from './utils' | 6 | import { broadcastToFollowers } from './utils' |
7 | import { getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' | 7 | import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' |
8 | import { logger } from '../../../helpers/logger' | 8 | import { logger } from '../../../helpers/logger' |
9 | 9 | ||
10 | async function buildVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { | 10 | async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { |
11 | const announcedObject = video.url | 11 | const announcedObject = video.url |
12 | 12 | ||
13 | const accountsToForwardView = await getActorsInvolvedInVideo(video, t) | 13 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) |
14 | const audience = getObjectFollowersAudience(accountsToForwardView) | 14 | const audience = getObjectFollowersAudience(actorsInvolvedInVideo) |
15 | return announceActivityData(videoShare.url, byActor, announcedObject, audience) | 15 | |
16 | const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience) | ||
17 | |||
18 | return { activity, actorsInvolvedInVideo } | ||
16 | } | 19 | } |
17 | 20 | ||
18 | async function sendVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { | 21 | async function sendVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { |
19 | const data = await buildVideoAnnounce(byActor, videoShare, video, t) | 22 | const { activity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t) |
20 | 23 | ||
21 | logger.info('Creating job to send announce %s.', videoShare.url) | 24 | logger.info('Creating job to send announce %s.', videoShare.url) |
22 | 25 | ||
23 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | ||
24 | const followersException = [ byActor ] | 26 | const followersException = [ byActor ] |
25 | 27 | return broadcastToFollowers(activity, byActor, actorsInvolvedInVideo, t, followersException) | |
26 | return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException) | ||
27 | } | 28 | } |
28 | 29 | ||
29 | function announceActivityData (url: string, byActor: ActorModel, object: string, audience?: ActivityAudience): ActivityAnnounce { | 30 | function buildAnnounceActivity (url: string, byActor: ActorModel, object: string, audience?: ActivityAudience): ActivityAnnounce { |
30 | if (!audience) audience = getAudience(byActor) | 31 | if (!audience) audience = getAudience(byActor) |
31 | 32 | ||
32 | return { | 33 | return audiencify({ |
33 | type: 'Announce', | 34 | type: 'Announce' as 'Announce', |
34 | to: audience.to, | ||
35 | cc: audience.cc, | ||
36 | id: url, | 35 | id: url, |
37 | actor: byActor.url, | 36 | actor: byActor.url, |
38 | object | 37 | object |
39 | } | 38 | }, audience) |
40 | } | 39 | } |
41 | 40 | ||
42 | // --------------------------------------------------------------------------- | 41 | // --------------------------------------------------------------------------- |
43 | 42 | ||
44 | export { | 43 | export { |
45 | sendVideoAnnounce, | 44 | sendVideoAnnounce, |
46 | announceActivityData, | 45 | buildAnnounceActivity, |
47 | buildVideoAnnounce | 46 | buildAnnounceWithVideoAudience |
48 | } | 47 | } |
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index fc76cdd8a..6f89b1a22 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts | |||
@@ -17,6 +17,7 @@ import { | |||
17 | getVideoCommentAudience | 17 | getVideoCommentAudience |
18 | } from '../audience' | 18 | } from '../audience' |
19 | import { logger } from '../../../helpers/logger' | 19 | import { logger } from '../../../helpers/logger' |
20 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | ||
20 | 21 | ||
21 | async function sendCreateVideo (video: VideoModel, t: Transaction) { | 22 | async function sendCreateVideo (video: VideoModel, t: Transaction) { |
22 | if (video.privacy === VideoPrivacy.PRIVATE) return undefined | 23 | if (video.privacy === VideoPrivacy.PRIVATE) return undefined |
@@ -27,12 +28,12 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) { | |||
27 | const videoObject = video.toActivityPubObject() | 28 | const videoObject = video.toActivityPubObject() |
28 | 29 | ||
29 | const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) | 30 | const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) |
30 | const data = createActivityData(video.url, byActor, videoObject, audience) | 31 | const createActivity = buildCreateActivity(video.url, byActor, videoObject, audience) |
31 | 32 | ||
32 | return broadcastToFollowers(data, byActor, [ byActor ], t) | 33 | return broadcastToFollowers(createActivity, byActor, [ byActor ], t) |
33 | } | 34 | } |
34 | 35 | ||
35 | async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel, t: Transaction) { | 36 | async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { |
36 | if (!video.VideoChannel.Account.Actor.serverId) return // Local | 37 | if (!video.VideoChannel.Account.Actor.serverId) return // Local |
37 | 38 | ||
38 | const url = getVideoAbuseActivityPubUrl(videoAbuse) | 39 | const url = getVideoAbuseActivityPubUrl(videoAbuse) |
@@ -40,9 +41,23 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, | |||
40 | logger.info('Creating job to send video abuse %s.', url) | 41 | logger.info('Creating job to send video abuse %s.', url) |
41 | 42 | ||
42 | const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } | 43 | const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } |
43 | const data = createActivityData(url, byActor, videoAbuse.toActivityPubObject(), audience) | 44 | const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience) |
44 | 45 | ||
45 | return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | 46 | return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) |
47 | } | ||
48 | |||
49 | async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { | ||
50 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url) | ||
51 | |||
52 | const redundancyObject = fileRedundancy.toActivityPubObject() | ||
53 | |||
54 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) | ||
55 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined) | ||
56 | |||
57 | const audience = getVideoAudience(video, actorsInvolvedInVideo) | ||
58 | const createActivity = buildCreateActivity(fileRedundancy.url, byActor, redundancyObject, audience) | ||
59 | |||
60 | return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
46 | } | 61 | } |
47 | 62 | ||
48 | async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) { | 63 | async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) { |
@@ -66,73 +81,73 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio | |||
66 | audience = getObjectFollowersAudience(actorsInvolvedInComment.concat(parentsCommentActors)) | 81 | audience = getObjectFollowersAudience(actorsInvolvedInComment.concat(parentsCommentActors)) |
67 | } | 82 | } |
68 | 83 | ||
69 | const data = createActivityData(comment.url, byActor, commentObject, audience) | 84 | const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience) |
70 | 85 | ||
71 | // This was a reply, send it to the parent actors | 86 | // This was a reply, send it to the parent actors |
72 | const actorsException = [ byActor ] | 87 | const actorsException = [ byActor ] |
73 | await broadcastToActors(data, byActor, parentsCommentActors, actorsException) | 88 | await broadcastToActors(createActivity, byActor, parentsCommentActors, actorsException) |
74 | 89 | ||
75 | // Broadcast to our followers | 90 | // Broadcast to our followers |
76 | await broadcastToFollowers(data, byActor, [ byActor ], t) | 91 | await broadcastToFollowers(createActivity, byActor, [ byActor ], t) |
77 | 92 | ||
78 | // Send to actors involved in the comment | 93 | // Send to actors involved in the comment |
79 | if (isOrigin) return broadcastToFollowers(data, byActor, actorsInvolvedInComment, t, actorsException) | 94 | if (isOrigin) return broadcastToFollowers(createActivity, byActor, actorsInvolvedInComment, t, actorsException) |
80 | 95 | ||
81 | // Send to origin | 96 | // Send to origin |
82 | return unicastTo(data, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) | 97 | return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) |
83 | } | 98 | } |
84 | 99 | ||
85 | async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) { | 100 | async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) { |
86 | logger.info('Creating job to send view of %s.', video.url) | 101 | logger.info('Creating job to send view of %s.', video.url) |
87 | 102 | ||
88 | const url = getVideoViewActivityPubUrl(byActor, video) | 103 | const url = getVideoViewActivityPubUrl(byActor, video) |
89 | const viewActivityData = createViewActivityData(byActor, video) | 104 | const viewActivity = buildViewActivity(byActor, video) |
90 | 105 | ||
91 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | 106 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) |
92 | 107 | ||
93 | // Send to origin | 108 | // Send to origin |
94 | if (video.isOwned() === false) { | 109 | if (video.isOwned() === false) { |
95 | const audience = getVideoAudience(video, actorsInvolvedInVideo) | 110 | const audience = getVideoAudience(video, actorsInvolvedInVideo) |
96 | const data = createActivityData(url, byActor, viewActivityData, audience) | 111 | const createActivity = buildCreateActivity(url, byActor, viewActivity, audience) |
97 | 112 | ||
98 | return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | 113 | return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) |
99 | } | 114 | } |
100 | 115 | ||
101 | // Send to followers | 116 | // Send to followers |
102 | const audience = getObjectFollowersAudience(actorsInvolvedInVideo) | 117 | const audience = getObjectFollowersAudience(actorsInvolvedInVideo) |
103 | const data = createActivityData(url, byActor, viewActivityData, audience) | 118 | const createActivity = buildCreateActivity(url, byActor, viewActivity, audience) |
104 | 119 | ||
105 | // Use the server actor to send the view | 120 | // Use the server actor to send the view |
106 | const serverActor = await getServerActor() | 121 | const serverActor = await getServerActor() |
107 | const actorsException = [ byActor ] | 122 | const actorsException = [ byActor ] |
108 | return broadcastToFollowers(data, serverActor, actorsInvolvedInVideo, t, actorsException) | 123 | return broadcastToFollowers(createActivity, serverActor, actorsInvolvedInVideo, t, actorsException) |
109 | } | 124 | } |
110 | 125 | ||
111 | async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { | 126 | async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { |
112 | logger.info('Creating job to dislike %s.', video.url) | 127 | logger.info('Creating job to dislike %s.', video.url) |
113 | 128 | ||
114 | const url = getVideoDislikeActivityPubUrl(byActor, video) | 129 | const url = getVideoDislikeActivityPubUrl(byActor, video) |
115 | const dislikeActivityData = createDislikeActivityData(byActor, video) | 130 | const dislikeActivity = buildDislikeActivity(byActor, video) |
116 | 131 | ||
117 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | 132 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) |
118 | 133 | ||
119 | // Send to origin | 134 | // Send to origin |
120 | if (video.isOwned() === false) { | 135 | if (video.isOwned() === false) { |
121 | const audience = getVideoAudience(video, actorsInvolvedInVideo) | 136 | const audience = getVideoAudience(video, actorsInvolvedInVideo) |
122 | const data = createActivityData(url, byActor, dislikeActivityData, audience) | 137 | const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience) |
123 | 138 | ||
124 | return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | 139 | return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) |
125 | } | 140 | } |
126 | 141 | ||
127 | // Send to followers | 142 | // Send to followers |
128 | const audience = getObjectFollowersAudience(actorsInvolvedInVideo) | 143 | const audience = getObjectFollowersAudience(actorsInvolvedInVideo) |
129 | const data = createActivityData(url, byActor, dislikeActivityData, audience) | 144 | const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience) |
130 | 145 | ||
131 | const actorsException = [ byActor ] | 146 | const actorsException = [ byActor ] |
132 | return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, actorsException) | 147 | return broadcastToFollowers(createActivity, byActor, actorsInvolvedInVideo, t, actorsException) |
133 | } | 148 | } |
134 | 149 | ||
135 | function createActivityData (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { | 150 | function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { |
136 | if (!audience) audience = getAudience(byActor) | 151 | if (!audience) audience = getAudience(byActor) |
137 | 152 | ||
138 | return audiencify( | 153 | return audiencify( |
@@ -146,7 +161,7 @@ function createActivityData (url: string, byActor: ActorModel, object: any, audi | |||
146 | ) | 161 | ) |
147 | } | 162 | } |
148 | 163 | ||
149 | function createDislikeActivityData (byActor: ActorModel, video: VideoModel) { | 164 | function buildDislikeActivity (byActor: ActorModel, video: VideoModel) { |
150 | return { | 165 | return { |
151 | type: 'Dislike', | 166 | type: 'Dislike', |
152 | actor: byActor.url, | 167 | actor: byActor.url, |
@@ -154,7 +169,7 @@ function createDislikeActivityData (byActor: ActorModel, video: VideoModel) { | |||
154 | } | 169 | } |
155 | } | 170 | } |
156 | 171 | ||
157 | function createViewActivityData (byActor: ActorModel, video: VideoModel) { | 172 | function buildViewActivity (byActor: ActorModel, video: VideoModel) { |
158 | return { | 173 | return { |
159 | type: 'View', | 174 | type: 'View', |
160 | actor: byActor.url, | 175 | actor: byActor.url, |
@@ -167,9 +182,10 @@ function createViewActivityData (byActor: ActorModel, video: VideoModel) { | |||
167 | export { | 182 | export { |
168 | sendCreateVideo, | 183 | sendCreateVideo, |
169 | sendVideoAbuse, | 184 | sendVideoAbuse, |
170 | createActivityData, | 185 | buildCreateActivity, |
171 | sendCreateView, | 186 | sendCreateView, |
172 | sendCreateDislike, | 187 | sendCreateDislike, |
173 | createDislikeActivityData, | 188 | buildDislikeActivity, |
174 | sendCreateVideoComment | 189 | sendCreateVideoComment, |
190 | sendCreateCacheFile | ||
175 | } | 191 | } |
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts index 3d1dfb699..479182543 100644 --- a/server/lib/activitypub/send/send-delete.ts +++ b/server/lib/activitypub/send/send-delete.ts | |||
@@ -15,24 +15,23 @@ async function sendDeleteVideo (video: VideoModel, t: Transaction) { | |||
15 | const url = getDeleteActivityPubUrl(video.url) | 15 | const url = getDeleteActivityPubUrl(video.url) |
16 | const byActor = video.VideoChannel.Account.Actor | 16 | const byActor = video.VideoChannel.Account.Actor |
17 | 17 | ||
18 | const data = deleteActivityData(url, video.url, byActor) | 18 | const activity = buildDeleteActivity(url, video.url, byActor) |
19 | 19 | ||
20 | const actorsInvolved = await VideoShareModel.loadActorsByShare(video.id, t) | 20 | const actorsInvolved = await getActorsInvolvedInVideo(video, t) |
21 | actorsInvolved.push(byActor) | ||
22 | 21 | ||
23 | return broadcastToFollowers(data, byActor, actorsInvolved, t) | 22 | return broadcastToFollowers(activity, byActor, actorsInvolved, t) |
24 | } | 23 | } |
25 | 24 | ||
26 | async function sendDeleteActor (byActor: ActorModel, t: Transaction) { | 25 | async function sendDeleteActor (byActor: ActorModel, t: Transaction) { |
27 | logger.info('Creating job to broadcast delete of actor %s.', byActor.url) | 26 | logger.info('Creating job to broadcast delete of actor %s.', byActor.url) |
28 | 27 | ||
29 | const url = getDeleteActivityPubUrl(byActor.url) | 28 | const url = getDeleteActivityPubUrl(byActor.url) |
30 | const data = deleteActivityData(url, byActor.url, byActor) | 29 | const activity = buildDeleteActivity(url, byActor.url, byActor) |
31 | 30 | ||
32 | const actorsInvolved = await VideoShareModel.loadActorsByVideoOwner(byActor.id, t) | 31 | const actorsInvolved = await VideoShareModel.loadActorsByVideoOwner(byActor.id, t) |
33 | actorsInvolved.push(byActor) | 32 | actorsInvolved.push(byActor) |
34 | 33 | ||
35 | return broadcastToFollowers(data, byActor, actorsInvolved, t) | 34 | return broadcastToFollowers(activity, byActor, actorsInvolved, t) |
36 | } | 35 | } |
37 | 36 | ||
38 | async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Transaction) { | 37 | async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Transaction) { |
@@ -45,23 +44,23 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans | |||
45 | const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, t) | 44 | const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, t) |
46 | 45 | ||
47 | const actorsInvolvedInComment = await getActorsInvolvedInVideo(videoComment.Video, t) | 46 | const actorsInvolvedInComment = await getActorsInvolvedInVideo(videoComment.Video, t) |
48 | actorsInvolvedInComment.push(byActor) | 47 | actorsInvolvedInComment.push(byActor) // Add the actor that commented the video |
49 | 48 | ||
50 | const audience = getVideoCommentAudience(videoComment, threadParentComments, actorsInvolvedInComment, isVideoOrigin) | 49 | const audience = getVideoCommentAudience(videoComment, threadParentComments, actorsInvolvedInComment, isVideoOrigin) |
51 | const data = deleteActivityData(url, videoComment.url, byActor, audience) | 50 | const activity = buildDeleteActivity(url, videoComment.url, byActor, audience) |
52 | 51 | ||
53 | // This was a reply, send it to the parent actors | 52 | // This was a reply, send it to the parent actors |
54 | const actorsException = [ byActor ] | 53 | const actorsException = [ byActor ] |
55 | await broadcastToActors(data, byActor, threadParentComments.map(c => c.Account.Actor), actorsException) | 54 | await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), actorsException) |
56 | 55 | ||
57 | // Broadcast to our followers | 56 | // Broadcast to our followers |
58 | await broadcastToFollowers(data, byActor, [ byActor ], t) | 57 | await broadcastToFollowers(activity, byActor, [ byActor ], t) |
59 | 58 | ||
60 | // Send to actors involved in the comment | 59 | // Send to actors involved in the comment |
61 | if (isVideoOrigin) return broadcastToFollowers(data, byActor, actorsInvolvedInComment, t, actorsException) | 60 | if (isVideoOrigin) return broadcastToFollowers(activity, byActor, actorsInvolvedInComment, t, actorsException) |
62 | 61 | ||
63 | // Send to origin | 62 | // Send to origin |
64 | return unicastTo(data, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl) | 63 | return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl) |
65 | } | 64 | } |
66 | 65 | ||
67 | // --------------------------------------------------------------------------- | 66 | // --------------------------------------------------------------------------- |
@@ -74,7 +73,7 @@ export { | |||
74 | 73 | ||
75 | // --------------------------------------------------------------------------- | 74 | // --------------------------------------------------------------------------- |
76 | 75 | ||
77 | function deleteActivityData (url: string, object: string, byActor: ActorModel, audience?: ActivityAudience): ActivityDelete { | 76 | function buildDeleteActivity (url: string, object: string, byActor: ActorModel, audience?: ActivityAudience): ActivityDelete { |
78 | const activity = { | 77 | const activity = { |
79 | type: 'Delete' as 'Delete', | 78 | type: 'Delete' as 'Delete', |
80 | id: url, | 79 | id: url, |
diff --git a/server/lib/activitypub/send/send-follow.ts b/server/lib/activitypub/send/send-follow.ts index 46d08c17b..170b46b48 100644 --- a/server/lib/activitypub/send/send-follow.ts +++ b/server/lib/activitypub/send/send-follow.ts | |||
@@ -15,12 +15,12 @@ function sendFollow (actorFollow: ActorFollowModel) { | |||
15 | logger.info('Creating job to send follow request to %s.', following.url) | 15 | logger.info('Creating job to send follow request to %s.', following.url) |
16 | 16 | ||
17 | const url = getActorFollowActivityPubUrl(actorFollow) | 17 | const url = getActorFollowActivityPubUrl(actorFollow) |
18 | const data = followActivityData(url, me, following) | 18 | const data = buildFollowActivity(url, me, following) |
19 | 19 | ||
20 | return unicastTo(data, me, following.inboxUrl) | 20 | return unicastTo(data, me, following.inboxUrl) |
21 | } | 21 | } |
22 | 22 | ||
23 | function followActivityData (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow { | 23 | function buildFollowActivity (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow { |
24 | return { | 24 | return { |
25 | type: 'Follow', | 25 | type: 'Follow', |
26 | id: url, | 26 | id: url, |
@@ -33,5 +33,5 @@ function followActivityData (url: string, byActor: ActorModel, targetActor: Acto | |||
33 | 33 | ||
34 | export { | 34 | export { |
35 | sendFollow, | 35 | sendFollow, |
36 | followActivityData | 36 | buildFollowActivity |
37 | } | 37 | } |
diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts index 83225f5df..a5408ac6a 100644 --- a/server/lib/activitypub/send/send-like.ts +++ b/server/lib/activitypub/send/send-like.ts | |||
@@ -17,20 +17,20 @@ async function sendLike (byActor: ActorModel, video: VideoModel, t: Transaction) | |||
17 | // Send to origin | 17 | // Send to origin |
18 | if (video.isOwned() === false) { | 18 | if (video.isOwned() === false) { |
19 | const audience = getVideoAudience(video, accountsInvolvedInVideo) | 19 | const audience = getVideoAudience(video, accountsInvolvedInVideo) |
20 | const data = likeActivityData(url, byActor, video, audience) | 20 | const data = buildLikeActivity(url, byActor, video, audience) |
21 | 21 | ||
22 | return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | 22 | return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) |
23 | } | 23 | } |
24 | 24 | ||
25 | // Send to followers | 25 | // Send to followers |
26 | const audience = getObjectFollowersAudience(accountsInvolvedInVideo) | 26 | const audience = getObjectFollowersAudience(accountsInvolvedInVideo) |
27 | const data = likeActivityData(url, byActor, video, audience) | 27 | const activity = buildLikeActivity(url, byActor, video, audience) |
28 | 28 | ||
29 | const followersException = [ byActor ] | 29 | const followersException = [ byActor ] |
30 | return broadcastToFollowers(data, byActor, accountsInvolvedInVideo, t, followersException) | 30 | return broadcastToFollowers(activity, byActor, accountsInvolvedInVideo, t, followersException) |
31 | } | 31 | } |
32 | 32 | ||
33 | function likeActivityData (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike { | 33 | function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike { |
34 | if (!audience) audience = getAudience(byActor) | 34 | if (!audience) audience = getAudience(byActor) |
35 | 35 | ||
36 | return audiencify( | 36 | return audiencify( |
@@ -48,5 +48,5 @@ function likeActivityData (url: string, byActor: ActorModel, video: VideoModel, | |||
48 | 48 | ||
49 | export { | 49 | export { |
50 | sendLike, | 50 | sendLike, |
51 | likeActivityData | 51 | buildLikeActivity |
52 | } | 52 | } |
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index 30d0fd98b..a50673c79 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts | |||
@@ -13,12 +13,13 @@ import { VideoModel } from '../../../models/video/video' | |||
13 | import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' | 13 | import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' |
14 | import { broadcastToFollowers, unicastTo } from './utils' | 14 | import { broadcastToFollowers, unicastTo } from './utils' |
15 | import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience' | 15 | import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience' |
16 | import { createActivityData, createDislikeActivityData } from './send-create' | 16 | import { buildCreateActivity, buildDislikeActivity } from './send-create' |
17 | import { followActivityData } from './send-follow' | 17 | import { buildFollowActivity } from './send-follow' |
18 | import { likeActivityData } from './send-like' | 18 | import { buildLikeActivity } from './send-like' |
19 | import { VideoShareModel } from '../../../models/video/video-share' | 19 | import { VideoShareModel } from '../../../models/video/video-share' |
20 | import { buildVideoAnnounce } from './send-announce' | 20 | import { buildAnnounceWithVideoAudience } from './send-announce' |
21 | import { logger } from '../../../helpers/logger' | 21 | import { logger } from '../../../helpers/logger' |
22 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | ||
22 | 23 | ||
23 | async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { | 24 | async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { |
24 | const me = actorFollow.ActorFollower | 25 | const me = actorFollow.ActorFollower |
@@ -32,10 +33,10 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { | |||
32 | const followUrl = getActorFollowActivityPubUrl(actorFollow) | 33 | const followUrl = getActorFollowActivityPubUrl(actorFollow) |
33 | const undoUrl = getUndoActivityPubUrl(followUrl) | 34 | const undoUrl = getUndoActivityPubUrl(followUrl) |
34 | 35 | ||
35 | const object = followActivityData(followUrl, me, following) | 36 | const followActivity = buildFollowActivity(followUrl, me, following) |
36 | const data = undoActivityData(undoUrl, me, object) | 37 | const undoActivity = undoActivityData(undoUrl, me, followActivity) |
37 | 38 | ||
38 | return unicastTo(data, me, following.inboxUrl) | 39 | return unicastTo(undoActivity, me, following.inboxUrl) |
39 | } | 40 | } |
40 | 41 | ||
41 | async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) { | 42 | async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) { |
@@ -45,21 +46,21 @@ async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transact | |||
45 | const undoUrl = getUndoActivityPubUrl(likeUrl) | 46 | const undoUrl = getUndoActivityPubUrl(likeUrl) |
46 | 47 | ||
47 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | 48 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) |
48 | const object = likeActivityData(likeUrl, byActor, video) | 49 | const likeActivity = buildLikeActivity(likeUrl, byActor, video) |
49 | 50 | ||
50 | // Send to origin | 51 | // Send to origin |
51 | if (video.isOwned() === false) { | 52 | if (video.isOwned() === false) { |
52 | const audience = getVideoAudience(video, actorsInvolvedInVideo) | 53 | const audience = getVideoAudience(video, actorsInvolvedInVideo) |
53 | const data = undoActivityData(undoUrl, byActor, object, audience) | 54 | const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience) |
54 | 55 | ||
55 | return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | 56 | return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) |
56 | } | 57 | } |
57 | 58 | ||
58 | const audience = getObjectFollowersAudience(actorsInvolvedInVideo) | 59 | const audience = getObjectFollowersAudience(actorsInvolvedInVideo) |
59 | const data = undoActivityData(undoUrl, byActor, object, audience) | 60 | const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience) |
60 | 61 | ||
61 | const followersException = [ byActor ] | 62 | const followersException = [ byActor ] |
62 | return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException) | 63 | return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException) |
63 | } | 64 | } |
64 | 65 | ||
65 | async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { | 66 | async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { |
@@ -69,20 +70,20 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans | |||
69 | const undoUrl = getUndoActivityPubUrl(dislikeUrl) | 70 | const undoUrl = getUndoActivityPubUrl(dislikeUrl) |
70 | 71 | ||
71 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | 72 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) |
72 | const dislikeActivity = createDislikeActivityData(byActor, video) | 73 | const dislikeActivity = buildDislikeActivity(byActor, video) |
73 | const object = createActivityData(dislikeUrl, byActor, dislikeActivity) | 74 | const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) |
74 | 75 | ||
75 | if (video.isOwned() === false) { | 76 | if (video.isOwned() === false) { |
76 | const audience = getVideoAudience(video, actorsInvolvedInVideo) | 77 | const audience = getVideoAudience(video, actorsInvolvedInVideo) |
77 | const data = undoActivityData(undoUrl, byActor, object, audience) | 78 | const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity, audience) |
78 | 79 | ||
79 | return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | 80 | return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) |
80 | } | 81 | } |
81 | 82 | ||
82 | const data = undoActivityData(undoUrl, byActor, object) | 83 | const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity) |
83 | 84 | ||
84 | const followersException = [ byActor ] | 85 | const followersException = [ byActor ] |
85 | return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException) | 86 | return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException) |
86 | } | 87 | } |
87 | 88 | ||
88 | async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { | 89 | async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { |
@@ -90,12 +91,27 @@ async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareMode | |||
90 | 91 | ||
91 | const undoUrl = getUndoActivityPubUrl(videoShare.url) | 92 | const undoUrl = getUndoActivityPubUrl(videoShare.url) |
92 | 93 | ||
93 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | 94 | const { activity: announceActivity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t) |
94 | const object = await buildVideoAnnounce(byActor, videoShare, video, t) | 95 | const undoActivity = undoActivityData(undoUrl, byActor, announceActivity) |
95 | const data = undoActivityData(undoUrl, byActor, object) | ||
96 | 96 | ||
97 | const followersException = [ byActor ] | 97 | const followersException = [ byActor ] |
98 | return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException) | 98 | return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException) |
99 | } | ||
100 | |||
101 | async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { | ||
102 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) | ||
103 | |||
104 | const undoUrl = getUndoActivityPubUrl(redundancyModel.url) | ||
105 | |||
106 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | ||
107 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | ||
108 | |||
109 | const audience = getVideoAudience(video, actorsInvolvedInVideo) | ||
110 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) | ||
111 | |||
112 | const undoActivity = undoActivityData(undoUrl, byActor, createActivity, audience) | ||
113 | |||
114 | return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
99 | } | 115 | } |
100 | 116 | ||
101 | // --------------------------------------------------------------------------- | 117 | // --------------------------------------------------------------------------- |
@@ -104,7 +120,8 @@ export { | |||
104 | sendUndoFollow, | 120 | sendUndoFollow, |
105 | sendUndoLike, | 121 | sendUndoLike, |
106 | sendUndoDislike, | 122 | sendUndoDislike, |
107 | sendUndoAnnounce | 123 | sendUndoAnnounce, |
124 | sendUndoCacheFile | ||
108 | } | 125 | } |
109 | 126 | ||
110 | // --------------------------------------------------------------------------- | 127 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index 6f1d80898..605473338 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts | |||
@@ -7,11 +7,11 @@ import { VideoModel } from '../../../models/video/video' | |||
7 | import { VideoChannelModel } from '../../../models/video/video-channel' | 7 | import { VideoChannelModel } from '../../../models/video/video-channel' |
8 | import { VideoShareModel } from '../../../models/video/video-share' | 8 | import { VideoShareModel } from '../../../models/video/video-share' |
9 | import { getUpdateActivityPubUrl } from '../url' | 9 | import { getUpdateActivityPubUrl } from '../url' |
10 | import { broadcastToFollowers } from './utils' | 10 | import { broadcastToFollowers, unicastTo } from './utils' |
11 | import { audiencify, getAudience } from '../audience' | 11 | import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' |
12 | import { logger } from '../../../helpers/logger' | 12 | import { logger } from '../../../helpers/logger' |
13 | import { videoFeedsValidator } from '../../../middlewares/validators' | ||
14 | import { VideoCaptionModel } from '../../../models/video/video-caption' | 13 | import { VideoCaptionModel } from '../../../models/video/video-caption' |
14 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | ||
15 | 15 | ||
16 | async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByActor?: ActorModel) { | 16 | async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByActor?: ActorModel) { |
17 | logger.info('Creating job to update video %s.', video.url) | 17 | logger.info('Creating job to update video %s.', video.url) |
@@ -26,12 +26,12 @@ async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByAct | |||
26 | const videoObject = video.toActivityPubObject() | 26 | const videoObject = video.toActivityPubObject() |
27 | const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) | 27 | const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) |
28 | 28 | ||
29 | const data = updateActivityData(url, byActor, videoObject, audience) | 29 | const updateActivity = buildUpdateActivity(url, byActor, videoObject, audience) |
30 | 30 | ||
31 | const actorsInvolved = await VideoShareModel.loadActorsByShare(video.id, t) | 31 | const actorsInvolved = await getActorsInvolvedInVideo(video, t) |
32 | actorsInvolved.push(byActor) | 32 | if (overrodeByActor) actorsInvolved.push(overrodeByActor) |
33 | 33 | ||
34 | return broadcastToFollowers(data, byActor, actorsInvolved, t) | 34 | return broadcastToFollowers(updateActivity, byActor, actorsInvolved, t) |
35 | } | 35 | } |
36 | 36 | ||
37 | async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelModel, t: Transaction) { | 37 | async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelModel, t: Transaction) { |
@@ -42,7 +42,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod | |||
42 | const url = getUpdateActivityPubUrl(byActor.url, byActor.updatedAt.toISOString()) | 42 | const url = getUpdateActivityPubUrl(byActor.url, byActor.updatedAt.toISOString()) |
43 | const accountOrChannelObject = accountOrChannel.toActivityPubObject() | 43 | const accountOrChannelObject = accountOrChannel.toActivityPubObject() |
44 | const audience = getAudience(byActor) | 44 | const audience = getAudience(byActor) |
45 | const data = updateActivityData(url, byActor, accountOrChannelObject, audience) | 45 | const updateActivity = buildUpdateActivity(url, byActor, accountOrChannelObject, audience) |
46 | 46 | ||
47 | let actorsInvolved: ActorModel[] | 47 | let actorsInvolved: ActorModel[] |
48 | if (accountOrChannel instanceof AccountModel) { | 48 | if (accountOrChannel instanceof AccountModel) { |
@@ -55,19 +55,35 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod | |||
55 | 55 | ||
56 | actorsInvolved.push(byActor) | 56 | actorsInvolved.push(byActor) |
57 | 57 | ||
58 | return broadcastToFollowers(data, byActor, actorsInvolved, t) | 58 | return broadcastToFollowers(updateActivity, byActor, actorsInvolved, t) |
59 | } | ||
60 | |||
61 | async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { | ||
62 | logger.info('Creating job to update cache file %s.', redundancyModel.url) | ||
63 | |||
64 | const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString()) | ||
65 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | ||
66 | |||
67 | const redundancyObject = redundancyModel.toActivityPubObject() | ||
68 | |||
69 | const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined) | ||
70 | const audience = getObjectFollowersAudience(accountsInvolvedInVideo) | ||
71 | |||
72 | const updateActivity = buildUpdateActivity(url, byActor, redundancyObject, audience) | ||
73 | return unicastTo(updateActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
59 | } | 74 | } |
60 | 75 | ||
61 | // --------------------------------------------------------------------------- | 76 | // --------------------------------------------------------------------------- |
62 | 77 | ||
63 | export { | 78 | export { |
64 | sendUpdateActor, | 79 | sendUpdateActor, |
65 | sendUpdateVideo | 80 | sendUpdateVideo, |
81 | sendUpdateCacheFile | ||
66 | } | 82 | } |
67 | 83 | ||
68 | // --------------------------------------------------------------------------- | 84 | // --------------------------------------------------------------------------- |
69 | 85 | ||
70 | function updateActivityData (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityUpdate { | 86 | function buildUpdateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityUpdate { |
71 | if (!audience) audience = getAudience(byActor) | 87 | if (!audience) audience = getAudience(byActor) |
72 | 88 | ||
73 | return audiencify( | 89 | return audiencify( |
diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts index da437292e..c20c15633 100644 --- a/server/lib/activitypub/send/utils.ts +++ b/server/lib/activitypub/send/utils.ts | |||
@@ -59,11 +59,11 @@ async function forwardActivity ( | |||
59 | async function broadcastToFollowers ( | 59 | async function broadcastToFollowers ( |
60 | data: any, | 60 | data: any, |
61 | byActor: ActorModel, | 61 | byActor: ActorModel, |
62 | toActorFollowers: ActorModel[], | 62 | toFollowersOf: ActorModel[], |
63 | t: Transaction, | 63 | t: Transaction, |
64 | actorsException: ActorModel[] = [] | 64 | actorsException: ActorModel[] = [] |
65 | ) { | 65 | ) { |
66 | const uris = await computeFollowerUris(toActorFollowers, actorsException, t) | 66 | const uris = await computeFollowerUris(toFollowersOf, actorsException, t) |
67 | return broadcastTo(uris, data, byActor) | 67 | return broadcastTo(uris, data, byActor) |
68 | } | 68 | } |
69 | 69 | ||
@@ -115,8 +115,8 @@ export { | |||
115 | 115 | ||
116 | // --------------------------------------------------------------------------- | 116 | // --------------------------------------------------------------------------- |
117 | 117 | ||
118 | async function computeFollowerUris (toActorFollower: ActorModel[], actorsException: ActorModel[], t: Transaction) { | 118 | async function computeFollowerUris (toFollowersOf: ActorModel[], actorsException: ActorModel[], t: Transaction) { |
119 | const toActorFollowerIds = toActorFollower.map(a => a.id) | 119 | const toActorFollowerIds = toFollowersOf.map(a => a.id) |
120 | 120 | ||
121 | const result = await ActorFollowModel.listAcceptedFollowerSharedInboxUrls(toActorFollowerIds, t) | 121 | const result = await ActorFollowModel.listAcceptedFollowerSharedInboxUrls(toActorFollowerIds, t) |
122 | const sharedInboxesException = await buildSharedInboxesException(actorsException) | 122 | const sharedInboxesException = await buildSharedInboxesException(actorsException) |
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts index 262463310..2e7c56955 100644 --- a/server/lib/activitypub/url.ts +++ b/server/lib/activitypub/url.ts | |||
@@ -4,11 +4,18 @@ import { ActorFollowModel } from '../../models/activitypub/actor-follow' | |||
4 | import { VideoModel } from '../../models/video/video' | 4 | import { VideoModel } from '../../models/video/video' |
5 | import { VideoAbuseModel } from '../../models/video/video-abuse' | 5 | import { VideoAbuseModel } from '../../models/video/video-abuse' |
6 | import { VideoCommentModel } from '../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../models/video/video-comment' |
7 | import { VideoFileModel } from '../../models/video/video-file' | ||
7 | 8 | ||
8 | function getVideoActivityPubUrl (video: VideoModel) { | 9 | function getVideoActivityPubUrl (video: VideoModel) { |
9 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid | 10 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid |
10 | } | 11 | } |
11 | 12 | ||
13 | function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) { | ||
14 | const suffixFPS = videoFile.fps ? '-' + videoFile.fps : '' | ||
15 | |||
16 | return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` | ||
17 | } | ||
18 | |||
12 | function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { | 19 | function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { |
13 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id | 20 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id |
14 | } | 21 | } |
@@ -101,5 +108,6 @@ export { | |||
101 | getVideoSharesActivityPubUrl, | 108 | getVideoSharesActivityPubUrl, |
102 | getVideoCommentsActivityPubUrl, | 109 | getVideoCommentsActivityPubUrl, |
103 | getVideoLikesActivityPubUrl, | 110 | getVideoLikesActivityPubUrl, |
104 | getVideoDislikesActivityPubUrl | 111 | getVideoDislikesActivityPubUrl, |
112 | getVideoCacheFileActivityPubUrl | ||
105 | } | 113 | } |
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 6c2095897..783f78d3e 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -3,12 +3,12 @@ import * as sequelize from 'sequelize' | |||
3 | import * as magnetUtil from 'magnet-uri' | 3 | import * as magnetUtil from 'magnet-uri' |
4 | import { join } from 'path' | 4 | import { join } from 'path' |
5 | import * as request from 'request' | 5 | import * as request from 'request' |
6 | import { ActivityIconObject, VideoState } from '../../../shared/index' | 6 | import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index' |
7 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 7 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
8 | import { VideoPrivacy } from '../../../shared/models/videos' | 8 | import { VideoPrivacy } from '../../../shared/models/videos' |
9 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' | 9 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' |
10 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' | 10 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' |
11 | import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' | 11 | import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' |
12 | import { logger } from '../../helpers/logger' | 12 | import { logger } from '../../helpers/logger' |
13 | import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' | 13 | import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' |
14 | import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' | 14 | import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' |
@@ -17,7 +17,7 @@ import { TagModel } from '../../models/video/tag' | |||
17 | import { VideoModel } from '../../models/video/video' | 17 | import { VideoModel } from '../../models/video/video' |
18 | import { VideoChannelModel } from '../../models/video/video-channel' | 18 | import { VideoChannelModel } from '../../models/video/video-channel' |
19 | import { VideoFileModel } from '../../models/video/video-file' | 19 | import { VideoFileModel } from '../../models/video/video-file' |
20 | import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor' | 20 | import { getOrCreateActorAndServerAndModel } from './actor' |
21 | import { addVideoComments } from './video-comments' | 21 | import { addVideoComments } from './video-comments' |
22 | import { crawlCollectionPage } from './crawl' | 22 | import { crawlCollectionPage } from './crawl' |
23 | import { sendCreateVideo, sendUpdateVideo } from './send' | 23 | import { sendCreateVideo, sendUpdateVideo } from './send' |
@@ -25,7 +25,6 @@ import { isArray } from '../../helpers/custom-validators/misc' | |||
25 | import { VideoCaptionModel } from '../../models/video/video-caption' | 25 | import { VideoCaptionModel } from '../../models/video/video-caption' |
26 | import { JobQueue } from '../job-queue' | 26 | import { JobQueue } from '../job-queue' |
27 | import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher' | 27 | import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher' |
28 | import { getUrlFromWebfinger } from '../../helpers/webfinger' | ||
29 | import { createRates } from './video-rates' | 28 | import { createRates } from './video-rates' |
30 | import { addVideoShares, shareVideoByServerAndChannel } from './share' | 29 | import { addVideoShares, shareVideoByServerAndChannel } from './share' |
31 | import { AccountModel } from '../../models/account/account' | 30 | import { AccountModel } from '../../models/account/account' |
@@ -137,10 +136,7 @@ async function videoActivityObjectToDBAttributes ( | |||
137 | } | 136 | } |
138 | 137 | ||
139 | function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { | 138 | function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { |
140 | const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) | 139 | const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] |
141 | const fileUrls = videoObject.url.filter(u => { | ||
142 | return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/') | ||
143 | }) | ||
144 | 140 | ||
145 | if (fileUrls.length === 0) { | 141 | if (fileUrls.length === 0) { |
146 | throw new Error('Cannot find video files for ' + videoCreated.url) | 142 | throw new Error('Cannot find video files for ' + videoCreated.url) |
@@ -331,8 +327,8 @@ async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> { | |||
331 | 327 | ||
332 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | 328 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) |
333 | const account = await AccountModel.load(channelActor.VideoChannel.accountId) | 329 | const account = await AccountModel.load(channelActor.VideoChannel.accountId) |
334 | return updateVideoFromAP(video, videoObject, account.Actor, channelActor) | ||
335 | 330 | ||
331 | return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) | ||
336 | } catch (err) { | 332 | } catch (err) { |
337 | logger.warn('Cannot refresh video.', { err }) | 333 | logger.warn('Cannot refresh video.', { err }) |
338 | return video | 334 | return video |
@@ -342,8 +338,8 @@ async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> { | |||
342 | async function updateVideoFromAP ( | 338 | async function updateVideoFromAP ( |
343 | video: VideoModel, | 339 | video: VideoModel, |
344 | videoObject: VideoTorrentObject, | 340 | videoObject: VideoTorrentObject, |
345 | accountActor: ActorModel, | 341 | account: AccountModel, |
346 | channelActor: ActorModel, | 342 | channel: VideoChannelModel, |
347 | overrideTo?: string[] | 343 | overrideTo?: string[] |
348 | ) { | 344 | ) { |
349 | logger.debug('Updating remote video "%s".', videoObject.uuid) | 345 | logger.debug('Updating remote video "%s".', videoObject.uuid) |
@@ -359,12 +355,12 @@ async function updateVideoFromAP ( | |||
359 | 355 | ||
360 | // Check actor has the right to update the video | 356 | // Check actor has the right to update the video |
361 | const videoChannel = video.VideoChannel | 357 | const videoChannel = video.VideoChannel |
362 | if (videoChannel.Account.Actor.id !== accountActor.id) { | 358 | if (videoChannel.Account.id !== account.id) { |
363 | throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url) | 359 | throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url) |
364 | } | 360 | } |
365 | 361 | ||
366 | const to = overrideTo ? overrideTo : videoObject.to | 362 | const to = overrideTo ? overrideTo : videoObject.to |
367 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to) | 363 | const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to) |
368 | video.set('name', videoData.name) | 364 | video.set('name', videoData.name) |
369 | video.set('uuid', videoData.uuid) | 365 | video.set('uuid', videoData.uuid) |
370 | video.set('url', videoData.url) | 366 | video.set('url', videoData.url) |
@@ -444,3 +440,11 @@ export { | |||
444 | addVideoShares, | 440 | addVideoShares, |
445 | createRates | 441 | createRates |
446 | } | 442 | } |
443 | |||
444 | // --------------------------------------------------------------------------- | ||
445 | |||
446 | function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { | ||
447 | const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) | ||
448 | |||
449 | return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') | ||
450 | } | ||
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts new file mode 100644 index 000000000..78221cc3d --- /dev/null +++ b/server/lib/redundancy.ts | |||
@@ -0,0 +1,18 @@ | |||
1 | import { VideoRedundancyModel } from '../models/redundancy/video-redundancy' | ||
2 | import { sendUndoCacheFile } from './activitypub/send' | ||
3 | import { Transaction } from 'sequelize' | ||
4 | import { getServerActor } from '../helpers/utils' | ||
5 | |||
6 | async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?: Transaction) { | ||
7 | const serverActor = await getServerActor() | ||
8 | |||
9 | await sendUndoCacheFile(serverActor, videoRedundancy, t) | ||
10 | |||
11 | await videoRedundancy.destroy({ transaction: t }) | ||
12 | } | ||
13 | |||
14 | // --------------------------------------------------------------------------- | ||
15 | |||
16 | export { | ||
17 | removeVideoRedundancy | ||
18 | } | ||
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts new file mode 100644 index 000000000..ee9ba1766 --- /dev/null +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -0,0 +1,161 @@ | |||
1 | import { AbstractScheduler } from './abstract-scheduler' | ||
2 | import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' | ||
3 | import { logger } from '../../helpers/logger' | ||
4 | import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' | ||
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | ||
6 | import { VideoFileModel } from '../../models/video/video-file' | ||
7 | import { sortBy } from 'lodash' | ||
8 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' | ||
9 | import { join } from 'path' | ||
10 | import { rename } from 'fs-extra' | ||
11 | import { getServerActor } from '../../helpers/utils' | ||
12 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | ||
13 | import { VideoModel } from '../../models/video/video' | ||
14 | import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' | ||
15 | import { removeVideoRedundancy } from '../redundancy' | ||
16 | import { isTestInstance } from '../../helpers/core-utils' | ||
17 | |||
18 | export class VideosRedundancyScheduler extends AbstractScheduler { | ||
19 | |||
20 | private static instance: AbstractScheduler | ||
21 | private executing = false | ||
22 | |||
23 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy | ||
24 | |||
25 | private constructor () { | ||
26 | super() | ||
27 | } | ||
28 | |||
29 | async execute () { | ||
30 | if (this.executing) return | ||
31 | |||
32 | this.executing = true | ||
33 | |||
34 | for (const obj of CONFIG.REDUNDANCY.VIDEOS) { | ||
35 | |||
36 | try { | ||
37 | const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy) | ||
38 | if (!videoToDuplicate) continue | ||
39 | |||
40 | const videoFiles = videoToDuplicate.VideoFiles | ||
41 | videoFiles.forEach(f => f.Video = videoToDuplicate) | ||
42 | |||
43 | const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy) | ||
44 | if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) { | ||
45 | if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) | ||
46 | continue | ||
47 | } | ||
48 | |||
49 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) | ||
50 | |||
51 | await this.createVideoRedundancy(obj.strategy, videoFiles) | ||
52 | } catch (err) { | ||
53 | logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) | ||
54 | } | ||
55 | } | ||
56 | |||
57 | const expired = await VideoRedundancyModel.listAllExpired() | ||
58 | |||
59 | for (const m of expired) { | ||
60 | logger.info('Removing expired video %s from our redundancy system.', this.buildEntryLogId(m)) | ||
61 | |||
62 | try { | ||
63 | await m.destroy() | ||
64 | } catch (err) { | ||
65 | logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) | ||
66 | } | ||
67 | } | ||
68 | |||
69 | this.executing = false | ||
70 | } | ||
71 | |||
72 | static get Instance () { | ||
73 | return this.instance || (this.instance = new this()) | ||
74 | } | ||
75 | |||
76 | private findVideoToDuplicate (strategy: VideoRedundancyStrategy) { | ||
77 | if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) | ||
78 | } | ||
79 | |||
80 | private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { | ||
81 | const serverActor = await getServerActor() | ||
82 | |||
83 | for (const file of filesToDuplicate) { | ||
84 | const existing = await VideoRedundancyModel.loadByFileId(file.id) | ||
85 | if (existing) { | ||
86 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', file.Video.url, file.resolution, strategy) | ||
87 | |||
88 | existing.expiresOn = this.buildNewExpiration() | ||
89 | await existing.save() | ||
90 | |||
91 | await sendUpdateCacheFile(serverActor, existing) | ||
92 | continue | ||
93 | } | ||
94 | |||
95 | // We need more attributes and check if the video still exists | ||
96 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id) | ||
97 | if (!video) continue | ||
98 | |||
99 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) | ||
100 | |||
101 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | ||
102 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) | ||
103 | |||
104 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, JOB_TTL['video-import']) | ||
105 | |||
106 | const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file)) | ||
107 | await rename(tmpPath, destPath) | ||
108 | |||
109 | const createdModel = await VideoRedundancyModel.create({ | ||
110 | expiresOn: new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS), | ||
111 | url: getVideoCacheFileActivityPubUrl(file), | ||
112 | fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL), | ||
113 | strategy, | ||
114 | videoFileId: file.id, | ||
115 | actorId: serverActor.id | ||
116 | }) | ||
117 | createdModel.VideoFile = file | ||
118 | |||
119 | await sendCreateCacheFile(serverActor, createdModel) | ||
120 | } | ||
121 | } | ||
122 | |||
123 | // Unused, but could be useful in the future, with a custom strategy | ||
124 | private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) { | ||
125 | const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt') | ||
126 | |||
127 | while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) { | ||
128 | const toDelete = sortedVideosRedundancy.shift() | ||
129 | |||
130 | const videoFile = toDelete.VideoFile | ||
131 | logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution) | ||
132 | |||
133 | await removeVideoRedundancy(toDelete, undefined) | ||
134 | } | ||
135 | |||
136 | return sortedVideosRedundancy | ||
137 | } | ||
138 | |||
139 | private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) { | ||
140 | const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) | ||
141 | |||
142 | const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size | ||
143 | const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0) | ||
144 | |||
145 | return totalDuplicated > maxSize | ||
146 | } | ||
147 | |||
148 | private buildNewExpiration () { | ||
149 | return new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS) | ||
150 | } | ||
151 | |||
152 | private buildEntryLogId (object: VideoRedundancyModel) { | ||
153 | return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` | ||
154 | } | ||
155 | |||
156 | private getTotalFileSizes (files: VideoFileModel[]) { | ||
157 | const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size | ||
158 | |||
159 | return files.reduce(fileReducer, 0) | ||
160 | } | ||
161 | } | ||
diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts new file mode 100644 index 000000000..d91b47574 --- /dev/null +++ b/server/middlewares/validators/redundancy.ts | |||
@@ -0,0 +1,80 @@ | |||
1 | import * as express from 'express' | ||
2 | import 'express-validator' | ||
3 | import { param, body } from 'express-validator/check' | ||
4 | import { exists, isBooleanValid, isIdOrUUIDValid, toIntOrNull } from '../../helpers/custom-validators/misc' | ||
5 | import { isVideoExist } from '../../helpers/custom-validators/videos' | ||
6 | import { logger } from '../../helpers/logger' | ||
7 | import { areValidationErrors } from './utils' | ||
8 | import { VideoModel } from '../../models/video/video' | ||
9 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | ||
10 | import { isHostValid } from '../../helpers/custom-validators/servers' | ||
11 | import { getServerActor } from '../../helpers/utils' | ||
12 | import { ActorFollowModel } from '../../models/activitypub/actor-follow' | ||
13 | import { SERVER_ACTOR_NAME } from '../../initializers' | ||
14 | import { ServerModel } from '../../models/server/server' | ||
15 | |||
16 | const videoRedundancyGetValidator = [ | ||
17 | param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), | ||
18 | param('resolution') | ||
19 | .customSanitizer(toIntOrNull) | ||
20 | .custom(exists).withMessage('Should have a valid resolution'), | ||
21 | param('fps') | ||
22 | .optional() | ||
23 | .customSanitizer(toIntOrNull) | ||
24 | .custom(exists).withMessage('Should have a valid fps'), | ||
25 | |||
26 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
27 | logger.debug('Checking videoRedundancyGetValidator parameters', { parameters: req.params }) | ||
28 | |||
29 | if (areValidationErrors(req, res)) return | ||
30 | if (!await isVideoExist(req.params.videoId, res)) return | ||
31 | |||
32 | const video: VideoModel = res.locals.video | ||
33 | const videoFile = video.VideoFiles.find(f => { | ||
34 | return f.resolution === req.params.resolution && (!req.params.fps || f.fps === req.params.fps) | ||
35 | }) | ||
36 | |||
37 | if (!videoFile) return res.status(404).json({ error: 'Video file not found.' }) | ||
38 | res.locals.videoFile = videoFile | ||
39 | |||
40 | const videoRedundancy = await VideoRedundancyModel.loadByFileId(videoFile.id) | ||
41 | if (!videoRedundancy)return res.status(404).json({ error: 'Video redundancy not found.' }) | ||
42 | res.locals.videoRedundancy = videoRedundancy | ||
43 | |||
44 | return next() | ||
45 | } | ||
46 | ] | ||
47 | |||
48 | const updateServerRedundancyValidator = [ | ||
49 | param('host').custom(isHostValid).withMessage('Should have a valid host'), | ||
50 | body('redundancyAllowed') | ||
51 | .toBoolean() | ||
52 | .custom(isBooleanValid).withMessage('Should have a valid redundancyAllowed attribute'), | ||
53 | |||
54 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
55 | logger.debug('Checking updateServerRedundancy parameters', { parameters: req.params }) | ||
56 | |||
57 | if (areValidationErrors(req, res)) return | ||
58 | |||
59 | const server = await ServerModel.loadByHost(req.params.host) | ||
60 | |||
61 | if (!server) { | ||
62 | return res | ||
63 | .status(404) | ||
64 | .json({ | ||
65 | error: `Server ${req.params.host} not found.` | ||
66 | }) | ||
67 | .end() | ||
68 | } | ||
69 | |||
70 | res.locals.server = server | ||
71 | return next() | ||
72 | } | ||
73 | ] | ||
74 | |||
75 | // --------------------------------------------------------------------------- | ||
76 | |||
77 | export { | ||
78 | videoRedundancyGetValidator, | ||
79 | updateServerRedundancyValidator | ||
80 | } | ||
diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts index 8bc095997..27bb43dae 100644 --- a/server/models/activitypub/actor-follow.ts +++ b/server/models/activitypub/actor-follow.ts | |||
@@ -19,7 +19,7 @@ import { | |||
19 | UpdatedAt | 19 | UpdatedAt |
20 | } from 'sequelize-typescript' | 20 | } from 'sequelize-typescript' |
21 | import { FollowState } from '../../../shared/models/actors' | 21 | import { FollowState } from '../../../shared/models/actors' |
22 | import { AccountFollow } from '../../../shared/models/actors/follow.model' | 22 | import { ActorFollow } from '../../../shared/models/actors/follow.model' |
23 | import { logger } from '../../helpers/logger' | 23 | import { logger } from '../../helpers/logger' |
24 | import { getServerActor } from '../../helpers/utils' | 24 | import { getServerActor } from '../../helpers/utils' |
25 | import { ACTOR_FOLLOW_SCORE } from '../../initializers' | 25 | import { ACTOR_FOLLOW_SCORE } from '../../initializers' |
@@ -529,7 +529,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> { | |||
529 | return ActorFollowModel.findAll(query) | 529 | return ActorFollowModel.findAll(query) |
530 | } | 530 | } |
531 | 531 | ||
532 | toFormattedJSON (): AccountFollow { | 532 | toFormattedJSON (): ActorFollow { |
533 | const follower = this.ActorFollower.toFormattedJSON() | 533 | const follower = this.ActorFollower.toFormattedJSON() |
534 | const following = this.ActorFollowing.toFormattedJSON() | 534 | const following = this.ActorFollowing.toFormattedJSON() |
535 | 535 | ||
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index 119d0c1da..ef8dd9f7c 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts | |||
@@ -76,7 +76,13 @@ export const unusedActorAttributesForAPI = [ | |||
76 | }, | 76 | }, |
77 | { | 77 | { |
78 | model: () => VideoChannelModel.unscoped(), | 78 | model: () => VideoChannelModel.unscoped(), |
79 | required: false | 79 | required: false, |
80 | include: [ | ||
81 | { | ||
82 | model: () => AccountModel, | ||
83 | required: true | ||
84 | } | ||
85 | ] | ||
80 | }, | 86 | }, |
81 | { | 87 | { |
82 | model: () => ServerModel, | 88 | model: () => ServerModel, |
@@ -337,6 +343,7 @@ export class ActorModel extends Model<ActorModel> { | |||
337 | uuid: this.uuid, | 343 | uuid: this.uuid, |
338 | name: this.preferredUsername, | 344 | name: this.preferredUsername, |
339 | host: this.getHost(), | 345 | host: this.getHost(), |
346 | hostRedundancyAllowed: this.getRedundancyAllowed(), | ||
340 | followingCount: this.followingCount, | 347 | followingCount: this.followingCount, |
341 | followersCount: this.followersCount, | 348 | followersCount: this.followersCount, |
342 | avatar, | 349 | avatar, |
@@ -440,6 +447,10 @@ export class ActorModel extends Model<ActorModel> { | |||
440 | return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST | 447 | return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST |
441 | } | 448 | } |
442 | 449 | ||
450 | getRedundancyAllowed () { | ||
451 | return this.Server ? this.Server.redundancyAllowed : false | ||
452 | } | ||
453 | |||
443 | getAvatarUrl () { | 454 | getAvatarUrl () { |
444 | if (!this.avatarId) return undefined | 455 | if (!this.avatarId) return undefined |
445 | 456 | ||
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts new file mode 100644 index 000000000..48ec77206 --- /dev/null +++ b/server/models/redundancy/video-redundancy.ts | |||
@@ -0,0 +1,249 @@ | |||
1 | import { | ||
2 | AfterDestroy, | ||
3 | AllowNull, | ||
4 | BelongsTo, | ||
5 | Column, | ||
6 | CreatedAt, | ||
7 | DataType, | ||
8 | ForeignKey, | ||
9 | Is, | ||
10 | Model, | ||
11 | Scopes, | ||
12 | Sequelize, | ||
13 | Table, | ||
14 | UpdatedAt | ||
15 | } from 'sequelize-typescript' | ||
16 | import { ActorModel } from '../activitypub/actor' | ||
17 | import { throwIfNotValid } from '../utils' | ||
18 | import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' | ||
19 | import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers' | ||
20 | import { VideoFileModel } from '../video/video-file' | ||
21 | import { isDateValid } from '../../helpers/custom-validators/misc' | ||
22 | import { getServerActor } from '../../helpers/utils' | ||
23 | import { VideoModel } from '../video/video' | ||
24 | import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' | ||
25 | import { logger } from '../../helpers/logger' | ||
26 | import { CacheFileObject } from '../../../shared' | ||
27 | import { VideoChannelModel } from '../video/video-channel' | ||
28 | import { ServerModel } from '../server/server' | ||
29 | import { sample } from 'lodash' | ||
30 | import { isTestInstance } from '../../helpers/core-utils' | ||
31 | |||
32 | export enum ScopeNames { | ||
33 | WITH_VIDEO = 'WITH_VIDEO' | ||
34 | } | ||
35 | |||
36 | @Scopes({ | ||
37 | [ ScopeNames.WITH_VIDEO ]: { | ||
38 | include: [ | ||
39 | { | ||
40 | model: () => VideoFileModel, | ||
41 | required: true, | ||
42 | include: [ | ||
43 | { | ||
44 | model: () => VideoModel, | ||
45 | required: true | ||
46 | } | ||
47 | ] | ||
48 | } | ||
49 | ] | ||
50 | } | ||
51 | }) | ||
52 | |||
53 | @Table({ | ||
54 | tableName: 'videoRedundancy', | ||
55 | indexes: [ | ||
56 | { | ||
57 | fields: [ 'videoFileId' ] | ||
58 | }, | ||
59 | { | ||
60 | fields: [ 'actorId' ] | ||
61 | }, | ||
62 | { | ||
63 | fields: [ 'url' ], | ||
64 | unique: true | ||
65 | } | ||
66 | ] | ||
67 | }) | ||
68 | export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | ||
69 | |||
70 | @CreatedAt | ||
71 | createdAt: Date | ||
72 | |||
73 | @UpdatedAt | ||
74 | updatedAt: Date | ||
75 | |||
76 | @AllowNull(false) | ||
77 | @Column | ||
78 | expiresOn: Date | ||
79 | |||
80 | @AllowNull(false) | ||
81 | @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl')) | ||
82 | @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max)) | ||
83 | fileUrl: string | ||
84 | |||
85 | @AllowNull(false) | ||
86 | @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) | ||
87 | @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max)) | ||
88 | url: string | ||
89 | |||
90 | @AllowNull(true) | ||
91 | @Column | ||
92 | strategy: string // Only used by us | ||
93 | |||
94 | @ForeignKey(() => VideoFileModel) | ||
95 | @Column | ||
96 | videoFileId: number | ||
97 | |||
98 | @BelongsTo(() => VideoFileModel, { | ||
99 | foreignKey: { | ||
100 | allowNull: false | ||
101 | }, | ||
102 | onDelete: 'cascade' | ||
103 | }) | ||
104 | VideoFile: VideoFileModel | ||
105 | |||
106 | @ForeignKey(() => ActorModel) | ||
107 | @Column | ||
108 | actorId: number | ||
109 | |||
110 | @BelongsTo(() => ActorModel, { | ||
111 | foreignKey: { | ||
112 | allowNull: false | ||
113 | }, | ||
114 | onDelete: 'cascade' | ||
115 | }) | ||
116 | Actor: ActorModel | ||
117 | |||
118 | @AfterDestroy | ||
119 | static removeFilesAndSendDelete (instance: VideoRedundancyModel) { | ||
120 | // Not us | ||
121 | if (!instance.strategy) return | ||
122 | |||
123 | logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution) | ||
124 | |||
125 | return instance.VideoFile.Video.removeFile(instance.VideoFile) | ||
126 | } | ||
127 | |||
128 | static loadByFileId (videoFileId: number) { | ||
129 | const query = { | ||
130 | where: { | ||
131 | videoFileId | ||
132 | } | ||
133 | } | ||
134 | |||
135 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) | ||
136 | } | ||
137 | |||
138 | static loadByUrl (url: string) { | ||
139 | const query = { | ||
140 | where: { | ||
141 | url | ||
142 | } | ||
143 | } | ||
144 | |||
145 | return VideoRedundancyModel.findOne(query) | ||
146 | } | ||
147 | |||
148 | static async findMostViewToDuplicate (randomizedFactor: number) { | ||
149 | // On VideoModel! | ||
150 | const query = { | ||
151 | logging: !isTestInstance(), | ||
152 | limit: randomizedFactor, | ||
153 | order: [ [ 'views', 'DESC' ] ], | ||
154 | include: [ | ||
155 | { | ||
156 | model: VideoFileModel.unscoped(), | ||
157 | required: true, | ||
158 | where: { | ||
159 | id: { | ||
160 | [ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn() | ||
161 | } | ||
162 | } | ||
163 | }, | ||
164 | { | ||
165 | attributes: [], | ||
166 | model: VideoChannelModel.unscoped(), | ||
167 | required: true, | ||
168 | include: [ | ||
169 | { | ||
170 | attributes: [], | ||
171 | model: ActorModel.unscoped(), | ||
172 | required: true, | ||
173 | include: [ | ||
174 | { | ||
175 | attributes: [], | ||
176 | model: ServerModel.unscoped(), | ||
177 | required: true, | ||
178 | where: { | ||
179 | redundancyAllowed: true | ||
180 | } | ||
181 | } | ||
182 | ] | ||
183 | } | ||
184 | ] | ||
185 | } | ||
186 | ] | ||
187 | } | ||
188 | |||
189 | const rows = await VideoModel.unscoped().findAll(query) | ||
190 | |||
191 | return sample(rows) | ||
192 | } | ||
193 | |||
194 | static async getVideoFiles (strategy: VideoRedundancyStrategy) { | ||
195 | const actor = await getServerActor() | ||
196 | |||
197 | const queryVideoFiles = { | ||
198 | logging: !isTestInstance(), | ||
199 | where: { | ||
200 | actorId: actor.id, | ||
201 | strategy | ||
202 | } | ||
203 | } | ||
204 | |||
205 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) | ||
206 | .findAll(queryVideoFiles) | ||
207 | } | ||
208 | |||
209 | static listAllExpired () { | ||
210 | const query = { | ||
211 | logging: !isTestInstance(), | ||
212 | where: { | ||
213 | expiresOn: { | ||
214 | [Sequelize.Op.lt]: new Date() | ||
215 | } | ||
216 | } | ||
217 | } | ||
218 | |||
219 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) | ||
220 | .findAll(query) | ||
221 | } | ||
222 | |||
223 | toActivityPubObject (): CacheFileObject { | ||
224 | return { | ||
225 | id: this.url, | ||
226 | type: 'CacheFile' as 'CacheFile', | ||
227 | object: this.VideoFile.Video.url, | ||
228 | expires: this.expiresOn.toISOString(), | ||
229 | url: { | ||
230 | type: 'Link', | ||
231 | mimeType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any, | ||
232 | href: this.fileUrl, | ||
233 | height: this.VideoFile.resolution, | ||
234 | size: this.VideoFile.size, | ||
235 | fps: this.VideoFile.fps | ||
236 | } | ||
237 | } | ||
238 | } | ||
239 | |||
240 | private static async buildExcludeIn () { | ||
241 | const actor = await getServerActor() | ||
242 | |||
243 | return Sequelize.literal( | ||
244 | '(' + | ||
245 | `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` + | ||
246 | ')' | ||
247 | ) | ||
248 | } | ||
249 | } | ||
diff --git a/server/models/server/server.ts b/server/models/server/server.ts index 9749f503e..ca3b24d51 100644 --- a/server/models/server/server.ts +++ b/server/models/server/server.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { AllowNull, Column, CreatedAt, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' | 1 | import { AllowNull, Column, CreatedAt, Default, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' |
2 | import { isHostValid } from '../../helpers/custom-validators/servers' | 2 | import { isHostValid } from '../../helpers/custom-validators/servers' |
3 | import { ActorModel } from '../activitypub/actor' | 3 | import { ActorModel } from '../activitypub/actor' |
4 | import { throwIfNotValid } from '../utils' | 4 | import { throwIfNotValid } from '../utils' |
@@ -19,6 +19,11 @@ export class ServerModel extends Model<ServerModel> { | |||
19 | @Column | 19 | @Column |
20 | host: string | 20 | host: string |
21 | 21 | ||
22 | @AllowNull(false) | ||
23 | @Default(false) | ||
24 | @Column | ||
25 | redundancyAllowed: boolean | ||
26 | |||
22 | @CreatedAt | 27 | @CreatedAt |
23 | createdAt: Date | 28 | createdAt: Date |
24 | 29 | ||
@@ -34,4 +39,14 @@ export class ServerModel extends Model<ServerModel> { | |||
34 | hooks: true | 39 | hooks: true |
35 | }) | 40 | }) |
36 | Actors: ActorModel[] | 41 | Actors: ActorModel[] |
42 | |||
43 | static loadByHost (host: string) { | ||
44 | const query = { | ||
45 | where: { | ||
46 | host | ||
47 | } | ||
48 | } | ||
49 | |||
50 | return ServerModel.findOne(query) | ||
51 | } | ||
37 | } | 52 | } |
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 3bc4855f3..0907ea569 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts | |||
@@ -1,5 +1,18 @@ | |||
1 | import { values } from 'lodash' | 1 | import { values } from 'lodash' |
2 | import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' | 2 | import { |
3 | AllowNull, | ||
4 | BelongsTo, | ||
5 | Column, | ||
6 | CreatedAt, | ||
7 | DataType, | ||
8 | Default, | ||
9 | ForeignKey, | ||
10 | HasMany, | ||
11 | Is, | ||
12 | Model, | ||
13 | Table, | ||
14 | UpdatedAt | ||
15 | } from 'sequelize-typescript' | ||
3 | import { | 16 | import { |
4 | isVideoFileInfoHashValid, | 17 | isVideoFileInfoHashValid, |
5 | isVideoFileResolutionValid, | 18 | isVideoFileResolutionValid, |
@@ -10,6 +23,7 @@ import { CONSTRAINTS_FIELDS } from '../../initializers' | |||
10 | import { throwIfNotValid } from '../utils' | 23 | import { throwIfNotValid } from '../utils' |
11 | import { VideoModel } from './video' | 24 | import { VideoModel } from './video' |
12 | import * as Sequelize from 'sequelize' | 25 | import * as Sequelize from 'sequelize' |
26 | import { VideoRedundancyModel } from '../redundancy/video-redundancy' | ||
13 | 27 | ||
14 | @Table({ | 28 | @Table({ |
15 | tableName: 'videoFile', | 29 | tableName: 'videoFile', |
@@ -70,6 +84,15 @@ export class VideoFileModel extends Model<VideoFileModel> { | |||
70 | }) | 84 | }) |
71 | Video: VideoModel | 85 | Video: VideoModel |
72 | 86 | ||
87 | @HasMany(() => VideoRedundancyModel, { | ||
88 | foreignKey: { | ||
89 | allowNull: false | ||
90 | }, | ||
91 | onDelete: 'CASCADE', | ||
92 | hooks: true | ||
93 | }) | ||
94 | RedundancyVideos: VideoRedundancyModel[] | ||
95 | |||
73 | static isInfohashExists (infoHash: string) { | 96 | static isInfohashExists (infoHash: string) { |
74 | const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' | 97 | const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' |
75 | const options = { | 98 | const options = { |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 86316653f..27c631dcd 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -27,13 +27,13 @@ import { | |||
27 | Table, | 27 | Table, |
28 | UpdatedAt | 28 | UpdatedAt |
29 | } from 'sequelize-typescript' | 29 | } from 'sequelize-typescript' |
30 | import { VideoPrivacy, VideoResolution, VideoState } from '../../../shared' | 30 | import { ActivityUrlObject, VideoPrivacy, VideoResolution, VideoState } from '../../../shared' |
31 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 31 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
32 | import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' | 32 | import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' |
33 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' | 33 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' |
34 | import { createTorrentPromise, peertubeTruncate } from '../../helpers/core-utils' | 34 | import { createTorrentPromise, peertubeTruncate } from '../../helpers/core-utils' |
35 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | 35 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' |
36 | import { isBooleanValid } from '../../helpers/custom-validators/misc' | 36 | import { isArray, isBooleanValid } from '../../helpers/custom-validators/misc' |
37 | import { | 37 | import { |
38 | isVideoCategoryValid, | 38 | isVideoCategoryValid, |
39 | isVideoDescriptionValid, | 39 | isVideoDescriptionValid, |
@@ -90,6 +90,7 @@ import { VideoCaptionModel } from './video-caption' | |||
90 | import { VideoBlacklistModel } from './video-blacklist' | 90 | import { VideoBlacklistModel } from './video-blacklist' |
91 | import { copy, remove, rename, stat, writeFile } from 'fs-extra' | 91 | import { copy, remove, rename, stat, writeFile } from 'fs-extra' |
92 | import { VideoViewModel } from './video-views' | 92 | import { VideoViewModel } from './video-views' |
93 | import { VideoRedundancyModel } from '../redundancy/video-redundancy' | ||
93 | 94 | ||
94 | // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation | 95 | // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation |
95 | const indexes: Sequelize.DefineIndexesOptions[] = [ | 96 | const indexes: Sequelize.DefineIndexesOptions[] = [ |
@@ -470,7 +471,13 @@ type AvailableForListIDsOptions = { | |||
470 | include: [ | 471 | include: [ |
471 | { | 472 | { |
472 | model: () => VideoFileModel.unscoped(), | 473 | model: () => VideoFileModel.unscoped(), |
473 | required: false | 474 | required: false, |
475 | include: [ | ||
476 | { | ||
477 | model: () => VideoRedundancyModel.unscoped(), | ||
478 | required: false | ||
479 | } | ||
480 | ] | ||
474 | } | 481 | } |
475 | ] | 482 | ] |
476 | }, | 483 | }, |
@@ -633,6 +640,7 @@ export class VideoModel extends Model<VideoModel> { | |||
633 | name: 'videoId', | 640 | name: 'videoId', |
634 | allowNull: false | 641 | allowNull: false |
635 | }, | 642 | }, |
643 | hooks: true, | ||
636 | onDelete: 'cascade' | 644 | onDelete: 'cascade' |
637 | }) | 645 | }) |
638 | VideoFiles: VideoFileModel[] | 646 | VideoFiles: VideoFileModel[] |
@@ -1325,9 +1333,7 @@ export class VideoModel extends Model<VideoModel> { | |||
1325 | [ CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + '/tracker/socket' ], | 1333 | [ CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + '/tracker/socket' ], |
1326 | [ CONFIG.WEBSERVER.URL + '/tracker/announce' ] | 1334 | [ CONFIG.WEBSERVER.URL + '/tracker/announce' ] |
1327 | ], | 1335 | ], |
1328 | urlList: [ | 1336 | urlList: [ CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) ] |
1329 | CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) | ||
1330 | ] | ||
1331 | } | 1337 | } |
1332 | 1338 | ||
1333 | const torrent = await createTorrentPromise(this.getVideoFilePath(videoFile), options) | 1339 | const torrent = await createTorrentPromise(this.getVideoFilePath(videoFile), options) |
@@ -1535,11 +1541,11 @@ export class VideoModel extends Model<VideoModel> { | |||
1535 | } | 1541 | } |
1536 | } | 1542 | } |
1537 | 1543 | ||
1538 | const url = [] | 1544 | const url: ActivityUrlObject[] = [] |
1539 | for (const file of this.VideoFiles) { | 1545 | for (const file of this.VideoFiles) { |
1540 | url.push({ | 1546 | url.push({ |
1541 | type: 'Link', | 1547 | type: 'Link', |
1542 | mimeType: VIDEO_EXT_MIMETYPE[ file.extname ], | 1548 | mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, |
1543 | href: this.getVideoFileUrl(file, baseUrlHttp), | 1549 | href: this.getVideoFileUrl(file, baseUrlHttp), |
1544 | height: file.resolution, | 1550 | height: file.resolution, |
1545 | size: file.size, | 1551 | size: file.size, |
@@ -1548,14 +1554,14 @@ export class VideoModel extends Model<VideoModel> { | |||
1548 | 1554 | ||
1549 | url.push({ | 1555 | url.push({ |
1550 | type: 'Link', | 1556 | type: 'Link', |
1551 | mimeType: 'application/x-bittorrent', | 1557 | mimeType: 'application/x-bittorrent' as 'application/x-bittorrent', |
1552 | href: this.getTorrentUrl(file, baseUrlHttp), | 1558 | href: this.getTorrentUrl(file, baseUrlHttp), |
1553 | height: file.resolution | 1559 | height: file.resolution |
1554 | }) | 1560 | }) |
1555 | 1561 | ||
1556 | url.push({ | 1562 | url.push({ |
1557 | type: 'Link', | 1563 | type: 'Link', |
1558 | mimeType: 'application/x-bittorrent;x-scheme-handler/magnet', | 1564 | mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', |
1559 | href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs), | 1565 | href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs), |
1560 | height: file.resolution | 1566 | height: file.resolution |
1561 | }) | 1567 | }) |
@@ -1796,7 +1802,7 @@ export class VideoModel extends Model<VideoModel> { | |||
1796 | (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL | 1802 | (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL |
1797 | } | 1803 | } |
1798 | 1804 | ||
1799 | private getBaseUrls () { | 1805 | getBaseUrls () { |
1800 | let baseUrlHttp | 1806 | let baseUrlHttp |
1801 | let baseUrlWs | 1807 | let baseUrlWs |
1802 | 1808 | ||
@@ -1811,39 +1817,42 @@ export class VideoModel extends Model<VideoModel> { | |||
1811 | return { baseUrlHttp, baseUrlWs } | 1817 | return { baseUrlHttp, baseUrlWs } |
1812 | } | 1818 | } |
1813 | 1819 | ||
1814 | private getThumbnailUrl (baseUrlHttp: string) { | 1820 | generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { |
1821 | const xs = this.getTorrentUrl(videoFile, baseUrlHttp) | ||
1822 | const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] | ||
1823 | let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] | ||
1824 | |||
1825 | const redundancies = videoFile.RedundancyVideos | ||
1826 | if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) | ||
1827 | |||
1828 | const magnetHash = { | ||
1829 | xs, | ||
1830 | announce, | ||
1831 | urlList, | ||
1832 | infoHash: videoFile.infoHash, | ||
1833 | name: this.name | ||
1834 | } | ||
1835 | |||
1836 | return magnetUtil.encode(magnetHash) | ||
1837 | } | ||
1838 | |||
1839 | getThumbnailUrl (baseUrlHttp: string) { | ||
1815 | return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() | 1840 | return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() |
1816 | } | 1841 | } |
1817 | 1842 | ||
1818 | private getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) { | 1843 | getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) { |
1819 | return baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentFileName(videoFile) | 1844 | return baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentFileName(videoFile) |
1820 | } | 1845 | } |
1821 | 1846 | ||
1822 | private getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { | 1847 | getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { |
1823 | return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + this.getTorrentFileName(videoFile) | 1848 | return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + this.getTorrentFileName(videoFile) |
1824 | } | 1849 | } |
1825 | 1850 | ||
1826 | private getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) { | 1851 | getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) { |
1827 | return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) | 1852 | return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) |
1828 | } | 1853 | } |
1829 | 1854 | ||
1830 | private getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { | 1855 | getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { |
1831 | return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) | 1856 | return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) |
1832 | } | 1857 | } |
1833 | |||
1834 | private generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { | ||
1835 | const xs = this.getTorrentUrl(videoFile, baseUrlHttp) | ||
1836 | const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] | ||
1837 | const urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] | ||
1838 | |||
1839 | const magnetHash = { | ||
1840 | xs, | ||
1841 | announce, | ||
1842 | urlList, | ||
1843 | infoHash: videoFile.infoHash, | ||
1844 | name: this.name | ||
1845 | } | ||
1846 | |||
1847 | return magnetUtil.encode(magnetHash) | ||
1848 | } | ||
1849 | } | 1858 | } |
diff --git a/server/tests/api/check-params/follows.ts b/server/tests/api/check-params/follows.ts index 2bc3b27d9..cdc95c81a 100644 --- a/server/tests/api/check-params/follows.ts +++ b/server/tests/api/check-params/follows.ts | |||
@@ -169,15 +169,6 @@ describe('Test server follows API validators', function () { | |||
169 | statusCodeExpected: 404 | 169 | statusCodeExpected: 404 |
170 | }) | 170 | }) |
171 | }) | 171 | }) |
172 | |||
173 | it('Should succeed with the correct parameters', async function () { | ||
174 | await makeDeleteRequest({ | ||
175 | url: server.url, | ||
176 | path: path + '/localhost:9002', | ||
177 | token: server.accessToken, | ||
178 | statusCodeExpected: 404 | ||
179 | }) | ||
180 | }) | ||
181 | }) | 172 | }) |
182 | }) | 173 | }) |
183 | 174 | ||
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index 777acbb0f..44460a167 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts | |||
@@ -1,15 +1,17 @@ | |||
1 | // Order of the tests we want to execute | 1 | // Order of the tests we want to execute |
2 | import './accounts' | 2 | import './accounts' |
3 | import './config' | ||
3 | import './follows' | 4 | import './follows' |
4 | import './jobs' | 5 | import './jobs' |
6 | import './redundancy' | ||
7 | import './search' | ||
5 | import './services' | 8 | import './services' |
9 | import './user-subscriptions' | ||
6 | import './users' | 10 | import './users' |
7 | import './video-abuses' | 11 | import './video-abuses' |
8 | import './video-blacklist' | 12 | import './video-blacklist' |
9 | import './video-captions' | 13 | import './video-captions' |
10 | import './video-channels' | 14 | import './video-channels' |
11 | import './video-comments' | 15 | import './video-comments' |
12 | import './videos' | ||
13 | import './video-imports' | 16 | import './video-imports' |
14 | import './search' | 17 | import './videos' |
15 | import './user-subscriptions' | ||
diff --git a/server/tests/api/check-params/redundancy.ts b/server/tests/api/check-params/redundancy.ts new file mode 100644 index 000000000..aa588e3dd --- /dev/null +++ b/server/tests/api/check-params/redundancy.ts | |||
@@ -0,0 +1,103 @@ | |||
1 | /* tslint:disable:no-unused-expression */ | ||
2 | |||
3 | import 'mocha' | ||
4 | |||
5 | import { | ||
6 | createUser, | ||
7 | doubleFollow, | ||
8 | flushAndRunMultipleServers, | ||
9 | flushTests, | ||
10 | killallServers, | ||
11 | makePutBodyRequest, | ||
12 | ServerInfo, | ||
13 | setAccessTokensToServers, | ||
14 | userLogin | ||
15 | } from '../../utils' | ||
16 | |||
17 | describe('Test server redundancy API validators', function () { | ||
18 | let servers: ServerInfo[] | ||
19 | let userAccessToken = null | ||
20 | |||
21 | // --------------------------------------------------------------- | ||
22 | |||
23 | before(async function () { | ||
24 | this.timeout(30000) | ||
25 | |||
26 | await flushTests() | ||
27 | servers = await flushAndRunMultipleServers(2) | ||
28 | |||
29 | await setAccessTokensToServers(servers) | ||
30 | await doubleFollow(servers[0], servers[1]) | ||
31 | |||
32 | const user = { | ||
33 | username: 'user1', | ||
34 | password: 'password' | ||
35 | } | ||
36 | |||
37 | await createUser(servers[0].url, servers[0].accessToken, user.username, user.password) | ||
38 | userAccessToken = await userLogin(servers[0], user) | ||
39 | }) | ||
40 | |||
41 | describe('When updating redundancy', function () { | ||
42 | const path = '/api/v1/server/redundancy' | ||
43 | |||
44 | it('Should fail with an invalid token', async function () { | ||
45 | await makePutBodyRequest({ | ||
46 | url: servers[0].url, | ||
47 | path: path + '/localhost:9002', | ||
48 | fields: { redundancyAllowed: true }, | ||
49 | token: 'fake_token', | ||
50 | statusCodeExpected: 401 | ||
51 | }) | ||
52 | }) | ||
53 | |||
54 | it('Should fail if the user is not an administrator', async function () { | ||
55 | await makePutBodyRequest({ | ||
56 | url: servers[0].url, | ||
57 | path: path + '/localhost:9002', | ||
58 | fields: { redundancyAllowed: true }, | ||
59 | token: userAccessToken, | ||
60 | statusCodeExpected: 403 | ||
61 | }) | ||
62 | }) | ||
63 | |||
64 | it('Should fail if we do not follow this server', async function () { | ||
65 | await makePutBodyRequest({ | ||
66 | url: servers[0].url, | ||
67 | path: path + '/example.com', | ||
68 | fields: { redundancyAllowed: true }, | ||
69 | token: servers[0].accessToken, | ||
70 | statusCodeExpected: 404 | ||
71 | }) | ||
72 | }) | ||
73 | |||
74 | it('Should fail without de redundancyAllowed param', async function () { | ||
75 | await makePutBodyRequest({ | ||
76 | url: servers[0].url, | ||
77 | path: path + '/localhost:9002', | ||
78 | fields: { blabla: true }, | ||
79 | token: servers[0].accessToken, | ||
80 | statusCodeExpected: 400 | ||
81 | }) | ||
82 | }) | ||
83 | |||
84 | it('Should succeed with the correct parameters', async function () { | ||
85 | await makePutBodyRequest({ | ||
86 | url: servers[0].url, | ||
87 | path: path + '/localhost:9002', | ||
88 | fields: { redundancyAllowed: true }, | ||
89 | token: servers[0].accessToken, | ||
90 | statusCodeExpected: 204 | ||
91 | }) | ||
92 | }) | ||
93 | }) | ||
94 | |||
95 | after(async function () { | ||
96 | killallServers(servers) | ||
97 | |||
98 | // Keep the logs if the test failed | ||
99 | if (this['ok']) { | ||
100 | await flushTests() | ||
101 | } | ||
102 | }) | ||
103 | }) | ||
diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts index eeb8b7a28..c74c68a33 100644 --- a/server/tests/api/server/index.ts +++ b/server/tests/api/server/index.ts | |||
@@ -3,6 +3,7 @@ import './email' | |||
3 | import './follows' | 3 | import './follows' |
4 | import './handle-down' | 4 | import './handle-down' |
5 | import './jobs' | 5 | import './jobs' |
6 | import './redundancy' | ||
6 | import './reverse-proxy' | 7 | import './reverse-proxy' |
7 | import './stats' | 8 | import './stats' |
8 | import './tracker' | 9 | import './tracker' |
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts new file mode 100644 index 000000000..c0ec75a45 --- /dev/null +++ b/server/tests/api/server/redundancy.ts | |||
@@ -0,0 +1,140 @@ | |||
1 | /* tslint:disable:no-unused-expression */ | ||
2 | |||
3 | import * as chai from 'chai' | ||
4 | import 'mocha' | ||
5 | import { VideoDetails } from '../../../../shared/models/videos' | ||
6 | import { | ||
7 | doubleFollow, | ||
8 | flushAndRunMultipleServers, | ||
9 | flushTests, | ||
10 | getFollowingListPaginationAndSort, | ||
11 | getVideo, | ||
12 | killallServers, | ||
13 | ServerInfo, | ||
14 | setAccessTokensToServers, | ||
15 | uploadVideo, | ||
16 | wait, | ||
17 | root, viewVideo | ||
18 | } from '../../utils' | ||
19 | import { waitJobs } from '../../utils/server/jobs' | ||
20 | import * as magnetUtil from 'magnet-uri' | ||
21 | import { updateRedundancy } from '../../utils/server/redundancy' | ||
22 | import { ActorFollow } from '../../../../shared/models/actors' | ||
23 | import { readdir } from 'fs-extra' | ||
24 | import { join } from 'path' | ||
25 | |||
26 | const expect = chai.expect | ||
27 | |||
28 | function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) { | ||
29 | const parsed = magnetUtil.decode(file.magnetUri) | ||
30 | |||
31 | for (const ws of baseWebseeds) { | ||
32 | const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) | ||
33 | expect(found, `Webseed ${ws} not found in ${file.magnetUri}`).to.not.be.undefined | ||
34 | } | ||
35 | } | ||
36 | |||
37 | describe('Test videos redundancy', function () { | ||
38 | let servers: ServerInfo[] = [] | ||
39 | let video1Server2UUID: string | ||
40 | let video2Server2UUID: string | ||
41 | |||
42 | before(async function () { | ||
43 | this.timeout(120000) | ||
44 | |||
45 | servers = await flushAndRunMultipleServers(3) | ||
46 | |||
47 | // Get the access tokens | ||
48 | await setAccessTokensToServers(servers) | ||
49 | |||
50 | { | ||
51 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) | ||
52 | video1Server2UUID = res.body.video.uuid | ||
53 | |||
54 | await viewVideo(servers[1].url, video1Server2UUID) | ||
55 | } | ||
56 | |||
57 | { | ||
58 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) | ||
59 | video2Server2UUID = res.body.video.uuid | ||
60 | } | ||
61 | |||
62 | await waitJobs(servers) | ||
63 | |||
64 | // Server 1 and server 2 follow each other | ||
65 | await doubleFollow(servers[0], servers[1]) | ||
66 | // Server 1 and server 3 follow each other | ||
67 | await doubleFollow(servers[0], servers[2]) | ||
68 | // Server 2 and server 3 follow each other | ||
69 | await doubleFollow(servers[1], servers[2]) | ||
70 | |||
71 | await waitJobs(servers) | ||
72 | }) | ||
73 | |||
74 | it('Should have 1 webseed on the first video', async function () { | ||
75 | const webseeds = [ | ||
76 | 'http://localhost:9002/static/webseed/' + video1Server2UUID | ||
77 | ] | ||
78 | |||
79 | for (const server of servers) { | ||
80 | const res = await getVideo(server.url, video1Server2UUID) | ||
81 | |||
82 | const video: VideoDetails = res.body | ||
83 | video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) | ||
84 | } | ||
85 | }) | ||
86 | |||
87 | it('Should enable redundancy on server 1', async function () { | ||
88 | await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true) | ||
89 | |||
90 | const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt') | ||
91 | const follows: ActorFollow[] = res.body.data | ||
92 | const server2 = follows.find(f => f.following.host === 'localhost:9002') | ||
93 | const server3 = follows.find(f => f.following.host === 'localhost:9003') | ||
94 | |||
95 | expect(server3).to.not.be.undefined | ||
96 | expect(server3.following.hostRedundancyAllowed).to.be.false | ||
97 | |||
98 | expect(server2).to.not.be.undefined | ||
99 | expect(server2.following.hostRedundancyAllowed).to.be.true | ||
100 | }) | ||
101 | |||
102 | it('Should have 2 webseed on the first video', async function () { | ||
103 | this.timeout(40000) | ||
104 | |||
105 | await waitJobs(servers) | ||
106 | await wait(15000) | ||
107 | await waitJobs(servers) | ||
108 | |||
109 | const webseeds = [ | ||
110 | 'http://localhost:9001/static/webseed/' + video1Server2UUID, | ||
111 | 'http://localhost:9002/static/webseed/' + video1Server2UUID | ||
112 | ] | ||
113 | |||
114 | for (const server of servers) { | ||
115 | const res = await getVideo(server.url, video1Server2UUID) | ||
116 | |||
117 | const video: VideoDetails = res.body | ||
118 | |||
119 | for (const file of video.files) { | ||
120 | checkMagnetWebseeds(file, webseeds) | ||
121 | } | ||
122 | } | ||
123 | |||
124 | const files = await readdir(join(root(), 'test1', 'videos')) | ||
125 | expect(files).to.have.lengthOf(4) | ||
126 | |||
127 | for (const resolution of [ 240, 360, 480, 720 ]) { | ||
128 | expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined | ||
129 | } | ||
130 | }) | ||
131 | |||
132 | after(async function () { | ||
133 | killallServers(servers) | ||
134 | |||
135 | // Keep the logs if the test failed | ||
136 | if (this['ok']) { | ||
137 | await flushTests() | ||
138 | } | ||
139 | }) | ||
140 | }) | ||
diff --git a/server/tests/utils/server/follows.ts b/server/tests/utils/server/follows.ts index d21fb5e58..8a65a958b 100644 --- a/server/tests/utils/server/follows.ts +++ b/server/tests/utils/server/follows.ts | |||
@@ -1,5 +1,4 @@ | |||
1 | import * as request from 'supertest' | 1 | import * as request from 'supertest' |
2 | import { wait } from '../miscs/miscs' | ||
3 | import { ServerInfo } from './servers' | 2 | import { ServerInfo } from './servers' |
4 | import { waitJobs } from './jobs' | 3 | import { waitJobs } from './jobs' |
5 | 4 | ||
diff --git a/server/tests/utils/server/redundancy.ts b/server/tests/utils/server/redundancy.ts new file mode 100644 index 000000000..c39ff2c8b --- /dev/null +++ b/server/tests/utils/server/redundancy.ts | |||
@@ -0,0 +1,17 @@ | |||
1 | import { makePutBodyRequest } from '../requests/requests' | ||
2 | |||
3 | async function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) { | ||
4 | const path = '/api/v1/server/redundancy/' + host | ||
5 | |||
6 | return makePutBodyRequest({ | ||
7 | url, | ||
8 | path, | ||
9 | token: accessToken, | ||
10 | fields: { redundancyAllowed }, | ||
11 | statusCodeExpected: expectedStatus | ||
12 | }) | ||
13 | } | ||
14 | |||
15 | export { | ||
16 | updateRedundancy | ||
17 | } | ||