aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-09-11 16:27:07 +0200
committerChocobozzz <me@florianbigard.com>2018-09-13 14:05:49 +0200
commitc48e82b5e0478434de30626d14594a97f2402e7c (patch)
treea78e5272bd0fe4f5b41831e571e02d05f1515b82 /server
parenta651038487faa838bda3ce04695b08bc65baff70 (diff)
downloadPeerTube-c48e82b5e0478434de30626d14594a97f2402e7c.tar.gz
PeerTube-c48e82b5e0478434de30626d14594a97f2402e7c.tar.zst
PeerTube-c48e82b5e0478434de30626d14594a97f2402e7c.zip
Basic video redundancy implementation
Diffstat (limited to 'server')
-rw-r--r--server/controllers/activitypub/client.ts34
-rw-r--r--server/controllers/activitypub/outbox.ts6
-rw-r--r--server/controllers/api/search.ts2
-rw-r--r--server/controllers/api/server/follows.ts5
-rw-r--r--server/controllers/api/server/index.ts2
-rw-r--r--server/controllers/api/server/redundancy.ts32
-rw-r--r--server/controllers/api/videos/abuse.ts2
-rw-r--r--server/helpers/activitypub.ts28
-rw-r--r--server/helpers/custom-validators/activitypub/activity.ts14
-rw-r--r--server/helpers/custom-validators/activitypub/cache-file.ts28
-rw-r--r--server/helpers/custom-validators/activitypub/misc.ts10
-rw-r--r--server/helpers/custom-validators/activitypub/undo.ts4
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts49
-rw-r--r--server/helpers/webtorrent.ts61
-rw-r--r--server/initializers/checker.ts17
-rw-r--r--server/initializers/constants.ts36
-rw-r--r--server/initializers/database.ts4
-rw-r--r--server/initializers/migrations/0270-server-redundancy.ts24
-rw-r--r--server/lib/activitypub/actor.ts6
-rw-r--r--server/lib/activitypub/cache-file.ts47
-rw-r--r--server/lib/activitypub/process/process-create.ts21
-rw-r--r--server/lib/activitypub/process/process-undo.ts44
-rw-r--r--server/lib/activitypub/process/process-update.ts34
-rw-r--r--server/lib/activitypub/send/send-accept.ts8
-rw-r--r--server/lib/activitypub/send/send-announce.ts33
-rw-r--r--server/lib/activitypub/send/send-create.ts68
-rw-r--r--server/lib/activitypub/send/send-delete.ts25
-rw-r--r--server/lib/activitypub/send/send-follow.ts6
-rw-r--r--server/lib/activitypub/send/send-like.ts10
-rw-r--r--server/lib/activitypub/send/send-undo.ts63
-rw-r--r--server/lib/activitypub/send/send-update.ts38
-rw-r--r--server/lib/activitypub/send/utils.ts8
-rw-r--r--server/lib/activitypub/url.ts10
-rw-r--r--server/lib/activitypub/videos.ts32
-rw-r--r--server/lib/redundancy.ts18
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts161
-rw-r--r--server/middlewares/validators/redundancy.ts80
-rw-r--r--server/models/activitypub/actor-follow.ts4
-rw-r--r--server/models/activitypub/actor.ts13
-rw-r--r--server/models/redundancy/video-redundancy.ts249
-rw-r--r--server/models/server/server.ts17
-rw-r--r--server/models/video/video-file.ts25
-rw-r--r--server/models/video/video.ts73
-rw-r--r--server/tests/api/check-params/follows.ts9
-rw-r--r--server/tests/api/check-params/index.ts8
-rw-r--r--server/tests/api/check-params/redundancy.ts103
-rw-r--r--server/tests/api/server/index.ts1
-rw-r--r--server/tests/api/server/redundancy.ts140
-rw-r--r--server/tests/utils/server/follows.ts1
-rw-r--r--server/tests/utils/server/redundancy.ts17
50 files changed, 1471 insertions, 259 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts
index 54cf44419..2e168ea78 100644
--- a/server/controllers/activitypub/client.ts
+++ b/server/controllers/activitypub/client.ts
@@ -3,9 +3,9 @@ import * as express from 'express'
3import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos' 3import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos'
4import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub' 4import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub'
5import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers' 5import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers'
6import { buildVideoAnnounce } from '../../lib/activitypub/send' 6import { buildAnnounceWithVideoAudience } from '../../lib/activitypub/send'
7import { audiencify, getAudience } from '../../lib/activitypub/audience' 7import { audiencify, getAudience } from '../../lib/activitypub/audience'
8import { createActivityData } from '../../lib/activitypub/send/send-create' 8import { buildCreateActivity } from '../../lib/activitypub/send/send-create'
9import { asyncMiddleware, executeIfActivityPub, localAccountValidator, localVideoChannelValidator } from '../../middlewares' 9import { asyncMiddleware, executeIfActivityPub, localAccountValidator, localVideoChannelValidator } from '../../middlewares'
10import { videosGetValidator, videosShareValidator } from '../../middlewares/validators' 10import { videosGetValidator, videosShareValidator } from '../../middlewares/validators'
11import { videoCommentGetValidator } from '../../middlewares/validators/video-comments' 11import { videoCommentGetValidator } from '../../middlewares/validators/video-comments'
@@ -26,6 +26,8 @@ import {
26 getVideoSharesActivityPubUrl 26 getVideoSharesActivityPubUrl
27} from '../../lib/activitypub' 27} from '../../lib/activitypub'
28import { VideoCaptionModel } from '../../models/video/video-caption' 28import { VideoCaptionModel } from '../../models/video/video-caption'
29import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy'
30import { getServerActor } from '../../helpers/utils'
29 31
30const activityPubClientRouter = express.Router() 32const activityPubClientRouter = express.Router()
31 33
@@ -93,6 +95,11 @@ activityPubClientRouter.get('/video-channels/:name/following',
93 executeIfActivityPub(asyncMiddleware(videoChannelFollowingController)) 95 executeIfActivityPub(asyncMiddleware(videoChannelFollowingController))
94) 96)
95 97
98activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?',
99 executeIfActivityPub(asyncMiddleware(videoRedundancyGetValidator)),
100 executeIfActivityPub(asyncMiddleware(videoRedundancyController))
101)
102
96// --------------------------------------------------------------------------- 103// ---------------------------------------------------------------------------
97 104
98export { 105export {
@@ -131,7 +138,7 @@ async function videoController (req: express.Request, res: express.Response, nex
131 const videoObject = audiencify(video.toActivityPubObject(), audience) 138 const videoObject = audiencify(video.toActivityPubObject(), audience)
132 139
133 if (req.path.endsWith('/activity')) { 140 if (req.path.endsWith('/activity')) {
134 const data = createActivityData(video.url, video.VideoChannel.Account.Actor, videoObject, audience) 141 const data = buildCreateActivity(video.url, video.VideoChannel.Account.Actor, videoObject, audience)
135 return activityPubResponse(activityPubContextify(data), res) 142 return activityPubResponse(activityPubContextify(data), res)
136 } 143 }
137 144
@@ -140,9 +147,9 @@ async function videoController (req: express.Request, res: express.Response, nex
140 147
141async function videoAnnounceController (req: express.Request, res: express.Response, next: express.NextFunction) { 148async function videoAnnounceController (req: express.Request, res: express.Response, next: express.NextFunction) {
142 const share = res.locals.videoShare as VideoShareModel 149 const share = res.locals.videoShare as VideoShareModel
143 const object = await buildVideoAnnounce(share.Actor, share, res.locals.video, undefined) 150 const { activity } = await buildAnnounceWithVideoAudience(share.Actor, share, res.locals.video, undefined)
144 151
145 return activityPubResponse(activityPubContextify(object), res) 152 return activityPubResponse(activityPubContextify(activity), res)
146} 153}
147 154
148async function videoAnnouncesController (req: express.Request, res: express.Response, next: express.NextFunction) { 155async function videoAnnouncesController (req: express.Request, res: express.Response, next: express.NextFunction) {
@@ -219,13 +226,28 @@ async function videoCommentController (req: express.Request, res: express.Respon
219 const videoCommentObject = audiencify(videoComment.toActivityPubObject(threadParentComments), audience) 226 const videoCommentObject = audiencify(videoComment.toActivityPubObject(threadParentComments), audience)
220 227
221 if (req.path.endsWith('/activity')) { 228 if (req.path.endsWith('/activity')) {
222 const data = createActivityData(videoComment.url, videoComment.Account.Actor, videoCommentObject, audience) 229 const data = buildCreateActivity(videoComment.url, videoComment.Account.Actor, videoCommentObject, audience)
223 return activityPubResponse(activityPubContextify(data), res) 230 return activityPubResponse(activityPubContextify(data), res)
224 } 231 }
225 232
226 return activityPubResponse(activityPubContextify(videoCommentObject), res) 233 return activityPubResponse(activityPubContextify(videoCommentObject), res)
227} 234}
228 235
236async function videoRedundancyController (req: express.Request, res: express.Response) {
237 const videoRedundancy = res.locals.videoRedundancy
238 const serverActor = await getServerActor()
239
240 const audience = getAudience(serverActor)
241 const object = audiencify(videoRedundancy.toActivityPubObject(), audience)
242
243 if (req.path.endsWith('/activity')) {
244 const data = buildCreateActivity(videoRedundancy.url, serverActor, object, audience)
245 return activityPubResponse(activityPubContextify(data), res)
246 }
247
248 return activityPubResponse(activityPubContextify(object), res)
249}
250
229// --------------------------------------------------------------------------- 251// ---------------------------------------------------------------------------
230 252
231async function actorFollowing (req: express.Request, actor: ActorModel) { 253async function actorFollowing (req: express.Request, actor: ActorModel) {
diff --git a/server/controllers/activitypub/outbox.ts b/server/controllers/activitypub/outbox.ts
index db69ae54b..bd0e4fe9d 100644
--- a/server/controllers/activitypub/outbox.ts
+++ b/server/controllers/activitypub/outbox.ts
@@ -3,7 +3,7 @@ import { Activity } from '../../../shared/models/activitypub/activity'
3import { VideoPrivacy } from '../../../shared/models/videos' 3import { VideoPrivacy } from '../../../shared/models/videos'
4import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub' 4import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub'
5import { logger } from '../../helpers/logger' 5import { logger } from '../../helpers/logger'
6import { announceActivityData, createActivityData } from '../../lib/activitypub/send' 6import { buildAnnounceActivity, buildCreateActivity } from '../../lib/activitypub/send'
7import { buildAudience } from '../../lib/activitypub/audience' 7import { buildAudience } from '../../lib/activitypub/audience'
8import { asyncMiddleware, localAccountValidator, localVideoChannelValidator } from '../../middlewares' 8import { asyncMiddleware, localAccountValidator, localVideoChannelValidator } from '../../middlewares'
9import { AccountModel } from '../../models/account/account' 9import { AccountModel } from '../../models/account/account'
@@ -60,12 +60,12 @@ async function buildActivities (actor: ActorModel, start: number, count: number)
60 // This is a shared video 60 // This is a shared video
61 if (video.VideoShares !== undefined && video.VideoShares.length !== 0) { 61 if (video.VideoShares !== undefined && video.VideoShares.length !== 0) {
62 const videoShare = video.VideoShares[0] 62 const videoShare = video.VideoShares[0]
63 const announceActivity = announceActivityData(videoShare.url, actor, video.url, createActivityAudience) 63 const announceActivity = buildAnnounceActivity(videoShare.url, actor, video.url, createActivityAudience)
64 64
65 activities.push(announceActivity) 65 activities.push(announceActivity)
66 } else { 66 } else {
67 const videoObject = video.toActivityPubObject() 67 const videoObject = video.toActivityPubObject()
68 const createActivity = createActivityData(video.url, byActor, videoObject, createActivityAudience) 68 const createActivity = buildCreateActivity(video.url, byActor, videoObject, createActivityAudience)
69 69
70 activities.push(createActivity) 70 activities.push(createActivity)
71 } 71 }
diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts
index bb7174891..28a7a04ca 100644
--- a/server/controllers/api/search.ts
+++ b/server/controllers/api/search.ts
@@ -17,8 +17,6 @@ import {
17import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search' 17import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search'
18import { getOrCreateActorAndServerAndModel, getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub' 18import { getOrCreateActorAndServerAndModel, getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub'
19import { logger } from '../../helpers/logger' 19import { logger } from '../../helpers/logger'
20import { User } from '../../../shared/models/users'
21import { CONFIG } from '../../initializers/constants'
22import { VideoChannelModel } from '../../models/video/video-channel' 20import { VideoChannelModel } from '../../models/video/video-channel'
23import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger' 21import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
24 22
diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts
index 23308445f..a4eae6b45 100644
--- a/server/controllers/api/server/follows.ts
+++ b/server/controllers/api/server/follows.ts
@@ -96,6 +96,11 @@ async function removeFollow (req: express.Request, res: express.Response, next:
96 await sequelizeTypescript.transaction(async t => { 96 await sequelizeTypescript.transaction(async t => {
97 if (follow.state === 'accepted') await sendUndoFollow(follow, t) 97 if (follow.state === 'accepted') await sendUndoFollow(follow, t)
98 98
99 // Disable redundancy on unfollowed instances
100 const server = follow.ActorFollowing.Server
101 server.redundancyAllowed = false
102 await server.save({ transaction: t })
103
99 await follow.destroy({ transaction: t }) 104 await follow.destroy({ transaction: t })
100 }) 105 })
101 106
diff --git a/server/controllers/api/server/index.ts b/server/controllers/api/server/index.ts
index 850a52cdb..43bca2c10 100644
--- a/server/controllers/api/server/index.ts
+++ b/server/controllers/api/server/index.ts
@@ -1,10 +1,12 @@
1import * as express from 'express' 1import * as express from 'express'
2import { serverFollowsRouter } from './follows' 2import { serverFollowsRouter } from './follows'
3import { statsRouter } from './stats' 3import { statsRouter } from './stats'
4import { serverRedundancyRouter } from './redundancy'
4 5
5const serverRouter = express.Router() 6const serverRouter = express.Router()
6 7
7serverRouter.use('/', serverFollowsRouter) 8serverRouter.use('/', serverFollowsRouter)
9serverRouter.use('/', serverRedundancyRouter)
8serverRouter.use('/', statsRouter) 10serverRouter.use('/', statsRouter)
9 11
10// --------------------------------------------------------------------------- 12// ---------------------------------------------------------------------------
diff --git a/server/controllers/api/server/redundancy.ts b/server/controllers/api/server/redundancy.ts
new file mode 100644
index 000000000..4216b9e35
--- /dev/null
+++ b/server/controllers/api/server/redundancy.ts
@@ -0,0 +1,32 @@
1import * as express from 'express'
2import { UserRight } from '../../../../shared/models/users'
3import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares'
4import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy'
5import { ServerModel } from '../../../models/server/server'
6
7const serverRedundancyRouter = express.Router()
8
9serverRedundancyRouter.put('/redundancy/:host',
10 authenticate,
11 ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW),
12 asyncMiddleware(updateServerRedundancyValidator),
13 asyncMiddleware(updateRedundancy)
14)
15
16// ---------------------------------------------------------------------------
17
18export {
19 serverRedundancyRouter
20}
21
22// ---------------------------------------------------------------------------
23
24async function updateRedundancy (req: express.Request, res: express.Response, next: express.NextFunction) {
25 const server = res.locals.server as ServerModel
26
27 server.redundancyAllowed = req.body.redundancyAllowed
28
29 await server.save()
30
31 return res.sendStatus(204)
32}
diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts
index 59bdf6257..08e11b00b 100644
--- a/server/controllers/api/videos/abuse.ts
+++ b/server/controllers/api/videos/abuse.ts
@@ -112,7 +112,7 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) {
112 112
113 // We send the video abuse to the origin server 113 // We send the video abuse to the origin server
114 if (videoInstance.isOwned() === false) { 114 if (videoInstance.isOwned() === false) {
115 await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t) 115 await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance)
116 } 116 }
117 117
118 auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON())) 118 auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON()))
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index a9de11fb0..1304c7559 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -14,20 +14,24 @@ function activityPubContextify <T> (data: T) {
14 'https://w3id.org/security/v1', 14 'https://w3id.org/security/v1',
15 { 15 {
16 RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', 16 RsaSignature2017: 'https://w3id.org/security#RsaSignature2017',
17 pt: 'https://joinpeertube.org/ns',
18 schema: 'http://schema.org#',
17 Hashtag: 'as:Hashtag', 19 Hashtag: 'as:Hashtag',
18 uuid: 'http://schema.org/identifier', 20 uuid: 'schema:identifier',
19 category: 'http://schema.org/category', 21 category: 'schema:category',
20 licence: 'http://schema.org/license', 22 licence: 'schema:license',
21 subtitleLanguage: 'http://schema.org/subtitleLanguage', 23 subtitleLanguage: 'schema:subtitleLanguage',
22 sensitive: 'as:sensitive', 24 sensitive: 'as:sensitive',
23 language: 'http://schema.org/inLanguage', 25 language: 'schema:inLanguage',
24 views: 'http://schema.org/Number', 26 views: 'schema:Number',
25 stats: 'http://schema.org/Number', 27 stats: 'schema:Number',
26 size: 'http://schema.org/Number', 28 size: 'schema:Number',
27 fps: 'http://schema.org/Number', 29 fps: 'schema:Number',
28 commentsEnabled: 'http://schema.org/Boolean', 30 commentsEnabled: 'schema:Boolean',
29 waitTranscoding: 'http://schema.org/Boolean', 31 waitTranscoding: 'schema:Boolean',
30 support: 'http://schema.org/Text' 32 expires: 'schema:expires',
33 support: 'schema:Text',
34 CacheFile: 'pt:CacheFile'
31 }, 35 },
32 { 36 {
33 likes: { 37 likes: {
diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts
index 381a29e66..2562ead9b 100644
--- a/server/helpers/custom-validators/activitypub/activity.ts
+++ b/server/helpers/custom-validators/activitypub/activity.ts
@@ -1,7 +1,10 @@
1import * as validator from 'validator' 1import * as validator from 'validator'
2import { Activity, ActivityType } from '../../../../shared/models/activitypub' 2import { Activity, ActivityType } from '../../../../shared/models/activitypub'
3import { 3import {
4 isActorAcceptActivityValid, isActorDeleteActivityValid, isActorFollowActivityValid, isActorRejectActivityValid, 4 isActorAcceptActivityValid,
5 isActorDeleteActivityValid,
6 isActorFollowActivityValid,
7 isActorRejectActivityValid,
5 isActorUpdateActivityValid 8 isActorUpdateActivityValid
6} from './actor' 9} from './actor'
7import { isAnnounceActivityValid } from './announce' 10import { isAnnounceActivityValid } from './announce'
@@ -11,12 +14,13 @@ import { isUndoActivityValid } from './undo'
11import { isVideoCommentCreateActivityValid, isVideoCommentDeleteActivityValid } from './video-comments' 14import { isVideoCommentCreateActivityValid, isVideoCommentDeleteActivityValid } from './video-comments'
12import { 15import {
13 isVideoFlagValid, 16 isVideoFlagValid,
14 sanitizeAndCheckVideoTorrentCreateActivity,
15 isVideoTorrentDeleteActivityValid, 17 isVideoTorrentDeleteActivityValid,
18 sanitizeAndCheckVideoTorrentCreateActivity,
16 sanitizeAndCheckVideoTorrentUpdateActivity 19 sanitizeAndCheckVideoTorrentUpdateActivity
17} from './videos' 20} from './videos'
18import { isViewActivityValid } from './view' 21import { isViewActivityValid } from './view'
19import { exists } from '../misc' 22import { exists } from '../misc'
23import { isCacheFileCreateActivityValid, isCacheFileUpdateActivityValid } from './cache-file'
20 24
21function isRootActivityValid (activity: any) { 25function isRootActivityValid (activity: any) {
22 return Array.isArray(activity['@context']) && ( 26 return Array.isArray(activity['@context']) && (
@@ -67,11 +71,13 @@ function checkCreateActivity (activity: any) {
67 isDislikeActivityValid(activity) || 71 isDislikeActivityValid(activity) ||
68 sanitizeAndCheckVideoTorrentCreateActivity(activity) || 72 sanitizeAndCheckVideoTorrentCreateActivity(activity) ||
69 isVideoFlagValid(activity) || 73 isVideoFlagValid(activity) ||
70 isVideoCommentCreateActivityValid(activity) 74 isVideoCommentCreateActivityValid(activity) ||
75 isCacheFileCreateActivityValid(activity)
71} 76}
72 77
73function checkUpdateActivity (activity: any) { 78function checkUpdateActivity (activity: any) {
74 return sanitizeAndCheckVideoTorrentUpdateActivity(activity) || 79 return isCacheFileUpdateActivityValid(activity) ||
80 sanitizeAndCheckVideoTorrentUpdateActivity(activity) ||
75 isActorUpdateActivityValid(activity) 81 isActorUpdateActivityValid(activity)
76} 82}
77 83
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts
new file mode 100644
index 000000000..bd70934c8
--- /dev/null
+++ b/server/helpers/custom-validators/activitypub/cache-file.ts
@@ -0,0 +1,28 @@
1import { isActivityPubUrlValid, isBaseActivityValid } from './misc'
2import { isRemoteVideoUrlValid } from './videos'
3import { isDateValid, exists } from '../misc'
4import { CacheFileObject } from '../../../../shared/models/activitypub/objects'
5
6function isCacheFileCreateActivityValid (activity: any) {
7 return isBaseActivityValid(activity, 'Create') &&
8 isCacheFileObjectValid(activity.object)
9}
10
11function isCacheFileUpdateActivityValid (activity: any) {
12 return isBaseActivityValid(activity, 'Update') &&
13 isCacheFileObjectValid(activity.object)
14}
15
16function isCacheFileObjectValid (object: CacheFileObject) {
17 return exists(object) &&
18 object.type === 'CacheFile' &&
19 isDateValid(object.expires) &&
20 isActivityPubUrlValid(object.object) &&
21 isRemoteVideoUrlValid(object.url)
22}
23
24export {
25 isCacheFileUpdateActivityValid,
26 isCacheFileCreateActivityValid,
27 isCacheFileObjectValid
28}
diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts
index 6c5c7abca..4e2c57f04 100644
--- a/server/helpers/custom-validators/activitypub/misc.ts
+++ b/server/helpers/custom-validators/activitypub/misc.ts
@@ -3,7 +3,7 @@ import { CONSTRAINTS_FIELDS } from '../../../initializers'
3import { isTestInstance } from '../../core-utils' 3import { isTestInstance } from '../../core-utils'
4import { exists } from '../misc' 4import { exists } from '../misc'
5 5
6function isActivityPubUrlValid (url: string) { 6function isUrlValid (url: string) {
7 const isURLOptions = { 7 const isURLOptions = {
8 require_host: true, 8 require_host: true,
9 require_tld: true, 9 require_tld: true,
@@ -17,13 +17,18 @@ function isActivityPubUrlValid (url: string) {
17 isURLOptions.require_tld = false 17 isURLOptions.require_tld = false
18 } 18 }
19 19
20 return exists(url) && validator.isURL('' + url, isURLOptions) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL) 20 return exists(url) && validator.isURL('' + url, isURLOptions)
21}
22
23function isActivityPubUrlValid (url: string) {
24 return isUrlValid(url) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL)
21} 25}
22 26
23function isBaseActivityValid (activity: any, type: string) { 27function isBaseActivityValid (activity: any, type: string) {
24 return (activity['@context'] === undefined || Array.isArray(activity['@context'])) && 28 return (activity['@context'] === undefined || Array.isArray(activity['@context'])) &&
25 activity.type === type && 29 activity.type === type &&
26 isActivityPubUrlValid(activity.id) && 30 isActivityPubUrlValid(activity.id) &&
31 exists(activity.actor) &&
27 (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) && 32 (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) &&
28 ( 33 (
29 activity.to === undefined || 34 activity.to === undefined ||
@@ -49,6 +54,7 @@ function setValidAttributedTo (obj: any) {
49} 54}
50 55
51export { 56export {
57 isUrlValid,
52 isActivityPubUrlValid, 58 isActivityPubUrlValid,
53 isBaseActivityValid, 59 isBaseActivityValid,
54 setValidAttributedTo 60 setValidAttributedTo
diff --git a/server/helpers/custom-validators/activitypub/undo.ts b/server/helpers/custom-validators/activitypub/undo.ts
index f50f224fa..578035893 100644
--- a/server/helpers/custom-validators/activitypub/undo.ts
+++ b/server/helpers/custom-validators/activitypub/undo.ts
@@ -2,6 +2,7 @@ import { isActorFollowActivityValid } from './actor'
2import { isBaseActivityValid } from './misc' 2import { isBaseActivityValid } from './misc'
3import { isDislikeActivityValid, isLikeActivityValid } from './rate' 3import { isDislikeActivityValid, isLikeActivityValid } from './rate'
4import { isAnnounceActivityValid } from './announce' 4import { isAnnounceActivityValid } from './announce'
5import { isCacheFileCreateActivityValid } from './cache-file'
5 6
6function isUndoActivityValid (activity: any) { 7function isUndoActivityValid (activity: any) {
7 return isBaseActivityValid(activity, 'Undo') && 8 return isBaseActivityValid(activity, 'Undo') &&
@@ -9,7 +10,8 @@ function isUndoActivityValid (activity: any) {
9 isActorFollowActivityValid(activity.object) || 10 isActorFollowActivityValid(activity.object) ||
10 isLikeActivityValid(activity.object) || 11 isLikeActivityValid(activity.object) ||
11 isDislikeActivityValid(activity.object) || 12 isDislikeActivityValid(activity.object) ||
12 isAnnounceActivityValid(activity.object) 13 isAnnounceActivityValid(activity.object) ||
14 isCacheFileCreateActivityValid(activity.object)
13 ) 15 )
14} 16}
15 17
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index 0362f43ab..f76eba474 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -75,6 +75,30 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
75 video.attributedTo.length !== 0 75 video.attributedTo.length !== 0
76} 76}
77 77
78function isRemoteVideoUrlValid (url: any) {
79 // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few release (currently beta.11)
80 if (url.width && !url.height) url.height = url.width
81
82 return url.type === 'Link' &&
83 (
84 ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 &&
85 isActivityPubUrlValid(url.href) &&
86 validator.isInt(url.height + '', { min: 0 }) &&
87 validator.isInt(url.size + '', { min: 0 }) &&
88 (!url.fps || validator.isInt(url.fps + '', { min: 0 }))
89 ) ||
90 (
91 ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 &&
92 isActivityPubUrlValid(url.href) &&
93 validator.isInt(url.height + '', { min: 0 })
94 ) ||
95 (
96 ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 &&
97 validator.isLength(url.href, { min: 5 }) &&
98 validator.isInt(url.height + '', { min: 0 })
99 )
100}
101
78// --------------------------------------------------------------------------- 102// ---------------------------------------------------------------------------
79 103
80export { 104export {
@@ -83,7 +107,8 @@ export {
83 isVideoTorrentDeleteActivityValid, 107 isVideoTorrentDeleteActivityValid,
84 isRemoteStringIdentifierValid, 108 isRemoteStringIdentifierValid,
85 isVideoFlagValid, 109 isVideoFlagValid,
86 sanitizeAndCheckVideoTorrentObject 110 sanitizeAndCheckVideoTorrentObject,
111 isRemoteVideoUrlValid
87} 112}
88 113
89// --------------------------------------------------------------------------- 114// ---------------------------------------------------------------------------
@@ -147,26 +172,4 @@ function setRemoteVideoTruncatedContent (video: any) {
147 return true 172 return true
148} 173}
149 174
150function isRemoteVideoUrlValid (url: any) {
151 // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few realease (currently beta.11)
152 if (url.width && !url.height) url.height = url.width
153 175
154 return url.type === 'Link' &&
155 (
156 ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 &&
157 isActivityPubUrlValid(url.href) &&
158 validator.isInt(url.height + '', { min: 0 }) &&
159 validator.isInt(url.size + '', { min: 0 }) &&
160 (!url.fps || validator.isInt(url.fps + '', { min: 0 }))
161 ) ||
162 (
163 ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 &&
164 isActivityPubUrlValid(url.href) &&
165 validator.isInt(url.height + '', { min: 0 })
166 ) ||
167 (
168 ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 &&
169 validator.isLength(url.href, { min: 5 }) &&
170 validator.isInt(url.height + '', { min: 0 })
171 )
172}
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 1c0cc7058..2fdfd1876 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -5,44 +5,49 @@ import { createWriteStream, remove } from 'fs-extra'
5import { CONFIG } from '../initializers' 5import { CONFIG } from '../initializers'
6import { join } from 'path' 6import { join } from 'path'
7 7
8function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: string }) { 8function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout?: number) {
9 const id = target.magnetUri || target.torrentName 9 const id = target.magnetUri || target.torrentName
10 let timer
10 11
11 const path = generateVideoTmpPath(id) 12 const path = generateVideoTmpPath(id)
12 logger.info('Importing torrent video %s', id) 13 logger.info('Importing torrent video %s', id)
13 14
14 return new Promise<string>((res, rej) => { 15 return new Promise<string>((res, rej) => {
15 const webtorrent = new WebTorrent() 16 const webtorrent = new WebTorrent()
17 let file: WebTorrent.TorrentFile
16 18
17 const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) 19 const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName)
18 20
19 const options = { path: CONFIG.STORAGE.VIDEOS_DIR } 21 const options = { path: CONFIG.STORAGE.VIDEOS_DIR }
20 const torrent = webtorrent.add(torrentId, options, torrent => { 22 const torrent = webtorrent.add(torrentId, options, torrent => {
21 if (torrent.files.length !== 1) return rej(new Error('The number of files is not equal to 1 for ' + torrentId)) 23 if (torrent.files.length !== 1) {
24 if (timer) clearTimeout(timer)
22 25
23 const file = torrent.files[ 0 ] 26 return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName)
27 .then(() => rej(new Error('The number of files is not equal to 1 for ' + torrentId)))
28 }
29
30 file = torrent.files[ 0 ]
24 31
25 const writeStream = createWriteStream(path) 32 const writeStream = createWriteStream(path)
26 writeStream.on('finish', () => { 33 writeStream.on('finish', () => {
27 webtorrent.destroy(async err => { 34 if (timer) clearTimeout(timer)
28 if (err) return rej(err)
29
30 if (target.torrentName) {
31 remove(torrentId)
32 .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))
33 }
34 35
35 remove(join(CONFIG.STORAGE.VIDEOS_DIR, file.name)) 36 return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName)
36 .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', file.name, { err })) 37 .then(() => res(path))
37
38 res(path)
39 })
40 }) 38 })
41 39
42 file.createReadStream().pipe(writeStream) 40 file.createReadStream().pipe(writeStream)
43 }) 41 })
44 42
45 torrent.on('error', err => rej(err)) 43 torrent.on('error', err => rej(err))
44
45 if (timeout) {
46 timer = setTimeout(async () => {
47 return safeWebtorrentDestroy(webtorrent, torrentId, file ? file.name : undefined, target.torrentName)
48 .then(() => rej(new Error('Webtorrent download timeout.')))
49 }, timeout)
50 }
46 }) 51 })
47} 52}
48 53
@@ -51,3 +56,29 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: stri
51export { 56export {
52 downloadWebTorrentVideo 57 downloadWebTorrentVideo
53} 58}
59
60// ---------------------------------------------------------------------------
61
62function safeWebtorrentDestroy (webtorrent: WebTorrent.Instance, torrentId: string, filename?: string, torrentName?: string) {
63 return new Promise(res => {
64 webtorrent.destroy(err => {
65 // Delete torrent file
66 if (torrentName) {
67 remove(torrentId)
68 .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))
69 }
70
71 // Delete downloaded file
72 if (filename) {
73 remove(join(CONFIG.STORAGE.VIDEOS_DIR, filename))
74 .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', filename, { err }))
75 }
76
77 if (err) {
78 logger.warn('Cannot destroy webtorrent in timeout.', { err })
79 }
80
81 return res()
82 })
83 })
84}
diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts
index 9dd104035..6a2badd35 100644
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker.ts
@@ -7,6 +7,9 @@ import { parse } from 'url'
7import { CONFIG } from './constants' 7import { CONFIG } from './constants'
8import { logger } from '../helpers/logger' 8import { logger } from '../helpers/logger'
9import { getServerActor } from '../helpers/utils' 9import { getServerActor } from '../helpers/utils'
10import { VideosRedundancy } from '../../shared/models/redundancy'
11import { isArray } from '../helpers/custom-validators/misc'
12import { uniq } from 'lodash'
10 13
11async function checkActivityPubUrls () { 14async function checkActivityPubUrls () {
12 const actor = await getServerActor() 15 const actor = await getServerActor()
@@ -35,6 +38,20 @@ function checkConfig () {
35 return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy 38 return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy
36 } 39 }
37 40
41 const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos')
42 if (isArray(redundancyVideos)) {
43 for (const r of redundancyVideos) {
44 if ([ 'most-views' ].indexOf(r.strategy) === -1) {
45 return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy
46 }
47 }
48
49 const filtered = uniq(redundancyVideos.map(r => r.strategy))
50 if (filtered.length !== redundancyVideos.length) {
51 return 'Redundancy video entries should have uniq strategies'
52 }
53 }
54
38 return null 55 return null
39} 56}
40 57
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 5b7ea5d6c..6b4afbfd8 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -1,6 +1,6 @@
1import { IConfig } from 'config' 1import { IConfig } from 'config'
2import { dirname, join } from 'path' 2import { dirname, join } from 'path'
3import { JobType, VideoRateType, VideoState } from '../../shared/models' 3import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models'
4import { ActivityPubActorType } from '../../shared/models/activitypub' 4import { ActivityPubActorType } from '../../shared/models/activitypub'
5import { FollowState } from '../../shared/models/actors' 5import { FollowState } from '../../shared/models/actors'
6import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' 6import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos'
@@ -9,13 +9,14 @@ import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../h
9import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' 9import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
10import { invert } from 'lodash' 10import { invert } from 'lodash'
11import { CronRepeatOptions, EveryRepeatOptions } from 'bull' 11import { CronRepeatOptions, EveryRepeatOptions } from 'bull'
12import * as bytes from 'bytes'
12 13
13// Use a variable to reload the configuration if we need 14// Use a variable to reload the configuration if we need
14let config: IConfig = require('config') 15let config: IConfig = require('config')
15 16
16// --------------------------------------------------------------------------- 17// ---------------------------------------------------------------------------
17 18
18const LAST_MIGRATION_VERSION = 265 19const LAST_MIGRATION_VERSION = 270
19 20
20// --------------------------------------------------------------------------- 21// ---------------------------------------------------------------------------
21 22
@@ -137,7 +138,8 @@ let SCHEDULER_INTERVALS_MS = {
137 badActorFollow: 60000 * 60, // 1 hour 138 badActorFollow: 60000 * 60, // 1 hour
138 removeOldJobs: 60000 * 60, // 1 hour 139 removeOldJobs: 60000 * 60, // 1 hour
139 updateVideos: 60000, // 1 minute 140 updateVideos: 60000, // 1 minute
140 youtubeDLUpdate: 60000 * 60 * 24 // 1 day 141 youtubeDLUpdate: 60000 * 60 * 24, // 1 day
142 videosRedundancy: 60000 * 2 // 2 hours
141} 143}
142 144
143// --------------------------------------------------------------------------- 145// ---------------------------------------------------------------------------
@@ -208,6 +210,9 @@ const CONFIG = {
208 INTERVAL_DAYS: config.get<number>('trending.videos.interval_days') 210 INTERVAL_DAYS: config.get<number>('trending.videos.interval_days')
209 } 211 }
210 }, 212 },
213 REDUNDANCY: {
214 VIDEOS: buildVideosRedundancy(config.get<any[]>('redundancy.videos'))
215 },
211 ADMIN: { 216 ADMIN: {
212 get EMAIL () { return config.get<string>('admin.email') } 217 get EMAIL () { return config.get<string>('admin.email') }
213 }, 218 },
@@ -321,6 +326,9 @@ const CONSTRAINTS_FIELDS = {
321 } 326 }
322 } 327 }
323 }, 328 },
329 VIDEOS_REDUNDANCY: {
330 URL: { min: 3, max: 2000 } // Length
331 },
324 VIDEOS: { 332 VIDEOS: {
325 NAME: { min: 3, max: 120 }, // Length 333 NAME: { min: 3, max: 120 }, // Length
326 LANGUAGE: { min: 1, max: 10 }, // Length 334 LANGUAGE: { min: 1, max: 10 }, // Length
@@ -584,6 +592,13 @@ const CACHE = {
584 } 592 }
585} 593}
586 594
595const REDUNDANCY = {
596 VIDEOS: {
597 EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
598 RANDOMIZED_FACTOR: 5
599 }
600}
601
587const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS) 602const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS)
588 603
589// --------------------------------------------------------------------------- 604// ---------------------------------------------------------------------------
@@ -629,8 +644,11 @@ if (isTestInstance() === true) {
629 SCHEDULER_INTERVALS_MS.badActorFollow = 10000 644 SCHEDULER_INTERVALS_MS.badActorFollow = 10000
630 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 645 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
631 SCHEDULER_INTERVALS_MS.updateVideos = 5000 646 SCHEDULER_INTERVALS_MS.updateVideos = 5000
647 SCHEDULER_INTERVALS_MS.videosRedundancy = 5000
632 REPEAT_JOBS['videos-views'] = { every: 5000 } 648 REPEAT_JOBS['videos-views'] = { every: 5000 }
633 649
650 REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
651
634 VIDEO_VIEW_LIFETIME = 1000 // 1 second 652 VIDEO_VIEW_LIFETIME = 1000 // 1 second
635 653
636 JOB_ATTEMPTS['email'] = 1 654 JOB_ATTEMPTS['email'] = 1
@@ -653,6 +671,7 @@ export {
653 CONFIG, 671 CONFIG,
654 CONSTRAINTS_FIELDS, 672 CONSTRAINTS_FIELDS,
655 EMBED_SIZE, 673 EMBED_SIZE,
674 REDUNDANCY,
656 JOB_CONCURRENCY, 675 JOB_CONCURRENCY,
657 JOB_ATTEMPTS, 676 JOB_ATTEMPTS,
658 LAST_MIGRATION_VERSION, 677 LAST_MIGRATION_VERSION,
@@ -722,6 +741,17 @@ function updateWebserverConfig () {
722 CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) 741 CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP)
723} 742}
724 743
744function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] {
745 if (!objs) return []
746
747 return objs.map(obj => {
748 return {
749 strategy: obj.strategy,
750 size: bytes.parse(obj.size)
751 }
752 })
753}
754
725function buildLanguages () { 755function buildLanguages () {
726 const iso639 = require('iso-639-3') 756 const iso639 = require('iso-639-3')
727 757
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index b68e1a882..4d57bf8aa 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -27,6 +27,7 @@ import { VideoCaptionModel } from '../models/video/video-caption'
27import { VideoImportModel } from '../models/video/video-import' 27import { VideoImportModel } from '../models/video/video-import'
28import { VideoViewModel } from '../models/video/video-views' 28import { VideoViewModel } from '../models/video/video-views'
29import { VideoChangeOwnershipModel } from '../models/video/video-change-ownership' 29import { VideoChangeOwnershipModel } from '../models/video/video-change-ownership'
30import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
30 31
31require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string 32require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
32 33
@@ -87,7 +88,8 @@ async function initDatabaseModels (silent: boolean) {
87 VideoCommentModel, 88 VideoCommentModel,
88 ScheduleVideoUpdateModel, 89 ScheduleVideoUpdateModel,
89 VideoImportModel, 90 VideoImportModel,
90 VideoViewModel 91 VideoViewModel,
92 VideoRedundancyModel
91 ]) 93 ])
92 94
93 // Check extensions exist in the database 95 // Check extensions exist in the database
diff --git a/server/initializers/migrations/0270-server-redundancy.ts b/server/initializers/migrations/0270-server-redundancy.ts
new file mode 100644
index 000000000..903ba8a85
--- /dev/null
+++ b/server/initializers/migrations/0270-server-redundancy.ts
@@ -0,0 +1,24 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction
5 queryInterface: Sequelize.QueryInterface
6 sequelize: Sequelize.Sequelize
7}): Promise<any> {
8 {
9 const data = {
10 type: Sequelize.BOOLEAN,
11 allowNull: false,
12 defaultValue: false
13 }
14
15 await utils.queryInterface.addColumn('server', 'redundancyAllowed', data)
16 }
17
18}
19
20function down (options) {
21 throw new Error('Not implemented.')
22}
23
24export { up, down }
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 1657262d7..3464add03 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -400,17 +400,15 @@ async function refreshActorIfNeeded (actor: ActorModel): Promise<{ actor: ActorM
400 await actor.save({ transaction: t }) 400 await actor.save({ transaction: t })
401 401
402 if (actor.Account) { 402 if (actor.Account) {
403 await actor.save({ transaction: t })
404
405 actor.Account.set('name', result.name) 403 actor.Account.set('name', result.name)
406 actor.Account.set('description', result.summary) 404 actor.Account.set('description', result.summary)
405
407 await actor.Account.save({ transaction: t }) 406 await actor.Account.save({ transaction: t })
408 } else if (actor.VideoChannel) { 407 } else if (actor.VideoChannel) {
409 await actor.save({ transaction: t })
410
411 actor.VideoChannel.set('name', result.name) 408 actor.VideoChannel.set('name', result.name)
412 actor.VideoChannel.set('description', result.summary) 409 actor.VideoChannel.set('description', result.summary)
413 actor.VideoChannel.set('support', result.support) 410 actor.VideoChannel.set('support', result.support)
411
414 await actor.VideoChannel.save({ transaction: t }) 412 await actor.VideoChannel.save({ transaction: t })
415 } 413 }
416 414
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
new file mode 100644
index 000000000..7325ddcb6
--- /dev/null
+++ b/server/lib/activitypub/cache-file.ts
@@ -0,0 +1,47 @@
1import { CacheFileObject } from '../../../shared/index'
2import { VideoModel } from '../../models/video/video'
3import { ActorModel } from '../../models/activitypub/actor'
4import { sequelizeTypescript } from '../../initializers'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6
7function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) {
8 const url = cacheFileObject.url
9
10 const videoFile = video.VideoFiles.find(f => {
11 return f.resolution === url.height && f.fps === url.fps
12 })
13
14 if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
15
16 return {
17 expiresOn: new Date(cacheFileObject.expires),
18 url: cacheFileObject.id,
19 fileUrl: cacheFileObject.url.href,
20 strategy: null,
21 videoFileId: videoFile.id,
22 actorId: byActor.id
23 }
24}
25
26function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) {
27 return sequelizeTypescript.transaction(async t => {
28 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
29
30 return VideoRedundancyModel.create(attributes, { transaction: t })
31 })
32}
33
34function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: ActorModel) {
35 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor)
36
37 redundancyModel.set('expires', attributes.expiresOn)
38 redundancyModel.set('fileUrl', attributes.fileUrl)
39
40 return redundancyModel.save()
41}
42
43export {
44 createCacheFile,
45 updateCacheFile,
46 cacheFileActivityObjectToDBAttributes
47}
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 16f426e23..32e555acf 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -1,4 +1,4 @@
1import { ActivityCreate, VideoAbuseState, VideoTorrentObject } from '../../../../shared' 1import { ActivityCreate, CacheFileObject, VideoAbuseState, VideoTorrentObject } from '../../../../shared'
2import { DislikeObject, VideoAbuseObject, ViewObject } from '../../../../shared/models/activitypub/objects' 2import { DislikeObject, VideoAbuseObject, ViewObject } from '../../../../shared/models/activitypub/objects'
3import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object' 3import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object'
4import { retryTransactionWrapper } from '../../../helpers/database-utils' 4import { retryTransactionWrapper } from '../../../helpers/database-utils'
@@ -12,6 +12,7 @@ import { addVideoComment, resolveThread } from '../video-comments'
12import { getOrCreateVideoAndAccountAndChannel } from '../videos' 12import { getOrCreateVideoAndAccountAndChannel } from '../videos'
13import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' 13import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils'
14import { Redis } from '../../redis' 14import { Redis } from '../../redis'
15import { createCacheFile } from '../cache-file'
15 16
16async function processCreateActivity (activity: ActivityCreate) { 17async function processCreateActivity (activity: ActivityCreate) {
17 const activityObject = activity.object 18 const activityObject = activity.object
@@ -28,6 +29,8 @@ async function processCreateActivity (activity: ActivityCreate) {
28 return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject) 29 return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject)
29 } else if (activityType === 'Note') { 30 } else if (activityType === 'Note') {
30 return retryTransactionWrapper(processCreateVideoComment, actor, activity) 31 return retryTransactionWrapper(processCreateVideoComment, actor, activity)
32 } else if (activityType === 'CacheFile') {
33 return retryTransactionWrapper(processCacheFile, actor, activity)
31 } 34 }
32 35
33 logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) 36 logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })
@@ -97,6 +100,20 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate)
97 } 100 }
98} 101}
99 102
103async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) {
104 const cacheFile = activity.object as CacheFileObject
105
106 const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object)
107
108 await createCacheFile(cacheFile, video, byActor)
109
110 if (video.isOwned()) {
111 // Don't resend the activity to the sender
112 const exceptions = [ byActor ]
113 await forwardActivity(activity, undefined, exceptions)
114 }
115}
116
100async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { 117async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) {
101 logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) 118 logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object)
102 119
@@ -113,7 +130,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat
113 state: VideoAbuseState.PENDING 130 state: VideoAbuseState.PENDING
114 } 131 }
115 132
116 await VideoAbuseModel.create(videoAbuseData) 133 await VideoAbuseModel.create(videoAbuseData, { transaction: t })
117 134
118 logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object) 135 logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object)
119 }) 136 })
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index 1c1de8827..0eb5fa392 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -1,4 +1,4 @@
1import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo } from '../../../../shared/models/activitypub' 1import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub'
2import { DislikeObject } from '../../../../shared/models/activitypub/objects' 2import { DislikeObject } from '../../../../shared/models/activitypub/objects'
3import { getActorUrl } from '../../../helpers/activitypub' 3import { getActorUrl } from '../../../helpers/activitypub'
4import { retryTransactionWrapper } from '../../../helpers/database-utils' 4import { retryTransactionWrapper } from '../../../helpers/database-utils'
@@ -11,6 +11,7 @@ import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
11import { forwardVideoRelatedActivity } from '../send/utils' 11import { forwardVideoRelatedActivity } from '../send/utils'
12import { getOrCreateVideoAndAccountAndChannel } from '../videos' 12import { getOrCreateVideoAndAccountAndChannel } from '../videos'
13import { VideoShareModel } from '../../../models/video/video-share' 13import { VideoShareModel } from '../../../models/video/video-share'
14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
14 15
15async function processUndoActivity (activity: ActivityUndo) { 16async function processUndoActivity (activity: ActivityUndo) {
16 const activityToUndo = activity.object 17 const activityToUndo = activity.object
@@ -19,11 +20,21 @@ async function processUndoActivity (activity: ActivityUndo) {
19 20
20 if (activityToUndo.type === 'Like') { 21 if (activityToUndo.type === 'Like') {
21 return retryTransactionWrapper(processUndoLike, actorUrl, activity) 22 return retryTransactionWrapper(processUndoLike, actorUrl, activity)
22 } else if (activityToUndo.type === 'Create' && activityToUndo.object.type === 'Dislike') { 23 }
23 return retryTransactionWrapper(processUndoDislike, actorUrl, activity) 24
24 } else if (activityToUndo.type === 'Follow') { 25 if (activityToUndo.type === 'Create') {
26 if (activityToUndo.object.type === 'Dislike') {
27 return retryTransactionWrapper(processUndoDislike, actorUrl, activity)
28 } else if (activityToUndo.object.type === 'CacheFile') {
29 return retryTransactionWrapper(processUndoCacheFile, actorUrl, activity)
30 }
31 }
32
33 if (activityToUndo.type === 'Follow') {
25 return retryTransactionWrapper(processUndoFollow, actorUrl, activityToUndo) 34 return retryTransactionWrapper(processUndoFollow, actorUrl, activityToUndo)
26 } else if (activityToUndo.type === 'Announce') { 35 }
36
37 if (activityToUndo.type === 'Announce') {
27 return retryTransactionWrapper(processUndoAnnounce, actorUrl, activityToUndo) 38 return retryTransactionWrapper(processUndoAnnounce, actorUrl, activityToUndo)
28 } 39 }
29 40
@@ -88,6 +99,29 @@ async function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
88 }) 99 })
89} 100}
90 101
102async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) {
103 const cacheFileObject = activity.object.object as CacheFileObject
104
105 const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object)
106
107 return sequelizeTypescript.transaction(async t => {
108 const byActor = await ActorModel.loadByUrl(actorUrl)
109 if (!byActor) throw new Error('Unknown actor ' + actorUrl)
110
111 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
112 if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url)
113
114 await cacheFile.destroy()
115
116 if (video.isOwned()) {
117 // Don't resend the activity to the sender
118 const exceptions = [ byActor ]
119
120 await forwardVideoRelatedActivity(activity, t, exceptions, video)
121 }
122 })
123}
124
91function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) { 125function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) {
92 return sequelizeTypescript.transaction(async t => { 126 return sequelizeTypescript.transaction(async t => {
93 const follower = await ActorModel.loadByUrl(actorUrl, t) 127 const follower = await ActorModel.loadByUrl(actorUrl, t)
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index d2ad738a2..d3af1a181 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -1,4 +1,4 @@
1import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub' 1import { ActivityUpdate, CacheFileObject, VideoTorrentObject } from '../../../../shared/models/activitypub'
2import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' 2import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor'
3import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' 3import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
4import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
@@ -7,8 +7,11 @@ import { AccountModel } from '../../../models/account/account'
7import { ActorModel } from '../../../models/activitypub/actor' 7import { ActorModel } from '../../../models/activitypub/actor'
8import { VideoChannelModel } from '../../../models/video/video-channel' 8import { VideoChannelModel } from '../../../models/video/video-channel'
9import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' 9import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor'
10import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos' 10import { getOrCreateVideoAndAccountAndChannel, updateVideoFromAP, getOrCreateVideoChannelFromVideoObject } from '../videos'
11import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' 11import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
13import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
14import { createCacheFile, updateCacheFile } from '../cache-file'
12 15
13async function processUpdateActivity (activity: ActivityUpdate) { 16async function processUpdateActivity (activity: ActivityUpdate) {
14 const actor = await getOrCreateActorAndServerAndModel(activity.actor) 17 const actor = await getOrCreateActorAndServerAndModel(activity.actor)
@@ -16,10 +19,16 @@ async function processUpdateActivity (activity: ActivityUpdate) {
16 19
17 if (objectType === 'Video') { 20 if (objectType === 'Video') {
18 return retryTransactionWrapper(processUpdateVideo, actor, activity) 21 return retryTransactionWrapper(processUpdateVideo, actor, activity)
19 } else if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { 22 }
23
24 if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') {
20 return retryTransactionWrapper(processUpdateActor, actor, activity) 25 return retryTransactionWrapper(processUpdateActor, actor, activity)
21 } 26 }
22 27
28 if (objectType === 'CacheFile') {
29 return retryTransactionWrapper(processUpdateCacheFile, actor, activity)
30 }
31
23 return undefined 32 return undefined
24} 33}
25 34
@@ -42,7 +51,24 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
42 const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) 51 const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id)
43 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) 52 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
44 53
45 return updateVideoFromAP(video, videoObject, actor, channelActor, activity.to) 54 return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to)
55}
56
57async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) {
58 const cacheFileObject = activity.object as CacheFileObject
59
60 if (!isCacheFileObjectValid(cacheFileObject) === false) {
61 logger.debug('Cahe file object sent by update is not valid.', { cacheFileObject })
62 return undefined
63 }
64
65 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
66 if (!redundancyModel) {
67 const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id)
68 return createCacheFile(cacheFileObject, video, byActor)
69 }
70
71 return updateCacheFile(cacheFileObject, redundancyModel, byActor)
46} 72}
47 73
48async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { 74async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) {
diff --git a/server/lib/activitypub/send/send-accept.ts b/server/lib/activitypub/send/send-accept.ts
index ef679707b..b6abde13d 100644
--- a/server/lib/activitypub/send/send-accept.ts
+++ b/server/lib/activitypub/send/send-accept.ts
@@ -3,7 +3,7 @@ import { ActorModel } from '../../../models/activitypub/actor'
3import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 3import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
4import { getActorFollowAcceptActivityPubUrl, getActorFollowActivityPubUrl } from '../url' 4import { getActorFollowAcceptActivityPubUrl, getActorFollowActivityPubUrl } from '../url'
5import { unicastTo } from './utils' 5import { unicastTo } from './utils'
6import { followActivityData } from './send-follow' 6import { buildFollowActivity } from './send-follow'
7import { logger } from '../../../helpers/logger' 7import { logger } from '../../../helpers/logger'
8 8
9async function sendAccept (actorFollow: ActorFollowModel) { 9async function sendAccept (actorFollow: ActorFollowModel) {
@@ -18,10 +18,10 @@ async function sendAccept (actorFollow: ActorFollowModel) {
18 logger.info('Creating job to accept follower %s.', follower.url) 18 logger.info('Creating job to accept follower %s.', follower.url)
19 19
20 const followUrl = getActorFollowActivityPubUrl(actorFollow) 20 const followUrl = getActorFollowActivityPubUrl(actorFollow)
21 const followData = followActivityData(followUrl, follower, me) 21 const followData = buildFollowActivity(followUrl, follower, me)
22 22
23 const url = getActorFollowAcceptActivityPubUrl(actorFollow) 23 const url = getActorFollowAcceptActivityPubUrl(actorFollow)
24 const data = acceptActivityData(url, me, followData) 24 const data = buildAcceptActivity(url, me, followData)
25 25
26 return unicastTo(data, me, follower.inboxUrl) 26 return unicastTo(data, me, follower.inboxUrl)
27} 27}
@@ -34,7 +34,7 @@ export {
34 34
35// --------------------------------------------------------------------------- 35// ---------------------------------------------------------------------------
36 36
37function acceptActivityData (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityAccept { 37function buildAcceptActivity (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityAccept {
38 return { 38 return {
39 type: 'Accept', 39 type: 'Accept',
40 id: url, 40 id: url,
diff --git a/server/lib/activitypub/send/send-announce.ts b/server/lib/activitypub/send/send-announce.ts
index 352813d73..f137217f8 100644
--- a/server/lib/activitypub/send/send-announce.ts
+++ b/server/lib/activitypub/send/send-announce.ts
@@ -4,45 +4,44 @@ import { ActorModel } from '../../../models/activitypub/actor'
4import { VideoModel } from '../../../models/video/video' 4import { VideoModel } from '../../../models/video/video'
5import { VideoShareModel } from '../../../models/video/video-share' 5import { VideoShareModel } from '../../../models/video/video-share'
6import { broadcastToFollowers } from './utils' 6import { broadcastToFollowers } from './utils'
7import { getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' 7import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience'
8import { logger } from '../../../helpers/logger' 8import { logger } from '../../../helpers/logger'
9 9
10async function buildVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { 10async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
11 const announcedObject = video.url 11 const announcedObject = video.url
12 12
13 const accountsToForwardView = await getActorsInvolvedInVideo(video, t) 13 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
14 const audience = getObjectFollowersAudience(accountsToForwardView) 14 const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
15 return announceActivityData(videoShare.url, byActor, announcedObject, audience) 15
16 const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience)
17
18 return { activity, actorsInvolvedInVideo }
16} 19}
17 20
18async function sendVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { 21async function sendVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
19 const data = await buildVideoAnnounce(byActor, videoShare, video, t) 22 const { activity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t)
20 23
21 logger.info('Creating job to send announce %s.', videoShare.url) 24 logger.info('Creating job to send announce %s.', videoShare.url)
22 25
23 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
24 const followersException = [ byActor ] 26 const followersException = [ byActor ]
25 27 return broadcastToFollowers(activity, byActor, actorsInvolvedInVideo, t, followersException)
26 return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException)
27} 28}
28 29
29function announceActivityData (url: string, byActor: ActorModel, object: string, audience?: ActivityAudience): ActivityAnnounce { 30function buildAnnounceActivity (url: string, byActor: ActorModel, object: string, audience?: ActivityAudience): ActivityAnnounce {
30 if (!audience) audience = getAudience(byActor) 31 if (!audience) audience = getAudience(byActor)
31 32
32 return { 33 return audiencify({
33 type: 'Announce', 34 type: 'Announce' as 'Announce',
34 to: audience.to,
35 cc: audience.cc,
36 id: url, 35 id: url,
37 actor: byActor.url, 36 actor: byActor.url,
38 object 37 object
39 } 38 }, audience)
40} 39}
41 40
42// --------------------------------------------------------------------------- 41// ---------------------------------------------------------------------------
43 42
44export { 43export {
45 sendVideoAnnounce, 44 sendVideoAnnounce,
46 announceActivityData, 45 buildAnnounceActivity,
47 buildVideoAnnounce 46 buildAnnounceWithVideoAudience
48} 47}
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index fc76cdd8a..6f89b1a22 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -17,6 +17,7 @@ import {
17 getVideoCommentAudience 17 getVideoCommentAudience
18} from '../audience' 18} from '../audience'
19import { logger } from '../../../helpers/logger' 19import { logger } from '../../../helpers/logger'
20import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
20 21
21async function sendCreateVideo (video: VideoModel, t: Transaction) { 22async function sendCreateVideo (video: VideoModel, t: Transaction) {
22 if (video.privacy === VideoPrivacy.PRIVATE) return undefined 23 if (video.privacy === VideoPrivacy.PRIVATE) return undefined
@@ -27,12 +28,12 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) {
27 const videoObject = video.toActivityPubObject() 28 const videoObject = video.toActivityPubObject()
28 29
29 const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) 30 const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
30 const data = createActivityData(video.url, byActor, videoObject, audience) 31 const createActivity = buildCreateActivity(video.url, byActor, videoObject, audience)
31 32
32 return broadcastToFollowers(data, byActor, [ byActor ], t) 33 return broadcastToFollowers(createActivity, byActor, [ byActor ], t)
33} 34}
34 35
35async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel, t: Transaction) { 36async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) {
36 if (!video.VideoChannel.Account.Actor.serverId) return // Local 37 if (!video.VideoChannel.Account.Actor.serverId) return // Local
37 38
38 const url = getVideoAbuseActivityPubUrl(videoAbuse) 39 const url = getVideoAbuseActivityPubUrl(videoAbuse)
@@ -40,9 +41,23 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
40 logger.info('Creating job to send video abuse %s.', url) 41 logger.info('Creating job to send video abuse %s.', url)
41 42
42 const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } 43 const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
43 const data = createActivityData(url, byActor, videoAbuse.toActivityPubObject(), audience) 44 const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience)
44 45
45 return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 46 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
47}
48
49async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) {
50 logger.info('Creating job to send file cache of %s.', fileRedundancy.url)
51
52 const redundancyObject = fileRedundancy.toActivityPubObject()
53
54 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
55 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined)
56
57 const audience = getVideoAudience(video, actorsInvolvedInVideo)
58 const createActivity = buildCreateActivity(fileRedundancy.url, byActor, redundancyObject, audience)
59
60 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
46} 61}
47 62
48async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) { 63async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) {
@@ -66,73 +81,73 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
66 audience = getObjectFollowersAudience(actorsInvolvedInComment.concat(parentsCommentActors)) 81 audience = getObjectFollowersAudience(actorsInvolvedInComment.concat(parentsCommentActors))
67 } 82 }
68 83
69 const data = createActivityData(comment.url, byActor, commentObject, audience) 84 const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience)
70 85
71 // This was a reply, send it to the parent actors 86 // This was a reply, send it to the parent actors
72 const actorsException = [ byActor ] 87 const actorsException = [ byActor ]
73 await broadcastToActors(data, byActor, parentsCommentActors, actorsException) 88 await broadcastToActors(createActivity, byActor, parentsCommentActors, actorsException)
74 89
75 // Broadcast to our followers 90 // Broadcast to our followers
76 await broadcastToFollowers(data, byActor, [ byActor ], t) 91 await broadcastToFollowers(createActivity, byActor, [ byActor ], t)
77 92
78 // Send to actors involved in the comment 93 // Send to actors involved in the comment
79 if (isOrigin) return broadcastToFollowers(data, byActor, actorsInvolvedInComment, t, actorsException) 94 if (isOrigin) return broadcastToFollowers(createActivity, byActor, actorsInvolvedInComment, t, actorsException)
80 95
81 // Send to origin 96 // Send to origin
82 return unicastTo(data, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) 97 return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
83} 98}
84 99
85async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) { 100async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) {
86 logger.info('Creating job to send view of %s.', video.url) 101 logger.info('Creating job to send view of %s.', video.url)
87 102
88 const url = getVideoViewActivityPubUrl(byActor, video) 103 const url = getVideoViewActivityPubUrl(byActor, video)
89 const viewActivityData = createViewActivityData(byActor, video) 104 const viewActivity = buildViewActivity(byActor, video)
90 105
91 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 106 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
92 107
93 // Send to origin 108 // Send to origin
94 if (video.isOwned() === false) { 109 if (video.isOwned() === false) {
95 const audience = getVideoAudience(video, actorsInvolvedInVideo) 110 const audience = getVideoAudience(video, actorsInvolvedInVideo)
96 const data = createActivityData(url, byActor, viewActivityData, audience) 111 const createActivity = buildCreateActivity(url, byActor, viewActivity, audience)
97 112
98 return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 113 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
99 } 114 }
100 115
101 // Send to followers 116 // Send to followers
102 const audience = getObjectFollowersAudience(actorsInvolvedInVideo) 117 const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
103 const data = createActivityData(url, byActor, viewActivityData, audience) 118 const createActivity = buildCreateActivity(url, byActor, viewActivity, audience)
104 119
105 // Use the server actor to send the view 120 // Use the server actor to send the view
106 const serverActor = await getServerActor() 121 const serverActor = await getServerActor()
107 const actorsException = [ byActor ] 122 const actorsException = [ byActor ]
108 return broadcastToFollowers(data, serverActor, actorsInvolvedInVideo, t, actorsException) 123 return broadcastToFollowers(createActivity, serverActor, actorsInvolvedInVideo, t, actorsException)
109} 124}
110 125
111async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { 126async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
112 logger.info('Creating job to dislike %s.', video.url) 127 logger.info('Creating job to dislike %s.', video.url)
113 128
114 const url = getVideoDislikeActivityPubUrl(byActor, video) 129 const url = getVideoDislikeActivityPubUrl(byActor, video)
115 const dislikeActivityData = createDislikeActivityData(byActor, video) 130 const dislikeActivity = buildDislikeActivity(byActor, video)
116 131
117 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 132 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
118 133
119 // Send to origin 134 // Send to origin
120 if (video.isOwned() === false) { 135 if (video.isOwned() === false) {
121 const audience = getVideoAudience(video, actorsInvolvedInVideo) 136 const audience = getVideoAudience(video, actorsInvolvedInVideo)
122 const data = createActivityData(url, byActor, dislikeActivityData, audience) 137 const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience)
123 138
124 return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 139 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
125 } 140 }
126 141
127 // Send to followers 142 // Send to followers
128 const audience = getObjectFollowersAudience(actorsInvolvedInVideo) 143 const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
129 const data = createActivityData(url, byActor, dislikeActivityData, audience) 144 const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience)
130 145
131 const actorsException = [ byActor ] 146 const actorsException = [ byActor ]
132 return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, actorsException) 147 return broadcastToFollowers(createActivity, byActor, actorsInvolvedInVideo, t, actorsException)
133} 148}
134 149
135function createActivityData (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { 150function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
136 if (!audience) audience = getAudience(byActor) 151 if (!audience) audience = getAudience(byActor)
137 152
138 return audiencify( 153 return audiencify(
@@ -146,7 +161,7 @@ function createActivityData (url: string, byActor: ActorModel, object: any, audi
146 ) 161 )
147} 162}
148 163
149function createDislikeActivityData (byActor: ActorModel, video: VideoModel) { 164function buildDislikeActivity (byActor: ActorModel, video: VideoModel) {
150 return { 165 return {
151 type: 'Dislike', 166 type: 'Dislike',
152 actor: byActor.url, 167 actor: byActor.url,
@@ -154,7 +169,7 @@ function createDislikeActivityData (byActor: ActorModel, video: VideoModel) {
154 } 169 }
155} 170}
156 171
157function createViewActivityData (byActor: ActorModel, video: VideoModel) { 172function buildViewActivity (byActor: ActorModel, video: VideoModel) {
158 return { 173 return {
159 type: 'View', 174 type: 'View',
160 actor: byActor.url, 175 actor: byActor.url,
@@ -167,9 +182,10 @@ function createViewActivityData (byActor: ActorModel, video: VideoModel) {
167export { 182export {
168 sendCreateVideo, 183 sendCreateVideo,
169 sendVideoAbuse, 184 sendVideoAbuse,
170 createActivityData, 185 buildCreateActivity,
171 sendCreateView, 186 sendCreateView,
172 sendCreateDislike, 187 sendCreateDislike,
173 createDislikeActivityData, 188 buildDislikeActivity,
174 sendCreateVideoComment 189 sendCreateVideoComment,
190 sendCreateCacheFile
175} 191}
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts
index 3d1dfb699..479182543 100644
--- a/server/lib/activitypub/send/send-delete.ts
+++ b/server/lib/activitypub/send/send-delete.ts
@@ -15,24 +15,23 @@ async function sendDeleteVideo (video: VideoModel, t: Transaction) {
15 const url = getDeleteActivityPubUrl(video.url) 15 const url = getDeleteActivityPubUrl(video.url)
16 const byActor = video.VideoChannel.Account.Actor 16 const byActor = video.VideoChannel.Account.Actor
17 17
18 const data = deleteActivityData(url, video.url, byActor) 18 const activity = buildDeleteActivity(url, video.url, byActor)
19 19
20 const actorsInvolved = await VideoShareModel.loadActorsByShare(video.id, t) 20 const actorsInvolved = await getActorsInvolvedInVideo(video, t)
21 actorsInvolved.push(byActor)
22 21
23 return broadcastToFollowers(data, byActor, actorsInvolved, t) 22 return broadcastToFollowers(activity, byActor, actorsInvolved, t)
24} 23}
25 24
26async function sendDeleteActor (byActor: ActorModel, t: Transaction) { 25async function sendDeleteActor (byActor: ActorModel, t: Transaction) {
27 logger.info('Creating job to broadcast delete of actor %s.', byActor.url) 26 logger.info('Creating job to broadcast delete of actor %s.', byActor.url)
28 27
29 const url = getDeleteActivityPubUrl(byActor.url) 28 const url = getDeleteActivityPubUrl(byActor.url)
30 const data = deleteActivityData(url, byActor.url, byActor) 29 const activity = buildDeleteActivity(url, byActor.url, byActor)
31 30
32 const actorsInvolved = await VideoShareModel.loadActorsByVideoOwner(byActor.id, t) 31 const actorsInvolved = await VideoShareModel.loadActorsByVideoOwner(byActor.id, t)
33 actorsInvolved.push(byActor) 32 actorsInvolved.push(byActor)
34 33
35 return broadcastToFollowers(data, byActor, actorsInvolved, t) 34 return broadcastToFollowers(activity, byActor, actorsInvolved, t)
36} 35}
37 36
38async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Transaction) { 37async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Transaction) {
@@ -45,23 +44,23 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
45 const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, t) 44 const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, t)
46 45
47 const actorsInvolvedInComment = await getActorsInvolvedInVideo(videoComment.Video, t) 46 const actorsInvolvedInComment = await getActorsInvolvedInVideo(videoComment.Video, t)
48 actorsInvolvedInComment.push(byActor) 47 actorsInvolvedInComment.push(byActor) // Add the actor that commented the video
49 48
50 const audience = getVideoCommentAudience(videoComment, threadParentComments, actorsInvolvedInComment, isVideoOrigin) 49 const audience = getVideoCommentAudience(videoComment, threadParentComments, actorsInvolvedInComment, isVideoOrigin)
51 const data = deleteActivityData(url, videoComment.url, byActor, audience) 50 const activity = buildDeleteActivity(url, videoComment.url, byActor, audience)
52 51
53 // This was a reply, send it to the parent actors 52 // This was a reply, send it to the parent actors
54 const actorsException = [ byActor ] 53 const actorsException = [ byActor ]
55 await broadcastToActors(data, byActor, threadParentComments.map(c => c.Account.Actor), actorsException) 54 await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), actorsException)
56 55
57 // Broadcast to our followers 56 // Broadcast to our followers
58 await broadcastToFollowers(data, byActor, [ byActor ], t) 57 await broadcastToFollowers(activity, byActor, [ byActor ], t)
59 58
60 // Send to actors involved in the comment 59 // Send to actors involved in the comment
61 if (isVideoOrigin) return broadcastToFollowers(data, byActor, actorsInvolvedInComment, t, actorsException) 60 if (isVideoOrigin) return broadcastToFollowers(activity, byActor, actorsInvolvedInComment, t, actorsException)
62 61
63 // Send to origin 62 // Send to origin
64 return unicastTo(data, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl) 63 return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
65} 64}
66 65
67// --------------------------------------------------------------------------- 66// ---------------------------------------------------------------------------
@@ -74,7 +73,7 @@ export {
74 73
75// --------------------------------------------------------------------------- 74// ---------------------------------------------------------------------------
76 75
77function deleteActivityData (url: string, object: string, byActor: ActorModel, audience?: ActivityAudience): ActivityDelete { 76function buildDeleteActivity (url: string, object: string, byActor: ActorModel, audience?: ActivityAudience): ActivityDelete {
78 const activity = { 77 const activity = {
79 type: 'Delete' as 'Delete', 78 type: 'Delete' as 'Delete',
80 id: url, 79 id: url,
diff --git a/server/lib/activitypub/send/send-follow.ts b/server/lib/activitypub/send/send-follow.ts
index 46d08c17b..170b46b48 100644
--- a/server/lib/activitypub/send/send-follow.ts
+++ b/server/lib/activitypub/send/send-follow.ts
@@ -15,12 +15,12 @@ function sendFollow (actorFollow: ActorFollowModel) {
15 logger.info('Creating job to send follow request to %s.', following.url) 15 logger.info('Creating job to send follow request to %s.', following.url)
16 16
17 const url = getActorFollowActivityPubUrl(actorFollow) 17 const url = getActorFollowActivityPubUrl(actorFollow)
18 const data = followActivityData(url, me, following) 18 const data = buildFollowActivity(url, me, following)
19 19
20 return unicastTo(data, me, following.inboxUrl) 20 return unicastTo(data, me, following.inboxUrl)
21} 21}
22 22
23function followActivityData (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow { 23function buildFollowActivity (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow {
24 return { 24 return {
25 type: 'Follow', 25 type: 'Follow',
26 id: url, 26 id: url,
@@ -33,5 +33,5 @@ function followActivityData (url: string, byActor: ActorModel, targetActor: Acto
33 33
34export { 34export {
35 sendFollow, 35 sendFollow,
36 followActivityData 36 buildFollowActivity
37} 37}
diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts
index 83225f5df..a5408ac6a 100644
--- a/server/lib/activitypub/send/send-like.ts
+++ b/server/lib/activitypub/send/send-like.ts
@@ -17,20 +17,20 @@ async function sendLike (byActor: ActorModel, video: VideoModel, t: Transaction)
17 // Send to origin 17 // Send to origin
18 if (video.isOwned() === false) { 18 if (video.isOwned() === false) {
19 const audience = getVideoAudience(video, accountsInvolvedInVideo) 19 const audience = getVideoAudience(video, accountsInvolvedInVideo)
20 const data = likeActivityData(url, byActor, video, audience) 20 const data = buildLikeActivity(url, byActor, video, audience)
21 21
22 return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 22 return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
23 } 23 }
24 24
25 // Send to followers 25 // Send to followers
26 const audience = getObjectFollowersAudience(accountsInvolvedInVideo) 26 const audience = getObjectFollowersAudience(accountsInvolvedInVideo)
27 const data = likeActivityData(url, byActor, video, audience) 27 const activity = buildLikeActivity(url, byActor, video, audience)
28 28
29 const followersException = [ byActor ] 29 const followersException = [ byActor ]
30 return broadcastToFollowers(data, byActor, accountsInvolvedInVideo, t, followersException) 30 return broadcastToFollowers(activity, byActor, accountsInvolvedInVideo, t, followersException)
31} 31}
32 32
33function likeActivityData (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike { 33function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike {
34 if (!audience) audience = getAudience(byActor) 34 if (!audience) audience = getAudience(byActor)
35 35
36 return audiencify( 36 return audiencify(
@@ -48,5 +48,5 @@ function likeActivityData (url: string, byActor: ActorModel, video: VideoModel,
48 48
49export { 49export {
50 sendLike, 50 sendLike,
51 likeActivityData 51 buildLikeActivity
52} 52}
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts
index 30d0fd98b..a50673c79 100644
--- a/server/lib/activitypub/send/send-undo.ts
+++ b/server/lib/activitypub/send/send-undo.ts
@@ -13,12 +13,13 @@ import { VideoModel } from '../../../models/video/video'
13import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' 13import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url'
14import { broadcastToFollowers, unicastTo } from './utils' 14import { broadcastToFollowers, unicastTo } from './utils'
15import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience' 15import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience'
16import { createActivityData, createDislikeActivityData } from './send-create' 16import { buildCreateActivity, buildDislikeActivity } from './send-create'
17import { followActivityData } from './send-follow' 17import { buildFollowActivity } from './send-follow'
18import { likeActivityData } from './send-like' 18import { buildLikeActivity } from './send-like'
19import { VideoShareModel } from '../../../models/video/video-share' 19import { VideoShareModel } from '../../../models/video/video-share'
20import { buildVideoAnnounce } from './send-announce' 20import { buildAnnounceWithVideoAudience } from './send-announce'
21import { logger } from '../../../helpers/logger' 21import { logger } from '../../../helpers/logger'
22import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
22 23
23async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { 24async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
24 const me = actorFollow.ActorFollower 25 const me = actorFollow.ActorFollower
@@ -32,10 +33,10 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
32 const followUrl = getActorFollowActivityPubUrl(actorFollow) 33 const followUrl = getActorFollowActivityPubUrl(actorFollow)
33 const undoUrl = getUndoActivityPubUrl(followUrl) 34 const undoUrl = getUndoActivityPubUrl(followUrl)
34 35
35 const object = followActivityData(followUrl, me, following) 36 const followActivity = buildFollowActivity(followUrl, me, following)
36 const data = undoActivityData(undoUrl, me, object) 37 const undoActivity = undoActivityData(undoUrl, me, followActivity)
37 38
38 return unicastTo(data, me, following.inboxUrl) 39 return unicastTo(undoActivity, me, following.inboxUrl)
39} 40}
40 41
41async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) { 42async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) {
@@ -45,21 +46,21 @@ async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transact
45 const undoUrl = getUndoActivityPubUrl(likeUrl) 46 const undoUrl = getUndoActivityPubUrl(likeUrl)
46 47
47 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 48 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
48 const object = likeActivityData(likeUrl, byActor, video) 49 const likeActivity = buildLikeActivity(likeUrl, byActor, video)
49 50
50 // Send to origin 51 // Send to origin
51 if (video.isOwned() === false) { 52 if (video.isOwned() === false) {
52 const audience = getVideoAudience(video, actorsInvolvedInVideo) 53 const audience = getVideoAudience(video, actorsInvolvedInVideo)
53 const data = undoActivityData(undoUrl, byActor, object, audience) 54 const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience)
54 55
55 return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 56 return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
56 } 57 }
57 58
58 const audience = getObjectFollowersAudience(actorsInvolvedInVideo) 59 const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
59 const data = undoActivityData(undoUrl, byActor, object, audience) 60 const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience)
60 61
61 const followersException = [ byActor ] 62 const followersException = [ byActor ]
62 return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException) 63 return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
63} 64}
64 65
65async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { 66async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
@@ -69,20 +70,20 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans
69 const undoUrl = getUndoActivityPubUrl(dislikeUrl) 70 const undoUrl = getUndoActivityPubUrl(dislikeUrl)
70 71
71 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 72 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
72 const dislikeActivity = createDislikeActivityData(byActor, video) 73 const dislikeActivity = buildDislikeActivity(byActor, video)
73 const object = createActivityData(dislikeUrl, byActor, dislikeActivity) 74 const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity)
74 75
75 if (video.isOwned() === false) { 76 if (video.isOwned() === false) {
76 const audience = getVideoAudience(video, actorsInvolvedInVideo) 77 const audience = getVideoAudience(video, actorsInvolvedInVideo)
77 const data = undoActivityData(undoUrl, byActor, object, audience) 78 const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity, audience)
78 79
79 return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 80 return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
80 } 81 }
81 82
82 const data = undoActivityData(undoUrl, byActor, object) 83 const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity)
83 84
84 const followersException = [ byActor ] 85 const followersException = [ byActor ]
85 return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException) 86 return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
86} 87}
87 88
88async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { 89async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
@@ -90,12 +91,27 @@ async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareMode
90 91
91 const undoUrl = getUndoActivityPubUrl(videoShare.url) 92 const undoUrl = getUndoActivityPubUrl(videoShare.url)
92 93
93 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 94 const { activity: announceActivity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t)
94 const object = await buildVideoAnnounce(byActor, videoShare, video, t) 95 const undoActivity = undoActivityData(undoUrl, byActor, announceActivity)
95 const data = undoActivityData(undoUrl, byActor, object)
96 96
97 const followersException = [ byActor ] 97 const followersException = [ byActor ]
98 return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException) 98 return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
99}
100
101async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
102 logger.info('Creating job to undo cache file %s.', redundancyModel.url)
103
104 const undoUrl = getUndoActivityPubUrl(redundancyModel.url)
105
106 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
107 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
108
109 const audience = getVideoAudience(video, actorsInvolvedInVideo)
110 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())
111
112 const undoActivity = undoActivityData(undoUrl, byActor, createActivity, audience)
113
114 return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
99} 115}
100 116
101// --------------------------------------------------------------------------- 117// ---------------------------------------------------------------------------
@@ -104,7 +120,8 @@ export {
104 sendUndoFollow, 120 sendUndoFollow,
105 sendUndoLike, 121 sendUndoLike,
106 sendUndoDislike, 122 sendUndoDislike,
107 sendUndoAnnounce 123 sendUndoAnnounce,
124 sendUndoCacheFile
108} 125}
109 126
110// --------------------------------------------------------------------------- 127// ---------------------------------------------------------------------------
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index 6f1d80898..605473338 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -7,11 +7,11 @@ import { VideoModel } from '../../../models/video/video'
7import { VideoChannelModel } from '../../../models/video/video-channel' 7import { VideoChannelModel } from '../../../models/video/video-channel'
8import { VideoShareModel } from '../../../models/video/video-share' 8import { VideoShareModel } from '../../../models/video/video-share'
9import { getUpdateActivityPubUrl } from '../url' 9import { getUpdateActivityPubUrl } from '../url'
10import { broadcastToFollowers } from './utils' 10import { broadcastToFollowers, unicastTo } from './utils'
11import { audiencify, getAudience } from '../audience' 11import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience'
12import { logger } from '../../../helpers/logger' 12import { logger } from '../../../helpers/logger'
13import { videoFeedsValidator } from '../../../middlewares/validators'
14import { VideoCaptionModel } from '../../../models/video/video-caption' 13import { VideoCaptionModel } from '../../../models/video/video-caption'
14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
15 15
16async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByActor?: ActorModel) { 16async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByActor?: ActorModel) {
17 logger.info('Creating job to update video %s.', video.url) 17 logger.info('Creating job to update video %s.', video.url)
@@ -26,12 +26,12 @@ async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByAct
26 const videoObject = video.toActivityPubObject() 26 const videoObject = video.toActivityPubObject()
27 const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) 27 const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
28 28
29 const data = updateActivityData(url, byActor, videoObject, audience) 29 const updateActivity = buildUpdateActivity(url, byActor, videoObject, audience)
30 30
31 const actorsInvolved = await VideoShareModel.loadActorsByShare(video.id, t) 31 const actorsInvolved = await getActorsInvolvedInVideo(video, t)
32 actorsInvolved.push(byActor) 32 if (overrodeByActor) actorsInvolved.push(overrodeByActor)
33 33
34 return broadcastToFollowers(data, byActor, actorsInvolved, t) 34 return broadcastToFollowers(updateActivity, byActor, actorsInvolved, t)
35} 35}
36 36
37async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelModel, t: Transaction) { 37async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelModel, t: Transaction) {
@@ -42,7 +42,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
42 const url = getUpdateActivityPubUrl(byActor.url, byActor.updatedAt.toISOString()) 42 const url = getUpdateActivityPubUrl(byActor.url, byActor.updatedAt.toISOString())
43 const accountOrChannelObject = accountOrChannel.toActivityPubObject() 43 const accountOrChannelObject = accountOrChannel.toActivityPubObject()
44 const audience = getAudience(byActor) 44 const audience = getAudience(byActor)
45 const data = updateActivityData(url, byActor, accountOrChannelObject, audience) 45 const updateActivity = buildUpdateActivity(url, byActor, accountOrChannelObject, audience)
46 46
47 let actorsInvolved: ActorModel[] 47 let actorsInvolved: ActorModel[]
48 if (accountOrChannel instanceof AccountModel) { 48 if (accountOrChannel instanceof AccountModel) {
@@ -55,19 +55,35 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
55 55
56 actorsInvolved.push(byActor) 56 actorsInvolved.push(byActor)
57 57
58 return broadcastToFollowers(data, byActor, actorsInvolved, t) 58 return broadcastToFollowers(updateActivity, byActor, actorsInvolved, t)
59}
60
61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
62 logger.info('Creating job to update cache file %s.', redundancyModel.url)
63
64 const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString())
65 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
66
67 const redundancyObject = redundancyModel.toActivityPubObject()
68
69 const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined)
70 const audience = getObjectFollowersAudience(accountsInvolvedInVideo)
71
72 const updateActivity = buildUpdateActivity(url, byActor, redundancyObject, audience)
73 return unicastTo(updateActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
59} 74}
60 75
61// --------------------------------------------------------------------------- 76// ---------------------------------------------------------------------------
62 77
63export { 78export {
64 sendUpdateActor, 79 sendUpdateActor,
65 sendUpdateVideo 80 sendUpdateVideo,
81 sendUpdateCacheFile
66} 82}
67 83
68// --------------------------------------------------------------------------- 84// ---------------------------------------------------------------------------
69 85
70function updateActivityData (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityUpdate { 86function buildUpdateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityUpdate {
71 if (!audience) audience = getAudience(byActor) 87 if (!audience) audience = getAudience(byActor)
72 88
73 return audiencify( 89 return audiencify(
diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts
index da437292e..c20c15633 100644
--- a/server/lib/activitypub/send/utils.ts
+++ b/server/lib/activitypub/send/utils.ts
@@ -59,11 +59,11 @@ async function forwardActivity (
59async function broadcastToFollowers ( 59async function broadcastToFollowers (
60 data: any, 60 data: any,
61 byActor: ActorModel, 61 byActor: ActorModel,
62 toActorFollowers: ActorModel[], 62 toFollowersOf: ActorModel[],
63 t: Transaction, 63 t: Transaction,
64 actorsException: ActorModel[] = [] 64 actorsException: ActorModel[] = []
65) { 65) {
66 const uris = await computeFollowerUris(toActorFollowers, actorsException, t) 66 const uris = await computeFollowerUris(toFollowersOf, actorsException, t)
67 return broadcastTo(uris, data, byActor) 67 return broadcastTo(uris, data, byActor)
68} 68}
69 69
@@ -115,8 +115,8 @@ export {
115 115
116// --------------------------------------------------------------------------- 116// ---------------------------------------------------------------------------
117 117
118async function computeFollowerUris (toActorFollower: ActorModel[], actorsException: ActorModel[], t: Transaction) { 118async function computeFollowerUris (toFollowersOf: ActorModel[], actorsException: ActorModel[], t: Transaction) {
119 const toActorFollowerIds = toActorFollower.map(a => a.id) 119 const toActorFollowerIds = toFollowersOf.map(a => a.id)
120 120
121 const result = await ActorFollowModel.listAcceptedFollowerSharedInboxUrls(toActorFollowerIds, t) 121 const result = await ActorFollowModel.listAcceptedFollowerSharedInboxUrls(toActorFollowerIds, t)
122 const sharedInboxesException = await buildSharedInboxesException(actorsException) 122 const sharedInboxesException = await buildSharedInboxesException(actorsException)
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts
index 262463310..2e7c56955 100644
--- a/server/lib/activitypub/url.ts
+++ b/server/lib/activitypub/url.ts
@@ -4,11 +4,18 @@ import { ActorFollowModel } from '../../models/activitypub/actor-follow'
4import { VideoModel } from '../../models/video/video' 4import { VideoModel } from '../../models/video/video'
5import { VideoAbuseModel } from '../../models/video/video-abuse' 5import { VideoAbuseModel } from '../../models/video/video-abuse'
6import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
7import { VideoFileModel } from '../../models/video/video-file'
7 8
8function getVideoActivityPubUrl (video: VideoModel) { 9function getVideoActivityPubUrl (video: VideoModel) {
9 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid 10 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid
10} 11}
11 12
13function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) {
14 const suffixFPS = videoFile.fps ? '-' + videoFile.fps : ''
15
16 return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}`
17}
18
12function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { 19function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) {
13 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id 20 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id
14} 21}
@@ -101,5 +108,6 @@ export {
101 getVideoSharesActivityPubUrl, 108 getVideoSharesActivityPubUrl,
102 getVideoCommentsActivityPubUrl, 109 getVideoCommentsActivityPubUrl,
103 getVideoLikesActivityPubUrl, 110 getVideoLikesActivityPubUrl,
104 getVideoDislikesActivityPubUrl 111 getVideoDislikesActivityPubUrl,
112 getVideoCacheFileActivityPubUrl
105} 113}
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 6c2095897..783f78d3e 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -3,12 +3,12 @@ import * as sequelize from 'sequelize'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import { join } from 'path' 4import { join } from 'path'
5import * as request from 'request' 5import * as request from 'request'
6import { ActivityIconObject, VideoState } from '../../../shared/index' 6import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index'
7import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 7import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
8import { VideoPrivacy } from '../../../shared/models/videos' 8import { VideoPrivacy } from '../../../shared/models/videos'
9import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' 9import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
10import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' 10import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
11import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' 11import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
12import { logger } from '../../helpers/logger' 12import { logger } from '../../helpers/logger'
13import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' 13import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
14import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' 14import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
@@ -17,7 +17,7 @@ import { TagModel } from '../../models/video/tag'
17import { VideoModel } from '../../models/video/video' 17import { VideoModel } from '../../models/video/video'
18import { VideoChannelModel } from '../../models/video/video-channel' 18import { VideoChannelModel } from '../../models/video/video-channel'
19import { VideoFileModel } from '../../models/video/video-file' 19import { VideoFileModel } from '../../models/video/video-file'
20import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor' 20import { getOrCreateActorAndServerAndModel } from './actor'
21import { addVideoComments } from './video-comments' 21import { addVideoComments } from './video-comments'
22import { crawlCollectionPage } from './crawl' 22import { crawlCollectionPage } from './crawl'
23import { sendCreateVideo, sendUpdateVideo } from './send' 23import { sendCreateVideo, sendUpdateVideo } from './send'
@@ -25,7 +25,6 @@ import { isArray } from '../../helpers/custom-validators/misc'
25import { VideoCaptionModel } from '../../models/video/video-caption' 25import { VideoCaptionModel } from '../../models/video/video-caption'
26import { JobQueue } from '../job-queue' 26import { JobQueue } from '../job-queue'
27import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher' 27import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
28import { getUrlFromWebfinger } from '../../helpers/webfinger'
29import { createRates } from './video-rates' 28import { createRates } from './video-rates'
30import { addVideoShares, shareVideoByServerAndChannel } from './share' 29import { addVideoShares, shareVideoByServerAndChannel } from './share'
31import { AccountModel } from '../../models/account/account' 30import { AccountModel } from '../../models/account/account'
@@ -137,10 +136,7 @@ async function videoActivityObjectToDBAttributes (
137} 136}
138 137
139function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { 138function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
140 const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) 139 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
141 const fileUrls = videoObject.url.filter(u => {
142 return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/')
143 })
144 140
145 if (fileUrls.length === 0) { 141 if (fileUrls.length === 0) {
146 throw new Error('Cannot find video files for ' + videoCreated.url) 142 throw new Error('Cannot find video files for ' + videoCreated.url)
@@ -331,8 +327,8 @@ async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
331 327
332 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) 328 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
333 const account = await AccountModel.load(channelActor.VideoChannel.accountId) 329 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
334 return updateVideoFromAP(video, videoObject, account.Actor, channelActor)
335 330
331 return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel)
336 } catch (err) { 332 } catch (err) {
337 logger.warn('Cannot refresh video.', { err }) 333 logger.warn('Cannot refresh video.', { err })
338 return video 334 return video
@@ -342,8 +338,8 @@ async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
342async function updateVideoFromAP ( 338async function updateVideoFromAP (
343 video: VideoModel, 339 video: VideoModel,
344 videoObject: VideoTorrentObject, 340 videoObject: VideoTorrentObject,
345 accountActor: ActorModel, 341 account: AccountModel,
346 channelActor: ActorModel, 342 channel: VideoChannelModel,
347 overrideTo?: string[] 343 overrideTo?: string[]
348) { 344) {
349 logger.debug('Updating remote video "%s".', videoObject.uuid) 345 logger.debug('Updating remote video "%s".', videoObject.uuid)
@@ -359,12 +355,12 @@ async function updateVideoFromAP (
359 355
360 // Check actor has the right to update the video 356 // Check actor has the right to update the video
361 const videoChannel = video.VideoChannel 357 const videoChannel = video.VideoChannel
362 if (videoChannel.Account.Actor.id !== accountActor.id) { 358 if (videoChannel.Account.id !== account.id) {
363 throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url) 359 throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
364 } 360 }
365 361
366 const to = overrideTo ? overrideTo : videoObject.to 362 const to = overrideTo ? overrideTo : videoObject.to
367 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to) 363 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to)
368 video.set('name', videoData.name) 364 video.set('name', videoData.name)
369 video.set('uuid', videoData.uuid) 365 video.set('uuid', videoData.uuid)
370 video.set('url', videoData.url) 366 video.set('url', videoData.url)
@@ -444,3 +440,11 @@ export {
444 addVideoShares, 440 addVideoShares,
445 createRates 441 createRates
446} 442}
443
444// ---------------------------------------------------------------------------
445
446function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
447 const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
448
449 return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/')
450}
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts
new file mode 100644
index 000000000..78221cc3d
--- /dev/null
+++ b/server/lib/redundancy.ts
@@ -0,0 +1,18 @@
1import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
2import { sendUndoCacheFile } from './activitypub/send'
3import { Transaction } from 'sequelize'
4import { getServerActor } from '../helpers/utils'
5
6async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?: Transaction) {
7 const serverActor = await getServerActor()
8
9 await sendUndoCacheFile(serverActor, videoRedundancy, t)
10
11 await videoRedundancy.destroy({ transaction: t })
12}
13
14// ---------------------------------------------------------------------------
15
16export {
17 removeVideoRedundancy
18}
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
new file mode 100644
index 000000000..ee9ba1766
--- /dev/null
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -0,0 +1,161 @@
1import { AbstractScheduler } from './abstract-scheduler'
2import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers'
3import { logger } from '../../helpers/logger'
4import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6import { VideoFileModel } from '../../models/video/video-file'
7import { sortBy } from 'lodash'
8import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
9import { join } from 'path'
10import { rename } from 'fs-extra'
11import { getServerActor } from '../../helpers/utils'
12import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
13import { VideoModel } from '../../models/video/video'
14import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
15import { removeVideoRedundancy } from '../redundancy'
16import { isTestInstance } from '../../helpers/core-utils'
17
18export class VideosRedundancyScheduler extends AbstractScheduler {
19
20 private static instance: AbstractScheduler
21 private executing = false
22
23 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy
24
25 private constructor () {
26 super()
27 }
28
29 async execute () {
30 if (this.executing) return
31
32 this.executing = true
33
34 for (const obj of CONFIG.REDUNDANCY.VIDEOS) {
35
36 try {
37 const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy)
38 if (!videoToDuplicate) continue
39
40 const videoFiles = videoToDuplicate.VideoFiles
41 videoFiles.forEach(f => f.Video = videoToDuplicate)
42
43 const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy)
44 if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) {
45 if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
46 continue
47 }
48
49 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy)
50
51 await this.createVideoRedundancy(obj.strategy, videoFiles)
52 } catch (err) {
53 logger.error('Cannot run videos redundancy %s.', obj.strategy, { err })
54 }
55 }
56
57 const expired = await VideoRedundancyModel.listAllExpired()
58
59 for (const m of expired) {
60 logger.info('Removing expired video %s from our redundancy system.', this.buildEntryLogId(m))
61
62 try {
63 await m.destroy()
64 } catch (err) {
65 logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m))
66 }
67 }
68
69 this.executing = false
70 }
71
72 static get Instance () {
73 return this.instance || (this.instance = new this())
74 }
75
76 private findVideoToDuplicate (strategy: VideoRedundancyStrategy) {
77 if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
78 }
79
80 private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) {
81 const serverActor = await getServerActor()
82
83 for (const file of filesToDuplicate) {
84 const existing = await VideoRedundancyModel.loadByFileId(file.id)
85 if (existing) {
86 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', file.Video.url, file.resolution, strategy)
87
88 existing.expiresOn = this.buildNewExpiration()
89 await existing.save()
90
91 await sendUpdateCacheFile(serverActor, existing)
92 continue
93 }
94
95 // We need more attributes and check if the video still exists
96 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id)
97 if (!video) continue
98
99 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy)
100
101 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
102 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
103
104 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, JOB_TTL['video-import'])
105
106 const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file))
107 await rename(tmpPath, destPath)
108
109 const createdModel = await VideoRedundancyModel.create({
110 expiresOn: new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS),
111 url: getVideoCacheFileActivityPubUrl(file),
112 fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL),
113 strategy,
114 videoFileId: file.id,
115 actorId: serverActor.id
116 })
117 createdModel.VideoFile = file
118
119 await sendCreateCacheFile(serverActor, createdModel)
120 }
121 }
122
123 // Unused, but could be useful in the future, with a custom strategy
124 private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) {
125 const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt')
126
127 while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) {
128 const toDelete = sortedVideosRedundancy.shift()
129
130 const videoFile = toDelete.VideoFile
131 logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution)
132
133 await removeVideoRedundancy(toDelete, undefined)
134 }
135
136 return sortedVideosRedundancy
137 }
138
139 private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
140 const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate)
141
142 const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size
143 const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0)
144
145 return totalDuplicated > maxSize
146 }
147
148 private buildNewExpiration () {
149 return new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS)
150 }
151
152 private buildEntryLogId (object: VideoRedundancyModel) {
153 return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
154 }
155
156 private getTotalFileSizes (files: VideoFileModel[]) {
157 const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size
158
159 return files.reduce(fileReducer, 0)
160 }
161}
diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts
new file mode 100644
index 000000000..d91b47574
--- /dev/null
+++ b/server/middlewares/validators/redundancy.ts
@@ -0,0 +1,80 @@
1import * as express from 'express'
2import 'express-validator'
3import { param, body } from 'express-validator/check'
4import { exists, isBooleanValid, isIdOrUUIDValid, toIntOrNull } from '../../helpers/custom-validators/misc'
5import { isVideoExist } from '../../helpers/custom-validators/videos'
6import { logger } from '../../helpers/logger'
7import { areValidationErrors } from './utils'
8import { VideoModel } from '../../models/video/video'
9import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
10import { isHostValid } from '../../helpers/custom-validators/servers'
11import { getServerActor } from '../../helpers/utils'
12import { ActorFollowModel } from '../../models/activitypub/actor-follow'
13import { SERVER_ACTOR_NAME } from '../../initializers'
14import { ServerModel } from '../../models/server/server'
15
16const videoRedundancyGetValidator = [
17 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
18 param('resolution')
19 .customSanitizer(toIntOrNull)
20 .custom(exists).withMessage('Should have a valid resolution'),
21 param('fps')
22 .optional()
23 .customSanitizer(toIntOrNull)
24 .custom(exists).withMessage('Should have a valid fps'),
25
26 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
27 logger.debug('Checking videoRedundancyGetValidator parameters', { parameters: req.params })
28
29 if (areValidationErrors(req, res)) return
30 if (!await isVideoExist(req.params.videoId, res)) return
31
32 const video: VideoModel = res.locals.video
33 const videoFile = video.VideoFiles.find(f => {
34 return f.resolution === req.params.resolution && (!req.params.fps || f.fps === req.params.fps)
35 })
36
37 if (!videoFile) return res.status(404).json({ error: 'Video file not found.' })
38 res.locals.videoFile = videoFile
39
40 const videoRedundancy = await VideoRedundancyModel.loadByFileId(videoFile.id)
41 if (!videoRedundancy)return res.status(404).json({ error: 'Video redundancy not found.' })
42 res.locals.videoRedundancy = videoRedundancy
43
44 return next()
45 }
46]
47
48const updateServerRedundancyValidator = [
49 param('host').custom(isHostValid).withMessage('Should have a valid host'),
50 body('redundancyAllowed')
51 .toBoolean()
52 .custom(isBooleanValid).withMessage('Should have a valid redundancyAllowed attribute'),
53
54 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
55 logger.debug('Checking updateServerRedundancy parameters', { parameters: req.params })
56
57 if (areValidationErrors(req, res)) return
58
59 const server = await ServerModel.loadByHost(req.params.host)
60
61 if (!server) {
62 return res
63 .status(404)
64 .json({
65 error: `Server ${req.params.host} not found.`
66 })
67 .end()
68 }
69
70 res.locals.server = server
71 return next()
72 }
73]
74
75// ---------------------------------------------------------------------------
76
77export {
78 videoRedundancyGetValidator,
79 updateServerRedundancyValidator
80}
diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts
index 8bc095997..27bb43dae 100644
--- a/server/models/activitypub/actor-follow.ts
+++ b/server/models/activitypub/actor-follow.ts
@@ -19,7 +19,7 @@ import {
19 UpdatedAt 19 UpdatedAt
20} from 'sequelize-typescript' 20} from 'sequelize-typescript'
21import { FollowState } from '../../../shared/models/actors' 21import { FollowState } from '../../../shared/models/actors'
22import { AccountFollow } from '../../../shared/models/actors/follow.model' 22import { ActorFollow } from '../../../shared/models/actors/follow.model'
23import { logger } from '../../helpers/logger' 23import { logger } from '../../helpers/logger'
24import { getServerActor } from '../../helpers/utils' 24import { getServerActor } from '../../helpers/utils'
25import { ACTOR_FOLLOW_SCORE } from '../../initializers' 25import { ACTOR_FOLLOW_SCORE } from '../../initializers'
@@ -529,7 +529,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
529 return ActorFollowModel.findAll(query) 529 return ActorFollowModel.findAll(query)
530 } 530 }
531 531
532 toFormattedJSON (): AccountFollow { 532 toFormattedJSON (): ActorFollow {
533 const follower = this.ActorFollower.toFormattedJSON() 533 const follower = this.ActorFollower.toFormattedJSON()
534 const following = this.ActorFollowing.toFormattedJSON() 534 const following = this.ActorFollowing.toFormattedJSON()
535 535
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index 119d0c1da..ef8dd9f7c 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -76,7 +76,13 @@ export const unusedActorAttributesForAPI = [
76 }, 76 },
77 { 77 {
78 model: () => VideoChannelModel.unscoped(), 78 model: () => VideoChannelModel.unscoped(),
79 required: false 79 required: false,
80 include: [
81 {
82 model: () => AccountModel,
83 required: true
84 }
85 ]
80 }, 86 },
81 { 87 {
82 model: () => ServerModel, 88 model: () => ServerModel,
@@ -337,6 +343,7 @@ export class ActorModel extends Model<ActorModel> {
337 uuid: this.uuid, 343 uuid: this.uuid,
338 name: this.preferredUsername, 344 name: this.preferredUsername,
339 host: this.getHost(), 345 host: this.getHost(),
346 hostRedundancyAllowed: this.getRedundancyAllowed(),
340 followingCount: this.followingCount, 347 followingCount: this.followingCount,
341 followersCount: this.followersCount, 348 followersCount: this.followersCount,
342 avatar, 349 avatar,
@@ -440,6 +447,10 @@ export class ActorModel extends Model<ActorModel> {
440 return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST 447 return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST
441 } 448 }
442 449
450 getRedundancyAllowed () {
451 return this.Server ? this.Server.redundancyAllowed : false
452 }
453
443 getAvatarUrl () { 454 getAvatarUrl () {
444 if (!this.avatarId) return undefined 455 if (!this.avatarId) return undefined
445 456
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
new file mode 100644
index 000000000..48ec77206
--- /dev/null
+++ b/server/models/redundancy/video-redundancy.ts
@@ -0,0 +1,249 @@
1import {
2 AfterDestroy,
3 AllowNull,
4 BelongsTo,
5 Column,
6 CreatedAt,
7 DataType,
8 ForeignKey,
9 Is,
10 Model,
11 Scopes,
12 Sequelize,
13 Table,
14 UpdatedAt
15} from 'sequelize-typescript'
16import { ActorModel } from '../activitypub/actor'
17import { throwIfNotValid } from '../utils'
18import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
19import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers'
20import { VideoFileModel } from '../video/video-file'
21import { isDateValid } from '../../helpers/custom-validators/misc'
22import { getServerActor } from '../../helpers/utils'
23import { VideoModel } from '../video/video'
24import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
25import { logger } from '../../helpers/logger'
26import { CacheFileObject } from '../../../shared'
27import { VideoChannelModel } from '../video/video-channel'
28import { ServerModel } from '../server/server'
29import { sample } from 'lodash'
30import { isTestInstance } from '../../helpers/core-utils'
31
32export enum ScopeNames {
33 WITH_VIDEO = 'WITH_VIDEO'
34}
35
36@Scopes({
37 [ ScopeNames.WITH_VIDEO ]: {
38 include: [
39 {
40 model: () => VideoFileModel,
41 required: true,
42 include: [
43 {
44 model: () => VideoModel,
45 required: true
46 }
47 ]
48 }
49 ]
50 }
51})
52
53@Table({
54 tableName: 'videoRedundancy',
55 indexes: [
56 {
57 fields: [ 'videoFileId' ]
58 },
59 {
60 fields: [ 'actorId' ]
61 },
62 {
63 fields: [ 'url' ],
64 unique: true
65 }
66 ]
67})
68export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
69
70 @CreatedAt
71 createdAt: Date
72
73 @UpdatedAt
74 updatedAt: Date
75
76 @AllowNull(false)
77 @Column
78 expiresOn: Date
79
80 @AllowNull(false)
81 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
82 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
83 fileUrl: string
84
85 @AllowNull(false)
86 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
87 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
88 url: string
89
90 @AllowNull(true)
91 @Column
92 strategy: string // Only used by us
93
94 @ForeignKey(() => VideoFileModel)
95 @Column
96 videoFileId: number
97
98 @BelongsTo(() => VideoFileModel, {
99 foreignKey: {
100 allowNull: false
101 },
102 onDelete: 'cascade'
103 })
104 VideoFile: VideoFileModel
105
106 @ForeignKey(() => ActorModel)
107 @Column
108 actorId: number
109
110 @BelongsTo(() => ActorModel, {
111 foreignKey: {
112 allowNull: false
113 },
114 onDelete: 'cascade'
115 })
116 Actor: ActorModel
117
118 @AfterDestroy
119 static removeFilesAndSendDelete (instance: VideoRedundancyModel) {
120 // Not us
121 if (!instance.strategy) return
122
123 logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution)
124
125 return instance.VideoFile.Video.removeFile(instance.VideoFile)
126 }
127
128 static loadByFileId (videoFileId: number) {
129 const query = {
130 where: {
131 videoFileId
132 }
133 }
134
135 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
136 }
137
138 static loadByUrl (url: string) {
139 const query = {
140 where: {
141 url
142 }
143 }
144
145 return VideoRedundancyModel.findOne(query)
146 }
147
148 static async findMostViewToDuplicate (randomizedFactor: number) {
149 // On VideoModel!
150 const query = {
151 logging: !isTestInstance(),
152 limit: randomizedFactor,
153 order: [ [ 'views', 'DESC' ] ],
154 include: [
155 {
156 model: VideoFileModel.unscoped(),
157 required: true,
158 where: {
159 id: {
160 [ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn()
161 }
162 }
163 },
164 {
165 attributes: [],
166 model: VideoChannelModel.unscoped(),
167 required: true,
168 include: [
169 {
170 attributes: [],
171 model: ActorModel.unscoped(),
172 required: true,
173 include: [
174 {
175 attributes: [],
176 model: ServerModel.unscoped(),
177 required: true,
178 where: {
179 redundancyAllowed: true
180 }
181 }
182 ]
183 }
184 ]
185 }
186 ]
187 }
188
189 const rows = await VideoModel.unscoped().findAll(query)
190
191 return sample(rows)
192 }
193
194 static async getVideoFiles (strategy: VideoRedundancyStrategy) {
195 const actor = await getServerActor()
196
197 const queryVideoFiles = {
198 logging: !isTestInstance(),
199 where: {
200 actorId: actor.id,
201 strategy
202 }
203 }
204
205 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
206 .findAll(queryVideoFiles)
207 }
208
209 static listAllExpired () {
210 const query = {
211 logging: !isTestInstance(),
212 where: {
213 expiresOn: {
214 [Sequelize.Op.lt]: new Date()
215 }
216 }
217 }
218
219 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
220 .findAll(query)
221 }
222
223 toActivityPubObject (): CacheFileObject {
224 return {
225 id: this.url,
226 type: 'CacheFile' as 'CacheFile',
227 object: this.VideoFile.Video.url,
228 expires: this.expiresOn.toISOString(),
229 url: {
230 type: 'Link',
231 mimeType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any,
232 href: this.fileUrl,
233 height: this.VideoFile.resolution,
234 size: this.VideoFile.size,
235 fps: this.VideoFile.fps
236 }
237 }
238 }
239
240 private static async buildExcludeIn () {
241 const actor = await getServerActor()
242
243 return Sequelize.literal(
244 '(' +
245 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` +
246 ')'
247 )
248 }
249}
diff --git a/server/models/server/server.ts b/server/models/server/server.ts
index 9749f503e..ca3b24d51 100644
--- a/server/models/server/server.ts
+++ b/server/models/server/server.ts
@@ -1,4 +1,4 @@
1import { AllowNull, Column, CreatedAt, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' 1import { AllowNull, Column, CreatedAt, Default, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
2import { isHostValid } from '../../helpers/custom-validators/servers' 2import { isHostValid } from '../../helpers/custom-validators/servers'
3import { ActorModel } from '../activitypub/actor' 3import { ActorModel } from '../activitypub/actor'
4import { throwIfNotValid } from '../utils' 4import { throwIfNotValid } from '../utils'
@@ -19,6 +19,11 @@ export class ServerModel extends Model<ServerModel> {
19 @Column 19 @Column
20 host: string 20 host: string
21 21
22 @AllowNull(false)
23 @Default(false)
24 @Column
25 redundancyAllowed: boolean
26
22 @CreatedAt 27 @CreatedAt
23 createdAt: Date 28 createdAt: Date
24 29
@@ -34,4 +39,14 @@ export class ServerModel extends Model<ServerModel> {
34 hooks: true 39 hooks: true
35 }) 40 })
36 Actors: ActorModel[] 41 Actors: ActorModel[]
42
43 static loadByHost (host: string) {
44 const query = {
45 where: {
46 host
47 }
48 }
49
50 return ServerModel.findOne(query)
51 }
37} 52}
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 3bc4855f3..0907ea569 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -1,5 +1,18 @@
1import { values } from 'lodash' 1import { values } from 'lodash'
2import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' 2import {
3 AllowNull,
4 BelongsTo,
5 Column,
6 CreatedAt,
7 DataType,
8 Default,
9 ForeignKey,
10 HasMany,
11 Is,
12 Model,
13 Table,
14 UpdatedAt
15} from 'sequelize-typescript'
3import { 16import {
4 isVideoFileInfoHashValid, 17 isVideoFileInfoHashValid,
5 isVideoFileResolutionValid, 18 isVideoFileResolutionValid,
@@ -10,6 +23,7 @@ import { CONSTRAINTS_FIELDS } from '../../initializers'
10import { throwIfNotValid } from '../utils' 23import { throwIfNotValid } from '../utils'
11import { VideoModel } from './video' 24import { VideoModel } from './video'
12import * as Sequelize from 'sequelize' 25import * as Sequelize from 'sequelize'
26import { VideoRedundancyModel } from '../redundancy/video-redundancy'
13 27
14@Table({ 28@Table({
15 tableName: 'videoFile', 29 tableName: 'videoFile',
@@ -70,6 +84,15 @@ export class VideoFileModel extends Model<VideoFileModel> {
70 }) 84 })
71 Video: VideoModel 85 Video: VideoModel
72 86
87 @HasMany(() => VideoRedundancyModel, {
88 foreignKey: {
89 allowNull: false
90 },
91 onDelete: 'CASCADE',
92 hooks: true
93 })
94 RedundancyVideos: VideoRedundancyModel[]
95
73 static isInfohashExists (infoHash: string) { 96 static isInfohashExists (infoHash: string) {
74 const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' 97 const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
75 const options = { 98 const options = {
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 86316653f..27c631dcd 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -27,13 +27,13 @@ import {
27 Table, 27 Table,
28 UpdatedAt 28 UpdatedAt
29} from 'sequelize-typescript' 29} from 'sequelize-typescript'
30import { VideoPrivacy, VideoResolution, VideoState } from '../../../shared' 30import { ActivityUrlObject, VideoPrivacy, VideoResolution, VideoState } from '../../../shared'
31import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 31import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
32import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' 32import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
33import { VideoFilter } from '../../../shared/models/videos/video-query.type' 33import { VideoFilter } from '../../../shared/models/videos/video-query.type'
34import { createTorrentPromise, peertubeTruncate } from '../../helpers/core-utils' 34import { createTorrentPromise, peertubeTruncate } from '../../helpers/core-utils'
35import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 35import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
36import { isBooleanValid } from '../../helpers/custom-validators/misc' 36import { isArray, isBooleanValid } from '../../helpers/custom-validators/misc'
37import { 37import {
38 isVideoCategoryValid, 38 isVideoCategoryValid,
39 isVideoDescriptionValid, 39 isVideoDescriptionValid,
@@ -90,6 +90,7 @@ import { VideoCaptionModel } from './video-caption'
90import { VideoBlacklistModel } from './video-blacklist' 90import { VideoBlacklistModel } from './video-blacklist'
91import { copy, remove, rename, stat, writeFile } from 'fs-extra' 91import { copy, remove, rename, stat, writeFile } from 'fs-extra'
92import { VideoViewModel } from './video-views' 92import { VideoViewModel } from './video-views'
93import { VideoRedundancyModel } from '../redundancy/video-redundancy'
93 94
94// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation 95// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
95const indexes: Sequelize.DefineIndexesOptions[] = [ 96const indexes: Sequelize.DefineIndexesOptions[] = [
@@ -470,7 +471,13 @@ type AvailableForListIDsOptions = {
470 include: [ 471 include: [
471 { 472 {
472 model: () => VideoFileModel.unscoped(), 473 model: () => VideoFileModel.unscoped(),
473 required: false 474 required: false,
475 include: [
476 {
477 model: () => VideoRedundancyModel.unscoped(),
478 required: false
479 }
480 ]
474 } 481 }
475 ] 482 ]
476 }, 483 },
@@ -633,6 +640,7 @@ export class VideoModel extends Model<VideoModel> {
633 name: 'videoId', 640 name: 'videoId',
634 allowNull: false 641 allowNull: false
635 }, 642 },
643 hooks: true,
636 onDelete: 'cascade' 644 onDelete: 'cascade'
637 }) 645 })
638 VideoFiles: VideoFileModel[] 646 VideoFiles: VideoFileModel[]
@@ -1325,9 +1333,7 @@ export class VideoModel extends Model<VideoModel> {
1325 [ CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + '/tracker/socket' ], 1333 [ CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + '/tracker/socket' ],
1326 [ CONFIG.WEBSERVER.URL + '/tracker/announce' ] 1334 [ CONFIG.WEBSERVER.URL + '/tracker/announce' ]
1327 ], 1335 ],
1328 urlList: [ 1336 urlList: [ CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) ]
1329 CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
1330 ]
1331 } 1337 }
1332 1338
1333 const torrent = await createTorrentPromise(this.getVideoFilePath(videoFile), options) 1339 const torrent = await createTorrentPromise(this.getVideoFilePath(videoFile), options)
@@ -1535,11 +1541,11 @@ export class VideoModel extends Model<VideoModel> {
1535 } 1541 }
1536 } 1542 }
1537 1543
1538 const url = [] 1544 const url: ActivityUrlObject[] = []
1539 for (const file of this.VideoFiles) { 1545 for (const file of this.VideoFiles) {
1540 url.push({ 1546 url.push({
1541 type: 'Link', 1547 type: 'Link',
1542 mimeType: VIDEO_EXT_MIMETYPE[ file.extname ], 1548 mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any,
1543 href: this.getVideoFileUrl(file, baseUrlHttp), 1549 href: this.getVideoFileUrl(file, baseUrlHttp),
1544 height: file.resolution, 1550 height: file.resolution,
1545 size: file.size, 1551 size: file.size,
@@ -1548,14 +1554,14 @@ export class VideoModel extends Model<VideoModel> {
1548 1554
1549 url.push({ 1555 url.push({
1550 type: 'Link', 1556 type: 'Link',
1551 mimeType: 'application/x-bittorrent', 1557 mimeType: 'application/x-bittorrent' as 'application/x-bittorrent',
1552 href: this.getTorrentUrl(file, baseUrlHttp), 1558 href: this.getTorrentUrl(file, baseUrlHttp),
1553 height: file.resolution 1559 height: file.resolution
1554 }) 1560 })
1555 1561
1556 url.push({ 1562 url.push({
1557 type: 'Link', 1563 type: 'Link',
1558 mimeType: 'application/x-bittorrent;x-scheme-handler/magnet', 1564 mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet',
1559 href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs), 1565 href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs),
1560 height: file.resolution 1566 height: file.resolution
1561 }) 1567 })
@@ -1796,7 +1802,7 @@ export class VideoModel extends Model<VideoModel> {
1796 (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL 1802 (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL
1797 } 1803 }
1798 1804
1799 private getBaseUrls () { 1805 getBaseUrls () {
1800 let baseUrlHttp 1806 let baseUrlHttp
1801 let baseUrlWs 1807 let baseUrlWs
1802 1808
@@ -1811,39 +1817,42 @@ export class VideoModel extends Model<VideoModel> {
1811 return { baseUrlHttp, baseUrlWs } 1817 return { baseUrlHttp, baseUrlWs }
1812 } 1818 }
1813 1819
1814 private getThumbnailUrl (baseUrlHttp: string) { 1820 generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
1821 const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
1822 const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
1823 let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
1824
1825 const redundancies = videoFile.RedundancyVideos
1826 if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl))
1827
1828 const magnetHash = {
1829 xs,
1830 announce,
1831 urlList,
1832 infoHash: videoFile.infoHash,
1833 name: this.name
1834 }
1835
1836 return magnetUtil.encode(magnetHash)
1837 }
1838
1839 getThumbnailUrl (baseUrlHttp: string) {
1815 return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() 1840 return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName()
1816 } 1841 }
1817 1842
1818 private getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) { 1843 getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
1819 return baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentFileName(videoFile) 1844 return baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
1820 } 1845 }
1821 1846
1822 private getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { 1847 getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
1823 return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + this.getTorrentFileName(videoFile) 1848 return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
1824 } 1849 }
1825 1850
1826 private getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) { 1851 getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
1827 return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) 1852 return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
1828 } 1853 }
1829 1854
1830 private getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { 1855 getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
1831 return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) 1856 return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
1832 } 1857 }
1833
1834 private generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
1835 const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
1836 const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
1837 const urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
1838
1839 const magnetHash = {
1840 xs,
1841 announce,
1842 urlList,
1843 infoHash: videoFile.infoHash,
1844 name: this.name
1845 }
1846
1847 return magnetUtil.encode(magnetHash)
1848 }
1849} 1858}
diff --git a/server/tests/api/check-params/follows.ts b/server/tests/api/check-params/follows.ts
index 2bc3b27d9..cdc95c81a 100644
--- a/server/tests/api/check-params/follows.ts
+++ b/server/tests/api/check-params/follows.ts
@@ -169,15 +169,6 @@ describe('Test server follows API validators', function () {
169 statusCodeExpected: 404 169 statusCodeExpected: 404
170 }) 170 })
171 }) 171 })
172
173 it('Should succeed with the correct parameters', async function () {
174 await makeDeleteRequest({
175 url: server.url,
176 path: path + '/localhost:9002',
177 token: server.accessToken,
178 statusCodeExpected: 404
179 })
180 })
181 }) 172 })
182 }) 173 })
183 174
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index 777acbb0f..44460a167 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -1,15 +1,17 @@
1// Order of the tests we want to execute 1// Order of the tests we want to execute
2import './accounts' 2import './accounts'
3import './config'
3import './follows' 4import './follows'
4import './jobs' 5import './jobs'
6import './redundancy'
7import './search'
5import './services' 8import './services'
9import './user-subscriptions'
6import './users' 10import './users'
7import './video-abuses' 11import './video-abuses'
8import './video-blacklist' 12import './video-blacklist'
9import './video-captions' 13import './video-captions'
10import './video-channels' 14import './video-channels'
11import './video-comments' 15import './video-comments'
12import './videos'
13import './video-imports' 16import './video-imports'
14import './search' 17import './videos'
15import './user-subscriptions'
diff --git a/server/tests/api/check-params/redundancy.ts b/server/tests/api/check-params/redundancy.ts
new file mode 100644
index 000000000..aa588e3dd
--- /dev/null
+++ b/server/tests/api/check-params/redundancy.ts
@@ -0,0 +1,103 @@
1/* tslint:disable:no-unused-expression */
2
3import 'mocha'
4
5import {
6 createUser,
7 doubleFollow,
8 flushAndRunMultipleServers,
9 flushTests,
10 killallServers,
11 makePutBodyRequest,
12 ServerInfo,
13 setAccessTokensToServers,
14 userLogin
15} from '../../utils'
16
17describe('Test server redundancy API validators', function () {
18 let servers: ServerInfo[]
19 let userAccessToken = null
20
21 // ---------------------------------------------------------------
22
23 before(async function () {
24 this.timeout(30000)
25
26 await flushTests()
27 servers = await flushAndRunMultipleServers(2)
28
29 await setAccessTokensToServers(servers)
30 await doubleFollow(servers[0], servers[1])
31
32 const user = {
33 username: 'user1',
34 password: 'password'
35 }
36
37 await createUser(servers[0].url, servers[0].accessToken, user.username, user.password)
38 userAccessToken = await userLogin(servers[0], user)
39 })
40
41 describe('When updating redundancy', function () {
42 const path = '/api/v1/server/redundancy'
43
44 it('Should fail with an invalid token', async function () {
45 await makePutBodyRequest({
46 url: servers[0].url,
47 path: path + '/localhost:9002',
48 fields: { redundancyAllowed: true },
49 token: 'fake_token',
50 statusCodeExpected: 401
51 })
52 })
53
54 it('Should fail if the user is not an administrator', async function () {
55 await makePutBodyRequest({
56 url: servers[0].url,
57 path: path + '/localhost:9002',
58 fields: { redundancyAllowed: true },
59 token: userAccessToken,
60 statusCodeExpected: 403
61 })
62 })
63
64 it('Should fail if we do not follow this server', async function () {
65 await makePutBodyRequest({
66 url: servers[0].url,
67 path: path + '/example.com',
68 fields: { redundancyAllowed: true },
69 token: servers[0].accessToken,
70 statusCodeExpected: 404
71 })
72 })
73
74 it('Should fail without de redundancyAllowed param', async function () {
75 await makePutBodyRequest({
76 url: servers[0].url,
77 path: path + '/localhost:9002',
78 fields: { blabla: true },
79 token: servers[0].accessToken,
80 statusCodeExpected: 400
81 })
82 })
83
84 it('Should succeed with the correct parameters', async function () {
85 await makePutBodyRequest({
86 url: servers[0].url,
87 path: path + '/localhost:9002',
88 fields: { redundancyAllowed: true },
89 token: servers[0].accessToken,
90 statusCodeExpected: 204
91 })
92 })
93 })
94
95 after(async function () {
96 killallServers(servers)
97
98 // Keep the logs if the test failed
99 if (this['ok']) {
100 await flushTests()
101 }
102 })
103})
diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts
index eeb8b7a28..c74c68a33 100644
--- a/server/tests/api/server/index.ts
+++ b/server/tests/api/server/index.ts
@@ -3,6 +3,7 @@ import './email'
3import './follows' 3import './follows'
4import './handle-down' 4import './handle-down'
5import './jobs' 5import './jobs'
6import './redundancy'
6import './reverse-proxy' 7import './reverse-proxy'
7import './stats' 8import './stats'
8import './tracker' 9import './tracker'
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts
new file mode 100644
index 000000000..c0ec75a45
--- /dev/null
+++ b/server/tests/api/server/redundancy.ts
@@ -0,0 +1,140 @@
1/* tslint:disable:no-unused-expression */
2
3import * as chai from 'chai'
4import 'mocha'
5import { VideoDetails } from '../../../../shared/models/videos'
6import {
7 doubleFollow,
8 flushAndRunMultipleServers,
9 flushTests,
10 getFollowingListPaginationAndSort,
11 getVideo,
12 killallServers,
13 ServerInfo,
14 setAccessTokensToServers,
15 uploadVideo,
16 wait,
17 root, viewVideo
18} from '../../utils'
19import { waitJobs } from '../../utils/server/jobs'
20import * as magnetUtil from 'magnet-uri'
21import { updateRedundancy } from '../../utils/server/redundancy'
22import { ActorFollow } from '../../../../shared/models/actors'
23import { readdir } from 'fs-extra'
24import { join } from 'path'
25
26const expect = chai.expect
27
28function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) {
29 const parsed = magnetUtil.decode(file.magnetUri)
30
31 for (const ws of baseWebseeds) {
32 const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
33 expect(found, `Webseed ${ws} not found in ${file.magnetUri}`).to.not.be.undefined
34 }
35}
36
37describe('Test videos redundancy', function () {
38 let servers: ServerInfo[] = []
39 let video1Server2UUID: string
40 let video2Server2UUID: string
41
42 before(async function () {
43 this.timeout(120000)
44
45 servers = await flushAndRunMultipleServers(3)
46
47 // Get the access tokens
48 await setAccessTokensToServers(servers)
49
50 {
51 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
52 video1Server2UUID = res.body.video.uuid
53
54 await viewVideo(servers[1].url, video1Server2UUID)
55 }
56
57 {
58 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
59 video2Server2UUID = res.body.video.uuid
60 }
61
62 await waitJobs(servers)
63
64 // Server 1 and server 2 follow each other
65 await doubleFollow(servers[0], servers[1])
66 // Server 1 and server 3 follow each other
67 await doubleFollow(servers[0], servers[2])
68 // Server 2 and server 3 follow each other
69 await doubleFollow(servers[1], servers[2])
70
71 await waitJobs(servers)
72 })
73
74 it('Should have 1 webseed on the first video', async function () {
75 const webseeds = [
76 'http://localhost:9002/static/webseed/' + video1Server2UUID
77 ]
78
79 for (const server of servers) {
80 const res = await getVideo(server.url, video1Server2UUID)
81
82 const video: VideoDetails = res.body
83 video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
84 }
85 })
86
87 it('Should enable redundancy on server 1', async function () {
88 await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
89
90 const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt')
91 const follows: ActorFollow[] = res.body.data
92 const server2 = follows.find(f => f.following.host === 'localhost:9002')
93 const server3 = follows.find(f => f.following.host === 'localhost:9003')
94
95 expect(server3).to.not.be.undefined
96 expect(server3.following.hostRedundancyAllowed).to.be.false
97
98 expect(server2).to.not.be.undefined
99 expect(server2.following.hostRedundancyAllowed).to.be.true
100 })
101
102 it('Should have 2 webseed on the first video', async function () {
103 this.timeout(40000)
104
105 await waitJobs(servers)
106 await wait(15000)
107 await waitJobs(servers)
108
109 const webseeds = [
110 'http://localhost:9001/static/webseed/' + video1Server2UUID,
111 'http://localhost:9002/static/webseed/' + video1Server2UUID
112 ]
113
114 for (const server of servers) {
115 const res = await getVideo(server.url, video1Server2UUID)
116
117 const video: VideoDetails = res.body
118
119 for (const file of video.files) {
120 checkMagnetWebseeds(file, webseeds)
121 }
122 }
123
124 const files = await readdir(join(root(), 'test1', 'videos'))
125 expect(files).to.have.lengthOf(4)
126
127 for (const resolution of [ 240, 360, 480, 720 ]) {
128 expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
129 }
130 })
131
132 after(async function () {
133 killallServers(servers)
134
135 // Keep the logs if the test failed
136 if (this['ok']) {
137 await flushTests()
138 }
139 })
140})
diff --git a/server/tests/utils/server/follows.ts b/server/tests/utils/server/follows.ts
index d21fb5e58..8a65a958b 100644
--- a/server/tests/utils/server/follows.ts
+++ b/server/tests/utils/server/follows.ts
@@ -1,5 +1,4 @@
1import * as request from 'supertest' 1import * as request from 'supertest'
2import { wait } from '../miscs/miscs'
3import { ServerInfo } from './servers' 2import { ServerInfo } from './servers'
4import { waitJobs } from './jobs' 3import { waitJobs } from './jobs'
5 4
diff --git a/server/tests/utils/server/redundancy.ts b/server/tests/utils/server/redundancy.ts
new file mode 100644
index 000000000..c39ff2c8b
--- /dev/null
+++ b/server/tests/utils/server/redundancy.ts
@@ -0,0 +1,17 @@
1import { makePutBodyRequest } from '../requests/requests'
2
3async function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) {
4 const path = '/api/v1/server/redundancy/' + host
5
6 return makePutBodyRequest({
7 url,
8 path,
9 token: accessToken,
10 fields: { redundancyAllowed },
11 statusCodeExpected: expectedStatus
12 })
13}
14
15export {
16 updateRedundancy
17}