aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/actor.ts13
-rw-r--r--server/lib/activitypub/crawl.ts5
-rw-r--r--server/lib/activitypub/process/index.ts8
-rw-r--r--server/lib/activitypub/process/process-create.ts19
-rw-r--r--server/lib/activitypub/process/process-like.ts4
-rw-r--r--server/lib/activitypub/process/process-undo.ts6
-rw-r--r--server/lib/activitypub/process/process.ts25
-rw-r--r--server/lib/activitypub/send/send-create.ts10
-rw-r--r--server/lib/activitypub/send/send-like.ts2
-rw-r--r--server/lib/activitypub/send/send-undo.ts2
-rw-r--r--server/lib/activitypub/share.ts15
-rw-r--r--server/lib/activitypub/url.ts12
-rw-r--r--server/lib/activitypub/video-comments.ts17
-rw-r--r--server/lib/activitypub/video-rates.ts36
-rw-r--r--server/lib/activitypub/videos.ts7
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-fetcher.ts2
-rw-r--r--server/lib/job-queue/handlers/utils/activitypub-http-utils.ts13
-rw-r--r--server/lib/job-queue/handlers/video-views.ts8
-rw-r--r--server/lib/job-queue/job-queue.ts4
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts5
20 files changed, 145 insertions, 68 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 45dd4443d..b16a00669 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -5,7 +5,7 @@ import * as url from 'url'
5import * as uuidv4 from 'uuid/v4' 5import * as uuidv4 from 'uuid/v4'
6import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub' 6import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
7import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' 7import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
8import { getActorUrl } from '../../helpers/activitypub' 8import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub'
9import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor' 9import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
10import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 10import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
11import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' 11import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
@@ -65,8 +65,12 @@ async function getOrCreateActorAndServerAndModel (
65 const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person') 65 const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
66 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url) 66 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
67 67
68 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
69 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
70 }
71
68 try { 72 try {
69 // Assert we don't recurse another time 73 // Don't recurse another time
70 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false) 74 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
71 } catch (err) { 75 } catch (err) {
72 logger.error('Cannot get or create account attributed to video channel ' + actor.url) 76 logger.error('Cannot get or create account attributed to video channel ' + actor.url)
@@ -297,12 +301,15 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
297 normalizeActor(requestResult.body) 301 normalizeActor(requestResult.body)
298 302
299 const actorJSON: ActivityPubActor = requestResult.body 303 const actorJSON: ActivityPubActor = requestResult.body
300
301 if (isActorObjectValid(actorJSON) === false) { 304 if (isActorObjectValid(actorJSON) === false) {
302 logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON }) 305 logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON })
303 return { result: undefined, statusCode: requestResult.response.statusCode } 306 return { result: undefined, statusCode: requestResult.response.statusCode }
304 } 307 }
305 308
309 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
310 throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
311 }
312
306 const followersCount = await fetchActorTotalItems(actorJSON.followers) 313 const followersCount = await fetchActorTotalItems(actorJSON.followers)
307 const followingCount = await fetchActorTotalItems(actorJSON.following) 314 const followingCount = await fetchActorTotalItems(actorJSON.following)
308 315
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts
index db9ce3293..1b9b14c2e 100644
--- a/server/lib/activitypub/crawl.ts
+++ b/server/lib/activitypub/crawl.ts
@@ -2,6 +2,7 @@ import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers'
2import { doRequest } from '../../helpers/requests' 2import { doRequest } from '../../helpers/requests'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import * as Bluebird from 'bluebird' 4import * as Bluebird from 'bluebird'
5import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
5 6
6async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any> | Bluebird<any>) { 7async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any> | Bluebird<any>) {
7 logger.info('Crawling ActivityPub data on %s.', uri) 8 logger.info('Crawling ActivityPub data on %s.', uri)
@@ -14,7 +15,7 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Pr
14 timeout: JOB_REQUEST_TIMEOUT 15 timeout: JOB_REQUEST_TIMEOUT
15 } 16 }
16 17
17 const response = await doRequest(options) 18 const response = await doRequest<ActivityPubOrderedCollection<T>>(options)
18 const firstBody = response.body 19 const firstBody = response.body
19 20
20 let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT 21 let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
@@ -23,7 +24,7 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Pr
23 while (nextLink && i < limit) { 24 while (nextLink && i < limit) {
24 options.uri = nextLink 25 options.uri = nextLink
25 26
26 const { body } = await doRequest(options) 27 const { body } = await doRequest<ActivityPubOrderedCollection<T>>(options)
27 nextLink = body.next 28 nextLink = body.next
28 i++ 29 i++
29 30
diff --git a/server/lib/activitypub/process/index.ts b/server/lib/activitypub/process/index.ts
index db4980a72..5466739c1 100644
--- a/server/lib/activitypub/process/index.ts
+++ b/server/lib/activitypub/process/index.ts
@@ -1,9 +1 @@
1export * from './process' export * from './process'
2export * from './process-accept'
3export * from './process-announce'
4export * from './process-create'
5export * from './process-delete'
6export * from './process-follow'
7export * from './process-like'
8export * from './process-undo'
9export * from './process-update'
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index cefe89db0..214e14546 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -12,6 +12,9 @@ import { getOrCreateVideoAndAccountAndChannel } from '../videos'
12import { forwardVideoRelatedActivity } from '../send/utils' 12import { forwardVideoRelatedActivity } from '../send/utils'
13import { Redis } from '../../redis' 13import { Redis } from '../../redis'
14import { createOrUpdateCacheFile } from '../cache-file' 14import { createOrUpdateCacheFile } from '../cache-file'
15import { immutableAssign } from '../../../tests/utils'
16import { getVideoDislikeActivityPubUrl } from '../url'
17import { VideoModel } from '../../../models/video/video'
15 18
16async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { 19async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) {
17 const activityObject = activity.object 20 const activityObject = activity.object
@@ -65,9 +68,10 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
65 videoId: video.id, 68 videoId: video.id,
66 accountId: byAccount.id 69 accountId: byAccount.id
67 } 70 }
71
68 const [ , created ] = await AccountVideoRateModel.findOrCreate({ 72 const [ , created ] = await AccountVideoRateModel.findOrCreate({
69 where: rate, 73 where: rate,
70 defaults: rate, 74 defaults: immutableAssign(rate, { url: getVideoDislikeActivityPubUrl(byActor, video) }),
71 transaction: t 75 transaction: t
72 }) 76 })
73 if (created === true) await video.increment('dislikes', { transaction: t }) 77 if (created === true) await video.increment('dislikes', { transaction: t })
@@ -84,19 +88,10 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
84async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { 88async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
85 const view = activity.object as ViewObject 89 const view = activity.object as ViewObject
86 90
87 const options = { 91 const video = await VideoModel.loadByUrl(view.object)
88 videoObject: view.object, 92 if (!video || video.isOwned() === false) return
89 fetchType: 'only-video' as 'only-video'
90 }
91 const { video } = await getOrCreateVideoAndAccountAndChannel(options)
92 93
93 await Redis.Instance.addVideoView(video.id) 94 await Redis.Instance.addVideoView(video.id)
94
95 if (video.isOwned()) {
96 // Don't resend the activity to the sender
97 const exceptions = [ byActor ]
98 await forwardVideoRelatedActivity(activity, undefined, exceptions, video)
99 }
100} 95}
101 96
102async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { 97async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) {
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts
index f7200db61..0dca17551 100644
--- a/server/lib/activitypub/process/process-like.ts
+++ b/server/lib/activitypub/process/process-like.ts
@@ -5,6 +5,8 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/activitypub/actor'
6import { forwardVideoRelatedActivity } from '../send/utils' 6import { forwardVideoRelatedActivity } from '../send/utils'
7import { getOrCreateVideoAndAccountAndChannel } from '../videos' 7import { getOrCreateVideoAndAccountAndChannel } from '../videos'
8import { immutableAssign } from '../../../tests/utils'
9import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url'
8 10
9async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) { 11async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) {
10 return retryTransactionWrapper(processLikeVideo, byActor, activity) 12 return retryTransactionWrapper(processLikeVideo, byActor, activity)
@@ -34,7 +36,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) {
34 } 36 }
35 const [ , created ] = await AccountVideoRateModel.findOrCreate({ 37 const [ , created ] = await AccountVideoRateModel.findOrCreate({
36 where: rate, 38 where: rate,
37 defaults: rate, 39 defaults: immutableAssign(rate, { url: getVideoLikeActivityPubUrl(byActor, video) }),
38 transaction: t 40 transaction: t
39 }) 41 })
40 if (created === true) await video.increment('likes', { transaction: t }) 42 if (created === true) await video.increment('likes', { transaction: t })
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index ff019cd8c..438a013b6 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -55,7 +55,8 @@ async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) {
55 return sequelizeTypescript.transaction(async t => { 55 return sequelizeTypescript.transaction(async t => {
56 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) 56 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
57 57
58 const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) 58 let rate = await AccountVideoRateModel.loadByUrl(likeActivity.id, t)
59 if (!rate) rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t)
59 if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) 60 if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`)
60 61
61 await rate.destroy({ transaction: t }) 62 await rate.destroy({ transaction: t })
@@ -78,7 +79,8 @@ async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo)
78 return sequelizeTypescript.transaction(async t => { 79 return sequelizeTypescript.transaction(async t => {
79 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) 80 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
80 81
81 const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) 82 let rate = await AccountVideoRateModel.loadByUrl(dislike.id, t)
83 if (!rate) rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t)
82 if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) 84 if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`)
83 85
84 await rate.destroy({ transaction: t }) 86 await rate.destroy({ transaction: t })
diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts
index b263f1ea2..b9b255ddf 100644
--- a/server/lib/activitypub/process/process.ts
+++ b/server/lib/activitypub/process/process.ts
@@ -1,5 +1,5 @@
1import { Activity, ActivityType } from '../../../../shared/models/activitypub' 1import { Activity, ActivityType } from '../../../../shared/models/activitypub'
2import { getActorUrl } from '../../../helpers/activitypub' 2import { checkUrlsSameHost, getActorUrl } from '../../../helpers/activitypub'
3import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
4import { ActorModel } from '../../../models/activitypub/actor' 4import { ActorModel } from '../../../models/activitypub/actor'
5import { processAcceptActivity } from './process-accept' 5import { processAcceptActivity } from './process-accept'
@@ -25,11 +25,17 @@ const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: Ac
25 Like: processLikeActivity 25 Like: processLikeActivity
26} 26}
27 27
28async function processActivities (activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel) { 28async function processActivities (
29 activities: Activity[],
30 options: {
31 signatureActor?: ActorModel
32 inboxActor?: ActorModel
33 outboxUrl?: string
34 } = {}) {
29 const actorsCache: { [ url: string ]: ActorModel } = {} 35 const actorsCache: { [ url: string ]: ActorModel } = {}
30 36
31 for (const activity of activities) { 37 for (const activity of activities) {
32 if (!signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) { 38 if (!options.signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) {
33 logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) 39 logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type)
34 continue 40 continue
35 } 41 }
@@ -37,12 +43,17 @@ async function processActivities (activities: Activity[], signatureActor?: Actor
37 const actorUrl = getActorUrl(activity.actor) 43 const actorUrl = getActorUrl(activity.actor)
38 44
39 // When we fetch remote data, we don't have signature 45 // When we fetch remote data, we don't have signature
40 if (signatureActor && actorUrl !== signatureActor.url) { 46 if (options.signatureActor && actorUrl !== options.signatureActor.url) {
41 logger.warn('Signature mismatch between %s and %s.', actorUrl, signatureActor.url) 47 logger.warn('Signature mismatch between %s and %s, skipping.', actorUrl, options.signatureActor.url)
42 continue 48 continue
43 } 49 }
44 50
45 const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl) 51 if (options.outboxUrl && checkUrlsSameHost(options.outboxUrl, actorUrl) !== true) {
52 logger.warn('Host mismatch between outbox URL %s and actor URL %s, skipping.', options.outboxUrl, actorUrl)
53 continue
54 }
55
56 const byActor = options.signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl)
46 actorsCache[actorUrl] = byActor 57 actorsCache[actorUrl] = byActor
47 58
48 const activityProcessor = processActivity[activity.type] 59 const activityProcessor = processActivity[activity.type]
@@ -52,7 +63,7 @@ async function processActivities (activities: Activity[], signatureActor?: Actor
52 } 63 }
53 64
54 try { 65 try {
55 await activityProcessor(activity, byActor, inboxActor) 66 await activityProcessor(activity, byActor, options.inboxActor)
56 } catch (err) { 67 } catch (err) {
57 logger.warn('Cannot process activity %s.', activity.type, { err }) 68 logger.warn('Cannot process activity %s.', activity.type, { err })
58 } 69 }
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index 285edba3b..e3fca0a17 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -95,7 +95,7 @@ async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transa
95 logger.info('Creating job to send view of %s.', video.url) 95 logger.info('Creating job to send view of %s.', video.url)
96 96
97 const url = getVideoViewActivityPubUrl(byActor, video) 97 const url = getVideoViewActivityPubUrl(byActor, video)
98 const viewActivity = buildViewActivity(byActor, video) 98 const viewActivity = buildViewActivity(url, byActor, video)
99 99
100 return sendVideoRelatedCreateActivity({ 100 return sendVideoRelatedCreateActivity({
101 // Use the server actor to send the view 101 // Use the server actor to send the view
@@ -111,7 +111,7 @@ async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Tra
111 logger.info('Creating job to dislike %s.', video.url) 111 logger.info('Creating job to dislike %s.', video.url)
112 112
113 const url = getVideoDislikeActivityPubUrl(byActor, video) 113 const url = getVideoDislikeActivityPubUrl(byActor, video)
114 const dislikeActivity = buildDislikeActivity(byActor, video) 114 const dislikeActivity = buildDislikeActivity(url, byActor, video)
115 115
116 return sendVideoRelatedCreateActivity({ 116 return sendVideoRelatedCreateActivity({
117 byActor, 117 byActor,
@@ -136,16 +136,18 @@ function buildCreateActivity (url: string, byActor: ActorModel, object: any, aud
136 ) 136 )
137} 137}
138 138
139function buildDislikeActivity (byActor: ActorModel, video: VideoModel) { 139function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel) {
140 return { 140 return {
141 id: url,
141 type: 'Dislike', 142 type: 'Dislike',
142 actor: byActor.url, 143 actor: byActor.url,
143 object: video.url 144 object: video.url
144 } 145 }
145} 146}
146 147
147function buildViewActivity (byActor: ActorModel, video: VideoModel) { 148function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel) {
148 return { 149 return {
150 id: url,
149 type: 'View', 151 type: 'View',
150 actor: byActor.url, 152 actor: byActor.url,
151 object: video.url 153 object: video.url
diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts
index 89307acc6..35227887a 100644
--- a/server/lib/activitypub/send/send-like.ts
+++ b/server/lib/activitypub/send/send-like.ts
@@ -24,8 +24,8 @@ function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel,
24 24
25 return audiencify( 25 return audiencify(
26 { 26 {
27 type: 'Like' as 'Like',
28 id: url, 27 id: url,
28 type: 'Like' as 'Like',
29 actor: byActor.url, 29 actor: byActor.url,
30 object: video.url 30 object: video.url
31 }, 31 },
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts
index 5236d2cb3..bf1b6e117 100644
--- a/server/lib/activitypub/send/send-undo.ts
+++ b/server/lib/activitypub/send/send-undo.ts
@@ -64,7 +64,7 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans
64 logger.info('Creating job to undo a dislike of video %s.', video.url) 64 logger.info('Creating job to undo a dislike of video %s.', video.url)
65 65
66 const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) 66 const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video)
67 const dislikeActivity = buildDislikeActivity(byActor, video) 67 const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video)
68 const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) 68 const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity)
69 69
70 return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t }) 70 return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t })
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts
index 3ff60a97c..d2649e2d5 100644
--- a/server/lib/activitypub/share.ts
+++ b/server/lib/activitypub/share.ts
@@ -4,13 +4,14 @@ import { getServerActor } from '../../helpers/utils'
4import { VideoModel } from '../../models/video/video' 4import { VideoModel } from '../../models/video/video'
5import { VideoShareModel } from '../../models/video/video-share' 5import { VideoShareModel } from '../../models/video/video-share'
6import { sendUndoAnnounce, sendVideoAnnounce } from './send' 6import { sendUndoAnnounce, sendVideoAnnounce } from './send'
7import { getAnnounceActivityPubUrl } from './url' 7import { getVideoAnnounceActivityPubUrl } from './url'
8import { VideoChannelModel } from '../../models/video/video-channel' 8import { VideoChannelModel } from '../../models/video/video-channel'
9import * as Bluebird from 'bluebird' 9import * as Bluebird from 'bluebird'
10import { doRequest } from '../../helpers/requests' 10import { doRequest } from '../../helpers/requests'
11import { getOrCreateActorAndServerAndModel } from './actor' 11import { getOrCreateActorAndServerAndModel } from './actor'
12import { logger } from '../../helpers/logger' 12import { logger } from '../../helpers/logger'
13import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' 13import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
14import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub'
14 15
15async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { 16async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) {
16 if (video.privacy === VideoPrivacy.PRIVATE) return undefined 17 if (video.privacy === VideoPrivacy.PRIVATE) return undefined
@@ -38,9 +39,13 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) {
38 json: true, 39 json: true,
39 activityPub: true 40 activityPub: true
40 }) 41 })
41 if (!body || !body.actor) throw new Error('Body of body actor is invalid') 42 if (!body || !body.actor) throw new Error('Body or body actor is invalid')
43
44 const actorUrl = getActorUrl(body.actor)
45 if (checkUrlsSameHost(shareUrl, actorUrl) !== true) {
46 throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`)
47 }
42 48
43 const actorUrl = body.actor
44 const actor = await getOrCreateActorAndServerAndModel(actorUrl) 49 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
45 50
46 const entry = { 51 const entry = {
@@ -72,7 +77,7 @@ export {
72async function shareByServer (video: VideoModel, t: Transaction) { 77async function shareByServer (video: VideoModel, t: Transaction) {
73 const serverActor = await getServerActor() 78 const serverActor = await getServerActor()
74 79
75 const serverShareUrl = getAnnounceActivityPubUrl(video.url, serverActor) 80 const serverShareUrl = getVideoAnnounceActivityPubUrl(serverActor, video)
76 return VideoShareModel.findOrCreate({ 81 return VideoShareModel.findOrCreate({
77 defaults: { 82 defaults: {
78 actorId: serverActor.id, 83 actorId: serverActor.id,
@@ -91,7 +96,7 @@ async function shareByServer (video: VideoModel, t: Transaction) {
91} 96}
92 97
93async function shareByVideoChannel (video: VideoModel, t: Transaction) { 98async function shareByVideoChannel (video: VideoModel, t: Transaction) {
94 const videoChannelShareUrl = getAnnounceActivityPubUrl(video.url, video.VideoChannel.Actor) 99 const videoChannelShareUrl = getVideoAnnounceActivityPubUrl(video.VideoChannel.Actor, video)
95 return VideoShareModel.findOrCreate({ 100 return VideoShareModel.findOrCreate({
96 defaults: { 101 defaults: {
97 actorId: video.VideoChannel.actorId, 102 actorId: video.VideoChannel.actorId,
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts
index e792be698..38f15448c 100644
--- a/server/lib/activitypub/url.ts
+++ b/server/lib/activitypub/url.ts
@@ -33,14 +33,14 @@ function getVideoAbuseActivityPubUrl (videoAbuse: VideoAbuseModel) {
33} 33}
34 34
35function getVideoViewActivityPubUrl (byActor: ActorModel, video: VideoModel) { 35function getVideoViewActivityPubUrl (byActor: ActorModel, video: VideoModel) {
36 return video.url + '/views/' + byActor.uuid + '/' + new Date().toISOString() 36 return byActor.url + '/views/videos/' + video.id + '/' + new Date().toISOString()
37} 37}
38 38
39function getVideoLikeActivityPubUrl (byActor: ActorModel, video: VideoModel) { 39function getVideoLikeActivityPubUrl (byActor: ActorModel, video: VideoModel | { id: number }) {
40 return byActor.url + '/likes/' + video.id 40 return byActor.url + '/likes/' + video.id
41} 41}
42 42
43function getVideoDislikeActivityPubUrl (byActor: ActorModel, video: VideoModel) { 43function getVideoDislikeActivityPubUrl (byActor: ActorModel, video: VideoModel | { id: number }) {
44 return byActor.url + '/dislikes/' + video.id 44 return byActor.url + '/dislikes/' + video.id
45} 45}
46 46
@@ -74,8 +74,8 @@ function getActorFollowAcceptActivityPubUrl (actorFollow: ActorFollowModel) {
74 return follower.url + '/accepts/follows/' + me.id 74 return follower.url + '/accepts/follows/' + me.id
75} 75}
76 76
77function getAnnounceActivityPubUrl (originalUrl: string, byActor: ActorModel) { 77function getVideoAnnounceActivityPubUrl (byActor: ActorModel, video: VideoModel) {
78 return originalUrl + '/announces/' + byActor.id 78 return video.url + '/announces/' + byActor.id
79} 79}
80 80
81function getDeleteActivityPubUrl (originalUrl: string) { 81function getDeleteActivityPubUrl (originalUrl: string) {
@@ -97,7 +97,7 @@ export {
97 getVideoAbuseActivityPubUrl, 97 getVideoAbuseActivityPubUrl,
98 getActorFollowActivityPubUrl, 98 getActorFollowActivityPubUrl,
99 getActorFollowAcceptActivityPubUrl, 99 getActorFollowAcceptActivityPubUrl,
100 getAnnounceActivityPubUrl, 100 getVideoAnnounceActivityPubUrl,
101 getUpdateActivityPubUrl, 101 getUpdateActivityPubUrl,
102 getUndoActivityPubUrl, 102 getUndoActivityPubUrl,
103 getVideoViewActivityPubUrl, 103 getVideoViewActivityPubUrl,
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index c8c17f4c4..5868e7297 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -9,6 +9,7 @@ import { VideoCommentModel } from '../../models/video/video-comment'
9import { getOrCreateActorAndServerAndModel } from './actor' 9import { getOrCreateActorAndServerAndModel } from './actor'
10import { getOrCreateVideoAndAccountAndChannel } from './videos' 10import { getOrCreateVideoAndAccountAndChannel } from './videos'
11import * as Bluebird from 'bluebird' 11import * as Bluebird from 'bluebird'
12import { checkUrlsSameHost } from '../../helpers/activitypub'
12 13
13async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { 14async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) {
14 let originCommentId: number = null 15 let originCommentId: number = null
@@ -61,6 +62,14 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) {
61 const actorUrl = body.attributedTo 62 const actorUrl = body.attributedTo
62 if (!actorUrl) return { created: false } 63 if (!actorUrl) return { created: false }
63 64
65 if (checkUrlsSameHost(commentUrl, actorUrl) !== true) {
66 throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${commentUrl}`)
67 }
68
69 if (checkUrlsSameHost(body.id, commentUrl) !== true) {
70 throw new Error(`Comment url ${commentUrl} host is different from the AP object id ${body.id}`)
71 }
72
64 const actor = await getOrCreateActorAndServerAndModel(actorUrl) 73 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
65 const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) 74 const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body)
66 if (!entry) return { created: false } 75 if (!entry) return { created: false }
@@ -134,6 +143,14 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) {
134 const actorUrl = body.attributedTo 143 const actorUrl = body.attributedTo
135 if (!actorUrl) throw new Error('Miss attributed to in comment') 144 if (!actorUrl) throw new Error('Miss attributed to in comment')
136 145
146 if (checkUrlsSameHost(url, actorUrl) !== true) {
147 throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`)
148 }
149
150 if (checkUrlsSameHost(body.id, url) !== true) {
151 throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`)
152 }
153
137 const actor = await getOrCreateActorAndServerAndModel(actorUrl) 154 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
138 const comment = new VideoCommentModel({ 155 const comment = new VideoCommentModel({
139 url: body.id, 156 url: body.id,
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts
index 1619251c3..1854b44c4 100644
--- a/server/lib/activitypub/video-rates.ts
+++ b/server/lib/activitypub/video-rates.ts
@@ -8,13 +8,35 @@ import { getOrCreateActorAndServerAndModel } from './actor'
8import { AccountVideoRateModel } from '../../models/account/account-video-rate' 8import { AccountVideoRateModel } from '../../models/account/account-video-rate'
9import { logger } from '../../helpers/logger' 9import { logger } from '../../helpers/logger'
10import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' 10import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
11import { doRequest } from '../../helpers/requests'
12import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub'
13import { ActorModel } from '../../models/activitypub/actor'
14import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url'
11 15
12async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { 16async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRateType) {
13 let rateCounts = 0 17 let rateCounts = 0
14 18
15 await Bluebird.map(actorUrls, async actorUrl => { 19 await Bluebird.map(ratesUrl, async rateUrl => {
16 try { 20 try {
21 // Fetch url
22 const { body } = await doRequest({
23 uri: rateUrl,
24 json: true,
25 activityPub: true
26 })
27 if (!body || !body.actor) throw new Error('Body or body actor is invalid')
28
29 const actorUrl = getActorUrl(body.actor)
30 if (checkUrlsSameHost(actorUrl, rateUrl) !== true) {
31 throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`)
32 }
33
34 if (checkUrlsSameHost(body.id, rateUrl) !== true) {
35 throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`)
36 }
37
17 const actor = await getOrCreateActorAndServerAndModel(actorUrl) 38 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
39
18 const [ , created ] = await AccountVideoRateModel 40 const [ , created ] = await AccountVideoRateModel
19 .findOrCreate({ 41 .findOrCreate({
20 where: { 42 where: {
@@ -24,13 +46,14 @@ async function createRates (actorUrls: string[], video: VideoModel, rate: VideoR
24 defaults: { 46 defaults: {
25 videoId: video.id, 47 videoId: video.id,
26 accountId: actor.Account.id, 48 accountId: actor.Account.id,
27 type: rate 49 type: rate,
50 url: body.id
28 } 51 }
29 }) 52 })
30 53
31 if (created) rateCounts += 1 54 if (created) rateCounts += 1
32 } catch (err) { 55 } catch (err) {
33 logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err }) 56 logger.warn('Cannot add rate %s.', rateUrl, { err })
34 } 57 }
35 }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) 58 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
36 59
@@ -62,7 +85,12 @@ async function sendVideoRateChange (account: AccountModel,
62 if (dislikes > 0) await sendCreateDislike(actor, video, t) 85 if (dislikes > 0) await sendCreateDislike(actor, video, t)
63} 86}
64 87
88function getRateUrl (rateType: VideoRateType, actor: ActorModel, video: VideoModel) {
89 return rateType === 'like' ? getVideoLikeActivityPubUrl(actor, video) : getVideoDislikeActivityPubUrl(actor, video)
90}
91
65export { 92export {
93 getRateUrl,
66 createRates, 94 createRates,
67 sendVideoRateChange 95 sendVideoRateChange
68} 96}
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 3da363c0a..5bd03c8c6 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -29,6 +29,7 @@ import { createRates } from './video-rates'
29import { addVideoShares, shareVideoByServerAndChannel } from './share' 29import { addVideoShares, shareVideoByServerAndChannel } from './share'
30import { AccountModel } from '../../models/account/account' 30import { AccountModel } from '../../models/account/account'
31import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' 31import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
32import { checkUrlsSameHost } from '../../helpers/activitypub'
32 33
33async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 34async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
34 // If the video is not private and published, we federate it 35 // If the video is not private and published, we federate it
@@ -63,7 +64,7 @@ async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.
63 64
64 const { response, body } = await doRequest(options) 65 const { response, body } = await doRequest(options)
65 66
66 if (sanitizeAndCheckVideoTorrentObject(body) === false) { 67 if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
67 logger.debug('Remote video JSON is not valid.', { body }) 68 logger.debug('Remote video JSON is not valid.', { body })
68 return { response, videoObject: undefined } 69 return { response, videoObject: undefined }
69 } 70 }
@@ -107,6 +108,10 @@ function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject
107 const channel = videoObject.attributedTo.find(a => a.type === 'Group') 108 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
108 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) 109 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
109 110
111 if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
112 throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
113 }
114
110 return getOrCreateActorAndServerAndModel(channel.id, 'all') 115 return getOrCreateActorAndServerAndModel(channel.id, 'all')
111} 116}
112 117
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index 42217c27c..67ccfa995 100644
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -23,7 +23,7 @@ async function processActivityPubHttpFetcher (job: Bull.Job) {
23 if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId) 23 if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId)
24 24
25 const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = { 25 const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = {
26 'activity': items => processActivities(items), 26 'activity': items => processActivities(items, { outboxUrl: payload.uri }),
27 'video-likes': items => createRates(items, video, 'like'), 27 'video-likes': items => createRates(items, video, 'like'),
28 'video-dislikes': items => createRates(items, video, 'dislike'), 28 'video-dislikes': items => createRates(items, video, 'dislike'),
29 'video-shares': items => addVideoShares(items, video), 29 'video-shares': items => addVideoShares(items, video),
diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
index fd9c74341..4961d4502 100644
--- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
+++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
@@ -38,15 +38,20 @@ async function buildSignedRequestOptions (payload: Payload) {
38 } 38 }
39} 39}
40 40
41function buildGlobalHeaders (body: object) { 41function buildGlobalHeaders (body: any) {
42 const digest = 'SHA-256=' + sha256(JSON.stringify(body), 'base64')
43
44 return { 42 return {
45 'Digest': digest 43 'Digest': buildDigest(body)
46 } 44 }
47} 45}
48 46
47function buildDigest (body: any) {
48 const rawBody = typeof body === 'string' ? body : JSON.stringify(body)
49
50 return 'SHA-256=' + sha256(rawBody, 'base64')
51}
52
49export { 53export {
54 buildDigest,
50 buildGlobalHeaders, 55 buildGlobalHeaders,
51 computeBody, 56 computeBody,
52 buildSignedRequestOptions 57 buildSignedRequestOptions
diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts
index cf180a11a..f44c3c727 100644
--- a/server/lib/job-queue/handlers/video-views.ts
+++ b/server/lib/job-queue/handlers/video-views.ts
@@ -3,8 +3,9 @@ import { logger } from '../../../helpers/logger'
3import { VideoModel } from '../../../models/video/video' 3import { VideoModel } from '../../../models/video/video'
4import { VideoViewModel } from '../../../models/video/video-views' 4import { VideoViewModel } from '../../../models/video/video-views'
5import { isTestInstance } from '../../../helpers/core-utils' 5import { isTestInstance } from '../../../helpers/core-utils'
6import { federateVideoIfNeeded } from '../../activitypub'
6 7
7async function processVideosViewsViews () { 8async function processVideosViews () {
8 const lastHour = new Date() 9 const lastHour = new Date()
9 10
10 // In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour 11 // In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour
@@ -36,6 +37,9 @@ async function processVideosViewsViews () {
36 views, 37 views,
37 videoId 38 videoId
38 }) 39 })
40
41 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
42 if (video.isOwned()) await federateVideoIfNeeded(video, false)
39 } catch (err) { 43 } catch (err) {
40 logger.debug('Cannot create video views for video %d in hour %d. Maybe the video does not exist anymore?', videoId, hour) 44 logger.debug('Cannot create video views for video %d in hour %d. Maybe the video does not exist anymore?', videoId, hour)
41 } 45 }
@@ -51,5 +55,5 @@ async function processVideosViewsViews () {
51// --------------------------------------------------------------------------- 55// ---------------------------------------------------------------------------
52 56
53export { 57export {
54 processVideosViewsViews 58 processVideosViews
55} 59}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 0696ba43c..4cfd4d253 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -10,7 +10,7 @@ import { EmailPayload, processEmail } from './handlers/email'
10import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' 10import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file'
11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' 11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow'
12import { processVideoImport, VideoImportPayload } from './handlers/video-import' 12import { processVideoImport, VideoImportPayload } from './handlers/video-import'
13import { processVideosViewsViews } from './handlers/video-views' 13import { processVideosViews } from './handlers/video-views'
14 14
15type CreateJobArgument = 15type CreateJobArgument =
16 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 16 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -32,7 +32,7 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
32 'video-file': processVideoFile, 32 'video-file': processVideoFile,
33 'email': processEmail, 33 'email': processEmail,
34 'video-import': processVideoImport, 34 'video-import': processVideoImport,
35 'videos-views': processVideosViewsViews 35 'videos-views': processVideosViews
36} 36}
37 37
38const jobTypes: JobType[] = [ 38const jobTypes: JobType[] = [
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index c49a8c89a..8b7f33539 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -185,11 +185,12 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
185 } 185 }
186 186
187 private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 187 private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
188 const maxSize = redundancy.size - this.getTotalFileSizes(filesToDuplicate) 188 const maxSize = redundancy.size
189 189
190 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) 190 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy)
191 const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate)
191 192
192 return totalDuplicated > maxSize 193 return totalWillDuplicate > maxSize
193 } 194 }
194 195
195 private buildNewExpiration (expiresAfterMs: number) { 196 private buildNewExpiration (expiresAfterMs: number) {