aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/activitypub/client.ts6
-rw-r--r--server/helpers/custom-validators/activitypub/video-comments.ts3
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts3
-rw-r--r--server/initializers/constants.ts1
-rw-r--r--server/initializers/installer.ts2
-rw-r--r--server/initializers/migrator.ts4
-rw-r--r--server/lib/activitypub/index.ts5
-rw-r--r--server/lib/activitypub/process/misc.ts194
-rw-r--r--server/lib/activitypub/process/process-announce.ts19
-rw-r--r--server/lib/activitypub/process/process-create.ts127
-rw-r--r--server/lib/activitypub/process/process-like.ts10
-rw-r--r--server/lib/activitypub/process/process-undo.ts16
-rw-r--r--server/lib/activitypub/process/process-update.ts10
-rw-r--r--server/lib/activitypub/video-comments.ts156
-rw-r--r--server/lib/activitypub/video-rates.ts52
-rw-r--r--server/lib/activitypub/videos.ts273
-rw-r--r--server/models/activitypub/actor.ts3
-rw-r--r--server/models/video/video-comment.ts15
-rw-r--r--server/models/video/video.ts12
-rw-r--r--server/tests/api/server/handle-down.ts147
20 files changed, 673 insertions, 385 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts
index d1a761724..ec3f72b64 100644
--- a/server/controllers/activitypub/client.ts
+++ b/server/controllers/activitypub/client.ts
@@ -1,9 +1,11 @@
1// Intercept ActivityPub client requests 1// Intercept ActivityPub client requests
2import * as express from 'express' 2import * as express from 'express'
3import { VideoPrivacy } from '../../../shared/models/videos'
3import { activityPubCollectionPagination } from '../../helpers/activitypub' 4import { activityPubCollectionPagination } from '../../helpers/activitypub'
4import { pageToStartAndCount } from '../../helpers/core-utils' 5import { pageToStartAndCount } from '../../helpers/core-utils'
5import { ACTIVITY_PUB, CONFIG } from '../../initializers' 6import { ACTIVITY_PUB, CONFIG } from '../../initializers'
6import { buildVideoAnnounceToFollowers } from '../../lib/activitypub/send' 7import { buildVideoAnnounceToFollowers } from '../../lib/activitypub/send'
8import { audiencify, getAudience } from '../../lib/activitypub/send/misc'
7import { asyncMiddleware, executeIfActivityPub, localAccountValidator } from '../../middlewares' 9import { asyncMiddleware, executeIfActivityPub, localAccountValidator } from '../../middlewares'
8import { videoChannelsGetValidator, videosGetValidator, videosShareValidator } from '../../middlewares/validators' 10import { videoChannelsGetValidator, videosGetValidator, videosShareValidator } from '../../middlewares/validators'
9import { videoCommentGetValidator } from '../../middlewares/validators/video-comments' 11import { videoCommentGetValidator } from '../../middlewares/validators/video-comments'
@@ -95,7 +97,9 @@ async function videoController (req: express.Request, res: express.Response, nex
95 97
96 // We need more attributes 98 // We need more attributes
97 const videoAll = await VideoModel.loadAndPopulateAll(video.id) 99 const videoAll = await VideoModel.loadAndPopulateAll(video.id)
98 return res.json(videoAll.toActivityPubObject()) 100 const audience = await getAudience(video.VideoChannel.Account.Actor, undefined, video.privacy === VideoPrivacy.PUBLIC)
101
102 return res.json(audiencify(videoAll.toActivityPubObject(), audience))
99} 103}
100 104
101async function videoAnnounceController (req: express.Request, res: express.Response, next: express.NextFunction) { 105async function videoAnnounceController (req: express.Request, res: express.Response, next: express.NextFunction) {
diff --git a/server/helpers/custom-validators/activitypub/video-comments.ts b/server/helpers/custom-validators/activitypub/video-comments.ts
index ce1209035..cbd4dac5c 100644
--- a/server/helpers/custom-validators/activitypub/video-comments.ts
+++ b/server/helpers/custom-validators/activitypub/video-comments.ts
@@ -24,7 +24,8 @@ function isVideoCommentDeleteActivityValid (activity: any) {
24 24
25export { 25export {
26 isVideoCommentCreateActivityValid, 26 isVideoCommentCreateActivityValid,
27 isVideoCommentDeleteActivityValid 27 isVideoCommentDeleteActivityValid,
28 isVideoCommentObjectValid
28} 29}
29 30
30// --------------------------------------------------------------------------- 31// ---------------------------------------------------------------------------
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index 37cd6965a..fb1d2d094 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -70,7 +70,8 @@ export {
70 isVideoTorrentCreateActivityValid, 70 isVideoTorrentCreateActivityValid,
71 isVideoTorrentUpdateActivityValid, 71 isVideoTorrentUpdateActivityValid,
72 isVideoTorrentDeleteActivityValid, 72 isVideoTorrentDeleteActivityValid,
73 isVideoFlagValid 73 isVideoFlagValid,
74 isVideoTorrentObjectValid
74} 75}
75 76
76// --------------------------------------------------------------------------- 77// ---------------------------------------------------------------------------
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index c902e0cf6..c735e6daf 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -279,6 +279,7 @@ const ACTIVITY_PUB = {
279 TORRENT: [ 'application/x-bittorrent' ], 279 TORRENT: [ 'application/x-bittorrent' ],
280 MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ] 280 MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ]
281 }, 281 },
282 MAX_RECURSION_COMMENTS: 100,
282 ACTOR_REFRESH_INTERVAL: 3600 * 24 // 1 day 283 ACTOR_REFRESH_INTERVAL: 3600 * 24 // 1 day
283} 284}
284 285
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index 58713c2c4..324a2c2e5 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -20,7 +20,7 @@ async function installApplication () {
20 await createOAuthAdminIfNotExist() 20 await createOAuthAdminIfNotExist()
21 } catch (err) { 21 } catch (err) {
22 logger.error('Cannot install application.', err) 22 logger.error('Cannot install application.', err)
23 throw err 23 process.exit(-1)
24 } 24 }
25} 25}
26 26
diff --git a/server/initializers/migrator.ts b/server/initializers/migrator.ts
index 29310b913..9ebc57f07 100644
--- a/server/initializers/migrator.ts
+++ b/server/initializers/migrator.ts
@@ -44,7 +44,7 @@ async function migrate () {
44 await executeMigration(actualVersion, migrationScript) 44 await executeMigration(actualVersion, migrationScript)
45 } catch (err) { 45 } catch (err) {
46 logger.error('Cannot execute migration %s.', migrationScript.version, err) 46 logger.error('Cannot execute migration %s.', migrationScript.version, err)
47 process.exit(0) 47 process.exit(-1)
48 } 48 }
49 } 49 }
50 50
@@ -92,7 +92,7 @@ async function executeMigration (actualVersion: number, entity: { version: strin
92 92
93 const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) 93 const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName))
94 94
95 await sequelizeTypescript.transaction(async t => { 95 return sequelizeTypescript.transaction(async t => {
96 const options = { 96 const options = {
97 transaction: t, 97 transaction: t,
98 queryInterface: sequelizeTypescript.getQueryInterface(), 98 queryInterface: sequelizeTypescript.getQueryInterface(),
diff --git a/server/lib/activitypub/index.ts b/server/lib/activitypub/index.ts
index 94ed1edaa..0779d1e91 100644
--- a/server/lib/activitypub/index.ts
+++ b/server/lib/activitypub/index.ts
@@ -5,3 +5,8 @@ export * from './fetch'
5export * from './share' 5export * from './share'
6export * from './videos' 6export * from './videos'
7export * from './url' 7export * from './url'
8export { videoCommentActivityObjectToDBAttributes } from './video-comments'
9export { addVideoComments } from './video-comments'
10export { addVideoComment } from './video-comments'
11export { sendVideoRateChangeToFollowers } from './video-rates'
12export { sendVideoRateChangeToOrigin } from './video-rates'
diff --git a/server/lib/activitypub/process/misc.ts b/server/lib/activitypub/process/misc.ts
deleted file mode 100644
index 461619ea7..000000000
--- a/server/lib/activitypub/process/misc.ts
+++ /dev/null
@@ -1,194 +0,0 @@
1import * as magnetUtil from 'magnet-uri'
2import { VideoTorrentObject } from '../../../../shared'
3import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object'
4import { VideoPrivacy } from '../../../../shared/models/videos'
5import { isVideoFileInfoHashValid } from '../../../helpers/custom-validators/videos'
6import { logger } from '../../../helpers/logger'
7import { doRequest } from '../../../helpers/requests'
8import { ACTIVITY_PUB, VIDEO_MIMETYPE_EXT } from '../../../initializers'
9import { ActorModel } from '../../../models/activitypub/actor'
10import { VideoModel } from '../../../models/video/video'
11import { VideoChannelModel } from '../../../models/video/video-channel'
12import { VideoCommentModel } from '../../../models/video/video-comment'
13import { VideoShareModel } from '../../../models/video/video-share'
14import { getOrCreateActorAndServerAndModel } from '../actor'
15
16async function videoActivityObjectToDBAttributes (
17 videoChannel: VideoChannelModel,
18 videoObject: VideoTorrentObject,
19 to: string[] = [],
20 cc: string[] = []
21) {
22 let privacy = VideoPrivacy.PRIVATE
23 if (to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.PUBLIC
24 else if (cc.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.UNLISTED
25
26 const duration = videoObject.duration.replace(/[^\d]+/, '')
27 let language = null
28 if (videoObject.language) {
29 language = parseInt(videoObject.language.identifier, 10)
30 }
31
32 let category = null
33 if (videoObject.category) {
34 category = parseInt(videoObject.category.identifier, 10)
35 }
36
37 let licence = null
38 if (videoObject.licence) {
39 licence = parseInt(videoObject.licence.identifier, 10)
40 }
41
42 let description = null
43 if (videoObject.content) {
44 description = videoObject.content
45 }
46
47 return {
48 name: videoObject.name,
49 uuid: videoObject.uuid,
50 url: videoObject.id,
51 category,
52 licence,
53 language,
54 description,
55 nsfw: videoObject.nsfw,
56 commentsEnabled: videoObject.commentsEnabled,
57 channelId: videoChannel.id,
58 duration: parseInt(duration, 10),
59 createdAt: new Date(videoObject.published),
60 // FIXME: updatedAt does not seems to be considered by Sequelize
61 updatedAt: new Date(videoObject.updated),
62 views: videoObject.views,
63 likes: 0,
64 dislikes: 0,
65 remote: true,
66 privacy
67 }
68}
69
70function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
71 const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
72 const fileUrls = videoObject.url.filter(u => {
73 return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/')
74 })
75
76 if (fileUrls.length === 0) {
77 throw new Error('Cannot find video files for ' + videoCreated.url)
78 }
79
80 const attributes = []
81 for (const fileUrl of fileUrls) {
82 // Fetch associated magnet uri
83 const magnet = videoObject.url.find(u => {
84 return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.width === fileUrl.width
85 })
86
87 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.url)
88
89 const parsed = magnetUtil.decode(magnet.url)
90 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) throw new Error('Cannot parse magnet URI ' + magnet.url)
91
92 const attribute = {
93 extname: VIDEO_MIMETYPE_EXT[fileUrl.mimeType],
94 infoHash: parsed.infoHash,
95 resolution: fileUrl.width,
96 size: fileUrl.size,
97 videoId: videoCreated.id
98 }
99 attributes.push(attribute)
100 }
101
102 return attributes
103}
104
105async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) {
106 let originCommentId: number = null
107 let inReplyToCommentId: number = null
108
109 // If this is not a reply to the video (thread), create or get the parent comment
110 if (video.url !== comment.inReplyTo) {
111 const [ parent ] = await addVideoComment(video, comment.inReplyTo)
112 if (!parent) {
113 logger.warn('Cannot fetch or get parent comment %s of comment %s.', comment.inReplyTo, comment.id)
114 return undefined
115 }
116
117 originCommentId = parent.originCommentId || parent.id
118 inReplyToCommentId = parent.id
119 }
120
121 return {
122 url: comment.url,
123 text: comment.content,
124 videoId: video.id,
125 accountId: actor.Account.id,
126 inReplyToCommentId,
127 originCommentId,
128 createdAt: new Date(comment.published),
129 updatedAt: new Date(comment.updated)
130 }
131}
132
133async function addVideoShares (instance: VideoModel, shareUrls: string[]) {
134 for (const shareUrl of shareUrls) {
135 // Fetch url
136 const { body } = await doRequest({
137 uri: shareUrl,
138 json: true,
139 activityPub: true
140 })
141 const actorUrl = body.actor
142 if (!actorUrl) continue
143
144 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
145
146 const entry = {
147 actorId: actor.id,
148 videoId: instance.id
149 }
150
151 await VideoShareModel.findOrCreate({
152 where: entry,
153 defaults: entry
154 })
155 }
156}
157
158async function addVideoComments (instance: VideoModel, commentUrls: string[]) {
159 for (const commentUrl of commentUrls) {
160 await addVideoComment(instance, commentUrl)
161 }
162}
163
164async function addVideoComment (instance: VideoModel, commentUrl: string) {
165 // Fetch url
166 const { body } = await doRequest({
167 uri: commentUrl,
168 json: true,
169 activityPub: true
170 })
171
172 const actorUrl = body.attributedTo
173 if (!actorUrl) return []
174
175 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
176 const entry = await videoCommentActivityObjectToDBAttributes(instance, actor, body)
177 if (!entry) return []
178
179 return VideoCommentModel.findOrCreate({
180 where: {
181 url: body.id
182 },
183 defaults: entry
184 })
185}
186
187// ---------------------------------------------------------------------------
188
189export {
190 videoFileActivityUrlToDBAttributes,
191 videoActivityObjectToDBAttributes,
192 addVideoShares,
193 addVideoComments
194}
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts
index 9adb40e01..bf7d7879d 100644
--- a/server/lib/activitypub/process/process-announce.ts
+++ b/server/lib/activitypub/process/process-announce.ts
@@ -7,6 +7,7 @@ import { VideoModel } from '../../../models/video/video'
7import { VideoShareModel } from '../../../models/video/video-share' 7import { VideoShareModel } from '../../../models/video/video-share'
8import { getOrCreateActorAndServerAndModel } from '../actor' 8import { getOrCreateActorAndServerAndModel } from '../actor'
9import { forwardActivity } from '../send/misc' 9import { forwardActivity } from '../send/misc'
10import { getOrCreateAccountAndVideoAndChannel } from '../videos'
10import { processCreateActivity } from './process-create' 11import { processCreateActivity } from './process-create'
11 12
12async function processAnnounceActivity (activity: ActivityAnnounce) { 13async function processAnnounceActivity (activity: ActivityAnnounce) {
@@ -44,19 +45,19 @@ function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnoun
44 return retryTransactionWrapper(shareVideo, options) 45 return retryTransactionWrapper(shareVideo, options)
45} 46}
46 47
47function shareVideo (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { 48async function shareVideo (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
48 const announced = activity.object 49 const announced = activity.object
50 let video: VideoModel
51
52 if (typeof announced === 'string') {
53 const res = await getOrCreateAccountAndVideoAndChannel(announced)
54 video = res.video
55 } else {
56 video = await processCreateActivity(announced)
57 }
49 58
50 return sequelizeTypescript.transaction(async t => { 59 return sequelizeTypescript.transaction(async t => {
51 // Add share entry 60 // Add share entry
52 let video: VideoModel
53
54 if (typeof announced === 'string') {
55 video = await VideoModel.loadByUrlAndPopulateAccount(announced)
56 if (!video) throw new Error('Unknown video to share ' + announced)
57 } else {
58 video = await processCreateActivity(announced)
59 }
60 61
61 const share = { 62 const share = {
62 actorId: actorAnnouncer.id, 63 actorId: actorAnnouncer.id,
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index e65b257c0..08d61996a 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -8,15 +8,13 @@ import { logger } from '../../../helpers/logger'
8import { sequelizeTypescript } from '../../../initializers' 8import { sequelizeTypescript } from '../../../initializers'
9import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 9import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
10import { ActorModel } from '../../../models/activitypub/actor' 10import { ActorModel } from '../../../models/activitypub/actor'
11import { TagModel } from '../../../models/video/tag'
12import { VideoModel } from '../../../models/video/video' 11import { VideoModel } from '../../../models/video/video'
13import { VideoAbuseModel } from '../../../models/video/video-abuse' 12import { VideoAbuseModel } from '../../../models/video/video-abuse'
14import { VideoCommentModel } from '../../../models/video/video-comment' 13import { VideoCommentModel } from '../../../models/video/video-comment'
15import { VideoFileModel } from '../../../models/video/video-file'
16import { getOrCreateActorAndServerAndModel } from '../actor' 14import { getOrCreateActorAndServerAndModel } from '../actor'
17import { forwardActivity, getActorsInvolvedInVideo } from '../send/misc' 15import { forwardActivity, getActorsInvolvedInVideo } from '../send/misc'
18import { generateThumbnailFromUrl } from '../videos' 16import { addVideoComments, resolveThread } from '../video-comments'
19import { addVideoComments, addVideoShares, videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' 17import { addVideoShares, getOrCreateAccountAndVideoAndChannel } from '../videos'
20 18
21async function processCreateActivity (activity: ActivityCreate) { 19async function processCreateActivity (activity: ActivityCreate) {
22 const activityObject = activity.object 20 const activityObject = activity.object
@@ -53,17 +51,7 @@ async function processCreateVideo (
53) { 51) {
54 const videoToCreateData = activity.object as VideoTorrentObject 52 const videoToCreateData = activity.object as VideoTorrentObject
55 53
56 const channel = videoToCreateData.attributedTo.find(a => a.type === 'Group') 54 const { video } = await getOrCreateAccountAndVideoAndChannel(videoToCreateData, actor)
57 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoToCreateData.url)
58
59 const channelActor = await getOrCreateActorAndServerAndModel(channel.id)
60
61 const options = {
62 arguments: [ actor, activity, videoToCreateData, channelActor ],
63 errorMessage: 'Cannot insert the remote video with many retries.'
64 }
65
66 const video = await retryTransactionWrapper(createRemoteVideo, options)
67 55
68 // Process outside the transaction because we could fetch remote data 56 // Process outside the transaction because we could fetch remote data
69 if (videoToCreateData.likes && Array.isArray(videoToCreateData.likes.orderedItems)) { 57 if (videoToCreateData.likes && Array.isArray(videoToCreateData.likes.orderedItems)) {
@@ -89,48 +77,6 @@ async function processCreateVideo (
89 return video 77 return video
90} 78}
91 79
92function createRemoteVideo (
93 account: ActorModel,
94 activity: ActivityCreate,
95 videoToCreateData: VideoTorrentObject,
96 channelActor: ActorModel
97) {
98 logger.debug('Adding remote video %s.', videoToCreateData.id)
99
100 return sequelizeTypescript.transaction(async t => {
101 const sequelizeOptions = {
102 transaction: t
103 }
104 const videoFromDatabase = await VideoModel.loadByUUIDOrURL(videoToCreateData.uuid, videoToCreateData.id, t)
105 if (videoFromDatabase) return videoFromDatabase
106
107 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoToCreateData, activity.to, activity.cc)
108 const video = VideoModel.build(videoData)
109
110 // Don't block on request
111 generateThumbnailFromUrl(video, videoToCreateData.icon)
112 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoToCreateData.id, err))
113
114 const videoCreated = await video.save(sequelizeOptions)
115
116 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoToCreateData)
117 if (videoFileAttributes.length === 0) {
118 throw new Error('Cannot find valid files for video %s ' + videoToCreateData.url)
119 }
120
121 const tasks: Bluebird<any>[] = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
122 await Promise.all(tasks)
123
124 const tags = videoToCreateData.tag.map(t => t.name)
125 const tagInstances = await TagModel.findOrCreateTags(tags, t)
126 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
127
128 logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid)
129
130 return videoCreated
131 })
132}
133
134async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { 80async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) {
135 let rateCounts = 0 81 let rateCounts = 0
136 const tasks: Bluebird<any>[] = [] 82 const tasks: Bluebird<any>[] = []
@@ -167,16 +113,15 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
167 return retryTransactionWrapper(createVideoDislike, options) 113 return retryTransactionWrapper(createVideoDislike, options)
168} 114}
169 115
170function createVideoDislike (byActor: ActorModel, activity: ActivityCreate) { 116async function createVideoDislike (byActor: ActorModel, activity: ActivityCreate) {
171 const dislike = activity.object as DislikeObject 117 const dislike = activity.object as DislikeObject
172 const byAccount = byActor.Account 118 const byAccount = byActor.Account
173 119
174 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) 120 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
175 121
176 return sequelizeTypescript.transaction(async t => { 122 const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object)
177 const video = await VideoModel.loadByUrlAndPopulateAccount(dislike.object, t)
178 if (!video) throw new Error('Unknown video ' + dislike.object)
179 123
124 return sequelizeTypescript.transaction(async t => {
180 const rate = { 125 const rate = {
181 type: 'dislike' as 'dislike', 126 type: 'dislike' as 'dislike',
182 videoId: video.id, 127 videoId: video.id,
@@ -200,9 +145,7 @@ function createVideoDislike (byActor: ActorModel, activity: ActivityCreate) {
200async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { 145async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
201 const view = activity.object as ViewObject 146 const view = activity.object as ViewObject
202 147
203 const video = await VideoModel.loadByUrlAndPopulateAccount(view.object) 148 const { video } = await getOrCreateAccountAndVideoAndChannel(view.object)
204
205 if (!video) throw new Error('Unknown video ' + view.object)
206 149
207 const account = await ActorModel.loadByUrl(view.actor) 150 const account = await ActorModel.loadByUrl(view.actor)
208 if (!account) throw new Error('Unknown account ' + view.actor) 151 if (!account) throw new Error('Unknown account ' + view.actor)
@@ -225,19 +168,15 @@ function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: Vid
225 return retryTransactionWrapper(addRemoteVideoAbuse, options) 168 return retryTransactionWrapper(addRemoteVideoAbuse, options)
226} 169}
227 170
228function addRemoteVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { 171async function addRemoteVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) {
229 logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) 172 logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object)
230 173
231 const account = actor.Account 174 const account = actor.Account
232 if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) 175 if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url)
233 176
234 return sequelizeTypescript.transaction(async t => { 177 const { video } = await getOrCreateAccountAndVideoAndChannel(videoAbuseToCreateData.object)
235 const video = await VideoModel.loadByUrlAndPopulateAccount(videoAbuseToCreateData.object, t)
236 if (!video) {
237 logger.warn('Unknown video %s for remote video abuse.', videoAbuseToCreateData.object)
238 return undefined
239 }
240 178
179 return sequelizeTypescript.transaction(async t => {
241 const videoAbuseData = { 180 const videoAbuseData = {
242 reporterAccountId: account.id, 181 reporterAccountId: account.id,
243 reason: videoAbuseToCreateData.content, 182 reason: videoAbuseToCreateData.content,
@@ -259,41 +198,33 @@ function processCreateVideoComment (byActor: ActorModel, activity: ActivityCreat
259 return retryTransactionWrapper(createVideoComment, options) 198 return retryTransactionWrapper(createVideoComment, options)
260} 199}
261 200
262function createVideoComment (byActor: ActorModel, activity: ActivityCreate) { 201async function createVideoComment (byActor: ActorModel, activity: ActivityCreate) {
263 const comment = activity.object as VideoCommentObject 202 const comment = activity.object as VideoCommentObject
264 const byAccount = byActor.Account 203 const byAccount = byActor.Account
265 204
266 if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url) 205 if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url)
267 206
207 const { video, parents } = await resolveThread(comment.inReplyTo)
208
268 return sequelizeTypescript.transaction(async t => { 209 return sequelizeTypescript.transaction(async t => {
269 let video = await VideoModel.loadByUrlAndPopulateAccount(comment.inReplyTo, t) 210 let originCommentId = null
270 let objectToCreate 211 let inReplyToCommentId = null
212
213 if (parents.length !== 0) {
214 const parent = parents[0]
215
216 originCommentId = parent.getThreadId()
217 inReplyToCommentId = parent.id
218 }
271 219
272 // This is a new thread 220 // This is a new thread
273 if (video) { 221 const objectToCreate = {
274 objectToCreate = { 222 url: comment.id,
275 url: comment.id, 223 text: comment.content,
276 text: comment.content, 224 originCommentId,
277 originCommentId: null, 225 inReplyToCommentId,
278 inReplyToComment: null, 226 videoId: video.id,
279 videoId: video.id, 227 accountId: byAccount.id
280 accountId: byAccount.id
281 }
282 } else {
283 const inReplyToComment = await VideoCommentModel.loadByUrl(comment.inReplyTo, t)
284 if (!inReplyToComment) throw new Error('Unknown replied comment ' + comment.inReplyTo)
285
286 video = await VideoModel.loadAndPopulateAccount(inReplyToComment.videoId)
287
288 const originCommentId = inReplyToComment.originCommentId || inReplyToComment.id
289 objectToCreate = {
290 url: comment.id,
291 text: comment.content,
292 originCommentId,
293 inReplyToCommentId: inReplyToComment.id,
294 videoId: video.id,
295 accountId: byAccount.id
296 }
297 } 228 }
298 229
299 const options = { 230 const options = {
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts
index 77fadabe1..0d161b126 100644
--- a/server/lib/activitypub/process/process-like.ts
+++ b/server/lib/activitypub/process/process-like.ts
@@ -3,9 +3,9 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
3import { sequelizeTypescript } from '../../../initializers' 3import { sequelizeTypescript } from '../../../initializers'
4import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 4import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/activitypub/actor'
6import { VideoModel } from '../../../models/video/video'
7import { getOrCreateActorAndServerAndModel } from '../actor' 6import { getOrCreateActorAndServerAndModel } from '../actor'
8import { forwardActivity } from '../send/misc' 7import { forwardActivity } from '../send/misc'
8import { getOrCreateAccountAndVideoAndChannel } from '../videos'
9 9
10async function processLikeActivity (activity: ActivityLike) { 10async function processLikeActivity (activity: ActivityLike) {
11 const actor = await getOrCreateActorAndServerAndModel(activity.actor) 11 const actor = await getOrCreateActorAndServerAndModel(activity.actor)
@@ -30,17 +30,15 @@ async function processLikeVideo (actor: ActorModel, activity: ActivityLike) {
30 return retryTransactionWrapper(createVideoLike, options) 30 return retryTransactionWrapper(createVideoLike, options)
31} 31}
32 32
33function createVideoLike (byActor: ActorModel, activity: ActivityLike) { 33async function createVideoLike (byActor: ActorModel, activity: ActivityLike) {
34 const videoUrl = activity.object 34 const videoUrl = activity.object
35 35
36 const byAccount = byActor.Account 36 const byAccount = byActor.Account
37 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) 37 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
38 38
39 return sequelizeTypescript.transaction(async t => { 39 const { video } = await getOrCreateAccountAndVideoAndChannel(videoUrl)
40 const video = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
41
42 if (!video) throw new Error('Unknown video ' + videoUrl)
43 40
41 return sequelizeTypescript.transaction(async t => {
44 const rate = { 42 const rate = {
45 type: 'like' as 'like', 43 type: 'like' as 'like',
46 videoId: video.id, 44 videoId: video.id,
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index 9cad59233..5a770bb97 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -7,8 +7,8 @@ import { AccountModel } from '../../../models/account/account'
7import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 7import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
8import { ActorModel } from '../../../models/activitypub/actor' 8import { ActorModel } from '../../../models/activitypub/actor'
9import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 9import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
10import { VideoModel } from '../../../models/video/video'
11import { forwardActivity } from '../send/misc' 10import { forwardActivity } from '../send/misc'
11import { getOrCreateAccountAndVideoAndChannel } from '../videos'
12 12
13async function processUndoActivity (activity: ActivityUndo) { 13async function processUndoActivity (activity: ActivityUndo) {
14 const activityToUndo = activity.object 14 const activityToUndo = activity.object
@@ -43,16 +43,15 @@ function processUndoLike (actorUrl: string, activity: ActivityUndo) {
43 return retryTransactionWrapper(undoLike, options) 43 return retryTransactionWrapper(undoLike, options)
44} 44}
45 45
46function undoLike (actorUrl: string, activity: ActivityUndo) { 46async function undoLike (actorUrl: string, activity: ActivityUndo) {
47 const likeActivity = activity.object as ActivityLike 47 const likeActivity = activity.object as ActivityLike
48 48
49 const { video } = await getOrCreateAccountAndVideoAndChannel(likeActivity.object)
50
49 return sequelizeTypescript.transaction(async t => { 51 return sequelizeTypescript.transaction(async t => {
50 const byAccount = await AccountModel.loadByUrl(actorUrl, t) 52 const byAccount = await AccountModel.loadByUrl(actorUrl, t)
51 if (!byAccount) throw new Error('Unknown account ' + actorUrl) 53 if (!byAccount) throw new Error('Unknown account ' + actorUrl)
52 54
53 const video = await VideoModel.loadByUrlAndPopulateAccount(likeActivity.object, t)
54 if (!video) throw new Error('Unknown video ' + likeActivity.actor)
55
56 const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) 55 const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t)
57 if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) 56 if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`)
58 57
@@ -76,16 +75,15 @@ function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
76 return retryTransactionWrapper(undoDislike, options) 75 return retryTransactionWrapper(undoDislike, options)
77} 76}
78 77
79function undoDislike (actorUrl: string, activity: ActivityUndo) { 78async function undoDislike (actorUrl: string, activity: ActivityUndo) {
80 const dislike = activity.object.object as DislikeObject 79 const dislike = activity.object.object as DislikeObject
81 80
81 const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object)
82
82 return sequelizeTypescript.transaction(async t => { 83 return sequelizeTypescript.transaction(async t => {
83 const byAccount = await AccountModel.loadByUrl(actorUrl, t) 84 const byAccount = await AccountModel.loadByUrl(actorUrl, t)
84 if (!byAccount) throw new Error('Unknown account ' + actorUrl) 85 if (!byAccount) throw new Error('Unknown account ' + actorUrl)
85 86
86 const video = await VideoModel.loadByUrlAndPopulateAccount(dislike.object, t)
87 if (!video) throw new Error('Unknown video ' + dislike.actor)
88
89 const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) 87 const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t)
90 if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) 88 if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`)
91 89
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index 2c094f7ca..a5431c76b 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -9,10 +9,9 @@ import { sequelizeTypescript } from '../../../initializers'
9import { AccountModel } from '../../../models/account/account' 9import { AccountModel } from '../../../models/account/account'
10import { ActorModel } from '../../../models/activitypub/actor' 10import { ActorModel } from '../../../models/activitypub/actor'
11import { TagModel } from '../../../models/video/tag' 11import { TagModel } from '../../../models/video/tag'
12import { VideoModel } from '../../../models/video/video'
13import { VideoFileModel } from '../../../models/video/video-file' 12import { VideoFileModel } from '../../../models/video/video-file'
14import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' 13import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor'
15import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' 14import { getOrCreateAccountAndVideoAndChannel, videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from '../videos'
16 15
17async function processUpdateActivity (activity: ActivityUpdate) { 16async function processUpdateActivity (activity: ActivityUpdate) {
18 const actor = await getOrCreateActorAndServerAndModel(activity.actor) 17 const actor = await getOrCreateActorAndServerAndModel(activity.actor)
@@ -46,8 +45,10 @@ function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) {
46async function updateRemoteVideo (actor: ActorModel, activity: ActivityUpdate) { 45async function updateRemoteVideo (actor: ActorModel, activity: ActivityUpdate) {
47 const videoAttributesToUpdate = activity.object as VideoTorrentObject 46 const videoAttributesToUpdate = activity.object as VideoTorrentObject
48 47
48 const res = await getOrCreateAccountAndVideoAndChannel(videoAttributesToUpdate.id)
49
49 logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid) 50 logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid)
50 let videoInstance: VideoModel 51 let videoInstance = res.video
51 let videoFieldsSave: any 52 let videoFieldsSave: any
52 53
53 try { 54 try {
@@ -56,9 +57,6 @@ async function updateRemoteVideo (actor: ActorModel, activity: ActivityUpdate) {
56 transaction: t 57 transaction: t
57 } 58 }
58 59
59 const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(videoAttributesToUpdate.id, t)
60 if (!videoInstance) throw new Error('Video ' + videoAttributesToUpdate.id + ' not found.')
61
62 videoFieldsSave = videoInstance.toJSON() 60 videoFieldsSave = videoInstance.toJSON()
63 61
64 const videoChannel = videoInstance.VideoChannel 62 const videoChannel = videoInstance.VideoChannel
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
new file mode 100644
index 000000000..17c86a381
--- /dev/null
+++ b/server/lib/activitypub/video-comments.ts
@@ -0,0 +1,156 @@
1import { VideoCommentObject } from '../../../shared/models/activitypub/objects/video-comment-object'
2import { isVideoCommentObjectValid } from '../../helpers/custom-validators/activitypub/video-comments'
3import { logger } from '../../helpers/logger'
4import { doRequest } from '../../helpers/requests'
5import { ACTIVITY_PUB } from '../../initializers'
6import { ActorModel } from '../../models/activitypub/actor'
7import { VideoModel } from '../../models/video/video'
8import { VideoCommentModel } from '../../models/video/video-comment'
9import { getOrCreateActorAndServerAndModel } from './actor'
10import { getOrCreateAccountAndVideoAndChannel } from './videos'
11
12async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) {
13 let originCommentId: number = null
14 let inReplyToCommentId: number = null
15
16 // If this is not a reply to the video (thread), create or get the parent comment
17 if (video.url !== comment.inReplyTo) {
18 const [ parent ] = await addVideoComment(video, comment.inReplyTo)
19 if (!parent) {
20 logger.warn('Cannot fetch or get parent comment %s of comment %s.', comment.inReplyTo, comment.id)
21 return undefined
22 }
23
24 originCommentId = parent.originCommentId || parent.id
25 inReplyToCommentId = parent.id
26 }
27
28 return {
29 url: comment.url,
30 text: comment.content,
31 videoId: video.id,
32 accountId: actor.Account.id,
33 inReplyToCommentId,
34 originCommentId,
35 createdAt: new Date(comment.published),
36 updatedAt: new Date(comment.updated)
37 }
38}
39
40async function addVideoComments (instance: VideoModel, commentUrls: string[]) {
41 for (const commentUrl of commentUrls) {
42 await addVideoComment(instance, commentUrl)
43 }
44}
45
46async function addVideoComment (videoInstance: VideoModel, commentUrl: string) {
47 logger.info('Fetching remote video comment %s.', commentUrl)
48
49 const { body } = await doRequest({
50 uri: commentUrl,
51 json: true,
52 activityPub: true
53 })
54
55 if (isVideoCommentObjectValid(body) === false) {
56 logger.debug('Remote video comment JSON is not valid.', { body })
57 return undefined
58 }
59
60 const actorUrl = body.attributedTo
61 if (!actorUrl) return []
62
63 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
64 const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body)
65 if (!entry) return []
66
67 return VideoCommentModel.findOrCreate({
68 where: {
69 url: body.id
70 },
71 defaults: entry
72 })
73}
74
75async function resolveThread (url: string, comments: VideoCommentModel[] = []) {
76 // Already have this comment?
77 const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideo(url)
78 if (commentFromDatabase) {
79 let parentComments = comments.concat([ commentFromDatabase ])
80
81 // Speed up things and resolve directly the thread
82 if (commentFromDatabase.InReplyToVideoComment) {
83 const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC')
84 console.log(data)
85
86 parentComments = parentComments.concat(data)
87 }
88
89 return resolveThread(commentFromDatabase.Video.url, parentComments)
90 }
91
92 try {
93 // Maybe it's a reply to a video?
94 const { video } = await getOrCreateAccountAndVideoAndChannel(url)
95
96 if (comments.length !== 0) {
97 const firstReply = comments[ comments.length - 1 ]
98 firstReply.inReplyToCommentId = null
99 firstReply.originCommentId = null
100 firstReply.videoId = video.id
101 comments[comments.length - 1] = await firstReply.save()
102
103 for (let i = comments.length - 2; i >= 0; i--) {
104 const comment = comments[ i ]
105 comment.originCommentId = firstReply.id
106 comment.inReplyToCommentId = comments[ i + 1 ].id
107 comment.videoId = video.id
108
109 comments[i] = await comment.save()
110 }
111 }
112
113 return { video, parents: comments }
114 } catch (err) {
115 logger.debug('Cannot get or create account and video and channel for reply %s, fetch comment', url, err)
116
117 if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) {
118 throw new Error('Recursion limit reached when resolving a thread')
119 }
120
121 const { body } = await doRequest({
122 uri: url,
123 json: true,
124 activityPub: true
125 })
126
127 if (isVideoCommentObjectValid(body) === false) {
128 throw new Error('Remote video comment JSON is not valid :' + JSON.stringify(body))
129 }
130
131 const actorUrl = body.attributedTo
132 if (!actorUrl) throw new Error('Miss attributed to in comment')
133
134 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
135 const comment = new VideoCommentModel({
136 url: body.url,
137 text: body.content,
138 videoId: null,
139 accountId: actor.Account.id,
140 inReplyToCommentId: null,
141 originCommentId: null,
142 createdAt: new Date(body.published),
143 updatedAt: new Date(body.updated)
144 })
145
146 return resolveThread(body.inReplyTo, comments.concat([ comment ]))
147 }
148
149}
150
151export {
152 videoCommentActivityObjectToDBAttributes,
153 addVideoComments,
154 addVideoComment,
155 resolveThread
156}
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts
new file mode 100644
index 000000000..1b2958cca
--- /dev/null
+++ b/server/lib/activitypub/video-rates.ts
@@ -0,0 +1,52 @@
1import { Transaction } from 'sequelize'
2import { AccountModel } from '../../models/account/account'
3import { VideoModel } from '../../models/video/video'
4import {
5 sendCreateDislikeToOrigin, sendCreateDislikeToVideoFollowers, sendLikeToOrigin, sendLikeToVideoFollowers, sendUndoDislikeToOrigin,
6 sendUndoDislikeToVideoFollowers, sendUndoLikeToOrigin, sendUndoLikeToVideoFollowers
7} from './send'
8
9async function sendVideoRateChangeToFollowers (account: AccountModel,
10 video: VideoModel,
11 likes: number,
12 dislikes: number,
13 t: Transaction) {
14 const actor = account.Actor
15
16 // Keep the order: first we undo and then we create
17
18 // Undo Like
19 if (likes < 0) await sendUndoLikeToVideoFollowers(actor, video, t)
20 // Undo Dislike
21 if (dislikes < 0) await sendUndoDislikeToVideoFollowers(actor, video, t)
22
23 // Like
24 if (likes > 0) await sendLikeToVideoFollowers(actor, video, t)
25 // Dislike
26 if (dislikes > 0) await sendCreateDislikeToVideoFollowers(actor, video, t)
27}
28
29async function sendVideoRateChangeToOrigin (account: AccountModel,
30 video: VideoModel,
31 likes: number,
32 dislikes: number,
33 t: Transaction) {
34 const actor = account.Actor
35
36 // Keep the order: first we undo and then we create
37
38 // Undo Like
39 if (likes < 0) await sendUndoLikeToOrigin(actor, video, t)
40 // Undo Dislike
41 if (dislikes < 0) await sendUndoDislikeToOrigin(actor, video, t)
42
43 // Like
44 if (likes > 0) await sendLikeToOrigin(actor, video, t)
45 // Dislike
46 if (dislikes > 0) await sendCreateDislikeToOrigin(actor, video, t)
47}
48
49export {
50 sendVideoRateChangeToFollowers,
51 sendVideoRateChangeToOrigin
52}
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 8bc928b93..708f4a897 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -1,15 +1,23 @@
1import * as Bluebird from 'bluebird'
2import * as magnetUtil from 'magnet-uri'
1import { join } from 'path' 3import { join } from 'path'
2import * as request from 'request' 4import * as request from 'request'
3import { Transaction } from 'sequelize'
4import { ActivityIconObject } from '../../../shared/index' 5import { ActivityIconObject } from '../../../shared/index'
6import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
7import { VideoPrivacy } from '../../../shared/models/videos'
8import { isVideoTorrentObjectValid } from '../../helpers/custom-validators/activitypub/videos'
9import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
10import { retryTransactionWrapper } from '../../helpers/database-utils'
11import { logger } from '../../helpers/logger'
5import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' 12import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
6import { CONFIG, REMOTE_SCHEME, STATIC_PATHS } from '../../initializers' 13import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, STATIC_PATHS, VIDEO_MIMETYPE_EXT } from '../../initializers'
7import { AccountModel } from '../../models/account/account' 14import { ActorModel } from '../../models/activitypub/actor'
15import { TagModel } from '../../models/video/tag'
8import { VideoModel } from '../../models/video/video' 16import { VideoModel } from '../../models/video/video'
9import { 17import { VideoChannelModel } from '../../models/video/video-channel'
10 sendCreateDislikeToOrigin, sendCreateDislikeToVideoFollowers, sendLikeToOrigin, sendLikeToVideoFollowers, sendUndoDislikeToOrigin, 18import { VideoFileModel } from '../../models/video/video-file'
11 sendUndoDislikeToVideoFollowers, sendUndoLikeToOrigin, sendUndoLikeToVideoFollowers 19import { VideoShareModel } from '../../models/video/video-share'
12} from './send' 20import { getOrCreateActorAndServerAndModel } from './actor'
13 21
14function fetchRemoteVideoPreview (video: VideoModel, reject: Function) { 22function fetchRemoteVideoPreview (video: VideoModel, reject: Function) {
15 // FIXME: use url 23 // FIXME: use url
@@ -45,54 +53,221 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject)
45 return doRequestAndSaveToFile(options, thumbnailPath) 53 return doRequestAndSaveToFile(options, thumbnailPath)
46} 54}
47 55
48async function sendVideoRateChangeToFollowers ( 56async function videoActivityObjectToDBAttributes (videoChannel: VideoChannelModel,
49 account: AccountModel, 57 videoObject: VideoTorrentObject,
50 video: VideoModel, 58 to: string[] = [],
51 likes: number, 59 cc: string[] = []) {
52 dislikes: number, 60 let privacy = VideoPrivacy.PRIVATE
53 t: Transaction 61 if (to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.PUBLIC
54) { 62 else if (cc.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.UNLISTED
55 const actor = account.Actor 63
56 64 const duration = videoObject.duration.replace(/[^\d]+/, '')
57 // Keep the order: first we undo and then we create 65 let language = null
58 66 if (videoObject.language) {
59 // Undo Like 67 language = parseInt(videoObject.language.identifier, 10)
60 if (likes < 0) await sendUndoLikeToVideoFollowers(actor, video, t) 68 }
61 // Undo Dislike 69
62 if (dislikes < 0) await sendUndoDislikeToVideoFollowers(actor, video, t) 70 let category = null
63 71 if (videoObject.category) {
64 // Like 72 category = parseInt(videoObject.category.identifier, 10)
65 if (likes > 0) await sendLikeToVideoFollowers(actor, video, t) 73 }
66 // Dislike 74
67 if (dislikes > 0) await sendCreateDislikeToVideoFollowers(actor, video, t) 75 let licence = null
76 if (videoObject.licence) {
77 licence = parseInt(videoObject.licence.identifier, 10)
78 }
79
80 let description = null
81 if (videoObject.content) {
82 description = videoObject.content
83 }
84
85 return {
86 name: videoObject.name,
87 uuid: videoObject.uuid,
88 url: videoObject.id,
89 category,
90 licence,
91 language,
92 description,
93 nsfw: videoObject.nsfw,
94 commentsEnabled: videoObject.commentsEnabled,
95 channelId: videoChannel.id,
96 duration: parseInt(duration, 10),
97 createdAt: new Date(videoObject.published),
98 // FIXME: updatedAt does not seems to be considered by Sequelize
99 updatedAt: new Date(videoObject.updated),
100 views: videoObject.views,
101 likes: 0,
102 dislikes: 0,
103 remote: true,
104 privacy
105 }
106}
107
108function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
109 const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
110 const fileUrls = videoObject.url.filter(u => {
111 return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/')
112 })
113
114 if (fileUrls.length === 0) {
115 throw new Error('Cannot find video files for ' + videoCreated.url)
116 }
117
118 const attributes = []
119 for (const fileUrl of fileUrls) {
120 // Fetch associated magnet uri
121 const magnet = videoObject.url.find(u => {
122 return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.width === fileUrl.width
123 })
124
125 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.url)
126
127 const parsed = magnetUtil.decode(magnet.url)
128 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) throw new Error('Cannot parse magnet URI ' + magnet.url)
129
130 const attribute = {
131 extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
132 infoHash: parsed.infoHash,
133 resolution: fileUrl.width,
134 size: fileUrl.size,
135 videoId: videoCreated.id
136 }
137 attributes.push(attribute)
138 }
139
140 return attributes
141}
142
143async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel) {
144 logger.debug('Adding remote video %s.', videoObject.id)
145
146 return sequelizeTypescript.transaction(async t => {
147 const sequelizeOptions = {
148 transaction: t
149 }
150 const videoFromDatabase = await VideoModel.loadByUUIDOrURLAndPopulateAccount(videoObject.uuid, videoObject.id, t)
151 if (videoFromDatabase) return videoFromDatabase
152
153 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to, videoObject.cc)
154 const video = VideoModel.build(videoData)
155
156 // Don't block on request
157 generateThumbnailFromUrl(video, videoObject.icon)
158 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, err))
159
160 const videoCreated = await video.save(sequelizeOptions)
161
162 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
163 if (videoFileAttributes.length === 0) {
164 throw new Error('Cannot find valid files for video %s ' + videoObject.url)
165 }
166
167 const tasks: Bluebird<any>[] = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
168 await Promise.all(tasks)
169
170 const tags = videoObject.tag.map(t => t.name)
171 const tagInstances = await TagModel.findOrCreateTags(tags, t)
172 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
173
174 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
175
176 videoCreated.VideoChannel = channelActor.VideoChannel
177 return videoCreated
178 })
68} 179}
69 180
70async function sendVideoRateChangeToOrigin ( 181async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentObject | string, actor?: ActorModel) {
71 account: AccountModel, 182 if (typeof videoObject === 'string') {
72 video: VideoModel, 183 const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoObject)
73 likes: number, 184 if (videoFromDatabase) {
74 dislikes: number, 185 return {
75 t: Transaction 186 video: videoFromDatabase,
76) { 187 actor: videoFromDatabase.VideoChannel.Account.Actor,
77 const actor = account.Actor 188 channelActor: videoFromDatabase.VideoChannel.Actor
78 189 }
79 // Keep the order: first we undo and then we create 190 }
80 191
81 // Undo Like 192 videoObject = await fetchRemoteVideo(videoObject)
82 if (likes < 0) await sendUndoLikeToOrigin(actor, video, t) 193 if (!videoObject) throw new Error('Cannot fetch remote video')
83 // Undo Dislike 194 }
84 if (dislikes < 0) await sendUndoDislikeToOrigin(actor, video, t) 195
85 196 if (!actor) {
86 // Like 197 const actorObj = videoObject.attributedTo.find(a => a.type === 'Person')
87 if (likes > 0) await sendLikeToOrigin(actor, video, t) 198 if (!actorObj) throw new Error('Cannot find associated actor to video ' + videoObject.url)
88 // Dislike 199
89 if (dislikes > 0) await sendCreateDislikeToOrigin(actor, video, t) 200 actor = await getOrCreateActorAndServerAndModel(actorObj.id)
201 }
202
203 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
204 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
205
206 const channelActor = await getOrCreateActorAndServerAndModel(channel.id)
207
208 const options = {
209 arguments: [ videoObject, channelActor ],
210 errorMessage: 'Cannot insert the remote video with many retries.'
211 }
212
213 const video = await retryTransactionWrapper(getOrCreateVideo, options)
214
215 return { actor, channelActor, video }
216}
217
218async function addVideoShares (instance: VideoModel, shareUrls: string[]) {
219 for (const shareUrl of shareUrls) {
220 // Fetch url
221 const { body } = await doRequest({
222 uri: shareUrl,
223 json: true,
224 activityPub: true
225 })
226 const actorUrl = body.actor
227 if (!actorUrl) continue
228
229 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
230
231 const entry = {
232 actorId: actor.id,
233 videoId: instance.id
234 }
235
236 await VideoShareModel.findOrCreate({
237 where: entry,
238 defaults: entry
239 })
240 }
90} 241}
91 242
92export { 243export {
244 getOrCreateAccountAndVideoAndChannel,
93 fetchRemoteVideoPreview, 245 fetchRemoteVideoPreview,
94 fetchRemoteVideoDescription, 246 fetchRemoteVideoDescription,
95 generateThumbnailFromUrl, 247 generateThumbnailFromUrl,
96 sendVideoRateChangeToFollowers, 248 videoActivityObjectToDBAttributes,
97 sendVideoRateChangeToOrigin 249 videoFileActivityUrlToDBAttributes,
250 getOrCreateVideo,
251 addVideoShares}
252
253// ---------------------------------------------------------------------------
254
255async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject> {
256 const options = {
257 uri: videoUrl,
258 method: 'GET',
259 json: true,
260 activityPub: true
261 }
262
263 logger.info('Fetching remote video %s.', videoUrl)
264
265 const { body } = await doRequest(options)
266
267 if (isVideoTorrentObjectValid(body) === false) {
268 logger.debug('Remote video JSON is not valid.', { body })
269 return undefined
270 }
271
272 return body
98} 273}
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index 707f140af..b88e06b41 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -63,6 +63,9 @@ enum ScopeNames {
63 tableName: 'actor', 63 tableName: 'actor',
64 indexes: [ 64 indexes: [
65 { 65 {
66 fields: [ 'url' ]
67 },
68 {
66 fields: [ 'preferredUsername', 'serverId' ], 69 fields: [ 'preferredUsername', 'serverId' ],
67 unique: true 70 unique: true
68 } 71 }
diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts
index dbb2fe429..fffa4bb57 100644
--- a/server/models/video/video-comment.ts
+++ b/server/models/video/video-comment.ts
@@ -208,7 +208,7 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
208 .findOne(query) 208 .findOne(query)
209 } 209 }
210 210
211 static loadByUrl (url: string, t?: Sequelize.Transaction) { 211 static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) {
212 const query: IFindOptions<VideoCommentModel> = { 212 const query: IFindOptions<VideoCommentModel> = {
213 where: { 213 where: {
214 url 214 url
@@ -217,10 +217,10 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
217 217
218 if (t !== undefined) query.transaction = t 218 if (t !== undefined) query.transaction = t
219 219
220 return VideoCommentModel.findOne(query) 220 return VideoCommentModel.scope([ ScopeNames.WITH_ACCOUNT ]).findOne(query)
221 } 221 }
222 222
223 static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) { 223 static loadByUrlAndPopulateReplyAndVideo (url: string, t?: Sequelize.Transaction) {
224 const query: IFindOptions<VideoCommentModel> = { 224 const query: IFindOptions<VideoCommentModel> = {
225 where: { 225 where: {
226 url 226 url
@@ -229,7 +229,7 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
229 229
230 if (t !== undefined) query.transaction = t 230 if (t !== undefined) query.transaction = t
231 231
232 return VideoCommentModel.scope([ ScopeNames.WITH_ACCOUNT ]).findOne(query) 232 return VideoCommentModel.scope([ ScopeNames.WITH_IN_REPLY_TO, ScopeNames.WITH_VIDEO ]).findOne(query)
233 } 233 }
234 234
235 static listThreadsForApi (videoId: number, start: number, count: number, sort: string) { 235 static listThreadsForApi (videoId: number, start: number, count: number, sort: string) {
@@ -271,9 +271,9 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
271 }) 271 })
272 } 272 }
273 273
274 static listThreadParentComments (comment: VideoCommentModel, t: Sequelize.Transaction) { 274 static listThreadParentComments (comment: VideoCommentModel, t: Sequelize.Transaction, order: 'ASC' | 'DESC' = 'ASC') {
275 const query = { 275 const query = {
276 order: [ [ 'createdAt', 'ASC' ] ], 276 order: [ [ 'createdAt', order ] ],
277 where: { 277 where: {
278 [ Sequelize.Op.or ]: [ 278 [ Sequelize.Op.or ]: [
279 { id: comment.getThreadId() }, 279 { id: comment.getThreadId() },
@@ -281,6 +281,9 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
281 ], 281 ],
282 id: { 282 id: {
283 [ Sequelize.Op.ne ]: comment.id 283 [ Sequelize.Op.ne ]: comment.id
284 },
285 createdAt: {
286 [ Sequelize.Op.lt ]: comment.createdAt
284 } 287 }
285 }, 288 },
286 transaction: t 289 transaction: t
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 918892938..6b825bf93 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -178,6 +178,10 @@ enum ScopeNames {
178 }, 178 },
179 { 179 {
180 fields: [ 'id', 'privacy' ] 180 fields: [ 'id', 'privacy' ]
181 },
182 {
183 fields: [ 'url'],
184 unique: true
181 } 185 }
182 ] 186 ]
183}) 187})
@@ -535,7 +539,7 @@ export class VideoModel extends Model<VideoModel> {
535 return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) 539 return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
536 } 540 }
537 541
538 static loadByUUIDOrURL (uuid: string, url: string, t?: Sequelize.Transaction) { 542 static loadByUUIDOrURLAndPopulateAccount (uuid: string, url: string, t?: Sequelize.Transaction) {
539 const query: IFindOptions<VideoModel> = { 543 const query: IFindOptions<VideoModel> = {
540 where: { 544 where: {
541 [Sequelize.Op.or]: [ 545 [Sequelize.Op.or]: [
@@ -547,7 +551,7 @@ export class VideoModel extends Model<VideoModel> {
547 551
548 if (t !== undefined) query.transaction = t 552 if (t !== undefined) query.transaction = t
549 553
550 return VideoModel.scope(ScopeNames.WITH_FILES).findOne(query) 554 return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
551 } 555 }
552 556
553 static loadAndPopulateAccountAndServerAndTags (id: number) { 557 static loadAndPopulateAccountAndServerAndTags (id: number) {
@@ -983,6 +987,10 @@ export class VideoModel extends Model<VideoModel> {
983 { 987 {
984 type: 'Group', 988 type: 'Group',
985 id: this.VideoChannel.Actor.url 989 id: this.VideoChannel.Actor.url
990 },
991 {
992 type: 'Person',
993 id: this.VideoChannel.Account.Actor.url
986 } 994 }
987 ] 995 ]
988 } 996 }
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts
new file mode 100644
index 000000000..da134c09f
--- /dev/null
+++ b/server/tests/api/server/handle-down.ts
@@ -0,0 +1,147 @@
1/* tslint:disable:no-unused-expression */
2
3import * as chai from 'chai'
4import 'mocha'
5import { VideoPrivacy } from '../../../../shared/models/videos'
6import { completeVideoCheck, runServer, viewVideo } from '../../utils'
7
8import {
9 flushAndRunMultipleServers, flushTests, getVideosList, killallServers, ServerInfo, setAccessTokensToServers, uploadVideo,
10 wait
11} from '../../utils/index'
12import { follow, getFollowersListPaginationAndSort } from '../../utils/server/follows'
13import { getJobsListPaginationAndSort } from '../../utils/server/jobs'
14
15const expect = chai.expect
16
17describe('Test handle downs', function () {
18 let servers: ServerInfo[] = []
19
20 const videoAttributes = {
21 name: 'my super name for server 1',
22 category: 5,
23 licence: 4,
24 language: 9,
25 nsfw: true,
26 description: 'my super description for server 1',
27 tags: [ 'tag1p1', 'tag2p1' ],
28 fixture: 'video_short1.webm'
29 }
30
31 const checkAttributes = {
32 name: 'my super name for server 1',
33 category: 5,
34 licence: 4,
35 language: 9,
36 nsfw: true,
37 description: 'my super description for server 1',
38 host: 'localhost:9001',
39 account: 'root',
40 isLocal: false,
41 duration: 10,
42 tags: [ 'tag1p1', 'tag2p1' ],
43 privacy: VideoPrivacy.PUBLIC,
44 commentsEnabled: true,
45 channel: {
46 name: 'Default root channel',
47 description: '',
48 isLocal: false
49 },
50 fixture: 'video_short1.webm',
51 files: [
52 {
53 resolution: 720,
54 size: 572456
55 }
56 ]
57 }
58
59 before(async function () {
60 this.timeout(20000)
61
62 servers = await flushAndRunMultipleServers(2)
63
64 // Get the access tokens
65 await setAccessTokensToServers(servers)
66 })
67
68 it('Should remove followers that are often down', async function () {
69 this.timeout(60000)
70
71 await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken)
72
73 await wait(5000)
74
75 await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
76
77 await wait(5000)
78
79 for (const server of servers) {
80 const res = await getVideosList(server.url)
81 expect(res.body.data).to.be.an('array')
82 expect(res.body.data).to.have.lengthOf(1)
83 }
84
85 // Kill server 1
86 killallServers([ servers[1] ])
87
88 // Remove server 2 follower
89 for (let i = 0; i < 10; i++) {
90 await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, videoAttributes)
91 }
92
93 await wait(10000)
94
95 const res = await getFollowersListPaginationAndSort(servers[0].url, 0, 1, 'createdAt')
96 expect(res.body.data).to.be.an('array')
97 expect(res.body.data).to.have.lengthOf(0)
98 })
99
100 it('Should not have pending/processing jobs anymore', async function () {
101 const res = await getJobsListPaginationAndSort(servers[0].url, servers[0].accessToken, 0, 50, '-createdAt')
102 const jobs = res.body.data
103
104 for (const job of jobs) {
105 expect(job.state).not.to.equal('pending')
106 expect(job.state).not.to.equal('processing')
107 }
108 })
109
110 it('Should follow server 1', async function () {
111 servers[1] = await runServer(2)
112
113 await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken)
114
115 await wait(5000)
116
117 const res = await getFollowersListPaginationAndSort(servers[0].url, 0, 1, 'createdAt')
118 expect(res.body.data).to.be.an('array')
119 expect(res.body.data).to.have.lengthOf(1)
120 })
121
122 it('Should send a view to server 2, and automatically fetch the video', async function () {
123 const resVideo = await getVideosList(servers[0].url)
124 const videoServer1 = resVideo.body.data[0]
125
126 await viewVideo(servers[0].url, videoServer1.uuid)
127
128 await wait(5000)
129
130 const res = await getVideosList(servers[1].url)
131 const videoServer2 = res.body.data.find(v => v.url === videoServer1.url)
132
133 expect(videoServer2).not.to.be.undefined
134
135 await completeVideoCheck(servers[1].url, videoServer2, checkAttributes)
136
137 })
138
139 after(async function () {
140 killallServers(servers)
141
142 // Keep the logs if the test failed
143 if (this['ok']) {
144 await flushTests()
145 }
146 })
147})
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-03 18:15:17 +0000