aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--client/src/app/header/header.component.scss2
-rw-r--r--client/src/app/shared/user-subscription/index.ts2
-rw-r--r--client/src/app/shared/video-channel/video-channel.service.ts10
-rw-r--r--server/controllers/api/search.ts35
-rw-r--r--server/initializers/constants.ts2
-rw-r--r--server/lib/activitypub/actor.ts4
-rw-r--r--server/lib/activitypub/crawl.ts3
-rw-r--r--server/lib/activitypub/process/process-announce.ts4
-rw-r--r--server/lib/activitypub/process/process-create.ts9
-rw-r--r--server/lib/activitypub/video-comments.ts9
-rw-r--r--server/lib/activitypub/videos.ts182
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-fetcher.ts26
-rw-r--r--server/tests/api/check-params/videos.ts6
-rw-r--r--server/tests/api/server/follows.ts3
-rw-r--r--server/tests/api/server/handle-down.ts3
-rw-r--r--server/tests/api/users/user-subscriptions.ts28
-rw-r--r--server/tests/api/videos/multiple-servers.ts18
-rw-r--r--server/tests/api/videos/single-server.ts6
-rw-r--r--server/tests/utils/videos/videos.ts25
19 files changed, 243 insertions, 134 deletions
diff --git a/client/src/app/header/header.component.scss b/client/src/app/header/header.component.scss
index d79e6274b..6ce92fc22 100644
--- a/client/src/app/header/header.component.scss
+++ b/client/src/app/header/header.component.scss
@@ -4,7 +4,7 @@
4#search-video { 4#search-video {
5 @include peertube-input-text($search-input-width); 5 @include peertube-input-text($search-input-width);
6 margin-right: 15px; 6 margin-right: 15px;
7 padding-right: 25px; // For the search icon 7 padding-right: 40px; // For the search icon
8 8
9 &::placeholder { 9 &::placeholder {
10 color: #000; 10 color: #000;
diff --git a/client/src/app/shared/user-subscription/index.ts b/client/src/app/shared/user-subscription/index.ts
index 024b36a41..faddae66a 100644
--- a/client/src/app/shared/user-subscription/index.ts
+++ b/client/src/app/shared/user-subscription/index.ts
@@ -1,2 +1,2 @@
1export * from './user-subscription.service' 1export * from './user-subscription.service'
2export * from './subscribe-button.component' \ No newline at end of file 2export * from './subscribe-button.component'
diff --git a/client/src/app/shared/video-channel/video-channel.service.ts b/client/src/app/shared/video-channel/video-channel.service.ts
index 46b121790..c94411146 100644
--- a/client/src/app/shared/video-channel/video-channel.service.ts
+++ b/client/src/app/shared/video-channel/video-channel.service.ts
@@ -17,11 +17,6 @@ export class VideoChannelService {
17 17
18 videoChannelLoaded = new ReplaySubject<VideoChannel>(1) 18 videoChannelLoaded = new ReplaySubject<VideoChannel>(1)
19 19
20 constructor (
21 private authHttp: HttpClient,
22 private restExtractor: RestExtractor
23 ) {}
24
25 static extractVideoChannels (result: ResultList<VideoChannelServer>) { 20 static extractVideoChannels (result: ResultList<VideoChannelServer>) {
26 const videoChannels: VideoChannel[] = [] 21 const videoChannels: VideoChannel[] = []
27 22
@@ -32,6 +27,11 @@ export class VideoChannelService {
32 return { data: videoChannels, total: result.total } 27 return { data: videoChannels, total: result.total }
33 } 28 }
34 29
30 constructor (
31 private authHttp: HttpClient,
32 private restExtractor: RestExtractor
33 ) { }
34
35 getVideoChannel (videoChannelName: string) { 35 getVideoChannel (videoChannelName: string) {
36 return this.authHttp.get<VideoChannel>(VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannelName) 36 return this.authHttp.get<VideoChannel>(VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannelName)
37 .pipe( 37 .pipe(
diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts
index 7a7504b7d..9c2c7d6c1 100644
--- a/server/controllers/api/search.ts
+++ b/server/controllers/api/search.ts
@@ -13,6 +13,8 @@ import {
13 videosSearchSortValidator 13 videosSearchSortValidator
14} from '../../middlewares' 14} from '../../middlewares'
15import { VideosSearchQuery } from '../../../shared/models/search' 15import { VideosSearchQuery } from '../../../shared/models/search'
16import { getOrCreateAccountAndVideoAndChannel } from '../../lib/activitypub'
17import { logger } from '../../helpers/logger'
16 18
17const searchRouter = express.Router() 19const searchRouter = express.Router()
18 20
@@ -33,9 +35,16 @@ export { searchRouter }
33 35
34// --------------------------------------------------------------------------- 36// ---------------------------------------------------------------------------
35 37
36async function searchVideos (req: express.Request, res: express.Response) { 38function searchVideos (req: express.Request, res: express.Response) {
37 const query: VideosSearchQuery = req.query 39 const query: VideosSearchQuery = req.query
40 if (query.search.startsWith('http://') || query.search.startsWith('https://')) {
41 return searchVideoUrl(query.search, res)
42 }
38 43
44 return searchVideosDB(query, res)
45}
46
47async function searchVideosDB (query: VideosSearchQuery, res: express.Response) {
39 const options = Object.assign(query, { 48 const options = Object.assign(query, {
40 includeLocalVideos: true, 49 includeLocalVideos: true,
41 nsfw: buildNSFWFilter(res, query.nsfw) 50 nsfw: buildNSFWFilter(res, query.nsfw)
@@ -44,3 +53,27 @@ async function searchVideos (req: express.Request, res: express.Response) {
44 53
45 return res.json(getFormattedObjects(resultList.data, resultList.total)) 54 return res.json(getFormattedObjects(resultList.data, resultList.total))
46} 55}
56
57async function searchVideoUrl (url: string, res: express.Response) {
58 let video: VideoModel
59
60 try {
61 const syncParam = {
62 likes: false,
63 dislikes: false,
64 shares: false,
65 comments: false,
66 thumbnail: true
67 }
68
69 const res = await getOrCreateAccountAndVideoAndChannel(url, syncParam)
70 video = res ? res.video : undefined
71 } catch (err) {
72 logger.info('Cannot search remote video %s.', url)
73 }
74
75 return res.json({
76 total: video ? 1 : 0,
77 data: video ? [ video.toFormattedJSON() ] : []
78 })
79}
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 7e865fe3b..99b10a7fc 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -112,6 +112,7 @@ const JOB_TTL: { [ id in JobType ]: number } = {
112 'email': 60000 * 10 // 10 minutes 112 'email': 60000 * 10 // 10 minutes
113} 113}
114const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job 114const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job
115const CRAWL_REQUEST_CONCURRENCY = 5 // How many requests in parallel to fetch remote data (likes, shares...)
115const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds 116const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds
116const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days 117const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
117 118
@@ -643,6 +644,7 @@ export {
643 STATIC_DOWNLOAD_PATHS, 644 STATIC_DOWNLOAD_PATHS,
644 RATES_LIMIT, 645 RATES_LIMIT,
645 VIDEO_EXT_MIMETYPE, 646 VIDEO_EXT_MIMETYPE,
647 CRAWL_REQUEST_CONCURRENCY,
646 JOB_COMPLETED_LIFETIME, 648 JOB_COMPLETED_LIFETIME,
647 VIDEO_IMPORT_STATES, 649 VIDEO_IMPORT_STATES,
648 VIDEO_VIEW_LIFETIME, 650 VIDEO_VIEW_LIFETIME,
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index d84b465b2..9922229d2 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -177,7 +177,8 @@ async function addFetchOutboxJob (actor: ActorModel) {
177 } 177 }
178 178
179 const payload = { 179 const payload = {
180 uris: [ actor.outboxUrl ] 180 uri: actor.outboxUrl,
181 type: 'activity' as 'activity'
181 } 182 }
182 183
183 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) 184 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
@@ -248,6 +249,7 @@ function saveActorAndServerAndModelIfNotExist (
248 } else if (actorCreated.type === 'Group') { // Video channel 249 } else if (actorCreated.type === 'Group') { // Video channel
249 actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t) 250 actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
250 actorCreated.VideoChannel.Actor = actorCreated 251 actorCreated.VideoChannel.Actor = actorCreated
252 actorCreated.VideoChannel.Account = ownerActor.Account
251 } 253 }
252 254
253 return actorCreated 255 return actorCreated
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts
index d4fc786f7..55912341c 100644
--- a/server/lib/activitypub/crawl.ts
+++ b/server/lib/activitypub/crawl.ts
@@ -1,8 +1,9 @@
1import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers' 1import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers'
2import { doRequest } from '../../helpers/requests' 2import { doRequest } from '../../helpers/requests'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import Bluebird = require('bluebird')
4 5
5async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any>) { 6async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any> | Bluebird<any>) {
6 logger.info('Crawling ActivityPub data on %s.', uri) 7 logger.info('Crawling ActivityPub data on %s.', uri)
7 8
8 const options = { 9 const options = {
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts
index d8ca59425..b08156aa1 100644
--- a/server/lib/activitypub/process/process-announce.ts
+++ b/server/lib/activitypub/process/process-announce.ts
@@ -24,10 +24,8 @@ export {
24 24
25async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { 25async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
26 const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id 26 const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id
27 let video: VideoModel
28 27
29 const res = await getOrCreateAccountAndVideoAndChannel(objectUri) 28 const { video } = await getOrCreateAccountAndVideoAndChannel(objectUri)
30 video = res.video
31 29
32 return sequelizeTypescript.transaction(async t => { 30 return sequelizeTypescript.transaction(async t => {
33 // Add share entry 31 // Add share entry
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 791148919..9655d015f 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -23,7 +23,7 @@ async function processCreateActivity (activity: ActivityCreate) {
23 } else if (activityType === 'Dislike') { 23 } else if (activityType === 'Dislike') {
24 return retryTransactionWrapper(processCreateDislike, actor, activity) 24 return retryTransactionWrapper(processCreateDislike, actor, activity)
25 } else if (activityType === 'Video') { 25 } else if (activityType === 'Video') {
26 return processCreateVideo(actor, activity) 26 return processCreateVideo(activity)
27 } else if (activityType === 'Flag') { 27 } else if (activityType === 'Flag') {
28 return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject) 28 return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject)
29 } else if (activityType === 'Note') { 29 } else if (activityType === 'Note') {
@@ -42,13 +42,10 @@ export {
42 42
43// --------------------------------------------------------------------------- 43// ---------------------------------------------------------------------------
44 44
45async function processCreateVideo ( 45async function processCreateVideo (activity: ActivityCreate) {
46 actor: ActorModel,
47 activity: ActivityCreate
48) {
49 const videoToCreateData = activity.object as VideoTorrentObject 46 const videoToCreateData = activity.object as VideoTorrentObject
50 47
51 const { video } = await getOrCreateAccountAndVideoAndChannel(videoToCreateData, actor) 48 const { video } = await getOrCreateAccountAndVideoAndChannel(videoToCreateData)
52 49
53 return video 50 return video
54} 51}
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index fd03710c2..14c7fde69 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -2,12 +2,13 @@ import { VideoCommentObject } from '../../../shared/models/activitypub/objects/v
2import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments' 2import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { doRequest } from '../../helpers/requests' 4import { doRequest } from '../../helpers/requests'
5import { ACTIVITY_PUB } from '../../initializers' 5import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
6import { ActorModel } from '../../models/activitypub/actor' 6import { ActorModel } from '../../models/activitypub/actor'
7import { VideoModel } from '../../models/video/video' 7import { VideoModel } from '../../models/video/video'
8import { VideoCommentModel } from '../../models/video/video-comment' 8import { VideoCommentModel } from '../../models/video/video-comment'
9import { getOrCreateActorAndServerAndModel } from './actor' 9import { getOrCreateActorAndServerAndModel } from './actor'
10import { getOrCreateAccountAndVideoAndChannel } from './videos' 10import { getOrCreateAccountAndVideoAndChannel } from './videos'
11import * as Bluebird from 'bluebird'
11 12
12async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { 13async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) {
13 let originCommentId: number = null 14 let originCommentId: number = null
@@ -38,9 +39,9 @@ async function videoCommentActivityObjectToDBAttributes (video: VideoModel, acto
38} 39}
39 40
40async function addVideoComments (commentUrls: string[], instance: VideoModel) { 41async function addVideoComments (commentUrls: string[], instance: VideoModel) {
41 for (const commentUrl of commentUrls) { 42 return Bluebird.map(commentUrls, commentUrl => {
42 await addVideoComment(instance, commentUrl) 43 return addVideoComment(instance, commentUrl)
43 } 44 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
44} 45}
45 46
46async function addVideoComment (videoInstance: VideoModel, commentUrl: string) { 47async function addVideoComment (videoInstance: VideoModel, commentUrl: string) {
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index d1888556c..fac1d3fc7 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -11,7 +11,7 @@ import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos
11import { retryTransactionWrapper } from '../../helpers/database-utils' 11import { retryTransactionWrapper } from '../../helpers/database-utils'
12import { logger } from '../../helpers/logger' 12import { logger } from '../../helpers/logger'
13import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' 13import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
14import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' 14import { ACTIVITY_PUB, CONFIG, CRAWL_REQUEST_CONCURRENCY, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
15import { AccountVideoRateModel } from '../../models/account/account-video-rate' 15import { AccountVideoRateModel } from '../../models/account/account-video-rate'
16import { ActorModel } from '../../models/activitypub/actor' 16import { ActorModel } from '../../models/activitypub/actor'
17import { TagModel } from '../../models/video/tag' 17import { TagModel } from '../../models/video/tag'
@@ -26,6 +26,8 @@ import { sendCreateVideo, sendUpdateVideo } from './send'
26import { shareVideoByServerAndChannel } from './index' 26import { shareVideoByServerAndChannel } from './index'
27import { isArray } from '../../helpers/custom-validators/misc' 27import { isArray } from '../../helpers/custom-validators/misc'
28import { VideoCaptionModel } from '../../models/video/video-caption' 28import { VideoCaptionModel } from '../../models/video/video-caption'
29import { JobQueue } from '../job-queue'
30import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
29 31
30async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 32async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
31 // If the video is not private and published, we federate it 33 // If the video is not private and published, we federate it
@@ -178,10 +180,10 @@ function getOrCreateVideoChannel (videoObject: VideoTorrentObject) {
178 return getOrCreateActorAndServerAndModel(channel.id) 180 return getOrCreateActorAndServerAndModel(channel.id)
179} 181}
180 182
181async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel) { 183async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
182 logger.debug('Adding remote video %s.', videoObject.id) 184 logger.debug('Adding remote video %s.', videoObject.id)
183 185
184 return sequelizeTypescript.transaction(async t => { 186 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
185 const sequelizeOptions = { 187 const sequelizeOptions = {
186 transaction: t 188 transaction: t
187 } 189 }
@@ -191,10 +193,6 @@ async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor:
191 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) 193 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
192 const video = VideoModel.build(videoData) 194 const video = VideoModel.build(videoData)
193 195
194 // Don't block on remote HTTP request (we are in a transaction!)
195 generateThumbnailFromUrl(video, videoObject.icon)
196 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
197
198 const videoCreated = await video.save(sequelizeOptions) 196 const videoCreated = await video.save(sequelizeOptions)
199 197
200 // Process files 198 // Process files
@@ -222,68 +220,100 @@ async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor:
222 videoCreated.VideoChannel = channelActor.VideoChannel 220 videoCreated.VideoChannel = channelActor.VideoChannel
223 return videoCreated 221 return videoCreated
224 }) 222 })
223
224 const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
225 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
226
227 if (waitThumbnail === true) await p
228
229 return videoCreated
225} 230}
226 231
227async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentObject | string, actor?: ActorModel) { 232type SyncParam = {
233 likes: boolean,
234 dislikes: boolean,
235 shares: boolean,
236 comments: boolean,
237 thumbnail: boolean
238}
239async function getOrCreateAccountAndVideoAndChannel (
240 videoObject: VideoTorrentObject | string,
241 syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true }
242) {
228 const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id 243 const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
229 244
230 const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) 245 const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
231 if (videoFromDatabase) { 246 if (videoFromDatabase) return { video: videoFromDatabase }
232 return {
233 video: videoFromDatabase,
234 actor: videoFromDatabase.VideoChannel.Account.Actor,
235 channelActor: videoFromDatabase.VideoChannel.Actor
236 }
237 }
238 247
239 videoObject = await fetchRemoteVideo(videoUrl) 248 const fetchedVideo = await fetchRemoteVideo(videoUrl)
240 if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) 249 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
241 250
242 if (!actor) { 251 const channelActor = await getOrCreateVideoChannel(fetchedVideo)
243 const actorObj = videoObject.attributedTo.find(a => a.type === 'Person') 252 const video = await retryTransactionWrapper(getOrCreateVideo, fetchedVideo, channelActor, syncParam.thumbnail)
244 if (!actorObj) throw new Error('Cannot find associated actor to video ' + videoObject.url)
245 253
246 actor = await getOrCreateActorAndServerAndModel(actorObj.id) 254 // Process outside the transaction because we could fetch remote data
247 }
248 255
249 const channelActor = await getOrCreateVideoChannel(videoObject) 256 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
250 257
251 const video = await retryTransactionWrapper(getOrCreateVideo, videoObject, channelActor) 258 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
252 259
253 // Process outside the transaction because we could fetch remote data 260 if (syncParam.likes === true) {
254 logger.info('Adding likes of video %s.', video.uuid) 261 await crawlCollectionPage<string>(fetchedVideo.likes, items => createRates(items, video, 'like'))
255 await crawlCollectionPage<string>(videoObject.likes, (items) => createRates(items, video, 'like')) 262 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
263 } else {
264 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
265 }
256 266
257 logger.info('Adding dislikes of video %s.', video.uuid) 267 if (syncParam.dislikes === true) {
258 await crawlCollectionPage<string>(videoObject.dislikes, (items) => createRates(items, video, 'dislike')) 268 await crawlCollectionPage<string>(fetchedVideo.dislikes, items => createRates(items, video, 'dislike'))
269 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
270 } else {
271 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
272 }
273
274 if (syncParam.shares === true) {
275 await crawlCollectionPage<string>(fetchedVideo.shares, items => addVideoShares(items, video))
276 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
277 } else {
278 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
279 }
259 280
260 logger.info('Adding shares of video %s.', video.uuid) 281 if (syncParam.comments === true) {
261 await crawlCollectionPage<string>(videoObject.shares, (items) => addVideoShares(items, video)) 282 await crawlCollectionPage<string>(fetchedVideo.comments, items => addVideoComments(items, video))
283 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
284 } else {
285 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
286 }
262 287
263 logger.info('Adding comments of video %s.', video.uuid) 288 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
264 await crawlCollectionPage<string>(videoObject.comments, (items) => addVideoComments(items, video))
265 289
266 return { actor, channelActor, video } 290 return { video }
267} 291}
268 292
269async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { 293async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) {
270 let rateCounts = 0 294 let rateCounts = 0
271 const tasks: Bluebird<number>[] = []
272
273 for (const actorUrl of actorUrls) {
274 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
275 const p = AccountVideoRateModel
276 .create({
277 videoId: video.id,
278 accountId: actor.Account.id,
279 type: rate
280 })
281 .then(() => rateCounts += 1)
282
283 tasks.push(p)
284 }
285 295
286 await Promise.all(tasks) 296 await Bluebird.map(actorUrls, async actorUrl => {
297 try {
298 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
299 const [ , created ] = await AccountVideoRateModel
300 .findOrCreate({
301 where: {
302 videoId: video.id,
303 accountId: actor.Account.id
304 },
305 defaults: {
306 videoId: video.id,
307 accountId: actor.Account.id,
308 type: rate
309 }
310 })
311
312 if (created) rateCounts += 1
313 } catch (err) {
314 logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err })
315 }
316 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
287 317
288 logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid) 318 logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid)
289 319
@@ -294,34 +324,35 @@ async function createRates (actorUrls: string[], video: VideoModel, rate: VideoR
294} 324}
295 325
296async function addVideoShares (shareUrls: string[], instance: VideoModel) { 326async function addVideoShares (shareUrls: string[], instance: VideoModel) {
297 for (const shareUrl of shareUrls) { 327 await Bluebird.map(shareUrls, async shareUrl => {
298 // Fetch url 328 try {
299 const { body } = await doRequest({ 329 // Fetch url
300 uri: shareUrl, 330 const { body } = await doRequest({
301 json: true, 331 uri: shareUrl,
302 activityPub: true 332 json: true,
303 }) 333 activityPub: true
304 if (!body || !body.actor) { 334 })
305 logger.warn('Cannot add remote share with url: %s, skipping...', shareUrl) 335 if (!body || !body.actor) throw new Error('Body of body actor is invalid')
306 continue
307 }
308
309 const actorUrl = body.actor
310 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
311 336
312 const entry = { 337 const actorUrl = body.actor
313 actorId: actor.id, 338 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
314 videoId: instance.id,
315 url: shareUrl
316 }
317 339
318 await VideoShareModel.findOrCreate({ 340 const entry = {
319 where: { 341 actorId: actor.id,
342 videoId: instance.id,
320 url: shareUrl 343 url: shareUrl
321 }, 344 }
322 defaults: entry 345
323 }) 346 await VideoShareModel.findOrCreate({
324 } 347 where: {
348 url: shareUrl
349 },
350 defaults: entry
351 })
352 } catch (err) {
353 logger.warn('Cannot add share %s.', shareUrl, { err })
354 }
355 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
325} 356}
326 357
327async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject> { 358async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject> {
@@ -355,5 +386,6 @@ export {
355 videoFileActivityUrlToDBAttributes, 386 videoFileActivityUrlToDBAttributes,
356 getOrCreateVideo, 387 getOrCreateVideo,
357 getOrCreateVideoChannel, 388 getOrCreateVideoChannel,
358 addVideoShares 389 addVideoShares,
390 createRates
359} 391}
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index f21da087e..72d670277 100644
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -1,22 +1,36 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger' 2import { logger } from '../../../helpers/logger'
3import { processActivities } from '../../activitypub/process' 3import { processActivities } from '../../activitypub/process'
4import { ActivitypubHttpBroadcastPayload } from './activitypub-http-broadcast' 4import { VideoModel } from '../../../models/video/video'
5import { addVideoShares, createRates } from '../../activitypub/videos'
6import { addVideoComments } from '../../activitypub/video-comments'
5import { crawlCollectionPage } from '../../activitypub/crawl' 7import { crawlCollectionPage } from '../../activitypub/crawl'
6import { Activity } from '../../../../shared/models/activitypub' 8
9type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments'
7 10
8export type ActivitypubHttpFetcherPayload = { 11export type ActivitypubHttpFetcherPayload = {
9 uris: string[] 12 uri: string
13 type: FetchType
14 videoId?: number
10} 15}
11 16
12async function processActivityPubHttpFetcher (job: Bull.Job) { 17async function processActivityPubHttpFetcher (job: Bull.Job) {
13 logger.info('Processing ActivityPub fetcher in job %d.', job.id) 18 logger.info('Processing ActivityPub fetcher in job %d.', job.id)
14 19
15 const payload = job.data as ActivitypubHttpBroadcastPayload 20 const payload = job.data as ActivitypubHttpFetcherPayload
21
22 let video: VideoModel
23 if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId)
16 24
17 for (const uri of payload.uris) { 25 const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = {
18 await crawlCollectionPage<Activity>(uri, (items) => processActivities(items)) 26 'activity': items => processActivities(items),
27 'video-likes': items => createRates(items, video, 'like'),
28 'video-dislikes': items => createRates(items, video, 'dislike'),
29 'video-shares': items => addVideoShares(items, video),
30 'video-comments': items => addVideoComments(items, video)
19 } 31 }
32
33 return crawlCollectionPage(payload.uri, fetcherType[payload.type])
20} 34}
21 35
22// --------------------------------------------------------------------------- 36// ---------------------------------------------------------------------------
diff --git a/server/tests/api/check-params/videos.ts b/server/tests/api/check-params/videos.ts
index 7fce8ba7c..904d22870 100644
--- a/server/tests/api/check-params/videos.ts
+++ b/server/tests/api/check-params/videos.ts
@@ -20,7 +20,7 @@ describe('Test videos API validator', function () {
20 let userAccessToken = '' 20 let userAccessToken = ''
21 let accountName: string 21 let accountName: string
22 let channelId: number 22 let channelId: number
23 let channelUUID: string 23 let channelName: string
24 let videoId 24 let videoId
25 25
26 // --------------------------------------------------------------- 26 // ---------------------------------------------------------------
@@ -42,7 +42,7 @@ describe('Test videos API validator', function () {
42 { 42 {
43 const res = await getMyUserInformation(server.url, server.accessToken) 43 const res = await getMyUserInformation(server.url, server.accessToken)
44 channelId = res.body.videoChannels[ 0 ].id 44 channelId = res.body.videoChannels[ 0 ].id
45 channelUUID = res.body.videoChannels[ 0 ].uuid 45 channelName = res.body.videoChannels[ 0 ].name
46 accountName = res.body.account.name + '@' + res.body.account.host 46 accountName = res.body.account.name + '@' + res.body.account.host
47 } 47 }
48 }) 48 })
@@ -140,7 +140,7 @@ describe('Test videos API validator', function () {
140 let path: string 140 let path: string
141 141
142 before(async function () { 142 before(async function () {
143 path = '/api/v1/video-channels/' + channelUUID + '/videos' 143 path = '/api/v1/video-channels/' + channelName + '/videos'
144 }) 144 })
145 145
146 it('Should fail with a bad start pagination', async function () { 146 it('Should fail with a bad start pagination', async function () {
diff --git a/server/tests/api/server/follows.ts b/server/tests/api/server/follows.ts
index 243fcd4e7..310c291bf 100644
--- a/server/tests/api/server/follows.ts
+++ b/server/tests/api/server/follows.ts
@@ -311,7 +311,8 @@ describe('Test follows', function () {
311 likes: 1, 311 likes: 1,
312 dislikes: 1, 312 dislikes: 1,
313 channel: { 313 channel: {
314 name: 'Main root channel', 314 displayName: 'Main root channel',
315 name: 'root_channel',
315 description: '', 316 description: '',
316 isLocal 317 isLocal
317 }, 318 },
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts
index df35b36eb..ed15c8090 100644
--- a/server/tests/api/server/handle-down.ts
+++ b/server/tests/api/server/handle-down.ts
@@ -71,7 +71,8 @@ describe('Test handle downs', function () {
71 privacy: VideoPrivacy.PUBLIC, 71 privacy: VideoPrivacy.PUBLIC,
72 commentsEnabled: true, 72 commentsEnabled: true,
73 channel: { 73 channel: {
74 name: 'Main root channel', 74 name: 'root_channel',
75 displayName: 'Main root channel',
75 description: '', 76 description: '',
76 isLocal: false 77 isLocal: false
77 }, 78 },
diff --git a/server/tests/api/users/user-subscriptions.ts b/server/tests/api/users/user-subscriptions.ts
index 2fbda6828..cb7d94b0b 100644
--- a/server/tests/api/users/user-subscriptions.ts
+++ b/server/tests/api/users/user-subscriptions.ts
@@ -2,7 +2,7 @@
2 2
3import * as chai from 'chai' 3import * as chai from 'chai'
4import 'mocha' 4import 'mocha'
5import { createUser, doubleFollow, flushAndRunMultipleServers, follow, getVideosList, unfollow, userLogin } from '../../utils' 5import { createUser, doubleFollow, flushAndRunMultipleServers, follow, getVideosList, unfollow, updateVideo, userLogin } from '../../utils'
6import { killallServers, ServerInfo, uploadVideo } from '../../utils/index' 6import { killallServers, ServerInfo, uploadVideo } from '../../utils/index'
7import { setAccessTokensToServers } from '../../utils/users/login' 7import { setAccessTokensToServers } from '../../utils/users/login'
8import { Video, VideoChannel } from '../../../../shared/models/videos' 8import { Video, VideoChannel } from '../../../../shared/models/videos'
@@ -20,6 +20,7 @@ const expect = chai.expect
20describe('Test users subscriptions', function () { 20describe('Test users subscriptions', function () {
21 let servers: ServerInfo[] = [] 21 let servers: ServerInfo[] = []
22 const users: { accessToken: string }[] = [] 22 const users: { accessToken: string }[] = []
23 let video3UUID: string
23 24
24 before(async function () { 25 before(async function () {
25 this.timeout(120000) 26 this.timeout(120000)
@@ -65,7 +66,8 @@ describe('Test users subscriptions', function () {
65 66
66 await waitJobs(servers) 67 await waitJobs(servers)
67 68
68 await uploadVideo(servers[2].url, users[2].accessToken, { name: 'video server 3 added after follow' }) 69 const res = await uploadVideo(servers[2].url, users[2].accessToken, { name: 'video server 3 added after follow' })
70 video3UUID = res.body.video.uuid
69 71
70 await waitJobs(servers) 72 await waitJobs(servers)
71 }) 73 })
@@ -247,7 +249,21 @@ describe('Test users subscriptions', function () {
247 } 249 }
248 }) 250 })
249 251
252 it('Should update a video of server 3 and see the updated video on server 1', async function () {
253 this.timeout(30000)
254
255 await updateVideo(servers[2].url, users[2].accessToken, video3UUID, { name: 'video server 3 added after follow updated' })
256
257 await waitJobs(servers)
258
259 const res = await listUserSubscriptionVideos(servers[0].url, users[0].accessToken, 'createdAt')
260 const videos: Video[] = res.body.data
261 expect(videos[2].name).to.equal('video server 3 added after follow updated')
262 })
263
250 it('Should remove user of server 3 subscription', async function () { 264 it('Should remove user of server 3 subscription', async function () {
265 this.timeout(30000)
266
251 await removeUserSubscription(servers[0].url, users[0].accessToken, 'user3_channel@localhost:9003') 267 await removeUserSubscription(servers[0].url, users[0].accessToken, 'user3_channel@localhost:9003')
252 268
253 await waitJobs(servers) 269 await waitJobs(servers)
@@ -267,6 +283,8 @@ describe('Test users subscriptions', function () {
267 }) 283 })
268 284
269 it('Should remove the root subscription and not display the videos anymore', async function () { 285 it('Should remove the root subscription and not display the videos anymore', async function () {
286 this.timeout(30000)
287
270 await removeUserSubscription(servers[0].url, users[0].accessToken, 'root_channel@localhost:9001') 288 await removeUserSubscription(servers[0].url, users[0].accessToken, 'root_channel@localhost:9001')
271 289
272 await waitJobs(servers) 290 await waitJobs(servers)
@@ -288,7 +306,7 @@ describe('Test users subscriptions', function () {
288 for (const video of res.body.data) { 306 for (const video of res.body.data) {
289 expect(video.name).to.not.contain('1-3') 307 expect(video.name).to.not.contain('1-3')
290 expect(video.name).to.not.contain('2-3') 308 expect(video.name).to.not.contain('2-3')
291 expect(video.name).to.not.contain('video server 3 added after follow') 309 expect(video.name).to.not.contain('video server 3 added after follow updated')
292 } 310 }
293 }) 311 })
294 312
@@ -309,7 +327,7 @@ describe('Test users subscriptions', function () {
309 327
310 expect(videos[0].name).to.equal('video 1-3') 328 expect(videos[0].name).to.equal('video 1-3')
311 expect(videos[1].name).to.equal('video 2-3') 329 expect(videos[1].name).to.equal('video 2-3')
312 expect(videos[2].name).to.equal('video server 3 added after follow') 330 expect(videos[2].name).to.equal('video server 3 added after follow updated')
313 } 331 }
314 332
315 { 333 {
@@ -319,7 +337,7 @@ describe('Test users subscriptions', function () {
319 for (const video of res.body.data) { 337 for (const video of res.body.data) {
320 expect(video.name).to.not.contain('1-3') 338 expect(video.name).to.not.contain('1-3')
321 expect(video.name).to.not.contain('2-3') 339 expect(video.name).to.not.contain('2-3')
322 expect(video.name).to.not.contain('video server 3 added after follow') 340 expect(video.name).to.not.contain('video server 3 added after follow updated')
323 } 341 }
324 } 342 }
325 }) 343 })
diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts
index 3c3839338..c551ccc59 100644
--- a/server/tests/api/videos/multiple-servers.ts
+++ b/server/tests/api/videos/multiple-servers.ts
@@ -128,7 +128,8 @@ describe('Test multiple servers', function () {
128 privacy: VideoPrivacy.PUBLIC, 128 privacy: VideoPrivacy.PUBLIC,
129 commentsEnabled: true, 129 commentsEnabled: true,
130 channel: { 130 channel: {
131 name: 'my channel', 131 displayName: 'my channel',
132 name: 'super_channel_name',
132 description: 'super channel', 133 description: 'super channel',
133 isLocal 134 isLocal
134 }, 135 },
@@ -201,7 +202,8 @@ describe('Test multiple servers', function () {
201 tags: [ 'tag1p2', 'tag2p2', 'tag3p2' ], 202 tags: [ 'tag1p2', 'tag2p2', 'tag3p2' ],
202 privacy: VideoPrivacy.PUBLIC, 203 privacy: VideoPrivacy.PUBLIC,
203 channel: { 204 channel: {
204 name: 'Main user1 channel', 205 displayName: 'Main user1 channel',
206 name: 'user1_channel',
205 description: 'super channel', 207 description: 'super channel',
206 isLocal 208 isLocal
207 }, 209 },
@@ -307,7 +309,8 @@ describe('Test multiple servers', function () {
307 tags: [ 'tag1p3' ], 309 tags: [ 'tag1p3' ],
308 privacy: VideoPrivacy.PUBLIC, 310 privacy: VideoPrivacy.PUBLIC,
309 channel: { 311 channel: {
310 name: 'Main root channel', 312 displayName: 'Main root channel',
313 name: 'root_channel',
311 description: '', 314 description: '',
312 isLocal 315 isLocal
313 }, 316 },
@@ -339,7 +342,8 @@ describe('Test multiple servers', function () {
339 tags: [ 'tag2p3', 'tag3p3', 'tag4p3' ], 342 tags: [ 'tag2p3', 'tag3p3', 'tag4p3' ],
340 privacy: VideoPrivacy.PUBLIC, 343 privacy: VideoPrivacy.PUBLIC,
341 channel: { 344 channel: {
342 name: 'Main root channel', 345 displayName: 'Main root channel',
346 name: 'root_channel',
343 description: '', 347 description: '',
344 isLocal 348 isLocal
345 }, 349 },
@@ -647,7 +651,8 @@ describe('Test multiple servers', function () {
647 tags: [ 'tag_up_1', 'tag_up_2' ], 651 tags: [ 'tag_up_1', 'tag_up_2' ],
648 privacy: VideoPrivacy.PUBLIC, 652 privacy: VideoPrivacy.PUBLIC,
649 channel: { 653 channel: {
650 name: 'Main root channel', 654 displayName: 'Main root channel',
655 name: 'root_channel',
651 description: '', 656 description: '',
652 isLocal 657 isLocal
653 }, 658 },
@@ -967,7 +972,8 @@ describe('Test multiple servers', function () {
967 tags: [ ], 972 tags: [ ],
968 privacy: VideoPrivacy.PUBLIC, 973 privacy: VideoPrivacy.PUBLIC,
969 channel: { 974 channel: {
970 name: 'Main root channel', 975 displayName: 'Main root channel',
976 name: 'root_channel',
971 description: '', 977 description: '',
972 isLocal 978 isLocal
973 }, 979 },
diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts
index 12181ad67..a757ad9da 100644
--- a/server/tests/api/videos/single-server.ts
+++ b/server/tests/api/videos/single-server.ts
@@ -56,7 +56,8 @@ describe('Test a single server', function () {
56 privacy: VideoPrivacy.PUBLIC, 56 privacy: VideoPrivacy.PUBLIC,
57 commentsEnabled: true, 57 commentsEnabled: true,
58 channel: { 58 channel: {
59 name: 'Main root channel', 59 displayName: 'Main root channel',
60 name: 'root_channel',
60 description: '', 61 description: '',
61 isLocal: true 62 isLocal: true
62 }, 63 },
@@ -87,7 +88,8 @@ describe('Test a single server', function () {
87 duration: 5, 88 duration: 5,
88 commentsEnabled: false, 89 commentsEnabled: false,
89 channel: { 90 channel: {
90 name: 'Main root channel', 91 name: 'root_channel',
92 displayName: 'Main root channel',
91 description: '', 93 description: '',
92 isLocal: true 94 isLocal: true
93 }, 95 },
diff --git a/server/tests/utils/videos/videos.ts b/server/tests/utils/videos/videos.ts
index 592248144..674a92df9 100644
--- a/server/tests/utils/videos/videos.ts
+++ b/server/tests/utils/videos/videos.ts
@@ -438,18 +438,19 @@ async function completeVideoCheck (
438 name: string 438 name: string
439 host: string 439 host: string
440 } 440 }
441 isLocal: boolean, 441 isLocal: boolean
442 tags: string[], 442 tags: string[]
443 privacy: number, 443 privacy: number
444 likes?: number, 444 likes?: number
445 dislikes?: number, 445 dislikes?: number
446 duration: number, 446 duration: number
447 channel: { 447 channel: {
448 name: string, 448 displayName: string
449 name: string
449 description 450 description
450 isLocal: boolean 451 isLocal: boolean
451 } 452 }
452 fixture: string, 453 fixture: string
453 files: { 454 files: {
454 resolution: number 455 resolution: number
455 size: number 456 size: number
@@ -476,8 +477,8 @@ async function completeVideoCheck (
476 expect(video.account.uuid).to.be.a('string') 477 expect(video.account.uuid).to.be.a('string')
477 expect(video.account.host).to.equal(attributes.account.host) 478 expect(video.account.host).to.equal(attributes.account.host)
478 expect(video.account.name).to.equal(attributes.account.name) 479 expect(video.account.name).to.equal(attributes.account.name)
479 expect(video.channel.displayName).to.equal(attributes.channel.name) 480 expect(video.channel.displayName).to.equal(attributes.channel.displayName)
480 expect(video.channel.name).to.have.lengthOf(36) 481 expect(video.channel.name).to.equal(attributes.channel.name)
481 expect(video.likes).to.equal(attributes.likes) 482 expect(video.likes).to.equal(attributes.likes)
482 expect(video.dislikes).to.equal(attributes.dislikes) 483 expect(video.dislikes).to.equal(attributes.dislikes)
483 expect(video.isLocal).to.equal(attributes.isLocal) 484 expect(video.isLocal).to.equal(attributes.isLocal)
@@ -497,8 +498,8 @@ async function completeVideoCheck (
497 expect(videoDetails.tags).to.deep.equal(attributes.tags) 498 expect(videoDetails.tags).to.deep.equal(attributes.tags)
498 expect(videoDetails.account.name).to.equal(attributes.account.name) 499 expect(videoDetails.account.name).to.equal(attributes.account.name)
499 expect(videoDetails.account.host).to.equal(attributes.account.host) 500 expect(videoDetails.account.host).to.equal(attributes.account.host)
500 expect(videoDetails.channel.displayName).to.equal(attributes.channel.name) 501 expect(video.channel.displayName).to.equal(attributes.channel.displayName)
501 expect(videoDetails.channel.name).to.have.lengthOf(36) 502 expect(video.channel.name).to.equal(attributes.channel.name)
502 expect(videoDetails.channel.host).to.equal(attributes.account.host) 503 expect(videoDetails.channel.host).to.equal(attributes.account.host)
503 expect(videoDetails.channel.isLocal).to.equal(attributes.channel.isLocal) 504 expect(videoDetails.channel.isLocal).to.equal(attributes.channel.isLocal)
504 expect(dateIsValid(videoDetails.channel.createdAt.toString())).to.be.true 505 expect(dateIsValid(videoDetails.channel.createdAt.toString())).to.be.true