aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-08-22 16:15:35 +0200
committerChocobozzz <me@florianbigard.com>2018-08-27 09:41:54 +0200
commit1297eb5db651a230474670c5da1517862fb9cc3e (patch)
treeecad4a0ceb0bb09e3c775262691ac68e9e0aca0c /server/lib
parentf6eebcb336c067e160a62020a5140d8d992ba384 (diff)
downloadPeerTube-1297eb5db651a230474670c5da1517862fb9cc3e.tar.gz
PeerTube-1297eb5db651a230474670c5da1517862fb9cc3e.tar.zst
PeerTube-1297eb5db651a230474670c5da1517862fb9cc3e.zip
Add refresh video on search
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/process/process-announce.ts4
-rw-r--r--server/lib/activitypub/process/process-create.ts10
-rw-r--r--server/lib/activitypub/process/process-like.ts4
-rw-r--r--server/lib/activitypub/process/process-undo.ts6
-rw-r--r--server/lib/activitypub/process/process-update.ts99
-rw-r--r--server/lib/activitypub/share.ts38
-rw-r--r--server/lib/activitypub/video-comments.ts4
-rw-r--r--server/lib/activitypub/video-rates.ts40
-rw-r--r--server/lib/activitypub/videos.ts241
9 files changed, 244 insertions, 202 deletions
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts
index b08156aa1..814556817 100644
--- a/server/lib/activitypub/process/process-announce.ts
+++ b/server/lib/activitypub/process/process-announce.ts
@@ -6,7 +6,7 @@ import { VideoModel } from '../../../models/video/video'
6import { VideoShareModel } from '../../../models/video/video-share' 6import { VideoShareModel } from '../../../models/video/video-share'
7import { getOrCreateActorAndServerAndModel } from '../actor' 7import { getOrCreateActorAndServerAndModel } from '../actor'
8import { forwardVideoRelatedActivity } from '../send/utils' 8import { forwardVideoRelatedActivity } from '../send/utils'
9import { getOrCreateAccountAndVideoAndChannel } from '../videos' 9import { getOrCreateVideoAndAccountAndChannel } from '../videos'
10 10
11async function processAnnounceActivity (activity: ActivityAnnounce) { 11async function processAnnounceActivity (activity: ActivityAnnounce) {
12 const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor) 12 const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor)
@@ -25,7 +25,7 @@ export {
25async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { 25async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
26 const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id 26 const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id
27 27
28 const { video } = await getOrCreateAccountAndVideoAndChannel(objectUri) 28 const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri)
29 29
30 return sequelizeTypescript.transaction(async t => { 30 return sequelizeTypescript.transaction(async t => {
31 // Add share entry 31 // Add share entry
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 9655d015f..e8f5ade06 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -10,7 +10,7 @@ import { VideoAbuseModel } from '../../../models/video/video-abuse'
10import { VideoCommentModel } from '../../../models/video/video-comment' 10import { VideoCommentModel } from '../../../models/video/video-comment'
11import { getOrCreateActorAndServerAndModel } from '../actor' 11import { getOrCreateActorAndServerAndModel } from '../actor'
12import { resolveThread } from '../video-comments' 12import { resolveThread } from '../video-comments'
13import { getOrCreateAccountAndVideoAndChannel } from '../videos' 13import { getOrCreateVideoAndAccountAndChannel } from '../videos'
14import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' 14import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils'
15 15
16async function processCreateActivity (activity: ActivityCreate) { 16async function processCreateActivity (activity: ActivityCreate) {
@@ -45,7 +45,7 @@ export {
45async function processCreateVideo (activity: ActivityCreate) { 45async function processCreateVideo (activity: ActivityCreate) {
46 const videoToCreateData = activity.object as VideoTorrentObject 46 const videoToCreateData = activity.object as VideoTorrentObject
47 47
48 const { video } = await getOrCreateAccountAndVideoAndChannel(videoToCreateData) 48 const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData)
49 49
50 return video 50 return video
51} 51}
@@ -56,7 +56,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
56 56
57 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) 57 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
58 58
59 const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object) 59 const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object)
60 60
61 return sequelizeTypescript.transaction(async t => { 61 return sequelizeTypescript.transaction(async t => {
62 const rate = { 62 const rate = {
@@ -83,7 +83,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
83async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { 83async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
84 const view = activity.object as ViewObject 84 const view = activity.object as ViewObject
85 85
86 const { video } = await getOrCreateAccountAndVideoAndChannel(view.object) 86 const { video } = await getOrCreateVideoAndAccountAndChannel(view.object)
87 87
88 const actor = await ActorModel.loadByUrl(view.actor) 88 const actor = await ActorModel.loadByUrl(view.actor)
89 if (!actor) throw new Error('Unknown actor ' + view.actor) 89 if (!actor) throw new Error('Unknown actor ' + view.actor)
@@ -103,7 +103,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat
103 const account = actor.Account 103 const account = actor.Account
104 if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) 104 if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url)
105 105
106 const { video } = await getOrCreateAccountAndVideoAndChannel(videoAbuseToCreateData.object) 106 const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object)
107 107
108 return sequelizeTypescript.transaction(async t => { 108 return sequelizeTypescript.transaction(async t => {
109 const videoAbuseData = { 109 const videoAbuseData = {
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts
index d0865b78c..9e1664fd8 100644
--- a/server/lib/activitypub/process/process-like.ts
+++ b/server/lib/activitypub/process/process-like.ts
@@ -5,7 +5,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/activitypub/actor'
6import { getOrCreateActorAndServerAndModel } from '../actor' 6import { getOrCreateActorAndServerAndModel } from '../actor'
7import { forwardVideoRelatedActivity } from '../send/utils' 7import { forwardVideoRelatedActivity } from '../send/utils'
8import { getOrCreateAccountAndVideoAndChannel } from '../videos' 8import { getOrCreateVideoAndAccountAndChannel } from '../videos'
9 9
10async function processLikeActivity (activity: ActivityLike) { 10async function processLikeActivity (activity: ActivityLike) {
11 const actor = await getOrCreateActorAndServerAndModel(activity.actor) 11 const actor = await getOrCreateActorAndServerAndModel(activity.actor)
@@ -27,7 +27,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) {
27 const byAccount = byActor.Account 27 const byAccount = byActor.Account
28 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) 28 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
29 29
30 const { video } = await getOrCreateAccountAndVideoAndChannel(videoUrl) 30 const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl)
31 31
32 return sequelizeTypescript.transaction(async t => { 32 return sequelizeTypescript.transaction(async t => {
33 const rate = { 33 const rate = {
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index b6de107ad..eab9e3d61 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -9,7 +9,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
9import { ActorModel } from '../../../models/activitypub/actor' 9import { ActorModel } from '../../../models/activitypub/actor'
10import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 10import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
11import { forwardVideoRelatedActivity } from '../send/utils' 11import { forwardVideoRelatedActivity } from '../send/utils'
12import { getOrCreateAccountAndVideoAndChannel } from '../videos' 12import { getOrCreateVideoAndAccountAndChannel } from '../videos'
13import { VideoShareModel } from '../../../models/video/video-share' 13import { VideoShareModel } from '../../../models/video/video-share'
14 14
15async function processUndoActivity (activity: ActivityUndo) { 15async function processUndoActivity (activity: ActivityUndo) {
@@ -43,7 +43,7 @@ export {
43async function processUndoLike (actorUrl: string, activity: ActivityUndo) { 43async function processUndoLike (actorUrl: string, activity: ActivityUndo) {
44 const likeActivity = activity.object as ActivityLike 44 const likeActivity = activity.object as ActivityLike
45 45
46 const { video } = await getOrCreateAccountAndVideoAndChannel(likeActivity.object) 46 const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object)
47 47
48 return sequelizeTypescript.transaction(async t => { 48 return sequelizeTypescript.transaction(async t => {
49 const byAccount = await AccountModel.loadByUrl(actorUrl, t) 49 const byAccount = await AccountModel.loadByUrl(actorUrl, t)
@@ -67,7 +67,7 @@ async function processUndoLike (actorUrl: string, activity: ActivityUndo) {
67async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { 67async function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
68 const dislike = activity.object.object as DislikeObject 68 const dislike = activity.object.object as DislikeObject
69 69
70 const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object) 70 const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object)
71 71
72 return sequelizeTypescript.transaction(async t => { 72 return sequelizeTypescript.transaction(async t => {
73 const byAccount = await AccountModel.loadByUrl(actorUrl, t) 73 const byAccount = await AccountModel.loadByUrl(actorUrl, t)
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index 11226e275..07a5ff92f 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -1,4 +1,3 @@
1import * as Bluebird from 'bluebird'
2import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub' 1import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub'
3import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' 2import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor'
4import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' 3import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
@@ -6,19 +5,10 @@ import { logger } from '../../../helpers/logger'
6import { sequelizeTypescript } from '../../../initializers' 5import { sequelizeTypescript } from '../../../initializers'
7import { AccountModel } from '../../../models/account/account' 6import { AccountModel } from '../../../models/account/account'
8import { ActorModel } from '../../../models/activitypub/actor' 7import { ActorModel } from '../../../models/activitypub/actor'
9import { TagModel } from '../../../models/video/tag'
10import { VideoChannelModel } from '../../../models/video/video-channel' 8import { VideoChannelModel } from '../../../models/video/video-channel'
11import { VideoFileModel } from '../../../models/video/video-file'
12import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' 9import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor'
13import { 10import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannel, updateVideoFromAP } from '../videos'
14 generateThumbnailFromUrl,
15 getOrCreateAccountAndVideoAndChannel,
16 getOrCreateVideoChannel,
17 videoActivityObjectToDBAttributes,
18 videoFileActivityUrlToDBAttributes
19} from '../videos'
20import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' 11import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
21import { VideoCaptionModel } from '../../../models/video/video-caption'
22 12
23async function processUpdateActivity (activity: ActivityUpdate) { 13async function processUpdateActivity (activity: ActivityUpdate) {
24 const actor = await getOrCreateActorAndServerAndModel(activity.actor) 14 const actor = await getOrCreateActorAndServerAndModel(activity.actor)
@@ -49,91 +39,10 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
49 return undefined 39 return undefined
50 } 40 }
51 41
52 const res = await getOrCreateAccountAndVideoAndChannel(videoObject.id) 42 const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id)
43 const channelActor = await getOrCreateVideoChannel(videoObject)
53 44
54 // Fetch video channel outside the transaction 45 return updateVideoFromAP(video, videoObject, actor, channelActor, activity.to)
55 const newVideoChannelActor = await getOrCreateVideoChannel(videoObject)
56 const newVideoChannel = newVideoChannelActor.VideoChannel
57
58 logger.debug('Updating remote video "%s".', videoObject.uuid)
59 let videoInstance = res.video
60 let videoFieldsSave: any
61
62 try {
63 await sequelizeTypescript.transaction(async t => {
64 const sequelizeOptions = {
65 transaction: t
66 }
67
68 videoFieldsSave = videoInstance.toJSON()
69
70 // Check actor has the right to update the video
71 const videoChannel = videoInstance.VideoChannel
72 if (videoChannel.Account.Actor.id !== actor.id) {
73 throw new Error('Account ' + actor.url + ' does not own video channel ' + videoChannel.Actor.url)
74 }
75
76 const videoData = await videoActivityObjectToDBAttributes(newVideoChannel, videoObject, activity.to)
77 videoInstance.set('name', videoData.name)
78 videoInstance.set('uuid', videoData.uuid)
79 videoInstance.set('url', videoData.url)
80 videoInstance.set('category', videoData.category)
81 videoInstance.set('licence', videoData.licence)
82 videoInstance.set('language', videoData.language)
83 videoInstance.set('description', videoData.description)
84 videoInstance.set('support', videoData.support)
85 videoInstance.set('nsfw', videoData.nsfw)
86 videoInstance.set('commentsEnabled', videoData.commentsEnabled)
87 videoInstance.set('waitTranscoding', videoData.waitTranscoding)
88 videoInstance.set('state', videoData.state)
89 videoInstance.set('duration', videoData.duration)
90 videoInstance.set('createdAt', videoData.createdAt)
91 videoInstance.set('updatedAt', videoData.updatedAt)
92 videoInstance.set('views', videoData.views)
93 videoInstance.set('privacy', videoData.privacy)
94 videoInstance.set('channelId', videoData.channelId)
95
96 await videoInstance.save(sequelizeOptions)
97
98 // Don't block on request
99 generateThumbnailFromUrl(videoInstance, videoObject.icon)
100 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
101
102 // Remove old video files
103 const videoFileDestroyTasks: Bluebird<void>[] = []
104 for (const videoFile of videoInstance.VideoFiles) {
105 videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
106 }
107 await Promise.all(videoFileDestroyTasks)
108
109 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoInstance, videoObject)
110 const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
111 await Promise.all(tasks)
112
113 // Update Tags
114 const tags = videoObject.tag.map(tag => tag.name)
115 const tagInstances = await TagModel.findOrCreateTags(tags, t)
116 await videoInstance.$set('Tags', tagInstances, sequelizeOptions)
117
118 // Update captions
119 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoInstance.id, t)
120
121 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
122 return VideoCaptionModel.insertOrReplaceLanguage(videoInstance.id, c.identifier, t)
123 })
124 await Promise.all(videoCaptionsPromises)
125 })
126
127 logger.info('Remote video with uuid %s updated', videoObject.uuid)
128 } catch (err) {
129 if (videoInstance !== undefined && videoFieldsSave !== undefined) {
130 resetSequelizeInstance(videoInstance, videoFieldsSave)
131 }
132
133 // This is just a debug because we will retry the insert
134 logger.debug('Cannot update the remote video.', { err })
135 throw err
136 }
137} 46}
138 47
139async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { 48async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) {
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts
index 698414867..fe3d73e9b 100644
--- a/server/lib/activitypub/share.ts
+++ b/server/lib/activitypub/share.ts
@@ -6,6 +6,11 @@ import { VideoShareModel } from '../../models/video/video-share'
6import { sendUndoAnnounce, sendVideoAnnounce } from './send' 6import { sendUndoAnnounce, sendVideoAnnounce } from './send'
7import { getAnnounceActivityPubUrl } from './url' 7import { getAnnounceActivityPubUrl } from './url'
8import { VideoChannelModel } from '../../models/video/video-channel' 8import { VideoChannelModel } from '../../models/video/video-channel'
9import * as Bluebird from 'bluebird'
10import { doRequest } from '../../helpers/requests'
11import { getOrCreateActorAndServerAndModel } from './actor'
12import { logger } from '../../helpers/logger'
13import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
9 14
10async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { 15async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) {
11 if (video.privacy === VideoPrivacy.PRIVATE) return undefined 16 if (video.privacy === VideoPrivacy.PRIVATE) return undefined
@@ -22,8 +27,41 @@ async function changeVideoChannelShare (video: VideoModel, oldVideoChannel: Vide
22 await shareByVideoChannel(video, t) 27 await shareByVideoChannel(video, t)
23} 28}
24 29
30async function addVideoShares (shareUrls: string[], instance: VideoModel) {
31 await Bluebird.map(shareUrls, async shareUrl => {
32 try {
33 // Fetch url
34 const { body } = await doRequest({
35 uri: shareUrl,
36 json: true,
37 activityPub: true
38 })
39 if (!body || !body.actor) throw new Error('Body of body actor is invalid')
40
41 const actorUrl = body.actor
42 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
43
44 const entry = {
45 actorId: actor.id,
46 videoId: instance.id,
47 url: shareUrl
48 }
49
50 await VideoShareModel.findOrCreate({
51 where: {
52 url: shareUrl
53 },
54 defaults: entry
55 })
56 } catch (err) {
57 logger.warn('Cannot add share %s.', shareUrl, { err })
58 }
59 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
60}
61
25export { 62export {
26 changeVideoChannelShare, 63 changeVideoChannelShare,
64 addVideoShares,
27 shareVideoByServerAndChannel 65 shareVideoByServerAndChannel
28} 66}
29 67
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index 14c7fde69..beff557bc 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -7,7 +7,7 @@ import { ActorModel } from '../../models/activitypub/actor'
7import { VideoModel } from '../../models/video/video' 7import { VideoModel } from '../../models/video/video'
8import { VideoCommentModel } from '../../models/video/video-comment' 8import { VideoCommentModel } from '../../models/video/video-comment'
9import { getOrCreateActorAndServerAndModel } from './actor' 9import { getOrCreateActorAndServerAndModel } from './actor'
10import { getOrCreateAccountAndVideoAndChannel } from './videos' 10import { getOrCreateVideoAndAccountAndChannel } from './videos'
11import * as Bluebird from 'bluebird' 11import * as Bluebird from 'bluebird'
12 12
13async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { 13async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) {
@@ -91,7 +91,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) {
91 91
92 try { 92 try {
93 // Maybe it's a reply to a video? 93 // Maybe it's a reply to a video?
94 const { video } = await getOrCreateAccountAndVideoAndChannel(url) 94 const { video } = await getOrCreateVideoAndAccountAndChannel(url)
95 95
96 if (comments.length !== 0) { 96 if (comments.length !== 0) {
97 const firstReply = comments[ comments.length - 1 ] 97 const firstReply = comments[ comments.length - 1 ]
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts
index 19011b4ab..1619251c3 100644
--- a/server/lib/activitypub/video-rates.ts
+++ b/server/lib/activitypub/video-rates.ts
@@ -2,6 +2,45 @@ import { Transaction } from 'sequelize'
2import { AccountModel } from '../../models/account/account' 2import { AccountModel } from '../../models/account/account'
3import { VideoModel } from '../../models/video/video' 3import { VideoModel } from '../../models/video/video'
4import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send' 4import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send'
5import { VideoRateType } from '../../../shared/models/videos'
6import * as Bluebird from 'bluebird'
7import { getOrCreateActorAndServerAndModel } from './actor'
8import { AccountVideoRateModel } from '../../models/account/account-video-rate'
9import { logger } from '../../helpers/logger'
10import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
11
12async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) {
13 let rateCounts = 0
14
15 await Bluebird.map(actorUrls, async actorUrl => {
16 try {
17 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
18 const [ , created ] = await AccountVideoRateModel
19 .findOrCreate({
20 where: {
21 videoId: video.id,
22 accountId: actor.Account.id
23 },
24 defaults: {
25 videoId: video.id,
26 accountId: actor.Account.id,
27 type: rate
28 }
29 })
30
31 if (created) rateCounts += 1
32 } catch (err) {
33 logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err })
34 }
35 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
36
37 logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid)
38
39 // This is "likes" and "dislikes"
40 if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts })
41
42 return
43}
5 44
6async function sendVideoRateChange (account: AccountModel, 45async function sendVideoRateChange (account: AccountModel,
7 video: VideoModel, 46 video: VideoModel,
@@ -24,5 +63,6 @@ async function sendVideoRateChange (account: AccountModel,
24} 63}
25 64
26export { 65export {
66 createRates,
27 sendVideoRateChange 67 sendVideoRateChange
28} 68}
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index fac1d3fc7..388c31fe5 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -5,29 +5,30 @@ import { join } from 'path'
5import * as request from 'request' 5import * as request from 'request'
6import { ActivityIconObject, VideoState } from '../../../shared/index' 6import { ActivityIconObject, VideoState } from '../../../shared/index'
7import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 7import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
8import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos' 8import { VideoPrivacy } from '../../../shared/models/videos'
9import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' 9import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
10import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' 10import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
11import { retryTransactionWrapper } from '../../helpers/database-utils' 11import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
12import { logger } from '../../helpers/logger' 12import { logger } from '../../helpers/logger'
13import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' 13import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
14import { ACTIVITY_PUB, CONFIG, CRAWL_REQUEST_CONCURRENCY, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' 14import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
15import { AccountVideoRateModel } from '../../models/account/account-video-rate'
16import { ActorModel } from '../../models/activitypub/actor' 15import { ActorModel } from '../../models/activitypub/actor'
17import { TagModel } from '../../models/video/tag' 16import { TagModel } from '../../models/video/tag'
18import { VideoModel } from '../../models/video/video' 17import { VideoModel } from '../../models/video/video'
19import { VideoChannelModel } from '../../models/video/video-channel' 18import { VideoChannelModel } from '../../models/video/video-channel'
20import { VideoFileModel } from '../../models/video/video-file' 19import { VideoFileModel } from '../../models/video/video-file'
21import { VideoShareModel } from '../../models/video/video-share' 20import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor'
22import { getOrCreateActorAndServerAndModel } from './actor'
23import { addVideoComments } from './video-comments' 21import { addVideoComments } from './video-comments'
24import { crawlCollectionPage } from './crawl' 22import { crawlCollectionPage } from './crawl'
25import { sendCreateVideo, sendUpdateVideo } from './send' 23import { sendCreateVideo, sendUpdateVideo } from './send'
26import { shareVideoByServerAndChannel } from './index'
27import { isArray } from '../../helpers/custom-validators/misc' 24import { isArray } from '../../helpers/custom-validators/misc'
28import { VideoCaptionModel } from '../../models/video/video-caption' 25import { VideoCaptionModel } from '../../models/video/video-caption'
29import { JobQueue } from '../job-queue' 26import { JobQueue } from '../job-queue'
30import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher' 27import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
28import { getUrlFromWebfinger } from '../../helpers/webfinger'
29import { createRates } from './video-rates'
30import { addVideoShares, shareVideoByServerAndChannel } from './share'
31import { AccountModel } from '../../models/account/account'
31 32
32async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 33async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
33 // If the video is not private and published, we federate it 34 // If the video is not private and published, we federate it
@@ -180,15 +181,11 @@ function getOrCreateVideoChannel (videoObject: VideoTorrentObject) {
180 return getOrCreateActorAndServerAndModel(channel.id) 181 return getOrCreateActorAndServerAndModel(channel.id)
181} 182}
182 183
183async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { 184async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
184 logger.debug('Adding remote video %s.', videoObject.id) 185 logger.debug('Adding remote video %s.', videoObject.id)
185 186
186 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { 187 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
187 const sequelizeOptions = { 188 const sequelizeOptions = { transaction: t }
188 transaction: t
189 }
190 const videoFromDatabase = await VideoModel.loadByUUIDOrURLAndPopulateAccount(videoObject.uuid, videoObject.id, t)
191 if (videoFromDatabase) return videoFromDatabase
192 189
193 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) 190 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
194 const video = VideoModel.build(videoData) 191 const video = VideoModel.build(videoData)
@@ -230,26 +227,32 @@ async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor:
230} 227}
231 228
232type SyncParam = { 229type SyncParam = {
233 likes: boolean, 230 likes: boolean
234 dislikes: boolean, 231 dislikes: boolean
235 shares: boolean, 232 shares: boolean
236 comments: boolean, 233 comments: boolean
237 thumbnail: boolean 234 thumbnail: boolean
235 refreshVideo: boolean
238} 236}
239async function getOrCreateAccountAndVideoAndChannel ( 237async function getOrCreateVideoAndAccountAndChannel (
240 videoObject: VideoTorrentObject | string, 238 videoObject: VideoTorrentObject | string,
241 syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } 239 syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
242) { 240) {
243 const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id 241 const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
244 242
245 const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) 243 let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
246 if (videoFromDatabase) return { video: videoFromDatabase } 244 if (videoFromDatabase) {
245 const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase)
246 if (syncParam.refreshVideo === true) videoFromDatabase = await p
247
248 return { video: videoFromDatabase }
249 }
247 250
248 const fetchedVideo = await fetchRemoteVideo(videoUrl) 251 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
249 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) 252 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
250 253
251 const channelActor = await getOrCreateVideoChannel(fetchedVideo) 254 const channelActor = await getOrCreateVideoChannel(fetchedVideo)
252 const video = await retryTransactionWrapper(getOrCreateVideo, fetchedVideo, channelActor, syncParam.thumbnail) 255 const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
253 256
254 // Process outside the transaction because we could fetch remote data 257 // Process outside the transaction because we could fetch remote data
255 258
@@ -290,101 +293,153 @@ async function getOrCreateAccountAndVideoAndChannel (
290 return { video } 293 return { video }
291} 294}
292 295
293async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { 296async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
294 let rateCounts = 0 297 const options = {
295 298 uri: videoUrl,
296 await Bluebird.map(actorUrls, async actorUrl => { 299 method: 'GET',
297 try { 300 json: true,
298 const actor = await getOrCreateActorAndServerAndModel(actorUrl) 301 activityPub: true
299 const [ , created ] = await AccountVideoRateModel 302 }
300 .findOrCreate({ 303
301 where: { 304 logger.info('Fetching remote video %s.', videoUrl)
302 videoId: video.id, 305
303 accountId: actor.Account.id 306 const { response, body } = await doRequest(options)
304 }, 307
305 defaults: { 308 if (sanitizeAndCheckVideoTorrentObject(body) === false) {
306 videoId: video.id, 309 logger.debug('Remote video JSON is not valid.', { body })
307 accountId: actor.Account.id, 310 return { response, videoObject: undefined }
308 type: rate 311 }
309 } 312
310 }) 313 return { response, videoObject: body }
311 314}
312 if (created) rateCounts += 1 315
313 } catch (err) { 316async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
314 logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err }) 317 if (!video.isOutdated()) return video
318
319 try {
320 const { response, videoObject } = await fetchRemoteVideo(video.url)
321 if (response.statusCode === 404) {
322 // Video does not exist anymore
323 await video.destroy()
324 return undefined
315 } 325 }
316 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
317 326
318 logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid) 327 if (videoObject === undefined) {
328 logger.warn('Cannot refresh remote video: invalid body.')
329 return video
330 }
319 331
320 // This is "likes" and "dislikes" 332 const channelActor = await getOrCreateVideoChannel(videoObject)
321 if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts }) 333 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
334 return updateVideoFromAP(video, videoObject, account.Actor, channelActor)
322 335
323 return 336 } catch (err) {
337 logger.warn('Cannot refresh video.', { err })
338 return video
339 }
324} 340}
325 341
326async function addVideoShares (shareUrls: string[], instance: VideoModel) { 342async function updateVideoFromAP (
327 await Bluebird.map(shareUrls, async shareUrl => { 343 video: VideoModel,
328 try { 344 videoObject: VideoTorrentObject,
329 // Fetch url 345 accountActor: ActorModel,
330 const { body } = await doRequest({ 346 channelActor: ActorModel,
331 uri: shareUrl, 347 overrideTo?: string[]
332 json: true, 348) {
333 activityPub: true 349 logger.debug('Updating remote video "%s".', videoObject.uuid)
334 }) 350 let videoFieldsSave: any
335 if (!body || !body.actor) throw new Error('Body of body actor is invalid') 351
352 try {
353 const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => {
354 const sequelizeOptions = {
355 transaction: t
356 }
336 357
337 const actorUrl = body.actor 358 videoFieldsSave = video.toJSON()
338 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
339 359
340 const entry = { 360 // Check actor has the right to update the video
341 actorId: actor.id, 361 const videoChannel = video.VideoChannel
342 videoId: instance.id, 362 if (videoChannel.Account.Actor.id !== accountActor.id) {
343 url: shareUrl 363 throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url)
344 } 364 }
345 365
346 await VideoShareModel.findOrCreate({ 366 const to = overrideTo ? overrideTo : videoObject.to
347 where: { 367 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to)
348 url: shareUrl 368 video.set('name', videoData.name)
349 }, 369 video.set('uuid', videoData.uuid)
350 defaults: entry 370 video.set('url', videoData.url)
351 }) 371 video.set('category', videoData.category)
352 } catch (err) { 372 video.set('licence', videoData.licence)
353 logger.warn('Cannot add share %s.', shareUrl, { err }) 373 video.set('language', videoData.language)
354 } 374 video.set('description', videoData.description)
355 }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) 375 video.set('support', videoData.support)
356} 376 video.set('nsfw', videoData.nsfw)
377 video.set('commentsEnabled', videoData.commentsEnabled)
378 video.set('waitTranscoding', videoData.waitTranscoding)
379 video.set('state', videoData.state)
380 video.set('duration', videoData.duration)
381 video.set('createdAt', videoData.createdAt)
382 video.set('publishedAt', videoData.publishedAt)
383 video.set('views', videoData.views)
384 video.set('privacy', videoData.privacy)
385 video.set('channelId', videoData.channelId)
386
387 await video.save(sequelizeOptions)
388
389 // Don't block on request
390 generateThumbnailFromUrl(video, videoObject.icon)
391 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
392
393 // Remove old video files
394 const videoFileDestroyTasks: Bluebird<void>[] = []
395 for (const videoFile of video.VideoFiles) {
396 videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
397 }
398 await Promise.all(videoFileDestroyTasks)
357 399
358async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject> { 400 const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject)
359 const options = { 401 const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
360 uri: videoUrl, 402 await Promise.all(tasks)
361 method: 'GET',
362 json: true,
363 activityPub: true
364 }
365 403
366 logger.info('Fetching remote video %s.', videoUrl) 404 // Update Tags
405 const tags = videoObject.tag.map(tag => tag.name)
406 const tagInstances = await TagModel.findOrCreateTags(tags, t)
407 await video.$set('Tags', tagInstances, sequelizeOptions)
367 408
368 const { body } = await doRequest(options) 409 // Update captions
410 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t)
369 411
370 if (sanitizeAndCheckVideoTorrentObject(body) === false) { 412 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
371 logger.debug('Remote video JSON is not valid.', { body }) 413 return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t)
372 return undefined 414 })
373 } 415 await Promise.all(videoCaptionsPromises)
416 })
417
418 logger.info('Remote video with uuid %s updated', videoObject.uuid)
374 419
375 return body 420 return updatedVideo
421 } catch (err) {
422 if (video !== undefined && videoFieldsSave !== undefined) {
423 resetSequelizeInstance(video, videoFieldsSave)
424 }
425
426 // This is just a debug because we will retry the insert
427 logger.debug('Cannot update the remote video.', { err })
428 throw err
429 }
376} 430}
377 431
378export { 432export {
433 updateVideoFromAP,
379 federateVideoIfNeeded, 434 federateVideoIfNeeded,
380 fetchRemoteVideo, 435 fetchRemoteVideo,
381 getOrCreateAccountAndVideoAndChannel, 436 getOrCreateVideoAndAccountAndChannel,
382 fetchRemoteVideoStaticFile, 437 fetchRemoteVideoStaticFile,
383 fetchRemoteVideoDescription, 438 fetchRemoteVideoDescription,
384 generateThumbnailFromUrl, 439 generateThumbnailFromUrl,
385 videoActivityObjectToDBAttributes, 440 videoActivityObjectToDBAttributes,
386 videoFileActivityUrlToDBAttributes, 441 videoFileActivityUrlToDBAttributes,
387 getOrCreateVideo, 442 createVideo,
388 getOrCreateVideoChannel, 443 getOrCreateVideoChannel,
389 addVideoShares, 444 addVideoShares,
390 createRates 445 createRates