aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-09-20 16:24:31 +0200
committerGitHub <noreply@github.com>2018-09-20 16:24:31 +0200
commit0491173a61aed66205c017e0d7e0503ea316c144 (patch)
treece6621597505f9518cfdf0981977d097c63f9fad /server/lib
parent8704acf49efc770d73bf07c10468ed8c74d28a83 (diff)
parent6247b2057b792cea155a1abd9788c363ae7d2cc2 (diff)
downloadPeerTube-0491173a61aed66205c017e0d7e0503ea316c144.tar.gz
PeerTube-0491173a61aed66205c017e0d7e0503ea316c144.tar.zst
PeerTube-0491173a61aed66205c017e0d7e0503ea316c144.zip
Merge branch 'develop' into cli-wrapper
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/actor.ts18
-rw-r--r--server/lib/activitypub/audience.ts10
-rw-r--r--server/lib/activitypub/cache-file.ts11
-rw-r--r--server/lib/activitypub/process/process-accept.ts6
-rw-r--r--server/lib/activitypub/process/process-announce.ts8
-rw-r--r--server/lib/activitypub/process/process-create.ts38
-rw-r--r--server/lib/activitypub/process/process-delete.ts34
-rw-r--r--server/lib/activitypub/process/process-follow.ts8
-rw-r--r--server/lib/activitypub/process/process-like.ts9
-rw-r--r--server/lib/activitypub/process/process-reject.ts12
-rw-r--r--server/lib/activitypub/process/process-undo.ts61
-rw-r--r--server/lib/activitypub/process/process-update.ts31
-rw-r--r--server/lib/activitypub/process/process.ts15
-rw-r--r--server/lib/activitypub/send/send-announce.ts4
-rw-r--r--server/lib/activitypub/send/send-create.ts95
-rw-r--r--server/lib/activitypub/send/send-delete.ts13
-rw-r--r--server/lib/activitypub/send/send-like.ts23
-rw-r--r--server/lib/activitypub/send/send-undo.ts85
-rw-r--r--server/lib/activitypub/send/send-update.ts16
-rw-r--r--server/lib/activitypub/send/utils.ts30
-rw-r--r--server/lib/activitypub/video-comments.ts2
-rw-r--r--server/lib/activitypub/videos.ts531
-rw-r--r--server/lib/avatar.ts11
-rw-r--r--server/lib/cache/videos-caption-cache.ts2
-rw-r--r--server/lib/cache/videos-preview-cache.ts4
-rw-r--r--server/lib/client-html.ts9
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-fetcher.ts4
-rw-r--r--server/lib/job-queue/handlers/video-file.ts15
-rw-r--r--server/lib/job-queue/handlers/video-import.ts2
-rw-r--r--server/lib/oauth-model.ts40
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts64
-rw-r--r--server/lib/schedulers/youtube-dl-update-scheduler.ts65
-rw-r--r--server/lib/video-transcoding.ts130
33 files changed, 766 insertions, 640 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 3464add03..d37a695a7 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -21,6 +21,7 @@ import { ServerModel } from '../../models/server/server'
21import { VideoChannelModel } from '../../models/video/video-channel' 21import { VideoChannelModel } from '../../models/video/video-channel'
22import { JobQueue } from '../job-queue' 22import { JobQueue } from '../job-queue'
23import { getServerActor } from '../../helpers/utils' 23import { getServerActor } from '../../helpers/utils'
24import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
24 25
25// Set account keys, this could be long so process after the account creation and do not block the client 26// Set account keys, this could be long so process after the account creation and do not block the client
26function setAsyncActorKeys (actor: ActorModel) { 27function setAsyncActorKeys (actor: ActorModel) {
@@ -38,13 +39,14 @@ function setAsyncActorKeys (actor: ActorModel) {
38 39
39async function getOrCreateActorAndServerAndModel ( 40async function getOrCreateActorAndServerAndModel (
40 activityActor: string | ActivityPubActor, 41 activityActor: string | ActivityPubActor,
42 fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
41 recurseIfNeeded = true, 43 recurseIfNeeded = true,
42 updateCollections = false 44 updateCollections = false
43) { 45) {
44 const actorUrl = getActorUrl(activityActor) 46 const actorUrl = getActorUrl(activityActor)
45 let created = false 47 let created = false
46 48
47 let actor = await ActorModel.loadByUrl(actorUrl) 49 let actor = await fetchActorByUrl(actorUrl, fetchType)
48 // Orphan actor (not associated to an account of channel) so recreate it 50 // Orphan actor (not associated to an account of channel) so recreate it
49 if (actor && (!actor.Account && !actor.VideoChannel)) { 51 if (actor && (!actor.Account && !actor.VideoChannel)) {
50 await actor.destroy() 52 await actor.destroy()
@@ -65,7 +67,7 @@ async function getOrCreateActorAndServerAndModel (
65 67
66 try { 68 try {
67 // Assert we don't recurse another time 69 // Assert we don't recurse another time
68 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, false) 70 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
69 } catch (err) { 71 } catch (err) {
70 logger.error('Cannot get or create account attributed to video channel ' + actor.url) 72 logger.error('Cannot get or create account attributed to video channel ' + actor.url)
71 throw new Error(err) 73 throw new Error(err)
@@ -79,7 +81,7 @@ async function getOrCreateActorAndServerAndModel (
79 if (actor.Account) actor.Account.Actor = actor 81 if (actor.Account) actor.Account.Actor = actor
80 if (actor.VideoChannel) actor.VideoChannel.Actor = actor 82 if (actor.VideoChannel) actor.VideoChannel.Actor = actor
81 83
82 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor) 84 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
83 if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.') 85 if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
84 86
85 if ((created === true || refreshed === true) && updateCollections === true) { 87 if ((created === true || refreshed === true) && updateCollections === true) {
@@ -370,8 +372,14 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu
370 return videoChannelCreated 372 return videoChannelCreated
371} 373}
372 374
373async function refreshActorIfNeeded (actor: ActorModel): Promise<{ actor: ActorModel, refreshed: boolean }> { 375async function refreshActorIfNeeded (
374 if (!actor.isOutdated()) return { actor, refreshed: false } 376 actorArg: ActorModel,
377 fetchedType: ActorFetchByUrlType
378): Promise<{ actor: ActorModel, refreshed: boolean }> {
379 if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
380
381 // We need more attributes
382 const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
375 383
376 try { 384 try {
377 const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) 385 const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts
index 7b4067c11..a86428461 100644
--- a/server/lib/activitypub/audience.ts
+++ b/server/lib/activitypub/audience.ts
@@ -6,7 +6,7 @@ import { VideoModel } from '../../models/video/video'
6import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
7import { VideoShareModel } from '../../models/video/video-share' 7import { VideoShareModel } from '../../models/video/video-share'
8 8
9function getVideoAudience (video: VideoModel, actorsInvolvedInVideo: ActorModel[]) { 9function getRemoteVideoAudience (video: VideoModel, actorsInvolvedInVideo: ActorModel[]): ActivityAudience {
10 return { 10 return {
11 to: [ video.VideoChannel.Account.Actor.url ], 11 to: [ video.VideoChannel.Account.Actor.url ],
12 cc: actorsInvolvedInVideo.map(a => a.followersUrl) 12 cc: actorsInvolvedInVideo.map(a => a.followersUrl)
@@ -18,7 +18,7 @@ function getVideoCommentAudience (
18 threadParentComments: VideoCommentModel[], 18 threadParentComments: VideoCommentModel[],
19 actorsInvolvedInVideo: ActorModel[], 19 actorsInvolvedInVideo: ActorModel[],
20 isOrigin = false 20 isOrigin = false
21) { 21): ActivityAudience {
22 const to = [ ACTIVITY_PUB.PUBLIC ] 22 const to = [ ACTIVITY_PUB.PUBLIC ]
23 const cc: string[] = [] 23 const cc: string[] = []
24 24
@@ -41,7 +41,7 @@ function getVideoCommentAudience (
41 } 41 }
42} 42}
43 43
44function getObjectFollowersAudience (actorsInvolvedInObject: ActorModel[]) { 44function getAudienceFromFollowersOf (actorsInvolvedInObject: ActorModel[]): ActivityAudience {
45 return { 45 return {
46 to: [ ACTIVITY_PUB.PUBLIC ].concat(actorsInvolvedInObject.map(a => a.followersUrl)), 46 to: [ ACTIVITY_PUB.PUBLIC ].concat(actorsInvolvedInObject.map(a => a.followersUrl)),
47 cc: [] 47 cc: []
@@ -83,9 +83,9 @@ function audiencify<T> (object: T, audience: ActivityAudience) {
83export { 83export {
84 buildAudience, 84 buildAudience,
85 getAudience, 85 getAudience,
86 getVideoAudience, 86 getRemoteVideoAudience,
87 getActorsInvolvedInVideo, 87 getActorsInvolvedInVideo,
88 getObjectFollowersAudience, 88 getAudienceFromFollowersOf,
89 audiencify, 89 audiencify,
90 getVideoCommentAudience 90 getVideoCommentAudience
91} 91}
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index 7325ddcb6..87f8a4162 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -1,10 +1,9 @@
1import { CacheFileObject } from '../../../shared/index' 1import { CacheFileObject } from '../../../shared/index'
2import { VideoModel } from '../../models/video/video' 2import { VideoModel } from '../../models/video/video'
3import { ActorModel } from '../../models/activitypub/actor'
4import { sequelizeTypescript } from '../../initializers' 3import { sequelizeTypescript } from '../../initializers'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 4import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6 5
7function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) { 6function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
8 const url = cacheFileObject.url 7 const url = cacheFileObject.url
9 8
10 const videoFile = video.VideoFiles.find(f => { 9 const videoFile = video.VideoFiles.find(f => {
@@ -23,7 +22,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
23 } 22 }
24} 23}
25 24
26function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) { 25function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
27 return sequelizeTypescript.transaction(async t => { 26 return sequelizeTypescript.transaction(async t => {
28 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) 27 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
29 28
@@ -31,7 +30,11 @@ function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, b
31 }) 30 })
32} 31}
33 32
34function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: ActorModel) { 33function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: { id?: number }) {
34 if (redundancyModel.actorId !== byActor.id) {
35 throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.')
36 }
37
35 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor) 38 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor)
36 39
37 redundancyModel.set('expires', attributes.expiresOn) 40 redundancyModel.set('expires', attributes.expiresOn)
diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts
index 046370b79..89bda9c32 100644
--- a/server/lib/activitypub/process/process-accept.ts
+++ b/server/lib/activitypub/process/process-accept.ts
@@ -1,15 +1,11 @@
1import { ActivityAccept } from '../../../../shared/models/activitypub' 1import { ActivityAccept } from '../../../../shared/models/activitypub'
2import { getActorUrl } from '../../../helpers/activitypub'
3import { ActorModel } from '../../../models/activitypub/actor' 2import { ActorModel } from '../../../models/activitypub/actor'
4import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 3import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
5import { addFetchOutboxJob } from '../actor' 4import { addFetchOutboxJob } from '../actor'
6 5
7async function processAcceptActivity (activity: ActivityAccept, inboxActor?: ActorModel) { 6async function processAcceptActivity (activity: ActivityAccept, targetActor: ActorModel, inboxActor?: ActorModel) {
8 if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') 7 if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.')
9 8
10 const actorUrl = getActorUrl(activity.actor)
11 const targetActor = await ActorModel.loadByUrl(actorUrl)
12
13 return processAccept(inboxActor, targetActor) 9 return processAccept(inboxActor, targetActor)
14} 10}
15 11
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts
index 814556817..cc88b5423 100644
--- a/server/lib/activitypub/process/process-announce.ts
+++ b/server/lib/activitypub/process/process-announce.ts
@@ -2,15 +2,11 @@ import { ActivityAnnounce } from '../../../../shared/models/activitypub'
2import { retryTransactionWrapper } from '../../../helpers/database-utils' 2import { retryTransactionWrapper } from '../../../helpers/database-utils'
3import { sequelizeTypescript } from '../../../initializers' 3import { sequelizeTypescript } from '../../../initializers'
4import { ActorModel } from '../../../models/activitypub/actor' 4import { ActorModel } from '../../../models/activitypub/actor'
5import { VideoModel } from '../../../models/video/video'
6import { VideoShareModel } from '../../../models/video/video-share' 5import { VideoShareModel } from '../../../models/video/video-share'
7import { getOrCreateActorAndServerAndModel } from '../actor'
8import { forwardVideoRelatedActivity } from '../send/utils' 6import { forwardVideoRelatedActivity } from '../send/utils'
9import { getOrCreateVideoAndAccountAndChannel } from '../videos' 7import { getOrCreateVideoAndAccountAndChannel } from '../videos'
10 8
11async function processAnnounceActivity (activity: ActivityAnnounce) { 9async function processAnnounceActivity (activity: ActivityAnnounce, actorAnnouncer: ActorModel) {
12 const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor)
13
14 return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity) 10 return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity)
15} 11}
16 12
@@ -25,7 +21,7 @@ export {
25async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { 21async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
26 const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id 22 const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id
27 23
28 const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri) 24 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri })
29 25
30 return sequelizeTypescript.transaction(async t => { 26 return sequelizeTypescript.transaction(async t => {
31 // Add share entry 27 // Add share entry
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 32e555acf..5197dac73 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -7,30 +7,28 @@ import { sequelizeTypescript } from '../../../initializers'
7import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 7import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
8import { ActorModel } from '../../../models/activitypub/actor' 8import { ActorModel } from '../../../models/activitypub/actor'
9import { VideoAbuseModel } from '../../../models/video/video-abuse' 9import { VideoAbuseModel } from '../../../models/video/video-abuse'
10import { getOrCreateActorAndServerAndModel } from '../actor'
11import { addVideoComment, resolveThread } from '../video-comments' 10import { addVideoComment, resolveThread } from '../video-comments'
12import { getOrCreateVideoAndAccountAndChannel } from '../videos' 11import { getOrCreateVideoAndAccountAndChannel } from '../videos'
13import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' 12import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils'
14import { Redis } from '../../redis' 13import { Redis } from '../../redis'
15import { createCacheFile } from '../cache-file' 14import { createCacheFile } from '../cache-file'
16 15
17async function processCreateActivity (activity: ActivityCreate) { 16async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) {
18 const activityObject = activity.object 17 const activityObject = activity.object
19 const activityType = activityObject.type 18 const activityType = activityObject.type
20 const actor = await getOrCreateActorAndServerAndModel(activity.actor)
21 19
22 if (activityType === 'View') { 20 if (activityType === 'View') {
23 return processCreateView(actor, activity) 21 return processCreateView(byActor, activity)
24 } else if (activityType === 'Dislike') { 22 } else if (activityType === 'Dislike') {
25 return retryTransactionWrapper(processCreateDislike, actor, activity) 23 return retryTransactionWrapper(processCreateDislike, byActor, activity)
26 } else if (activityType === 'Video') { 24 } else if (activityType === 'Video') {
27 return processCreateVideo(activity) 25 return processCreateVideo(activity)
28 } else if (activityType === 'Flag') { 26 } else if (activityType === 'Flag') {
29 return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject) 27 return retryTransactionWrapper(processCreateVideoAbuse, byActor, activityObject as VideoAbuseObject)
30 } else if (activityType === 'Note') { 28 } else if (activityType === 'Note') {
31 return retryTransactionWrapper(processCreateVideoComment, actor, activity) 29 return retryTransactionWrapper(processCreateVideoComment, byActor, activity)
32 } else if (activityType === 'CacheFile') { 30 } else if (activityType === 'CacheFile') {
33 return retryTransactionWrapper(processCacheFile, actor, activity) 31 return retryTransactionWrapper(processCacheFile, byActor, activity)
34 } 32 }
35 33
36 logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) 34 logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })
@@ -48,7 +46,7 @@ export {
48async function processCreateVideo (activity: ActivityCreate) { 46async function processCreateVideo (activity: ActivityCreate) {
49 const videoToCreateData = activity.object as VideoTorrentObject 47 const videoToCreateData = activity.object as VideoTorrentObject
50 48
51 const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData) 49 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData })
52 50
53 return video 51 return video
54} 52}
@@ -59,7 +57,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
59 57
60 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) 58 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
61 59
62 const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) 60 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object })
63 61
64 return sequelizeTypescript.transaction(async t => { 62 return sequelizeTypescript.transaction(async t => {
65 const rate = { 63 const rate = {
@@ -86,10 +84,14 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
86async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { 84async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
87 const view = activity.object as ViewObject 85 const view = activity.object as ViewObject
88 86
89 const { video } = await getOrCreateVideoAndAccountAndChannel(view.object) 87 const options = {
88 videoObject: view.object,
89 fetchType: 'only-video' as 'only-video'
90 }
91 const { video } = await getOrCreateVideoAndAccountAndChannel(options)
90 92
91 const actor = await ActorModel.loadByUrl(view.actor) 93 const actorExists = await ActorModel.isActorUrlExist(view.actor)
92 if (!actor) throw new Error('Unknown actor ' + view.actor) 94 if (actorExists === false) throw new Error('Unknown actor ' + view.actor)
93 95
94 await Redis.Instance.addVideoView(video.id) 96 await Redis.Instance.addVideoView(video.id)
95 97
@@ -103,7 +105,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate)
103async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { 105async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) {
104 const cacheFile = activity.object as CacheFileObject 106 const cacheFile = activity.object as CacheFileObject
105 107
106 const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object) 108 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object })
107 109
108 await createCacheFile(cacheFile, video, byActor) 110 await createCacheFile(cacheFile, video, byActor)
109 111
@@ -114,13 +116,13 @@ async function processCacheFile (byActor: ActorModel, activity: ActivityCreate)
114 } 116 }
115} 117}
116 118
117async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { 119async function processCreateVideoAbuse (byActor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) {
118 logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) 120 logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object)
119 121
120 const account = actor.Account 122 const account = byActor.Account
121 if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) 123 if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
122 124
123 const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object) 125 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object })
124 126
125 return sequelizeTypescript.transaction(async t => { 127 return sequelizeTypescript.transaction(async t => {
126 const videoAbuseData = { 128 const videoAbuseData = {
diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts
index 3c830abea..038d8c4d3 100644
--- a/server/lib/activitypub/process/process-delete.ts
+++ b/server/lib/activitypub/process/process-delete.ts
@@ -7,41 +7,41 @@ import { ActorModel } from '../../../models/activitypub/actor'
7import { VideoModel } from '../../../models/video/video' 7import { VideoModel } from '../../../models/video/video'
8import { VideoChannelModel } from '../../../models/video/video-channel' 8import { VideoChannelModel } from '../../../models/video/video-channel'
9import { VideoCommentModel } from '../../../models/video/video-comment' 9import { VideoCommentModel } from '../../../models/video/video-comment'
10import { getOrCreateActorAndServerAndModel } from '../actor'
11import { forwardActivity } from '../send/utils' 10import { forwardActivity } from '../send/utils'
12 11
13async function processDeleteActivity (activity: ActivityDelete) { 12async function processDeleteActivity (activity: ActivityDelete, byActor: ActorModel) {
14 const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id 13 const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id
15 14
16 if (activity.actor === objectUrl) { 15 if (activity.actor === objectUrl) {
17 let actor = await ActorModel.loadByUrl(activity.actor) 16 // We need more attributes (all the account and channel)
18 if (!actor) return undefined 17 const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url)
19 18
20 if (actor.type === 'Person') { 19 if (byActorFull.type === 'Person') {
21 if (!actor.Account) throw new Error('Actor ' + actor.url + ' is a person but we cannot find it in database.') 20 if (!byActorFull.Account) throw new Error('Actor ' + byActorFull.url + ' is a person but we cannot find it in database.')
22 21
23 actor.Account.Actor = await actor.Account.$get('Actor') as ActorModel 22 byActorFull.Account.Actor = await byActorFull.Account.$get('Actor') as ActorModel
24 return retryTransactionWrapper(processDeleteAccount, actor.Account) 23 return retryTransactionWrapper(processDeleteAccount, byActorFull.Account)
25 } else if (actor.type === 'Group') { 24 } else if (byActorFull.type === 'Group') {
26 if (!actor.VideoChannel) throw new Error('Actor ' + actor.url + ' is a group but we cannot find it in database.') 25 if (!byActorFull.VideoChannel) throw new Error('Actor ' + byActorFull.url + ' is a group but we cannot find it in database.')
27 26
28 actor.VideoChannel.Actor = await actor.VideoChannel.$get('Actor') as ActorModel 27 byActorFull.VideoChannel.Actor = await byActorFull.VideoChannel.$get('Actor') as ActorModel
29 return retryTransactionWrapper(processDeleteVideoChannel, actor.VideoChannel) 28 return retryTransactionWrapper(processDeleteVideoChannel, byActorFull.VideoChannel)
30 } 29 }
31 } 30 }
32 31
33 const actor = await getOrCreateActorAndServerAndModel(activity.actor)
34 { 32 {
35 const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccount(objectUrl) 33 const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccount(objectUrl)
36 if (videoCommentInstance) { 34 if (videoCommentInstance) {
37 return retryTransactionWrapper(processDeleteVideoComment, actor, videoCommentInstance, activity) 35 return retryTransactionWrapper(processDeleteVideoComment, byActor, videoCommentInstance, activity)
38 } 36 }
39 } 37 }
40 38
41 { 39 {
42 const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl) 40 const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl)
43 if (videoInstance) { 41 if (videoInstance) {
44 return retryTransactionWrapper(processDeleteVideo, actor, videoInstance) 42 if (videoInstance.isOwned()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`)
43
44 return retryTransactionWrapper(processDeleteVideo, byActor, videoInstance)
45 } 45 }
46 } 46 }
47 47
@@ -94,6 +94,10 @@ function processDeleteVideoComment (byActor: ActorModel, videoComment: VideoComm
94 logger.debug('Removing remote video comment "%s".', videoComment.url) 94 logger.debug('Removing remote video comment "%s".', videoComment.url)
95 95
96 return sequelizeTypescript.transaction(async t => { 96 return sequelizeTypescript.transaction(async t => {
97 if (videoComment.Account.id !== byActor.Account.id) {
98 throw new Error('Account ' + byActor.url + ' does not own video comment ' + videoComment.url)
99 }
100
97 await videoComment.destroy({ transaction: t }) 101 await videoComment.destroy({ transaction: t })
98 102
99 if (videoComment.Video.isOwned()) { 103 if (videoComment.Video.isOwned()) {
diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts
index f34fd66cc..24c9085f7 100644
--- a/server/lib/activitypub/process/process-follow.ts
+++ b/server/lib/activitypub/process/process-follow.ts
@@ -4,14 +4,12 @@ import { logger } from '../../../helpers/logger'
4import { sequelizeTypescript } from '../../../initializers' 4import { sequelizeTypescript } from '../../../initializers'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/activitypub/actor'
6import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 6import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
7import { getOrCreateActorAndServerAndModel } from '../actor'
8import { sendAccept } from '../send' 7import { sendAccept } from '../send'
9 8
10async function processFollowActivity (activity: ActivityFollow) { 9async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) {
11 const activityObject = activity.object 10 const activityObject = activity.object
12 const actor = await getOrCreateActorAndServerAndModel(activity.actor)
13 11
14 return retryTransactionWrapper(processFollow, actor, activityObject) 12 return retryTransactionWrapper(processFollow, byActor, activityObject)
15} 13}
16 14
17// --------------------------------------------------------------------------- 15// ---------------------------------------------------------------------------
@@ -24,7 +22,7 @@ export {
24 22
25async function processFollow (actor: ActorModel, targetActorURL: string) { 23async function processFollow (actor: ActorModel, targetActorURL: string) {
26 await sequelizeTypescript.transaction(async t => { 24 await sequelizeTypescript.transaction(async t => {
27 const targetActor = await ActorModel.loadByUrl(targetActorURL, t) 25 const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t)
28 26
29 if (!targetActor) throw new Error('Unknown actor') 27 if (!targetActor) throw new Error('Unknown actor')
30 if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') 28 if (targetActor.isOwned() === false) throw new Error('This is not a local actor.')
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts
index 9e1664fd8..f7200db61 100644
--- a/server/lib/activitypub/process/process-like.ts
+++ b/server/lib/activitypub/process/process-like.ts
@@ -3,14 +3,11 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
3import { sequelizeTypescript } from '../../../initializers' 3import { sequelizeTypescript } from '../../../initializers'
4import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 4import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/activitypub/actor'
6import { getOrCreateActorAndServerAndModel } from '../actor'
7import { forwardVideoRelatedActivity } from '../send/utils' 6import { forwardVideoRelatedActivity } from '../send/utils'
8import { getOrCreateVideoAndAccountAndChannel } from '../videos' 7import { getOrCreateVideoAndAccountAndChannel } from '../videos'
9 8
10async function processLikeActivity (activity: ActivityLike) { 9async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) {
11 const actor = await getOrCreateActorAndServerAndModel(activity.actor) 10 return retryTransactionWrapper(processLikeVideo, byActor, activity)
12
13 return retryTransactionWrapper(processLikeVideo, actor, activity)
14} 11}
15 12
16// --------------------------------------------------------------------------- 13// ---------------------------------------------------------------------------
@@ -27,7 +24,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) {
27 const byAccount = byActor.Account 24 const byAccount = byActor.Account
28 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) 25 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
29 26
30 const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl) 27 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl })
31 28
32 return sequelizeTypescript.transaction(async t => { 29 return sequelizeTypescript.transaction(async t => {
33 const rate = { 30 const rate = {
diff --git a/server/lib/activitypub/process/process-reject.ts b/server/lib/activitypub/process/process-reject.ts
index f06b03772..709a65096 100644
--- a/server/lib/activitypub/process/process-reject.ts
+++ b/server/lib/activitypub/process/process-reject.ts
@@ -1,15 +1,11 @@
1import { ActivityReject } from '../../../../shared/models/activitypub/activity' 1import { ActivityReject } from '../../../../shared/models/activitypub/activity'
2import { getActorUrl } from '../../../helpers/activitypub'
3import { sequelizeTypescript } from '../../../initializers' 2import { sequelizeTypescript } from '../../../initializers'
4import { ActorModel } from '../../../models/activitypub/actor' 3import { ActorModel } from '../../../models/activitypub/actor'
5import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 4import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
6 5
7async function processRejectActivity (activity: ActivityReject, inboxActor?: ActorModel) { 6async function processRejectActivity (activity: ActivityReject, targetActor: ActorModel, inboxActor?: ActorModel) {
8 if (inboxActor === undefined) throw new Error('Need to reject on explicit inbox.') 7 if (inboxActor === undefined) throw new Error('Need to reject on explicit inbox.')
9 8
10 const actorUrl = getActorUrl(activity.actor)
11 const targetActor = await ActorModel.loadByUrl(actorUrl)
12
13 return processReject(inboxActor, targetActor) 9 return processReject(inboxActor, targetActor)
14} 10}
15 11
@@ -21,11 +17,11 @@ export {
21 17
22// --------------------------------------------------------------------------- 18// ---------------------------------------------------------------------------
23 19
24async function processReject (actor: ActorModel, targetActor: ActorModel) { 20async function processReject (follower: ActorModel, targetActor: ActorModel) {
25 return sequelizeTypescript.transaction(async t => { 21 return sequelizeTypescript.transaction(async t => {
26 const actorFollow = await ActorFollowModel.loadByActorAndTarget(actor.id, targetActor.id, t) 22 const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, targetActor.id, t)
27 23
28 if (!actorFollow) throw new Error(`'Unknown actor follow ${actor.id} -> ${targetActor.id}.`) 24 if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${targetActor.id}.`)
29 25
30 await actorFollow.destroy({ transaction: t }) 26 await actorFollow.destroy({ transaction: t })
31 27
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index 0eb5fa392..73ca0a17c 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -1,10 +1,8 @@
1import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub' 1import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub'
2import { DislikeObject } from '../../../../shared/models/activitypub/objects' 2import { DislikeObject } from '../../../../shared/models/activitypub/objects'
3import { getActorUrl } from '../../../helpers/activitypub'
4import { retryTransactionWrapper } from '../../../helpers/database-utils' 3import { retryTransactionWrapper } from '../../../helpers/database-utils'
5import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
6import { sequelizeTypescript } from '../../../initializers' 5import { sequelizeTypescript } from '../../../initializers'
7import { AccountModel } from '../../../models/account/account'
8import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 6import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
9import { ActorModel } from '../../../models/activitypub/actor' 7import { ActorModel } from '../../../models/activitypub/actor'
10import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 8import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
@@ -13,29 +11,27 @@ import { getOrCreateVideoAndAccountAndChannel } from '../videos'
13import { VideoShareModel } from '../../../models/video/video-share' 11import { VideoShareModel } from '../../../models/video/video-share'
14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 12import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
15 13
16async function processUndoActivity (activity: ActivityUndo) { 14async function processUndoActivity (activity: ActivityUndo, byActor: ActorModel) {
17 const activityToUndo = activity.object 15 const activityToUndo = activity.object
18 16
19 const actorUrl = getActorUrl(activity.actor)
20
21 if (activityToUndo.type === 'Like') { 17 if (activityToUndo.type === 'Like') {
22 return retryTransactionWrapper(processUndoLike, actorUrl, activity) 18 return retryTransactionWrapper(processUndoLike, byActor, activity)
23 } 19 }
24 20
25 if (activityToUndo.type === 'Create') { 21 if (activityToUndo.type === 'Create') {
26 if (activityToUndo.object.type === 'Dislike') { 22 if (activityToUndo.object.type === 'Dislike') {
27 return retryTransactionWrapper(processUndoDislike, actorUrl, activity) 23 return retryTransactionWrapper(processUndoDislike, byActor, activity)
28 } else if (activityToUndo.object.type === 'CacheFile') { 24 } else if (activityToUndo.object.type === 'CacheFile') {
29 return retryTransactionWrapper(processUndoCacheFile, actorUrl, activity) 25 return retryTransactionWrapper(processUndoCacheFile, byActor, activity)
30 } 26 }
31 } 27 }
32 28
33 if (activityToUndo.type === 'Follow') { 29 if (activityToUndo.type === 'Follow') {
34 return retryTransactionWrapper(processUndoFollow, actorUrl, activityToUndo) 30 return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo)
35 } 31 }
36 32
37 if (activityToUndo.type === 'Announce') { 33 if (activityToUndo.type === 'Announce') {
38 return retryTransactionWrapper(processUndoAnnounce, actorUrl, activityToUndo) 34 return retryTransactionWrapper(processUndoAnnounce, byActor, activityToUndo)
39 } 35 }
40 36
41 logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id }) 37 logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id })
@@ -51,66 +47,63 @@ export {
51 47
52// --------------------------------------------------------------------------- 48// ---------------------------------------------------------------------------
53 49
54async function processUndoLike (actorUrl: string, activity: ActivityUndo) { 50async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) {
55 const likeActivity = activity.object as ActivityLike 51 const likeActivity = activity.object as ActivityLike
56 52
57 const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object) 53 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object })
58 54
59 return sequelizeTypescript.transaction(async t => { 55 return sequelizeTypescript.transaction(async t => {
60 const byAccount = await AccountModel.loadByUrl(actorUrl, t) 56 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
61 if (!byAccount) throw new Error('Unknown account ' + actorUrl)
62 57
63 const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) 58 const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t)
64 if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) 59 if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`)
65 60
66 await rate.destroy({ transaction: t }) 61 await rate.destroy({ transaction: t })
67 await video.decrement('likes', { transaction: t }) 62 await video.decrement('likes', { transaction: t })
68 63
69 if (video.isOwned()) { 64 if (video.isOwned()) {
70 // Don't resend the activity to the sender 65 // Don't resend the activity to the sender
71 const exceptions = [ byAccount.Actor ] 66 const exceptions = [ byActor ]
72 67
73 await forwardVideoRelatedActivity(activity, t, exceptions, video) 68 await forwardVideoRelatedActivity(activity, t, exceptions, video)
74 } 69 }
75 }) 70 })
76} 71}
77 72
78async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { 73async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) {
79 const dislike = activity.object.object as DislikeObject 74 const dislike = activity.object.object as DislikeObject
80 75
81 const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) 76 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object })
82 77
83 return sequelizeTypescript.transaction(async t => { 78 return sequelizeTypescript.transaction(async t => {
84 const byAccount = await AccountModel.loadByUrl(actorUrl, t) 79 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
85 if (!byAccount) throw new Error('Unknown account ' + actorUrl)
86 80
87 const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) 81 const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t)
88 if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) 82 if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`)
89 83
90 await rate.destroy({ transaction: t }) 84 await rate.destroy({ transaction: t })
91 await video.decrement('dislikes', { transaction: t }) 85 await video.decrement('dislikes', { transaction: t })
92 86
93 if (video.isOwned()) { 87 if (video.isOwned()) {
94 // Don't resend the activity to the sender 88 // Don't resend the activity to the sender
95 const exceptions = [ byAccount.Actor ] 89 const exceptions = [ byActor ]
96 90
97 await forwardVideoRelatedActivity(activity, t, exceptions, video) 91 await forwardVideoRelatedActivity(activity, t, exceptions, video)
98 } 92 }
99 }) 93 })
100} 94}
101 95
102async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) { 96async function processUndoCacheFile (byActor: ActorModel, activity: ActivityUndo) {
103 const cacheFileObject = activity.object.object as CacheFileObject 97 const cacheFileObject = activity.object.object as CacheFileObject
104 98
105 const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object) 99 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object })
106 100
107 return sequelizeTypescript.transaction(async t => { 101 return sequelizeTypescript.transaction(async t => {
108 const byActor = await ActorModel.loadByUrl(actorUrl)
109 if (!byActor) throw new Error('Unknown actor ' + actorUrl)
110
111 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) 102 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
112 if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url) 103 if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url)
113 104
105 if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.')
106
114 await cacheFile.destroy() 107 await cacheFile.destroy()
115 108
116 if (video.isOwned()) { 109 if (video.isOwned()) {
@@ -122,10 +115,9 @@ async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) {
122 }) 115 })
123} 116}
124 117
125function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) { 118function processUndoFollow (follower: ActorModel, followActivity: ActivityFollow) {
126 return sequelizeTypescript.transaction(async t => { 119 return sequelizeTypescript.transaction(async t => {
127 const follower = await ActorModel.loadByUrl(actorUrl, t) 120 const following = await ActorModel.loadByUrlAndPopulateAccountAndChannel(followActivity.object, t)
128 const following = await ActorModel.loadByUrl(followActivity.object, t)
129 const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, following.id, t) 121 const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, following.id, t)
130 122
131 if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${following.id}.`) 123 if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${following.id}.`)
@@ -136,11 +128,8 @@ function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) {
136 }) 128 })
137} 129}
138 130
139function processUndoAnnounce (actorUrl: string, announceActivity: ActivityAnnounce) { 131function processUndoAnnounce (byActor: ActorModel, announceActivity: ActivityAnnounce) {
140 return sequelizeTypescript.transaction(async t => { 132 return sequelizeTypescript.transaction(async t => {
141 const byActor = await ActorModel.loadByUrl(actorUrl, t)
142 if (!byActor) throw new Error('Unknown actor ' + actorUrl)
143
144 const share = await VideoShareModel.loadByUrl(announceActivity.id, t) 133 const share = await VideoShareModel.loadByUrl(announceActivity.id, t)
145 if (!share) throw new Error(`Unknown video share ${announceActivity.id}.`) 134 if (!share) throw new Error(`Unknown video share ${announceActivity.id}.`)
146 135
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index d3af1a181..ed3489ebf 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -6,27 +6,30 @@ import { sequelizeTypescript } from '../../../initializers'
6import { AccountModel } from '../../../models/account/account' 6import { AccountModel } from '../../../models/account/account'
7import { ActorModel } from '../../../models/activitypub/actor' 7import { ActorModel } from '../../../models/activitypub/actor'
8import { VideoChannelModel } from '../../../models/video/video-channel' 8import { VideoChannelModel } from '../../../models/video/video-channel'
9import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' 9import { fetchAvatarIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor'
10import { getOrCreateVideoAndAccountAndChannel, updateVideoFromAP, getOrCreateVideoChannelFromVideoObject } from '../videos' 10import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos'
11import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' 11import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' 12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
13import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 13import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
14import { createCacheFile, updateCacheFile } from '../cache-file' 14import { createCacheFile, updateCacheFile } from '../cache-file'
15 15
16async function processUpdateActivity (activity: ActivityUpdate) { 16async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) {
17 const actor = await getOrCreateActorAndServerAndModel(activity.actor)
18 const objectType = activity.object.type 17 const objectType = activity.object.type
19 18
20 if (objectType === 'Video') { 19 if (objectType === 'Video') {
21 return retryTransactionWrapper(processUpdateVideo, actor, activity) 20 return retryTransactionWrapper(processUpdateVideo, byActor, activity)
22 } 21 }
23 22
24 if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { 23 if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') {
25 return retryTransactionWrapper(processUpdateActor, actor, activity) 24 // We need more attributes
25 const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url)
26 return retryTransactionWrapper(processUpdateActor, byActorFull, activity)
26 } 27 }
27 28
28 if (objectType === 'CacheFile') { 29 if (objectType === 'CacheFile') {
29 return retryTransactionWrapper(processUpdateCacheFile, actor, activity) 30 // We need more attributes
31 const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url)
32 return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity)
30 } 33 }
31 34
32 return undefined 35 return undefined
@@ -48,10 +51,18 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
48 return undefined 51 return undefined
49 } 52 }
50 53
51 const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) 54 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id })
52 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) 55 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
53 56
54 return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to) 57 const updateOptions = {
58 video,
59 videoObject,
60 account: actor.Account,
61 channel: channelActor.VideoChannel,
62 updateViews: true,
63 overrideTo: activity.to
64 }
65 return updateVideoFromAP(updateOptions)
55} 66}
56 67
57async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) { 68async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) {
@@ -64,7 +75,7 @@ async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUp
64 75
65 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) 76 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
66 if (!redundancyModel) { 77 if (!redundancyModel) {
67 const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id) 78 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.id })
68 return createCacheFile(cacheFileObject, video, byActor) 79 return createCacheFile(cacheFileObject, video, byActor)
69 } 80 }
70 81
diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts
index da91675ce..b263f1ea2 100644
--- a/server/lib/activitypub/process/process.ts
+++ b/server/lib/activitypub/process/process.ts
@@ -11,8 +11,9 @@ import { processLikeActivity } from './process-like'
11import { processRejectActivity } from './process-reject' 11import { processRejectActivity } from './process-reject'
12import { processUndoActivity } from './process-undo' 12import { processUndoActivity } from './process-undo'
13import { processUpdateActivity } from './process-update' 13import { processUpdateActivity } from './process-update'
14import { getOrCreateActorAndServerAndModel } from '../actor'
14 15
15const processActivity: { [ P in ActivityType ]: (activity: Activity, inboxActor?: ActorModel) => Promise<any> } = { 16const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: ActorModel, inboxActor?: ActorModel) => Promise<any> } = {
16 Create: processCreateActivity, 17 Create: processCreateActivity,
17 Update: processUpdateActivity, 18 Update: processUpdateActivity,
18 Delete: processDeleteActivity, 19 Delete: processDeleteActivity,
@@ -25,7 +26,14 @@ const processActivity: { [ P in ActivityType ]: (activity: Activity, inboxActor?
25} 26}
26 27
27async function processActivities (activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel) { 28async function processActivities (activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel) {
29 const actorsCache: { [ url: string ]: ActorModel } = {}
30
28 for (const activity of activities) { 31 for (const activity of activities) {
32 if (!signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) {
33 logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type)
34 continue
35 }
36
29 const actorUrl = getActorUrl(activity.actor) 37 const actorUrl = getActorUrl(activity.actor)
30 38
31 // When we fetch remote data, we don't have signature 39 // When we fetch remote data, we don't have signature
@@ -34,6 +42,9 @@ async function processActivities (activities: Activity[], signatureActor?: Actor
34 continue 42 continue
35 } 43 }
36 44
45 const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl)
46 actorsCache[actorUrl] = byActor
47
37 const activityProcessor = processActivity[activity.type] 48 const activityProcessor = processActivity[activity.type]
38 if (activityProcessor === undefined) { 49 if (activityProcessor === undefined) {
39 logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id }) 50 logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id })
@@ -41,7 +52,7 @@ async function processActivities (activities: Activity[], signatureActor?: Actor
41 } 52 }
42 53
43 try { 54 try {
44 await activityProcessor(activity, inboxActor) 55 await activityProcessor(activity, byActor, inboxActor)
45 } catch (err) { 56 } catch (err) {
46 logger.warn('Cannot process activity %s.', activity.type, { err }) 57 logger.warn('Cannot process activity %s.', activity.type, { err })
47 } 58 }
diff --git a/server/lib/activitypub/send/send-announce.ts b/server/lib/activitypub/send/send-announce.ts
index f137217f8..cd0cab7ee 100644
--- a/server/lib/activitypub/send/send-announce.ts
+++ b/server/lib/activitypub/send/send-announce.ts
@@ -4,14 +4,14 @@ import { ActorModel } from '../../../models/activitypub/actor'
4import { VideoModel } from '../../../models/video/video' 4import { VideoModel } from '../../../models/video/video'
5import { VideoShareModel } from '../../../models/video/video-share' 5import { VideoShareModel } from '../../../models/video/video-share'
6import { broadcastToFollowers } from './utils' 6import { broadcastToFollowers } from './utils'
7import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' 7import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf } from '../audience'
8import { logger } from '../../../helpers/logger' 8import { logger } from '../../../helpers/logger'
9 9
10async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { 10async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
11 const announcedObject = video.url 11 const announcedObject = video.url
12 12
13 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 13 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
14 const audience = getObjectFollowersAudience(actorsInvolvedInVideo) 14 const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo)
15 15
16 const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience) 16 const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience)
17 17
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index 6f89b1a22..285edba3b 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -1,21 +1,13 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub' 2import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub'
3import { VideoPrivacy } from '../../../../shared/models/videos' 3import { VideoPrivacy } from '../../../../shared/models/videos'
4import { getServerActor } from '../../../helpers/utils'
5import { ActorModel } from '../../../models/activitypub/actor' 4import { ActorModel } from '../../../models/activitypub/actor'
6import { VideoModel } from '../../../models/video/video' 5import { VideoModel } from '../../../models/video/video'
7import { VideoAbuseModel } from '../../../models/video/video-abuse' 6import { VideoAbuseModel } from '../../../models/video/video-abuse'
8import { VideoCommentModel } from '../../../models/video/video-comment' 7import { VideoCommentModel } from '../../../models/video/video-comment'
9import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url' 8import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url'
10import { broadcastToActors, broadcastToFollowers, unicastTo } from './utils' 9import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
11import { 10import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience'
12 audiencify,
13 getActorsInvolvedInVideo,
14 getAudience,
15 getObjectFollowersAudience,
16 getVideoAudience,
17 getVideoCommentAudience
18} from '../audience'
19import { logger } from '../../../helpers/logger' 11import { logger } from '../../../helpers/logger'
20import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 12import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
21 13
@@ -40,6 +32,7 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
40 32
41 logger.info('Creating job to send video abuse %s.', url) 33 logger.info('Creating job to send video abuse %s.', url)
42 34
35 // Custom audience, we only send the abuse to the origin instance
43 const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } 36 const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
44 const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience) 37 const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience)
45 38
@@ -49,15 +42,15 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
49async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { 42async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) {
50 logger.info('Creating job to send file cache of %s.', fileRedundancy.url) 43 logger.info('Creating job to send file cache of %s.', fileRedundancy.url)
51 44
52 const redundancyObject = fileRedundancy.toActivityPubObject()
53
54 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) 45 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
55 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined) 46 const redundancyObject = fileRedundancy.toActivityPubObject()
56
57 const audience = getVideoAudience(video, actorsInvolvedInVideo)
58 const createActivity = buildCreateActivity(fileRedundancy.url, byActor, redundancyObject, audience)
59 47
60 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 48 return sendVideoRelatedCreateActivity({
49 byActor,
50 video,
51 url: fileRedundancy.url,
52 object: redundancyObject
53 })
61} 54}
62 55
63async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) { 56async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) {
@@ -70,6 +63,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
70 const commentObject = comment.toActivityPubObject(threadParentComments) 63 const commentObject = comment.toActivityPubObject(threadParentComments)
71 64
72 const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, t) 65 const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, t)
66 // Add the actor that commented too
73 actorsInvolvedInComment.push(byActor) 67 actorsInvolvedInComment.push(byActor)
74 68
75 const parentsCommentActors = threadParentComments.map(c => c.Account.Actor) 69 const parentsCommentActors = threadParentComments.map(c => c.Account.Actor)
@@ -78,7 +72,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
78 if (isOrigin) { 72 if (isOrigin) {
79 audience = getVideoCommentAudience(comment, threadParentComments, actorsInvolvedInComment, isOrigin) 73 audience = getVideoCommentAudience(comment, threadParentComments, actorsInvolvedInComment, isOrigin)
80 } else { 74 } else {
81 audience = getObjectFollowersAudience(actorsInvolvedInComment.concat(parentsCommentActors)) 75 audience = getAudienceFromFollowersOf(actorsInvolvedInComment.concat(parentsCommentActors))
82 } 76 }
83 77
84 const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience) 78 const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience)
@@ -103,24 +97,14 @@ async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transa
103 const url = getVideoViewActivityPubUrl(byActor, video) 97 const url = getVideoViewActivityPubUrl(byActor, video)
104 const viewActivity = buildViewActivity(byActor, video) 98 const viewActivity = buildViewActivity(byActor, video)
105 99
106 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 100 return sendVideoRelatedCreateActivity({
107 101 // Use the server actor to send the view
108 // Send to origin 102 byActor,
109 if (video.isOwned() === false) { 103 video,
110 const audience = getVideoAudience(video, actorsInvolvedInVideo) 104 url,
111 const createActivity = buildCreateActivity(url, byActor, viewActivity, audience) 105 object: viewActivity,
112 106 transaction: t
113 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 107 })
114 }
115
116 // Send to followers
117 const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
118 const createActivity = buildCreateActivity(url, byActor, viewActivity, audience)
119
120 // Use the server actor to send the view
121 const serverActor = await getServerActor()
122 const actorsException = [ byActor ]
123 return broadcastToFollowers(createActivity, serverActor, actorsInvolvedInVideo, t, actorsException)
124} 108}
125 109
126async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { 110async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
@@ -129,22 +113,13 @@ async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Tra
129 const url = getVideoDislikeActivityPubUrl(byActor, video) 113 const url = getVideoDislikeActivityPubUrl(byActor, video)
130 const dislikeActivity = buildDislikeActivity(byActor, video) 114 const dislikeActivity = buildDislikeActivity(byActor, video)
131 115
132 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 116 return sendVideoRelatedCreateActivity({
133 117 byActor,
134 // Send to origin 118 video,
135 if (video.isOwned() === false) { 119 url,
136 const audience = getVideoAudience(video, actorsInvolvedInVideo) 120 object: dislikeActivity,
137 const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience) 121 transaction: t
138 122 })
139 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
140 }
141
142 // Send to followers
143 const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
144 const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience)
145
146 const actorsException = [ byActor ]
147 return broadcastToFollowers(createActivity, byActor, actorsInvolvedInVideo, t, actorsException)
148} 123}
149 124
150function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { 125function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
@@ -189,3 +164,19 @@ export {
189 sendCreateVideoComment, 164 sendCreateVideoComment,
190 sendCreateCacheFile 165 sendCreateCacheFile
191} 166}
167
168// ---------------------------------------------------------------------------
169
170async function sendVideoRelatedCreateActivity (options: {
171 byActor: ActorModel,
172 video: VideoModel,
173 url: string,
174 object: any,
175 transaction?: Transaction
176}) {
177 const activityBuilder = (audience: ActivityAudience) => {
178 return buildCreateActivity(options.url, options.byActor, options.object, audience)
179 }
180
181 return sendVideoRelatedActivity(activityBuilder, options)
182}
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts
index 479182543..18969433a 100644
--- a/server/lib/activitypub/send/send-delete.ts
+++ b/server/lib/activitypub/send/send-delete.ts
@@ -5,21 +5,22 @@ import { VideoModel } from '../../../models/video/video'
5import { VideoCommentModel } from '../../../models/video/video-comment' 5import { VideoCommentModel } from '../../../models/video/video-comment'
6import { VideoShareModel } from '../../../models/video/video-share' 6import { VideoShareModel } from '../../../models/video/video-share'
7import { getDeleteActivityPubUrl } from '../url' 7import { getDeleteActivityPubUrl } from '../url'
8import { broadcastToActors, broadcastToFollowers, unicastTo } from './utils' 8import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
9import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience' 9import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience'
10import { logger } from '../../../helpers/logger' 10import { logger } from '../../../helpers/logger'
11 11
12async function sendDeleteVideo (video: VideoModel, t: Transaction) { 12async function sendDeleteVideo (video: VideoModel, transaction: Transaction) {
13 logger.info('Creating job to broadcast delete of video %s.', video.url) 13 logger.info('Creating job to broadcast delete of video %s.', video.url)
14 14
15 const url = getDeleteActivityPubUrl(video.url)
16 const byActor = video.VideoChannel.Account.Actor 15 const byActor = video.VideoChannel.Account.Actor
17 16
18 const activity = buildDeleteActivity(url, video.url, byActor) 17 const activityBuilder = (audience: ActivityAudience) => {
18 const url = getDeleteActivityPubUrl(video.url)
19 19
20 const actorsInvolved = await getActorsInvolvedInVideo(video, t) 20 return buildDeleteActivity(url, video.url, byActor, audience)
21 }
21 22
22 return broadcastToFollowers(activity, byActor, actorsInvolved, t) 23 return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction })
23} 24}
24 25
25async function sendDeleteActor (byActor: ActorModel, t: Transaction) { 26async function sendDeleteActor (byActor: ActorModel, t: Transaction) {
diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts
index a5408ac6a..89307acc6 100644
--- a/server/lib/activitypub/send/send-like.ts
+++ b/server/lib/activitypub/send/send-like.ts
@@ -3,31 +3,20 @@ import { ActivityAudience, ActivityLike } from '../../../../shared/models/activi
3import { ActorModel } from '../../../models/activitypub/actor' 3import { ActorModel } from '../../../models/activitypub/actor'
4import { VideoModel } from '../../../models/video/video' 4import { VideoModel } from '../../../models/video/video'
5import { getVideoLikeActivityPubUrl } from '../url' 5import { getVideoLikeActivityPubUrl } from '../url'
6import { broadcastToFollowers, unicastTo } from './utils' 6import { sendVideoRelatedActivity } from './utils'
7import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience' 7import { audiencify, getAudience } from '../audience'
8import { logger } from '../../../helpers/logger' 8import { logger } from '../../../helpers/logger'
9 9
10async function sendLike (byActor: ActorModel, video: VideoModel, t: Transaction) { 10async function sendLike (byActor: ActorModel, video: VideoModel, t: Transaction) {
11 logger.info('Creating job to like %s.', video.url) 11 logger.info('Creating job to like %s.', video.url)
12 12
13 const url = getVideoLikeActivityPubUrl(byActor, video) 13 const activityBuilder = (audience: ActivityAudience) => {
14 const url = getVideoLikeActivityPubUrl(byActor, video)
14 15
15 const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 16 return buildLikeActivity(url, byActor, video, audience)
16
17 // Send to origin
18 if (video.isOwned() === false) {
19 const audience = getVideoAudience(video, accountsInvolvedInVideo)
20 const data = buildLikeActivity(url, byActor, video, audience)
21
22 return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
23 } 17 }
24 18
25 // Send to followers 19 return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t })
26 const audience = getObjectFollowersAudience(accountsInvolvedInVideo)
27 const activity = buildLikeActivity(url, byActor, video, audience)
28
29 const followersException = [ byActor ]
30 return broadcastToFollowers(activity, byActor, accountsInvolvedInVideo, t, followersException)
31} 20}
32 21
33function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike { 22function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike {
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts
index a50673c79..5236d2cb3 100644
--- a/server/lib/activitypub/send/send-undo.ts
+++ b/server/lib/activitypub/send/send-undo.ts
@@ -11,8 +11,8 @@ import { ActorModel } from '../../../models/activitypub/actor'
11import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 11import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
12import { VideoModel } from '../../../models/video/video' 12import { VideoModel } from '../../../models/video/video'
13import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' 13import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url'
14import { broadcastToFollowers, unicastTo } from './utils' 14import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
15import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience' 15import { audiencify, getAudience } from '../audience'
16import { buildCreateActivity, buildDislikeActivity } from './send-create' 16import { buildCreateActivity, buildDislikeActivity } from './send-create'
17import { buildFollowActivity } from './send-follow' 17import { buildFollowActivity } from './send-follow'
18import { buildLikeActivity } from './send-like' 18import { buildLikeActivity } from './send-like'
@@ -39,79 +39,44 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
39 return unicastTo(undoActivity, me, following.inboxUrl) 39 return unicastTo(undoActivity, me, following.inboxUrl)
40} 40}
41 41
42async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) { 42async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
43 logger.info('Creating job to undo a like of video %s.', video.url) 43 logger.info('Creating job to undo announce %s.', videoShare.url)
44 44
45 const likeUrl = getVideoLikeActivityPubUrl(byActor, video) 45 const undoUrl = getUndoActivityPubUrl(videoShare.url)
46 const undoUrl = getUndoActivityPubUrl(likeUrl)
47 46
48 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 47 const { activity: announceActivity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t)
49 const likeActivity = buildLikeActivity(likeUrl, byActor, video) 48 const undoActivity = undoActivityData(undoUrl, byActor, announceActivity)
50 49
51 // Send to origin 50 const followersException = [ byActor ]
52 if (video.isOwned() === false) { 51 return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
53 const audience = getVideoAudience(video, actorsInvolvedInVideo) 52}
54 const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience)
55 53
56 return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 54async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) {
57 } 55 logger.info('Creating job to undo a like of video %s.', video.url)
58 56
59 const audience = getObjectFollowersAudience(actorsInvolvedInVideo) 57 const likeUrl = getVideoLikeActivityPubUrl(byActor, video)
60 const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience) 58 const likeActivity = buildLikeActivity(likeUrl, byActor, video)
61 59
62 const followersException = [ byActor ] 60 return sendUndoVideoRelatedActivity({ byActor, video, url: likeUrl, activity: likeActivity, transaction: t })
63 return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
64} 61}
65 62
66async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { 63async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
67 logger.info('Creating job to undo a dislike of video %s.', video.url) 64 logger.info('Creating job to undo a dislike of video %s.', video.url)
68 65
69 const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) 66 const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video)
70 const undoUrl = getUndoActivityPubUrl(dislikeUrl)
71
72 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
73 const dislikeActivity = buildDislikeActivity(byActor, video) 67 const dislikeActivity = buildDislikeActivity(byActor, video)
74 const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) 68 const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity)
75 69
76 if (video.isOwned() === false) { 70 return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t })
77 const audience = getVideoAudience(video, actorsInvolvedInVideo)
78 const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity, audience)
79
80 return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
81 }
82
83 const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity)
84
85 const followersException = [ byActor ]
86 return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
87}
88
89async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
90 logger.info('Creating job to undo announce %s.', videoShare.url)
91
92 const undoUrl = getUndoActivityPubUrl(videoShare.url)
93
94 const { activity: announceActivity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t)
95 const undoActivity = undoActivityData(undoUrl, byActor, announceActivity)
96
97 const followersException = [ byActor ]
98 return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
99} 71}
100 72
101async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { 73async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
102 logger.info('Creating job to undo cache file %s.', redundancyModel.url) 74 logger.info('Creating job to undo cache file %s.', redundancyModel.url)
103 75
104 const undoUrl = getUndoActivityPubUrl(redundancyModel.url)
105
106 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 76 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
107 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
108
109 const audience = getVideoAudience(video, actorsInvolvedInVideo)
110 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) 77 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())
111 78
112 const undoActivity = undoActivityData(undoUrl, byActor, createActivity, audience) 79 return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t })
113
114 return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
115} 80}
116 81
117// --------------------------------------------------------------------------- 82// ---------------------------------------------------------------------------
@@ -144,3 +109,19 @@ function undoActivityData (
144 audience 109 audience
145 ) 110 )
146} 111}
112
113async function sendUndoVideoRelatedActivity (options: {
114 byActor: ActorModel,
115 video: VideoModel,
116 url: string,
117 activity: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce,
118 transaction: Transaction
119}) {
120 const activityBuilder = (audience: ActivityAudience) => {
121 const undoUrl = getUndoActivityPubUrl(options.url)
122
123 return undoActivityData(undoUrl, options.byActor, options.activity, audience)
124 }
125
126 return sendVideoRelatedActivity(activityBuilder, options)
127}
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index 605473338..ec46789b7 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -7,8 +7,8 @@ import { VideoModel } from '../../../models/video/video'
7import { VideoChannelModel } from '../../../models/video/video-channel' 7import { VideoChannelModel } from '../../../models/video/video-channel'
8import { VideoShareModel } from '../../../models/video/video-share' 8import { VideoShareModel } from '../../../models/video/video-share'
9import { getUpdateActivityPubUrl } from '../url' 9import { getUpdateActivityPubUrl } from '../url'
10import { broadcastToFollowers, unicastTo } from './utils' 10import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
11import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' 11import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf } from '../audience'
12import { logger } from '../../../helpers/logger' 12import { logger } from '../../../helpers/logger'
13import { VideoCaptionModel } from '../../../models/video/video-caption' 13import { VideoCaptionModel } from '../../../models/video/video-caption'
14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
@@ -61,16 +61,16 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { 61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
62 logger.info('Creating job to update cache file %s.', redundancyModel.url) 62 logger.info('Creating job to update cache file %s.', redundancyModel.url)
63 63
64 const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString())
65 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 64 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
66 65
67 const redundancyObject = redundancyModel.toActivityPubObject() 66 const activityBuilder = (audience: ActivityAudience) => {
67 const redundancyObject = redundancyModel.toActivityPubObject()
68 const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString())
68 69
69 const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined) 70 return buildUpdateActivity(url, byActor, redundancyObject, audience)
70 const audience = getObjectFollowersAudience(accountsInvolvedInVideo) 71 }
71 72
72 const updateActivity = buildUpdateActivity(url, byActor, redundancyObject, audience) 73 return sendVideoRelatedActivity(activityBuilder, { byActor, video })
73 return unicastTo(updateActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
74} 74}
75 75
76// --------------------------------------------------------------------------- 76// ---------------------------------------------------------------------------
diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts
index c20c15633..69706e620 100644
--- a/server/lib/activitypub/send/utils.ts
+++ b/server/lib/activitypub/send/utils.ts
@@ -1,13 +1,36 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { Activity } from '../../../../shared/models/activitypub' 2import { Activity, ActivityAudience } from '../../../../shared/models/activitypub'
3import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
4import { ActorModel } from '../../../models/activitypub/actor' 4import { ActorModel } from '../../../models/activitypub/actor'
5import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 5import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
6import { JobQueue } from '../../job-queue' 6import { JobQueue } from '../../job-queue'
7import { VideoModel } from '../../../models/video/video' 7import { VideoModel } from '../../../models/video/video'
8import { getActorsInvolvedInVideo } from '../audience' 8import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
9import { getServerActor } from '../../../helpers/utils' 9import { getServerActor } from '../../../helpers/utils'
10 10
11async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
12 byActor: ActorModel,
13 video: VideoModel,
14 transaction?: Transaction
15}) {
16 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(options.video, options.transaction)
17
18 // Send to origin
19 if (options.video.isOwned() === false) {
20 const audience = getRemoteVideoAudience(options.video, actorsInvolvedInVideo)
21 const activity = activityBuilder(audience)
22
23 return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl)
24 }
25
26 // Send to followers
27 const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo)
28 const activity = activityBuilder(audience)
29
30 const actorsException = [ options.byActor ]
31 return broadcastToFollowers(activity, options.byActor, actorsInvolvedInVideo, options.transaction, actorsException)
32}
33
11async function forwardVideoRelatedActivity ( 34async function forwardVideoRelatedActivity (
12 activity: Activity, 35 activity: Activity,
13 t: Transaction, 36 t: Transaction,
@@ -110,7 +133,8 @@ export {
110 unicastTo, 133 unicastTo,
111 forwardActivity, 134 forwardActivity,
112 broadcastToActors, 135 broadcastToActors,
113 forwardVideoRelatedActivity 136 forwardVideoRelatedActivity,
137 sendVideoRelatedActivity
114} 138}
115 139
116// --------------------------------------------------------------------------- 140// ---------------------------------------------------------------------------
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index ffbd3a64e..4ca8bf659 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -94,7 +94,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) {
94 try { 94 try {
95 // Maybe it's a reply to a video? 95 // Maybe it's a reply to a video?
96 // If yes, it's done: we resolved all the thread 96 // If yes, it's done: we resolved all the thread
97 const { video } = await getOrCreateVideoAndAccountAndChannel(url) 97 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url })
98 98
99 if (comments.length !== 0) { 99 if (comments.length !== 0) {
100 const firstReply = comments[ comments.length - 1 ] 100 const firstReply = comments[ comments.length - 1 ]
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 783f78d3e..48c0e0a5c 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -3,7 +3,7 @@ import * as sequelize from 'sequelize'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import { join } from 'path' 4import { join } from 'path'
5import * as request from 'request' 5import * as request from 'request'
6import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index' 6import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index'
7import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 7import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
8import { VideoPrivacy } from '../../../shared/models/videos' 8import { VideoPrivacy } from '../../../shared/models/videos'
9import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' 9import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
@@ -28,6 +28,7 @@ import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub
28import { createRates } from './video-rates' 28import { createRates } from './video-rates'
29import { addVideoShares, shareVideoByServerAndChannel } from './share' 29import { addVideoShares, shareVideoByServerAndChannel } from './share'
30import { AccountModel } from '../../models/account/account' 30import { AccountModel } from '../../models/account/account'
31import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
31 32
32async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 33async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
33 // If the video is not private and published, we federate it 34 // If the video is not private and published, we federate it
@@ -50,18 +51,29 @@ async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, tr
50 } 51 }
51} 52}
52 53
53function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { 54async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
54 const host = video.VideoChannel.Account.Actor.Server.host 55 const options = {
56 uri: videoUrl,
57 method: 'GET',
58 json: true,
59 activityPub: true
60 }
55 61
56 // We need to provide a callback, if no we could have an uncaught exception 62 logger.info('Fetching remote video %s.', videoUrl)
57 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { 63
58 if (err) reject(err) 64 const { response, body } = await doRequest(options)
59 }) 65
66 if (sanitizeAndCheckVideoTorrentObject(body) === false) {
67 logger.debug('Remote video JSON is not valid.', { body })
68 return { response, videoObject: undefined }
69 }
70
71 return { response, videoObject: body }
60} 72}
61 73
62async function fetchRemoteVideoDescription (video: VideoModel) { 74async function fetchRemoteVideoDescription (video: VideoModel) {
63 const host = video.VideoChannel.Account.Actor.Server.host 75 const host = video.VideoChannel.Account.Actor.Server.host
64 const path = video.getDescriptionPath() 76 const path = video.getDescriptionAPIPath()
65 const options = { 77 const options = {
66 uri: REMOTE_SCHEME.HTTP + '://' + host + path, 78 uri: REMOTE_SCHEME.HTTP + '://' + host + path,
67 json: true 79 json: true
@@ -71,6 +83,15 @@ async function fetchRemoteVideoDescription (video: VideoModel) {
71 return body.description ? body.description : '' 83 return body.description ? body.description : ''
72} 84}
73 85
86function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
87 const host = video.VideoChannel.Account.Actor.Server.host
88
89 // We need to provide a callback, if no we could have an uncaught exception
90 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => {
91 if (err) reject(err)
92 })
93}
94
74function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { 95function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
75 const thumbnailName = video.getThumbnailName() 96 const thumbnailName = video.getThumbnailName()
76 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) 97 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
@@ -82,144 +103,11 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject)
82 return doRequestAndSaveToFile(options, thumbnailPath) 103 return doRequestAndSaveToFile(options, thumbnailPath)
83} 104}
84 105
85async function videoActivityObjectToDBAttributes (
86 videoChannel: VideoChannelModel,
87 videoObject: VideoTorrentObject,
88 to: string[] = []
89) {
90 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
91 const duration = videoObject.duration.replace(/[^\d]+/, '')
92
93 let language: string | undefined
94 if (videoObject.language) {
95 language = videoObject.language.identifier
96 }
97
98 let category: number | undefined
99 if (videoObject.category) {
100 category = parseInt(videoObject.category.identifier, 10)
101 }
102
103 let licence: number | undefined
104 if (videoObject.licence) {
105 licence = parseInt(videoObject.licence.identifier, 10)
106 }
107
108 const description = videoObject.content || null
109 const support = videoObject.support || null
110
111 return {
112 name: videoObject.name,
113 uuid: videoObject.uuid,
114 url: videoObject.id,
115 category,
116 licence,
117 language,
118 description,
119 support,
120 nsfw: videoObject.sensitive,
121 commentsEnabled: videoObject.commentsEnabled,
122 waitTranscoding: videoObject.waitTranscoding,
123 state: videoObject.state,
124 channelId: videoChannel.id,
125 duration: parseInt(duration, 10),
126 createdAt: new Date(videoObject.published),
127 publishedAt: new Date(videoObject.published),
128 // FIXME: updatedAt does not seems to be considered by Sequelize
129 updatedAt: new Date(videoObject.updated),
130 views: videoObject.views,
131 likes: 0,
132 dislikes: 0,
133 remote: true,
134 privacy
135 }
136}
137
138function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
139 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
140
141 if (fileUrls.length === 0) {
142 throw new Error('Cannot find video files for ' + videoCreated.url)
143 }
144
145 const attributes: VideoFileModel[] = []
146 for (const fileUrl of fileUrls) {
147 // Fetch associated magnet uri
148 const magnet = videoObject.url.find(u => {
149 return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height
150 })
151
152 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
153
154 const parsed = magnetUtil.decode(magnet.href)
155 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
156 throw new Error('Cannot parse magnet URI ' + magnet.href)
157 }
158
159 const attribute = {
160 extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
161 infoHash: parsed.infoHash,
162 resolution: fileUrl.height,
163 size: fileUrl.size,
164 videoId: videoCreated.id,
165 fps: fileUrl.fps
166 } as VideoFileModel
167 attributes.push(attribute)
168 }
169
170 return attributes
171}
172
173function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { 106function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
174 const channel = videoObject.attributedTo.find(a => a.type === 'Group') 107 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
175 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) 108 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
176 109
177 return getOrCreateActorAndServerAndModel(channel.id) 110 return getOrCreateActorAndServerAndModel(channel.id, 'all')
178}
179
180async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
181 logger.debug('Adding remote video %s.', videoObject.id)
182
183 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
184 const sequelizeOptions = { transaction: t }
185
186 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
187 const video = VideoModel.build(videoData)
188
189 const videoCreated = await video.save(sequelizeOptions)
190
191 // Process files
192 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
193 if (videoFileAttributes.length === 0) {
194 throw new Error('Cannot find valid files for video %s ' + videoObject.url)
195 }
196
197 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
198 await Promise.all(videoFilePromises)
199
200 // Process tags
201 const tags = videoObject.tag.map(t => t.name)
202 const tagInstances = await TagModel.findOrCreateTags(tags, t)
203 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
204
205 // Process captions
206 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
207 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
208 })
209 await Promise.all(videoCaptionsPromises)
210
211 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
212
213 videoCreated.VideoChannel = channelActor.VideoChannel
214 return videoCreated
215 })
216
217 const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
218 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
219
220 if (waitThumbnail === true) await p
221
222 return videoCreated
223} 111}
224 112
225type SyncParam = { 113type SyncParam = {
@@ -230,28 +118,7 @@ type SyncParam = {
230 thumbnail: boolean 118 thumbnail: boolean
231 refreshVideo: boolean 119 refreshVideo: boolean
232} 120}
233async function getOrCreateVideoAndAccountAndChannel ( 121async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
234 videoObject: VideoTorrentObject | string,
235 syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
236) {
237 const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
238
239 let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
240 if (videoFromDatabase) {
241 const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase)
242 if (syncParam.refreshVideo === true) videoFromDatabase = await p
243
244 return { video: videoFromDatabase }
245 }
246
247 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
248 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
249
250 const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
251 const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
252
253 // Process outside the transaction because we could fetch remote data
254
255 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) 122 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
256 123
257 const jobPayloads: ActivitypubHttpFetcherPayload[] = [] 124 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
@@ -285,64 +152,56 @@ async function getOrCreateVideoAndAccountAndChannel (
285 } 152 }
286 153
287 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) 154 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
288
289 return { video }
290} 155}
291 156
292async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { 157async function getOrCreateVideoAndAccountAndChannel (options: {
293 const options = { 158 videoObject: VideoTorrentObject | string,
294 uri: videoUrl, 159 syncParam?: SyncParam,
295 method: 'GET', 160 fetchType?: VideoFetchByUrlType,
296 json: true, 161 refreshViews?: boolean
297 activityPub: true 162}) {
298 } 163 // Default params
299 164 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
300 logger.info('Fetching remote video %s.', videoUrl) 165 const fetchType = options.fetchType || 'all'
301 166 const refreshViews = options.refreshViews || false
302 const { response, body } = await doRequest(options) 167
168 // Get video url
169 const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id
170
171 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
172 if (videoFromDatabase) {
173 const refreshOptions = {
174 video: videoFromDatabase,
175 fetchedType: fetchType,
176 syncParam,
177 refreshViews
178 }
179 const p = retryTransactionWrapper(refreshVideoIfNeeded, refreshOptions)
180 if (syncParam.refreshVideo === true) videoFromDatabase = await p
303 181
304 if (sanitizeAndCheckVideoTorrentObject(body) === false) { 182 return { video: videoFromDatabase }
305 logger.debug('Remote video JSON is not valid.', { body })
306 return { response, videoObject: undefined }
307 } 183 }
308 184
309 return { response, videoObject: body } 185 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
310} 186 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
311
312async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
313 if (!video.isOutdated()) return video
314
315 try {
316 const { response, videoObject } = await fetchRemoteVideo(video.url)
317 if (response.statusCode === 404) {
318 // Video does not exist anymore
319 await video.destroy()
320 return undefined
321 }
322 187
323 if (videoObject === undefined) { 188 const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
324 logger.warn('Cannot refresh remote video: invalid body.') 189 const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
325 return video
326 }
327 190
328 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) 191 await syncVideoExternalAttributes(video, fetchedVideo, syncParam)
329 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
330 192
331 return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) 193 return { video }
332 } catch (err) {
333 logger.warn('Cannot refresh video.', { err })
334 return video
335 }
336} 194}
337 195
338async function updateVideoFromAP ( 196async function updateVideoFromAP (options: {
339 video: VideoModel, 197 video: VideoModel,
340 videoObject: VideoTorrentObject, 198 videoObject: VideoTorrentObject,
341 account: AccountModel, 199 account: AccountModel,
342 channel: VideoChannelModel, 200 channel: VideoChannelModel,
201 updateViews: boolean,
343 overrideTo?: string[] 202 overrideTo?: string[]
344) { 203}) {
345 logger.debug('Updating remote video "%s".', videoObject.uuid) 204 logger.debug('Updating remote video "%s".', options.videoObject.uuid)
346 let videoFieldsSave: any 205 let videoFieldsSave: any
347 206
348 try { 207 try {
@@ -351,72 +210,72 @@ async function updateVideoFromAP (
351 transaction: t 210 transaction: t
352 } 211 }
353 212
354 videoFieldsSave = video.toJSON() 213 videoFieldsSave = options.video.toJSON()
355 214
356 // Check actor has the right to update the video 215 // Check actor has the right to update the video
357 const videoChannel = video.VideoChannel 216 const videoChannel = options.video.VideoChannel
358 if (videoChannel.Account.id !== account.id) { 217 if (videoChannel.Account.id !== options.account.id) {
359 throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url) 218 throw new Error('Account ' + options.account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
360 } 219 }
361 220
362 const to = overrideTo ? overrideTo : videoObject.to 221 const to = options.overrideTo ? options.overrideTo : options.videoObject.to
363 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to) 222 const videoData = await videoActivityObjectToDBAttributes(options.channel, options.videoObject, to)
364 video.set('name', videoData.name) 223 options.video.set('name', videoData.name)
365 video.set('uuid', videoData.uuid) 224 options.video.set('uuid', videoData.uuid)
366 video.set('url', videoData.url) 225 options.video.set('url', videoData.url)
367 video.set('category', videoData.category) 226 options.video.set('category', videoData.category)
368 video.set('licence', videoData.licence) 227 options.video.set('licence', videoData.licence)
369 video.set('language', videoData.language) 228 options.video.set('language', videoData.language)
370 video.set('description', videoData.description) 229 options.video.set('description', videoData.description)
371 video.set('support', videoData.support) 230 options.video.set('support', videoData.support)
372 video.set('nsfw', videoData.nsfw) 231 options.video.set('nsfw', videoData.nsfw)
373 video.set('commentsEnabled', videoData.commentsEnabled) 232 options.video.set('commentsEnabled', videoData.commentsEnabled)
374 video.set('waitTranscoding', videoData.waitTranscoding) 233 options.video.set('waitTranscoding', videoData.waitTranscoding)
375 video.set('state', videoData.state) 234 options.video.set('state', videoData.state)
376 video.set('duration', videoData.duration) 235 options.video.set('duration', videoData.duration)
377 video.set('createdAt', videoData.createdAt) 236 options.video.set('createdAt', videoData.createdAt)
378 video.set('publishedAt', videoData.publishedAt) 237 options.video.set('publishedAt', videoData.publishedAt)
379 video.set('views', videoData.views) 238 options.video.set('privacy', videoData.privacy)
380 video.set('privacy', videoData.privacy) 239 options.video.set('channelId', videoData.channelId)
381 video.set('channelId', videoData.channelId) 240
382 241 if (options.updateViews === true) options.video.set('views', videoData.views)
383 await video.save(sequelizeOptions) 242 await options.video.save(sequelizeOptions)
384 243
385 // Don't block on request 244 // Don't block on request
386 generateThumbnailFromUrl(video, videoObject.icon) 245 generateThumbnailFromUrl(options.video, options.videoObject.icon)
387 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) 246 .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }))
388 247
389 // Remove old video files 248 // Remove old video files
390 const videoFileDestroyTasks: Bluebird<void>[] = [] 249 const videoFileDestroyTasks: Bluebird<void>[] = []
391 for (const videoFile of video.VideoFiles) { 250 for (const videoFile of options.video.VideoFiles) {
392 videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) 251 videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
393 } 252 }
394 await Promise.all(videoFileDestroyTasks) 253 await Promise.all(videoFileDestroyTasks)
395 254
396 const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject) 255 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject)
397 const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) 256 const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
398 await Promise.all(tasks) 257 await Promise.all(tasks)
399 258
400 // Update Tags 259 // Update Tags
401 const tags = videoObject.tag.map(tag => tag.name) 260 const tags = options.videoObject.tag.map(tag => tag.name)
402 const tagInstances = await TagModel.findOrCreateTags(tags, t) 261 const tagInstances = await TagModel.findOrCreateTags(tags, t)
403 await video.$set('Tags', tagInstances, sequelizeOptions) 262 await options.video.$set('Tags', tagInstances, sequelizeOptions)
404 263
405 // Update captions 264 // Update captions
406 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t) 265 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t)
407 266
408 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { 267 const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => {
409 return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t) 268 return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t)
410 }) 269 })
411 await Promise.all(videoCaptionsPromises) 270 await Promise.all(videoCaptionsPromises)
412 }) 271 })
413 272
414 logger.info('Remote video with uuid %s updated', videoObject.uuid) 273 logger.info('Remote video with uuid %s updated', options.videoObject.uuid)
415 274
416 return updatedVideo 275 return updatedVideo
417 } catch (err) { 276 } catch (err) {
418 if (video !== undefined && videoFieldsSave !== undefined) { 277 if (options.video !== undefined && videoFieldsSave !== undefined) {
419 resetSequelizeInstance(video, videoFieldsSave) 278 resetSequelizeInstance(options.video, videoFieldsSave)
420 } 279 }
421 280
422 // This is just a debug because we will retry the insert 281 // This is just a debug because we will retry the insert
@@ -433,12 +292,7 @@ export {
433 fetchRemoteVideoStaticFile, 292 fetchRemoteVideoStaticFile,
434 fetchRemoteVideoDescription, 293 fetchRemoteVideoDescription,
435 generateThumbnailFromUrl, 294 generateThumbnailFromUrl,
436 videoActivityObjectToDBAttributes, 295 getOrCreateVideoChannelFromVideoObject
437 videoFileActivityUrlToDBAttributes,
438 createVideo,
439 getOrCreateVideoChannelFromVideoObject,
440 addVideoShares,
441 createRates
442} 296}
443 297
444// --------------------------------------------------------------------------- 298// ---------------------------------------------------------------------------
@@ -448,3 +302,178 @@ function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideo
448 302
449 return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') 303 return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/')
450} 304}
305
306async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
307 logger.debug('Adding remote video %s.', videoObject.id)
308
309 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
310 const sequelizeOptions = { transaction: t }
311
312 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
313 const video = VideoModel.build(videoData)
314
315 const videoCreated = await video.save(sequelizeOptions)
316
317 // Process files
318 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
319 if (videoFileAttributes.length === 0) {
320 throw new Error('Cannot find valid files for video %s ' + videoObject.url)
321 }
322
323 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
324 await Promise.all(videoFilePromises)
325
326 // Process tags
327 const tags = videoObject.tag.map(t => t.name)
328 const tagInstances = await TagModel.findOrCreateTags(tags, t)
329 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
330
331 // Process captions
332 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
333 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
334 })
335 await Promise.all(videoCaptionsPromises)
336
337 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
338
339 videoCreated.VideoChannel = channelActor.VideoChannel
340 return videoCreated
341 })
342
343 const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
344 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
345
346 if (waitThumbnail === true) await p
347
348 return videoCreated
349}
350
351async function refreshVideoIfNeeded (options: {
352 video: VideoModel,
353 fetchedType: VideoFetchByUrlType,
354 syncParam: SyncParam,
355 refreshViews: boolean
356}): Promise<VideoModel> {
357 if (!options.video.isOutdated()) return options.video
358
359 // We need more attributes if the argument video was fetched with not enough joints
360 const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
361
362 try {
363 const { response, videoObject } = await fetchRemoteVideo(video.url)
364 if (response.statusCode === 404) {
365 // Video does not exist anymore
366 await video.destroy()
367 return undefined
368 }
369
370 if (videoObject === undefined) {
371 logger.warn('Cannot refresh remote video: invalid body.')
372 return video
373 }
374
375 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
376 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
377
378 const updateOptions = {
379 video,
380 videoObject,
381 account,
382 channel: channelActor.VideoChannel,
383 updateViews: options.refreshViews
384 }
385 await updateVideoFromAP(updateOptions)
386 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
387 } catch (err) {
388 logger.warn('Cannot refresh video.', { err })
389 return video
390 }
391}
392
393async function videoActivityObjectToDBAttributes (
394 videoChannel: VideoChannelModel,
395 videoObject: VideoTorrentObject,
396 to: string[] = []
397) {
398 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
399 const duration = videoObject.duration.replace(/[^\d]+/, '')
400
401 let language: string | undefined
402 if (videoObject.language) {
403 language = videoObject.language.identifier
404 }
405
406 let category: number | undefined
407 if (videoObject.category) {
408 category = parseInt(videoObject.category.identifier, 10)
409 }
410
411 let licence: number | undefined
412 if (videoObject.licence) {
413 licence = parseInt(videoObject.licence.identifier, 10)
414 }
415
416 const description = videoObject.content || null
417 const support = videoObject.support || null
418
419 return {
420 name: videoObject.name,
421 uuid: videoObject.uuid,
422 url: videoObject.id,
423 category,
424 licence,
425 language,
426 description,
427 support,
428 nsfw: videoObject.sensitive,
429 commentsEnabled: videoObject.commentsEnabled,
430 waitTranscoding: videoObject.waitTranscoding,
431 state: videoObject.state,
432 channelId: videoChannel.id,
433 duration: parseInt(duration, 10),
434 createdAt: new Date(videoObject.published),
435 publishedAt: new Date(videoObject.published),
436 // FIXME: updatedAt does not seems to be considered by Sequelize
437 updatedAt: new Date(videoObject.updated),
438 views: videoObject.views,
439 likes: 0,
440 dislikes: 0,
441 remote: true,
442 privacy
443 }
444}
445
446function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
447 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
448
449 if (fileUrls.length === 0) {
450 throw new Error('Cannot find video files for ' + videoCreated.url)
451 }
452
453 const attributes: VideoFileModel[] = []
454 for (const fileUrl of fileUrls) {
455 // Fetch associated magnet uri
456 const magnet = videoObject.url.find(u => {
457 return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height
458 })
459
460 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
461
462 const parsed = magnetUtil.decode(magnet.href)
463 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
464 throw new Error('Cannot parse magnet URI ' + magnet.href)
465 }
466
467 const attribute = {
468 extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
469 infoHash: parsed.infoHash,
470 resolution: fileUrl.height,
471 size: fileUrl.size,
472 videoId: videoCreated.id,
473 fps: fileUrl.fps
474 } as VideoFileModel
475 attributes.push(attribute)
476 }
477
478 return attributes
479}
diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts
index 5cfb81fc7..14f0a05f5 100644
--- a/server/lib/avatar.ts
+++ b/server/lib/avatar.ts
@@ -3,23 +3,18 @@ import { sendUpdateActor } from './activitypub/send'
3import { AVATARS_SIZE, CONFIG, sequelizeTypescript } from '../initializers' 3import { AVATARS_SIZE, CONFIG, sequelizeTypescript } from '../initializers'
4import { updateActorAvatarInstance } from './activitypub' 4import { updateActorAvatarInstance } from './activitypub'
5import { processImage } from '../helpers/image-utils' 5import { processImage } from '../helpers/image-utils'
6import { ActorModel } from '../models/activitypub/actor'
7import { AccountModel } from '../models/account/account' 6import { AccountModel } from '../models/account/account'
8import { VideoChannelModel } from '../models/video/video-channel' 7import { VideoChannelModel } from '../models/video/video-channel'
9import { extname, join } from 'path' 8import { extname, join } from 'path'
10 9
11async function updateActorAvatarFile ( 10async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) {
12 avatarPhysicalFile: Express.Multer.File,
13 actor: ActorModel,
14 accountOrChannel: AccountModel | VideoChannelModel
15) {
16 const extension = extname(avatarPhysicalFile.filename) 11 const extension = extname(avatarPhysicalFile.filename)
17 const avatarName = actor.uuid + extension 12 const avatarName = accountOrChannel.Actor.uuid + extension
18 const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) 13 const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
19 await processImage(avatarPhysicalFile, destination, AVATARS_SIZE) 14 await processImage(avatarPhysicalFile, destination, AVATARS_SIZE)
20 15
21 return sequelizeTypescript.transaction(async t => { 16 return sequelizeTypescript.transaction(async t => {
22 const updatedActor = await updateActorAvatarInstance(actor, avatarName, t) 17 const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarName, t)
23 await updatedActor.save({ transaction: t }) 18 await updatedActor.save({ transaction: t })
24 19
25 await sendUpdateActor(accountOrChannel, t) 20 await sendUpdateActor(accountOrChannel, t)
diff --git a/server/lib/cache/videos-caption-cache.ts b/server/lib/cache/videos-caption-cache.ts
index 380d42b2c..f240affbc 100644
--- a/server/lib/cache/videos-caption-cache.ts
+++ b/server/lib/cache/videos-caption-cache.ts
@@ -38,7 +38,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
38 if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.') 38 if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.')
39 39
40 // Used to fetch the path 40 // Used to fetch the path
41 const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(videoId) 41 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
42 if (!video) return undefined 42 if (!video) return undefined
43 43
44 const remoteStaticPath = videoCaption.getCaptionStaticPath() 44 const remoteStaticPath = videoCaption.getCaptionStaticPath()
diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/cache/videos-preview-cache.ts
index 22b6d9cb0..a5d6f5b62 100644
--- a/server/lib/cache/videos-preview-cache.ts
+++ b/server/lib/cache/videos-preview-cache.ts
@@ -16,7 +16,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
16 } 16 }
17 17
18 async getFilePath (videoUUID: string) { 18 async getFilePath (videoUUID: string) {
19 const video = await VideoModel.loadByUUID(videoUUID) 19 const video = await VideoModel.loadByUUIDWithFile(videoUUID)
20 if (!video) return undefined 20 if (!video) return undefined
21 21
22 if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()) 22 if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
@@ -25,7 +25,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
25 } 25 }
26 26
27 protected async loadRemoteFile (key: string) { 27 protected async loadRemoteFile (key: string) {
28 const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(key) 28 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(key)
29 if (!video) return undefined 29 if (!video) return undefined
30 30
31 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') 31 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts
index a69e09c32..fc013e0c3 100644
--- a/server/lib/client-html.ts
+++ b/server/lib/client-html.ts
@@ -8,6 +8,7 @@ import { VideoModel } from '../models/video/video'
8import * as validator from 'validator' 8import * as validator from 'validator'
9import { VideoPrivacy } from '../../shared/models/videos' 9import { VideoPrivacy } from '../../shared/models/videos'
10import { readFile } from 'fs-extra' 10import { readFile } from 'fs-extra'
11import { getActivityStreamDuration } from '../models/video/video-format-utils'
11 12
12export class ClientHtml { 13export class ClientHtml {
13 14
@@ -38,10 +39,8 @@ export class ClientHtml {
38 let videoPromise: Bluebird<VideoModel> 39 let videoPromise: Bluebird<VideoModel>
39 40
40 // Let Angular application handle errors 41 // Let Angular application handle errors
41 if (validator.isUUID(videoId, 4)) { 42 if (validator.isInt(videoId) || validator.isUUID(videoId, 4)) {
42 videoPromise = VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(videoId) 43 videoPromise = VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
43 } else if (validator.isInt(videoId)) {
44 videoPromise = VideoModel.loadAndPopulateAccountAndServerAndTags(+videoId)
45 } else { 44 } else {
46 return ClientHtml.getIndexHTML(req, res) 45 return ClientHtml.getIndexHTML(req, res)
47 } 46 }
@@ -150,7 +149,7 @@ export class ClientHtml {
150 description: videoDescriptionEscaped, 149 description: videoDescriptionEscaped,
151 thumbnailUrl: previewUrl, 150 thumbnailUrl: previewUrl,
152 uploadDate: video.createdAt.toISOString(), 151 uploadDate: video.createdAt.toISOString(),
153 duration: video.getActivityStreamDuration(), 152 duration: getActivityStreamDuration(video.duration),
154 contentUrl: videoUrl, 153 contentUrl: videoUrl,
155 embedUrl: embedUrl, 154 embedUrl: embedUrl,
156 interactionCount: video.views 155 interactionCount: video.views
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index 72d670277..42217c27c 100644
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -1,10 +1,10 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger' 2import { logger } from '../../../helpers/logger'
3import { processActivities } from '../../activitypub/process' 3import { processActivities } from '../../activitypub/process'
4import { VideoModel } from '../../../models/video/video'
5import { addVideoShares, createRates } from '../../activitypub/videos'
6import { addVideoComments } from '../../activitypub/video-comments' 4import { addVideoComments } from '../../activitypub/video-comments'
7import { crawlCollectionPage } from '../../activitypub/crawl' 5import { crawlCollectionPage } from '../../activitypub/crawl'
6import { VideoModel } from '../../../models/video/video'
7import { addVideoShares, createRates } from '../../activitypub'
8 8
9type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' 9type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments'
10 10
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts
index c6308f7a6..1463c93fc 100644
--- a/server/lib/job-queue/handlers/video-file.ts
+++ b/server/lib/job-queue/handlers/video-file.ts
@@ -8,6 +8,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
8import { sequelizeTypescript } from '../../../initializers' 8import { sequelizeTypescript } from '../../../initializers'
9import * as Bluebird from 'bluebird' 9import * as Bluebird from 'bluebird'
10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' 10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
11import { importVideoFile, transcodeOriginalVideofile, optimizeOriginalVideofile } from '../../video-transcoding'
11 12
12export type VideoFilePayload = { 13export type VideoFilePayload = {
13 videoUUID: string 14 videoUUID: string
@@ -25,14 +26,14 @@ async function processVideoFileImport (job: Bull.Job) {
25 const payload = job.data as VideoFileImportPayload 26 const payload = job.data as VideoFileImportPayload
26 logger.info('Processing video file import in job %d.', job.id) 27 logger.info('Processing video file import in job %d.', job.id)
27 28
28 const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID) 29 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID)
29 // No video, maybe deleted? 30 // No video, maybe deleted?
30 if (!video) { 31 if (!video) {
31 logger.info('Do not process job %d, video does not exist.', job.id) 32 logger.info('Do not process job %d, video does not exist.', job.id)
32 return undefined 33 return undefined
33 } 34 }
34 35
35 await video.importVideoFile(payload.filePath) 36 await importVideoFile(video, payload.filePath)
36 37
37 await onVideoFileTranscoderOrImportSuccess(video) 38 await onVideoFileTranscoderOrImportSuccess(video)
38 return video 39 return video
@@ -42,7 +43,7 @@ async function processVideoFile (job: Bull.Job) {
42 const payload = job.data as VideoFilePayload 43 const payload = job.data as VideoFilePayload
43 logger.info('Processing video file in job %d.', job.id) 44 logger.info('Processing video file in job %d.', job.id)
44 45
45 const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID) 46 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID)
46 // No video, maybe deleted? 47 // No video, maybe deleted?
47 if (!video) { 48 if (!video) {
48 logger.info('Do not process job %d, video does not exist.', job.id) 49 logger.info('Do not process job %d, video does not exist.', job.id)
@@ -51,11 +52,11 @@ async function processVideoFile (job: Bull.Job) {
51 52
52 // Transcoding in other resolution 53 // Transcoding in other resolution
53 if (payload.resolution) { 54 if (payload.resolution) {
54 await video.transcodeOriginalVideofile(payload.resolution, payload.isPortraitMode || false) 55 await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false)
55 56
56 await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) 57 await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video)
57 } else { 58 } else {
58 await video.optimizeOriginalVideofile() 59 await optimizeOriginalVideofile(video)
59 60
60 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) 61 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo)
61 } 62 }
@@ -68,7 +69,7 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) {
68 69
69 return sequelizeTypescript.transaction(async t => { 70 return sequelizeTypescript.transaction(async t => {
70 // Maybe the video changed in database, refresh it 71 // Maybe the video changed in database, refresh it
71 let videoDatabase = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t) 72 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
72 // Video does not exist anymore 73 // Video does not exist anymore
73 if (!videoDatabase) return undefined 74 if (!videoDatabase) return undefined
74 75
@@ -98,7 +99,7 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole
98 99
99 return sequelizeTypescript.transaction(async t => { 100 return sequelizeTypescript.transaction(async t => {
100 // Maybe the video changed in database, refresh it 101 // Maybe the video changed in database, refresh it
101 const videoDatabase = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t) 102 const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
102 // Video does not exist anymore 103 // Video does not exist anymore
103 if (!videoDatabase) return undefined 104 if (!videoDatabase) return undefined
104 105
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index ebcb2090c..9e14e57e6 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -183,7 +183,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
183 const videoUpdated = await video.save({ transaction: t }) 183 const videoUpdated = await video.save({ transaction: t })
184 184
185 // Now we can federate the video (reload from database, we need more attributes) 185 // Now we can federate the video (reload from database, we need more attributes)
186 const videoForFederation = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t) 186 const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
187 await federateVideoIfNeeded(videoForFederation, true, t) 187 await federateVideoIfNeeded(videoForFederation, true, t)
188 188
189 // Update video import object 189 // Update video import object
diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts
index 2f8667e19..5cbe60b82 100644
--- a/server/lib/oauth-model.ts
+++ b/server/lib/oauth-model.ts
@@ -4,15 +4,50 @@ import { UserModel } from '../models/account/user'
4import { OAuthClientModel } from '../models/oauth/oauth-client' 4import { OAuthClientModel } from '../models/oauth/oauth-client'
5import { OAuthTokenModel } from '../models/oauth/oauth-token' 5import { OAuthTokenModel } from '../models/oauth/oauth-token'
6import { CONFIG } from '../initializers/constants' 6import { CONFIG } from '../initializers/constants'
7import { Transaction } from 'sequelize'
7 8
8type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } 9type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date }
10const accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
11const userHavingToken: { [ userId: number ]: string } = {}
9 12
10// --------------------------------------------------------------------------- 13// ---------------------------------------------------------------------------
11 14
15function deleteUserToken (userId: number, t?: Transaction) {
16 clearCacheByUserId(userId)
17
18 return OAuthTokenModel.deleteUserToken(userId, t)
19}
20
21function clearCacheByUserId (userId: number) {
22 const token = userHavingToken[userId]
23 if (token !== undefined) {
24 accessTokenCache[ token ] = undefined
25 userHavingToken[ userId ] = undefined
26 }
27}
28
29function clearCacheByToken (token: string) {
30 const tokenModel = accessTokenCache[ token ]
31 if (tokenModel !== undefined) {
32 userHavingToken[tokenModel.userId] = undefined
33 accessTokenCache[ token ] = undefined
34 }
35}
36
12function getAccessToken (bearerToken: string) { 37function getAccessToken (bearerToken: string) {
13 logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') 38 logger.debug('Getting access token (bearerToken: ' + bearerToken + ').')
14 39
40 if (accessTokenCache[bearerToken] !== undefined) return accessTokenCache[bearerToken]
41
15 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) 42 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
43 .then(tokenModel => {
44 if (tokenModel) {
45 accessTokenCache[ bearerToken ] = tokenModel
46 userHavingToken[ tokenModel.userId ] = tokenModel.accessToken
47 }
48
49 return tokenModel
50 })
16} 51}
17 52
18function getClient (clientId: string, clientSecret: string) { 53function getClient (clientId: string, clientSecret: string) {
@@ -48,6 +83,8 @@ async function getUser (usernameOrEmail: string, password: string) {
48async function revokeToken (tokenInfo: TokenInfo) { 83async function revokeToken (tokenInfo: TokenInfo) {
49 const token = await OAuthTokenModel.getByRefreshTokenAndPopulateUser(tokenInfo.refreshToken) 84 const token = await OAuthTokenModel.getByRefreshTokenAndPopulateUser(tokenInfo.refreshToken)
50 if (token) { 85 if (token) {
86 clearCacheByToken(token.accessToken)
87
51 token.destroy() 88 token.destroy()
52 .catch(err => logger.error('Cannot destroy token when revoking token.', { err })) 89 .catch(err => logger.error('Cannot destroy token when revoking token.', { err }))
53 } 90 }
@@ -85,6 +122,9 @@ async function saveToken (token: TokenInfo, client: OAuthClientModel, user: User
85 122
86// See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications 123// See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications
87export { 124export {
125 deleteUserToken,
126 clearCacheByUserId,
127 clearCacheByToken,
88 getAccessToken, 128 getAccessToken,
89 getClient, 129 getClient,
90 getRefreshToken, 130 getRefreshToken,
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index ee9ba1766..960651712 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,10 +1,9 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' 2import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' 4import { VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6import { VideoFileModel } from '../../models/video/video-file' 6import { VideoFileModel } from '../../models/video/video-file'
7import { sortBy } from 'lodash'
8import { downloadWebTorrentVideo } from '../../helpers/webtorrent' 7import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
9import { join } from 'path' 8import { join } from 'path'
10import { rename } from 'fs-extra' 9import { rename } from 'fs-extra'
@@ -12,7 +11,6 @@ import { getServerActor } from '../../helpers/utils'
12import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
13import { VideoModel } from '../../models/video/video' 12import { VideoModel } from '../../models/video/video'
14import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' 13import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
15import { removeVideoRedundancy } from '../redundancy'
16import { isTestInstance } from '../../helpers/core-utils' 14import { isTestInstance } from '../../helpers/core-utils'
17 15
18export class VideosRedundancyScheduler extends AbstractScheduler { 16export class VideosRedundancyScheduler extends AbstractScheduler {
@@ -20,7 +18,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
20 private static instance: AbstractScheduler 18 private static instance: AbstractScheduler
21 private executing = false 19 private executing = false
22 20
23 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy 21 protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
24 22
25 private constructor () { 23 private constructor () {
26 super() 24 super()
@@ -31,17 +29,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
31 29
32 this.executing = true 30 this.executing = true
33 31
34 for (const obj of CONFIG.REDUNDANCY.VIDEOS) { 32 for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
35
36 try { 33 try {
37 const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy) 34 const videoToDuplicate = await this.findVideoToDuplicate(obj)
38 if (!videoToDuplicate) continue 35 if (!videoToDuplicate) continue
39 36
40 const videoFiles = videoToDuplicate.VideoFiles 37 const videoFiles = videoToDuplicate.VideoFiles
41 videoFiles.forEach(f => f.Video = videoToDuplicate) 38 videoFiles.forEach(f => f.Video = videoToDuplicate)
42 39
43 const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy) 40 if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) {
44 if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) {
45 if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) 41 if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
46 continue 42 continue
47 } 43 }
@@ -54,6 +50,16 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
54 } 50 }
55 } 51 }
56 52
53 await this.removeExpired()
54
55 this.executing = false
56 }
57
58 static get Instance () {
59 return this.instance || (this.instance = new this())
60 }
61
62 private async removeExpired () {
57 const expired = await VideoRedundancyModel.listAllExpired() 63 const expired = await VideoRedundancyModel.listAllExpired()
58 64
59 for (const m of expired) { 65 for (const m of expired) {
@@ -65,16 +71,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
65 logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) 71 logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m))
66 } 72 }
67 } 73 }
68
69 this.executing = false
70 } 74 }
71 75
72 static get Instance () { 76 private findVideoToDuplicate (cache: VideosRedundancy) {
73 return this.instance || (this.instance = new this()) 77 if (cache.strategy === 'most-views') {
74 } 78 return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
79 }
80
81 if (cache.strategy === 'trending') {
82 return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
83 }
75 84
76 private findVideoToDuplicate (strategy: VideoRedundancyStrategy) { 85 if (cache.strategy === 'recently-added') {
77 if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) 86 const minViews = cache.minViews
87 return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews)
88 }
78 } 89 }
79 90
80 private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { 91 private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) {
@@ -120,27 +131,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
120 } 131 }
121 } 132 }
122 133
123 // Unused, but could be useful in the future, with a custom strategy 134 private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
124 private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) {
125 const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt')
126
127 while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) {
128 const toDelete = sortedVideosRedundancy.shift()
129
130 const videoFile = toDelete.VideoFile
131 logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution)
132
133 await removeVideoRedundancy(toDelete, undefined)
134 }
135
136 return sortedVideosRedundancy
137 }
138
139 private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
140 const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) 135 const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate)
141 136
142 const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size 137 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy)
143 const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0)
144 138
145 return totalDuplicated > maxSize 139 return totalDuplicated > maxSize
146 } 140 }
diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts
index faadb4334..461cd045e 100644
--- a/server/lib/schedulers/youtube-dl-update-scheduler.ts
+++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts
@@ -1,13 +1,6 @@
1// Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
2// We rewrote it to avoid sync calls
3
4import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
5import { SCHEDULER_INTERVALS_MS } from '../../initializers' 2import { SCHEDULER_INTERVALS_MS } from '../../initializers'
6import { logger } from '../../helpers/logger' 3import { updateYoutubeDLBinary } from '../../helpers/youtube-dl'
7import * as request from 'request'
8import { createWriteStream, ensureDir, writeFile } from 'fs-extra'
9import { join } from 'path'
10import { root } from '../../helpers/core-utils'
11 4
12export class YoutubeDlUpdateScheduler extends AbstractScheduler { 5export class YoutubeDlUpdateScheduler extends AbstractScheduler {
13 6
@@ -19,60 +12,8 @@ export class YoutubeDlUpdateScheduler extends AbstractScheduler {
19 super() 12 super()
20 } 13 }
21 14
22 async execute () { 15 execute () {
23 logger.info('Updating youtubeDL binary.') 16 return updateYoutubeDLBinary()
24
25 const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
26 const bin = join(binDirectory, 'youtube-dl')
27 const detailsPath = join(binDirectory, 'details')
28 const url = 'https://yt-dl.org/downloads/latest/youtube-dl'
29
30 await ensureDir(binDirectory)
31
32 return new Promise(res => {
33 request.get(url, { followRedirect: false }, (err, result) => {
34 if (err) {
35 logger.error('Cannot update youtube-dl.', { err })
36 return res()
37 }
38
39 if (result.statusCode !== 302) {
40 logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
41 return res()
42 }
43
44 const url = result.headers.location
45 const downloadFile = request.get(url)
46 const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ]
47
48 downloadFile.on('response', result => {
49 if (result.statusCode !== 200) {
50 logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
51 return res()
52 }
53
54 downloadFile.pipe(createWriteStream(bin, { mode: 493 }))
55 })
56
57 downloadFile.on('error', err => {
58 logger.error('youtube-dl update error.', { err })
59 return res()
60 })
61
62 downloadFile.on('end', () => {
63 const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
64 writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
65 if (err) {
66 logger.error('youtube-dl update error: cannot write details.', { err })
67 return res()
68 }
69
70 logger.info('youtube-dl updated to version %s.', newVersion)
71 return res()
72 })
73 })
74 })
75 })
76 } 17 }
77 18
78 static get Instance () { 19 static get Instance () {
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
new file mode 100644
index 000000000..bf3ff78c2
--- /dev/null
+++ b/server/lib/video-transcoding.ts
@@ -0,0 +1,130 @@
1import { CONFIG } from '../initializers'
2import { join, extname } from 'path'
3import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils'
4import { copy, remove, rename, stat } from 'fs-extra'
5import { logger } from '../helpers/logger'
6import { VideoResolution } from '../../shared/models/videos'
7import { VideoFileModel } from '../models/video/video-file'
8import { VideoModel } from '../models/video/video'
9
10async function optimizeOriginalVideofile (video: VideoModel) {
11 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
12 const newExtname = '.mp4'
13 const inputVideoFile = video.getOriginalFile()
14 const videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile))
15 const videoTranscodedPath = join(videosDirectory, video.id + '-transcoded' + newExtname)
16
17 const transcodeOptions = {
18 inputPath: videoInputPath,
19 outputPath: videoTranscodedPath
20 }
21
22 // Could be very long!
23 await transcode(transcodeOptions)
24
25 try {
26 await remove(videoInputPath)
27
28 // Important to do this before getVideoFilename() to take in account the new file extension
29 inputVideoFile.set('extname', newExtname)
30
31 const videoOutputPath = video.getVideoFilePath(inputVideoFile)
32 await rename(videoTranscodedPath, videoOutputPath)
33 const stats = await stat(videoOutputPath)
34 const fps = await getVideoFileFPS(videoOutputPath)
35
36 inputVideoFile.set('size', stats.size)
37 inputVideoFile.set('fps', fps)
38
39 await video.createTorrentAndSetInfoHash(inputVideoFile)
40 await inputVideoFile.save()
41 } catch (err) {
42 // Auto destruction...
43 video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err }))
44
45 throw err
46 }
47}
48
49async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) {
50 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
51 const extname = '.mp4'
52
53 // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed
54 const videoInputPath = join(videosDirectory, video.getVideoFilename(video.getOriginalFile()))
55
56 const newVideoFile = new VideoFileModel({
57 resolution,
58 extname,
59 size: 0,
60 videoId: video.id
61 })
62 const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile))
63
64 const transcodeOptions = {
65 inputPath: videoInputPath,
66 outputPath: videoOutputPath,
67 resolution,
68 isPortraitMode
69 }
70
71 await transcode(transcodeOptions)
72
73 const stats = await stat(videoOutputPath)
74 const fps = await getVideoFileFPS(videoOutputPath)
75
76 newVideoFile.set('size', stats.size)
77 newVideoFile.set('fps', fps)
78
79 await video.createTorrentAndSetInfoHash(newVideoFile)
80
81 await newVideoFile.save()
82
83 video.VideoFiles.push(newVideoFile)
84}
85
86async function importVideoFile (video: VideoModel, inputFilePath: string) {
87 const { videoFileResolution } = await getVideoFileResolution(inputFilePath)
88 const { size } = await stat(inputFilePath)
89 const fps = await getVideoFileFPS(inputFilePath)
90
91 let updatedVideoFile = new VideoFileModel({
92 resolution: videoFileResolution,
93 extname: extname(inputFilePath),
94 size,
95 fps,
96 videoId: video.id
97 })
98
99 const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution)
100
101 if (currentVideoFile) {
102 // Remove old file and old torrent
103 await video.removeFile(currentVideoFile)
104 await video.removeTorrent(currentVideoFile)
105 // Remove the old video file from the array
106 video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile)
107
108 // Update the database
109 currentVideoFile.set('extname', updatedVideoFile.extname)
110 currentVideoFile.set('size', updatedVideoFile.size)
111 currentVideoFile.set('fps', updatedVideoFile.fps)
112
113 updatedVideoFile = currentVideoFile
114 }
115
116 const outputPath = video.getVideoFilePath(updatedVideoFile)
117 await copy(inputFilePath, outputPath)
118
119 await video.createTorrentAndSetInfoHash(updatedVideoFile)
120
121 await updatedVideoFile.save()
122
123 video.VideoFiles.push(updatedVideoFile)
124}
125
126export {
127 optimizeOriginalVideofile,
128 transcodeOriginalVideofile,
129 importVideoFile
130}