diff options
Diffstat (limited to 'server/lib')
28 files changed, 940 insertions, 361 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index f7bf7c65a..a3f379b76 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts | |||
@@ -4,7 +4,7 @@ import * as url from 'url' | |||
4 | import * as uuidv4 from 'uuid/v4' | 4 | import * as uuidv4 from 'uuid/v4' |
5 | import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub' | 5 | import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub' |
6 | import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' | 6 | import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' |
7 | import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' | 7 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
8 | import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor' | 8 | import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor' |
9 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | 9 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' |
10 | import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' | 10 | import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' |
@@ -42,7 +42,7 @@ async function getOrCreateActorAndServerAndModel ( | |||
42 | recurseIfNeeded = true, | 42 | recurseIfNeeded = true, |
43 | updateCollections = false | 43 | updateCollections = false |
44 | ) { | 44 | ) { |
45 | const actorUrl = getAPUrl(activityActor) | 45 | const actorUrl = getAPId(activityActor) |
46 | let created = false | 46 | let created = false |
47 | 47 | ||
48 | let actor = await fetchActorByUrl(actorUrl, fetchType) | 48 | let actor = await fetchActorByUrl(actorUrl, fetchType) |
@@ -201,6 +201,69 @@ async function addFetchOutboxJob (actor: ActorModel) { | |||
201 | return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) | 201 | return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) |
202 | } | 202 | } |
203 | 203 | ||
204 | async function refreshActorIfNeeded ( | ||
205 | actorArg: ActorModel, | ||
206 | fetchedType: ActorFetchByUrlType | ||
207 | ): Promise<{ actor: ActorModel, refreshed: boolean }> { | ||
208 | if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } | ||
209 | |||
210 | // We need more attributes | ||
211 | const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) | ||
212 | |||
213 | try { | ||
214 | let actorUrl: string | ||
215 | try { | ||
216 | actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) | ||
217 | } catch (err) { | ||
218 | logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err) | ||
219 | actorUrl = actor.url | ||
220 | } | ||
221 | |||
222 | const { result, statusCode } = await fetchRemoteActor(actorUrl) | ||
223 | |||
224 | if (statusCode === 404) { | ||
225 | logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) | ||
226 | actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() | ||
227 | return { actor: undefined, refreshed: false } | ||
228 | } | ||
229 | |||
230 | if (result === undefined) { | ||
231 | logger.warn('Cannot fetch remote actor in refresh actor.') | ||
232 | return { actor, refreshed: false } | ||
233 | } | ||
234 | |||
235 | return sequelizeTypescript.transaction(async t => { | ||
236 | updateInstanceWithAnother(actor, result.actor) | ||
237 | |||
238 | if (result.avatarName !== undefined) { | ||
239 | await updateActorAvatarInstance(actor, result.avatarName, t) | ||
240 | } | ||
241 | |||
242 | // Force update | ||
243 | actor.setDataValue('updatedAt', new Date()) | ||
244 | await actor.save({ transaction: t }) | ||
245 | |||
246 | if (actor.Account) { | ||
247 | actor.Account.set('name', result.name) | ||
248 | actor.Account.set('description', result.summary) | ||
249 | |||
250 | await actor.Account.save({ transaction: t }) | ||
251 | } else if (actor.VideoChannel) { | ||
252 | actor.VideoChannel.set('name', result.name) | ||
253 | actor.VideoChannel.set('description', result.summary) | ||
254 | actor.VideoChannel.set('support', result.support) | ||
255 | |||
256 | await actor.VideoChannel.save({ transaction: t }) | ||
257 | } | ||
258 | |||
259 | return { refreshed: true, actor } | ||
260 | }) | ||
261 | } catch (err) { | ||
262 | logger.warn('Cannot refresh actor.', { err }) | ||
263 | return { actor, refreshed: false } | ||
264 | } | ||
265 | } | ||
266 | |||
204 | export { | 267 | export { |
205 | getOrCreateActorAndServerAndModel, | 268 | getOrCreateActorAndServerAndModel, |
206 | buildActorInstance, | 269 | buildActorInstance, |
@@ -208,6 +271,7 @@ export { | |||
208 | fetchActorTotalItems, | 271 | fetchActorTotalItems, |
209 | fetchAvatarIfExists, | 272 | fetchAvatarIfExists, |
210 | updateActorInstance, | 273 | updateActorInstance, |
274 | refreshActorIfNeeded, | ||
211 | updateActorAvatarInstance, | 275 | updateActorAvatarInstance, |
212 | addFetchOutboxJob | 276 | addFetchOutboxJob |
213 | } | 277 | } |
@@ -291,12 +355,12 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe | |||
291 | 355 | ||
292 | logger.info('Fetching remote actor %s.', actorUrl) | 356 | logger.info('Fetching remote actor %s.', actorUrl) |
293 | 357 | ||
294 | const requestResult = await doRequest(options) | 358 | const requestResult = await doRequest<ActivityPubActor>(options) |
295 | normalizeActor(requestResult.body) | 359 | normalizeActor(requestResult.body) |
296 | 360 | ||
297 | const actorJSON: ActivityPubActor = requestResult.body | 361 | const actorJSON = requestResult.body |
298 | if (isActorObjectValid(actorJSON) === false) { | 362 | if (isActorObjectValid(actorJSON) === false) { |
299 | logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON }) | 363 | logger.debug('Remote actor JSON is not valid.', { actorJSON }) |
300 | return { result: undefined, statusCode: requestResult.response.statusCode } | 364 | return { result: undefined, statusCode: requestResult.response.statusCode } |
301 | } | 365 | } |
302 | 366 | ||
@@ -372,59 +436,3 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu | |||
372 | 436 | ||
373 | return videoChannelCreated | 437 | return videoChannelCreated |
374 | } | 438 | } |
375 | |||
376 | async function refreshActorIfNeeded ( | ||
377 | actorArg: ActorModel, | ||
378 | fetchedType: ActorFetchByUrlType | ||
379 | ): Promise<{ actor: ActorModel, refreshed: boolean }> { | ||
380 | if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } | ||
381 | |||
382 | // We need more attributes | ||
383 | const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) | ||
384 | |||
385 | try { | ||
386 | const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) | ||
387 | const { result, statusCode } = await fetchRemoteActor(actorUrl) | ||
388 | |||
389 | if (statusCode === 404) { | ||
390 | logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) | ||
391 | actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() | ||
392 | return { actor: undefined, refreshed: false } | ||
393 | } | ||
394 | |||
395 | if (result === undefined) { | ||
396 | logger.warn('Cannot fetch remote actor in refresh actor.') | ||
397 | return { actor, refreshed: false } | ||
398 | } | ||
399 | |||
400 | return sequelizeTypescript.transaction(async t => { | ||
401 | updateInstanceWithAnother(actor, result.actor) | ||
402 | |||
403 | if (result.avatarName !== undefined) { | ||
404 | await updateActorAvatarInstance(actor, result.avatarName, t) | ||
405 | } | ||
406 | |||
407 | // Force update | ||
408 | actor.setDataValue('updatedAt', new Date()) | ||
409 | await actor.save({ transaction: t }) | ||
410 | |||
411 | if (actor.Account) { | ||
412 | actor.Account.set('name', result.name) | ||
413 | actor.Account.set('description', result.summary) | ||
414 | |||
415 | await actor.Account.save({ transaction: t }) | ||
416 | } else if (actor.VideoChannel) { | ||
417 | actor.VideoChannel.set('name', result.name) | ||
418 | actor.VideoChannel.set('description', result.summary) | ||
419 | actor.VideoChannel.set('support', result.support) | ||
420 | |||
421 | await actor.VideoChannel.save({ transaction: t }) | ||
422 | } | ||
423 | |||
424 | return { refreshed: true, actor } | ||
425 | }) | ||
426 | } catch (err) { | ||
427 | logger.warn('Cannot refresh actor.', { err }) | ||
428 | return { actor, refreshed: false } | ||
429 | } | ||
430 | } | ||
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index f6f068b45..9a40414bb 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts | |||
@@ -1,11 +1,28 @@ | |||
1 | import { CacheFileObject } from '../../../shared/index' | 1 | import { ActivityPlaylistUrlObject, ActivityVideoUrlObject, CacheFileObject } from '../../../shared/index' |
2 | import { VideoModel } from '../../models/video/video' | 2 | import { VideoModel } from '../../models/video/video' |
3 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 3 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
4 | import { Transaction } from 'sequelize' | 4 | import { Transaction } from 'sequelize' |
5 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
5 | 6 | ||
6 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { | 7 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { |
7 | const url = cacheFileObject.url | ||
8 | 8 | ||
9 | if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { | ||
10 | const url = cacheFileObject.url | ||
11 | |||
12 | const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) | ||
13 | if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) | ||
14 | |||
15 | return { | ||
16 | expiresOn: new Date(cacheFileObject.expires), | ||
17 | url: cacheFileObject.id, | ||
18 | fileUrl: url.href, | ||
19 | strategy: null, | ||
20 | videoStreamingPlaylistId: playlist.id, | ||
21 | actorId: byActor.id | ||
22 | } | ||
23 | } | ||
24 | |||
25 | const url = cacheFileObject.url | ||
9 | const videoFile = video.VideoFiles.find(f => { | 26 | const videoFile = video.VideoFiles.find(f => { |
10 | return f.resolution === url.height && f.fps === url.fps | 27 | return f.resolution === url.height && f.fps === url.fps |
11 | }) | 28 | }) |
@@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject | |||
15 | return { | 32 | return { |
16 | expiresOn: new Date(cacheFileObject.expires), | 33 | expiresOn: new Date(cacheFileObject.expires), |
17 | url: cacheFileObject.id, | 34 | url: cacheFileObject.id, |
18 | fileUrl: cacheFileObject.url.href, | 35 | fileUrl: url.href, |
19 | strategy: null, | 36 | strategy: null, |
20 | videoFileId: videoFile.id, | 37 | videoFileId: videoFile.id, |
21 | actorId: byActor.id | 38 | actorId: byActor.id |
diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts index 605705ad3..ebb275e34 100644 --- a/server/lib/activitypub/process/process-accept.ts +++ b/server/lib/activitypub/process/process-accept.ts | |||
@@ -2,7 +2,6 @@ import { ActivityAccept } from '../../../../shared/models/activitypub' | |||
2 | import { ActorModel } from '../../../models/activitypub/actor' | 2 | import { ActorModel } from '../../../models/activitypub/actor' |
3 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 3 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
4 | import { addFetchOutboxJob } from '../actor' | 4 | import { addFetchOutboxJob } from '../actor' |
5 | import { Notifier } from '../../notifier' | ||
6 | 5 | ||
7 | async function processAcceptActivity (activity: ActivityAccept, targetActor: ActorModel, inboxActor?: ActorModel) { | 6 | async function processAcceptActivity (activity: ActivityAccept, targetActor: ActorModel, inboxActor?: ActorModel) { |
8 | if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') | 7 | if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') |
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 2e04ee843..5f4d793a5 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -1,36 +1,44 @@ | |||
1 | import { ActivityCreate, CacheFileObject, VideoAbuseState, VideoTorrentObject } from '../../../../shared' | 1 | import { ActivityCreate, CacheFileObject, VideoTorrentObject } from '../../../../shared' |
2 | import { DislikeObject, VideoAbuseObject, ViewObject } from '../../../../shared/models/activitypub/objects' | ||
3 | import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object' | 2 | import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object' |
4 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 3 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
5 | import { logger } from '../../../helpers/logger' | 4 | import { logger } from '../../../helpers/logger' |
6 | import { sequelizeTypescript } from '../../../initializers' | 5 | import { sequelizeTypescript } from '../../../initializers' |
7 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | ||
8 | import { ActorModel } from '../../../models/activitypub/actor' | 6 | import { ActorModel } from '../../../models/activitypub/actor' |
9 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | ||
10 | import { addVideoComment, resolveThread } from '../video-comments' | 7 | import { addVideoComment, resolveThread } from '../video-comments' |
11 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 8 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
12 | import { forwardVideoRelatedActivity } from '../send/utils' | 9 | import { forwardVideoRelatedActivity } from '../send/utils' |
13 | import { Redis } from '../../redis' | ||
14 | import { createOrUpdateCacheFile } from '../cache-file' | 10 | import { createOrUpdateCacheFile } from '../cache-file' |
15 | import { getVideoDislikeActivityPubUrl } from '../url' | ||
16 | import { Notifier } from '../../notifier' | 11 | import { Notifier } from '../../notifier' |
12 | import { processViewActivity } from './process-view' | ||
13 | import { processDislikeActivity } from './process-dislike' | ||
14 | import { processFlagActivity } from './process-flag' | ||
17 | 15 | ||
18 | async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { | 16 | async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { |
19 | const activityObject = activity.object | 17 | const activityObject = activity.object |
20 | const activityType = activityObject.type | 18 | const activityType = activityObject.type |
21 | 19 | ||
22 | if (activityType === 'View') { | 20 | if (activityType === 'View') { |
23 | return processCreateView(byActor, activity) | 21 | return processViewActivity(activity, byActor) |
24 | } else if (activityType === 'Dislike') { | 22 | } |
25 | return retryTransactionWrapper(processCreateDislike, byActor, activity) | 23 | |
26 | } else if (activityType === 'Video') { | 24 | if (activityType === 'Dislike') { |
25 | return retryTransactionWrapper(processDislikeActivity, activity, byActor) | ||
26 | } | ||
27 | |||
28 | if (activityType === 'Flag') { | ||
29 | return retryTransactionWrapper(processFlagActivity, activity, byActor) | ||
30 | } | ||
31 | |||
32 | if (activityType === 'Video') { | ||
27 | return processCreateVideo(activity) | 33 | return processCreateVideo(activity) |
28 | } else if (activityType === 'Flag') { | 34 | } |
29 | return retryTransactionWrapper(processCreateVideoAbuse, byActor, activityObject as VideoAbuseObject) | 35 | |
30 | } else if (activityType === 'Note') { | 36 | if (activityType === 'Note') { |
31 | return retryTransactionWrapper(processCreateVideoComment, byActor, activity) | 37 | return retryTransactionWrapper(processCreateVideoComment, activity, byActor) |
32 | } else if (activityType === 'CacheFile') { | 38 | } |
33 | return retryTransactionWrapper(processCacheFile, byActor, activity) | 39 | |
40 | if (activityType === 'CacheFile') { | ||
41 | return retryTransactionWrapper(processCacheFile, activity, byActor) | ||
34 | } | 42 | } |
35 | 43 | ||
36 | logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) | 44 | logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) |
@@ -55,56 +63,7 @@ async function processCreateVideo (activity: ActivityCreate) { | |||
55 | return video | 63 | return video |
56 | } | 64 | } |
57 | 65 | ||
58 | async function processCreateDislike (byActor: ActorModel, activity: ActivityCreate) { | 66 | async function processCacheFile (activity: ActivityCreate, byActor: ActorModel) { |
59 | const dislike = activity.object as DislikeObject | ||
60 | const byAccount = byActor.Account | ||
61 | |||
62 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | ||
63 | |||
64 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) | ||
65 | |||
66 | return sequelizeTypescript.transaction(async t => { | ||
67 | const rate = { | ||
68 | type: 'dislike' as 'dislike', | ||
69 | videoId: video.id, | ||
70 | accountId: byAccount.id | ||
71 | } | ||
72 | |||
73 | const [ , created ] = await AccountVideoRateModel.findOrCreate({ | ||
74 | where: rate, | ||
75 | defaults: Object.assign({}, rate, { url: getVideoDislikeActivityPubUrl(byActor, video) }), | ||
76 | transaction: t | ||
77 | }) | ||
78 | if (created === true) await video.increment('dislikes', { transaction: t }) | ||
79 | |||
80 | if (video.isOwned() && created === true) { | ||
81 | // Don't resend the activity to the sender | ||
82 | const exceptions = [ byActor ] | ||
83 | |||
84 | await forwardVideoRelatedActivity(activity, t, exceptions, video) | ||
85 | } | ||
86 | }) | ||
87 | } | ||
88 | |||
89 | async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { | ||
90 | const view = activity.object as ViewObject | ||
91 | |||
92 | const options = { | ||
93 | videoObject: view.object, | ||
94 | fetchType: 'only-video' as 'only-video' | ||
95 | } | ||
96 | const { video } = await getOrCreateVideoAndAccountAndChannel(options) | ||
97 | |||
98 | await Redis.Instance.addVideoView(video.id) | ||
99 | |||
100 | if (video.isOwned()) { | ||
101 | // Don't resend the activity to the sender | ||
102 | const exceptions = [ byActor ] | ||
103 | await forwardVideoRelatedActivity(activity, undefined, exceptions, video) | ||
104 | } | ||
105 | } | ||
106 | |||
107 | async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { | ||
108 | const cacheFile = activity.object as CacheFileObject | 67 | const cacheFile = activity.object as CacheFileObject |
109 | 68 | ||
110 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) | 69 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) |
@@ -120,32 +79,7 @@ async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) | |||
120 | } | 79 | } |
121 | } | 80 | } |
122 | 81 | ||
123 | async function processCreateVideoAbuse (byActor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { | 82 | async function processCreateVideoComment (activity: ActivityCreate, byActor: ActorModel) { |
124 | logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) | ||
125 | |||
126 | const account = byActor.Account | ||
127 | if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | ||
128 | |||
129 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object }) | ||
130 | |||
131 | return sequelizeTypescript.transaction(async t => { | ||
132 | const videoAbuseData = { | ||
133 | reporterAccountId: account.id, | ||
134 | reason: videoAbuseToCreateData.content, | ||
135 | videoId: video.id, | ||
136 | state: VideoAbuseState.PENDING | ||
137 | } | ||
138 | |||
139 | const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t }) | ||
140 | videoAbuseInstance.Video = video | ||
141 | |||
142 | Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance) | ||
143 | |||
144 | logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object) | ||
145 | }) | ||
146 | } | ||
147 | |||
148 | async function processCreateVideoComment (byActor: ActorModel, activity: ActivityCreate) { | ||
149 | const commentObject = activity.object as VideoCommentObject | 83 | const commentObject = activity.object as VideoCommentObject |
150 | const byAccount = byActor.Account | 84 | const byAccount = byActor.Account |
151 | 85 | ||
diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts new file mode 100644 index 000000000..bfd69e07a --- /dev/null +++ b/server/lib/activitypub/process/process-dislike.ts | |||
@@ -0,0 +1,52 @@ | |||
1 | import { ActivityCreate, ActivityDislike } from '../../../../shared' | ||
2 | import { DislikeObject } from '../../../../shared/models/activitypub/objects' | ||
3 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
4 | import { sequelizeTypescript } from '../../../initializers' | ||
5 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | ||
6 | import { ActorModel } from '../../../models/activitypub/actor' | ||
7 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | ||
8 | import { forwardVideoRelatedActivity } from '../send/utils' | ||
9 | import { getVideoDislikeActivityPubUrl } from '../url' | ||
10 | |||
11 | async function processDislikeActivity (activity: ActivityCreate | ActivityDislike, byActor: ActorModel) { | ||
12 | return retryTransactionWrapper(processDislike, activity, byActor) | ||
13 | } | ||
14 | |||
15 | // --------------------------------------------------------------------------- | ||
16 | |||
17 | export { | ||
18 | processDislikeActivity | ||
19 | } | ||
20 | |||
21 | // --------------------------------------------------------------------------- | ||
22 | |||
23 | async function processDislike (activity: ActivityCreate | ActivityDislike, byActor: ActorModel) { | ||
24 | const dislikeObject = activity.type === 'Dislike' ? activity.object : (activity.object as DislikeObject).object | ||
25 | const byAccount = byActor.Account | ||
26 | |||
27 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | ||
28 | |||
29 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislikeObject }) | ||
30 | |||
31 | return sequelizeTypescript.transaction(async t => { | ||
32 | const rate = { | ||
33 | type: 'dislike' as 'dislike', | ||
34 | videoId: video.id, | ||
35 | accountId: byAccount.id | ||
36 | } | ||
37 | |||
38 | const [ , created ] = await AccountVideoRateModel.findOrCreate({ | ||
39 | where: rate, | ||
40 | defaults: Object.assign({}, rate, { url: getVideoDislikeActivityPubUrl(byActor, video) }), | ||
41 | transaction: t | ||
42 | }) | ||
43 | if (created === true) await video.increment('dislikes', { transaction: t }) | ||
44 | |||
45 | if (video.isOwned() && created === true) { | ||
46 | // Don't resend the activity to the sender | ||
47 | const exceptions = [ byActor ] | ||
48 | |||
49 | await forwardVideoRelatedActivity(activity, t, exceptions, video) | ||
50 | } | ||
51 | }) | ||
52 | } | ||
diff --git a/server/lib/activitypub/process/process-flag.ts b/server/lib/activitypub/process/process-flag.ts new file mode 100644 index 000000000..79ce6fb41 --- /dev/null +++ b/server/lib/activitypub/process/process-flag.ts | |||
@@ -0,0 +1,49 @@ | |||
1 | import { ActivityCreate, ActivityFlag, VideoAbuseState } from '../../../../shared' | ||
2 | import { VideoAbuseObject } from '../../../../shared/models/activitypub/objects' | ||
3 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
4 | import { logger } from '../../../helpers/logger' | ||
5 | import { sequelizeTypescript } from '../../../initializers' | ||
6 | import { ActorModel } from '../../../models/activitypub/actor' | ||
7 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | ||
8 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | ||
9 | import { Notifier } from '../../notifier' | ||
10 | import { getAPId } from '../../../helpers/activitypub' | ||
11 | |||
12 | async function processFlagActivity (activity: ActivityCreate | ActivityFlag, byActor: ActorModel) { | ||
13 | return retryTransactionWrapper(processCreateVideoAbuse, activity, byActor) | ||
14 | } | ||
15 | |||
16 | // --------------------------------------------------------------------------- | ||
17 | |||
18 | export { | ||
19 | processFlagActivity | ||
20 | } | ||
21 | |||
22 | // --------------------------------------------------------------------------- | ||
23 | |||
24 | async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag, byActor: ActorModel) { | ||
25 | const flag = activity.type === 'Flag' ? activity : (activity.object as VideoAbuseObject) | ||
26 | |||
27 | logger.debug('Reporting remote abuse for video %s.', getAPId(flag.object)) | ||
28 | |||
29 | const account = byActor.Account | ||
30 | if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | ||
31 | |||
32 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: flag.object }) | ||
33 | |||
34 | return sequelizeTypescript.transaction(async t => { | ||
35 | const videoAbuseData = { | ||
36 | reporterAccountId: account.id, | ||
37 | reason: flag.content, | ||
38 | videoId: video.id, | ||
39 | state: VideoAbuseState.PENDING | ||
40 | } | ||
41 | |||
42 | const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t }) | ||
43 | videoAbuseInstance.Video = video | ||
44 | |||
45 | Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance) | ||
46 | |||
47 | logger.info('Remote abuse for video uuid %s created', flag.object) | ||
48 | }) | ||
49 | } | ||
diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts index a67892440..0cd537187 100644 --- a/server/lib/activitypub/process/process-follow.ts +++ b/server/lib/activitypub/process/process-follow.ts | |||
@@ -6,9 +6,10 @@ import { ActorModel } from '../../../models/activitypub/actor' | |||
6 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 6 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
7 | import { sendAccept } from '../send' | 7 | import { sendAccept } from '../send' |
8 | import { Notifier } from '../../notifier' | 8 | import { Notifier } from '../../notifier' |
9 | import { getAPId } from '../../../helpers/activitypub' | ||
9 | 10 | ||
10 | async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) { | 11 | async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) { |
11 | const activityObject = activity.object | 12 | const activityObject = getAPId(activity.object) |
12 | 13 | ||
13 | return retryTransactionWrapper(processFollow, byActor, activityObject) | 14 | return retryTransactionWrapper(processFollow, byActor, activityObject) |
14 | } | 15 | } |
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index e8e97eece..2a04167d7 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts | |||
@@ -6,6 +6,7 @@ import { ActorModel } from '../../../models/activitypub/actor' | |||
6 | import { forwardVideoRelatedActivity } from '../send/utils' | 6 | import { forwardVideoRelatedActivity } from '../send/utils' |
7 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 7 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
8 | import { getVideoLikeActivityPubUrl } from '../url' | 8 | import { getVideoLikeActivityPubUrl } from '../url' |
9 | import { getAPId } from '../../../helpers/activitypub' | ||
9 | 10 | ||
10 | async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) { | 11 | async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) { |
11 | return retryTransactionWrapper(processLikeVideo, byActor, activity) | 12 | return retryTransactionWrapper(processLikeVideo, byActor, activity) |
@@ -20,7 +21,7 @@ export { | |||
20 | // --------------------------------------------------------------------------- | 21 | // --------------------------------------------------------------------------- |
21 | 22 | ||
22 | async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { | 23 | async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { |
23 | const videoUrl = activity.object | 24 | const videoUrl = getAPId(activity.object) |
24 | 25 | ||
25 | const byAccount = byActor.Account | 26 | const byAccount = byActor.Account |
26 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) | 27 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) |
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 438a013b6..ed0177a67 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts | |||
@@ -26,6 +26,10 @@ async function processUndoActivity (activity: ActivityUndo, byActor: ActorModel) | |||
26 | } | 26 | } |
27 | } | 27 | } |
28 | 28 | ||
29 | if (activityToUndo.type === 'Dislike') { | ||
30 | return retryTransactionWrapper(processUndoDislike, byActor, activity) | ||
31 | } | ||
32 | |||
29 | if (activityToUndo.type === 'Follow') { | 33 | if (activityToUndo.type === 'Follow') { |
30 | return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo) | 34 | return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo) |
31 | } | 35 | } |
@@ -72,7 +76,9 @@ async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) { | |||
72 | } | 76 | } |
73 | 77 | ||
74 | async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) { | 78 | async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) { |
75 | const dislike = activity.object.object as DislikeObject | 79 | const dislike = activity.object.type === 'Dislike' |
80 | ? activity.object | ||
81 | : activity.object.object as DislikeObject | ||
76 | 82 | ||
77 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) | 83 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) |
78 | 84 | ||
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts new file mode 100644 index 000000000..8f66d3630 --- /dev/null +++ b/server/lib/activitypub/process/process-view.ts | |||
@@ -0,0 +1,35 @@ | |||
1 | import { ActorModel } from '../../../models/activitypub/actor' | ||
2 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | ||
3 | import { forwardVideoRelatedActivity } from '../send/utils' | ||
4 | import { Redis } from '../../redis' | ||
5 | import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' | ||
6 | |||
7 | async function processViewActivity (activity: ActivityView | ActivityCreate, byActor: ActorModel) { | ||
8 | return processCreateView(activity, byActor) | ||
9 | } | ||
10 | |||
11 | // --------------------------------------------------------------------------- | ||
12 | |||
13 | export { | ||
14 | processViewActivity | ||
15 | } | ||
16 | |||
17 | // --------------------------------------------------------------------------- | ||
18 | |||
19 | async function processCreateView (activity: ActivityView | ActivityCreate, byActor: ActorModel) { | ||
20 | const videoObject = activity.type === 'View' ? activity.object : (activity.object as ViewObject).object | ||
21 | |||
22 | const options = { | ||
23 | videoObject: videoObject, | ||
24 | fetchType: 'only-video' as 'only-video' | ||
25 | } | ||
26 | const { video } = await getOrCreateVideoAndAccountAndChannel(options) | ||
27 | |||
28 | await Redis.Instance.addVideoView(video.id) | ||
29 | |||
30 | if (video.isOwned()) { | ||
31 | // Don't resend the activity to the sender | ||
32 | const exceptions = [ byActor ] | ||
33 | await forwardVideoRelatedActivity(activity, undefined, exceptions, video) | ||
34 | } | ||
35 | } | ||
diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts index bcc5cac7a..9dd241402 100644 --- a/server/lib/activitypub/process/process.ts +++ b/server/lib/activitypub/process/process.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' | 1 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' |
2 | import { checkUrlsSameHost, getAPUrl } from '../../../helpers/activitypub' | 2 | import { checkUrlsSameHost, getAPId } from '../../../helpers/activitypub' |
3 | import { logger } from '../../../helpers/logger' | 3 | import { logger } from '../../../helpers/logger' |
4 | import { ActorModel } from '../../../models/activitypub/actor' | 4 | import { ActorModel } from '../../../models/activitypub/actor' |
5 | import { processAcceptActivity } from './process-accept' | 5 | import { processAcceptActivity } from './process-accept' |
@@ -12,6 +12,9 @@ import { processRejectActivity } from './process-reject' | |||
12 | import { processUndoActivity } from './process-undo' | 12 | import { processUndoActivity } from './process-undo' |
13 | import { processUpdateActivity } from './process-update' | 13 | import { processUpdateActivity } from './process-update' |
14 | import { getOrCreateActorAndServerAndModel } from '../actor' | 14 | import { getOrCreateActorAndServerAndModel } from '../actor' |
15 | import { processDislikeActivity } from './process-dislike' | ||
16 | import { processFlagActivity } from './process-flag' | ||
17 | import { processViewActivity } from './process-view' | ||
15 | 18 | ||
16 | const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: ActorModel, inboxActor?: ActorModel) => Promise<any> } = { | 19 | const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: ActorModel, inboxActor?: ActorModel) => Promise<any> } = { |
17 | Create: processCreateActivity, | 20 | Create: processCreateActivity, |
@@ -22,7 +25,10 @@ const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: Ac | |||
22 | Reject: processRejectActivity, | 25 | Reject: processRejectActivity, |
23 | Announce: processAnnounceActivity, | 26 | Announce: processAnnounceActivity, |
24 | Undo: processUndoActivity, | 27 | Undo: processUndoActivity, |
25 | Like: processLikeActivity | 28 | Like: processLikeActivity, |
29 | Dislike: processDislikeActivity, | ||
30 | Flag: processFlagActivity, | ||
31 | View: processViewActivity | ||
26 | } | 32 | } |
27 | 33 | ||
28 | async function processActivities ( | 34 | async function processActivities ( |
@@ -35,12 +41,12 @@ async function processActivities ( | |||
35 | const actorsCache: { [ url: string ]: ActorModel } = {} | 41 | const actorsCache: { [ url: string ]: ActorModel } = {} |
36 | 42 | ||
37 | for (const activity of activities) { | 43 | for (const activity of activities) { |
38 | if (!options.signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) { | 44 | if (!options.signatureActor && [ 'Create', 'Announce', 'Like' ].includes(activity.type) === false) { |
39 | logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) | 45 | logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) |
40 | continue | 46 | continue |
41 | } | 47 | } |
42 | 48 | ||
43 | const actorUrl = getAPUrl(activity.actor) | 49 | const actorUrl = getAPId(activity.actor) |
44 | 50 | ||
45 | // When we fetch remote data, we don't have signature | 51 | // When we fetch remote data, we don't have signature |
46 | if (options.signatureActor && actorUrl !== options.signatureActor.url) { | 52 | if (options.signatureActor && actorUrl !== options.signatureActor.url) { |
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index e3fca0a17..ef20e404c 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts | |||
@@ -3,9 +3,7 @@ import { ActivityAudience, ActivityCreate } from '../../../../shared/models/acti | |||
3 | import { VideoPrivacy } from '../../../../shared/models/videos' | 3 | import { VideoPrivacy } from '../../../../shared/models/videos' |
4 | import { ActorModel } from '../../../models/activitypub/actor' | 4 | import { ActorModel } from '../../../models/activitypub/actor' |
5 | import { VideoModel } from '../../../models/video/video' | 5 | import { VideoModel } from '../../../models/video/video' |
6 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | ||
7 | import { VideoCommentModel } from '../../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../../models/video/video-comment' |
8 | import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url' | ||
9 | import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' | 7 | import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' |
10 | import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience' | 8 | import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience' |
11 | import { logger } from '../../../helpers/logger' | 9 | import { logger } from '../../../helpers/logger' |
@@ -25,31 +23,14 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) { | |||
25 | return broadcastToFollowers(createActivity, byActor, [ byActor ], t) | 23 | return broadcastToFollowers(createActivity, byActor, [ byActor ], t) |
26 | } | 24 | } |
27 | 25 | ||
28 | async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { | 26 | async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) { |
29 | if (!video.VideoChannel.Account.Actor.serverId) return // Local | ||
30 | |||
31 | const url = getVideoAbuseActivityPubUrl(videoAbuse) | ||
32 | |||
33 | logger.info('Creating job to send video abuse %s.', url) | ||
34 | |||
35 | // Custom audience, we only send the abuse to the origin instance | ||
36 | const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } | ||
37 | const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience) | ||
38 | |||
39 | return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
40 | } | ||
41 | |||
42 | async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { | ||
43 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url) | 27 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url) |
44 | 28 | ||
45 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) | ||
46 | const redundancyObject = fileRedundancy.toActivityPubObject() | ||
47 | |||
48 | return sendVideoRelatedCreateActivity({ | 29 | return sendVideoRelatedCreateActivity({ |
49 | byActor, | 30 | byActor, |
50 | video, | 31 | video, |
51 | url: fileRedundancy.url, | 32 | url: fileRedundancy.url, |
52 | object: redundancyObject | 33 | object: fileRedundancy.toActivityPubObject() |
53 | }) | 34 | }) |
54 | } | 35 | } |
55 | 36 | ||
@@ -91,37 +72,6 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio | |||
91 | return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) | 72 | return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) |
92 | } | 73 | } |
93 | 74 | ||
94 | async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) { | ||
95 | logger.info('Creating job to send view of %s.', video.url) | ||
96 | |||
97 | const url = getVideoViewActivityPubUrl(byActor, video) | ||
98 | const viewActivity = buildViewActivity(url, byActor, video) | ||
99 | |||
100 | return sendVideoRelatedCreateActivity({ | ||
101 | // Use the server actor to send the view | ||
102 | byActor, | ||
103 | video, | ||
104 | url, | ||
105 | object: viewActivity, | ||
106 | transaction: t | ||
107 | }) | ||
108 | } | ||
109 | |||
110 | async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { | ||
111 | logger.info('Creating job to dislike %s.', video.url) | ||
112 | |||
113 | const url = getVideoDislikeActivityPubUrl(byActor, video) | ||
114 | const dislikeActivity = buildDislikeActivity(url, byActor, video) | ||
115 | |||
116 | return sendVideoRelatedCreateActivity({ | ||
117 | byActor, | ||
118 | video, | ||
119 | url, | ||
120 | object: dislikeActivity, | ||
121 | transaction: t | ||
122 | }) | ||
123 | } | ||
124 | |||
125 | function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { | 75 | function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { |
126 | if (!audience) audience = getAudience(byActor) | 76 | if (!audience) audience = getAudience(byActor) |
127 | 77 | ||
@@ -136,33 +86,11 @@ function buildCreateActivity (url: string, byActor: ActorModel, object: any, aud | |||
136 | ) | 86 | ) |
137 | } | 87 | } |
138 | 88 | ||
139 | function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel) { | ||
140 | return { | ||
141 | id: url, | ||
142 | type: 'Dislike', | ||
143 | actor: byActor.url, | ||
144 | object: video.url | ||
145 | } | ||
146 | } | ||
147 | |||
148 | function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel) { | ||
149 | return { | ||
150 | id: url, | ||
151 | type: 'View', | ||
152 | actor: byActor.url, | ||
153 | object: video.url | ||
154 | } | ||
155 | } | ||
156 | |||
157 | // --------------------------------------------------------------------------- | 89 | // --------------------------------------------------------------------------- |
158 | 90 | ||
159 | export { | 91 | export { |
160 | sendCreateVideo, | 92 | sendCreateVideo, |
161 | sendVideoAbuse, | ||
162 | buildCreateActivity, | 93 | buildCreateActivity, |
163 | sendCreateView, | ||
164 | sendCreateDislike, | ||
165 | buildDislikeActivity, | ||
166 | sendCreateVideoComment, | 94 | sendCreateVideoComment, |
167 | sendCreateCacheFile | 95 | sendCreateCacheFile |
168 | } | 96 | } |
diff --git a/server/lib/activitypub/send/send-dislike.ts b/server/lib/activitypub/send/send-dislike.ts new file mode 100644 index 000000000..a88436f2c --- /dev/null +++ b/server/lib/activitypub/send/send-dislike.ts | |||
@@ -0,0 +1,41 @@ | |||
1 | import { Transaction } from 'sequelize' | ||
2 | import { ActorModel } from '../../../models/activitypub/actor' | ||
3 | import { VideoModel } from '../../../models/video/video' | ||
4 | import { getVideoDislikeActivityPubUrl } from '../url' | ||
5 | import { logger } from '../../../helpers/logger' | ||
6 | import { ActivityAudience, ActivityDislike } from '../../../../shared/models/activitypub' | ||
7 | import { sendVideoRelatedActivity } from './utils' | ||
8 | import { audiencify, getAudience } from '../audience' | ||
9 | |||
10 | async function sendDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { | ||
11 | logger.info('Creating job to dislike %s.', video.url) | ||
12 | |||
13 | const activityBuilder = (audience: ActivityAudience) => { | ||
14 | const url = getVideoDislikeActivityPubUrl(byActor, video) | ||
15 | |||
16 | return buildDislikeActivity(url, byActor, video, audience) | ||
17 | } | ||
18 | |||
19 | return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t }) | ||
20 | } | ||
21 | |||
22 | function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityDislike { | ||
23 | if (!audience) audience = getAudience(byActor) | ||
24 | |||
25 | return audiencify( | ||
26 | { | ||
27 | id: url, | ||
28 | type: 'Dislike' as 'Dislike', | ||
29 | actor: byActor.url, | ||
30 | object: video.url | ||
31 | }, | ||
32 | audience | ||
33 | ) | ||
34 | } | ||
35 | |||
36 | // --------------------------------------------------------------------------- | ||
37 | |||
38 | export { | ||
39 | sendDislike, | ||
40 | buildDislikeActivity | ||
41 | } | ||
diff --git a/server/lib/activitypub/send/send-flag.ts b/server/lib/activitypub/send/send-flag.ts new file mode 100644 index 000000000..96a7311b9 --- /dev/null +++ b/server/lib/activitypub/send/send-flag.ts | |||
@@ -0,0 +1,39 @@ | |||
1 | import { ActorModel } from '../../../models/activitypub/actor' | ||
2 | import { VideoModel } from '../../../models/video/video' | ||
3 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | ||
4 | import { getVideoAbuseActivityPubUrl } from '../url' | ||
5 | import { unicastTo } from './utils' | ||
6 | import { logger } from '../../../helpers/logger' | ||
7 | import { ActivityAudience, ActivityFlag } from '../../../../shared/models/activitypub' | ||
8 | import { audiencify, getAudience } from '../audience' | ||
9 | |||
10 | async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { | ||
11 | if (!video.VideoChannel.Account.Actor.serverId) return // Local user | ||
12 | |||
13 | const url = getVideoAbuseActivityPubUrl(videoAbuse) | ||
14 | |||
15 | logger.info('Creating job to send video abuse %s.', url) | ||
16 | |||
17 | // Custom audience, we only send the abuse to the origin instance | ||
18 | const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } | ||
19 | const flagActivity = buildFlagActivity(url, byActor, videoAbuse, audience) | ||
20 | |||
21 | return unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
22 | } | ||
23 | |||
24 | function buildFlagActivity (url: string, byActor: ActorModel, videoAbuse: VideoAbuseModel, audience: ActivityAudience): ActivityFlag { | ||
25 | if (!audience) audience = getAudience(byActor) | ||
26 | |||
27 | const activity = Object.assign( | ||
28 | { id: url, actor: byActor.url }, | ||
29 | videoAbuse.toActivityPubObject() | ||
30 | ) | ||
31 | |||
32 | return audiencify(activity, audience) | ||
33 | } | ||
34 | |||
35 | // --------------------------------------------------------------------------- | ||
36 | |||
37 | export { | ||
38 | sendVideoAbuse | ||
39 | } | ||
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index bf1b6e117..ecbf605d6 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts | |||
@@ -2,7 +2,7 @@ import { Transaction } from 'sequelize' | |||
2 | import { | 2 | import { |
3 | ActivityAnnounce, | 3 | ActivityAnnounce, |
4 | ActivityAudience, | 4 | ActivityAudience, |
5 | ActivityCreate, | 5 | ActivityCreate, ActivityDislike, |
6 | ActivityFollow, | 6 | ActivityFollow, |
7 | ActivityLike, | 7 | ActivityLike, |
8 | ActivityUndo | 8 | ActivityUndo |
@@ -13,13 +13,14 @@ import { VideoModel } from '../../../models/video/video' | |||
13 | import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' | 13 | import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' |
14 | import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' | 14 | import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' |
15 | import { audiencify, getAudience } from '../audience' | 15 | import { audiencify, getAudience } from '../audience' |
16 | import { buildCreateActivity, buildDislikeActivity } from './send-create' | 16 | import { buildCreateActivity } from './send-create' |
17 | import { buildFollowActivity } from './send-follow' | 17 | import { buildFollowActivity } from './send-follow' |
18 | import { buildLikeActivity } from './send-like' | 18 | import { buildLikeActivity } from './send-like' |
19 | import { VideoShareModel } from '../../../models/video/video-share' | 19 | import { VideoShareModel } from '../../../models/video/video-share' |
20 | import { buildAnnounceWithVideoAudience } from './send-announce' | 20 | import { buildAnnounceWithVideoAudience } from './send-announce' |
21 | import { logger } from '../../../helpers/logger' | 21 | import { logger } from '../../../helpers/logger' |
22 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | 22 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' |
23 | import { buildDislikeActivity } from './send-dislike' | ||
23 | 24 | ||
24 | async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { | 25 | async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { |
25 | const me = actorFollow.ActorFollower | 26 | const me = actorFollow.ActorFollower |
@@ -65,15 +66,15 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans | |||
65 | 66 | ||
66 | const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) | 67 | const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) |
67 | const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video) | 68 | const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video) |
68 | const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) | ||
69 | 69 | ||
70 | return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t }) | 70 | return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: dislikeActivity, transaction: t }) |
71 | } | 71 | } |
72 | 72 | ||
73 | async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { | 73 | async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { |
74 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) | 74 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) |
75 | 75 | ||
76 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | 76 | const videoId = redundancyModel.getVideo().id |
77 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
77 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) | 78 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) |
78 | 79 | ||
79 | return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) | 80 | return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) |
@@ -94,7 +95,7 @@ export { | |||
94 | function undoActivityData ( | 95 | function undoActivityData ( |
95 | url: string, | 96 | url: string, |
96 | byActor: ActorModel, | 97 | byActor: ActorModel, |
97 | object: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, | 98 | object: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce, |
98 | audience?: ActivityAudience | 99 | audience?: ActivityAudience |
99 | ): ActivityUndo { | 100 | ): ActivityUndo { |
100 | if (!audience) audience = getAudience(byActor) | 101 | if (!audience) audience = getAudience(byActor) |
@@ -114,7 +115,7 @@ async function sendUndoVideoRelatedActivity (options: { | |||
114 | byActor: ActorModel, | 115 | byActor: ActorModel, |
115 | video: VideoModel, | 116 | video: VideoModel, |
116 | url: string, | 117 | url: string, |
117 | activity: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, | 118 | activity: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce, |
118 | transaction: Transaction | 119 | transaction: Transaction |
119 | }) { | 120 | }) { |
120 | const activityBuilder = (audience: ActivityAudience) => { | 121 | const activityBuilder = (audience: ActivityAudience) => { |
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index a68f03edf..839f66470 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts | |||
@@ -61,7 +61,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod | |||
61 | async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { | 61 | async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { |
62 | logger.info('Creating job to update cache file %s.', redundancyModel.url) | 62 | logger.info('Creating job to update cache file %s.', redundancyModel.url) |
63 | 63 | ||
64 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | 64 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id) |
65 | 65 | ||
66 | const activityBuilder = (audience: ActivityAudience) => { | 66 | const activityBuilder = (audience: ActivityAudience) => { |
67 | const redundancyObject = redundancyModel.toActivityPubObject() | 67 | const redundancyObject = redundancyModel.toActivityPubObject() |
diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts new file mode 100644 index 000000000..8ad126be0 --- /dev/null +++ b/server/lib/activitypub/send/send-view.ts | |||
@@ -0,0 +1,40 @@ | |||
1 | import { Transaction } from 'sequelize' | ||
2 | import { ActivityAudience, ActivityView } from '../../../../shared/models/activitypub' | ||
3 | import { ActorModel } from '../../../models/activitypub/actor' | ||
4 | import { VideoModel } from '../../../models/video/video' | ||
5 | import { getVideoLikeActivityPubUrl } from '../url' | ||
6 | import { sendVideoRelatedActivity } from './utils' | ||
7 | import { audiencify, getAudience } from '../audience' | ||
8 | import { logger } from '../../../helpers/logger' | ||
9 | |||
10 | async function sendView (byActor: ActorModel, video: VideoModel, t: Transaction) { | ||
11 | logger.info('Creating job to send view of %s.', video.url) | ||
12 | |||
13 | const activityBuilder = (audience: ActivityAudience) => { | ||
14 | const url = getVideoLikeActivityPubUrl(byActor, video) | ||
15 | |||
16 | return buildViewActivity(url, byActor, video, audience) | ||
17 | } | ||
18 | |||
19 | return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t }) | ||
20 | } | ||
21 | |||
22 | function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityView { | ||
23 | if (!audience) audience = getAudience(byActor) | ||
24 | |||
25 | return audiencify( | ||
26 | { | ||
27 | id: url, | ||
28 | type: 'View' as 'View', | ||
29 | actor: byActor.url, | ||
30 | object: video.url | ||
31 | }, | ||
32 | audience | ||
33 | ) | ||
34 | } | ||
35 | |||
36 | // --------------------------------------------------------------------------- | ||
37 | |||
38 | export { | ||
39 | sendView | ||
40 | } | ||
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 5dcba778c..1767df0ae 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts | |||
@@ -11,7 +11,7 @@ import { doRequest } from '../../helpers/requests' | |||
11 | import { getOrCreateActorAndServerAndModel } from './actor' | 11 | import { getOrCreateActorAndServerAndModel } from './actor' |
12 | import { logger } from '../../helpers/logger' | 12 | import { logger } from '../../helpers/logger' |
13 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' | 13 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' |
14 | import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' | 14 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
15 | 15 | ||
16 | async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { | 16 | async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { |
17 | if (video.privacy === VideoPrivacy.PRIVATE) return undefined | 17 | if (video.privacy === VideoPrivacy.PRIVATE) return undefined |
@@ -41,7 +41,7 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) { | |||
41 | }) | 41 | }) |
42 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') | 42 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') |
43 | 43 | ||
44 | const actorUrl = getAPUrl(body.actor) | 44 | const actorUrl = getAPId(body.actor) |
45 | if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { | 45 | if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { |
46 | throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) | 46 | throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) |
47 | } | 47 | } |
@@ -78,7 +78,7 @@ async function shareByServer (video: VideoModel, t: Transaction) { | |||
78 | const serverActor = await getServerActor() | 78 | const serverActor = await getServerActor() |
79 | 79 | ||
80 | const serverShareUrl = getVideoAnnounceActivityPubUrl(serverActor, video) | 80 | const serverShareUrl = getVideoAnnounceActivityPubUrl(serverActor, video) |
81 | return VideoShareModel.findOrCreate({ | 81 | const [ serverShare ] = await VideoShareModel.findOrCreate({ |
82 | defaults: { | 82 | defaults: { |
83 | actorId: serverActor.id, | 83 | actorId: serverActor.id, |
84 | videoId: video.id, | 84 | videoId: video.id, |
@@ -88,16 +88,14 @@ async function shareByServer (video: VideoModel, t: Transaction) { | |||
88 | url: serverShareUrl | 88 | url: serverShareUrl |
89 | }, | 89 | }, |
90 | transaction: t | 90 | transaction: t |
91 | }).then(([ serverShare, created ]) => { | ||
92 | if (created) return sendVideoAnnounce(serverActor, serverShare, video, t) | ||
93 | |||
94 | return undefined | ||
95 | }) | 91 | }) |
92 | |||
93 | return sendVideoAnnounce(serverActor, serverShare, video, t) | ||
96 | } | 94 | } |
97 | 95 | ||
98 | async function shareByVideoChannel (video: VideoModel, t: Transaction) { | 96 | async function shareByVideoChannel (video: VideoModel, t: Transaction) { |
99 | const videoChannelShareUrl = getVideoAnnounceActivityPubUrl(video.VideoChannel.Actor, video) | 97 | const videoChannelShareUrl = getVideoAnnounceActivityPubUrl(video.VideoChannel.Actor, video) |
100 | return VideoShareModel.findOrCreate({ | 98 | const [ videoChannelShare ] = await VideoShareModel.findOrCreate({ |
101 | defaults: { | 99 | defaults: { |
102 | actorId: video.VideoChannel.actorId, | 100 | actorId: video.VideoChannel.actorId, |
103 | videoId: video.id, | 101 | videoId: video.id, |
@@ -107,11 +105,9 @@ async function shareByVideoChannel (video: VideoModel, t: Transaction) { | |||
107 | url: videoChannelShareUrl | 105 | url: videoChannelShareUrl |
108 | }, | 106 | }, |
109 | transaction: t | 107 | transaction: t |
110 | }).then(([ videoChannelShare, created ]) => { | ||
111 | if (created) return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t) | ||
112 | |||
113 | return undefined | ||
114 | }) | 108 | }) |
109 | |||
110 | return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t) | ||
115 | } | 111 | } |
116 | 112 | ||
117 | async function undoShareByVideoChannel (video: VideoModel, oldVideoChannel: VideoChannelModel, t: Transaction) { | 113 | async function undoShareByVideoChannel (video: VideoModel, oldVideoChannel: VideoChannelModel, t: Transaction) { |
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts index 38f15448c..4229fe094 100644 --- a/server/lib/activitypub/url.ts +++ b/server/lib/activitypub/url.ts | |||
@@ -5,6 +5,8 @@ import { VideoModel } from '../../models/video/video' | |||
5 | import { VideoAbuseModel } from '../../models/video/video-abuse' | 5 | import { VideoAbuseModel } from '../../models/video/video-abuse' |
6 | import { VideoCommentModel } from '../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../models/video/video-comment' |
7 | import { VideoFileModel } from '../../models/video/video-file' | 7 | import { VideoFileModel } from '../../models/video/video-file' |
8 | import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' | ||
9 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
8 | 10 | ||
9 | function getVideoActivityPubUrl (video: VideoModel) { | 11 | function getVideoActivityPubUrl (video: VideoModel) { |
10 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid | 12 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid |
@@ -16,6 +18,10 @@ function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) { | |||
16 | return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` | 18 | return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` |
17 | } | 19 | } |
18 | 20 | ||
21 | function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) { | ||
22 | return `${CONFIG.WEBSERVER.URL}/redundancy/video-playlists/${playlist.getStringType()}/${video.uuid}` | ||
23 | } | ||
24 | |||
19 | function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { | 25 | function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { |
20 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id | 26 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id |
21 | } | 27 | } |
@@ -92,6 +98,7 @@ function getUndoActivityPubUrl (originalUrl: string) { | |||
92 | 98 | ||
93 | export { | 99 | export { |
94 | getVideoActivityPubUrl, | 100 | getVideoActivityPubUrl, |
101 | getVideoCacheStreamingPlaylistActivityPubUrl, | ||
95 | getVideoChannelActivityPubUrl, | 102 | getVideoChannelActivityPubUrl, |
96 | getAccountActivityPubUrl, | 103 | getAccountActivityPubUrl, |
97 | getVideoAbuseActivityPubUrl, | 104 | getVideoAbuseActivityPubUrl, |
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 2cce67f0c..7aac79118 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
2 | import { AccountModel } from '../../models/account/account' | 2 | import { AccountModel } from '../../models/account/account' |
3 | import { VideoModel } from '../../models/video/video' | 3 | import { VideoModel } from '../../models/video/video' |
4 | import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send' | 4 | import { sendLike, sendUndoDislike, sendUndoLike } from './send' |
5 | import { VideoRateType } from '../../../shared/models/videos' | 5 | import { VideoRateType } from '../../../shared/models/videos' |
6 | import * as Bluebird from 'bluebird' | 6 | import * as Bluebird from 'bluebird' |
7 | import { getOrCreateActorAndServerAndModel } from './actor' | 7 | import { getOrCreateActorAndServerAndModel } from './actor' |
@@ -9,9 +9,10 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate' | |||
9 | import { logger } from '../../helpers/logger' | 9 | import { logger } from '../../helpers/logger' |
10 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' | 10 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' |
11 | import { doRequest } from '../../helpers/requests' | 11 | import { doRequest } from '../../helpers/requests' |
12 | import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' | 12 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
13 | import { ActorModel } from '../../models/activitypub/actor' | 13 | import { ActorModel } from '../../models/activitypub/actor' |
14 | import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url' | 14 | import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url' |
15 | import { sendDislike } from './send/send-dislike' | ||
15 | 16 | ||
16 | async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRateType) { | 17 | async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRateType) { |
17 | let rateCounts = 0 | 18 | let rateCounts = 0 |
@@ -26,7 +27,7 @@ async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRa | |||
26 | }) | 27 | }) |
27 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') | 28 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') |
28 | 29 | ||
29 | const actorUrl = getAPUrl(body.actor) | 30 | const actorUrl = getAPId(body.actor) |
30 | if (checkUrlsSameHost(actorUrl, rateUrl) !== true) { | 31 | if (checkUrlsSameHost(actorUrl, rateUrl) !== true) { |
31 | throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`) | 32 | throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`) |
32 | } | 33 | } |
@@ -82,7 +83,7 @@ async function sendVideoRateChange (account: AccountModel, | |||
82 | // Like | 83 | // Like |
83 | if (likes > 0) await sendLike(actor, video, t) | 84 | if (likes > 0) await sendLike(actor, video, t) |
84 | // Dislike | 85 | // Dislike |
85 | if (dislikes > 0) await sendCreateDislike(actor, video, t) | 86 | if (dislikes > 0) await sendDislike(actor, video, t) |
86 | } | 87 | } |
87 | 88 | ||
88 | function getRateUrl (rateType: VideoRateType, actor: ActorModel, video: VideoModel) { | 89 | function getRateUrl (rateType: VideoRateType, actor: ActorModel, video: VideoModel) { |
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 893768769..710929aac 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -2,7 +2,14 @@ import * as Bluebird from 'bluebird' | |||
2 | import * as sequelize from 'sequelize' | 2 | import * as sequelize from 'sequelize' |
3 | import * as magnetUtil from 'magnet-uri' | 3 | import * as magnetUtil from 'magnet-uri' |
4 | import * as request from 'request' | 4 | import * as request from 'request' |
5 | import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' | 5 | import { |
6 | ActivityIconObject, | ||
7 | ActivityPlaylistSegmentHashesObject, | ||
8 | ActivityPlaylistUrlObject, | ||
9 | ActivityUrlObject, | ||
10 | ActivityVideoUrlObject, | ||
11 | VideoState | ||
12 | } from '../../../shared/index' | ||
6 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 13 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
7 | import { VideoPrivacy } from '../../../shared/models/videos' | 14 | import { VideoPrivacy } from '../../../shared/models/videos' |
8 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' | 15 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' |
@@ -28,8 +35,11 @@ import { createRates } from './video-rates' | |||
28 | import { addVideoShares, shareVideoByServerAndChannel } from './share' | 35 | import { addVideoShares, shareVideoByServerAndChannel } from './share' |
29 | import { AccountModel } from '../../models/account/account' | 36 | import { AccountModel } from '../../models/account/account' |
30 | import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' | 37 | import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' |
31 | import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' | 38 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
32 | import { Notifier } from '../notifier' | 39 | import { Notifier } from '../notifier' |
40 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
41 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
42 | import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' | ||
33 | 43 | ||
34 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { | 44 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { |
35 | // If the video is not private and published, we federate it | 45 | // If the video is not private and published, we federate it |
@@ -155,7 +165,7 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid | |||
155 | } | 165 | } |
156 | 166 | ||
157 | async function getOrCreateVideoAndAccountAndChannel (options: { | 167 | async function getOrCreateVideoAndAccountAndChannel (options: { |
158 | videoObject: VideoTorrentObject | string, | 168 | videoObject: { id: string } | string, |
159 | syncParam?: SyncParam, | 169 | syncParam?: SyncParam, |
160 | fetchType?: VideoFetchByUrlType, | 170 | fetchType?: VideoFetchByUrlType, |
161 | allowRefresh?: boolean // true by default | 171 | allowRefresh?: boolean // true by default |
@@ -166,7 +176,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { | |||
166 | const allowRefresh = options.allowRefresh !== false | 176 | const allowRefresh = options.allowRefresh !== false |
167 | 177 | ||
168 | // Get video url | 178 | // Get video url |
169 | const videoUrl = getAPUrl(options.videoObject) | 179 | const videoUrl = getAPId(options.videoObject) |
170 | 180 | ||
171 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | 181 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) |
172 | if (videoFromDatabase) { | 182 | if (videoFromDatabase) { |
@@ -179,7 +189,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { | |||
179 | } | 189 | } |
180 | 190 | ||
181 | if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) | 191 | if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) |
182 | else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoFromDatabase.url } }) | 192 | else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } }) |
183 | } | 193 | } |
184 | 194 | ||
185 | return { video: videoFromDatabase, created: false } | 195 | return { video: videoFromDatabase, created: false } |
@@ -233,6 +243,7 @@ async function updateVideoFromAP (options: { | |||
233 | options.video.set('support', videoData.support) | 243 | options.video.set('support', videoData.support) |
234 | options.video.set('nsfw', videoData.nsfw) | 244 | options.video.set('nsfw', videoData.nsfw) |
235 | options.video.set('commentsEnabled', videoData.commentsEnabled) | 245 | options.video.set('commentsEnabled', videoData.commentsEnabled) |
246 | options.video.set('downloadEnabled', videoData.downloadEnabled) | ||
236 | options.video.set('waitTranscoding', videoData.waitTranscoding) | 247 | options.video.set('waitTranscoding', videoData.waitTranscoding) |
237 | options.video.set('state', videoData.state) | 248 | options.video.set('state', videoData.state) |
238 | options.video.set('duration', videoData.duration) | 249 | options.video.set('duration', videoData.duration) |
@@ -264,6 +275,25 @@ async function updateVideoFromAP (options: { | |||
264 | } | 275 | } |
265 | 276 | ||
266 | { | 277 | { |
278 | const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject) | ||
279 | const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) | ||
280 | |||
281 | // Remove video files that do not exist anymore | ||
282 | const destroyTasks = options.video.VideoStreamingPlaylists | ||
283 | .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f))) | ||
284 | .map(f => f.destroy(sequelizeOptions)) | ||
285 | await Promise.all(destroyTasks) | ||
286 | |||
287 | // Update or add other one | ||
288 | const upsertTasks = streamingPlaylistAttributes.map(a => { | ||
289 | return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t }) | ||
290 | .then(([ streamingPlaylist ]) => streamingPlaylist) | ||
291 | }) | ||
292 | |||
293 | options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks) | ||
294 | } | ||
295 | |||
296 | { | ||
267 | // Update Tags | 297 | // Update Tags |
268 | const tags = options.videoObject.tag.map(tag => tag.name) | 298 | const tags = options.videoObject.tag.map(tag => tag.name) |
269 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 299 | const tagInstances = await TagModel.findOrCreateTags(tags, t) |
@@ -367,13 +397,25 @@ export { | |||
367 | 397 | ||
368 | // --------------------------------------------------------------------------- | 398 | // --------------------------------------------------------------------------- |
369 | 399 | ||
370 | function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { | 400 | function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { |
371 | const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) | 401 | const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) |
372 | 402 | ||
373 | const urlMediaType = url.mediaType || url.mimeType | 403 | const urlMediaType = url.mediaType || url.mimeType |
374 | return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') | 404 | return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') |
375 | } | 405 | } |
376 | 406 | ||
407 | function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject { | ||
408 | const urlMediaType = url.mediaType || url.mimeType | ||
409 | |||
410 | return urlMediaType === 'application/x-mpegURL' | ||
411 | } | ||
412 | |||
413 | function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { | ||
414 | const urlMediaType = tag.mediaType || tag.mimeType | ||
415 | |||
416 | return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json' | ||
417 | } | ||
418 | |||
377 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { | 419 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { |
378 | logger.debug('Adding remote video %s.', videoObject.id) | 420 | logger.debug('Adding remote video %s.', videoObject.id) |
379 | 421 | ||
@@ -394,8 +436,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor | |||
394 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) | 436 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) |
395 | await Promise.all(videoFilePromises) | 437 | await Promise.all(videoFilePromises) |
396 | 438 | ||
439 | const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject) | ||
440 | const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t })) | ||
441 | await Promise.all(playlistPromises) | ||
442 | |||
397 | // Process tags | 443 | // Process tags |
398 | const tags = videoObject.tag.map(t => t.name) | 444 | const tags = videoObject.tag |
445 | .filter(t => t.type === 'Hashtag') | ||
446 | .map(t => t.name) | ||
399 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 447 | const tagInstances = await TagModel.findOrCreateTags(tags, t) |
400 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) | 448 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) |
401 | 449 | ||
@@ -456,6 +504,7 @@ async function videoActivityObjectToDBAttributes ( | |||
456 | support, | 504 | support, |
457 | nsfw: videoObject.sensitive, | 505 | nsfw: videoObject.sensitive, |
458 | commentsEnabled: videoObject.commentsEnabled, | 506 | commentsEnabled: videoObject.commentsEnabled, |
507 | downloadEnabled: videoObject.downloadEnabled, | ||
459 | waitTranscoding: videoObject.waitTranscoding, | 508 | waitTranscoding: videoObject.waitTranscoding, |
460 | state: videoObject.state, | 509 | state: videoObject.state, |
461 | channelId: videoChannel.id, | 510 | channelId: videoChannel.id, |
@@ -473,13 +522,13 @@ async function videoActivityObjectToDBAttributes ( | |||
473 | } | 522 | } |
474 | 523 | ||
475 | function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { | 524 | function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { |
476 | const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] | 525 | const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] |
477 | 526 | ||
478 | if (fileUrls.length === 0) { | 527 | if (fileUrls.length === 0) { |
479 | throw new Error('Cannot find video files for ' + video.url) | 528 | throw new Error('Cannot find video files for ' + video.url) |
480 | } | 529 | } |
481 | 530 | ||
482 | const attributes: VideoFileModel[] = [] | 531 | const attributes: FilteredModelAttributes<VideoFileModel>[] = [] |
483 | for (const fileUrl of fileUrls) { | 532 | for (const fileUrl of fileUrls) { |
484 | // Fetch associated magnet uri | 533 | // Fetch associated magnet uri |
485 | const magnet = videoObject.url.find(u => { | 534 | const magnet = videoObject.url.find(u => { |
@@ -502,7 +551,45 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid | |||
502 | size: fileUrl.size, | 551 | size: fileUrl.size, |
503 | videoId: video.id, | 552 | videoId: video.id, |
504 | fps: fileUrl.fps || -1 | 553 | fps: fileUrl.fps || -1 |
505 | } as VideoFileModel | 554 | } |
555 | |||
556 | attributes.push(attribute) | ||
557 | } | ||
558 | |||
559 | return attributes | ||
560 | } | ||
561 | |||
562 | function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { | ||
563 | const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] | ||
564 | if (playlistUrls.length === 0) return [] | ||
565 | |||
566 | const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = [] | ||
567 | for (const playlistUrlObject of playlistUrls) { | ||
568 | const p2pMediaLoaderInfohashes = playlistUrlObject.tag | ||
569 | .filter(t => t.type === 'Infohash') | ||
570 | .map(t => t.name) | ||
571 | if (p2pMediaLoaderInfohashes.length === 0) { | ||
572 | logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject }) | ||
573 | continue | ||
574 | } | ||
575 | |||
576 | const segmentsSha256UrlObject = playlistUrlObject.tag | ||
577 | .find(t => { | ||
578 | return isAPPlaylistSegmentHashesUrlObject(t) | ||
579 | }) as ActivityPlaylistSegmentHashesObject | ||
580 | if (!segmentsSha256UrlObject) { | ||
581 | logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) | ||
582 | continue | ||
583 | } | ||
584 | |||
585 | const attribute = { | ||
586 | type: VideoStreamingPlaylistType.HLS, | ||
587 | playlistUrl: playlistUrlObject.href, | ||
588 | segmentsSha256Url: segmentsSha256UrlObject.href, | ||
589 | p2pMediaLoaderInfohashes, | ||
590 | videoId: video.id | ||
591 | } | ||
592 | |||
506 | attributes.push(attribute) | 593 | attributes.push(attribute) |
507 | } | 594 | } |
508 | 595 | ||
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index 1875ec1fc..b2c376e20 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import * as Bluebird from 'bluebird' | 2 | import * as Bluebird from 'bluebird' |
3 | import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/models/i18n/i18n' | 3 | import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/models/i18n/i18n' |
4 | import { CONFIG, CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE, STATIC_PATHS } from '../initializers' | 4 | import { CONFIG, CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE } from '../initializers' |
5 | import { join } from 'path' | 5 | import { join } from 'path' |
6 | import { escapeHTML } from '../helpers/core-utils' | 6 | import { escapeHTML } from '../helpers/core-utils' |
7 | import { VideoModel } from '../models/video/video' | 7 | import { VideoModel } from '../models/video/video' |
@@ -187,8 +187,8 @@ export class ClientHtml { | |||
187 | // Schema.org | 187 | // Schema.org |
188 | tagsString += `<script type="application/ld+json">${JSON.stringify(schemaTags)}</script>` | 188 | tagsString += `<script type="application/ld+json">${JSON.stringify(schemaTags)}</script>` |
189 | 189 | ||
190 | // SEO | 190 | // SEO, use origin video url so Google does not index remote videos |
191 | tagsString += `<link rel="canonical" href="${videoUrl}" />` | 191 | tagsString += `<link rel="canonical" href="${video.url}" />` |
192 | 192 | ||
193 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.OPENGRAPH_AND_OEMBED, tagsString) | 193 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.OPENGRAPH_AND_OEMBED, tagsString) |
194 | } | 194 | } |
diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index f384a254e..672414cc0 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts | |||
@@ -296,9 +296,9 @@ class Emailer { | |||
296 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | 296 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) |
297 | } | 297 | } |
298 | 298 | ||
299 | addForgetPasswordEmailJob (to: string, resetPasswordUrl: string) { | 299 | addPasswordResetEmailJob (to: string, resetPasswordUrl: string) { |
300 | const text = `Hi dear user,\n\n` + | 300 | const text = `Hi dear user,\n\n` + |
301 | `It seems you forgot your password on ${CONFIG.WEBSERVER.HOST}! ` + | 301 | `A reset password procedure for your account ${to} has been requested on ${CONFIG.WEBSERVER.HOST} ` + |
302 | `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + | 302 | `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + |
303 | `If you are not the person who initiated this request, please ignore this email.\n\n` + | 303 | `If you are not the person who initiated this request, please ignore this email.\n\n` + |
304 | `Cheers,\n` + | 304 | `Cheers,\n` + |
diff --git a/server/lib/hls.ts b/server/lib/hls.ts new file mode 100644 index 000000000..3575981f4 --- /dev/null +++ b/server/lib/hls.ts | |||
@@ -0,0 +1,164 @@ | |||
1 | import { VideoModel } from '../models/video/video' | ||
2 | import { basename, join, dirname } from 'path' | ||
3 | import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers' | ||
4 | import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' | ||
5 | import { getVideoFileSize } from '../helpers/ffmpeg-utils' | ||
6 | import { sha256 } from '../helpers/core-utils' | ||
7 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
8 | import { logger } from '../helpers/logger' | ||
9 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' | ||
10 | import { generateRandomString } from '../helpers/utils' | ||
11 | import { flatten, uniq } from 'lodash' | ||
12 | |||
13 | async function updateMasterHLSPlaylist (video: VideoModel) { | ||
14 | const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) | ||
15 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] | ||
16 | const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) | ||
17 | |||
18 | for (const file of video.VideoFiles) { | ||
19 | // If we did not generated a playlist for this resolution, skip | ||
20 | const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
21 | if (await pathExists(filePlaylistPath) === false) continue | ||
22 | |||
23 | const videoFilePath = video.getVideoFilePath(file) | ||
24 | |||
25 | const size = await getVideoFileSize(videoFilePath) | ||
26 | |||
27 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | ||
28 | const resolution = `RESOLUTION=${size.width}x${size.height}` | ||
29 | |||
30 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | ||
31 | if (file.fps) line += ',FRAME-RATE=' + file.fps | ||
32 | |||
33 | masterPlaylists.push(line) | ||
34 | masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
35 | } | ||
36 | |||
37 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | ||
38 | } | ||
39 | |||
40 | async function updateSha256Segments (video: VideoModel) { | ||
41 | const json: { [filename: string]: { [range: string]: string } } = {} | ||
42 | |||
43 | const playlistDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) | ||
44 | |||
45 | // For all the resolutions available for this video | ||
46 | for (const file of video.VideoFiles) { | ||
47 | const rangeHashes: { [range: string]: string } = {} | ||
48 | |||
49 | const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)) | ||
50 | const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
51 | |||
52 | // Maybe the playlist is not generated for this resolution yet | ||
53 | if (!await pathExists(playlistPath)) continue | ||
54 | |||
55 | const playlistContent = await readFile(playlistPath) | ||
56 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | ||
57 | |||
58 | const fd = await open(videoPath, 'r') | ||
59 | for (const range of ranges) { | ||
60 | const buf = Buffer.alloc(range.length) | ||
61 | await read(fd, buf, 0, range.length, range.offset) | ||
62 | |||
63 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) | ||
64 | } | ||
65 | await close(fd) | ||
66 | |||
67 | const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution) | ||
68 | json[videoFilename] = rangeHashes | ||
69 | } | ||
70 | |||
71 | const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) | ||
72 | await outputJSON(outputPath, json) | ||
73 | } | ||
74 | |||
75 | function getRangesFromPlaylist (playlistContent: string) { | ||
76 | const ranges: { offset: number, length: number }[] = [] | ||
77 | const lines = playlistContent.split('\n') | ||
78 | const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ | ||
79 | |||
80 | for (const line of lines) { | ||
81 | const captured = regex.exec(line) | ||
82 | |||
83 | if (captured) { | ||
84 | ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) | ||
85 | } | ||
86 | } | ||
87 | |||
88 | return ranges | ||
89 | } | ||
90 | |||
91 | function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { | ||
92 | let timer | ||
93 | |||
94 | logger.info('Importing HLS playlist %s', playlistUrl) | ||
95 | |||
96 | return new Promise<string>(async (res, rej) => { | ||
97 | const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) | ||
98 | |||
99 | await ensureDir(tmpDirectory) | ||
100 | |||
101 | timer = setTimeout(() => { | ||
102 | deleteTmpDirectory(tmpDirectory) | ||
103 | |||
104 | return rej(new Error('HLS download timeout.')) | ||
105 | }, timeout) | ||
106 | |||
107 | try { | ||
108 | // Fetch master playlist | ||
109 | const subPlaylistUrls = await fetchUniqUrls(playlistUrl) | ||
110 | |||
111 | const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) | ||
112 | const fileUrls = uniq(flatten(await Promise.all(subRequests))) | ||
113 | |||
114 | logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) | ||
115 | |||
116 | for (const fileUrl of fileUrls) { | ||
117 | const destPath = join(tmpDirectory, basename(fileUrl)) | ||
118 | |||
119 | await doRequestAndSaveToFile({ uri: fileUrl }, destPath) | ||
120 | } | ||
121 | |||
122 | clearTimeout(timer) | ||
123 | |||
124 | await move(tmpDirectory, destinationDir, { overwrite: true }) | ||
125 | |||
126 | return res() | ||
127 | } catch (err) { | ||
128 | deleteTmpDirectory(tmpDirectory) | ||
129 | |||
130 | return rej(err) | ||
131 | } | ||
132 | }) | ||
133 | |||
134 | function deleteTmpDirectory (directory: string) { | ||
135 | remove(directory) | ||
136 | .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) | ||
137 | } | ||
138 | |||
139 | async function fetchUniqUrls (playlistUrl: string) { | ||
140 | const { body } = await doRequest<string>({ uri: playlistUrl }) | ||
141 | |||
142 | if (!body) return [] | ||
143 | |||
144 | const urls = body.split('\n') | ||
145 | .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) | ||
146 | .map(url => { | ||
147 | if (url.startsWith('http://') || url.startsWith('https://')) return url | ||
148 | |||
149 | return `${dirname(playlistUrl)}/${url}` | ||
150 | }) | ||
151 | |||
152 | return uniq(urls) | ||
153 | } | ||
154 | } | ||
155 | |||
156 | // --------------------------------------------------------------------------- | ||
157 | |||
158 | export { | ||
159 | updateMasterHLSPlaylist, | ||
160 | updateSha256Segments, | ||
161 | downloadPlaylistSegments | ||
162 | } | ||
163 | |||
164 | // --------------------------------------------------------------------------- | ||
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts index 671b0f487..454b975fe 100644 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts | |||
@@ -1,30 +1,33 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { logger } from '../../../helpers/logger' | 2 | import { logger } from '../../../helpers/logger' |
3 | import { fetchVideoByUrl } from '../../../helpers/video' | 3 | import { fetchVideoByUrl } from '../../../helpers/video' |
4 | import { refreshVideoIfNeeded } from '../../activitypub' | 4 | import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub' |
5 | import { ActorModel } from '../../../models/activitypub/actor' | ||
5 | 6 | ||
6 | export type RefreshPayload = { | 7 | export type RefreshPayload = { |
7 | videoUrl: string | 8 | type: 'video' | 'actor' |
8 | type: 'video' | 9 | url: string |
9 | } | 10 | } |
10 | 11 | ||
11 | async function refreshAPObject (job: Bull.Job) { | 12 | async function refreshAPObject (job: Bull.Job) { |
12 | const payload = job.data as RefreshPayload | 13 | const payload = job.data as RefreshPayload |
13 | 14 | ||
14 | logger.info('Processing AP refresher in job %d for video %s.', job.id, payload.videoUrl) | 15 | logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url) |
15 | 16 | ||
16 | if (payload.type === 'video') return refreshAPVideo(payload.videoUrl) | 17 | if (payload.type === 'video') return refreshVideo(payload.url) |
18 | if (payload.type === 'actor') return refreshActor(payload.url) | ||
17 | } | 19 | } |
18 | 20 | ||
19 | // --------------------------------------------------------------------------- | 21 | // --------------------------------------------------------------------------- |
20 | 22 | ||
21 | export { | 23 | export { |
24 | refreshActor, | ||
22 | refreshAPObject | 25 | refreshAPObject |
23 | } | 26 | } |
24 | 27 | ||
25 | // --------------------------------------------------------------------------- | 28 | // --------------------------------------------------------------------------- |
26 | 29 | ||
27 | async function refreshAPVideo (videoUrl: string) { | 30 | async function refreshVideo (videoUrl: string) { |
28 | const fetchType = 'all' as 'all' | 31 | const fetchType = 'all' as 'all' |
29 | const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } | 32 | const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } |
30 | 33 | ||
@@ -39,3 +42,13 @@ async function refreshAPVideo (videoUrl: string) { | |||
39 | await refreshVideoIfNeeded(refreshOptions) | 42 | await refreshVideoIfNeeded(refreshOptions) |
40 | } | 43 | } |
41 | } | 44 | } |
45 | |||
46 | async function refreshActor (actorUrl: string) { | ||
47 | const fetchType = 'all' as 'all' | ||
48 | const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl) | ||
49 | |||
50 | if (actor) { | ||
51 | await refreshActorIfNeeded(actor, fetchType) | ||
52 | } | ||
53 | |||
54 | } | ||
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 593e43cc5..04983155c 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts | |||
@@ -5,17 +5,18 @@ import { VideoModel } from '../../../models/video/video' | |||
5 | import { JobQueue } from '../job-queue' | 5 | import { JobQueue } from '../job-queue' |
6 | import { federateVideoIfNeeded } from '../../activitypub' | 6 | import { federateVideoIfNeeded } from '../../activitypub' |
7 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 7 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
8 | import { sequelizeTypescript } from '../../../initializers' | 8 | import { sequelizeTypescript, CONFIG } from '../../../initializers' |
9 | import * as Bluebird from 'bluebird' | 9 | import * as Bluebird from 'bluebird' |
10 | import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' | 10 | import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' |
11 | import { importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' | 11 | import { generateHlsPlaylist, importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' |
12 | import { Notifier } from '../../notifier' | 12 | import { Notifier } from '../../notifier' |
13 | 13 | ||
14 | export type VideoFilePayload = { | 14 | export type VideoFilePayload = { |
15 | videoUUID: string | 15 | videoUUID: string |
16 | isNewVideo?: boolean | ||
17 | resolution?: VideoResolution | 16 | resolution?: VideoResolution |
17 | isNewVideo?: boolean | ||
18 | isPortraitMode?: boolean | 18 | isPortraitMode?: boolean |
19 | generateHlsPlaylist?: boolean | ||
19 | } | 20 | } |
20 | 21 | ||
21 | export type VideoFileImportPayload = { | 22 | export type VideoFileImportPayload = { |
@@ -51,21 +52,38 @@ async function processVideoFile (job: Bull.Job) { | |||
51 | return undefined | 52 | return undefined |
52 | } | 53 | } |
53 | 54 | ||
54 | // Transcoding in other resolution | 55 | if (payload.generateHlsPlaylist) { |
55 | if (payload.resolution) { | 56 | await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false) |
57 | |||
58 | await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video) | ||
59 | } else if (payload.resolution) { // Transcoding in other resolution | ||
56 | await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) | 60 | await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) |
57 | 61 | ||
58 | await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) | 62 | await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video, payload) |
59 | } else { | 63 | } else { |
60 | await optimizeVideofile(video) | 64 | await optimizeVideofile(video) |
61 | 65 | ||
62 | await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) | 66 | await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload) |
63 | } | 67 | } |
64 | 68 | ||
65 | return video | 69 | return video |
66 | } | 70 | } |
67 | 71 | ||
68 | async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { | 72 | async function onHlsPlaylistGenerationSuccess (video: VideoModel) { |
73 | if (video === undefined) return undefined | ||
74 | |||
75 | await sequelizeTypescript.transaction(async t => { | ||
76 | // Maybe the video changed in database, refresh it | ||
77 | let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | ||
78 | // Video does not exist anymore | ||
79 | if (!videoDatabase) return undefined | ||
80 | |||
81 | // If the video was not published, we consider it is a new one for other instances | ||
82 | await federateVideoIfNeeded(videoDatabase, false, t) | ||
83 | }) | ||
84 | } | ||
85 | |||
86 | async function onVideoFileTranscoderOrImportSuccess (video: VideoModel, payload?: VideoFilePayload) { | ||
69 | if (video === undefined) return undefined | 87 | if (video === undefined) return undefined |
70 | 88 | ||
71 | const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { | 89 | const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { |
@@ -91,13 +109,16 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { | |||
91 | return { videoDatabase, videoPublished } | 109 | return { videoDatabase, videoPublished } |
92 | }) | 110 | }) |
93 | 111 | ||
94 | if (videoPublished) { | 112 | // don't notify prior to scheduled video update |
113 | if (videoPublished && !videoDatabase.ScheduleVideoUpdate) { | ||
95 | Notifier.Instance.notifyOnNewVideo(videoDatabase) | 114 | Notifier.Instance.notifyOnNewVideo(videoDatabase) |
96 | Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) | 115 | Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) |
97 | } | 116 | } |
117 | |||
118 | await createHlsJobIfEnabled(payload) | ||
98 | } | 119 | } |
99 | 120 | ||
100 | async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) { | 121 | async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoFilePayload) { |
101 | if (videoArg === undefined) return undefined | 122 | if (videoArg === undefined) return undefined |
102 | 123 | ||
103 | // Outside the transaction (IO on disk) | 124 | // Outside the transaction (IO on disk) |
@@ -144,13 +165,18 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo | |||
144 | logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) | 165 | logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) |
145 | } | 166 | } |
146 | 167 | ||
147 | await federateVideoIfNeeded(videoDatabase, isNewVideo, t) | 168 | await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t) |
148 | 169 | ||
149 | return { videoDatabase, videoPublished } | 170 | return { videoDatabase, videoPublished } |
150 | }) | 171 | }) |
151 | 172 | ||
152 | if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) | 173 | // don't notify prior to scheduled video update |
153 | if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) | 174 | if (!videoDatabase.ScheduleVideoUpdate) { |
175 | if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) | ||
176 | if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) | ||
177 | } | ||
178 | |||
179 | await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution })) | ||
154 | } | 180 | } |
155 | 181 | ||
156 | // --------------------------------------------------------------------------- | 182 | // --------------------------------------------------------------------------- |
@@ -159,3 +185,20 @@ export { | |||
159 | processVideoFile, | 185 | processVideoFile, |
160 | processVideoFileImport | 186 | processVideoFileImport |
161 | } | 187 | } |
188 | |||
189 | // --------------------------------------------------------------------------- | ||
190 | |||
191 | function createHlsJobIfEnabled (payload?: VideoFilePayload) { | ||
192 | // Generate HLS playlist? | ||
193 | if (payload && CONFIG.TRANSCODING.HLS.ENABLED) { | ||
194 | const hlsTranscodingPayload = { | ||
195 | videoUUID: payload.videoUUID, | ||
196 | resolution: payload.resolution, | ||
197 | isPortraitMode: payload.isPortraitMode, | ||
198 | |||
199 | generateHlsPlaylist: true | ||
200 | } | ||
201 | |||
202 | return JobQueue.Instance.createJob({ type: 'video-file', payload: hlsTranscodingPayload }) | ||
203 | } | ||
204 | } | ||
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index f643ee226..1a48f2bd0 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { AbstractScheduler } from './abstract-scheduler' | 1 | import { AbstractScheduler } from './abstract-scheduler' |
2 | import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' | 2 | import { CONFIG, HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { VideosRedundancy } from '../../../shared/models/redundancy' | 4 | import { VideosRedundancy } from '../../../shared/models/redundancy' |
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
@@ -9,9 +9,19 @@ import { join } from 'path' | |||
9 | import { move } from 'fs-extra' | 9 | import { move } from 'fs-extra' |
10 | import { getServerActor } from '../../helpers/utils' | 10 | import { getServerActor } from '../../helpers/utils' |
11 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | 11 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' |
12 | import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' | 12 | import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' |
13 | import { removeVideoRedundancy } from '../redundancy' | 13 | import { removeVideoRedundancy } from '../redundancy' |
14 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' | 14 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' |
15 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
16 | import { VideoModel } from '../../models/video/video' | ||
17 | import { downloadPlaylistSegments } from '../hls' | ||
18 | |||
19 | type CandidateToDuplicate = { | ||
20 | redundancy: VideosRedundancy, | ||
21 | video: VideoModel, | ||
22 | files: VideoFileModel[], | ||
23 | streamingPlaylists: VideoStreamingPlaylistModel[] | ||
24 | } | ||
15 | 25 | ||
16 | export class VideosRedundancyScheduler extends AbstractScheduler { | 26 | export class VideosRedundancyScheduler extends AbstractScheduler { |
17 | 27 | ||
@@ -24,28 +34,32 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
24 | } | 34 | } |
25 | 35 | ||
26 | protected async internalExecute () { | 36 | protected async internalExecute () { |
27 | for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { | 37 | for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { |
28 | logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) | 38 | logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) |
29 | 39 | ||
30 | try { | 40 | try { |
31 | const videoToDuplicate = await this.findVideoToDuplicate(obj) | 41 | const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig) |
32 | if (!videoToDuplicate) continue | 42 | if (!videoToDuplicate) continue |
33 | 43 | ||
34 | const videoFiles = videoToDuplicate.VideoFiles | 44 | const candidateToDuplicate = { |
35 | videoFiles.forEach(f => f.Video = videoToDuplicate) | 45 | video: videoToDuplicate, |
46 | redundancy: redundancyConfig, | ||
47 | files: videoToDuplicate.VideoFiles, | ||
48 | streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists | ||
49 | } | ||
36 | 50 | ||
37 | await this.purgeCacheIfNeeded(obj, videoFiles) | 51 | await this.purgeCacheIfNeeded(candidateToDuplicate) |
38 | 52 | ||
39 | if (await this.isTooHeavy(obj, videoFiles)) { | 53 | if (await this.isTooHeavy(candidateToDuplicate)) { |
40 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) | 54 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) |
41 | continue | 55 | continue |
42 | } | 56 | } |
43 | 57 | ||
44 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) | 58 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy) |
45 | 59 | ||
46 | await this.createVideoRedundancy(obj, videoFiles) | 60 | await this.createVideoRedundancies(candidateToDuplicate) |
47 | } catch (err) { | 61 | } catch (err) { |
48 | logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) | 62 | logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err }) |
49 | } | 63 | } |
50 | } | 64 | } |
51 | 65 | ||
@@ -63,25 +77,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
63 | 77 | ||
64 | for (const redundancyModel of expired) { | 78 | for (const redundancyModel of expired) { |
65 | try { | 79 | try { |
66 | await this.extendsOrDeleteRedundancy(redundancyModel) | 80 | const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) |
81 | const candidate = { | ||
82 | redundancy: redundancyConfig, | ||
83 | video: null, | ||
84 | files: [], | ||
85 | streamingPlaylists: [] | ||
86 | } | ||
87 | |||
88 | // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it | ||
89 | if (!redundancyConfig || await this.isTooHeavy(candidate)) { | ||
90 | logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) | ||
91 | await removeVideoRedundancy(redundancyModel) | ||
92 | } else { | ||
93 | await this.extendsRedundancy(redundancyModel) | ||
94 | } | ||
67 | } catch (err) { | 95 | } catch (err) { |
68 | logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel)) | 96 | logger.error( |
97 | 'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel), | ||
98 | { err } | ||
99 | ) | ||
69 | } | 100 | } |
70 | } | 101 | } |
71 | } | 102 | } |
72 | 103 | ||
73 | private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) { | 104 | private async extendsRedundancy (redundancyModel: VideoRedundancyModel) { |
74 | // Refresh the video, maybe it was deleted | ||
75 | const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url) | ||
76 | |||
77 | if (!video) { | ||
78 | logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url) | ||
79 | |||
80 | await redundancyModel.destroy() | ||
81 | return | ||
82 | } | ||
83 | |||
84 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) | 105 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) |
106 | // Redundancy strategy disabled, remove our redundancy instead of extending expiration | ||
107 | if (!redundancy) await removeVideoRedundancy(redundancyModel) | ||
108 | |||
85 | await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) | 109 | await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) |
86 | } | 110 | } |
87 | 111 | ||
@@ -112,49 +136,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
112 | } | 136 | } |
113 | } | 137 | } |
114 | 138 | ||
115 | private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 139 | private async createVideoRedundancies (data: CandidateToDuplicate) { |
116 | const serverActor = await getServerActor() | 140 | const video = await this.loadAndRefreshVideo(data.video.url) |
141 | |||
142 | if (!video) { | ||
143 | logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url) | ||
117 | 144 | ||
118 | for (const file of filesToDuplicate) { | 145 | return |
119 | const video = await this.loadAndRefreshVideo(file.Video.url) | 146 | } |
120 | 147 | ||
148 | for (const file of data.files) { | ||
121 | const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) | 149 | const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) |
122 | if (existingRedundancy) { | 150 | if (existingRedundancy) { |
123 | await this.extendsOrDeleteRedundancy(existingRedundancy) | 151 | await this.extendsRedundancy(existingRedundancy) |
124 | 152 | ||
125 | continue | 153 | continue |
126 | } | 154 | } |
127 | 155 | ||
128 | if (!video) { | 156 | await this.createVideoFileRedundancy(data.redundancy, video, file) |
129 | logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url) | 157 | } |
158 | |||
159 | for (const streamingPlaylist of data.streamingPlaylists) { | ||
160 | const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id) | ||
161 | if (existingRedundancy) { | ||
162 | await this.extendsRedundancy(existingRedundancy) | ||
130 | 163 | ||
131 | continue | 164 | continue |
132 | } | 165 | } |
133 | 166 | ||
134 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) | 167 | await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist) |
168 | } | ||
169 | } | ||
135 | 170 | ||
136 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | 171 | private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) { |
137 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) | 172 | file.Video = video |
138 | 173 | ||
139 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) | 174 | const serverActor = await getServerActor() |
140 | 175 | ||
141 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) | 176 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) |
142 | await move(tmpPath, destPath) | ||
143 | 177 | ||
144 | const createdModel = await VideoRedundancyModel.create({ | 178 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() |
145 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | 179 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) |
146 | url: getVideoCacheFileActivityPubUrl(file), | ||
147 | fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), | ||
148 | strategy: redundancy.strategy, | ||
149 | videoFileId: file.id, | ||
150 | actorId: serverActor.id | ||
151 | }) | ||
152 | createdModel.VideoFile = file | ||
153 | 180 | ||
154 | await sendCreateCacheFile(serverActor, createdModel) | 181 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) |
155 | 182 | ||
156 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) | 183 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) |
157 | } | 184 | await move(tmpPath, destPath) |
185 | |||
186 | const createdModel = await VideoRedundancyModel.create({ | ||
187 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | ||
188 | url: getVideoCacheFileActivityPubUrl(file), | ||
189 | fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), | ||
190 | strategy: redundancy.strategy, | ||
191 | videoFileId: file.id, | ||
192 | actorId: serverActor.id | ||
193 | }) | ||
194 | |||
195 | createdModel.VideoFile = file | ||
196 | |||
197 | await sendCreateCacheFile(serverActor, video, createdModel) | ||
198 | |||
199 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) | ||
200 | } | ||
201 | |||
202 | private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) { | ||
203 | playlist.Video = video | ||
204 | |||
205 | const serverActor = await getServerActor() | ||
206 | |||
207 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) | ||
208 | |||
209 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | ||
210 | await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) | ||
211 | |||
212 | const createdModel = await VideoRedundancyModel.create({ | ||
213 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | ||
214 | url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), | ||
215 | fileUrl: playlist.getVideoRedundancyUrl(CONFIG.WEBSERVER.URL), | ||
216 | strategy: redundancy.strategy, | ||
217 | videoStreamingPlaylistId: playlist.id, | ||
218 | actorId: serverActor.id | ||
219 | }) | ||
220 | |||
221 | createdModel.VideoStreamingPlaylist = playlist | ||
222 | |||
223 | await sendCreateCacheFile(serverActor, video, createdModel) | ||
224 | |||
225 | logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) | ||
158 | } | 226 | } |
159 | 227 | ||
160 | private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { | 228 | private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { |
@@ -168,8 +236,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
168 | await sendUpdateCacheFile(serverActor, redundancy) | 236 | await sendUpdateCacheFile(serverActor, redundancy) |
169 | } | 237 | } |
170 | 238 | ||
171 | private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 239 | private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { |
172 | while (this.isTooHeavy(redundancy, filesToDuplicate)) { | 240 | while (this.isTooHeavy(candidateToDuplicate)) { |
241 | const redundancy = candidateToDuplicate.redundancy | ||
173 | const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) | 242 | const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) |
174 | if (!toDelete) return | 243 | if (!toDelete) return |
175 | 244 | ||
@@ -177,11 +246,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
177 | } | 246 | } |
178 | } | 247 | } |
179 | 248 | ||
180 | private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 249 | private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { |
181 | const maxSize = redundancy.size | 250 | const maxSize = candidateToDuplicate.redundancy.size |
182 | 251 | ||
183 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) | 252 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy) |
184 | const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate) | 253 | const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) |
185 | 254 | ||
186 | return totalWillDuplicate > maxSize | 255 | return totalWillDuplicate > maxSize |
187 | } | 256 | } |
@@ -191,13 +260,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
191 | } | 260 | } |
192 | 261 | ||
193 | private buildEntryLogId (object: VideoRedundancyModel) { | 262 | private buildEntryLogId (object: VideoRedundancyModel) { |
194 | return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` | 263 | if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` |
264 | |||
265 | return `${object.VideoStreamingPlaylist.playlistUrl}` | ||
195 | } | 266 | } |
196 | 267 | ||
197 | private getTotalFileSizes (files: VideoFileModel[]) { | 268 | private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) { |
198 | const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size | 269 | const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size |
199 | 270 | ||
200 | return files.reduce(fileReducer, 0) | 271 | return files.reduce(fileReducer, 0) * playlists.length |
201 | } | 272 | } |
202 | 273 | ||
203 | private async loadAndRefreshVideo (videoUrl: string) { | 274 | private async loadAndRefreshVideo (videoUrl: string) { |
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index 4460f46e4..086b860a2 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts | |||
@@ -1,11 +1,14 @@ | |||
1 | import { CONFIG } from '../initializers' | 1 | import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers' |
2 | import { extname, join } from 'path' | 2 | import { extname, join } from 'path' |
3 | import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' | 3 | import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' |
4 | import { copy, remove, move, stat } from 'fs-extra' | 4 | import { copy, ensureDir, move, remove, stat } from 'fs-extra' |
5 | import { logger } from '../helpers/logger' | 5 | import { logger } from '../helpers/logger' |
6 | import { VideoResolution } from '../../shared/models/videos' | 6 | import { VideoResolution } from '../../shared/models/videos' |
7 | import { VideoFileModel } from '../models/video/video-file' | 7 | import { VideoFileModel } from '../models/video/video-file' |
8 | import { VideoModel } from '../models/video/video' | 8 | import { VideoModel } from '../models/video/video' |
9 | import { updateMasterHLSPlaylist, updateSha256Segments } from './hls' | ||
10 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
11 | import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type' | ||
9 | 12 | ||
10 | async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { | 13 | async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { |
11 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | 14 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR |
@@ -17,7 +20,8 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi | |||
17 | 20 | ||
18 | const transcodeOptions = { | 21 | const transcodeOptions = { |
19 | inputPath: videoInputPath, | 22 | inputPath: videoInputPath, |
20 | outputPath: videoTranscodedPath | 23 | outputPath: videoTranscodedPath, |
24 | resolution: inputVideoFile.resolution | ||
21 | } | 25 | } |
22 | 26 | ||
23 | // Could be very long! | 27 | // Could be very long! |
@@ -47,7 +51,7 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi | |||
47 | } | 51 | } |
48 | } | 52 | } |
49 | 53 | ||
50 | async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { | 54 | async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) { |
51 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | 55 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR |
52 | const extname = '.mp4' | 56 | const extname = '.mp4' |
53 | 57 | ||
@@ -60,13 +64,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR | |||
60 | size: 0, | 64 | size: 0, |
61 | videoId: video.id | 65 | videoId: video.id |
62 | }) | 66 | }) |
63 | const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) | 67 | const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile)) |
64 | 68 | ||
65 | const transcodeOptions = { | 69 | const transcodeOptions = { |
66 | inputPath: videoInputPath, | 70 | inputPath: videoInputPath, |
67 | outputPath: videoOutputPath, | 71 | outputPath: videoOutputPath, |
68 | resolution, | 72 | resolution, |
69 | isPortraitMode | 73 | isPortraitMode: isPortrait |
70 | } | 74 | } |
71 | 75 | ||
72 | await transcode(transcodeOptions) | 76 | await transcode(transcodeOptions) |
@@ -84,6 +88,41 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR | |||
84 | video.VideoFiles.push(newVideoFile) | 88 | video.VideoFiles.push(newVideoFile) |
85 | } | 89 | } |
86 | 90 | ||
91 | async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { | ||
92 | const baseHlsDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) | ||
93 | await ensureDir(join(HLS_PLAYLIST_DIRECTORY, video.uuid)) | ||
94 | |||
95 | const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile())) | ||
96 | const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) | ||
97 | |||
98 | const transcodeOptions = { | ||
99 | inputPath: videoInputPath, | ||
100 | outputPath, | ||
101 | resolution, | ||
102 | isPortraitMode, | ||
103 | |||
104 | hlsPlaylist: { | ||
105 | videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution) | ||
106 | } | ||
107 | } | ||
108 | |||
109 | await transcode(transcodeOptions) | ||
110 | |||
111 | await updateMasterHLSPlaylist(video) | ||
112 | await updateSha256Segments(video) | ||
113 | |||
114 | const playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) | ||
115 | |||
116 | await VideoStreamingPlaylistModel.upsert({ | ||
117 | videoId: video.id, | ||
118 | playlistUrl, | ||
119 | segmentsSha256Url: CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid), | ||
120 | p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles), | ||
121 | |||
122 | type: VideoStreamingPlaylistType.HLS | ||
123 | }) | ||
124 | } | ||
125 | |||
87 | async function importVideoFile (video: VideoModel, inputFilePath: string) { | 126 | async function importVideoFile (video: VideoModel, inputFilePath: string) { |
88 | const { videoFileResolution } = await getVideoFileResolution(inputFilePath) | 127 | const { videoFileResolution } = await getVideoFileResolution(inputFilePath) |
89 | const { size } = await stat(inputFilePath) | 128 | const { size } = await stat(inputFilePath) |
@@ -125,6 +164,7 @@ async function importVideoFile (video: VideoModel, inputFilePath: string) { | |||
125 | } | 164 | } |
126 | 165 | ||
127 | export { | 166 | export { |
167 | generateHlsPlaylist, | ||
128 | optimizeVideofile, | 168 | optimizeVideofile, |
129 | transcodeOriginalVideofile, | 169 | transcodeOriginalVideofile, |
130 | importVideoFile | 170 | importVideoFile |